PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach $IPV6RE);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QemuConfig;
  31 use PVE::QMPClient;
  32 use PVE::RPCEnvironment;
  33 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
  34 use PVE::QemuServer::Memory;
  35 use PVE::QemuServer::USB qw(parse_usb_device);
  36 use PVE::QemuServer::Cloudinit;
  37 use Time::HiRes qw(gettimeofday);
  38 use File::Copy qw(copy);
  39 use URI::Escape;
  40
  41 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  42 my $OVMF_CODE = "$EDK2_FW_BASE/OVMF_CODE.fd";
  43 my $OVMF_VARS = "$EDK2_FW_BASE/OVMF_VARS.fd";
  44
  45 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  46
  47 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  48
  49 my $QEMU_FORMAT_RE = qr/raw|cow|qcow|qcow2|qed|vmdk|cloop/;
  50
  51 # Note about locking: we use flock on the config file protect
  52 # against concurent actions.
  53 # Aditionaly, we have a 'lock' setting in the config file. This
  54 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  55 # allowed when such lock is set. But you can ignore this kind of
  56 # lock with the --skiplock flag.
  57
  58 cfs_register_file('/qemu-server/',
  59                   \&parse_vm_config,
  60                   \&write_vm_config);
  61
  62 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  63     description => "Some command save/restore state from this location.",
  64     type => 'string',
  65     maxLength => 128,
  66     optional => 1,
  67 });
  68
  69 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  70     description => "The name of the snapshot.",
  71     type => 'string', format => 'pve-configid',
  72     maxLength => 40,
  73 });
  74
  75 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  76     type => 'string',
  77     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  78     description => "The drive's backing file's data format.",
  79     optional => 1,
  80 });
  81
  82 #no warnings 'redefine';
  83
  84 sub cgroups_write {
  85    my ($controller, $vmid, $option, $value) = @_;
  86
  87    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  88    PVE::ProcFSTools::write_proc_entry($path, $value);
  89
  90 }
  91
  92 my $nodename = PVE::INotify::nodename();
  93
  94 mkdir "/etc/pve/nodes/$nodename";
  95 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  96 mkdir $confdir;
  97
  98 my $var_run_tmpdir = "/var/run/qemu-server";
  99 mkdir $var_run_tmpdir;
 100
 101 my $lock_dir = "/var/lock/qemu-server";
 102 mkdir $lock_dir;
 103
 104 my $pcisysfs = "/sys/bus/pci";
 105
 106 my $cpu_vendor_list = {
 107     # Intel CPUs
 108     486 => 'GenuineIntel',
 109     pentium => 'GenuineIntel',
 110     pentium2  => 'GenuineIntel',
 111     pentium3  => 'GenuineIntel',
 112     coreduo => 'GenuineIntel',
 113     core2duo => 'GenuineIntel',
 114     Conroe  => 'GenuineIntel',
 115     Penryn  => 'GenuineIntel',
 116     Nehalem  => 'GenuineIntel',
 117     'Nehalem-IBRS'  => 'GenuineIntel',
 118     Westmere => 'GenuineIntel',
 119     'Westmere-IBRS' => 'GenuineIntel',
 120     SandyBridge => 'GenuineIntel',
 121     'SandyBridge-IBRS' => 'GenuineIntel',
 122     IvyBridge => 'GenuineIntel',
 123     'IvyBridge-IBRS' => 'GenuineIntel',
 124     Haswell => 'GenuineIntel',
 125     'Haswell-IBRS' => 'GenuineIntel',
 126     'Haswell-noTSX' => 'GenuineIntel',
 127     'Haswell-noTSX-IBRS' => 'GenuineIntel',
 128     Broadwell => 'GenuineIntel',
 129     'Broadwell-IBRS' => 'GenuineIntel',
 130     'Broadwell-noTSX' => 'GenuineIntel',
 131     'Broadwell-noTSX-IBRS' => 'GenuineIntel',
 132     'Skylake-Client' => 'GenuineIntel',
 133     'Skylake-Client-IBRS' => 'GenuineIntel',
 134     'Skylake-Server' => 'GenuineIntel',
 135     'Skylake-Server-IBRS' => 'GenuineIntel',
 136
 137     # AMD CPUs
 138     athlon => 'AuthenticAMD',
 139     phenom  => 'AuthenticAMD',
 140     Opteron_G1  => 'AuthenticAMD',
 141     Opteron_G2  => 'AuthenticAMD',
 142     Opteron_G3  => 'AuthenticAMD',
 143     Opteron_G4  => 'AuthenticAMD',
 144     Opteron_G5  => 'AuthenticAMD',
 145     EPYC => 'AuthenticAMD',
 146     'EPYC-IBPB' => 'AuthenticAMD',
 147
 148     # generic types, use vendor from host node
 149     host => 'default',
 150     kvm32 => 'default',
 151     kvm64 => 'default',
 152     qemu32 => 'default',
 153     qemu64 => 'default',
 154     max => 'default',
 155 };
 156
 157 my $cpu_flag = qr/[+-](pcid|spec-ctrl)/;
 158
 159 my $cpu_fmt = {
 160     cputype => {
 161         description => "Emulated CPU type.",
 162         type => 'string',
 163         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 164         default => 'kvm64',
 165         default_key => 1,
 166     },
 167     hidden => {
 168         description => "Do not identify as a KVM virtual machine.",
 169         type => 'boolean',
 170         optional => 1,
 171         default => 0
 172     },
 173     flags => {
 174         description => "List of additional CPU flags separated by ';'."
 175                      . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
 176                      . " Currently supported flags: 'pcid', 'spec-ctrl'.",
 177         format_description => '+FLAG[;-FLAG...]',
 178         type => 'string',
 179         pattern => qr/$cpu_flag(;$cpu_flag)*/,
 180         optional => 1,
 181     },
 182 };
 183
 184 my $watchdog_fmt = {
 185     model => {
 186         default_key => 1,
 187         type => 'string',
 188         enum => [qw(i6300esb ib700)],
 189         description => "Watchdog type to emulate.",
 190         default => 'i6300esb',
 191         optional => 1,
 192     },
 193     action => {
 194         type => 'string',
 195         enum => [qw(reset shutdown poweroff pause debug none)],
 196         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 197         optional => 1,
 198     },
 199 };
 200 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 201
 202 my $confdesc = {
 203     onboot => {
 204         optional => 1,
 205         type => 'boolean',
 206         description => "Specifies whether a VM will be started during system bootup.",
 207         default => 0,
 208     },
 209     autostart => {
 210         optional => 1,
 211         type => 'boolean',
 212         description => "Automatic restart after crash (currently ignored).",
 213         default => 0,
 214     },
 215     hotplug => {
 216         optional => 1,
 217         type => 'string', format => 'pve-hotplug-features',
 218         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 219         default => 'network,disk,usb',
 220     },
 221     reboot => {
 222         optional => 1,
 223         type => 'boolean',
 224         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 225         default => 1,
 226     },
 227     lock => {
 228         optional => 1,
 229         type => 'string',
 230         description => "Lock/unlock the VM.",
 231         enum => [qw(migrate backup snapshot rollback)],
 232     },
 233     cpulimit => {
 234         optional => 1,
 235         type => 'number',
 236         description => "Limit of CPU usage.",
 237         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 238         minimum => 0,
 239         maximum => 128,
 240         default => 0,
 241     },
 242     cpuunits => {
 243         optional => 1,
 244         type => 'integer',
 245         description => "CPU weight for a VM.",
 246         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 247         minimum => 2,
 248         maximum => 262144,
 249         default => 1024,
 250     },
 251     memory => {
 252         optional => 1,
 253         type => 'integer',
 254         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 255         minimum => 16,
 256         default => 512,
 257     },
 258     balloon => {
 259         optional => 1,
 260         type => 'integer',
 261         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 262         minimum => 0,
 263     },
 264     shares => {
 265         optional => 1,
 266         type => 'integer',
 267         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 268         minimum => 0,
 269         maximum => 50000,
 270         default => 1000,
 271     },
 272     keyboard => {
 273         optional => 1,
 274         type => 'string',
 275         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 276                        "It should not be necessary to set it.",
 277         enum => PVE::Tools::kvmkeymaplist(),
 278         default => undef,
 279     },
 280     name => {
 281         optional => 1,
 282         type => 'string', format => 'dns-name',
 283         description => "Set a name for the VM. Only used on the configuration web interface.",
 284     },
 285     scsihw => {
 286         optional => 1,
 287         type => 'string',
 288         description => "SCSI controller model",
 289         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 290         default => 'lsi',
 291     },
 292     description => {
 293         optional => 1,
 294         type => 'string',
 295         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 296     },
 297     ostype => {
 298         optional => 1,
 299         type => 'string',
 300         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 301         description => "Specify guest operating system.",
 302         verbose_description => <<EODESC,
 303 Specify guest operating system. This is used to enable special
 304 optimization/features for specific operating systems:
 305
 306 [horizontal]
 307 other;; unspecified OS
 308 wxp;; Microsoft Windows XP
 309 w2k;; Microsoft Windows 2000
 310 w2k3;; Microsoft Windows 2003
 311 w2k8;; Microsoft Windows 2008
 312 wvista;; Microsoft Windows Vista
 313 win7;; Microsoft Windows 7
 314 win8;; Microsoft Windows 8/2012/2012r2
 315 win10;; Microsoft Windows 10/2016
 316 l24;; Linux 2.4 Kernel
 317 l26;; Linux 2.6/3.X Kernel
 318 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 319 EODESC
 320     },
 321     boot => {
 322         optional => 1,
 323         type => 'string',
 324         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 325         pattern => '[acdn]{1,4}',
 326         default => 'cdn',
 327     },
 328     bootdisk => {
 329         optional => 1,
 330         type => 'string', format => 'pve-qm-bootdisk',
 331         description => "Enable booting from specified disk.",
 332         pattern => '(ide|sata|scsi|virtio)\d+',
 333     },
 334     smp => {
 335         optional => 1,
 336         type => 'integer',
 337         description => "The number of CPUs. Please use option -sockets instead.",
 338         minimum => 1,
 339         default => 1,
 340     },
 341     sockets => {
 342         optional => 1,
 343         type => 'integer',
 344         description => "The number of CPU sockets.",
 345         minimum => 1,
 346         default => 1,
 347     },
 348     cores => {
 349         optional => 1,
 350         type => 'integer',
 351         description => "The number of cores per socket.",
 352         minimum => 1,
 353         default => 1,
 354     },
 355     numa => {
 356         optional => 1,
 357         type => 'boolean',
 358         description => "Enable/disable NUMA.",
 359         default => 0,
 360     },
 361     hugepages => {
 362         optional => 1,
 363         type => 'string',
 364         description => "Enable/disable hugepages memory.",
 365         enum => [qw(any 2 1024)],
 366     },
 367     vcpus => {
 368         optional => 1,
 369         type => 'integer',
 370         description => "Number of hotplugged vcpus.",
 371         minimum => 1,
 372         default => 0,
 373     },
 374     acpi => {
 375         optional => 1,
 376         type => 'boolean',
 377         description => "Enable/disable ACPI.",
 378         default => 1,
 379     },
 380     agent => {
 381         optional => 1,
 382         type => 'boolean',
 383         description => "Enable/disable Qemu GuestAgent.",
 384         default => 0,
 385     },
 386     kvm => {
 387         optional => 1,
 388         type => 'boolean',
 389         description => "Enable/disable KVM hardware virtualization.",
 390         default => 1,
 391     },
 392     tdf => {
 393         optional => 1,
 394         type => 'boolean',
 395         description => "Enable/disable time drift fix.",
 396         default => 0,
 397     },
 398     localtime => {
 399         optional => 1,
 400         type => 'boolean',
 401         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 402     },
 403     freeze => {
 404         optional => 1,
 405         type => 'boolean',
 406         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 407     },
 408     vga => {
 409         optional => 1,
 410         type => 'string',
 411         description => "Select the VGA type.",
 412         verbose_description => "Select the VGA type. If you want to use high resolution" .
 413             " modes (>= 1280x1024x16) then you should use the options " .
 414             "'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and " .
 415             "'cirrus' for other OS types. The 'qxl' option enables the SPICE " .
 416             "display sever. For win* OS you can select how many independent " .
 417             "displays you want, Linux guests can add displays them self. " .
 418             "You can also run without any graphic card, using a serial device" .
 419             " as terminal.",
 420         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 421     },
 422     watchdog => {
 423         optional => 1,
 424         type => 'string', format => 'pve-qm-watchdog',
 425         description => "Create a virtual hardware watchdog device.",
 426         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 427             " (by a guest action), the watchdog must be periodically polled " .
 428             "by an agent inside the guest or else the watchdog will reset " .
 429             "the guest (or execute the respective action specified)",
 430     },
 431     startdate => {
 432         optional => 1,
 433         type => 'string',
 434         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 435         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 436         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 437         default => 'now',
 438     },
 439     startup =>  get_standard_option('pve-startup-order'),
 440     template => {
 441         optional => 1,
 442         type => 'boolean',
 443         description => "Enable/disable Template.",
 444         default => 0,
 445     },
 446     args => {
 447         optional => 1,
 448         type => 'string',
 449         description => "Arbitrary arguments passed to kvm.",
 450         verbose_description => <<EODESCR,
 451 Arbitrary arguments passed to kvm, for example:
 452
 453 args: -no-reboot -no-hpet
 454
 455 NOTE: this option is for experts only.
 456 EODESCR
 457     },
 458     tablet => {
 459         optional => 1,
 460         type => 'boolean',
 461         default => 1,
 462         description => "Enable/disable the USB tablet device.",
 463         verbose_description => "Enable/disable the USB tablet device. This device is " .
 464             "usually needed to allow absolute mouse positioning with VNC. " .
 465             "Else the mouse runs out of sync with normal VNC clients. " .
 466             "If you're running lots of console-only guests on one host, " .
 467             "you may consider disabling this to save some context switches. " .
 468             "This is turned off by default if you use spice (-vga=qxl).",
 469     },
 470     migrate_speed => {
 471         optional => 1,
 472         type => 'integer',
 473         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 474         minimum => 0,
 475         default => 0,
 476     },
 477     migrate_downtime => {
 478         optional => 1,
 479         type => 'number',
 480         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 481         minimum => 0,
 482         default => 0.1,
 483     },
 484     cdrom => {
 485         optional => 1,
 486         type => 'string', format => 'pve-qm-ide',
 487         typetext => '<volume>',
 488         description => "This is an alias for option -ide2",
 489     },
 490     cpu => {
 491         optional => 1,
 492         description => "Emulated CPU type.",
 493         type => 'string',
 494         format => $cpu_fmt,
 495     },
 496     parent => get_standard_option('pve-snapshot-name', {
 497         optional => 1,
 498         description => "Parent snapshot name. This is used internally, and should not be modified.",
 499     }),
 500     snaptime => {
 501         optional => 1,
 502         description => "Timestamp for snapshots.",
 503         type => 'integer',
 504         minimum => 0,
 505     },
 506     vmstate => {
 507         optional => 1,
 508         type => 'string', format => 'pve-volume-id',
 509         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 510     },
 511     vmstatestorage => get_standard_option('pve-storage-id', {
 512         description => "Default storage for VM state volumes/files.",
 513         optional => 1,
 514     }),
 515     machine => {
 516         description => "Specific the Qemu machine type.",
 517         type => 'string',
 518         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
 519         maxLength => 40,
 520         optional => 1,
 521     },
 522     smbios1 => {
 523         description => "Specify SMBIOS type 1 fields.",
 524         type => 'string', format => 'pve-qm-smbios1',
 525         maxLength => 256,
 526         optional => 1,
 527     },
 528     protection => {
 529         optional => 1,
 530         type => 'boolean',
 531         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 532         default => 0,
 533     },
 534     bios => {
 535         optional => 1,
 536         type => 'string',
 537         enum => [ qw(seabios ovmf) ],
 538         description => "Select BIOS implementation.",
 539         default => 'seabios',
 540     },
 541 };
 542
 543 my $confdesc_cloudinit = {
 544     citype => {
 545         optional => 1,
 546         type => 'string',
 547         description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
 548         enum => ['configdrive2', 'nocloud'],
 549     },
 550     ciuser => {
 551         optional => 1,
 552         type => 'string',
 553         description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
 554     },
 555     cipassword => {
 556         optional => 1,
 557         type => 'string',
 558         description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
 559     },
 560     searchdomain => {
 561         optional => 1,
 562         type => 'string',
 563         description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 564     },
 565     nameserver => {
 566         optional => 1,
 567         type => 'string', format => 'address-list',
 568         description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 569     },
 570     sshkeys => {
 571         optional => 1,
 572         type => 'string',
 573         format => 'urlencoded',
 574         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 575     },
 576 };
 577
 578 # what about other qemu settings ?
 579 #cpu => 'string',
 580 #machine => 'string',
 581 #fda => 'file',
 582 #fdb => 'file',
 583 #mtdblock => 'file',
 584 #sd => 'file',
 585 #pflash => 'file',
 586 #snapshot => 'bool',
 587 #bootp => 'file',
 588 ##tftp => 'dir',
 589 ##smb => 'dir',
 590 #kernel => 'file',
 591 #append => 'string',
 592 #initrd => 'file',
 593 ##soundhw => 'string',
 594
 595 while (my ($k, $v) = each %$confdesc) {
 596     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 597 }
 598
 599 my $MAX_IDE_DISKS = 4;
 600 my $MAX_SCSI_DISKS = 14;
 601 my $MAX_VIRTIO_DISKS = 16;
 602 my $MAX_SATA_DISKS = 6;
 603 my $MAX_USB_DEVICES = 5;
 604 my $MAX_NETS = 32;
 605 my $MAX_UNUSED_DISKS = 8;
 606 my $MAX_HOSTPCI_DEVICES = 4;
 607 my $MAX_SERIAL_PORTS = 4;
 608 my $MAX_PARALLEL_PORTS = 3;
 609 my $MAX_NUMA = 8;
 610
 611 my $numa_fmt = {
 612     cpus => {
 613         type => "string",
 614         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 615         description => "CPUs accessing this NUMA node.",
 616         format_description => "id[-id];...",
 617     },
 618     memory => {
 619         type => "number",
 620         description => "Amount of memory this NUMA node provides.",
 621         optional => 1,
 622     },
 623     hostnodes => {
 624         type => "string",
 625         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 626         description => "Host NUMA nodes to use.",
 627         format_description => "id[-id];...",
 628         optional => 1,
 629     },
 630     policy => {
 631         type => 'string',
 632         enum => [qw(preferred bind interleave)],
 633         description => "NUMA allocation policy.",
 634         optional => 1,
 635     },
 636 };
 637 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 638 my $numadesc = {
 639     optional => 1,
 640     type => 'string', format => $numa_fmt,
 641     description => "NUMA topology.",
 642 };
 643 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 644
 645 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 646     $confdesc->{"numa$i"} = $numadesc;
 647 }
 648
 649 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 650                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 651                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 652 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 653
 654 my $net_fmt_bridge_descr = <<__EOD__;
 655 Bridge to attach the network device to. The Proxmox VE standard bridge
 656 is called 'vmbr0'.
 657
 658 If you do not specify a bridge, we create a kvm user (NATed) network
 659 device, which provides DHCP and DNS services. The following addresses
 660 are used:
 661
 662  10.0.2.2   Gateway
 663  10.0.2.3   DNS Server
 664  10.0.2.4   SMB Server
 665
 666 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 667 __EOD__
 668
 669 my $net_fmt = {
 670     macaddr => {
 671         type => 'string',
 672         pattern => qr/[0-9a-f]{2}(?::[0-9a-f]{2}){5}/i,
 673         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 674         format_description => "XX:XX:XX:XX:XX:XX",
 675         optional => 1,
 676     },
 677     model => {
 678         type => 'string',
 679         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 680         enum => $nic_model_list,
 681         default_key => 1,
 682     },
 683     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 684     bridge => {
 685         type => 'string',
 686         description => $net_fmt_bridge_descr,
 687         format_description => 'bridge',
 688         optional => 1,
 689     },
 690     queues => {
 691         type => 'integer',
 692         minimum => 0, maximum => 16,
 693         description => 'Number of packet queues to be used on the device.',
 694         optional => 1,
 695     },
 696     rate => {
 697         type => 'number',
 698         minimum => 0,
 699         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 700         optional => 1,
 701     },
 702     tag => {
 703         type => 'integer',
 704         minimum => 1, maximum => 4094,
 705         description => 'VLAN tag to apply to packets on this interface.',
 706         optional => 1,
 707     },
 708     trunks => {
 709         type => 'string',
 710         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 711         description => 'VLAN trunks to pass through this interface.',
 712         format_description => 'vlanid[;vlanid...]',
 713         optional => 1,
 714     },
 715     firewall => {
 716         type => 'boolean',
 717         description => 'Whether this interface should be protected by the firewall.',
 718         optional => 1,
 719     },
 720     link_down => {
 721         type => 'boolean',
 722         description => 'Whether this interface should be disconnected (like pulling the plug).',
 723         optional => 1,
 724     },
 725 };
 726
 727 my $netdesc = {
 728     optional => 1,
 729     type => 'string', format => $net_fmt,
 730     description => "Specify network devices.",
 731 };
 732
 733 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 734
 735 my $ipconfig_fmt = {
 736     ip => {
 737         type => 'string',
 738         format => 'pve-ipv4-config',
 739         format_description => 'IPv4Format/CIDR',
 740         description => 'IPv4 address in CIDR format.',
 741         optional => 1,
 742         default => 'dhcp',
 743     },
 744     gw => {
 745         type => 'string',
 746         format => 'ipv4',
 747         format_description => 'GatewayIPv4',
 748         description => 'Default gateway for IPv4 traffic.',
 749         optional => 1,
 750         requires => 'ip',
 751     },
 752     ip6 => {
 753         type => 'string',
 754         format => 'pve-ipv6-config',
 755         format_description => 'IPv6Format/CIDR',
 756         description => 'IPv6 address in CIDR format.',
 757         optional => 1,
 758         default => 'dhcp',
 759     },
 760     gw6 => {
 761         type => 'string',
 762         format => 'ipv6',
 763         format_description => 'GatewayIPv6',
 764         description => 'Default gateway for IPv6 traffic.',
 765         optional => 1,
 766         requires => 'ip6',
 767     },
 768 };
 769 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 770 my $ipconfigdesc = {
 771     optional => 1,
 772     type => 'string', format => 'pve-qm-ipconfig',
 773     description => <<'EODESCR',
 774 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 775
 776 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 777
 778 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
 779 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 780
 781 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
 782 EODESCR
 783 };
 784 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 785
 786 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 787     $confdesc->{"net$i"} = $netdesc;
 788     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 789 }
 790
 791 foreach my $key (keys %$confdesc_cloudinit) {
 792     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 793 }
 794
 795 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 796 sub verify_volume_id_or_qm_path {
 797     my ($volid, $noerr) = @_;
 798
 799     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 800         return $volid;
 801     }
 802
 803     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 804     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 805     if ($@) {
 806         return undef if $noerr;
 807         die $@;
 808     }
 809     return $volid;
 810 }
 811
 812 my $drivename_hash;
 813
 814 my %drivedesc_base = (
 815     volume => { alias => 'file' },
 816     file => {
 817         type => 'string',
 818         format => 'pve-volume-id-or-qm-path',
 819         default_key => 1,
 820         format_description => 'volume',
 821         description => "The drive's backing volume.",
 822     },
 823     media => {
 824         type => 'string',
 825         enum => [qw(cdrom disk)],
 826         description => "The drive's media type.",
 827         default => 'disk',
 828         optional => 1
 829     },
 830     cyls => {
 831         type => 'integer',
 832         description => "Force the drive's physical geometry to have a specific cylinder count.",
 833         optional => 1
 834     },
 835     heads => {
 836         type => 'integer',
 837         description => "Force the drive's physical geometry to have a specific head count.",
 838         optional => 1
 839     },
 840     secs => {
 841         type => 'integer',
 842         description => "Force the drive's physical geometry to have a specific sector count.",
 843         optional => 1
 844     },
 845     trans => {
 846         type => 'string',
 847         enum => [qw(none lba auto)],
 848         description => "Force disk geometry bios translation mode.",
 849         optional => 1,
 850     },
 851     snapshot => {
 852         type => 'boolean',
 853         description => "Controls qemu's snapshot mode feature."
 854             . " If activated, changes made to the disk are temporary and will"
 855             . " be discarded when the VM is shutdown.",
 856         optional => 1,
 857     },
 858     cache => {
 859         type => 'string',
 860         enum => [qw(none writethrough writeback unsafe directsync)],
 861         description => "The drive's cache mode",
 862         optional => 1,
 863     },
 864     format => get_standard_option('pve-qm-image-format'),
 865     size => {
 866         type => 'string',
 867         format => 'disk-size',
 868         format_description => 'DiskSize',
 869         description => "Disk size. This is purely informational and has no effect.",
 870         optional => 1,
 871     },
 872     backup => {
 873         type => 'boolean',
 874         description => "Whether the drive should be included when making backups.",
 875         optional => 1,
 876     },
 877     replicate => {
 878         type => 'boolean',
 879         description => 'Whether the drive should considered for replication jobs.',
 880         optional => 1,
 881         default => 1,
 882     },
 883     rerror => {
 884         type => 'string',
 885         enum => [qw(ignore report stop)],
 886         description => 'Read error action.',
 887         optional => 1,
 888     },
 889     werror => {
 890         type => 'string',
 891         enum => [qw(enospc ignore report stop)],
 892         description => 'Write error action.',
 893         optional => 1,
 894     },
 895     aio => {
 896         type => 'string',
 897         enum => [qw(native threads)],
 898         description => 'AIO type to use.',
 899         optional => 1,
 900     },
 901     discard => {
 902         type => 'string',
 903         enum => [qw(ignore on)],
 904         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 905         optional => 1,
 906     },
 907     detect_zeroes => {
 908         type => 'boolean',
 909         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 910         optional => 1,
 911     },
 912     serial => {
 913         type => 'string',
 914         format => 'urlencoded',
 915         format_description => 'serial',
 916         maxLength => 20*3, # *3 since it's %xx url enoded
 917         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
 918         optional => 1,
 919     },
 920     shared => {
 921         type => 'boolean',
 922         description => 'Mark this locally-managed volume as available on all nodes',
 923         verbose_description => "Mark this locally-managed volume as available on all nodes.\n\nWARNING: This option does not share the volume automatically, it assumes it is shared already!",
 924         optional => 1,
 925         default => 0,
 926     }
 927 );
 928
 929 my %iothread_fmt = ( iothread => {
 930         type => 'boolean',
 931         description => "Whether to use iothreads for this drive",
 932         optional => 1,
 933 });
 934
 935 my %model_fmt = (
 936     model => {
 937         type => 'string',
 938         format => 'urlencoded',
 939         format_description => 'model',
 940         maxLength => 40*3, # *3 since it's %xx url enoded
 941         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
 942         optional => 1,
 943     },
 944 );
 945
 946 my %queues_fmt = (
 947     queues => {
 948         type => 'integer',
 949         description => "Number of queues.",
 950         minimum => 2,
 951         optional => 1
 952     }
 953 );
 954
 955 my %scsiblock_fmt = (
 956     scsiblock => {
 957         type => 'boolean',
 958         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
 959         optional => 1,
 960         default => 0,
 961     },
 962 );
 963
 964 my $add_throttle_desc = sub {
 965     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
 966     my $d = {
 967         type => $type,
 968         format_description => $unit,
 969         description => "Maximum $what in $longunit.",
 970         optional => 1,
 971     };
 972     $d->{minimum} = $minimum if defined($minimum);
 973     $drivedesc_base{$key} = $d;
 974 };
 975 # throughput: (leaky bucket)
 976 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
 977 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
 978 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
 979 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
 980 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
 981 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
 982 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
 983 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
 984 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
 985
 986 # pools: (pool of IO before throttling starts taking effect)
 987 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
 988 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
 989 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
 990 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
 991 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
 992 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
 993
 994 # burst lengths
 995 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
 996 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
 997 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
 998 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
 999 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1000 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1001
1002 # legacy support
1003 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
1004 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
1005 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
1006 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
1007
1008 my $ide_fmt = {
1009     %drivedesc_base,
1010     %model_fmt,
1011 };
1012 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
1013
1014 my $idedesc = {
1015     optional => 1,
1016     type => 'string', format => $ide_fmt,
1017     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
1018 };
1019 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
1020
1021 my $scsi_fmt = {
1022     %drivedesc_base,
1023     %iothread_fmt,
1024     %queues_fmt,
1025     %scsiblock_fmt,
1026 };
1027 my $scsidesc = {
1028     optional => 1,
1029     type => 'string', format => $scsi_fmt,
1030     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
1031 };
1032 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
1033
1034 my $sata_fmt = {
1035     %drivedesc_base,
1036 };
1037 my $satadesc = {
1038     optional => 1,
1039     type => 'string', format => $sata_fmt,
1040     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
1041 };
1042 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
1043
1044 my $virtio_fmt = {
1045     %drivedesc_base,
1046     %iothread_fmt,
1047 };
1048 my $virtiodesc = {
1049     optional => 1,
1050     type => 'string', format => $virtio_fmt,
1051     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
1052 };
1053 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
1054
1055 my $alldrive_fmt = {
1056     %drivedesc_base,
1057     %iothread_fmt,
1058     %model_fmt,
1059     %queues_fmt,
1060     %scsiblock_fmt,
1061 };
1062
1063 my $efidisk_fmt = {
1064     volume => { alias => 'file' },
1065     file => {
1066         type => 'string',
1067         format => 'pve-volume-id-or-qm-path',
1068         default_key => 1,
1069         format_description => 'volume',
1070         description => "The drive's backing volume.",
1071     },
1072     format => get_standard_option('pve-qm-image-format'),
1073     size => {
1074         type => 'string',
1075         format => 'disk-size',
1076         format_description => 'DiskSize',
1077         description => "Disk size. This is purely informational and has no effect.",
1078         optional => 1,
1079     },
1080 };
1081
1082 my $efidisk_desc = {
1083     optional => 1,
1084     type => 'string', format => $efidisk_fmt,
1085     description => "Configure a Disk for storing EFI vars",
1086 };
1087
1088 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
1089
1090 my $usb_fmt = {
1091     host => {
1092         default_key => 1,
1093         type => 'string', format => 'pve-qm-usb-device',
1094         format_description => 'HOSTUSBDEVICE|spice',
1095         description => <<EODESCR,
1096 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1097
1098  'bus-port(.port)*' (decimal numbers) or
1099  'vendor_id:product_id' (hexadeciaml numbers) or
1100  'spice'
1101
1102 You can use the 'lsusb -t' command to list existing usb devices.
1103
1104 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1105
1106 The value 'spice' can be used to add a usb redirection devices for spice.
1107 EODESCR
1108     },
1109     usb3 => {
1110         optional => 1,
1111         type => 'boolean',
1112         description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
1113         default => 0,
1114     },
1115 };
1116
1117 my $usbdesc = {
1118     optional => 1,
1119     type => 'string', format => $usb_fmt,
1120     description => "Configure an USB device (n is 0 to 4).",
1121 };
1122 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1123
1124 # NOTE: the match-groups of this regex are used in parse_hostpci
1125 my $PCIRE = qr/([a-f0-9]{2}:[a-f0-9]{2})(?:\.([a-f0-9]))?/;
1126 my $hostpci_fmt = {
1127     host => {
1128         default_key => 1,
1129         type => 'string',
1130         pattern => qr/$PCIRE(;$PCIRE)*/,
1131         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1132         description => <<EODESCR,
1133 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1134 of PCI virtual functions of the host. HOSTPCIID syntax is:
1135
1136 'bus:dev.func' (hexadecimal numbers)
1137
1138 You can us the 'lspci' command to list existing PCI devices.
1139 EODESCR
1140     },
1141     rombar => {
1142         type => 'boolean',
1143         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1144         optional => 1,
1145         default => 1,
1146     },
1147     romfile => {
1148         type => 'string',
1149         pattern => '[^,;]+',
1150         format_description => 'string',
1151         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1152         optional => 1,
1153     },
1154     pcie => {
1155         type => 'boolean',
1156         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1157         optional => 1,
1158         default => 0,
1159     },
1160     'x-vga' => {
1161         type => 'boolean',
1162         description =>  "Enable vfio-vga device support.",
1163         optional => 1,
1164         default => 0,
1165     },
1166 };
1167 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1168
1169 my $hostpcidesc = {
1170         optional => 1,
1171         type => 'string', format => 'pve-qm-hostpci',
1172         description => "Map host PCI devices into guest.",
1173         verbose_description =>  <<EODESCR,
1174 Map host PCI devices into guest.
1175
1176 NOTE: This option allows direct access to host hardware. So it is no longer
1177 possible to migrate such machines - use with special care.
1178
1179 CAUTION: Experimental! User reported problems with this option.
1180 EODESCR
1181 };
1182 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1183
1184 my $serialdesc = {
1185         optional => 1,
1186         type => 'string',
1187         pattern => '(/dev/.+|socket)',
1188         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1189         verbose_description =>  <<EODESCR,
1190 Create a serial device inside the VM (n is 0 to 3), and pass through a
1191 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1192 host side (use 'qm terminal' to open a terminal connection).
1193
1194 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1195
1196 CAUTION: Experimental! User reported problems with this option.
1197 EODESCR
1198 };
1199
1200 my $paralleldesc= {
1201         optional => 1,
1202         type => 'string',
1203         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1204         description =>  "Map host parallel devices (n is 0 to 2).",
1205         verbose_description =>  <<EODESCR,
1206 Map host parallel devices (n is 0 to 2).
1207
1208 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1209
1210 CAUTION: Experimental! User reported problems with this option.
1211 EODESCR
1212 };
1213
1214 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1215     $confdesc->{"parallel$i"} = $paralleldesc;
1216 }
1217
1218 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1219     $confdesc->{"serial$i"} = $serialdesc;
1220 }
1221
1222 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1223     $confdesc->{"hostpci$i"} = $hostpcidesc;
1224 }
1225
1226 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1227     $drivename_hash->{"ide$i"} = 1;
1228     $confdesc->{"ide$i"} = $idedesc;
1229 }
1230
1231 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1232     $drivename_hash->{"sata$i"} = 1;
1233     $confdesc->{"sata$i"} = $satadesc;
1234 }
1235
1236 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1237     $drivename_hash->{"scsi$i"} = 1;
1238     $confdesc->{"scsi$i"} = $scsidesc ;
1239 }
1240
1241 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1242     $drivename_hash->{"virtio$i"} = 1;
1243     $confdesc->{"virtio$i"} = $virtiodesc;
1244 }
1245
1246 $drivename_hash->{efidisk0} = 1;
1247 $confdesc->{efidisk0} = $efidisk_desc;
1248
1249 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1250     $confdesc->{"usb$i"} = $usbdesc;
1251 }
1252
1253 my $unuseddesc = {
1254     optional => 1,
1255     type => 'string', format => 'pve-volume-id',
1256     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1257 };
1258
1259 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1260     $confdesc->{"unused$i"} = $unuseddesc;
1261 }
1262
1263 my $kvm_api_version = 0;
1264
1265 sub kvm_version {
1266
1267     return $kvm_api_version if $kvm_api_version;
1268
1269     my $fh = IO::File->new("</dev/kvm") ||
1270         return 0;
1271
1272     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
1273         $kvm_api_version = $v;
1274     }
1275
1276     $fh->close();
1277
1278     return  $kvm_api_version;
1279 }
1280
1281 my $kvm_user_version;
1282
1283 sub kvm_user_version {
1284
1285     return $kvm_user_version if $kvm_user_version;
1286
1287     $kvm_user_version = 'unknown';
1288
1289     my $code = sub {
1290         my $line = shift;
1291         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1292             $kvm_user_version = $2;
1293         }
1294     };
1295
1296     eval { run_command("kvm -version", outfunc => $code); };
1297     warn $@ if $@;
1298
1299     return $kvm_user_version;
1300
1301 }
1302
1303 my $kernel_has_vhost_net = -c '/dev/vhost-net';
1304
1305 sub valid_drive_names {
1306     # order is important - used to autoselect boot disk
1307     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1308             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1309             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1310             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1311             'efidisk0');
1312 }
1313
1314 sub is_valid_drivename {
1315     my $dev = shift;
1316
1317     return defined($drivename_hash->{$dev});
1318 }
1319
1320 sub option_exists {
1321     my $key = shift;
1322     return defined($confdesc->{$key});
1323 }
1324
1325 sub nic_models {
1326     return $nic_model_list;
1327 }
1328
1329 sub os_list_description {
1330
1331     return {
1332         other => 'Other',
1333         wxp => 'Windows XP',
1334         w2k => 'Windows 2000',
1335         w2k3 =>, 'Windows 2003',
1336         w2k8 => 'Windows 2008',
1337         wvista => 'Windows Vista',
1338         win7 => 'Windows 7',
1339         win8 => 'Windows 8/2012',
1340         win10 => 'Windows 10/2016',
1341         l24 => 'Linux 2.4',
1342         l26 => 'Linux 2.6',
1343     };
1344 }
1345
1346 my $cdrom_path;
1347
1348 sub get_cdrom_path {
1349
1350     return  $cdrom_path if $cdrom_path;
1351
1352     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1353     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1354     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1355 }
1356
1357 sub get_iso_path {
1358     my ($storecfg, $vmid, $cdrom) = @_;
1359
1360     if ($cdrom eq 'cdrom') {
1361         return get_cdrom_path();
1362     } elsif ($cdrom eq 'none') {
1363         return '';
1364     } elsif ($cdrom =~ m|^/|) {
1365         return $cdrom;
1366     } else {
1367         return PVE::Storage::path($storecfg, $cdrom);
1368     }
1369 }
1370
1371 # try to convert old style file names to volume IDs
1372 sub filename_to_volume_id {
1373     my ($vmid, $file, $media) = @_;
1374
1375      if (!($file eq 'none' || $file eq 'cdrom' ||
1376           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1377
1378         return undef if $file =~ m|/|;
1379
1380         if ($media && $media eq 'cdrom') {
1381             $file = "local:iso/$file";
1382         } else {
1383             $file = "local:$vmid/$file";
1384         }
1385     }
1386
1387     return $file;
1388 }
1389
1390 sub verify_media_type {
1391     my ($opt, $vtype, $media) = @_;
1392
1393     return if !$media;
1394
1395     my $etype;
1396     if ($media eq 'disk') {
1397         $etype = 'images';
1398     } elsif ($media eq 'cdrom') {
1399         $etype = 'iso';
1400     } else {
1401         die "internal error";
1402     }
1403
1404     return if ($vtype eq $etype);
1405
1406     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1407 }
1408
1409 sub cleanup_drive_path {
1410     my ($opt, $storecfg, $drive) = @_;
1411
1412     # try to convert filesystem paths to volume IDs
1413
1414     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1415         ($drive->{file} !~ m|^/dev/.+|) &&
1416         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1417         ($drive->{file} !~ m/^\d+$/)) {
1418         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1419         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1420         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1421         verify_media_type($opt, $vtype, $drive->{media});
1422         $drive->{file} = $volid;
1423     }
1424
1425     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1426 }
1427
1428 sub parse_hotplug_features {
1429     my ($data) = @_;
1430
1431     my $res = {};
1432
1433     return $res if $data eq '0';
1434
1435     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1436
1437     foreach my $feature (PVE::Tools::split_list($data)) {
1438         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1439             $res->{$1} = 1;
1440         } else {
1441             die "invalid hotplug feature '$feature'\n";
1442         }
1443     }
1444     return $res;
1445 }
1446
1447 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1448 sub pve_verify_hotplug_features {
1449     my ($value, $noerr) = @_;
1450
1451     return $value if parse_hotplug_features($value);
1452
1453     return undef if $noerr;
1454
1455     die "unable to parse hotplug option\n";
1456 }
1457
1458 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1459 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1460 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1461 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1462 #        [,iothread=on][,serial=serial][,model=model]
1463
1464 sub parse_drive {
1465     my ($key, $data) = @_;
1466
1467     my ($interface, $index);
1468
1469     if ($key =~ m/^([^\d]+)(\d+)$/) {
1470         $interface = $1;
1471         $index = $2;
1472     } else {
1473         return undef;
1474     }
1475
1476     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1477                                      : $confdesc->{$key}->{format};
1478     if (!$desc) {
1479         warn "invalid drive key: $key\n";
1480         return undef;
1481     }
1482     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1483     return undef if !$res;
1484     $res->{interface} = $interface;
1485     $res->{index} = $index;
1486
1487     my $error = 0;
1488     foreach my $opt (qw(bps bps_rd bps_wr)) {
1489         if (my $bps = defined(delete $res->{$opt})) {
1490             if (defined($res->{"m$opt"})) {
1491                 warn "both $opt and m$opt specified\n";
1492                 ++$error;
1493                 next;
1494             }
1495             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1496         }
1497     }
1498
1499     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1500     for my $requirement (
1501         [mbps_max => 'mbps'],
1502         [mbps_rd_max => 'mbps_rd'],
1503         [mbps_wr_max => 'mbps_wr'],
1504         [miops_max => 'miops'],
1505         [miops_rd_max => 'miops_rd'],
1506         [miops_wr_max => 'miops_wr'],
1507         [bps_max_length => 'mbps_max'],
1508         [bps_rd_max_length => 'mbps_rd_max'],
1509         [bps_wr_max_length => 'mbps_wr_max'],
1510         [iops_max_length => 'iops_max'],
1511         [iops_rd_max_length => 'iops_rd_max'],
1512         [iops_wr_max_length => 'iops_wr_max']) {
1513         my ($option, $requires) = @$requirement;
1514         if ($res->{$option} && !$res->{$requires}) {
1515             warn "$option requires $requires\n";
1516             ++$error;
1517         }
1518     }
1519
1520     return undef if $error;
1521
1522     return undef if $res->{mbps_rd} && $res->{mbps};
1523     return undef if $res->{mbps_wr} && $res->{mbps};
1524     return undef if $res->{iops_rd} && $res->{iops};
1525     return undef if $res->{iops_wr} && $res->{iops};
1526
1527     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1528         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1529         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1530         return undef if $res->{interface} eq 'virtio';
1531     }
1532
1533     if (my $size = $res->{size}) {
1534         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1535     }
1536
1537     return $res;
1538 }
1539
1540 sub print_drive {
1541     my ($vmid, $drive) = @_;
1542     my $data = { %$drive };
1543     delete $data->{$_} for qw(index interface);
1544     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1545 }
1546
1547 sub scsi_inquiry {
1548     my($fh, $noerr) = @_;
1549
1550     my $SG_IO = 0x2285;
1551     my $SG_GET_VERSION_NUM = 0x2282;
1552
1553     my $versionbuf = "\x00" x 8;
1554     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1555     if (!$ret) {
1556         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1557         return undef;
1558     }
1559     my $version = unpack("I", $versionbuf);
1560     if ($version < 30000) {
1561         die "scsi generic interface too old\n"  if !$noerr;
1562         return undef;
1563     }
1564
1565     my $buf = "\x00" x 36;
1566     my $sensebuf = "\x00" x 8;
1567     my $cmd = pack("C x3 C x1", 0x12, 36);
1568
1569     # see /usr/include/scsi/sg.h
1570     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1571
1572     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1573                       length($sensebuf), 0, length($buf), $buf,
1574                       $cmd, $sensebuf, 6000);
1575
1576     $ret = ioctl($fh, $SG_IO, $packet);
1577     if (!$ret) {
1578         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1579         return undef;
1580     }
1581
1582     my @res = unpack($sg_io_hdr_t, $packet);
1583     if ($res[17] || $res[18]) {
1584         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1585         return undef;
1586     }
1587
1588     my $res = {};
1589     (my $byte0, my $byte1, $res->{vendor},
1590      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1591
1592     $res->{removable} = $byte1 & 128 ? 1 : 0;
1593     $res->{type} = $byte0 & 31;
1594
1595     return $res;
1596 }
1597
1598 sub path_is_scsi {
1599     my ($path) = @_;
1600
1601     my $fh = IO::File->new("+<$path") || return undef;
1602     my $res = scsi_inquiry($fh, 1);
1603     close($fh);
1604
1605     return $res;
1606 }
1607
1608 sub machine_type_is_q35 {
1609     my ($conf) = @_;
1610
1611     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1612 }
1613
1614 sub print_tabletdevice_full {
1615     my ($conf) = @_;
1616
1617     my $q35 = machine_type_is_q35($conf);
1618
1619     # we use uhci for old VMs because tablet driver was buggy in older qemu
1620     my $usbbus = $q35 ? "ehci" : "uhci";
1621
1622     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1623 }
1624
1625 sub print_drivedevice_full {
1626     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1627
1628     my $device = '';
1629     my $maxdev = 0;
1630
1631     if ($drive->{interface} eq 'virtio') {
1632         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1633         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1634         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1635     } elsif ($drive->{interface} eq 'scsi') {
1636
1637         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1638         my $unit = $drive->{index} % $maxdev;
1639         my $devicetype = 'hd';
1640         my $path = '';
1641         if (drive_is_cdrom($drive)) {
1642             $devicetype = 'cd';
1643         } else {
1644             if ($drive->{file} =~ m|^/|) {
1645                 $path = $drive->{file};
1646                 if (my $info = path_is_scsi($path)) {
1647                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1648                         $devicetype = 'block';
1649                     } elsif ($info->{type} == 1) { # tape
1650                         $devicetype = 'generic';
1651                     }
1652                 }
1653             } else {
1654                  $path = PVE::Storage::path($storecfg, $drive->{file});
1655             }
1656
1657             if($path =~ m/^iscsi\:\/\//){
1658                 $devicetype = 'generic';
1659             }
1660         }
1661
1662         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1663             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1664         } else {
1665             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1666         }
1667
1668     } elsif ($drive->{interface} eq 'ide'){
1669         $maxdev = 2;
1670         my $controller = int($drive->{index} / $maxdev);
1671         my $unit = $drive->{index} % $maxdev;
1672         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1673
1674         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1675         if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
1676             $model = URI::Escape::uri_unescape($model);
1677             $device .= ",model=$model";
1678         }
1679     } elsif ($drive->{interface} eq 'sata'){
1680         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1681         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1682         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1683     } elsif ($drive->{interface} eq 'usb') {
1684         die "implement me";
1685         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1686     } else {
1687         die "unsupported interface type";
1688     }
1689
1690     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1691
1692     if (my $serial = $drive->{serial}) {
1693         $serial = URI::Escape::uri_unescape($serial);
1694         $device .= ",serial=$serial";
1695     }
1696
1697
1698     return $device;
1699 }
1700
1701 sub get_initiator_name {
1702     my $initiator;
1703
1704     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1705     while (defined(my $line = <$fh>)) {
1706         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1707         $initiator = $1;
1708         last;
1709     }
1710     $fh->close();
1711
1712     return $initiator;
1713 }
1714
1715 sub print_drive_full {
1716     my ($storecfg, $vmid, $drive) = @_;
1717
1718     my $path;
1719     my $volid = $drive->{file};
1720     my $format;
1721
1722     if (drive_is_cdrom($drive)) {
1723         $path = get_iso_path($storecfg, $vmid, $volid);
1724     } else {
1725         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1726         if ($storeid) {
1727             $path = PVE::Storage::path($storecfg, $volid);
1728             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1729             $format = qemu_img_format($scfg, $volname);
1730         } else {
1731             $path = $volid;
1732             $format = "raw";
1733         }
1734    }
1735
1736     my $opts = '';
1737     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1738     foreach my $o (@qemu_drive_options) {
1739         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1740     }
1741
1742     # snapshot only accepts on|off
1743     if (defined($drive->{snapshot})) {
1744         my $v = $drive->{snapshot} ? 'on' : 'off';
1745         $opts .= ",snapshot=$v";
1746     }
1747
1748     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1749         my ($dir, $qmpname) = @$type;
1750         if (my $v = $drive->{"mbps$dir"}) {
1751             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1752         }
1753         if (my $v = $drive->{"mbps${dir}_max"}) {
1754             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1755         }
1756         if (my $v = $drive->{"bps${dir}_max_length"}) {
1757             $opts .= ",throttling.bps$qmpname-max-length=$v";
1758         }
1759         if (my $v = $drive->{"iops${dir}"}) {
1760             $opts .= ",throttling.iops$qmpname=$v";
1761         }
1762         if (my $v = $drive->{"iops${dir}_max"}) {
1763             $opts .= ",throttling.iops$qmpname-max=$v";
1764         }
1765         if (my $v = $drive->{"iops${dir}_max_length"}) {
1766             $opts .= ",throttling.iops$qmpname-max-length=$v";
1767         }
1768     }
1769
1770     $opts .= ",format=$format" if $format && !$drive->{format};
1771
1772     my $cache_direct = 0;
1773
1774     if (my $cache = $drive->{cache}) {
1775         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1776     } elsif (!drive_is_cdrom($drive)) {
1777         $opts .= ",cache=none";
1778         $cache_direct = 1;
1779     }
1780
1781     # aio native works only with O_DIRECT
1782     if (!$drive->{aio}) {
1783         if($cache_direct) {
1784             $opts .= ",aio=native";
1785         } else {
1786             $opts .= ",aio=threads";
1787         }
1788     }
1789
1790     if (!drive_is_cdrom($drive)) {
1791         my $detectzeroes;
1792         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1793             $detectzeroes = 'off';
1794         } elsif ($drive->{discard}) {
1795             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1796         } else {
1797             # This used to be our default with discard not being specified:
1798             $detectzeroes = 'on';
1799         }
1800         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1801     }
1802
1803     my $pathinfo = $path ? "file=$path," : '';
1804
1805     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1806 }
1807
1808 sub print_netdevice_full {
1809     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1810
1811     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1812
1813     my $device = $net->{model};
1814     if ($net->{model} eq 'virtio') {
1815          $device = 'virtio-net-pci';
1816      };
1817
1818     my $pciaddr = print_pci_addr("$netid", $bridges);
1819     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1820     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1821         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1822         my $vectors = $net->{queues} * 2 + 2;
1823         $tmpstr .= ",vectors=$vectors,mq=on";
1824     }
1825     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1826
1827     if ($use_old_bios_files) {
1828         my $romfile;
1829         if ($device eq 'virtio-net-pci') {
1830             $romfile = 'pxe-virtio.rom';
1831         } elsif ($device eq 'e1000') {
1832             $romfile = 'pxe-e1000.rom';
1833         } elsif ($device eq 'ne2k') {
1834             $romfile = 'pxe-ne2k_pci.rom';
1835         } elsif ($device eq 'pcnet') {
1836             $romfile = 'pxe-pcnet.rom';
1837         } elsif ($device eq 'rtl8139') {
1838             $romfile = 'pxe-rtl8139.rom';
1839         }
1840         $tmpstr .= ",romfile=$romfile" if $romfile;
1841     }
1842
1843     return $tmpstr;
1844 }
1845
1846 sub print_netdev_full {
1847     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1848
1849     my $i = '';
1850     if ($netid =~ m/^net(\d+)$/) {
1851         $i = int($1);
1852     }
1853
1854     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1855
1856     my $ifname = "tap${vmid}i$i";
1857
1858     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1859     die "interface name '$ifname' is too long (max 15 character)\n"
1860         if length($ifname) >= 16;
1861
1862     my $vhostparam = '';
1863     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1864
1865     my $vmname = $conf->{name} || "vm$vmid";
1866
1867     my $netdev = "";
1868     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1869
1870     if ($net->{bridge}) {
1871         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1872     } else {
1873         $netdev = "type=user,id=$netid,hostname=$vmname";
1874     }
1875
1876     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1877
1878     return $netdev;
1879 }
1880
1881
1882 sub print_cpu_device {
1883     my ($conf, $id) = @_;
1884
1885     my $kvm = $conf->{kvm} // 1;
1886     my $cpu = $kvm ? "kvm64" : "qemu64";
1887     if (my $cputype = $conf->{cpu}) {
1888         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
1889             or die "Cannot parse cpu description: $cputype\n";
1890         $cpu = $cpuconf->{cputype};
1891     }
1892
1893     my $cores = $conf->{cores} || 1;
1894
1895     my $current_core = ($id - 1) % $cores;
1896     my $current_socket = int(($id - 1 - $current_core)/$cores);
1897
1898     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
1899 }
1900
1901 sub drive_is_cloudinit {
1902     my ($drive) = @_;
1903     return $drive->{file} =~ m@[:/]vm-\d+-cloudinit(?:\.$QEMU_FORMAT_RE)?$@;
1904 }
1905
1906 sub drive_is_cdrom {
1907     my ($drive, $exclude_cloudinit) = @_;
1908
1909     return 0 if $exclude_cloudinit && drive_is_cloudinit($drive);
1910
1911     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1912
1913 }
1914
1915 sub parse_number_sets {
1916     my ($set) = @_;
1917     my $res = [];
1918     foreach my $part (split(/;/, $set)) {
1919         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1920             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1921             push @$res, [ $1, $2 ];
1922         } else {
1923             die "invalid range: $part\n";
1924         }
1925     }
1926     return $res;
1927 }
1928
1929 sub parse_numa {
1930     my ($data) = @_;
1931
1932     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
1933     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1934     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1935     return $res;
1936 }
1937
1938 sub parse_hostpci {
1939     my ($value) = @_;
1940
1941     return undef if !$value;
1942
1943     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
1944
1945     my @idlist = split(/;/, $res->{host});
1946     delete $res->{host};
1947     foreach my $id (@idlist) {
1948         if ($id =~ /^$PCIRE$/) {
1949             if (defined($2)) {
1950                 push @{$res->{pciid}}, { id => $1, function => $2 };
1951             } else {
1952                 my $pcidevices = lspci($1);
1953                 $res->{pciid} = $pcidevices->{$1};
1954             }
1955         } else {
1956             # should have been caught by parse_property_string already
1957             die "failed to parse PCI id: $id\n";
1958         }
1959     }
1960     return $res;
1961 }
1962
1963 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1964 sub parse_net {
1965     my ($data) = @_;
1966
1967     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
1968     if ($@) {
1969         warn $@;
1970         return undef;
1971     }
1972     if (!defined($res->{macaddr})) {
1973         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1974         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1975     }
1976     return $res;
1977 }
1978
1979 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1980 sub parse_ipconfig {
1981     my ($data) = @_;
1982
1983     my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
1984     if ($@) {
1985         warn $@;
1986         return undef;
1987     }
1988
1989     if ($res->{gw} && !$res->{ip}) {
1990         warn 'gateway specified without specifying an IP address';
1991         return undef;
1992     }
1993     if ($res->{gw6} && !$res->{ip6}) {
1994         warn 'IPv6 gateway specified without specifying an IPv6 address';
1995         return undef;
1996     }
1997     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1998         warn 'gateway specified together with DHCP';
1999         return undef;
2000     }
2001     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
2002         # gw6 + auto/dhcp
2003         warn "IPv6 gateway specified together with $res->{ip6} address";
2004         return undef;
2005     }
2006
2007     if (!$res->{ip} && !$res->{ip6}) {
2008         return { ip => 'dhcp', ip6 => 'dhcp' };
2009     }
2010
2011     return $res;
2012 }
2013
2014 sub print_net {
2015     my $net = shift;
2016
2017     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2018 }
2019
2020 sub add_random_macs {
2021     my ($settings) = @_;
2022
2023     foreach my $opt (keys %$settings) {
2024         next if $opt !~ m/^net(\d+)$/;
2025         my $net = parse_net($settings->{$opt});
2026         next if !$net;
2027         $settings->{$opt} = print_net($net);
2028     }
2029 }
2030
2031 sub vm_is_volid_owner {
2032     my ($storecfg, $vmid, $volid) = @_;
2033
2034     if ($volid !~  m|^/|) {
2035         my ($path, $owner);
2036         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2037         if ($owner && ($owner == $vmid)) {
2038             return 1;
2039         }
2040     }
2041
2042     return undef;
2043 }
2044
2045 sub split_flagged_list {
2046     my $text = shift || '';
2047     $text =~ s/[,;]/ /g;
2048     $text =~ s/^\s+//;
2049     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
2050 }
2051
2052 sub join_flagged_list {
2053     my ($how, $lst) = @_;
2054     join $how, map { $lst->{$_} . $_ } keys %$lst;
2055 }
2056
2057 sub vmconfig_delete_pending_option {
2058     my ($conf, $key, $force) = @_;
2059
2060     delete $conf->{pending}->{$key};
2061     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2062     $pending_delete_hash->{$key} = $force ? '!' : '';
2063     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2064 }
2065
2066 sub vmconfig_undelete_pending_option {
2067     my ($conf, $key) = @_;
2068
2069     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2070     delete $pending_delete_hash->{$key};
2071
2072     if (%$pending_delete_hash) {
2073         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2074     } else {
2075         delete $conf->{pending}->{delete};
2076     }
2077 }
2078
2079 sub vmconfig_register_unused_drive {
2080     my ($storecfg, $vmid, $conf, $drive) = @_;
2081
2082     if (drive_is_cloudinit($drive)) {
2083         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2084         warn $@ if $@;
2085     } elsif (!drive_is_cdrom($drive)) {
2086         my $volid = $drive->{file};
2087         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2088             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2089         }
2090     }
2091 }
2092
2093 sub vmconfig_cleanup_pending {
2094     my ($conf) = @_;
2095
2096     # remove pending changes when nothing changed
2097     my $changes;
2098     foreach my $opt (keys %{$conf->{pending}}) {
2099         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
2100             $changes = 1;
2101             delete $conf->{pending}->{$opt};
2102         }
2103     }
2104
2105     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
2106     my $pending_delete_hash = {};
2107     while (my ($opt, $force) = each %$current_delete_hash) {
2108         if (defined($conf->{$opt})) {
2109             $pending_delete_hash->{$opt} = $force;
2110         } else {
2111             $changes = 1;
2112         }
2113     }
2114
2115     if (%$pending_delete_hash) {
2116         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2117     } else {
2118         delete $conf->{pending}->{delete};
2119     }
2120
2121     return $changes;
2122 }
2123
2124 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
2125 my $smbios1_fmt = {
2126     uuid => {
2127         type => 'string',
2128         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2129         format_description => 'UUID',
2130         description => "Set SMBIOS1 UUID.",
2131         optional => 1,
2132     },
2133     version => {
2134         type => 'string',
2135         pattern => '\S+',
2136         format_description => 'string',
2137         description => "Set SMBIOS1 version.",
2138         optional => 1,
2139     },
2140     serial => {
2141         type => 'string',
2142         pattern => '\S+',
2143         format_description => 'string',
2144         description => "Set SMBIOS1 serial number.",
2145         optional => 1,
2146     },
2147     manufacturer => {
2148         type => 'string',
2149         pattern => '\S+',
2150         format_description => 'string',
2151         description => "Set SMBIOS1 manufacturer.",
2152         optional => 1,
2153     },
2154     product => {
2155         type => 'string',
2156         pattern => '\S+',
2157         format_description => 'string',
2158         description => "Set SMBIOS1 product ID.",
2159         optional => 1,
2160     },
2161     sku => {
2162         type => 'string',
2163         pattern => '\S+',
2164         format_description => 'string',
2165         description => "Set SMBIOS1 SKU string.",
2166         optional => 1,
2167     },
2168     family => {
2169         type => 'string',
2170         pattern => '\S+',
2171         format_description => 'string',
2172         description => "Set SMBIOS1 family string.",
2173         optional => 1,
2174     },
2175 };
2176
2177 sub parse_smbios1 {
2178     my ($data) = @_;
2179
2180     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
2181     warn $@ if $@;
2182     return $res;
2183 }
2184
2185 sub print_smbios1 {
2186     my ($smbios1) = @_;
2187     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2188 }
2189
2190 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2191
2192 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2193 sub verify_bootdisk {
2194     my ($value, $noerr) = @_;
2195
2196     return $value if is_valid_drivename($value);
2197
2198     return undef if $noerr;
2199
2200     die "invalid boot disk '$value'\n";
2201 }
2202
2203 sub parse_watchdog {
2204     my ($value) = @_;
2205
2206     return undef if !$value;
2207
2208     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2209     warn $@ if $@;
2210     return $res;
2211 }
2212
2213 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2214 sub verify_usb_device {
2215     my ($value, $noerr) = @_;
2216
2217     return $value if parse_usb_device($value);
2218
2219     return undef if $noerr;
2220
2221     die "unable to parse usb device\n";
2222 }
2223
2224 # add JSON properties for create and set function
2225 sub json_config_properties {
2226     my $prop = shift;
2227
2228     foreach my $opt (keys %$confdesc) {
2229         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
2230         $prop->{$opt} = $confdesc->{$opt};
2231     }
2232
2233     return $prop;
2234 }
2235
2236 # return copy of $confdesc_cloudinit to generate documentation
2237 sub cloudinit_config_properties {
2238
2239     return dclone($confdesc_cloudinit);
2240 }
2241
2242 sub check_type {
2243     my ($key, $value) = @_;
2244
2245     die "unknown setting '$key'\n" if !$confdesc->{$key};
2246
2247     my $type = $confdesc->{$key}->{type};
2248
2249     if (!defined($value)) {
2250         die "got undefined value\n";
2251     }
2252
2253     if ($value =~ m/[\n\r]/) {
2254         die "property contains a line feed\n";
2255     }
2256
2257     if ($type eq 'boolean') {
2258         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2259         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2260         die "type check ('boolean') failed - got '$value'\n";
2261     } elsif ($type eq 'integer') {
2262         return int($1) if $value =~ m/^(\d+)$/;
2263         die "type check ('integer') failed - got '$value'\n";
2264     } elsif ($type eq 'number') {
2265         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2266         die "type check ('number') failed - got '$value'\n";
2267     } elsif ($type eq 'string') {
2268         if (my $fmt = $confdesc->{$key}->{format}) {
2269             PVE::JSONSchema::check_format($fmt, $value);
2270             return $value;
2271         }
2272         $value =~ s/^\"(.*)\"$/$1/;
2273         return $value;
2274     } else {
2275         die "internal error"
2276     }
2277 }
2278
2279 sub check_iommu_support{
2280     #fixme : need to check IOMMU support
2281     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
2282
2283     my $iommu=1;
2284     return $iommu;
2285
2286 }
2287
2288 sub touch_config {
2289     my ($vmid) = @_;
2290
2291     my $conf = PVE::QemuConfig->config_file($vmid);
2292     utime undef, undef, $conf;
2293 }
2294
2295 sub destroy_vm {
2296     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2297
2298     my $conffile = PVE::QemuConfig->config_file($vmid);
2299
2300     my $conf = PVE::QemuConfig->load_config($vmid);
2301
2302     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2303
2304     if ($conf->{template}) {
2305         # check if any base image is still used by a linked clone
2306         foreach_drive($conf, sub {
2307                 my ($ds, $drive) = @_;
2308
2309                 return if drive_is_cdrom($drive);
2310
2311                 my $volid = $drive->{file};
2312
2313                 return if !$volid || $volid =~ m|^/|;
2314
2315                 die "base volume '$volid' is still in use by linked cloned\n"
2316                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2317
2318         });
2319     }
2320
2321     # only remove disks owned by this VM
2322     foreach_drive($conf, sub {
2323         my ($ds, $drive) = @_;
2324
2325         return if drive_is_cdrom($drive, 1);
2326
2327         my $volid = $drive->{file};
2328
2329         return if !$volid || $volid =~ m|^/|;
2330
2331         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2332         return if !$path || !$owner || ($owner != $vmid);
2333
2334         eval {
2335             PVE::Storage::vdisk_free($storecfg, $volid);
2336         };
2337         warn "Could not remove disk '$volid', check manually: $@" if $@;
2338
2339     });
2340
2341     if ($keep_empty_config) {
2342         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2343     } else {
2344         unlink $conffile;
2345     }
2346
2347     # also remove unused disk
2348     eval {
2349         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2350
2351         eval {
2352             PVE::Storage::foreach_volid($dl, sub {
2353                 my ($volid, $sid, $volname, $d) = @_;
2354                 PVE::Storage::vdisk_free($storecfg, $volid);
2355             });
2356         };
2357         warn $@ if $@;
2358
2359     };
2360     warn $@ if $@;
2361 }
2362
2363 sub parse_vm_config {
2364     my ($filename, $raw) = @_;
2365
2366     return undef if !defined($raw);
2367
2368     my $res = {
2369         digest => Digest::SHA::sha1_hex($raw),
2370         snapshots => {},
2371         pending => {},
2372     };
2373
2374     $filename =~ m|/qemu-server/(\d+)\.conf$|
2375         || die "got strange filename '$filename'";
2376
2377     my $vmid = $1;
2378
2379     my $conf = $res;
2380     my $descr;
2381     my $section = '';
2382
2383     my @lines = split(/\n/, $raw);
2384     foreach my $line (@lines) {
2385         next if $line =~ m/^\s*$/;
2386
2387         if ($line =~ m/^\[PENDING\]\s*$/i) {
2388             $section = 'pending';
2389             if (defined($descr)) {
2390                 $descr =~ s/\s+$//;
2391                 $conf->{description} = $descr;
2392             }
2393             $descr = undef;
2394             $conf = $res->{$section} = {};
2395             next;
2396
2397         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2398             $section = $1;
2399             if (defined($descr)) {
2400                 $descr =~ s/\s+$//;
2401                 $conf->{description} = $descr;
2402             }
2403             $descr = undef;
2404             $conf = $res->{snapshots}->{$section} = {};
2405             next;
2406         }
2407
2408         if ($line =~ m/^\#(.*)\s*$/) {
2409             $descr = '' if !defined($descr);
2410             $descr .= PVE::Tools::decode_text($1) . "\n";
2411             next;
2412         }
2413
2414         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2415             $descr = '' if !defined($descr);
2416             $descr .= PVE::Tools::decode_text($2);
2417         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2418             $conf->{snapstate} = $1;
2419         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2420             my $key = $1;
2421             my $value = $2;
2422             $conf->{$key} = $value;
2423         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2424             my $value = $1;
2425             if ($section eq 'pending') {
2426                 $conf->{delete} = $value; # we parse this later
2427             } else {
2428                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2429             }
2430         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2431             my $key = $1;
2432             my $value = $2;
2433             eval { $value = check_type($key, $value); };
2434             if ($@) {
2435                 warn "vm $vmid - unable to parse value of '$key' - $@";
2436             } else {
2437                 $key = 'ide2' if $key eq 'cdrom';
2438                 my $fmt = $confdesc->{$key}->{format};
2439                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2440                     my $v = parse_drive($key, $value);
2441                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2442                         $v->{file} = $volid;
2443                         $value = print_drive($vmid, $v);
2444                     } else {
2445                         warn "vm $vmid - unable to parse value of '$key'\n";
2446                         next;
2447                     }
2448                 }
2449
2450                 $conf->{$key} = $value;
2451             }
2452         }
2453     }
2454
2455     if (defined($descr)) {
2456         $descr =~ s/\s+$//;
2457         $conf->{description} = $descr;
2458     }
2459     delete $res->{snapstate}; # just to be sure
2460
2461     return $res;
2462 }
2463
2464 sub write_vm_config {
2465     my ($filename, $conf) = @_;
2466
2467     delete $conf->{snapstate}; # just to be sure
2468
2469     if ($conf->{cdrom}) {
2470         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2471         $conf->{ide2} = $conf->{cdrom};
2472         delete $conf->{cdrom};
2473     }
2474
2475     # we do not use 'smp' any longer
2476     if ($conf->{sockets}) {
2477         delete $conf->{smp};
2478     } elsif ($conf->{smp}) {
2479         $conf->{sockets} = $conf->{smp};
2480         delete $conf->{cores};
2481         delete $conf->{smp};
2482     }
2483
2484     my $used_volids = {};
2485
2486     my $cleanup_config = sub {
2487         my ($cref, $pending, $snapname) = @_;
2488
2489         foreach my $key (keys %$cref) {
2490             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2491                 $key eq 'snapstate' || $key eq 'pending';
2492             my $value = $cref->{$key};
2493             if ($key eq 'delete') {
2494                 die "propertry 'delete' is only allowed in [PENDING]\n"
2495                     if !$pending;
2496                 # fixme: check syntax?
2497                 next;
2498             }
2499             eval { $value = check_type($key, $value); };
2500             die "unable to parse value of '$key' - $@" if $@;
2501
2502             $cref->{$key} = $value;
2503
2504             if (!$snapname && is_valid_drivename($key)) {
2505                 my $drive = parse_drive($key, $value);
2506                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2507             }
2508         }
2509     };
2510
2511     &$cleanup_config($conf);
2512
2513     &$cleanup_config($conf->{pending}, 1);
2514
2515     foreach my $snapname (keys %{$conf->{snapshots}}) {
2516         die "internal error" if $snapname eq 'pending';
2517         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2518     }
2519
2520     # remove 'unusedX' settings if we re-add a volume
2521     foreach my $key (keys %$conf) {
2522         my $value = $conf->{$key};
2523         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2524             delete $conf->{$key};
2525         }
2526     }
2527
2528     my $generate_raw_config = sub {
2529         my ($conf, $pending) = @_;
2530
2531         my $raw = '';
2532
2533         # add description as comment to top of file
2534         if (defined(my $descr = $conf->{description})) {
2535             if ($descr) {
2536                 foreach my $cl (split(/\n/, $descr)) {
2537                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2538                 }
2539             } else {
2540                 $raw .= "#\n" if $pending;
2541             }
2542         }
2543
2544         foreach my $key (sort keys %$conf) {
2545             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2546             $raw .= "$key: $conf->{$key}\n";
2547         }
2548         return $raw;
2549     };
2550
2551     my $raw = &$generate_raw_config($conf);
2552
2553     if (scalar(keys %{$conf->{pending}})){
2554         $raw .= "\n[PENDING]\n";
2555         $raw .= &$generate_raw_config($conf->{pending}, 1);
2556     }
2557
2558     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2559         $raw .= "\n[$snapname]\n";
2560         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2561     }
2562
2563     return $raw;
2564 }
2565
2566 sub load_defaults {
2567
2568     my $res = {};
2569
2570     # we use static defaults from our JSON schema configuration
2571     foreach my $key (keys %$confdesc) {
2572         if (defined(my $default = $confdesc->{$key}->{default})) {
2573             $res->{$key} = $default;
2574         }
2575     }
2576
2577     return $res;
2578 }
2579
2580 sub config_list {
2581     my $vmlist = PVE::Cluster::get_vmlist();
2582     my $res = {};
2583     return $res if !$vmlist || !$vmlist->{ids};
2584     my $ids = $vmlist->{ids};
2585
2586     foreach my $vmid (keys %$ids) {
2587         my $d = $ids->{$vmid};
2588         next if !$d->{node} || $d->{node} ne $nodename;
2589         next if !$d->{type} || $d->{type} ne 'qemu';
2590         $res->{$vmid}->{exists} = 1;
2591     }
2592     return $res;
2593 }
2594
2595 # test if VM uses local resources (to prevent migration)
2596 sub check_local_resources {
2597     my ($conf, $noerr) = @_;
2598
2599     my $loc_res = 0;
2600
2601     $loc_res = 1 if $conf->{hostusb}; # old syntax
2602     $loc_res = 1 if $conf->{hostpci}; # old syntax
2603
2604     foreach my $k (keys %$conf) {
2605         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2606         # sockets are safe: they will recreated be on the target side post-migrate
2607         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2608         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2609     }
2610
2611     die "VM uses local resources\n" if $loc_res && !$noerr;
2612
2613     return $loc_res;
2614 }
2615
2616 # check if used storages are available on all nodes (use by migrate)
2617 sub check_storage_availability {
2618     my ($storecfg, $conf, $node) = @_;
2619
2620     foreach_drive($conf, sub {
2621         my ($ds, $drive) = @_;
2622
2623         my $volid = $drive->{file};
2624         return if !$volid;
2625
2626         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2627         return if !$sid;
2628
2629         # check if storage is available on both nodes
2630         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2631         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2632    });
2633 }
2634
2635 # list nodes where all VM images are available (used by has_feature API)
2636 sub shared_nodes {
2637     my ($conf, $storecfg) = @_;
2638
2639     my $nodelist = PVE::Cluster::get_nodelist();
2640     my $nodehash = { map { $_ => 1 } @$nodelist };
2641     my $nodename = PVE::INotify::nodename();
2642
2643     foreach_drive($conf, sub {
2644         my ($ds, $drive) = @_;
2645
2646         my $volid = $drive->{file};
2647         return if !$volid;
2648
2649         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2650         if ($storeid) {
2651             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2652             if ($scfg->{disable}) {
2653                 $nodehash = {};
2654             } elsif (my $avail = $scfg->{nodes}) {
2655                 foreach my $node (keys %$nodehash) {
2656                     delete $nodehash->{$node} if !$avail->{$node};
2657                 }
2658             } elsif (!$scfg->{shared}) {
2659                 foreach my $node (keys %$nodehash) {
2660                     delete $nodehash->{$node} if $node ne $nodename
2661                 }
2662             }
2663         }
2664     });
2665
2666     return $nodehash
2667 }
2668
2669 sub check_cmdline {
2670     my ($pidfile, $pid) = @_;
2671
2672     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2673     if (defined($fh)) {
2674         my $line = <$fh>;
2675         $fh->close;
2676         return undef if !$line;
2677         my @param = split(/\0/, $line);
2678
2679         my $cmd = $param[0];
2680         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2681
2682         for (my $i = 0; $i < scalar (@param); $i++) {
2683             my $p = $param[$i];
2684             next if !$p;
2685             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2686                 my $p = $param[$i+1];
2687                 return 1 if $p && ($p eq $pidfile);
2688                 return undef;
2689             }
2690         }
2691     }
2692     return undef;
2693 }
2694
2695 sub check_running {
2696     my ($vmid, $nocheck, $node) = @_;
2697
2698     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2699
2700     die "unable to find configuration file for VM $vmid - no such machine\n"
2701         if !$nocheck && ! -f $filename;
2702
2703     my $pidfile = pidfile_name($vmid);
2704
2705     if (my $fd = IO::File->new("<$pidfile")) {
2706         my $st = stat($fd);
2707         my $line = <$fd>;
2708         close($fd);
2709
2710         my $mtime = $st->mtime;
2711         if ($mtime > time()) {
2712             warn "file '$filename' modified in future\n";
2713         }
2714
2715         if ($line =~ m/^(\d+)$/) {
2716             my $pid = $1;
2717             if (check_cmdline($pidfile, $pid)) {
2718                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2719                     return $pid;
2720                 }
2721             }
2722         }
2723     }
2724
2725     return undef;
2726 }
2727
2728 sub vzlist {
2729
2730     my $vzlist = config_list();
2731
2732     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2733
2734     while (defined(my $de = $fd->read)) {
2735         next if $de !~ m/^(\d+)\.pid$/;
2736         my $vmid = $1;
2737         next if !defined($vzlist->{$vmid});
2738         if (my $pid = check_running($vmid)) {
2739             $vzlist->{$vmid}->{pid} = $pid;
2740         }
2741     }
2742
2743     return $vzlist;
2744 }
2745
2746 sub disksize {
2747     my ($storecfg, $conf) = @_;
2748
2749     my $bootdisk = $conf->{bootdisk};
2750     return undef if !$bootdisk;
2751     return undef if !is_valid_drivename($bootdisk);
2752
2753     return undef if !$conf->{$bootdisk};
2754
2755     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2756     return undef if !defined($drive);
2757
2758     return undef if drive_is_cdrom($drive);
2759
2760     my $volid = $drive->{file};
2761     return undef if !$volid;
2762
2763     return $drive->{size};
2764 }
2765
2766 my $last_proc_pid_stat;
2767
2768 # get VM status information
2769 # This must be fast and should not block ($full == false)
2770 # We only query KVM using QMP if $full == true (this can be slow)
2771 sub vmstatus {
2772     my ($opt_vmid, $full) = @_;
2773
2774     my $res = {};
2775
2776     my $storecfg = PVE::Storage::config();
2777
2778     my $list = vzlist();
2779     my $defaults = load_defaults();
2780
2781     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2782
2783     my $cpucount = $cpuinfo->{cpus} || 1;
2784
2785     foreach my $vmid (keys %$list) {
2786         next if $opt_vmid && ($vmid ne $opt_vmid);
2787
2788         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
2789         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2790
2791         my $d = {};
2792         $d->{pid} = $list->{$vmid}->{pid};
2793
2794         # fixme: better status?
2795         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2796
2797         my $size = disksize($storecfg, $conf);
2798         if (defined($size)) {
2799             $d->{disk} = 0; # no info available
2800             $d->{maxdisk} = $size;
2801         } else {
2802             $d->{disk} = 0;
2803             $d->{maxdisk} = 0;
2804         }
2805
2806         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2807             * ($conf->{cores} || $defaults->{cores});
2808         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2809         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2810
2811         $d->{name} = $conf->{name} || "VM $vmid";
2812         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2813             : $defaults->{memory}*(1024*1024);
2814
2815         if ($conf->{balloon}) {
2816             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2817             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2818                 : $defaults->{shares};
2819         }
2820
2821         $d->{uptime} = 0;
2822         $d->{cpu} = 0;
2823         $d->{mem} = 0;
2824
2825         $d->{netout} = 0;
2826         $d->{netin} = 0;
2827
2828         $d->{diskread} = 0;
2829         $d->{diskwrite} = 0;
2830
2831         $d->{template} = PVE::QemuConfig->is_template($conf);
2832
2833         $d->{serial} = 1 if conf_has_serial($conf);
2834
2835         $res->{$vmid} = $d;
2836     }
2837
2838     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2839     foreach my $dev (keys %$netdev) {
2840         next if $dev !~ m/^tap([1-9]\d*)i/;
2841         my $vmid = $1;
2842         my $d = $res->{$vmid};
2843         next if !$d;
2844
2845         $d->{netout} += $netdev->{$dev}->{receive};
2846         $d->{netin} += $netdev->{$dev}->{transmit};
2847
2848         if ($full) {
2849             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2850             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2851         }
2852
2853     }
2854
2855     my $ctime = gettimeofday;
2856
2857     foreach my $vmid (keys %$list) {
2858
2859         my $d = $res->{$vmid};
2860         my $pid = $d->{pid};
2861         next if !$pid;
2862
2863         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2864         next if !$pstat; # not running
2865
2866         my $used = $pstat->{utime} + $pstat->{stime};
2867
2868         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2869
2870         if ($pstat->{vsize}) {
2871             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2872         }
2873
2874         my $old = $last_proc_pid_stat->{$pid};
2875         if (!$old) {
2876             $last_proc_pid_stat->{$pid} = {
2877                 time => $ctime,
2878                 used => $used,
2879                 cpu => 0,
2880             };
2881             next;
2882         }
2883
2884         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2885
2886         if ($dtime > 1000) {
2887             my $dutime = $used -  $old->{used};
2888
2889             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2890             $last_proc_pid_stat->{$pid} = {
2891                 time => $ctime,
2892                 used => $used,
2893                 cpu => $d->{cpu},
2894             };
2895         } else {
2896             $d->{cpu} = $old->{cpu};
2897         }
2898     }
2899
2900     return $res if !$full;
2901
2902     my $qmpclient = PVE::QMPClient->new();
2903
2904     my $ballooncb = sub {
2905         my ($vmid, $resp) = @_;
2906
2907         my $info = $resp->{'return'};
2908         return if !$info->{max_mem};
2909
2910         my $d = $res->{$vmid};
2911
2912         # use memory assigned to VM
2913         $d->{maxmem} = $info->{max_mem};
2914         $d->{balloon} = $info->{actual};
2915
2916         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2917             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2918             $d->{freemem} = $info->{free_mem};
2919         }
2920
2921         $d->{ballooninfo} = $info;
2922     };
2923
2924     my $blockstatscb = sub {
2925         my ($vmid, $resp) = @_;
2926         my $data = $resp->{'return'} || [];
2927         my $totalrdbytes = 0;
2928         my $totalwrbytes = 0;
2929
2930         for my $blockstat (@$data) {
2931             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2932             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2933
2934             $blockstat->{device} =~ s/drive-//;
2935             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2936         }
2937         $res->{$vmid}->{diskread} = $totalrdbytes;
2938         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2939     };
2940
2941     my $statuscb = sub {
2942         my ($vmid, $resp) = @_;
2943
2944         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2945         # this fails if ballon driver is not loaded, so this must be
2946         # the last commnand (following command are aborted if this fails).
2947         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2948
2949         my $status = 'unknown';
2950         if (!defined($status = $resp->{'return'}->{status})) {
2951             warn "unable to get VM status\n";
2952             return;
2953         }
2954
2955         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2956     };
2957
2958     foreach my $vmid (keys %$list) {
2959         next if $opt_vmid && ($vmid ne $opt_vmid);
2960         next if !$res->{$vmid}->{pid}; # not running
2961         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2962     }
2963
2964     $qmpclient->queue_execute(undef, 2);
2965
2966     foreach my $vmid (keys %$list) {
2967         next if $opt_vmid && ($vmid ne $opt_vmid);
2968         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2969     }
2970
2971     return $res;
2972 }
2973
2974 sub foreach_drive {
2975     my ($conf, $func, @param) = @_;
2976
2977     foreach my $ds (valid_drive_names()) {
2978         next if !defined($conf->{$ds});
2979
2980         my $drive = parse_drive($ds, $conf->{$ds});
2981         next if !$drive;
2982
2983         &$func($ds, $drive, @param);
2984     }
2985 }
2986
2987 sub foreach_volid {
2988     my ($conf, $func, @param) = @_;
2989
2990     my $volhash = {};
2991
2992     my $test_volid = sub {
2993         my ($volid, $is_cdrom, $replicate, $shared, $snapname) = @_;
2994
2995         return if !$volid;
2996
2997         $volhash->{$volid}->{cdrom} //= 1;
2998         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
2999
3000         $volhash->{$volid}->{replicate} //= 0;
3001         $volhash->{$volid}->{replicate} = 1 if $replicate;
3002
3003         $volhash->{$volid}->{shared} //= 0;
3004         $volhash->{$volid}->{shared} = 1 if $shared;
3005
3006         $volhash->{$volid}->{referenced_in_config} //= 0;
3007         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
3008
3009         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
3010             if defined($snapname);
3011     };
3012
3013     foreach_drive($conf, sub {
3014         my ($ds, $drive) = @_;
3015         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, undef);
3016     });
3017
3018     foreach my $snapname (keys %{$conf->{snapshots}}) {
3019         my $snap = $conf->{snapshots}->{$snapname};
3020         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
3021         foreach_drive($snap, sub {
3022             my ($ds, $drive) = @_;
3023             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, $snapname);
3024         });
3025     }
3026
3027     foreach my $volid (keys %$volhash) {
3028         &$func($volid, $volhash->{$volid}, @param);
3029     }
3030 }
3031
3032 sub conf_has_serial {
3033     my ($conf) = @_;
3034
3035     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3036         if ($conf->{"serial$i"}) {
3037             return 1;
3038         }
3039     }
3040
3041     return 0;
3042 }
3043
3044 sub vga_conf_has_spice {
3045     my ($vga) = @_;
3046
3047     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
3048
3049     return $1 || 1;
3050 }
3051
3052 sub config_to_command {
3053     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
3054
3055     my $cmd = [];
3056     my $globalFlags = [];
3057     my $machineFlags = [];
3058     my $rtcFlags = [];
3059     my $cpuFlags = [];
3060     my $devices = [];
3061     my $pciaddr = '';
3062     my $bridges = {};
3063     my $kvmver = kvm_user_version();
3064     my $vernum = 0; # unknown
3065     my $ostype = $conf->{ostype};
3066     my $winversion = windows_version($ostype);
3067     my $kvm = $conf->{kvm} // 1;
3068
3069     die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n" if (!$cpuinfo->{hvm} && $kvm);
3070
3071     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
3072         $vernum = $1*1000000+$2*1000;
3073     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
3074         $vernum = $1*1000000+$2*1000+$3;
3075     }
3076
3077     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
3078
3079     my $have_ovz = -f '/proc/vz/vestat';
3080
3081     my $q35 = machine_type_is_q35($conf);
3082     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3083     my $machine_type = $forcemachine || $conf->{machine};
3084     my $use_old_bios_files = undef;
3085     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3086
3087     my $cpuunits = defined($conf->{cpuunits}) ?
3088             $conf->{cpuunits} : $defaults->{cpuunits};
3089
3090     push @$cmd, '/usr/bin/kvm';
3091
3092     push @$cmd, '-id', $vmid;
3093
3094     my $vmname = $conf->{name} || "vm$vmid";
3095
3096     push @$cmd, '-name', $vmname;
3097
3098     my $use_virtio = 0;
3099
3100     my $qmpsocket = qmp_socket($vmid);
3101     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3102     push @$cmd, '-mon', "chardev=qmp,mode=control";
3103
3104
3105     push @$cmd, '-pidfile' , pidfile_name($vmid);
3106
3107     push @$cmd, '-daemonize';
3108
3109     if ($conf->{smbios1}) {
3110         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3111     }
3112
3113     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3114         die "uefi base image not found\n" if ! -f $OVMF_CODE;
3115
3116         my $path;
3117         my $format;
3118         if (my $efidisk = $conf->{efidisk0}) {
3119             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
3120             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3121             $format = $d->{format};
3122             if ($storeid) {
3123                 $path = PVE::Storage::path($storecfg, $d->{file});
3124                 if (!defined($format)) {
3125                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3126                     $format = qemu_img_format($scfg, $volname);
3127                 }
3128             } else {
3129                 $path = $d->{file};
3130                 die "efidisk format must be specified\n"
3131                     if !defined($format);
3132             }
3133         } else {
3134             warn "no efidisk configured! Using temporary efivars disk.\n";
3135             $path = "/tmp/$vmid-ovmf.fd";
3136             PVE::Tools::file_copy($OVMF_VARS, $path, -s $OVMF_VARS);
3137             $format = 'raw';
3138         }
3139
3140         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$OVMF_CODE";
3141         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
3142     }
3143
3144
3145     # add usb controllers
3146     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $q35, $usbdesc->{format}, $MAX_USB_DEVICES);
3147     push @$devices, @usbcontrollers if @usbcontrollers;
3148     my $vga = $conf->{vga};
3149
3150     my $qxlnum = vga_conf_has_spice($vga);
3151     $vga = 'qxl' if $qxlnum;
3152
3153     if (!$vga) {
3154         if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 9)) {
3155             $vga = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3156         } else {
3157             $vga = ($winversion >= 6) ? 'std' : 'cirrus';
3158         }
3159     }
3160
3161     # enable absolute mouse coordinates (needed by vnc)
3162     my $tablet;
3163     if (defined($conf->{tablet})) {
3164         $tablet = $conf->{tablet};
3165     } else {
3166         $tablet = $defaults->{tablet};
3167         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3168         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3169     }
3170
3171     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
3172
3173     my $kvm_off = 0;
3174     my $gpu_passthrough;
3175
3176     # host pci devices
3177     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3178         my $d = parse_hostpci($conf->{"hostpci$i"});
3179         next if !$d;
3180
3181         my $pcie = $d->{pcie};
3182         if($pcie){
3183             die "q35 machine model is not enabled" if !$q35;
3184             $pciaddr = print_pcie_addr("hostpci$i");
3185         }else{
3186             $pciaddr = print_pci_addr("hostpci$i", $bridges);
3187         }
3188
3189         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
3190         my $romfile = $d->{romfile};
3191
3192         my $xvga = '';
3193         if ($d->{'x-vga'}) {
3194             $xvga = ',x-vga=on';
3195             $kvm_off = 1;
3196             $vga = 'none';
3197             $gpu_passthrough = 1;
3198
3199             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3200                 $xvga = "";
3201             }
3202         }
3203         my $pcidevices = $d->{pciid};
3204         my $multifunction = 1 if @$pcidevices > 1;
3205
3206         my $j=0;
3207         foreach my $pcidevice (@$pcidevices) {
3208
3209             my $id = "hostpci$i";
3210             $id .= ".$j" if $multifunction;
3211             my $addr = $pciaddr;
3212             $addr .= ".$j" if $multifunction;
3213             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
3214
3215             if($j == 0){
3216                 $devicestr .= "$rombar$xvga";
3217                 $devicestr .= ",multifunction=on" if $multifunction;
3218                 $devicestr .= ",romfile=/usr/share/kvm/$romfile" if $romfile;
3219             }
3220
3221             push @$devices, '-device', $devicestr;
3222             $j++;
3223         }
3224     }
3225
3226     # usb devices
3227     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES);
3228     push @$devices, @usbdevices if @usbdevices;
3229     # serial devices
3230     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3231         if (my $path = $conf->{"serial$i"}) {
3232             if ($path eq 'socket') {
3233                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3234                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3235                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3236             } else {
3237                 die "no such serial device\n" if ! -c $path;
3238                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3239                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3240             }
3241         }
3242     }
3243
3244     # parallel devices
3245     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3246         if (my $path = $conf->{"parallel$i"}) {
3247             die "no such parallel device\n" if ! -c $path;
3248             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3249             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3250             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3251         }
3252     }
3253
3254
3255     my $sockets = 1;
3256     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3257     $sockets = $conf->{sockets} if  $conf->{sockets};
3258
3259     my $cores = $conf->{cores} || 1;
3260
3261     my $maxcpus = $sockets * $cores;
3262
3263     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3264
3265     my $allowed_vcpus = $cpuinfo->{cpus};
3266
3267     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3268         if ($allowed_vcpus < $maxcpus);
3269
3270     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
3271
3272         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3273         for (my $i = 2; $i <= $vcpus; $i++)  {
3274             my $cpustr = print_cpu_device($conf,$i);
3275             push @$cmd, '-device', $cpustr;
3276         }
3277
3278     } else {
3279
3280         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3281     }
3282     push @$cmd, '-nodefaults';
3283
3284     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3285
3286     my $bootindex_hash = {};
3287     my $i = 1;
3288     foreach my $o (split(//, $bootorder)) {
3289         $bootindex_hash->{$o} = $i*100;
3290         $i++;
3291     }
3292
3293     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3294
3295     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3296
3297     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3298
3299     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
3300
3301     if ($vga && $vga !~ m/^serial\d+$/ && $vga ne 'none'){
3302         my $socket = vnc_socket($vmid);
3303         push @$cmd,  '-vnc', "unix:$socket,x509,password";
3304     } else {
3305         push @$cmd, '-nographic';
3306     }
3307
3308     # time drift fix
3309     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3310
3311     my $useLocaltime = $conf->{localtime};
3312
3313     if ($winversion >= 5) { # windows
3314         $useLocaltime = 1 if !defined($conf->{localtime});
3315
3316         # use time drift fix when acpi is enabled
3317         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3318             $tdf = 1 if !defined($conf->{tdf});
3319         }
3320     }
3321
3322     if ($winversion >= 6) {
3323         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3324         push @$cmd, '-no-hpet';
3325     }
3326
3327     push @$rtcFlags, 'driftfix=slew' if $tdf;
3328
3329     if (!$kvm) {
3330         push @$machineFlags, 'accel=tcg';
3331     }
3332
3333     if ($machine_type) {
3334         push @$machineFlags, "type=${machine_type}";
3335     }
3336
3337     if ($conf->{startdate}) {
3338         push @$rtcFlags, "base=$conf->{startdate}";
3339     } elsif ($useLocaltime) {
3340         push @$rtcFlags, 'base=localtime';
3341     }
3342
3343     my $cpu = $kvm ? "kvm64" : "qemu64";
3344     if (my $cputype = $conf->{cpu}) {
3345         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3346             or die "Cannot parse cpu description: $cputype\n";
3347         $cpu = $cpuconf->{cputype};
3348         $kvm_off = 1 if $cpuconf->{hidden};
3349
3350         if (defined(my $flags = $cpuconf->{flags})) {
3351             push @$cpuFlags, split(";", $flags);
3352         }
3353     }
3354
3355     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
3356
3357     push @$cpuFlags , '-x2apic'
3358         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3359
3360     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3361
3362     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3363
3364     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3365
3366         push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3367         push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3368     }
3369
3370     add_hyperv_enlightenments($cpuFlags, $winversion, $machine_type, $kvmver, $conf->{bios}, $gpu_passthrough) if $kvm;
3371
3372     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm;
3373
3374     push @$cpuFlags, 'kvm=off' if $kvm_off;
3375
3376     my $cpu_vendor = $cpu_vendor_list->{$cpu} ||
3377         die "internal error"; # should not happen
3378
3379     push @$cpuFlags, "vendor=${cpu_vendor}"
3380         if $cpu_vendor ne 'default';
3381
3382     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3383
3384     push @$cmd, '-cpu', $cpu;
3385
3386     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3387
3388     push @$cmd, '-S' if $conf->{freeze};
3389
3390     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3391
3392     # enable sound
3393     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3394     #push @$cmd, '-soundhw', 'es1370';
3395     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3396
3397     if($conf->{agent}) {
3398         my $qgasocket = qmp_socket($vmid, 1);
3399         my $pciaddr = print_pci_addr("qga0", $bridges);
3400         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3401         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3402         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3403     }
3404
3405     my $spice_port;
3406
3407     if ($qxlnum) {
3408         if ($qxlnum > 1) {
3409             if ($winversion){
3410                 for(my $i = 1; $i < $qxlnum; $i++){
3411                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3412                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3413                 }
3414             } else {
3415                 # assume other OS works like Linux
3416                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3417                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3418             }
3419         }
3420
3421         my $pciaddr = print_pci_addr("spice", $bridges);
3422
3423         my $nodename = PVE::INotify::nodename();
3424         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3425         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3426         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3427         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3428         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3429
3430         push @$devices, '-spice', "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3431
3432         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3433         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3434         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3435     }
3436
3437     # enable balloon by default, unless explicitly disabled
3438     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3439         $pciaddr = print_pci_addr("balloon0", $bridges);
3440         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3441     }
3442
3443     if ($conf->{watchdog}) {
3444         my $wdopts = parse_watchdog($conf->{watchdog});
3445         $pciaddr = print_pci_addr("watchdog", $bridges);
3446         my $watchdog = $wdopts->{model} || 'i6300esb';
3447         push @$devices, '-device', "$watchdog$pciaddr";
3448         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3449     }
3450
3451     my $vollist = [];
3452     my $scsicontroller = {};
3453     my $ahcicontroller = {};
3454     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3455
3456     # Add iscsi initiator name if available
3457     if (my $initiator = get_initiator_name()) {
3458         push @$devices, '-iscsi', "initiator-name=$initiator";
3459     }
3460
3461     foreach_drive($conf, sub {
3462         my ($ds, $drive) = @_;
3463
3464         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3465             push @$vollist, $drive->{file};
3466         }
3467
3468         # ignore efidisk here, already added in bios/fw handling code above
3469         return if $drive->{interface} eq 'efidisk';
3470
3471         $use_virtio = 1 if $ds =~ m/^virtio/;
3472
3473         if (drive_is_cdrom ($drive)) {
3474             if ($bootindex_hash->{d}) {
3475                 $drive->{bootindex} = $bootindex_hash->{d};
3476                 $bootindex_hash->{d} += 1;
3477             }
3478         } else {
3479             if ($bootindex_hash->{c}) {
3480                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3481                 $bootindex_hash->{c} += 1;
3482             }
3483         }
3484
3485         if($drive->{interface} eq 'virtio'){
3486            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3487         }
3488
3489         if ($drive->{interface} eq 'scsi') {
3490
3491             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3492
3493             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3494             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3495
3496             my $iothread = '';
3497             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3498                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3499                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3500             } elsif ($drive->{iothread}) {
3501                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3502             }
3503
3504             my $queues = '';
3505             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3506                 $queues = ",num_queues=$drive->{queues}";
3507             }
3508
3509             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3510             $scsicontroller->{$controller}=1;
3511         }
3512
3513         if ($drive->{interface} eq 'sata') {
3514            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3515            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3516            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3517            $ahcicontroller->{$controller}=1;
3518         }
3519
3520         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3521         push @$devices, '-drive',$drive_cmd;
3522         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3523     });
3524
3525     for (my $i = 0; $i < $MAX_NETS; $i++) {
3526          next if !$conf->{"net$i"};
3527          my $d = parse_net($conf->{"net$i"});
3528          next if !$d;
3529
3530          $use_virtio = 1 if $d->{model} eq 'virtio';
3531
3532          if ($bootindex_hash->{n}) {
3533             $d->{bootindex} = $bootindex_hash->{n};
3534             $bootindex_hash->{n} += 1;
3535          }
3536
3537          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3538          push @$devices, '-netdev', $netdevfull;
3539
3540          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3541          push @$devices, '-device', $netdevicefull;
3542     }
3543
3544     if (!$q35) {
3545         # add pci bridges
3546         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3547            $bridges->{1} = 1;
3548            $bridges->{2} = 1;
3549         }
3550
3551         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3552
3553         while (my ($k, $v) = each %$bridges) {
3554             $pciaddr = print_pci_addr("pci.$k");
3555             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3556         }
3557     }
3558
3559     # add custom args
3560     if ($conf->{args}) {
3561         my $aa = PVE::Tools::split_args($conf->{args});
3562         push @$cmd, @$aa;
3563     }
3564
3565     push @$cmd, @$devices;
3566     push @$cmd, '-rtc', join(',', @$rtcFlags)
3567         if scalar(@$rtcFlags);
3568     push @$cmd, '-machine', join(',', @$machineFlags)
3569         if scalar(@$machineFlags);
3570     push @$cmd, '-global', join(',', @$globalFlags)
3571         if scalar(@$globalFlags);
3572
3573     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3574 }
3575
3576 sub vnc_socket {
3577     my ($vmid) = @_;
3578     return "${var_run_tmpdir}/$vmid.vnc";
3579 }
3580
3581 sub spice_port {
3582     my ($vmid) = @_;
3583
3584     my $res = vm_mon_cmd($vmid, 'query-spice');
3585
3586     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3587 }
3588
3589 sub qmp_socket {
3590     my ($vmid, $qga) = @_;
3591     my $sockettype = $qga ? 'qga' : 'qmp';
3592     return "${var_run_tmpdir}/$vmid.$sockettype";
3593 }
3594
3595 sub pidfile_name {
3596     my ($vmid) = @_;
3597     return "${var_run_tmpdir}/$vmid.pid";
3598 }
3599
3600 sub vm_devices_list {
3601     my ($vmid) = @_;
3602
3603     my $res = vm_mon_cmd($vmid, 'query-pci');
3604     my $devices_to_check = [];
3605     my $devices = {};
3606     foreach my $pcibus (@$res) {
3607         push @$devices_to_check, @{$pcibus->{devices}},
3608     }
3609
3610     while (@$devices_to_check) {
3611         my $to_check = [];
3612         for my $d (@$devices_to_check) {
3613             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
3614             next if !$d->{'pci_bridge'};
3615
3616             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
3617             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
3618         }
3619         $devices_to_check = $to_check;
3620     }
3621
3622     my $resblock = vm_mon_cmd($vmid, 'query-block');
3623     foreach my $block (@$resblock) {
3624         if($block->{device} =~ m/^drive-(\S+)/){
3625                 $devices->{$1} = 1;
3626         }
3627     }
3628
3629     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3630     foreach my $mice (@$resmice) {
3631         if ($mice->{name} eq 'QEMU HID Tablet') {
3632             $devices->{tablet} = 1;
3633             last;
3634         }
3635     }
3636
3637     # for usb devices there is no query-usb
3638     # but we can iterate over the entries in
3639     # qom-list path=/machine/peripheral
3640     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3641     foreach my $per (@$resperipheral) {
3642         if ($per->{name} =~ m/^usb\d+$/) {
3643             $devices->{$per->{name}} = 1;
3644         }
3645     }
3646
3647     return $devices;
3648 }
3649
3650 sub vm_deviceplug {
3651     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3652
3653     my $q35 = machine_type_is_q35($conf);
3654
3655     my $devices_list = vm_devices_list($vmid);
3656     return 1 if defined($devices_list->{$deviceid});
3657
3658     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3659
3660     if ($deviceid eq 'tablet') {
3661
3662         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3663
3664     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3665
3666         die "usb hotplug currently not reliable\n";
3667         # since we can't reliably hot unplug all added usb devices
3668         # and usb passthrough disables live migration
3669         # we disable usb hotplugging for now
3670         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3671
3672     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3673
3674         qemu_iothread_add($vmid, $deviceid, $device);
3675
3676         qemu_driveadd($storecfg, $vmid, $device);
3677         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3678
3679         qemu_deviceadd($vmid, $devicefull);
3680         eval { qemu_deviceaddverify($vmid, $deviceid); };
3681         if (my $err = $@) {
3682             eval { qemu_drivedel($vmid, $deviceid); };
3683             warn $@ if $@;
3684             die $err;
3685         }
3686
3687     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3688
3689
3690         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3691         my $pciaddr = print_pci_addr($deviceid);
3692         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3693
3694         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3695
3696         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3697             qemu_iothread_add($vmid, $deviceid, $device);
3698             $devicefull .= ",iothread=iothread-$deviceid";
3699         }
3700
3701         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3702             $devicefull .= ",num_queues=$device->{queues}";
3703         }
3704
3705         qemu_deviceadd($vmid, $devicefull);
3706         qemu_deviceaddverify($vmid, $deviceid);
3707
3708     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3709
3710         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3711         qemu_driveadd($storecfg, $vmid, $device);
3712
3713         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3714         eval { qemu_deviceadd($vmid, $devicefull); };
3715         if (my $err = $@) {
3716             eval { qemu_drivedel($vmid, $deviceid); };
3717             warn $@ if $@;
3718             die $err;
3719         }
3720
3721     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3722
3723         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3724
3725         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
3726         my $use_old_bios_files = undef;
3727         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3728
3729         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3730         qemu_deviceadd($vmid, $netdevicefull);
3731         eval { qemu_deviceaddverify($vmid, $deviceid); };
3732         if (my $err = $@) {
3733             eval { qemu_netdevdel($vmid, $deviceid); };
3734             warn $@ if $@;
3735             die $err;
3736         }
3737
3738     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3739
3740         my $bridgeid = $2;
3741         my $pciaddr = print_pci_addr($deviceid);
3742         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3743
3744         qemu_deviceadd($vmid, $devicefull);
3745         qemu_deviceaddverify($vmid, $deviceid);
3746
3747     } else {
3748         die "can't hotplug device '$deviceid'\n";
3749     }
3750
3751     return 1;
3752 }
3753
3754 # fixme: this should raise exceptions on error!
3755 sub vm_deviceunplug {
3756     my ($vmid, $conf, $deviceid) = @_;
3757
3758     my $devices_list = vm_devices_list($vmid);
3759     return 1 if !defined($devices_list->{$deviceid});
3760
3761     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3762
3763     if ($deviceid eq 'tablet') {
3764
3765         qemu_devicedel($vmid, $deviceid);
3766
3767     } elsif ($deviceid =~ m/^usb\d+$/) {
3768
3769         die "usb hotplug currently not reliable\n";
3770         # when unplugging usb devices this way,
3771         # there may be remaining usb controllers/hubs
3772         # so we disable it for now
3773         qemu_devicedel($vmid, $deviceid);
3774         qemu_devicedelverify($vmid, $deviceid);
3775
3776     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3777
3778         qemu_devicedel($vmid, $deviceid);
3779         qemu_devicedelverify($vmid, $deviceid);
3780         qemu_drivedel($vmid, $deviceid);
3781         qemu_iothread_del($conf, $vmid, $deviceid);
3782
3783     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3784
3785         qemu_devicedel($vmid, $deviceid);
3786         qemu_devicedelverify($vmid, $deviceid);
3787         qemu_iothread_del($conf, $vmid, $deviceid);
3788
3789     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3790
3791         qemu_devicedel($vmid, $deviceid);
3792         qemu_drivedel($vmid, $deviceid);
3793         qemu_deletescsihw($conf, $vmid, $deviceid);
3794
3795     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3796
3797         qemu_devicedel($vmid, $deviceid);
3798         qemu_devicedelverify($vmid, $deviceid);
3799         qemu_netdevdel($vmid, $deviceid);
3800
3801     } else {
3802         die "can't unplug device '$deviceid'\n";
3803     }
3804
3805     return 1;
3806 }
3807
3808 sub qemu_deviceadd {
3809     my ($vmid, $devicefull) = @_;
3810
3811     $devicefull = "driver=".$devicefull;
3812     my %options =  split(/[=,]/, $devicefull);
3813
3814     vm_mon_cmd($vmid, "device_add" , %options);
3815 }
3816
3817 sub qemu_devicedel {
3818     my ($vmid, $deviceid) = @_;
3819
3820     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3821 }
3822
3823 sub qemu_iothread_add {
3824     my($vmid, $deviceid, $device) = @_;
3825
3826     if ($device->{iothread}) {
3827         my $iothreads = vm_iothreads_list($vmid);
3828         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3829     }
3830 }
3831
3832 sub qemu_iothread_del {
3833     my($conf, $vmid, $deviceid) = @_;
3834
3835     my $device = parse_drive($deviceid, $conf->{$deviceid});
3836     if ($device->{iothread}) {
3837         my $iothreads = vm_iothreads_list($vmid);
3838         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3839     }
3840 }
3841
3842 sub qemu_objectadd {
3843     my($vmid, $objectid, $qomtype) = @_;
3844
3845     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3846
3847     return 1;
3848 }
3849
3850 sub qemu_objectdel {
3851     my($vmid, $objectid) = @_;
3852
3853     vm_mon_cmd($vmid, "object-del", id => $objectid);
3854
3855     return 1;
3856 }
3857
3858 sub qemu_driveadd {
3859     my ($storecfg, $vmid, $device) = @_;
3860
3861     my $drive = print_drive_full($storecfg, $vmid, $device);
3862     $drive =~ s/\\/\\\\/g;
3863     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3864
3865     # If the command succeeds qemu prints: "OK"
3866     return 1 if $ret =~ m/OK/s;
3867
3868     die "adding drive failed: $ret\n";
3869 }
3870
3871 sub qemu_drivedel {
3872     my($vmid, $deviceid) = @_;
3873
3874     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3875     $ret =~ s/^\s+//;
3876
3877     return 1 if $ret eq "";
3878
3879     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3880     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3881
3882     die "deleting drive $deviceid failed : $ret\n";
3883 }
3884
3885 sub qemu_deviceaddverify {
3886     my ($vmid, $deviceid) = @_;
3887
3888     for (my $i = 0; $i <= 5; $i++) {
3889          my $devices_list = vm_devices_list($vmid);
3890          return 1 if defined($devices_list->{$deviceid});
3891          sleep 1;
3892     }
3893
3894     die "error on hotplug device '$deviceid'\n";
3895 }
3896
3897
3898 sub qemu_devicedelverify {
3899     my ($vmid, $deviceid) = @_;
3900
3901     # need to verify that the device is correctly removed as device_del
3902     # is async and empty return is not reliable
3903
3904     for (my $i = 0; $i <= 5; $i++) {
3905          my $devices_list = vm_devices_list($vmid);
3906          return 1 if !defined($devices_list->{$deviceid});
3907          sleep 1;
3908     }
3909
3910     die "error on hot-unplugging device '$deviceid'\n";
3911 }
3912
3913 sub qemu_findorcreatescsihw {
3914     my ($storecfg, $conf, $vmid, $device) = @_;
3915
3916     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3917
3918     my $scsihwid="$controller_prefix$controller";
3919     my $devices_list = vm_devices_list($vmid);
3920
3921     if(!defined($devices_list->{$scsihwid})) {
3922         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3923     }
3924
3925     return 1;
3926 }
3927
3928 sub qemu_deletescsihw {
3929     my ($conf, $vmid, $opt) = @_;
3930
3931     my $device = parse_drive($opt, $conf->{$opt});
3932
3933     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3934         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3935         return 1;
3936     }
3937
3938     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3939
3940     my $devices_list = vm_devices_list($vmid);
3941     foreach my $opt (keys %{$devices_list}) {
3942         if (PVE::QemuServer::is_valid_drivename($opt)) {
3943             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3944             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3945                 return 1;
3946             }
3947         }
3948     }
3949
3950     my $scsihwid="scsihw$controller";
3951
3952     vm_deviceunplug($vmid, $conf, $scsihwid);
3953
3954     return 1;
3955 }
3956
3957 sub qemu_add_pci_bridge {
3958     my ($storecfg, $conf, $vmid, $device) = @_;
3959
3960     my $bridges = {};
3961
3962     my $bridgeid;
3963
3964     print_pci_addr($device, $bridges);
3965
3966     while (my ($k, $v) = each %$bridges) {
3967         $bridgeid = $k;
3968     }
3969     return 1 if !defined($bridgeid) || $bridgeid < 1;
3970
3971     my $bridge = "pci.$bridgeid";
3972     my $devices_list = vm_devices_list($vmid);
3973
3974     if (!defined($devices_list->{$bridge})) {
3975         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3976     }
3977
3978     return 1;
3979 }
3980
3981 sub qemu_set_link_status {
3982     my ($vmid, $device, $up) = @_;
3983
3984     vm_mon_cmd($vmid, "set_link", name => $device,
3985                up => $up ? JSON::true : JSON::false);
3986 }
3987
3988 sub qemu_netdevadd {
3989     my ($vmid, $conf, $device, $deviceid) = @_;
3990
3991     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
3992     my %options =  split(/[=,]/, $netdev);
3993
3994     vm_mon_cmd($vmid, "netdev_add",  %options);
3995     return 1;
3996 }
3997
3998 sub qemu_netdevdel {
3999     my ($vmid, $deviceid) = @_;
4000
4001     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
4002 }
4003
4004 sub qemu_usb_hotplug {
4005     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
4006
4007     return if !$device;
4008
4009     # remove the old one first
4010     vm_deviceunplug($vmid, $conf, $deviceid);
4011
4012     # check if xhci controller is necessary and available
4013     if ($device->{usb3}) {
4014
4015         my $devicelist = vm_devices_list($vmid);
4016
4017         if (!$devicelist->{xhci}) {
4018             my $pciaddr = print_pci_addr("xhci");
4019             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4020         }
4021     }
4022     my $d = parse_usb_device($device->{host});
4023     $d->{usb3} = $device->{usb3};
4024
4025     # add the new one
4026     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d);
4027 }
4028
4029 sub qemu_cpu_hotplug {
4030     my ($vmid, $conf, $vcpus) = @_;
4031
4032     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
4033
4034     my $sockets = 1;
4035     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4036     $sockets = $conf->{sockets} if  $conf->{sockets};
4037     my $cores = $conf->{cores} || 1;
4038     my $maxcpus = $sockets * $cores;
4039
4040     $vcpus = $maxcpus if !$vcpus;
4041
4042     die "you can't add more vcpus than maxcpus\n"
4043         if $vcpus > $maxcpus;
4044
4045     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4046
4047     if ($vcpus < $currentvcpus) {
4048
4049         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4050
4051             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4052                 qemu_devicedel($vmid, "cpu$i");
4053                 my $retry = 0;
4054                 my $currentrunningvcpus = undef;
4055                 while (1) {
4056                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4057                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4058                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4059                     $retry++;
4060                     sleep 1;
4061                 }
4062                 #update conf after each succesfull cpu unplug
4063                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4064                 PVE::QemuConfig->write_config($vmid, $conf);
4065             }
4066         } else {
4067             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4068         }
4069
4070         return;
4071     }
4072
4073     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4074     die "vcpus in running vm does not match its configuration\n"
4075         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4076
4077     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4078
4079         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4080             my $cpustr = print_cpu_device($conf, $i);
4081             qemu_deviceadd($vmid, $cpustr);
4082
4083             my $retry = 0;
4084             my $currentrunningvcpus = undef;
4085             while (1) {
4086                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4087                 last if scalar(@{$currentrunningvcpus}) == $i;
4088                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4089                 sleep 1;
4090                 $retry++;
4091             }
4092             #update conf after each succesfull cpu hotplug
4093             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4094             PVE::QemuConfig->write_config($vmid, $conf);
4095         }
4096     } else {
4097
4098         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4099             vm_mon_cmd($vmid, "cpu-add", id => int($i));
4100         }
4101     }
4102 }
4103
4104 sub qemu_block_set_io_throttle {
4105     my ($vmid, $deviceid,
4106         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4107         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4108         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4109         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4110
4111     return if !check_running($vmid) ;
4112
4113     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4114         bps => int($bps),
4115         bps_rd => int($bps_rd),
4116         bps_wr => int($bps_wr),
4117         iops => int($iops),
4118         iops_rd => int($iops_rd),
4119         iops_wr => int($iops_wr),
4120         bps_max => int($bps_max),
4121         bps_rd_max => int($bps_rd_max),
4122         bps_wr_max => int($bps_wr_max),
4123         iops_max => int($iops_max),
4124         iops_rd_max => int($iops_rd_max),
4125         iops_wr_max => int($iops_wr_max),
4126         bps_max_length => int($bps_max_length),
4127         bps_rd_max_length => int($bps_rd_max_length),
4128         bps_wr_max_length => int($bps_wr_max_length),
4129         iops_max_length => int($iops_max_length),
4130         iops_rd_max_length => int($iops_rd_max_length),
4131         iops_wr_max_length => int($iops_wr_max_length),
4132     );
4133
4134 }
4135
4136 # old code, only used to shutdown old VM after update
4137 sub __read_avail {
4138     my ($fh, $timeout) = @_;
4139
4140     my $sel = new IO::Select;
4141     $sel->add($fh);
4142
4143     my $res = '';
4144     my $buf;
4145
4146     my @ready;
4147     while (scalar (@ready = $sel->can_read($timeout))) {
4148         my $count;
4149         if ($count = $fh->sysread($buf, 8192)) {
4150             if ($buf =~ /^(.*)\(qemu\) $/s) {
4151                 $res .= $1;
4152                 last;
4153             } else {
4154                 $res .= $buf;
4155             }
4156         } else {
4157             if (!defined($count)) {
4158                 die "$!\n";
4159             }
4160             last;
4161         }
4162     }
4163
4164     die "monitor read timeout\n" if !scalar(@ready);
4165
4166     return $res;
4167 }
4168
4169 sub qemu_block_resize {
4170     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4171
4172     my $running = check_running($vmid);
4173
4174     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4175
4176     return if !$running;
4177
4178     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4179
4180 }
4181
4182 sub qemu_volume_snapshot {
4183     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4184
4185     my $running = check_running($vmid);
4186
4187     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4188         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
4189     } else {
4190         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4191     }
4192 }
4193
4194 sub qemu_volume_snapshot_delete {
4195     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4196
4197     my $running = check_running($vmid);
4198
4199     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4200         vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
4201     } else {
4202         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4203     }
4204 }
4205
4206 sub set_migration_caps {
4207     my ($vmid) = @_;
4208
4209     my $cap_ref = [];
4210
4211     my $enabled_cap = {
4212         "auto-converge" => 1,
4213         "xbzrle" => 1,
4214         "x-rdma-pin-all" => 0,
4215         "zero-blocks" => 0,
4216         "compress" => 0
4217     };
4218
4219     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4220
4221     for my $supported_capability (@$supported_capabilities) {
4222         push @$cap_ref, {
4223             capability => $supported_capability->{capability},
4224             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4225         };
4226     }
4227
4228     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4229 }
4230
4231 my $fast_plug_option = {
4232     'lock' => 1,
4233     'name' => 1,
4234     'onboot' => 1,
4235     'shares' => 1,
4236     'startup' => 1,
4237     'description' => 1,
4238     'protection' => 1,
4239     'vmstatestorage' => 1,
4240 };
4241
4242 # hotplug changes in [PENDING]
4243 # $selection hash can be used to only apply specified options, for
4244 # example: { cores => 1 } (only apply changed 'cores')
4245 # $errors ref is used to return error messages
4246 sub vmconfig_hotplug_pending {
4247     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4248
4249     my $defaults = load_defaults();
4250
4251     # commit values which do not have any impact on running VM first
4252     # Note: those option cannot raise errors, we we do not care about
4253     # $selection and always apply them.
4254
4255     my $add_error = sub {
4256         my ($opt, $msg) = @_;
4257         $errors->{$opt} = "hotplug problem - $msg";
4258     };
4259
4260     my $changes = 0;
4261     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4262         if ($fast_plug_option->{$opt}) {
4263             $conf->{$opt} = $conf->{pending}->{$opt};
4264             delete $conf->{pending}->{$opt};
4265             $changes = 1;
4266         }
4267     }
4268
4269     if ($changes) {
4270         PVE::QemuConfig->write_config($vmid, $conf);
4271         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4272     }
4273
4274     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4275
4276     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4277     while (my ($opt, $force) = each %$pending_delete_hash) {
4278         next if $selection && !$selection->{$opt};
4279         eval {
4280             if ($opt eq 'hotplug') {
4281                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4282             } elsif ($opt eq 'tablet') {
4283                 die "skip\n" if !$hotplug_features->{usb};
4284                 if ($defaults->{tablet}) {
4285                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4286                 } else {
4287                     vm_deviceunplug($vmid, $conf, $opt);
4288                 }
4289             } elsif ($opt =~ m/^usb\d+/) {
4290                 die "skip\n";
4291                 # since we cannot reliably hot unplug usb devices
4292                 # we are disabling it
4293                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4294                 vm_deviceunplug($vmid, $conf, $opt);
4295             } elsif ($opt eq 'vcpus') {
4296                 die "skip\n" if !$hotplug_features->{cpu};
4297                 qemu_cpu_hotplug($vmid, $conf, undef);
4298             } elsif ($opt eq 'balloon') {
4299                 # enable balloon device is not hotpluggable
4300                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4301                 # here we reset the ballooning value to memory
4302                 my $balloon = $conf->{memory} || $defaults->{memory};
4303                 vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4304             } elsif ($fast_plug_option->{$opt}) {
4305                 # do nothing
4306             } elsif ($opt =~ m/^net(\d+)$/) {
4307                 die "skip\n" if !$hotplug_features->{network};
4308                 vm_deviceunplug($vmid, $conf, $opt);
4309             } elsif (is_valid_drivename($opt)) {
4310                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4311                 vm_deviceunplug($vmid, $conf, $opt);
4312                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4313             } elsif ($opt =~ m/^memory$/) {
4314                 die "skip\n" if !$hotplug_features->{memory};
4315                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4316             } elsif ($opt eq 'cpuunits') {
4317                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4318             } elsif ($opt eq 'cpulimit') {
4319                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4320             } else {
4321                 die "skip\n";
4322             }
4323         };
4324         if (my $err = $@) {
4325             &$add_error($opt, $err) if $err ne "skip\n";
4326         } else {
4327             # save new config if hotplug was successful
4328             delete $conf->{$opt};
4329             vmconfig_undelete_pending_option($conf, $opt);
4330             PVE::QemuConfig->write_config($vmid, $conf);
4331             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4332         }
4333     }
4334
4335     my $apply_pending_cloudinit;
4336     $apply_pending_cloudinit = sub {
4337         my ($key, $value) = @_;
4338         $apply_pending_cloudinit = sub {}; # once is enough
4339
4340         my @cloudinit_opts = keys %$confdesc_cloudinit;
4341         foreach my $opt (keys %{$conf->{pending}}) {
4342             next if !grep { $_ eq $opt } @cloudinit_opts;
4343             $conf->{$opt} = delete $conf->{pending}->{$opt};
4344         }
4345
4346         my $new_conf = { %$conf };
4347         $new_conf->{$key} = $value;
4348         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4349     };
4350
4351     foreach my $opt (keys %{$conf->{pending}}) {
4352         next if $selection && !$selection->{$opt};
4353         my $value = $conf->{pending}->{$opt};
4354         eval {
4355             if ($opt eq 'hotplug') {
4356                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4357             } elsif ($opt eq 'tablet') {
4358                 die "skip\n" if !$hotplug_features->{usb};
4359                 if ($value == 1) {
4360                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4361                 } elsif ($value == 0) {
4362                     vm_deviceunplug($vmid, $conf, $opt);
4363                 }
4364             } elsif ($opt =~ m/^usb\d+$/) {
4365                 die "skip\n";
4366                 # since we cannot reliably hot unplug usb devices
4367                 # we are disabling it
4368                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4369                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4370                 die "skip\n" if !$d;
4371                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d);
4372             } elsif ($opt eq 'vcpus') {
4373                 die "skip\n" if !$hotplug_features->{cpu};
4374                 qemu_cpu_hotplug($vmid, $conf, $value);
4375             } elsif ($opt eq 'balloon') {
4376                 # enable/disable balloning device is not hotpluggable
4377                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4378                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4379                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4380
4381                 # allow manual ballooning if shares is set to zero
4382                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4383                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4384                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4385                 }
4386             } elsif ($opt =~ m/^net(\d+)$/) {
4387                 # some changes can be done without hotplug
4388                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4389                                     $vmid, $opt, $value);
4390             } elsif (is_valid_drivename($opt)) {
4391                 # some changes can be done without hotplug
4392                 my $drive = parse_drive($opt, $value);
4393                 if (drive_is_cloudinit($drive)) {
4394                     &$apply_pending_cloudinit($opt, $value);
4395                 }
4396                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4397                                      $vmid, $opt, $value, 1);
4398             } elsif ($opt =~ m/^memory$/) { #dimms
4399                 die "skip\n" if !$hotplug_features->{memory};
4400                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4401             } elsif ($opt eq 'cpuunits') {
4402                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4403             } elsif ($opt eq 'cpulimit') {
4404                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4405                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4406             } else {
4407                 die "skip\n";  # skip non-hot-pluggable options
4408             }
4409         };
4410         if (my $err = $@) {
4411             &$add_error($opt, $err) if $err ne "skip\n";
4412         } else {
4413             # save new config if hotplug was successful
4414             $conf->{$opt} = $value;
4415             delete $conf->{pending}->{$opt};
4416             PVE::QemuConfig->write_config($vmid, $conf);
4417             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4418         }
4419     }
4420 }
4421
4422 sub try_deallocate_drive {
4423     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4424
4425     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4426         my $volid = $drive->{file};
4427         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4428             my $sid = PVE::Storage::parse_volume_id($volid);
4429             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4430
4431             # check if the disk is really unused
4432             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4433                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4434             PVE::Storage::vdisk_free($storecfg, $volid);
4435             return 1;
4436         } else {
4437             # If vm is not owner of this disk remove from config
4438             return 1;
4439         }
4440     }
4441
4442     return undef;
4443 }
4444
4445 sub vmconfig_delete_or_detach_drive {
4446     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4447
4448     my $drive = parse_drive($opt, $conf->{$opt});
4449
4450     my $rpcenv = PVE::RPCEnvironment::get();
4451     my $authuser = $rpcenv->get_user();
4452
4453     if ($force) {
4454         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4455         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4456     } else {
4457         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4458     }
4459 }
4460
4461 sub vmconfig_apply_pending {
4462     my ($vmid, $conf, $storecfg) = @_;
4463
4464     # cold plug
4465
4466     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4467     while (my ($opt, $force) = each %$pending_delete_hash) {
4468         die "internal error" if $opt =~ m/^unused/;
4469         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4470         if (!defined($conf->{$opt})) {
4471             vmconfig_undelete_pending_option($conf, $opt);
4472             PVE::QemuConfig->write_config($vmid, $conf);
4473         } elsif (is_valid_drivename($opt)) {
4474             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4475             vmconfig_undelete_pending_option($conf, $opt);
4476             delete $conf->{$opt};
4477             PVE::QemuConfig->write_config($vmid, $conf);
4478         } else {
4479             vmconfig_undelete_pending_option($conf, $opt);
4480             delete $conf->{$opt};
4481             PVE::QemuConfig->write_config($vmid, $conf);
4482         }
4483     }
4484
4485     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4486
4487     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4488         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4489
4490         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4491             # skip if nothing changed
4492         } elsif (is_valid_drivename($opt)) {
4493             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4494                 if defined($conf->{$opt});
4495             $conf->{$opt} = $conf->{pending}->{$opt};
4496         } else {
4497             $conf->{$opt} = $conf->{pending}->{$opt};
4498         }
4499
4500         delete $conf->{pending}->{$opt};
4501         PVE::QemuConfig->write_config($vmid, $conf);
4502     }
4503 }
4504
4505 my $safe_num_ne = sub {
4506     my ($a, $b) = @_;
4507
4508     return 0 if !defined($a) && !defined($b);
4509     return 1 if !defined($a);
4510     return 1 if !defined($b);
4511
4512     return $a != $b;
4513 };
4514
4515 my $safe_string_ne = sub {
4516     my ($a, $b) = @_;
4517
4518     return 0 if !defined($a) && !defined($b);
4519     return 1 if !defined($a);
4520     return 1 if !defined($b);
4521
4522     return $a ne $b;
4523 };
4524
4525 sub vmconfig_update_net {
4526     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4527
4528     my $newnet = parse_net($value);
4529
4530     if ($conf->{$opt}) {
4531         my $oldnet = parse_net($conf->{$opt});
4532
4533         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4534             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4535             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4536             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4537
4538             # for non online change, we try to hot-unplug
4539             die "skip\n" if !$hotplug;
4540             vm_deviceunplug($vmid, $conf, $opt);
4541         } else {
4542
4543             die "internal error" if $opt !~ m/net(\d+)/;
4544             my $iface = "tap${vmid}i$1";
4545
4546             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4547                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4548                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4549                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4550                 PVE::Network::tap_unplug($iface);
4551                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4552             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4553                 # Rate can be applied on its own but any change above needs to
4554                 # include the rate in tap_plug since OVS resets everything.
4555                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4556             }
4557
4558             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4559                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4560             }
4561
4562             return 1;
4563         }
4564     }
4565
4566     if ($hotplug) {
4567         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4568     } else {
4569         die "skip\n";
4570     }
4571 }
4572
4573 sub vmconfig_update_disk {
4574     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4575
4576     # fixme: do we need force?
4577
4578     my $drive = parse_drive($opt, $value);
4579
4580     if ($conf->{$opt}) {
4581
4582         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4583
4584             my $media = $drive->{media} || 'disk';
4585             my $oldmedia = $old_drive->{media} || 'disk';
4586             die "unable to change media type\n" if $media ne $oldmedia;
4587
4588             if (!drive_is_cdrom($old_drive)) {
4589
4590                 if ($drive->{file} ne $old_drive->{file}) {
4591
4592                     die "skip\n" if !$hotplug;
4593
4594                     # unplug and register as unused
4595                     vm_deviceunplug($vmid, $conf, $opt);
4596                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4597
4598                 } else {
4599                     # update existing disk
4600
4601                     # skip non hotpluggable value
4602                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4603                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4604                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4605                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4606                         die "skip\n";
4607                     }
4608
4609                     # apply throttle
4610                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4611                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4612                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4613                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4614                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4615                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4616                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4617                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4618                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4619                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4620                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4621                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4622                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4623                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4624                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4625                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4626                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4627                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4628
4629                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4630                                                    ($drive->{mbps} || 0)*1024*1024,
4631                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4632                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4633                                                    $drive->{iops} || 0,
4634                                                    $drive->{iops_rd} || 0,
4635                                                    $drive->{iops_wr} || 0,
4636                                                    ($drive->{mbps_max} || 0)*1024*1024,
4637                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4638                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4639                                                    $drive->{iops_max} || 0,
4640                                                    $drive->{iops_rd_max} || 0,
4641                                                    $drive->{iops_wr_max} || 0,
4642                                                    $drive->{bps_max_length} || 1,
4643                                                    $drive->{bps_rd_max_length} || 1,
4644                                                    $drive->{bps_wr_max_length} || 1,
4645                                                    $drive->{iops_max_length} || 1,
4646                                                    $drive->{iops_rd_max_length} || 1,
4647                                                    $drive->{iops_wr_max_length} || 1);
4648
4649                     }
4650
4651                     return 1;
4652                 }
4653
4654             } else { # cdrom
4655
4656                 if ($drive->{file} eq 'none') {
4657                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4658                     if (drive_is_cloudinit($old_drive)) {
4659                         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
4660                     }
4661                 } else {
4662                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4663                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4664                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4665                 }
4666
4667                 return 1;
4668             }
4669         }
4670     }
4671
4672     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4673     # hotplug new disks
4674     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4675     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4676 }
4677
4678 sub vm_start {
4679     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4680         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
4681
4682     PVE::QemuConfig->lock_config($vmid, sub {
4683         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4684
4685         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
4686
4687         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4688
4689         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4690
4691         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4692             vmconfig_apply_pending($vmid, $conf, $storecfg);
4693             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4694         }
4695
4696         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
4697
4698         my $defaults = load_defaults();
4699
4700         # set environment variable useful inside network script
4701         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4702
4703         my $local_volumes = {};
4704
4705         if ($targetstorage) {
4706             foreach_drive($conf, sub {
4707                 my ($ds, $drive) = @_;
4708
4709                 return if drive_is_cdrom($drive);
4710
4711                 my $volid = $drive->{file};
4712
4713                 return if !$volid;
4714
4715                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4716
4717                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4718                 return if $scfg->{shared};
4719                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
4720             });
4721
4722             my $format = undef;
4723
4724             foreach my $opt (sort keys %$local_volumes) {
4725
4726                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
4727                 my $drive = parse_drive($opt, $conf->{$opt});
4728
4729                 #if remote storage is specified, use default format
4730                 if ($targetstorage && $targetstorage ne "1") {
4731                     $storeid = $targetstorage;
4732                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
4733                     $format = $defFormat;
4734                 } else {
4735                     #else we use same format than original
4736                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4737                     $format = qemu_img_format($scfg, $volid);
4738                 }
4739
4740                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
4741                 my $newdrive = $drive;
4742                 $newdrive->{format} = $format;
4743                 $newdrive->{file} = $newvolid;
4744                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
4745                 $local_volumes->{$opt} = $drivestr;
4746                 #pass drive to conf for command line
4747                 $conf->{$opt} = $drivestr;
4748             }
4749         }
4750
4751         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4752
4753         my $migrate_port = 0;
4754         my $migrate_uri;
4755         if ($statefile) {
4756             if ($statefile eq 'tcp') {
4757                 my $localip = "localhost";
4758                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4759                 my $nodename = PVE::INotify::nodename();
4760
4761                 if (!defined($migration_type)) {
4762                     if (defined($datacenterconf->{migration}->{type})) {
4763                         $migration_type = $datacenterconf->{migration}->{type};
4764                     } else {
4765                         $migration_type = 'secure';
4766                     }
4767                 }
4768
4769                 if ($migration_type eq 'insecure') {
4770                     my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4771                     if ($migrate_network_addr) {
4772                         $localip = $migrate_network_addr;
4773                     } else {
4774                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4775                     }
4776
4777                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4778                 }
4779
4780                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4781                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4782                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4783                 push @$cmd, '-incoming', $migrate_uri;
4784                 push @$cmd, '-S';
4785
4786             } elsif ($statefile eq 'unix') {
4787                 # should be default for secure migrations as a ssh TCP forward
4788                 # tunnel is not deterministic reliable ready and fails regurarly
4789                 # to set up in time, so use UNIX socket forwards
4790                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
4791                 unlink $socket_addr;
4792
4793                 $migrate_uri = "unix:$socket_addr";
4794
4795                 push @$cmd, '-incoming', $migrate_uri;
4796                 push @$cmd, '-S';
4797
4798             } else {
4799                 push @$cmd, '-loadstate', $statefile;
4800             }
4801         } elsif ($paused) {
4802             push @$cmd, '-S';
4803         }
4804
4805         # host pci devices
4806         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4807           my $d = parse_hostpci($conf->{"hostpci$i"});
4808           next if !$d;
4809           my $pcidevices = $d->{pciid};
4810           foreach my $pcidevice (@$pcidevices) {
4811                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4812
4813                 my $info = pci_device_info("0000:$pciid");
4814                 die "IOMMU not present\n" if !check_iommu_support();
4815                 die "no pci device info for device '$pciid'\n" if !$info;
4816                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4817                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4818           }
4819         }
4820
4821         PVE::Storage::activate_volumes($storecfg, $vollist);
4822
4823         if (!check_running($vmid, 1)) {
4824             eval {
4825                 run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
4826                     outfunc => sub {}, errfunc => sub {});
4827             };
4828         }
4829
4830         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
4831                                                   : $defaults->{cpuunits};
4832
4833         my $start_timeout = $conf->{hugepages} ? 300 : 30;
4834         my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077);
4835
4836         my %properties = (
4837             Slice => 'qemu.slice',
4838             KillMode => 'none',
4839             CPUShares => $cpuunits
4840         );
4841
4842         if (my $cpulimit = $conf->{cpulimit}) {
4843             $properties{CPUQuota} = int($cpulimit * 100);
4844         }
4845         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
4846
4847         my $run_qemu = sub {
4848             PVE::Tools::run_fork sub {
4849                 PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4850                 run_command($cmd, %run_params);
4851             };
4852         };
4853
4854         if ($conf->{hugepages}) {
4855
4856             my $code = sub {
4857                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
4858                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
4859
4860                 PVE::QemuServer::Memory::hugepages_mount();
4861                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
4862
4863                 eval { $run_qemu->() };
4864                 if (my $err = $@) {
4865                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
4866                     die $err;
4867                 }
4868
4869                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
4870             };
4871             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
4872
4873         } else {
4874             eval { $run_qemu->() };
4875         }
4876
4877         if (my $err = $@) {
4878             # deactivate volumes if start fails
4879             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
4880             die "start failed: $err";
4881         }
4882
4883         print "migration listens on $migrate_uri\n" if $migrate_uri;
4884
4885         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
4886             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4887             warn $@ if $@;
4888         }
4889
4890         #start nbd server for storage migration
4891         if ($targetstorage) {
4892             my $nodename = PVE::INotify::nodename();
4893             my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4894             my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
4895             my $pfamily = PVE::Tools::get_host_address_family($nodename);
4896             $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4897
4898             vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${migrate_port}" } } );
4899
4900             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4901
4902             foreach my $opt (sort keys %$local_volumes) {
4903                 my $volid = $local_volumes->{$opt};
4904                 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
4905                 my $migrate_storage_uri = "nbd:${localip}:${migrate_port}:exportname=drive-$opt";
4906                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
4907             }
4908         }
4909
4910         if ($migratedfrom) {
4911             eval {
4912                 set_migration_caps($vmid);
4913             };
4914             warn $@ if $@;
4915
4916             if ($spice_port) {
4917                 print "spice listens on port $spice_port\n";
4918                 if ($spice_ticket) {
4919                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4920                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4921                 }
4922             }
4923
4924         } else {
4925             vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4926                 if !$statefile && $conf->{balloon};
4927
4928             foreach my $opt (keys %$conf) {
4929                 next if $opt !~  m/^net\d+$/;
4930                 my $nicconf = parse_net($conf->{$opt});
4931                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4932             }
4933         }
4934
4935         vm_mon_cmd_nocheck($vmid, 'qom-set',
4936                     path => "machine/peripheral/balloon0",
4937                     property => "guest-stats-polling-interval",
4938                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4939
4940     });
4941 }
4942
4943 sub vm_mon_cmd {
4944     my ($vmid, $execute, %params) = @_;
4945
4946     my $cmd = { execute => $execute, arguments => \%params };
4947     vm_qmp_command($vmid, $cmd);
4948 }
4949
4950 sub vm_mon_cmd_nocheck {
4951     my ($vmid, $execute, %params) = @_;
4952
4953     my $cmd = { execute => $execute, arguments => \%params };
4954     vm_qmp_command($vmid, $cmd, 1);
4955 }
4956
4957 sub vm_qmp_command {
4958     my ($vmid, $cmd, $nocheck) = @_;
4959
4960     my $res;
4961
4962     my $timeout;
4963     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4964         $timeout = $cmd->{arguments}->{timeout};
4965         delete $cmd->{arguments}->{timeout};
4966     }
4967
4968     eval {
4969         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4970         my $sname = qmp_socket($vmid);
4971         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4972             my $qmpclient = PVE::QMPClient->new();
4973
4974             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4975         } else {
4976             die "unable to open monitor socket\n";
4977         }
4978     };
4979     if (my $err = $@) {
4980         syslog("err", "VM $vmid qmp command failed - $err");
4981         die $err;
4982     }
4983
4984     return $res;
4985 }
4986
4987 sub vm_human_monitor_command {
4988     my ($vmid, $cmdline) = @_;
4989
4990     my $res;
4991
4992     my $cmd = {
4993         execute => 'human-monitor-command',
4994         arguments => { 'command-line' => $cmdline},
4995     };
4996
4997     return vm_qmp_command($vmid, $cmd);
4998 }
4999
5000 sub vm_commandline {
5001     my ($storecfg, $vmid) = @_;
5002
5003     my $conf = PVE::QemuConfig->load_config($vmid);
5004
5005     my $defaults = load_defaults();
5006
5007     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
5008
5009     return PVE::Tools::cmd2string($cmd);
5010 }
5011
5012 sub vm_reset {
5013     my ($vmid, $skiplock) = @_;
5014
5015     PVE::QemuConfig->lock_config($vmid, sub {
5016
5017         my $conf = PVE::QemuConfig->load_config($vmid);
5018
5019         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5020
5021         vm_mon_cmd($vmid, "system_reset");
5022     });
5023 }
5024
5025 sub get_vm_volumes {
5026     my ($conf) = @_;
5027
5028     my $vollist = [];
5029     foreach_volid($conf, sub {
5030         my ($volid, $attr) = @_;
5031
5032         return if $volid =~ m|^/|;
5033
5034         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5035         return if !$sid;
5036
5037         push @$vollist, $volid;
5038     });
5039
5040     return $vollist;
5041 }
5042
5043 sub vm_stop_cleanup {
5044     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5045
5046     eval {
5047
5048         if (!$keepActive) {
5049             my $vollist = get_vm_volumes($conf);
5050             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5051         }
5052
5053         foreach my $ext (qw(mon qmp pid vnc qga)) {
5054             unlink "/var/run/qemu-server/${vmid}.$ext";
5055         }
5056
5057         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5058     };
5059     warn $@ if $@; # avoid errors - just warn
5060 }
5061
5062 # Note: use $nockeck to skip tests if VM configuration file exists.
5063 # We need that when migration VMs to other nodes (files already moved)
5064 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5065 sub vm_stop {
5066     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5067
5068     $force = 1 if !defined($force) && !$shutdown;
5069
5070     if ($migratedfrom){
5071         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5072         kill 15, $pid if $pid;
5073         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5074         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5075         return;
5076     }
5077
5078     PVE::QemuConfig->lock_config($vmid, sub {
5079
5080         my $pid = check_running($vmid, $nocheck);
5081         return if !$pid;
5082
5083         my $conf;
5084         if (!$nocheck) {
5085             $conf = PVE::QemuConfig->load_config($vmid);
5086             PVE::QemuConfig->check_lock($conf) if !$skiplock;
5087             if (!defined($timeout) && $shutdown && $conf->{startup}) {
5088                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5089                 $timeout = $opts->{down} if $opts->{down};
5090             }
5091         }
5092
5093         $timeout = 60 if !defined($timeout);
5094
5095         eval {
5096             if ($shutdown) {
5097                 if (defined($conf) && $conf->{agent}) {
5098                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
5099                 } else {
5100                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
5101                 }
5102             } else {
5103                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
5104             }
5105         };
5106         my $err = $@;
5107
5108         if (!$err) {
5109             my $count = 0;
5110             while (($count < $timeout) && check_running($vmid, $nocheck)) {
5111                 $count++;
5112                 sleep 1;
5113             }
5114
5115             if ($count >= $timeout) {
5116                 if ($force) {
5117                     warn "VM still running - terminating now with SIGTERM\n";
5118                     kill 15, $pid;
5119                 } else {
5120                     die "VM quit/powerdown failed - got timeout\n";
5121                 }
5122             } else {
5123                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5124                 return;
5125             }
5126         } else {
5127             if ($force) {
5128                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5129                 kill 15, $pid;
5130             } else {
5131                 die "VM quit/powerdown failed\n";
5132             }
5133         }
5134
5135         # wait again
5136         $timeout = 10;
5137
5138         my $count = 0;
5139         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5140             $count++;
5141             sleep 1;
5142         }
5143
5144         if ($count >= $timeout) {
5145             warn "VM still running - terminating now with SIGKILL\n";
5146             kill 9, $pid;
5147             sleep 1;
5148         }
5149
5150         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5151    });
5152 }
5153
5154 sub vm_suspend {
5155     my ($vmid, $skiplock) = @_;
5156
5157     PVE::QemuConfig->lock_config($vmid, sub {
5158
5159         my $conf = PVE::QemuConfig->load_config($vmid);
5160
5161         PVE::QemuConfig->check_lock($conf)
5162             if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5163
5164         vm_mon_cmd($vmid, "stop");
5165     });
5166 }
5167
5168 sub vm_resume {
5169     my ($vmid, $skiplock, $nocheck) = @_;
5170
5171     PVE::QemuConfig->lock_config($vmid, sub {
5172
5173         if (!$nocheck) {
5174
5175             my $conf = PVE::QemuConfig->load_config($vmid);
5176
5177             PVE::QemuConfig->check_lock($conf)
5178                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5179
5180             vm_mon_cmd($vmid, "cont");
5181
5182         } else {
5183             vm_mon_cmd_nocheck($vmid, "cont");
5184         }
5185     });
5186 }
5187
5188 sub vm_sendkey {
5189     my ($vmid, $skiplock, $key) = @_;
5190
5191     PVE::QemuConfig->lock_config($vmid, sub {
5192
5193         my $conf = PVE::QemuConfig->load_config($vmid);
5194
5195         # there is no qmp command, so we use the human monitor command
5196         vm_human_monitor_command($vmid, "sendkey $key");
5197     });
5198 }
5199
5200 sub vm_destroy {
5201     my ($storecfg, $vmid, $skiplock) = @_;
5202
5203     PVE::QemuConfig->lock_config($vmid, sub {
5204
5205         my $conf = PVE::QemuConfig->load_config($vmid);
5206
5207         if (!check_running($vmid)) {
5208             destroy_vm($storecfg, $vmid, undef, $skiplock);
5209         } else {
5210             die "VM $vmid is running - destroy failed\n";
5211         }
5212     });
5213 }
5214
5215 # pci helpers
5216
5217 sub file_write {
5218     my ($filename, $buf) = @_;
5219
5220     my $fh = IO::File->new($filename, "w");
5221     return undef if !$fh;
5222
5223     my $res = print $fh $buf;
5224
5225     $fh->close();
5226
5227     return $res;
5228 }
5229
5230 sub pci_device_info {
5231     my ($name) = @_;
5232
5233     my $res;
5234
5235     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
5236     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
5237
5238     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
5239     return undef if !defined($irq) || $irq !~ m/^\d+$/;
5240
5241     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
5242     return undef if !defined($vendor) || $vendor !~ s/^0x//;
5243
5244     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
5245     return undef if !defined($product) || $product !~ s/^0x//;
5246
5247     $res = {
5248         name => $name,
5249         vendor => $vendor,
5250         product => $product,
5251         domain => $domain,
5252         bus => $bus,
5253         slot => $slot,
5254         func => $func,
5255         irq => $irq,
5256         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
5257     };
5258
5259     return $res;
5260 }
5261
5262 sub pci_dev_reset {
5263     my ($dev) = @_;
5264
5265     my $name = $dev->{name};
5266
5267     my $fn = "$pcisysfs/devices/$name/reset";
5268
5269     return file_write($fn, "1");
5270 }
5271
5272 sub pci_dev_bind_to_vfio {
5273     my ($dev) = @_;
5274
5275     my $name = $dev->{name};
5276
5277     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5278
5279     if (!-d $vfio_basedir) {
5280         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5281     }
5282     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5283
5284     my $testdir = "$vfio_basedir/$name";
5285     return 1 if -d $testdir;
5286
5287     my $data = "$dev->{vendor} $dev->{product}";
5288     return undef if !file_write("$vfio_basedir/new_id", $data);
5289
5290     my $fn = "$pcisysfs/devices/$name/driver/unbind";
5291     if (!file_write($fn, $name)) {
5292         return undef if -f $fn;
5293     }
5294
5295     $fn = "$vfio_basedir/bind";
5296     if (! -d $testdir) {
5297         return undef if !file_write($fn, $name);
5298     }
5299
5300     return -d $testdir;
5301 }
5302
5303 sub pci_dev_group_bind_to_vfio {
5304     my ($pciid) = @_;
5305
5306     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5307
5308     if (!-d $vfio_basedir) {
5309         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5310     }
5311     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5312
5313     # get IOMMU group devices
5314     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
5315       my @devs = grep /^0000:/, readdir($D);
5316     closedir($D);
5317
5318     foreach my $pciid (@devs) {
5319         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
5320
5321         # pci bridges, switches or root ports are not supported
5322         # they have a pci_bus subdirectory so skip them
5323         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
5324
5325         my $info = pci_device_info($1);
5326         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
5327     }
5328
5329     return 1;
5330 }
5331
5332 # vzdump restore implementaion
5333
5334 sub tar_archive_read_firstfile {
5335     my $archive = shift;
5336
5337     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5338
5339     # try to detect archive type first
5340     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5341         die "unable to open file '$archive'\n";
5342     my $firstfile = <$fh>;
5343     kill 15, $pid;
5344     close $fh;
5345
5346     die "ERROR: archive contaions no data\n" if !$firstfile;
5347     chomp $firstfile;
5348
5349     return $firstfile;
5350 }
5351
5352 sub tar_restore_cleanup {
5353     my ($storecfg, $statfile) = @_;
5354
5355     print STDERR "starting cleanup\n";
5356
5357     if (my $fd = IO::File->new($statfile, "r")) {
5358         while (defined(my $line = <$fd>)) {
5359             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5360                 my $volid = $2;
5361                 eval {
5362                     if ($volid =~ m|^/|) {
5363                         unlink $volid || die 'unlink failed\n';
5364                     } else {
5365                         PVE::Storage::vdisk_free($storecfg, $volid);
5366                     }
5367                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5368                 };
5369                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5370             } else {
5371                 print STDERR "unable to parse line in statfile - $line";
5372             }
5373         }
5374         $fd->close();
5375     }
5376 }
5377
5378 sub restore_archive {
5379     my ($archive, $vmid, $user, $opts) = @_;
5380
5381     my $format = $opts->{format};
5382     my $comp;
5383
5384     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5385         $format = 'tar' if !$format;
5386         $comp = 'gzip';
5387     } elsif ($archive =~ m/\.tar$/) {
5388         $format = 'tar' if !$format;
5389     } elsif ($archive =~ m/.tar.lzo$/) {
5390         $format = 'tar' if !$format;
5391         $comp = 'lzop';
5392     } elsif ($archive =~ m/\.vma$/) {
5393         $format = 'vma' if !$format;
5394     } elsif ($archive =~ m/\.vma\.gz$/) {
5395         $format = 'vma' if !$format;
5396         $comp = 'gzip';
5397     } elsif ($archive =~ m/\.vma\.lzo$/) {
5398         $format = 'vma' if !$format;
5399         $comp = 'lzop';
5400     } else {
5401         $format = 'vma' if !$format; # default
5402     }
5403
5404     # try to detect archive format
5405     if ($format eq 'tar') {
5406         return restore_tar_archive($archive, $vmid, $user, $opts);
5407     } else {
5408         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5409     }
5410 }
5411
5412 sub restore_update_config_line {
5413     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5414
5415     return if $line =~ m/^\#qmdump\#/;
5416     return if $line =~ m/^\#vzdump\#/;
5417     return if $line =~ m/^lock:/;
5418     return if $line =~ m/^unused\d+:/;
5419     return if $line =~ m/^parent:/;
5420     return if $line =~ m/^template:/; # restored VM is never a template
5421
5422     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5423     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5424         # try to convert old 1.X settings
5425         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5426         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5427             my ($model, $macaddr) = split(/\=/, $devconfig);
5428             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5429             my $net = {
5430                 model => $model,
5431                 bridge => "vmbr$ind",
5432                 macaddr => $macaddr,
5433             };
5434             my $netstr = print_net($net);
5435
5436             print $outfd "net$cookie->{netcount}: $netstr\n";
5437             $cookie->{netcount}++;
5438         }
5439     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5440         my ($id, $netstr) = ($1, $2);
5441         my $net = parse_net($netstr);
5442         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5443         $netstr = print_net($net);
5444         print $outfd "$id: $netstr\n";
5445     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5446         my $virtdev = $1;
5447         my $value = $3;
5448         my $di = parse_drive($virtdev, $value);
5449         if (defined($di->{backup}) && !$di->{backup}) {
5450             print $outfd "#$line";
5451         } elsif ($map->{$virtdev}) {
5452             delete $di->{format}; # format can change on restore
5453             $di->{file} = $map->{$virtdev};
5454             $value = print_drive($vmid, $di);
5455             print $outfd "$virtdev: $value\n";
5456         } else {
5457             print $outfd $line;
5458         }
5459     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5460         my ($uuid, $uuid_str);
5461         UUID::generate($uuid);
5462         UUID::unparse($uuid, $uuid_str);
5463         my $smbios1 = parse_smbios1($2);
5464         $smbios1->{uuid} = $uuid_str;
5465         print $outfd $1.print_smbios1($smbios1)."\n";
5466     } else {
5467         print $outfd $line;
5468     }
5469 }
5470
5471 sub scan_volids {
5472     my ($cfg, $vmid) = @_;
5473
5474     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5475
5476     my $volid_hash = {};
5477     foreach my $storeid (keys %$info) {
5478         foreach my $item (@{$info->{$storeid}}) {
5479             next if !($item->{volid} && $item->{size});
5480             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5481             $volid_hash->{$item->{volid}} = $item;
5482         }
5483     }
5484
5485     return $volid_hash;
5486 }
5487
5488 sub is_volume_in_use {
5489     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5490
5491     my $path = PVE::Storage::path($storecfg, $volid);
5492
5493     my $scan_config = sub {
5494         my ($cref, $snapname) = @_;
5495
5496         foreach my $key (keys %$cref) {
5497             my $value = $cref->{$key};
5498             if (is_valid_drivename($key)) {
5499                 next if $skip_drive && $key eq $skip_drive;
5500                 my $drive = parse_drive($key, $value);
5501                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5502                 return 1 if $volid eq $drive->{file};
5503                 if ($drive->{file} =~ m!^/!) {
5504                     return 1 if $drive->{file} eq $path;
5505                 } else {
5506                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5507                     next if !$storeid;
5508                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5509                     next if !$scfg;
5510                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5511                 }
5512             }
5513         }
5514
5515         return 0;
5516     };
5517
5518     return 1 if &$scan_config($conf);
5519
5520     undef $skip_drive;
5521
5522     foreach my $snapname (keys %{$conf->{snapshots}}) {
5523         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5524     }
5525
5526     return 0;
5527 }
5528
5529 sub update_disksize {
5530     my ($vmid, $conf, $volid_hash) = @_;
5531
5532     my $changes;
5533
5534     # used and unused disks
5535     my $referenced = {};
5536
5537     # Note: it is allowed to define multiple storages with same path (alias), so
5538     # we need to check both 'volid' and real 'path' (two different volid can point
5539     # to the same path).
5540
5541     my $referencedpath = {};
5542
5543     # update size info
5544     foreach my $opt (keys %$conf) {
5545         if (is_valid_drivename($opt)) {
5546             my $drive = parse_drive($opt, $conf->{$opt});
5547             my $volid = $drive->{file};
5548             next if !$volid;
5549
5550             $referenced->{$volid} = 1;
5551             if ($volid_hash->{$volid} &&
5552                 (my $path = $volid_hash->{$volid}->{path})) {
5553                 $referencedpath->{$path} = 1;
5554             }
5555
5556             next if drive_is_cdrom($drive);
5557             next if !$volid_hash->{$volid};
5558
5559             $drive->{size} = $volid_hash->{$volid}->{size};
5560             my $new = print_drive($vmid, $drive);
5561             if ($new ne $conf->{$opt}) {
5562                 $changes = 1;
5563                 $conf->{$opt} = $new;
5564             }
5565         }
5566     }
5567
5568     # remove 'unusedX' entry if volume is used
5569     foreach my $opt (keys %$conf) {
5570         next if $opt !~ m/^unused\d+$/;
5571         my $volid = $conf->{$opt};
5572         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5573         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
5574             $changes = 1;
5575             delete $conf->{$opt};
5576         }
5577
5578         $referenced->{$volid} = 1;
5579         $referencedpath->{$path} = 1 if $path;
5580     }
5581
5582     foreach my $volid (sort keys %$volid_hash) {
5583         next if $volid =~ m/vm-$vmid-state-/;
5584         next if $referenced->{$volid};
5585         my $path = $volid_hash->{$volid}->{path};
5586         next if !$path; # just to be sure
5587         next if $referencedpath->{$path};
5588         $changes = 1;
5589         PVE::QemuConfig->add_unused_volume($conf, $volid);
5590         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
5591     }
5592
5593     return $changes;
5594 }
5595
5596 sub rescan {
5597     my ($vmid, $nolock) = @_;
5598
5599     my $cfg = PVE::Storage::config();
5600
5601     my $volid_hash = scan_volids($cfg, $vmid);
5602
5603     my $updatefn =  sub {
5604         my ($vmid) = @_;
5605
5606         my $conf = PVE::QemuConfig->load_config($vmid);
5607
5608         PVE::QemuConfig->check_lock($conf);
5609
5610         my $vm_volids = {};
5611         foreach my $volid (keys %$volid_hash) {
5612             my $info = $volid_hash->{$volid};
5613             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5614         }
5615
5616         my $changes = update_disksize($vmid, $conf, $vm_volids);
5617
5618         PVE::QemuConfig->write_config($vmid, $conf) if $changes;
5619     };
5620
5621     if (defined($vmid)) {
5622         if ($nolock) {
5623             &$updatefn($vmid);
5624         } else {
5625             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5626         }
5627     } else {
5628         my $vmlist = config_list();
5629         foreach my $vmid (keys %$vmlist) {
5630             if ($nolock) {
5631                 &$updatefn($vmid);
5632             } else {
5633                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5634             }
5635         }
5636     }
5637 }
5638
5639 sub restore_vma_archive {
5640     my ($archive, $vmid, $user, $opts, $comp) = @_;
5641
5642     my $readfrom = $archive;
5643
5644     my $cfg = PVE::Storage::config();
5645     my $commands = [];
5646     my $bwlimit = $opts->{bwlimit};
5647
5648     my $dbg_cmdstring = '';
5649     my $add_pipe = sub {
5650         my ($cmd) = @_;
5651         push @$commands, $cmd;
5652         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
5653         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
5654         $readfrom = '-';
5655     };
5656
5657     my $input = undef;
5658     if ($archive eq '-') {
5659         $input = '<&STDIN';
5660     } else {
5661         # If we use a backup from a PVE defined storage we also consider that
5662         # storage's rate limit:
5663         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
5664         if (defined($volid)) {
5665             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
5666             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
5667             if ($readlimit) {
5668                 print STDERR "applying read rate limit: $readlimit\n";
5669                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
5670                 $add_pipe->($cstream);
5671             }
5672         }
5673     }
5674
5675     if ($comp) {
5676         my $cmd;
5677         if ($comp eq 'gzip') {
5678             $cmd = ['zcat', $readfrom];
5679         } elsif ($comp eq 'lzop') {
5680             $cmd = ['lzop', '-d', '-c', $readfrom];
5681         } else {
5682             die "unknown compression method '$comp'\n";
5683         }
5684         $add_pipe->($cmd);
5685     }
5686
5687     my $tmpdir = "/var/tmp/vzdumptmp$$";
5688     rmtree $tmpdir;
5689
5690     # disable interrupts (always do cleanups)
5691     local $SIG{INT} =
5692         local $SIG{TERM} =
5693         local $SIG{QUIT} =
5694         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
5695
5696     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5697     POSIX::mkfifo($mapfifo, 0600);
5698     my $fifofh;
5699
5700     my $openfifo = sub {
5701         open($fifofh, '>', $mapfifo) || die $!;
5702     };
5703
5704     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
5705
5706     my $oldtimeout;
5707     my $timeout = 5;
5708
5709     my $devinfo = {};
5710
5711     my $rpcenv = PVE::RPCEnvironment::get();
5712
5713     my $conffile = PVE::QemuConfig->config_file($vmid);
5714     my $tmpfn = "$conffile.$$.tmp";
5715
5716     # Note: $oldconf is undef if VM does not exists
5717     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
5718     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
5719
5720     my %storage_limits;
5721
5722     my $print_devmap = sub {
5723         my $virtdev_hash = {};
5724
5725         my $cfgfn = "$tmpdir/qemu-server.conf";
5726
5727         # we can read the config - that is already extracted
5728         my $fh = IO::File->new($cfgfn, "r") ||
5729             "unable to read qemu-server.conf - $!\n";
5730
5731         my $fwcfgfn = "$tmpdir/qemu-server.fw";
5732         if (-f $fwcfgfn) {
5733             my $pve_firewall_dir = '/etc/pve/firewall';
5734             mkdir $pve_firewall_dir; # make sure the dir exists
5735             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
5736         }
5737
5738         while (defined(my $line = <$fh>)) {
5739             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5740                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5741                 die "archive does not contain data for drive '$virtdev'\n"
5742                     if !$devinfo->{$devname};
5743                 if (defined($opts->{storage})) {
5744                     $storeid = $opts->{storage} || 'local';
5745                 } elsif (!$storeid) {
5746                     $storeid = 'local';
5747                 }
5748                 $format = 'raw' if !$format;
5749                 $devinfo->{$devname}->{devname} = $devname;
5750                 $devinfo->{$devname}->{virtdev} = $virtdev;
5751                 $devinfo->{$devname}->{format} = $format;
5752                 $devinfo->{$devname}->{storeid} = $storeid;
5753
5754                 # check permission on storage
5755                 my $pool = $opts->{pool}; # todo: do we need that?
5756                 if ($user ne 'root@pam') {
5757                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5758                 }
5759
5760                 $storage_limits{$storeid} = $bwlimit;
5761
5762                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5763             }
5764         }
5765
5766         foreach my $key (keys %storage_limits) {
5767             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
5768             next if !$limit;
5769             print STDERR "rate limit for storage $key: $limit KiB/s\n";
5770             $storage_limits{$key} = $limit * 1024;
5771         }
5772
5773         foreach my $devname (keys %$devinfo) {
5774             die "found no device mapping information for device '$devname'\n"
5775                 if !$devinfo->{$devname}->{virtdev};
5776         }
5777
5778         # create empty/temp config
5779         if ($oldconf) {
5780             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5781             foreach_drive($oldconf, sub {
5782                 my ($ds, $drive) = @_;
5783
5784                 return if drive_is_cdrom($drive);
5785
5786                 my $volid = $drive->{file};
5787
5788                 return if !$volid || $volid =~ m|^/|;
5789
5790                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5791                 return if !$path || !$owner || ($owner != $vmid);
5792
5793                 # Note: only delete disk we want to restore
5794                 # other volumes will become unused
5795                 if ($virtdev_hash->{$ds}) {
5796                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
5797                     if (my $err = $@) {
5798                         warn $err;
5799                     }
5800                 }
5801             });
5802
5803             # delete vmstate files
5804             # since after the restore we have no snapshots anymore
5805             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5806                 my $snap = $oldconf->{snapshots}->{$snapname};
5807                 if ($snap->{vmstate}) {
5808                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
5809                     if (my $err = $@) {
5810                         warn $err;
5811                     }
5812                 }
5813             }
5814         }
5815
5816         my $map = {};
5817         foreach my $virtdev (sort keys %$virtdev_hash) {
5818             my $d = $virtdev_hash->{$virtdev};
5819             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5820             my $storeid = $d->{storeid};
5821             my $scfg = PVE::Storage::storage_config($cfg, $storeid);
5822
5823             my $map_opts = '';
5824             if (my $limit = $storage_limits{$storeid}) {
5825                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
5826             }
5827
5828             # test if requested format is supported
5829             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $storeid);
5830             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5831             $d->{format} = $defFormat if !$supported;
5832
5833             my $volid = PVE::Storage::vdisk_alloc($cfg, $storeid, $vmid,
5834                                                   $d->{format}, undef, $alloc_size);
5835             print STDERR "new volume ID is '$volid'\n";
5836             $d->{volid} = $volid;
5837             my $path = PVE::Storage::path($cfg, $volid);
5838
5839             PVE::Storage::activate_volumes($cfg,[$volid]);
5840
5841             my $write_zeros = 1;
5842             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
5843                 $write_zeros = 0;
5844             }
5845
5846             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
5847
5848             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5849             $map->{$virtdev} = $volid;
5850         }
5851
5852         $fh->seek(0, 0) || die "seek failed - $!\n";
5853
5854         my $outfd = new IO::File ($tmpfn, "w") ||
5855             die "unable to write config for VM $vmid\n";
5856
5857         my $cookie = { netcount => 0 };
5858         while (defined(my $line = <$fh>)) {
5859             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5860         }
5861
5862         $fh->close();
5863         $outfd->close();
5864     };
5865
5866     eval {
5867         # enable interrupts
5868         local $SIG{INT} =
5869             local $SIG{TERM} =
5870             local $SIG{QUIT} =
5871             local $SIG{HUP} =
5872             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
5873         local $SIG{ALRM} = sub { die "got timeout\n"; };
5874
5875         $oldtimeout = alarm($timeout);
5876
5877         my $parser = sub {
5878             my $line = shift;
5879
5880             print "$line\n";
5881
5882             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5883                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5884                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5885             } elsif ($line =~ m/^CTIME: /) {
5886                 # we correctly received the vma config, so we can disable
5887                 # the timeout now for disk allocation (set to 10 minutes, so
5888                 # that we always timeout if something goes wrong)
5889                 alarm(600);
5890                 &$print_devmap();
5891                 print $fifofh "done\n";
5892                 my $tmp = $oldtimeout || 0;
5893                 $oldtimeout = undef;
5894                 alarm($tmp);
5895                 close($fifofh);
5896             }
5897         };
5898
5899         print "restore vma archive: $dbg_cmdstring\n";
5900         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
5901     };
5902     my $err = $@;
5903
5904     alarm($oldtimeout) if $oldtimeout;
5905
5906     my $vollist = [];
5907     foreach my $devname (keys %$devinfo) {
5908         my $volid = $devinfo->{$devname}->{volid};
5909         push @$vollist, $volid if $volid;
5910     }
5911
5912     PVE::Storage::deactivate_volumes($cfg, $vollist);
5913
5914     unlink $mapfifo;
5915
5916     if ($err) {
5917         rmtree $tmpdir;
5918         unlink $tmpfn;
5919
5920         foreach my $devname (keys %$devinfo) {
5921             my $volid = $devinfo->{$devname}->{volid};
5922             next if !$volid;
5923             eval {
5924                 if ($volid =~ m|^/|) {
5925                     unlink $volid || die 'unlink failed\n';
5926                 } else {
5927                     PVE::Storage::vdisk_free($cfg, $volid);
5928                 }
5929                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5930             };
5931             print STDERR "unable to cleanup '$volid' - $@" if $@;
5932         }
5933         die $err;
5934     }
5935
5936     rmtree $tmpdir;
5937
5938     rename($tmpfn, $conffile) ||
5939         die "unable to commit configuration file '$conffile'\n";
5940
5941     PVE::Cluster::cfs_update(); # make sure we read new file
5942
5943     eval { rescan($vmid, 1); };
5944     warn $@ if $@;
5945 }
5946
5947 sub restore_tar_archive {
5948     my ($archive, $vmid, $user, $opts) = @_;
5949
5950     if ($archive ne '-') {
5951         my $firstfile = tar_archive_read_firstfile($archive);
5952         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5953             if $firstfile ne 'qemu-server.conf';
5954     }
5955
5956     my $storecfg = PVE::Storage::config();
5957
5958     # destroy existing data - keep empty config
5959     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
5960     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5961
5962     my $tocmd = "/usr/lib/qemu-server/qmextract";
5963
5964     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5965     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5966     $tocmd .= ' --prealloc' if $opts->{prealloc};
5967     $tocmd .= ' --info' if $opts->{info};
5968
5969     # tar option "xf" does not autodetect compression when read from STDIN,
5970     # so we pipe to zcat
5971     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5972         PVE::Tools::shellquote("--to-command=$tocmd");
5973
5974     my $tmpdir = "/var/tmp/vzdumptmp$$";
5975     mkpath $tmpdir;
5976
5977     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5978     local $ENV{VZDUMP_VMID} = $vmid;
5979     local $ENV{VZDUMP_USER} = $user;
5980
5981     my $conffile = PVE::QemuConfig->config_file($vmid);
5982     my $tmpfn = "$conffile.$$.tmp";
5983
5984     # disable interrupts (always do cleanups)
5985     local $SIG{INT} =
5986         local $SIG{TERM} =
5987         local $SIG{QUIT} =
5988         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
5989
5990     eval {
5991         # enable interrupts
5992         local $SIG{INT} =
5993             local $SIG{TERM} =
5994             local $SIG{QUIT} =
5995             local $SIG{HUP} =
5996             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
5997
5998         if ($archive eq '-') {
5999             print "extracting archive from STDIN\n";
6000             run_command($cmd, input => "<&STDIN");
6001         } else {
6002             print "extracting archive '$archive'\n";
6003             run_command($cmd);
6004         }
6005
6006         return if $opts->{info};
6007
6008         # read new mapping
6009         my $map = {};
6010         my $statfile = "$tmpdir/qmrestore.stat";
6011         if (my $fd = IO::File->new($statfile, "r")) {
6012             while (defined (my $line = <$fd>)) {
6013                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6014                     $map->{$1} = $2 if $1;
6015                 } else {
6016                     print STDERR "unable to parse line in statfile - $line\n";
6017                 }
6018             }
6019             $fd->close();
6020         }
6021
6022         my $confsrc = "$tmpdir/qemu-server.conf";
6023
6024         my $srcfd = new IO::File($confsrc, "r") ||
6025             die "unable to open file '$confsrc'\n";
6026
6027         my $outfd = new IO::File ($tmpfn, "w") ||
6028             die "unable to write config for VM $vmid\n";
6029
6030         my $cookie = { netcount => 0 };
6031         while (defined (my $line = <$srcfd>)) {
6032             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6033         }
6034
6035         $srcfd->close();
6036         $outfd->close();
6037     };
6038     my $err = $@;
6039
6040     if ($err) {
6041
6042         unlink $tmpfn;
6043
6044         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6045
6046         die $err;
6047     }
6048
6049     rmtree $tmpdir;
6050
6051     rename $tmpfn, $conffile ||
6052         die "unable to commit configuration file '$conffile'\n";
6053
6054     PVE::Cluster::cfs_update(); # make sure we read new file
6055
6056     eval { rescan($vmid, 1); };
6057     warn $@ if $@;
6058 };
6059
6060 sub foreach_storage_used_by_vm {
6061     my ($conf, $func) = @_;
6062
6063     my $sidhash = {};
6064
6065     foreach_drive($conf, sub {
6066         my ($ds, $drive) = @_;
6067         return if drive_is_cdrom($drive);
6068
6069         my $volid = $drive->{file};
6070
6071         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6072         $sidhash->{$sid} = $sid if $sid;
6073     });
6074
6075     foreach my $sid (sort keys %$sidhash) {
6076         &$func($sid);
6077     }
6078 }
6079
6080 sub do_snapshots_with_qemu {
6081     my ($storecfg, $volid) = @_;
6082
6083     my $storage_name = PVE::Storage::parse_volume_id($volid);
6084
6085     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
6086         && !$storecfg->{ids}->{$storage_name}->{krbd}){
6087         return 1;
6088     }
6089
6090     if ($volid =~ m/\.(qcow2|qed)$/){
6091         return 1;
6092     }
6093
6094     return undef;
6095 }
6096
6097 sub qga_check_running {
6098     my ($vmid, $nowarn) = @_;
6099
6100     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
6101     if ($@) {
6102         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6103         return 0;
6104     }
6105     return 1;
6106 }
6107
6108 sub template_create {
6109     my ($vmid, $conf, $disk) = @_;
6110
6111     my $storecfg = PVE::Storage::config();
6112
6113     foreach_drive($conf, sub {
6114         my ($ds, $drive) = @_;
6115
6116         return if drive_is_cdrom($drive);
6117         return if $disk && $ds ne $disk;
6118
6119         my $volid = $drive->{file};
6120         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6121
6122         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6123         $drive->{file} = $voliddst;
6124         $conf->{$ds} = print_drive($vmid, $drive);
6125         PVE::QemuConfig->write_config($vmid, $conf);
6126     });
6127 }
6128
6129 sub qemu_img_convert {
6130     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6131
6132     my $storecfg = PVE::Storage::config();
6133     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6134     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6135
6136     if ($src_storeid && $dst_storeid) {
6137
6138         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6139
6140         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6141         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6142
6143         my $src_format = qemu_img_format($src_scfg, $src_volname);
6144         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6145
6146         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6147         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6148
6149         my $cmd = [];
6150         push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6151         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
6152         push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6153         push @$cmd, '-T', 'none' if $src_scfg->{type} eq 'zfspool';
6154         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path;
6155         if ($is_zero_initialized) {
6156             push @$cmd, "zeroinit:$dst_path";
6157         } else {
6158             push @$cmd, $dst_path;
6159         }
6160
6161         my $parser = sub {
6162             my $line = shift;
6163             if($line =~ m/\((\S+)\/100\%\)/){
6164                 my $percent = $1;
6165                 my $transferred = int($size * $percent / 100);
6166                 my $remaining = $size - $transferred;
6167
6168                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6169             }
6170
6171         };
6172
6173         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6174         my $err = $@;
6175         die "copy failed: $err" if $err;
6176     }
6177 }
6178
6179 sub qemu_img_format {
6180     my ($scfg, $volname) = @_;
6181
6182     if ($scfg->{path} && $volname =~ m/\.($QEMU_FORMAT_RE)$/) {
6183         return $1;
6184     } else {
6185         return "raw";
6186     }
6187 }
6188
6189 sub qemu_drive_mirror {
6190     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga) = @_;
6191
6192     $jobs = {} if !$jobs;
6193
6194     my $qemu_target;
6195     my $format;
6196     $jobs->{"drive-$drive"} = {};
6197
6198     if ($dst_volid =~ /^nbd:/) {
6199         $qemu_target = $dst_volid;
6200         $format = "nbd";
6201     } else {
6202         my $storecfg = PVE::Storage::config();
6203         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6204
6205         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6206
6207         $format = qemu_img_format($dst_scfg, $dst_volname);
6208
6209         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6210
6211         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6212     }
6213
6214     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6215     $opts->{format} = $format if $format;
6216
6217     print "drive mirror is starting for drive-$drive\n";
6218
6219     eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); }; #if a job already run for this device,it's throw an error
6220
6221     if (my $err = $@) {
6222         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6223         die "mirroring error: $err";
6224     }
6225
6226     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6227 }
6228
6229 sub qemu_drive_mirror_monitor {
6230     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6231
6232     eval {
6233         my $err_complete = 0;
6234
6235         while (1) {
6236             die "storage migration timed out\n" if $err_complete > 300;
6237
6238             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6239
6240             my $running_mirror_jobs = {};
6241             foreach my $stat (@$stats) {
6242                 next if $stat->{type} ne 'mirror';
6243                 $running_mirror_jobs->{$stat->{device}} = $stat;
6244             }
6245
6246             my $readycounter = 0;
6247
6248             foreach my $job (keys %$jobs) {
6249
6250                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6251                     print "$job : finished\n";
6252                     delete $jobs->{$job};
6253                     next;
6254                 }
6255
6256                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6257
6258                 my $busy = $running_mirror_jobs->{$job}->{busy};
6259                 my $ready = $running_mirror_jobs->{$job}->{ready};
6260                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6261                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6262                     my $remaining = $total - $transferred;
6263                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6264
6265                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6266                 }
6267
6268                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6269             }
6270
6271             last if scalar(keys %$jobs) == 0;
6272
6273             if ($readycounter == scalar(keys %$jobs)) {
6274                 print "all mirroring jobs are ready \n";
6275                 last if $skipcomplete; #do the complete later
6276
6277                 if ($vmiddst && $vmiddst != $vmid) {
6278                     my $agent_running = $qga && qga_check_running($vmid);
6279                     if ($agent_running) {
6280                         print "freeze filesystem\n";
6281                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6282                     } else {
6283                         print "suspend vm\n";
6284                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6285                     }
6286
6287                     # if we clone a disk for a new target vm, we don't switch the disk
6288                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6289
6290                     if ($agent_running) {
6291                         print "unfreeze filesystem\n";
6292                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6293                     } else {
6294                         print "resume vm\n";
6295                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6296                     }
6297
6298                     last;
6299                 } else {
6300
6301                     foreach my $job (keys %$jobs) {
6302                         # try to switch the disk if source and destination are on the same guest
6303                         print "$job: Completing block job...\n";
6304
6305                         eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
6306                         if ($@ =~ m/cannot be completed/) {
6307                             print "$job: Block job cannot be completed, try again.\n";
6308                             $err_complete++;
6309                         }else {
6310                             print "$job: Completed successfully.\n";
6311                             $jobs->{$job}->{complete} = 1;
6312                         }
6313                     }
6314                 }
6315             }
6316             sleep 1;
6317         }
6318     };
6319     my $err = $@;
6320
6321     if ($err) {
6322         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6323         die "mirroring error: $err";
6324     }
6325
6326 }
6327
6328 sub qemu_blockjobs_cancel {
6329     my ($vmid, $jobs) = @_;
6330
6331     foreach my $job (keys %$jobs) {
6332         print "$job: Cancelling block job\n";
6333         eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
6334         $jobs->{$job}->{cancel} = 1;
6335     }
6336
6337     while (1) {
6338         my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6339
6340         my $running_jobs = {};
6341         foreach my $stat (@$stats) {
6342             $running_jobs->{$stat->{device}} = $stat;
6343         }
6344
6345         foreach my $job (keys %$jobs) {
6346
6347             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6348                 print "$job: Done.\n";
6349                 delete $jobs->{$job};
6350             }
6351         }
6352
6353         last if scalar(keys %$jobs) == 0;
6354
6355         sleep 1;
6356     }
6357 }
6358
6359 sub clone_disk {
6360     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6361         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga) = @_;
6362
6363     my $newvolid;
6364
6365     if (!$full) {
6366         print "create linked clone of drive $drivename ($drive->{file})\n";
6367         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6368         push @$newvollist, $newvolid;
6369     } else {
6370
6371         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6372         $storeid = $storage if $storage;
6373
6374         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
6375         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6376
6377         print "create full clone of drive $drivename ($drive->{file})\n";
6378         my $name = undef;
6379         if (drive_is_cloudinit($drive)) {
6380             $name = "vm-$newvmid-cloudinit";
6381             # cloudinit only supports raw and qcow2 atm:
6382             if ($dst_format eq 'qcow2') {
6383                 $name .= '.qcow2';
6384             } elsif ($dst_format ne 'raw') {
6385                 die "clone: unhandled format for cloudinit image\n";
6386             }
6387         }
6388         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
6389         push @$newvollist, $newvolid;
6390
6391         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
6392
6393         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6394         if (!$running || $snapname) {
6395             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6396         } else {
6397
6398             my $kvmver = get_running_qemu_version ($vmid);
6399             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
6400                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6401                     if $drive->{iothread};
6402             }
6403
6404             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga);
6405         }
6406     }
6407
6408     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6409
6410     my $disk = $drive;
6411     $disk->{format} = undef;
6412     $disk->{file} = $newvolid;
6413     $disk->{size} = $size;
6414
6415     return $disk;
6416 }
6417
6418 # this only works if VM is running
6419 sub get_current_qemu_machine {
6420     my ($vmid) = @_;
6421
6422     my $cmd = { execute => 'query-machines', arguments => {} };
6423     my $res = vm_qmp_command($vmid, $cmd);
6424
6425     my ($current, $default);
6426     foreach my $e (@$res) {
6427         $default = $e->{name} if $e->{'is-default'};
6428         $current = $e->{name} if $e->{'is-current'};
6429     }
6430
6431     # fallback to the default machine if current is not supported by qemu
6432     return $current || $default || 'pc';
6433 }
6434
6435 sub get_running_qemu_version {
6436     my ($vmid) = @_;
6437     my $cmd = { execute => 'query-version', arguments => {} };
6438     my $res = vm_qmp_command($vmid, $cmd);
6439     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
6440 }
6441
6442 sub qemu_machine_feature_enabled {
6443     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6444
6445     my $current_major;
6446     my $current_minor;
6447
6448     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6449
6450         $current_major = $3;
6451         $current_minor = $4;
6452
6453     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6454
6455         $current_major = $1;
6456         $current_minor = $2;
6457     }
6458
6459     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
6460
6461
6462 }
6463
6464 sub qemu_machine_pxe {
6465     my ($vmid, $conf, $machine) = @_;
6466
6467     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6468
6469     if ($conf->{machine} && $conf->{machine} =~ m/\.pxe$/) {
6470         $machine .= '.pxe';
6471     }
6472
6473     return $machine;
6474 }
6475
6476 sub qemu_use_old_bios_files {
6477     my ($machine_type) = @_;
6478
6479     return if !$machine_type;
6480
6481     my $use_old_bios_files = undef;
6482
6483     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6484         $machine_type = $1;
6485         $use_old_bios_files = 1;
6486     } else {
6487         my $kvmver = kvm_user_version();
6488         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6489         # load new efi bios files on migration. So this hack is required to allow
6490         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6491         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6492         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
6493     }
6494
6495     return ($use_old_bios_files, $machine_type);
6496 }
6497
6498 sub create_efidisk {
6499     my ($storecfg, $storeid, $vmid, $fmt) = @_;
6500
6501     die "EFI vars default image not found\n" if ! -f $OVMF_VARS;
6502
6503     my $vars_size = PVE::Tools::convert_size(-s $OVMF_VARS, 'b' => 'kb');
6504     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
6505     PVE::Storage::activate_volumes($storecfg, [$volid]);
6506
6507     my $path = PVE::Storage::path($storecfg, $volid);
6508     eval {
6509         run_command(['/usr/bin/qemu-img', 'convert', '-n', '-f', 'raw', '-O', $fmt, $OVMF_VARS, $path]);
6510     };
6511     die "Copying EFI vars image failed: $@" if $@;
6512
6513     return ($volid, $vars_size);
6514 }
6515
6516 sub lspci {
6517
6518     my $devices = {};
6519
6520     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6521             my (undef, $id, $function) = @_;
6522             my $res = { id => $id, function => $function};
6523             push @{$devices->{$id}}, $res;
6524     });
6525
6526     # Entries should be sorted by functions.
6527     foreach my $id (keys %$devices) {
6528         my $dev = $devices->{$id};
6529         $devices->{$id} = [ sort { $a->{function} <=> $b->{function} } @$dev ];
6530     }
6531
6532     return $devices;
6533 }
6534
6535 sub vm_iothreads_list {
6536     my ($vmid) = @_;
6537
6538     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6539
6540     my $iothreads = {};
6541     foreach my $iothread (@$res) {
6542         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6543     }
6544
6545     return $iothreads;
6546 }
6547
6548 sub scsihw_infos {
6549     my ($conf, $drive) = @_;
6550
6551     my $maxdev = 0;
6552
6553     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
6554         $maxdev = 7;
6555     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6556         $maxdev = 1;
6557     } else {
6558         $maxdev = 256;
6559     }
6560
6561     my $controller = int($drive->{index} / $maxdev);
6562     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6563
6564     return ($maxdev, $controller, $controller_prefix);
6565 }
6566
6567 sub add_hyperv_enlightenments {
6568     my ($cpuFlags, $winversion, $machine_type, $kvmver, $bios, $gpu_passthrough) = @_;
6569
6570     return if $winversion < 6;
6571     return if $bios && $bios eq 'ovmf' && $winversion < 8;
6572
6573     push @$cpuFlags , 'hv_vendor_id=proxmox' if $gpu_passthrough;
6574
6575     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
6576         push @$cpuFlags , 'hv_spinlocks=0x1fff';
6577         push @$cpuFlags , 'hv_vapic';
6578         push @$cpuFlags , 'hv_time';
6579     } else {
6580         push @$cpuFlags , 'hv_spinlocks=0xffff';
6581     }
6582
6583     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
6584         push @$cpuFlags , 'hv_reset';
6585         push @$cpuFlags , 'hv_vpindex';
6586         push @$cpuFlags , 'hv_runtime';
6587     }
6588
6589     if ($winversion >= 7) {
6590         push @$cpuFlags , 'hv_relaxed';
6591     }
6592 }
6593
6594 sub windows_version {
6595     my ($ostype) = @_;
6596
6597     return 0 if !$ostype;
6598
6599     my $winversion = 0;
6600
6601     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
6602         $winversion = 5;
6603     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
6604         $winversion = 6;
6605     } elsif ($ostype =~ m/^win(\d+)$/) {
6606         $winversion = $1;
6607     }
6608
6609     return $winversion;
6610 }
6611
6612 sub resolve_dst_disk_format {
6613         my ($storecfg, $storeid, $src_volname, $format) = @_;
6614         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6615
6616         if (!$format) {
6617             # if no target format is specified, use the source disk format as hint
6618             if ($src_volname) {
6619                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6620                 $format = qemu_img_format($scfg, $src_volname);
6621             } else {
6622                 return $defFormat;
6623             }
6624         }
6625
6626         # test if requested format is supported - else use default
6627         my $supported = grep { $_ eq $format } @$validFormats;
6628         $format = $defFormat if !$supported;
6629         return $format;
6630 }
6631
6632 sub resolve_first_disk {
6633     my $conf = shift;
6634     my @disks = PVE::QemuServer::valid_drive_names();
6635     my $firstdisk;
6636     foreach my $ds (reverse @disks) {
6637         next if !$conf->{$ds};
6638         my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
6639         next if PVE::QemuServer::drive_is_cdrom($disk);
6640         $firstdisk = $ds;
6641     }
6642     return $firstdisk;
6643 }
6644
6645 sub generate_smbios1_uuid {
6646     my ($uuid, $uuid_str);
6647     UUID::generate($uuid);
6648     UUID::unparse($uuid, $uuid_str);
6649     return "uuid=$uuid_str";
6650 }
6651
6652 # bash completion helper
6653
6654 sub complete_backup_archives {
6655     my ($cmdname, $pname, $cvalue) = @_;
6656
6657     my $cfg = PVE::Storage::config();
6658
6659     my $storeid;
6660
6661     if ($cvalue =~ m/^([^:]+):/) {
6662         $storeid = $1;
6663     }
6664
6665     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6666
6667     my $res = [];
6668     foreach my $id (keys %$data) {
6669         foreach my $item (@{$data->{$id}}) {
6670             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6671             push @$res, $item->{volid} if defined($item->{volid});
6672         }
6673     }
6674
6675     return $res;
6676 }
6677
6678 my $complete_vmid_full = sub {
6679     my ($running) = @_;
6680
6681     my $idlist = vmstatus();
6682
6683     my $res = [];
6684
6685     foreach my $id (keys %$idlist) {
6686         my $d = $idlist->{$id};
6687         if (defined($running)) {
6688             next if $d->{template};
6689             next if $running && $d->{status} ne 'running';
6690             next if !$running && $d->{status} eq 'running';
6691         }
6692         push @$res, $id;
6693
6694     }
6695     return $res;
6696 };
6697
6698 sub complete_vmid {
6699     return &$complete_vmid_full();
6700 }
6701
6702 sub complete_vmid_stopped {
6703     return &$complete_vmid_full(0);
6704 }
6705
6706 sub complete_vmid_running {
6707     return &$complete_vmid_full(1);
6708 }
6709
6710 sub complete_storage {
6711
6712     my $cfg = PVE::Storage::config();
6713     my $ids = $cfg->{ids};
6714
6715     my $res = [];
6716     foreach my $sid (keys %$ids) {
6717         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6718         next if !$ids->{$sid}->{content}->{images};
6719         push @$res, $sid;
6720     }
6721
6722     return $res;
6723 }
6724
6725 sub nbd_stop {
6726     my ($vmid) = @_;
6727
6728     vm_mon_cmd($vmid, 'nbd-server-stop');
6729 }
6730
6731 1;