PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach $IPV6RE);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QemuConfig;
  31 use PVE::QMPClient;
  32 use PVE::RPCEnvironment;
  33 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
  34 use PVE::QemuServer::Memory;
  35 use PVE::QemuServer::USB qw(parse_usb_device);
  36 use PVE::QemuServer::Cloudinit;
  37 use Time::HiRes qw(gettimeofday);
  38 use File::Copy qw(copy);
  39 use URI::Escape;
  40
  41 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  42 my $OVMF_CODE = "$EDK2_FW_BASE/OVMF_CODE.fd";
  43 my $OVMF_VARS = "$EDK2_FW_BASE/OVMF_VARS.fd";
  44
  45 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  46
  47 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  48
  49 my $QEMU_FORMAT_RE = qr/raw|cow|qcow|qcow2|qed|vmdk|cloop/;
  50
  51 # Note about locking: we use flock on the config file protect
  52 # against concurent actions.
  53 # Aditionaly, we have a 'lock' setting in the config file. This
  54 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  55 # allowed when such lock is set. But you can ignore this kind of
  56 # lock with the --skiplock flag.
  57
  58 cfs_register_file('/qemu-server/',
  59                   \&parse_vm_config,
  60                   \&write_vm_config);
  61
  62 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  63     description => "Some command save/restore state from this location.",
  64     type => 'string',
  65     maxLength => 128,
  66     optional => 1,
  67 });
  68
  69 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  70     description => "The name of the snapshot.",
  71     type => 'string', format => 'pve-configid',
  72     maxLength => 40,
  73 });
  74
  75 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  76     type => 'string',
  77     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  78     description => "The drive's backing file's data format.",
  79     optional => 1,
  80 });
  81
  82 #no warnings 'redefine';
  83
  84 sub cgroups_write {
  85    my ($controller, $vmid, $option, $value) = @_;
  86
  87    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  88    PVE::ProcFSTools::write_proc_entry($path, $value);
  89
  90 }
  91
  92 my $nodename = PVE::INotify::nodename();
  93
  94 mkdir "/etc/pve/nodes/$nodename";
  95 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  96 mkdir $confdir;
  97
  98 my $var_run_tmpdir = "/var/run/qemu-server";
  99 mkdir $var_run_tmpdir;
 100
 101 my $lock_dir = "/var/lock/qemu-server";
 102 mkdir $lock_dir;
 103
 104 my $pcisysfs = "/sys/bus/pci";
 105
 106 my $cpu_vendor_list = {
 107     # Intel CPUs
 108     486 => 'GenuineIntel',
 109     pentium => 'GenuineIntel',
 110     pentium2  => 'GenuineIntel',
 111     pentium3  => 'GenuineIntel',
 112     coreduo => 'GenuineIntel',
 113     core2duo => 'GenuineIntel',
 114     Conroe  => 'GenuineIntel',
 115     Penryn  => 'GenuineIntel',
 116     Nehalem  => 'GenuineIntel',
 117     'Nehalem-IBRS'  => 'GenuineIntel',
 118     Westmere => 'GenuineIntel',
 119     'Westmere-IBRS' => 'GenuineIntel',
 120     SandyBridge => 'GenuineIntel',
 121     'SandyBridge-IBRS' => 'GenuineIntel',
 122     IvyBridge => 'GenuineIntel',
 123     'IvyBridge-IBRS' => 'GenuineIntel',
 124     Haswell => 'GenuineIntel',
 125     'Haswell-IBRS' => 'GenuineIntel',
 126     'Haswell-noTSX' => 'GenuineIntel',
 127     'Haswell-noTSX-IBRS' => 'GenuineIntel',
 128     Broadwell => 'GenuineIntel',
 129     'Broadwell-IBRS' => 'GenuineIntel',
 130     'Broadwell-noTSX' => 'GenuineIntel',
 131     'Broadwell-noTSX-IBRS' => 'GenuineIntel',
 132     'Skylake-Client' => 'GenuineIntel',
 133     'Skylake-Client-IBRS' => 'GenuineIntel',
 134     'Skylake-Server' => 'GenuineIntel',
 135     'Skylake-Server-IBRS' => 'GenuineIntel',
 136
 137     # AMD CPUs
 138     athlon => 'AuthenticAMD',
 139     phenom  => 'AuthenticAMD',
 140     Opteron_G1  => 'AuthenticAMD',
 141     Opteron_G2  => 'AuthenticAMD',
 142     Opteron_G3  => 'AuthenticAMD',
 143     Opteron_G4  => 'AuthenticAMD',
 144     Opteron_G5  => 'AuthenticAMD',
 145     EPYC => 'AuthenticAMD',
 146     'EPYC-IBPB' => 'AuthenticAMD',
 147
 148     # generic types, use vendor from host node
 149     host => 'default',
 150     kvm32 => 'default',
 151     kvm64 => 'default',
 152     qemu32 => 'default',
 153     qemu64 => 'default',
 154     max => 'default',
 155 };
 156
 157 my $cpu_flag = qr/[+-](pcid|spec-ctrl)/;
 158
 159 my $cpu_fmt = {
 160     cputype => {
 161         description => "Emulated CPU type.",
 162         type => 'string',
 163         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 164         default => 'kvm64',
 165         default_key => 1,
 166     },
 167     hidden => {
 168         description => "Do not identify as a KVM virtual machine.",
 169         type => 'boolean',
 170         optional => 1,
 171         default => 0
 172     },
 173     flags => {
 174         description => "List of additional CPU flags separated by ';'."
 175                      . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
 176                      . " Currently supported flags: 'pcid', 'spec-ctrl'.",
 177         format_description => '+FLAG[;-FLAG...]',
 178         type => 'string',
 179         pattern => qr/$cpu_flag(;$cpu_flag)*/,
 180         optional => 1,
 181     },
 182 };
 183
 184 my $watchdog_fmt = {
 185     model => {
 186         default_key => 1,
 187         type => 'string',
 188         enum => [qw(i6300esb ib700)],
 189         description => "Watchdog type to emulate.",
 190         default => 'i6300esb',
 191         optional => 1,
 192     },
 193     action => {
 194         type => 'string',
 195         enum => [qw(reset shutdown poweroff pause debug none)],
 196         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 197         optional => 1,
 198     },
 199 };
 200 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 201
 202 my $confdesc = {
 203     onboot => {
 204         optional => 1,
 205         type => 'boolean',
 206         description => "Specifies whether a VM will be started during system bootup.",
 207         default => 0,
 208     },
 209     autostart => {
 210         optional => 1,
 211         type => 'boolean',
 212         description => "Automatic restart after crash (currently ignored).",
 213         default => 0,
 214     },
 215     hotplug => {
 216         optional => 1,
 217         type => 'string', format => 'pve-hotplug-features',
 218         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 219         default => 'network,disk,usb',
 220     },
 221     reboot => {
 222         optional => 1,
 223         type => 'boolean',
 224         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 225         default => 1,
 226     },
 227     lock => {
 228         optional => 1,
 229         type => 'string',
 230         description => "Lock/unlock the VM.",
 231         enum => [qw(migrate backup snapshot rollback)],
 232     },
 233     cpulimit => {
 234         optional => 1,
 235         type => 'number',
 236         description => "Limit of CPU usage.",
 237         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 238         minimum => 0,
 239         maximum => 128,
 240         default => 0,
 241     },
 242     cpuunits => {
 243         optional => 1,
 244         type => 'integer',
 245         description => "CPU weight for a VM.",
 246         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 247         minimum => 2,
 248         maximum => 262144,
 249         default => 1024,
 250     },
 251     memory => {
 252         optional => 1,
 253         type => 'integer',
 254         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 255         minimum => 16,
 256         default => 512,
 257     },
 258     balloon => {
 259         optional => 1,
 260         type => 'integer',
 261         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 262         minimum => 0,
 263     },
 264     shares => {
 265         optional => 1,
 266         type => 'integer',
 267         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 268         minimum => 0,
 269         maximum => 50000,
 270         default => 1000,
 271     },
 272     keyboard => {
 273         optional => 1,
 274         type => 'string',
 275         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.conf' configuration file.".
 276                        "It should not be necessary to set it.",
 277         enum => PVE::Tools::kvmkeymaplist(),
 278         default => undef,
 279     },
 280     name => {
 281         optional => 1,
 282         type => 'string', format => 'dns-name',
 283         description => "Set a name for the VM. Only used on the configuration web interface.",
 284     },
 285     scsihw => {
 286         optional => 1,
 287         type => 'string',
 288         description => "SCSI controller model",
 289         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 290         default => 'lsi',
 291     },
 292     description => {
 293         optional => 1,
 294         type => 'string',
 295         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 296     },
 297     ostype => {
 298         optional => 1,
 299         type => 'string',
 300         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 301         description => "Specify guest operating system.",
 302         verbose_description => <<EODESC,
 303 Specify guest operating system. This is used to enable special
 304 optimization/features for specific operating systems:
 305
 306 [horizontal]
 307 other;; unspecified OS
 308 wxp;; Microsoft Windows XP
 309 w2k;; Microsoft Windows 2000
 310 w2k3;; Microsoft Windows 2003
 311 w2k8;; Microsoft Windows 2008
 312 wvista;; Microsoft Windows Vista
 313 win7;; Microsoft Windows 7
 314 win8;; Microsoft Windows 8/2012/2012r2
 315 win10;; Microsoft Windows 10/2016
 316 l24;; Linux 2.4 Kernel
 317 l26;; Linux 2.6/3.X Kernel
 318 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 319 EODESC
 320     },
 321     boot => {
 322         optional => 1,
 323         type => 'string',
 324         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 325         pattern => '[acdn]{1,4}',
 326         default => 'cdn',
 327     },
 328     bootdisk => {
 329         optional => 1,
 330         type => 'string', format => 'pve-qm-bootdisk',
 331         description => "Enable booting from specified disk.",
 332         pattern => '(ide|sata|scsi|virtio)\d+',
 333     },
 334     smp => {
 335         optional => 1,
 336         type => 'integer',
 337         description => "The number of CPUs. Please use option -sockets instead.",
 338         minimum => 1,
 339         default => 1,
 340     },
 341     sockets => {
 342         optional => 1,
 343         type => 'integer',
 344         description => "The number of CPU sockets.",
 345         minimum => 1,
 346         default => 1,
 347     },
 348     cores => {
 349         optional => 1,
 350         type => 'integer',
 351         description => "The number of cores per socket.",
 352         minimum => 1,
 353         default => 1,
 354     },
 355     numa => {
 356         optional => 1,
 357         type => 'boolean',
 358         description => "Enable/disable NUMA.",
 359         default => 0,
 360     },
 361     hugepages => {
 362         optional => 1,
 363         type => 'string',
 364         description => "Enable/disable hugepages memory.",
 365         enum => [qw(any 2 1024)],
 366     },
 367     vcpus => {
 368         optional => 1,
 369         type => 'integer',
 370         description => "Number of hotplugged vcpus.",
 371         minimum => 1,
 372         default => 0,
 373     },
 374     acpi => {
 375         optional => 1,
 376         type => 'boolean',
 377         description => "Enable/disable ACPI.",
 378         default => 1,
 379     },
 380     agent => {
 381         optional => 1,
 382         type => 'boolean',
 383         description => "Enable/disable Qemu GuestAgent.",
 384         default => 0,
 385     },
 386     kvm => {
 387         optional => 1,
 388         type => 'boolean',
 389         description => "Enable/disable KVM hardware virtualization.",
 390         default => 1,
 391     },
 392     tdf => {
 393         optional => 1,
 394         type => 'boolean',
 395         description => "Enable/disable time drift fix.",
 396         default => 0,
 397     },
 398     localtime => {
 399         optional => 1,
 400         type => 'boolean',
 401         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 402     },
 403     freeze => {
 404         optional => 1,
 405         type => 'boolean',
 406         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 407     },
 408     vga => {
 409         optional => 1,
 410         type => 'string',
 411         description => "Select the VGA type.",
 412         verbose_description => "Select the VGA type. If you want to use high resolution" .
 413             " modes (>= 1280x1024x16) then you should use the options " .
 414             "'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and " .
 415             "'cirrus' for other OS types. The 'qxl' option enables the SPICE " .
 416             "display sever. For win* OS you can select how many independent " .
 417             "displays you want, Linux guests can add displays them self. " .
 418             "You can also run without any graphic card, using a serial device" .
 419             " as terminal.",
 420         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 421     },
 422     watchdog => {
 423         optional => 1,
 424         type => 'string', format => 'pve-qm-watchdog',
 425         description => "Create a virtual hardware watchdog device.",
 426         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 427             " (by a guest action), the watchdog must be periodically polled " .
 428             "by an agent inside the guest or else the watchdog will reset " .
 429             "the guest (or execute the respective action specified)",
 430     },
 431     startdate => {
 432         optional => 1,
 433         type => 'string',
 434         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 435         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 436         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 437         default => 'now',
 438     },
 439     startup =>  get_standard_option('pve-startup-order'),
 440     template => {
 441         optional => 1,
 442         type => 'boolean',
 443         description => "Enable/disable Template.",
 444         default => 0,
 445     },
 446     args => {
 447         optional => 1,
 448         type => 'string',
 449         description => "Arbitrary arguments passed to kvm.",
 450         verbose_description => <<EODESCR,
 451 Arbitrary arguments passed to kvm, for example:
 452
 453 args: -no-reboot -no-hpet
 454
 455 NOTE: this option is for experts only.
 456 EODESCR
 457     },
 458     tablet => {
 459         optional => 1,
 460         type => 'boolean',
 461         default => 1,
 462         description => "Enable/disable the USB tablet device.",
 463         verbose_description => "Enable/disable the USB tablet device. This device is " .
 464             "usually needed to allow absolute mouse positioning with VNC. " .
 465             "Else the mouse runs out of sync with normal VNC clients. " .
 466             "If you're running lots of console-only guests on one host, " .
 467             "you may consider disabling this to save some context switches. " .
 468             "This is turned off by default if you use spice (-vga=qxl).",
 469     },
 470     migrate_speed => {
 471         optional => 1,
 472         type => 'integer',
 473         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 474         minimum => 0,
 475         default => 0,
 476     },
 477     migrate_downtime => {
 478         optional => 1,
 479         type => 'number',
 480         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 481         minimum => 0,
 482         default => 0.1,
 483     },
 484     cdrom => {
 485         optional => 1,
 486         type => 'string', format => 'pve-qm-ide',
 487         typetext => '<volume>',
 488         description => "This is an alias for option -ide2",
 489     },
 490     cpu => {
 491         optional => 1,
 492         description => "Emulated CPU type.",
 493         type => 'string',
 494         format => $cpu_fmt,
 495     },
 496     parent => get_standard_option('pve-snapshot-name', {
 497         optional => 1,
 498         description => "Parent snapshot name. This is used internally, and should not be modified.",
 499     }),
 500     snaptime => {
 501         optional => 1,
 502         description => "Timestamp for snapshots.",
 503         type => 'integer',
 504         minimum => 0,
 505     },
 506     vmstate => {
 507         optional => 1,
 508         type => 'string', format => 'pve-volume-id',
 509         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 510     },
 511     vmstatestorage => get_standard_option('pve-storage-id', {
 512         description => "Default storage for VM state volumes/files.",
 513         optional => 1,
 514     }),
 515     machine => {
 516         description => "Specific the Qemu machine type.",
 517         type => 'string',
 518         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
 519         maxLength => 40,
 520         optional => 1,
 521     },
 522     smbios1 => {
 523         description => "Specify SMBIOS type 1 fields.",
 524         type => 'string', format => 'pve-qm-smbios1',
 525         maxLength => 256,
 526         optional => 1,
 527     },
 528     protection => {
 529         optional => 1,
 530         type => 'boolean',
 531         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 532         default => 0,
 533     },
 534     bios => {
 535         optional => 1,
 536         type => 'string',
 537         enum => [ qw(seabios ovmf) ],
 538         description => "Select BIOS implementation.",
 539         default => 'seabios',
 540     },
 541 };
 542
 543 my $confdesc_cloudinit = {
 544     citype => {
 545         optional => 1,
 546         type => 'string',
 547         description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
 548         enum => ['configdrive2', 'nocloud'],
 549     },
 550     ciuser => {
 551         optional => 1,
 552         type => 'string',
 553         description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
 554     },
 555     cipassword => {
 556         optional => 1,
 557         type => 'string',
 558         description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
 559     },
 560     searchdomain => {
 561         optional => 1,
 562         type => 'string',
 563         description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 564     },
 565     nameserver => {
 566         optional => 1,
 567         type => 'string', format => 'address-list',
 568         description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 569     },
 570     sshkeys => {
 571         optional => 1,
 572         type => 'string',
 573         format => 'urlencoded',
 574         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 575     },
 576 };
 577
 578 # what about other qemu settings ?
 579 #cpu => 'string',
 580 #machine => 'string',
 581 #fda => 'file',
 582 #fdb => 'file',
 583 #mtdblock => 'file',
 584 #sd => 'file',
 585 #pflash => 'file',
 586 #snapshot => 'bool',
 587 #bootp => 'file',
 588 ##tftp => 'dir',
 589 ##smb => 'dir',
 590 #kernel => 'file',
 591 #append => 'string',
 592 #initrd => 'file',
 593 ##soundhw => 'string',
 594
 595 while (my ($k, $v) = each %$confdesc) {
 596     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 597 }
 598
 599 my $MAX_IDE_DISKS = 4;
 600 my $MAX_SCSI_DISKS = 14;
 601 my $MAX_VIRTIO_DISKS = 16;
 602 my $MAX_SATA_DISKS = 6;
 603 my $MAX_USB_DEVICES = 5;
 604 my $MAX_NETS = 32;
 605 my $MAX_UNUSED_DISKS = 8;
 606 my $MAX_HOSTPCI_DEVICES = 4;
 607 my $MAX_SERIAL_PORTS = 4;
 608 my $MAX_PARALLEL_PORTS = 3;
 609 my $MAX_NUMA = 8;
 610
 611 my $numa_fmt = {
 612     cpus => {
 613         type => "string",
 614         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 615         description => "CPUs accessing this NUMA node.",
 616         format_description => "id[-id];...",
 617     },
 618     memory => {
 619         type => "number",
 620         description => "Amount of memory this NUMA node provides.",
 621         optional => 1,
 622     },
 623     hostnodes => {
 624         type => "string",
 625         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 626         description => "Host NUMA nodes to use.",
 627         format_description => "id[-id];...",
 628         optional => 1,
 629     },
 630     policy => {
 631         type => 'string',
 632         enum => [qw(preferred bind interleave)],
 633         description => "NUMA allocation policy.",
 634         optional => 1,
 635     },
 636 };
 637 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 638 my $numadesc = {
 639     optional => 1,
 640     type => 'string', format => $numa_fmt,
 641     description => "NUMA topology.",
 642 };
 643 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 644
 645 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 646     $confdesc->{"numa$i"} = $numadesc;
 647 }
 648
 649 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 650                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 651                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 652 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 653
 654 my $net_fmt_bridge_descr = <<__EOD__;
 655 Bridge to attach the network device to. The Proxmox VE standard bridge
 656 is called 'vmbr0'.
 657
 658 If you do not specify a bridge, we create a kvm user (NATed) network
 659 device, which provides DHCP and DNS services. The following addresses
 660 are used:
 661
 662  10.0.2.2   Gateway
 663  10.0.2.3   DNS Server
 664  10.0.2.4   SMB Server
 665
 666 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 667 __EOD__
 668
 669 my $net_fmt = {
 670     macaddr => {
 671         type => 'string',
 672         pattern => qr/[0-9a-f]{2}(?::[0-9a-f]{2}){5}/i,
 673         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 674         format_description => "XX:XX:XX:XX:XX:XX",
 675         optional => 1,
 676     },
 677     model => {
 678         type => 'string',
 679         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 680         enum => $nic_model_list,
 681         default_key => 1,
 682     },
 683     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 684     bridge => {
 685         type => 'string',
 686         description => $net_fmt_bridge_descr,
 687         format_description => 'bridge',
 688         optional => 1,
 689     },
 690     queues => {
 691         type => 'integer',
 692         minimum => 0, maximum => 16,
 693         description => 'Number of packet queues to be used on the device.',
 694         optional => 1,
 695     },
 696     rate => {
 697         type => 'number',
 698         minimum => 0,
 699         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 700         optional => 1,
 701     },
 702     tag => {
 703         type => 'integer',
 704         minimum => 1, maximum => 4094,
 705         description => 'VLAN tag to apply to packets on this interface.',
 706         optional => 1,
 707     },
 708     trunks => {
 709         type => 'string',
 710         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 711         description => 'VLAN trunks to pass through this interface.',
 712         format_description => 'vlanid[;vlanid...]',
 713         optional => 1,
 714     },
 715     firewall => {
 716         type => 'boolean',
 717         description => 'Whether this interface should be protected by the firewall.',
 718         optional => 1,
 719     },
 720     link_down => {
 721         type => 'boolean',
 722         description => 'Whether this interface should be disconnected (like pulling the plug).',
 723         optional => 1,
 724     },
 725 };
 726
 727 my $netdesc = {
 728     optional => 1,
 729     type => 'string', format => $net_fmt,
 730     description => "Specify network devices.",
 731 };
 732
 733 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 734
 735 my $ipconfig_fmt = {
 736     ip => {
 737         type => 'string',
 738         format => 'pve-ipv4-config',
 739         format_description => 'IPv4Format/CIDR',
 740         description => 'IPv4 address in CIDR format.',
 741         optional => 1,
 742         default => 'dhcp',
 743     },
 744     gw => {
 745         type => 'string',
 746         format => 'ipv4',
 747         format_description => 'GatewayIPv4',
 748         description => 'Default gateway for IPv4 traffic.',
 749         optional => 1,
 750         requires => 'ip',
 751     },
 752     ip6 => {
 753         type => 'string',
 754         format => 'pve-ipv6-config',
 755         format_description => 'IPv6Format/CIDR',
 756         description => 'IPv6 address in CIDR format.',
 757         optional => 1,
 758         default => 'dhcp',
 759     },
 760     gw6 => {
 761         type => 'string',
 762         format => 'ipv6',
 763         format_description => 'GatewayIPv6',
 764         description => 'Default gateway for IPv6 traffic.',
 765         optional => 1,
 766         requires => 'ip6',
 767     },
 768 };
 769 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 770 my $ipconfigdesc = {
 771     optional => 1,
 772     type => 'string', format => 'pve-qm-ipconfig',
 773     description => <<'EODESCR',
 774 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 775
 776 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 777
 778 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
 779 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 780
 781 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
 782 EODESCR
 783 };
 784 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 785
 786 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 787     $confdesc->{"net$i"} = $netdesc;
 788     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 789 }
 790
 791 foreach my $key (keys %$confdesc_cloudinit) {
 792     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 793 }
 794
 795 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 796 sub verify_volume_id_or_qm_path {
 797     my ($volid, $noerr) = @_;
 798
 799     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 800         return $volid;
 801     }
 802
 803     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 804     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 805     if ($@) {
 806         return undef if $noerr;
 807         die $@;
 808     }
 809     return $volid;
 810 }
 811
 812 my $drivename_hash;
 813
 814 my %drivedesc_base = (
 815     volume => { alias => 'file' },
 816     file => {
 817         type => 'string',
 818         format => 'pve-volume-id-or-qm-path',
 819         default_key => 1,
 820         format_description => 'volume',
 821         description => "The drive's backing volume.",
 822     },
 823     media => {
 824         type => 'string',
 825         enum => [qw(cdrom disk)],
 826         description => "The drive's media type.",
 827         default => 'disk',
 828         optional => 1
 829     },
 830     cyls => {
 831         type => 'integer',
 832         description => "Force the drive's physical geometry to have a specific cylinder count.",
 833         optional => 1
 834     },
 835     heads => {
 836         type => 'integer',
 837         description => "Force the drive's physical geometry to have a specific head count.",
 838         optional => 1
 839     },
 840     secs => {
 841         type => 'integer',
 842         description => "Force the drive's physical geometry to have a specific sector count.",
 843         optional => 1
 844     },
 845     trans => {
 846         type => 'string',
 847         enum => [qw(none lba auto)],
 848         description => "Force disk geometry bios translation mode.",
 849         optional => 1,
 850     },
 851     snapshot => {
 852         type => 'boolean',
 853         description => "Controls qemu's snapshot mode feature."
 854             . " If activated, changes made to the disk are temporary and will"
 855             . " be discarded when the VM is shutdown.",
 856         optional => 1,
 857     },
 858     cache => {
 859         type => 'string',
 860         enum => [qw(none writethrough writeback unsafe directsync)],
 861         description => "The drive's cache mode",
 862         optional => 1,
 863     },
 864     format => get_standard_option('pve-qm-image-format'),
 865     size => {
 866         type => 'string',
 867         format => 'disk-size',
 868         format_description => 'DiskSize',
 869         description => "Disk size. This is purely informational and has no effect.",
 870         optional => 1,
 871     },
 872     backup => {
 873         type => 'boolean',
 874         description => "Whether the drive should be included when making backups.",
 875         optional => 1,
 876     },
 877     replicate => {
 878         type => 'boolean',
 879         description => 'Whether the drive should considered for replication jobs.',
 880         optional => 1,
 881         default => 1,
 882     },
 883     rerror => {
 884         type => 'string',
 885         enum => [qw(ignore report stop)],
 886         description => 'Read error action.',
 887         optional => 1,
 888     },
 889     werror => {
 890         type => 'string',
 891         enum => [qw(enospc ignore report stop)],
 892         description => 'Write error action.',
 893         optional => 1,
 894     },
 895     aio => {
 896         type => 'string',
 897         enum => [qw(native threads)],
 898         description => 'AIO type to use.',
 899         optional => 1,
 900     },
 901     discard => {
 902         type => 'string',
 903         enum => [qw(ignore on)],
 904         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 905         optional => 1,
 906     },
 907     detect_zeroes => {
 908         type => 'boolean',
 909         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 910         optional => 1,
 911     },
 912     serial => {
 913         type => 'string',
 914         format => 'urlencoded',
 915         format_description => 'serial',
 916         maxLength => 20*3, # *3 since it's %xx url enoded
 917         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
 918         optional => 1,
 919     },
 920     shared => {
 921         type => 'boolean',
 922         description => 'Mark this locally-managed volume as available on all nodes',
 923         verbose_description => "Mark this locally-managed volume as available on all nodes.\n\nWARNING: This option does not share the volume automatically, it assumes it is shared already!",
 924         optional => 1,
 925         default => 0,
 926     }
 927 );
 928
 929 my %iothread_fmt = ( iothread => {
 930         type => 'boolean',
 931         description => "Whether to use iothreads for this drive",
 932         optional => 1,
 933 });
 934
 935 my %model_fmt = (
 936     model => {
 937         type => 'string',
 938         format => 'urlencoded',
 939         format_description => 'model',
 940         maxLength => 40*3, # *3 since it's %xx url enoded
 941         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
 942         optional => 1,
 943     },
 944 );
 945
 946 my %queues_fmt = (
 947     queues => {
 948         type => 'integer',
 949         description => "Number of queues.",
 950         minimum => 2,
 951         optional => 1
 952     }
 953 );
 954
 955 my %scsiblock_fmt = (
 956     scsiblock => {
 957         type => 'boolean',
 958         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
 959         optional => 1,
 960         default => 0,
 961     },
 962 );
 963
 964 my $add_throttle_desc = sub {
 965     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
 966     my $d = {
 967         type => $type,
 968         format_description => $unit,
 969         description => "Maximum $what in $longunit.",
 970         optional => 1,
 971     };
 972     $d->{minimum} = $minimum if defined($minimum);
 973     $drivedesc_base{$key} = $d;
 974 };
 975 # throughput: (leaky bucket)
 976 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
 977 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
 978 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
 979 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
 980 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
 981 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
 982 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
 983 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
 984 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
 985
 986 # pools: (pool of IO before throttling starts taking effect)
 987 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
 988 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
 989 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
 990 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
 991 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
 992 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
 993
 994 # burst lengths
 995 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
 996 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
 997 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
 998 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
 999 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1000 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1001
1002 # legacy support
1003 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
1004 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
1005 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
1006 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
1007
1008 my $ide_fmt = {
1009     %drivedesc_base,
1010     %model_fmt,
1011 };
1012 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
1013
1014 my $idedesc = {
1015     optional => 1,
1016     type => 'string', format => $ide_fmt,
1017     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
1018 };
1019 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
1020
1021 my $scsi_fmt = {
1022     %drivedesc_base,
1023     %iothread_fmt,
1024     %queues_fmt,
1025     %scsiblock_fmt,
1026 };
1027 my $scsidesc = {
1028     optional => 1,
1029     type => 'string', format => $scsi_fmt,
1030     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
1031 };
1032 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
1033
1034 my $sata_fmt = {
1035     %drivedesc_base,
1036 };
1037 my $satadesc = {
1038     optional => 1,
1039     type => 'string', format => $sata_fmt,
1040     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
1041 };
1042 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
1043
1044 my $virtio_fmt = {
1045     %drivedesc_base,
1046     %iothread_fmt,
1047 };
1048 my $virtiodesc = {
1049     optional => 1,
1050     type => 'string', format => $virtio_fmt,
1051     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
1052 };
1053 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
1054
1055 my $alldrive_fmt = {
1056     %drivedesc_base,
1057     %iothread_fmt,
1058     %model_fmt,
1059     %queues_fmt,
1060     %scsiblock_fmt,
1061 };
1062
1063 my $efidisk_fmt = {
1064     volume => { alias => 'file' },
1065     file => {
1066         type => 'string',
1067         format => 'pve-volume-id-or-qm-path',
1068         default_key => 1,
1069         format_description => 'volume',
1070         description => "The drive's backing volume.",
1071     },
1072     format => get_standard_option('pve-qm-image-format'),
1073     size => {
1074         type => 'string',
1075         format => 'disk-size',
1076         format_description => 'DiskSize',
1077         description => "Disk size. This is purely informational and has no effect.",
1078         optional => 1,
1079     },
1080 };
1081
1082 my $efidisk_desc = {
1083     optional => 1,
1084     type => 'string', format => $efidisk_fmt,
1085     description => "Configure a Disk for storing EFI vars",
1086 };
1087
1088 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
1089
1090 my $usb_fmt = {
1091     host => {
1092         default_key => 1,
1093         type => 'string', format => 'pve-qm-usb-device',
1094         format_description => 'HOSTUSBDEVICE|spice',
1095         description => <<EODESCR,
1096 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1097
1098  'bus-port(.port)*' (decimal numbers) or
1099  'vendor_id:product_id' (hexadeciaml numbers) or
1100  'spice'
1101
1102 You can use the 'lsusb -t' command to list existing usb devices.
1103
1104 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1105
1106 The value 'spice' can be used to add a usb redirection devices for spice.
1107 EODESCR
1108     },
1109     usb3 => {
1110         optional => 1,
1111         type => 'boolean',
1112         description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
1113         default => 0,
1114     },
1115 };
1116
1117 my $usbdesc = {
1118     optional => 1,
1119     type => 'string', format => $usb_fmt,
1120     description => "Configure an USB device (n is 0 to 4).",
1121 };
1122 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1123
1124 # NOTE: the match-groups of this regex are used in parse_hostpci
1125 my $PCIRE = qr/([a-f0-9]{2}:[a-f0-9]{2})(?:\.([a-f0-9]))?/;
1126 my $hostpci_fmt = {
1127     host => {
1128         default_key => 1,
1129         type => 'string',
1130         pattern => qr/$PCIRE(;$PCIRE)*/,
1131         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1132         description => <<EODESCR,
1133 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1134 of PCI virtual functions of the host. HOSTPCIID syntax is:
1135
1136 'bus:dev.func' (hexadecimal numbers)
1137
1138 You can us the 'lspci' command to list existing PCI devices.
1139 EODESCR
1140     },
1141     rombar => {
1142         type => 'boolean',
1143         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1144         optional => 1,
1145         default => 1,
1146     },
1147     romfile => {
1148         type => 'string',
1149         pattern => '[^,;]+',
1150         format_description => 'string',
1151         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1152         optional => 1,
1153     },
1154     pcie => {
1155         type => 'boolean',
1156         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1157         optional => 1,
1158         default => 0,
1159     },
1160     'x-vga' => {
1161         type => 'boolean',
1162         description =>  "Enable vfio-vga device support.",
1163         optional => 1,
1164         default => 0,
1165     },
1166 };
1167 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1168
1169 my $hostpcidesc = {
1170         optional => 1,
1171         type => 'string', format => 'pve-qm-hostpci',
1172         description => "Map host PCI devices into guest.",
1173         verbose_description =>  <<EODESCR,
1174 Map host PCI devices into guest.
1175
1176 NOTE: This option allows direct access to host hardware. So it is no longer
1177 possible to migrate such machines - use with special care.
1178
1179 CAUTION: Experimental! User reported problems with this option.
1180 EODESCR
1181 };
1182 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1183
1184 my $serialdesc = {
1185         optional => 1,
1186         type => 'string',
1187         pattern => '(/dev/.+|socket)',
1188         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1189         verbose_description =>  <<EODESCR,
1190 Create a serial device inside the VM (n is 0 to 3), and pass through a
1191 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1192 host side (use 'qm terminal' to open a terminal connection).
1193
1194 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1195
1196 CAUTION: Experimental! User reported problems with this option.
1197 EODESCR
1198 };
1199
1200 my $paralleldesc= {
1201         optional => 1,
1202         type => 'string',
1203         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1204         description =>  "Map host parallel devices (n is 0 to 2).",
1205         verbose_description =>  <<EODESCR,
1206 Map host parallel devices (n is 0 to 2).
1207
1208 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1209
1210 CAUTION: Experimental! User reported problems with this option.
1211 EODESCR
1212 };
1213
1214 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1215     $confdesc->{"parallel$i"} = $paralleldesc;
1216 }
1217
1218 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1219     $confdesc->{"serial$i"} = $serialdesc;
1220 }
1221
1222 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1223     $confdesc->{"hostpci$i"} = $hostpcidesc;
1224 }
1225
1226 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1227     $drivename_hash->{"ide$i"} = 1;
1228     $confdesc->{"ide$i"} = $idedesc;
1229 }
1230
1231 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1232     $drivename_hash->{"sata$i"} = 1;
1233     $confdesc->{"sata$i"} = $satadesc;
1234 }
1235
1236 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1237     $drivename_hash->{"scsi$i"} = 1;
1238     $confdesc->{"scsi$i"} = $scsidesc ;
1239 }
1240
1241 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1242     $drivename_hash->{"virtio$i"} = 1;
1243     $confdesc->{"virtio$i"} = $virtiodesc;
1244 }
1245
1246 $drivename_hash->{efidisk0} = 1;
1247 $confdesc->{efidisk0} = $efidisk_desc;
1248
1249 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1250     $confdesc->{"usb$i"} = $usbdesc;
1251 }
1252
1253 my $unuseddesc = {
1254     optional => 1,
1255     type => 'string', format => 'pve-volume-id',
1256     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1257 };
1258
1259 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1260     $confdesc->{"unused$i"} = $unuseddesc;
1261 }
1262
1263 my $kvm_api_version = 0;
1264
1265 sub kvm_version {
1266
1267     return $kvm_api_version if $kvm_api_version;
1268
1269     my $fh = IO::File->new("</dev/kvm") ||
1270         return 0;
1271
1272     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
1273         $kvm_api_version = $v;
1274     }
1275
1276     $fh->close();
1277
1278     return  $kvm_api_version;
1279 }
1280
1281 my $kvm_user_version;
1282
1283 sub kvm_user_version {
1284
1285     return $kvm_user_version if $kvm_user_version;
1286
1287     $kvm_user_version = 'unknown';
1288
1289     my $code = sub {
1290         my $line = shift;
1291         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1292             $kvm_user_version = $2;
1293         }
1294     };
1295
1296     eval { run_command("kvm -version", outfunc => $code); };
1297     warn $@ if $@;
1298
1299     return $kvm_user_version;
1300
1301 }
1302
1303 my $kernel_has_vhost_net = -c '/dev/vhost-net';
1304
1305 sub valid_drive_names {
1306     # order is important - used to autoselect boot disk
1307     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1308             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1309             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1310             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1311             'efidisk0');
1312 }
1313
1314 sub is_valid_drivename {
1315     my $dev = shift;
1316
1317     return defined($drivename_hash->{$dev});
1318 }
1319
1320 sub option_exists {
1321     my $key = shift;
1322     return defined($confdesc->{$key});
1323 }
1324
1325 sub nic_models {
1326     return $nic_model_list;
1327 }
1328
1329 sub os_list_description {
1330
1331     return {
1332         other => 'Other',
1333         wxp => 'Windows XP',
1334         w2k => 'Windows 2000',
1335         w2k3 =>, 'Windows 2003',
1336         w2k8 => 'Windows 2008',
1337         wvista => 'Windows Vista',
1338         win7 => 'Windows 7',
1339         win8 => 'Windows 8/2012',
1340         win10 => 'Windows 10/2016',
1341         l24 => 'Linux 2.4',
1342         l26 => 'Linux 2.6',
1343     };
1344 }
1345
1346 my $cdrom_path;
1347
1348 sub get_cdrom_path {
1349
1350     return  $cdrom_path if $cdrom_path;
1351
1352     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1353     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1354     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1355 }
1356
1357 sub get_iso_path {
1358     my ($storecfg, $vmid, $cdrom) = @_;
1359
1360     if ($cdrom eq 'cdrom') {
1361         return get_cdrom_path();
1362     } elsif ($cdrom eq 'none') {
1363         return '';
1364     } elsif ($cdrom =~ m|^/|) {
1365         return $cdrom;
1366     } else {
1367         return PVE::Storage::path($storecfg, $cdrom);
1368     }
1369 }
1370
1371 # try to convert old style file names to volume IDs
1372 sub filename_to_volume_id {
1373     my ($vmid, $file, $media) = @_;
1374
1375      if (!($file eq 'none' || $file eq 'cdrom' ||
1376           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1377
1378         return undef if $file =~ m|/|;
1379
1380         if ($media && $media eq 'cdrom') {
1381             $file = "local:iso/$file";
1382         } else {
1383             $file = "local:$vmid/$file";
1384         }
1385     }
1386
1387     return $file;
1388 }
1389
1390 sub verify_media_type {
1391     my ($opt, $vtype, $media) = @_;
1392
1393     return if !$media;
1394
1395     my $etype;
1396     if ($media eq 'disk') {
1397         $etype = 'images';
1398     } elsif ($media eq 'cdrom') {
1399         $etype = 'iso';
1400     } else {
1401         die "internal error";
1402     }
1403
1404     return if ($vtype eq $etype);
1405
1406     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1407 }
1408
1409 sub cleanup_drive_path {
1410     my ($opt, $storecfg, $drive) = @_;
1411
1412     # try to convert filesystem paths to volume IDs
1413
1414     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1415         ($drive->{file} !~ m|^/dev/.+|) &&
1416         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1417         ($drive->{file} !~ m/^\d+$/)) {
1418         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1419         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1420         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1421         verify_media_type($opt, $vtype, $drive->{media});
1422         $drive->{file} = $volid;
1423     }
1424
1425     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1426 }
1427
1428 sub parse_hotplug_features {
1429     my ($data) = @_;
1430
1431     my $res = {};
1432
1433     return $res if $data eq '0';
1434
1435     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1436
1437     foreach my $feature (PVE::Tools::split_list($data)) {
1438         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1439             $res->{$1} = 1;
1440         } else {
1441             die "invalid hotplug feature '$feature'\n";
1442         }
1443     }
1444     return $res;
1445 }
1446
1447 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1448 sub pve_verify_hotplug_features {
1449     my ($value, $noerr) = @_;
1450
1451     return $value if parse_hotplug_features($value);
1452
1453     return undef if $noerr;
1454
1455     die "unable to parse hotplug option\n";
1456 }
1457
1458 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1459 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1460 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1461 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1462 #        [,iothread=on][,serial=serial][,model=model]
1463
1464 sub parse_drive {
1465     my ($key, $data) = @_;
1466
1467     my ($interface, $index);
1468
1469     if ($key =~ m/^([^\d]+)(\d+)$/) {
1470         $interface = $1;
1471         $index = $2;
1472     } else {
1473         return undef;
1474     }
1475
1476     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1477                                      : $confdesc->{$key}->{format};
1478     if (!$desc) {
1479         warn "invalid drive key: $key\n";
1480         return undef;
1481     }
1482     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1483     return undef if !$res;
1484     $res->{interface} = $interface;
1485     $res->{index} = $index;
1486
1487     my $error = 0;
1488     foreach my $opt (qw(bps bps_rd bps_wr)) {
1489         if (my $bps = defined(delete $res->{$opt})) {
1490             if (defined($res->{"m$opt"})) {
1491                 warn "both $opt and m$opt specified\n";
1492                 ++$error;
1493                 next;
1494             }
1495             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1496         }
1497     }
1498
1499     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1500     for my $requirement (
1501         [mbps_max => 'mbps'],
1502         [mbps_rd_max => 'mbps_rd'],
1503         [mbps_wr_max => 'mbps_wr'],
1504         [miops_max => 'miops'],
1505         [miops_rd_max => 'miops_rd'],
1506         [miops_wr_max => 'miops_wr'],
1507         [bps_max_length => 'mbps_max'],
1508         [bps_rd_max_length => 'mbps_rd_max'],
1509         [bps_wr_max_length => 'mbps_wr_max'],
1510         [iops_max_length => 'iops_max'],
1511         [iops_rd_max_length => 'iops_rd_max'],
1512         [iops_wr_max_length => 'iops_wr_max']) {
1513         my ($option, $requires) = @$requirement;
1514         if ($res->{$option} && !$res->{$requires}) {
1515             warn "$option requires $requires\n";
1516             ++$error;
1517         }
1518     }
1519
1520     return undef if $error;
1521
1522     return undef if $res->{mbps_rd} && $res->{mbps};
1523     return undef if $res->{mbps_wr} && $res->{mbps};
1524     return undef if $res->{iops_rd} && $res->{iops};
1525     return undef if $res->{iops_wr} && $res->{iops};
1526
1527     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1528         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1529         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1530         return undef if $res->{interface} eq 'virtio';
1531     }
1532
1533     if (my $size = $res->{size}) {
1534         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1535     }
1536
1537     return $res;
1538 }
1539
1540 sub print_drive {
1541     my ($vmid, $drive) = @_;
1542     my $data = { %$drive };
1543     delete $data->{$_} for qw(index interface);
1544     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1545 }
1546
1547 sub scsi_inquiry {
1548     my($fh, $noerr) = @_;
1549
1550     my $SG_IO = 0x2285;
1551     my $SG_GET_VERSION_NUM = 0x2282;
1552
1553     my $versionbuf = "\x00" x 8;
1554     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1555     if (!$ret) {
1556         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1557         return undef;
1558     }
1559     my $version = unpack("I", $versionbuf);
1560     if ($version < 30000) {
1561         die "scsi generic interface too old\n"  if !$noerr;
1562         return undef;
1563     }
1564
1565     my $buf = "\x00" x 36;
1566     my $sensebuf = "\x00" x 8;
1567     my $cmd = pack("C x3 C x1", 0x12, 36);
1568
1569     # see /usr/include/scsi/sg.h
1570     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1571
1572     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1573                       length($sensebuf), 0, length($buf), $buf,
1574                       $cmd, $sensebuf, 6000);
1575
1576     $ret = ioctl($fh, $SG_IO, $packet);
1577     if (!$ret) {
1578         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1579         return undef;
1580     }
1581
1582     my @res = unpack($sg_io_hdr_t, $packet);
1583     if ($res[17] || $res[18]) {
1584         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1585         return undef;
1586     }
1587
1588     my $res = {};
1589     (my $byte0, my $byte1, $res->{vendor},
1590      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1591
1592     $res->{removable} = $byte1 & 128 ? 1 : 0;
1593     $res->{type} = $byte0 & 31;
1594
1595     return $res;
1596 }
1597
1598 sub path_is_scsi {
1599     my ($path) = @_;
1600
1601     my $fh = IO::File->new("+<$path") || return undef;
1602     my $res = scsi_inquiry($fh, 1);
1603     close($fh);
1604
1605     return $res;
1606 }
1607
1608 sub machine_type_is_q35 {
1609     my ($conf) = @_;
1610
1611     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1612 }
1613
1614 sub print_tabletdevice_full {
1615     my ($conf) = @_;
1616
1617     my $q35 = machine_type_is_q35($conf);
1618
1619     # we use uhci for old VMs because tablet driver was buggy in older qemu
1620     my $usbbus = $q35 ? "ehci" : "uhci";
1621
1622     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1623 }
1624
1625 sub print_drivedevice_full {
1626     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1627
1628     my $device = '';
1629     my $maxdev = 0;
1630
1631     if ($drive->{interface} eq 'virtio') {
1632         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1633         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1634         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1635     } elsif ($drive->{interface} eq 'scsi') {
1636
1637         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1638         my $unit = $drive->{index} % $maxdev;
1639         my $devicetype = 'hd';
1640         my $path = '';
1641         if (drive_is_cdrom($drive)) {
1642             $devicetype = 'cd';
1643         } else {
1644             if ($drive->{file} =~ m|^/|) {
1645                 $path = $drive->{file};
1646                 if (my $info = path_is_scsi($path)) {
1647                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1648                         $devicetype = 'block';
1649                     } elsif ($info->{type} == 1) { # tape
1650                         $devicetype = 'generic';
1651                     }
1652                 }
1653             } else {
1654                  $path = PVE::Storage::path($storecfg, $drive->{file});
1655             }
1656
1657             if($path =~ m/^iscsi\:\/\//){
1658                 $devicetype = 'generic';
1659             }
1660         }
1661
1662         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1663             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1664         } else {
1665             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1666         }
1667
1668     } elsif ($drive->{interface} eq 'ide'){
1669         $maxdev = 2;
1670         my $controller = int($drive->{index} / $maxdev);
1671         my $unit = $drive->{index} % $maxdev;
1672         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1673
1674         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1675         if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
1676             $model = URI::Escape::uri_unescape($model);
1677             $device .= ",model=$model";
1678         }
1679     } elsif ($drive->{interface} eq 'sata'){
1680         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1681         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1682         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1683     } elsif ($drive->{interface} eq 'usb') {
1684         die "implement me";
1685         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1686     } else {
1687         die "unsupported interface type";
1688     }
1689
1690     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1691
1692     return $device;
1693 }
1694
1695 sub get_initiator_name {
1696     my $initiator;
1697
1698     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1699     while (defined(my $line = <$fh>)) {
1700         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1701         $initiator = $1;
1702         last;
1703     }
1704     $fh->close();
1705
1706     return $initiator;
1707 }
1708
1709 sub print_drive_full {
1710     my ($storecfg, $vmid, $drive) = @_;
1711
1712     my $path;
1713     my $volid = $drive->{file};
1714     my $format;
1715
1716     if (drive_is_cdrom($drive)) {
1717         $path = get_iso_path($storecfg, $vmid, $volid);
1718     } else {
1719         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1720         if ($storeid) {
1721             $path = PVE::Storage::path($storecfg, $volid);
1722             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1723             $format = qemu_img_format($scfg, $volname);
1724         } else {
1725             $path = $volid;
1726             $format = "raw";
1727         }
1728    }
1729
1730     my $opts = '';
1731     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1732     foreach my $o (@qemu_drive_options) {
1733         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1734     }
1735
1736     # snapshot only accepts on|off
1737     if (defined($drive->{snapshot})) {
1738         my $v = $drive->{snapshot} ? 'on' : 'off';
1739         $opts .= ",snapshot=$v";
1740     }
1741
1742     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1743         my ($dir, $qmpname) = @$type;
1744         if (my $v = $drive->{"mbps$dir"}) {
1745             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1746         }
1747         if (my $v = $drive->{"mbps${dir}_max"}) {
1748             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1749         }
1750         if (my $v = $drive->{"bps${dir}_max_length"}) {
1751             $opts .= ",throttling.bps$qmpname-max-length=$v";
1752         }
1753         if (my $v = $drive->{"iops${dir}"}) {
1754             $opts .= ",throttling.iops$qmpname=$v";
1755         }
1756         if (my $v = $drive->{"iops${dir}_max"}) {
1757             $opts .= ",throttling.iops$qmpname-max=$v";
1758         }
1759         if (my $v = $drive->{"iops${dir}_max_length"}) {
1760             $opts .= ",throttling.iops$qmpname-max-length=$v";
1761         }
1762     }
1763
1764     if (my $serial = $drive->{serial}) {
1765         $serial = URI::Escape::uri_unescape($serial);
1766         $opts .= ",serial=$serial";
1767     }
1768
1769     $opts .= ",format=$format" if $format && !$drive->{format};
1770
1771     my $cache_direct = 0;
1772
1773     if (my $cache = $drive->{cache}) {
1774         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1775     } elsif (!drive_is_cdrom($drive)) {
1776         $opts .= ",cache=none";
1777         $cache_direct = 1;
1778     }
1779
1780     # aio native works only with O_DIRECT
1781     if (!$drive->{aio}) {
1782         if($cache_direct) {
1783             $opts .= ",aio=native";
1784         } else {
1785             $opts .= ",aio=threads";
1786         }
1787     }
1788
1789     if (!drive_is_cdrom($drive)) {
1790         my $detectzeroes;
1791         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1792             $detectzeroes = 'off';
1793         } elsif ($drive->{discard}) {
1794             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1795         } else {
1796             # This used to be our default with discard not being specified:
1797             $detectzeroes = 'on';
1798         }
1799         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1800     }
1801
1802     my $pathinfo = $path ? "file=$path," : '';
1803
1804     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1805 }
1806
1807 sub print_netdevice_full {
1808     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1809
1810     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1811
1812     my $device = $net->{model};
1813     if ($net->{model} eq 'virtio') {
1814          $device = 'virtio-net-pci';
1815      };
1816
1817     my $pciaddr = print_pci_addr("$netid", $bridges);
1818     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1819     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1820         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1821         my $vectors = $net->{queues} * 2 + 2;
1822         $tmpstr .= ",vectors=$vectors,mq=on";
1823     }
1824     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1825
1826     if ($use_old_bios_files) {
1827         my $romfile;
1828         if ($device eq 'virtio-net-pci') {
1829             $romfile = 'pxe-virtio.rom';
1830         } elsif ($device eq 'e1000') {
1831             $romfile = 'pxe-e1000.rom';
1832         } elsif ($device eq 'ne2k') {
1833             $romfile = 'pxe-ne2k_pci.rom';
1834         } elsif ($device eq 'pcnet') {
1835             $romfile = 'pxe-pcnet.rom';
1836         } elsif ($device eq 'rtl8139') {
1837             $romfile = 'pxe-rtl8139.rom';
1838         }
1839         $tmpstr .= ",romfile=$romfile" if $romfile;
1840     }
1841
1842     return $tmpstr;
1843 }
1844
1845 sub print_netdev_full {
1846     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1847
1848     my $i = '';
1849     if ($netid =~ m/^net(\d+)$/) {
1850         $i = int($1);
1851     }
1852
1853     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1854
1855     my $ifname = "tap${vmid}i$i";
1856
1857     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1858     die "interface name '$ifname' is too long (max 15 character)\n"
1859         if length($ifname) >= 16;
1860
1861     my $vhostparam = '';
1862     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1863
1864     my $vmname = $conf->{name} || "vm$vmid";
1865
1866     my $netdev = "";
1867     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1868
1869     if ($net->{bridge}) {
1870         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1871     } else {
1872         $netdev = "type=user,id=$netid,hostname=$vmname";
1873     }
1874
1875     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1876
1877     return $netdev;
1878 }
1879
1880
1881 sub print_cpu_device {
1882     my ($conf, $id) = @_;
1883
1884     my $kvm = $conf->{kvm} // 1;
1885     my $cpu = $kvm ? "kvm64" : "qemu64";
1886     if (my $cputype = $conf->{cpu}) {
1887         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
1888             or die "Cannot parse cpu description: $cputype\n";
1889         $cpu = $cpuconf->{cputype};
1890     }
1891
1892     my $cores = $conf->{cores} || 1;
1893
1894     my $current_core = ($id - 1) % $cores;
1895     my $current_socket = int(($id - 1 - $current_core)/$cores);
1896
1897     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
1898 }
1899
1900 sub drive_is_cloudinit {
1901     my ($drive) = @_;
1902     return $drive->{file} =~ m@[:/]vm-\d+-cloudinit(?:\.$QEMU_FORMAT_RE)?$@;
1903 }
1904
1905 sub drive_is_cdrom {
1906     my ($drive, $exclude_cloudinit) = @_;
1907
1908     return 0 if $exclude_cloudinit && drive_is_cloudinit($drive);
1909
1910     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1911
1912 }
1913
1914 sub parse_number_sets {
1915     my ($set) = @_;
1916     my $res = [];
1917     foreach my $part (split(/;/, $set)) {
1918         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1919             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1920             push @$res, [ $1, $2 ];
1921         } else {
1922             die "invalid range: $part\n";
1923         }
1924     }
1925     return $res;
1926 }
1927
1928 sub parse_numa {
1929     my ($data) = @_;
1930
1931     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
1932     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1933     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1934     return $res;
1935 }
1936
1937 sub parse_hostpci {
1938     my ($value) = @_;
1939
1940     return undef if !$value;
1941
1942     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
1943
1944     my @idlist = split(/;/, $res->{host});
1945     delete $res->{host};
1946     foreach my $id (@idlist) {
1947         if ($id =~ /^$PCIRE$/) {
1948             if (defined($2)) {
1949                 push @{$res->{pciid}}, { id => $1, function => $2 };
1950             } else {
1951                 my $pcidevices = lspci($1);
1952                 $res->{pciid} = $pcidevices->{$1};
1953             }
1954         } else {
1955             # should have been caught by parse_property_string already
1956             die "failed to parse PCI id: $id\n";
1957         }
1958     }
1959     return $res;
1960 }
1961
1962 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1963 sub parse_net {
1964     my ($data) = @_;
1965
1966     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
1967     if ($@) {
1968         warn $@;
1969         return undef;
1970     }
1971     if (!defined($res->{macaddr})) {
1972         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1973         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1974     }
1975     return $res;
1976 }
1977
1978 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1979 sub parse_ipconfig {
1980     my ($data) = @_;
1981
1982     my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
1983     if ($@) {
1984         warn $@;
1985         return undef;
1986     }
1987
1988     if ($res->{gw} && !$res->{ip}) {
1989         warn 'gateway specified without specifying an IP address';
1990         return undef;
1991     }
1992     if ($res->{gw6} && !$res->{ip6}) {
1993         warn 'IPv6 gateway specified without specifying an IPv6 address';
1994         return undef;
1995     }
1996     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1997         warn 'gateway specified together with DHCP';
1998         return undef;
1999     }
2000     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
2001         # gw6 + auto/dhcp
2002         warn "IPv6 gateway specified together with $res->{ip6} address";
2003         return undef;
2004     }
2005
2006     if (!$res->{ip} && !$res->{ip6}) {
2007         return { ip => 'dhcp', ip6 => 'dhcp' };
2008     }
2009
2010     return $res;
2011 }
2012
2013 sub print_net {
2014     my $net = shift;
2015
2016     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2017 }
2018
2019 sub add_random_macs {
2020     my ($settings) = @_;
2021
2022     foreach my $opt (keys %$settings) {
2023         next if $opt !~ m/^net(\d+)$/;
2024         my $net = parse_net($settings->{$opt});
2025         next if !$net;
2026         $settings->{$opt} = print_net($net);
2027     }
2028 }
2029
2030 sub vm_is_volid_owner {
2031     my ($storecfg, $vmid, $volid) = @_;
2032
2033     if ($volid !~  m|^/|) {
2034         my ($path, $owner);
2035         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2036         if ($owner && ($owner == $vmid)) {
2037             return 1;
2038         }
2039     }
2040
2041     return undef;
2042 }
2043
2044 sub split_flagged_list {
2045     my $text = shift || '';
2046     $text =~ s/[,;]/ /g;
2047     $text =~ s/^\s+//;
2048     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
2049 }
2050
2051 sub join_flagged_list {
2052     my ($how, $lst) = @_;
2053     join $how, map { $lst->{$_} . $_ } keys %$lst;
2054 }
2055
2056 sub vmconfig_delete_pending_option {
2057     my ($conf, $key, $force) = @_;
2058
2059     delete $conf->{pending}->{$key};
2060     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2061     $pending_delete_hash->{$key} = $force ? '!' : '';
2062     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2063 }
2064
2065 sub vmconfig_undelete_pending_option {
2066     my ($conf, $key) = @_;
2067
2068     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2069     delete $pending_delete_hash->{$key};
2070
2071     if (%$pending_delete_hash) {
2072         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2073     } else {
2074         delete $conf->{pending}->{delete};
2075     }
2076 }
2077
2078 sub vmconfig_register_unused_drive {
2079     my ($storecfg, $vmid, $conf, $drive) = @_;
2080
2081     if (drive_is_cloudinit($drive)) {
2082         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2083         warn $@ if $@;
2084     } elsif (!drive_is_cdrom($drive)) {
2085         my $volid = $drive->{file};
2086         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2087             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2088         }
2089     }
2090 }
2091
2092 sub vmconfig_cleanup_pending {
2093     my ($conf) = @_;
2094
2095     # remove pending changes when nothing changed
2096     my $changes;
2097     foreach my $opt (keys %{$conf->{pending}}) {
2098         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
2099             $changes = 1;
2100             delete $conf->{pending}->{$opt};
2101         }
2102     }
2103
2104     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
2105     my $pending_delete_hash = {};
2106     while (my ($opt, $force) = each %$current_delete_hash) {
2107         if (defined($conf->{$opt})) {
2108             $pending_delete_hash->{$opt} = $force;
2109         } else {
2110             $changes = 1;
2111         }
2112     }
2113
2114     if (%$pending_delete_hash) {
2115         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2116     } else {
2117         delete $conf->{pending}->{delete};
2118     }
2119
2120     return $changes;
2121 }
2122
2123 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
2124 my $smbios1_fmt = {
2125     uuid => {
2126         type => 'string',
2127         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2128         format_description => 'UUID',
2129         description => "Set SMBIOS1 UUID.",
2130         optional => 1,
2131     },
2132     version => {
2133         type => 'string',
2134         pattern => '\S+',
2135         format_description => 'string',
2136         description => "Set SMBIOS1 version.",
2137         optional => 1,
2138     },
2139     serial => {
2140         type => 'string',
2141         pattern => '\S+',
2142         format_description => 'string',
2143         description => "Set SMBIOS1 serial number.",
2144         optional => 1,
2145     },
2146     manufacturer => {
2147         type => 'string',
2148         pattern => '\S+',
2149         format_description => 'string',
2150         description => "Set SMBIOS1 manufacturer.",
2151         optional => 1,
2152     },
2153     product => {
2154         type => 'string',
2155         pattern => '\S+',
2156         format_description => 'string',
2157         description => "Set SMBIOS1 product ID.",
2158         optional => 1,
2159     },
2160     sku => {
2161         type => 'string',
2162         pattern => '\S+',
2163         format_description => 'string',
2164         description => "Set SMBIOS1 SKU string.",
2165         optional => 1,
2166     },
2167     family => {
2168         type => 'string',
2169         pattern => '\S+',
2170         format_description => 'string',
2171         description => "Set SMBIOS1 family string.",
2172         optional => 1,
2173     },
2174 };
2175
2176 sub parse_smbios1 {
2177     my ($data) = @_;
2178
2179     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
2180     warn $@ if $@;
2181     return $res;
2182 }
2183
2184 sub print_smbios1 {
2185     my ($smbios1) = @_;
2186     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2187 }
2188
2189 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2190
2191 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2192 sub verify_bootdisk {
2193     my ($value, $noerr) = @_;
2194
2195     return $value if is_valid_drivename($value);
2196
2197     return undef if $noerr;
2198
2199     die "invalid boot disk '$value'\n";
2200 }
2201
2202 sub parse_watchdog {
2203     my ($value) = @_;
2204
2205     return undef if !$value;
2206
2207     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2208     warn $@ if $@;
2209     return $res;
2210 }
2211
2212 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2213 sub verify_usb_device {
2214     my ($value, $noerr) = @_;
2215
2216     return $value if parse_usb_device($value);
2217
2218     return undef if $noerr;
2219
2220     die "unable to parse usb device\n";
2221 }
2222
2223 # add JSON properties for create and set function
2224 sub json_config_properties {
2225     my $prop = shift;
2226
2227     foreach my $opt (keys %$confdesc) {
2228         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
2229         $prop->{$opt} = $confdesc->{$opt};
2230     }
2231
2232     return $prop;
2233 }
2234
2235 # return copy of $confdesc_cloudinit to generate documentation
2236 sub cloudinit_config_properties {
2237
2238     return dclone($confdesc_cloudinit);
2239 }
2240
2241 sub check_type {
2242     my ($key, $value) = @_;
2243
2244     die "unknown setting '$key'\n" if !$confdesc->{$key};
2245
2246     my $type = $confdesc->{$key}->{type};
2247
2248     if (!defined($value)) {
2249         die "got undefined value\n";
2250     }
2251
2252     if ($value =~ m/[\n\r]/) {
2253         die "property contains a line feed\n";
2254     }
2255
2256     if ($type eq 'boolean') {
2257         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2258         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2259         die "type check ('boolean') failed - got '$value'\n";
2260     } elsif ($type eq 'integer') {
2261         return int($1) if $value =~ m/^(\d+)$/;
2262         die "type check ('integer') failed - got '$value'\n";
2263     } elsif ($type eq 'number') {
2264         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2265         die "type check ('number') failed - got '$value'\n";
2266     } elsif ($type eq 'string') {
2267         if (my $fmt = $confdesc->{$key}->{format}) {
2268             PVE::JSONSchema::check_format($fmt, $value);
2269             return $value;
2270         }
2271         $value =~ s/^\"(.*)\"$/$1/;
2272         return $value;
2273     } else {
2274         die "internal error"
2275     }
2276 }
2277
2278 sub check_iommu_support{
2279     #fixme : need to check IOMMU support
2280     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
2281
2282     my $iommu=1;
2283     return $iommu;
2284
2285 }
2286
2287 sub touch_config {
2288     my ($vmid) = @_;
2289
2290     my $conf = PVE::QemuConfig->config_file($vmid);
2291     utime undef, undef, $conf;
2292 }
2293
2294 sub destroy_vm {
2295     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2296
2297     my $conffile = PVE::QemuConfig->config_file($vmid);
2298
2299     my $conf = PVE::QemuConfig->load_config($vmid);
2300
2301     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2302
2303     if ($conf->{template}) {
2304         # check if any base image is still used by a linked clone
2305         foreach_drive($conf, sub {
2306                 my ($ds, $drive) = @_;
2307
2308                 return if drive_is_cdrom($drive);
2309
2310                 my $volid = $drive->{file};
2311
2312                 return if !$volid || $volid =~ m|^/|;
2313
2314                 die "base volume '$volid' is still in use by linked cloned\n"
2315                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2316
2317         });
2318     }
2319
2320     # only remove disks owned by this VM
2321     foreach_drive($conf, sub {
2322         my ($ds, $drive) = @_;
2323
2324         return if drive_is_cdrom($drive, 1);
2325
2326         my $volid = $drive->{file};
2327
2328         return if !$volid || $volid =~ m|^/|;
2329
2330         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2331         return if !$path || !$owner || ($owner != $vmid);
2332
2333         eval {
2334             PVE::Storage::vdisk_free($storecfg, $volid);
2335         };
2336         warn "Could not remove disk '$volid', check manually: $@" if $@;
2337
2338     });
2339
2340     if ($keep_empty_config) {
2341         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2342     } else {
2343         unlink $conffile;
2344     }
2345
2346     # also remove unused disk
2347     eval {
2348         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2349
2350         eval {
2351             PVE::Storage::foreach_volid($dl, sub {
2352                 my ($volid, $sid, $volname, $d) = @_;
2353                 PVE::Storage::vdisk_free($storecfg, $volid);
2354             });
2355         };
2356         warn $@ if $@;
2357
2358     };
2359     warn $@ if $@;
2360 }
2361
2362 sub parse_vm_config {
2363     my ($filename, $raw) = @_;
2364
2365     return undef if !defined($raw);
2366
2367     my $res = {
2368         digest => Digest::SHA::sha1_hex($raw),
2369         snapshots => {},
2370         pending => {},
2371     };
2372
2373     $filename =~ m|/qemu-server/(\d+)\.conf$|
2374         || die "got strange filename '$filename'";
2375
2376     my $vmid = $1;
2377
2378     my $conf = $res;
2379     my $descr;
2380     my $section = '';
2381
2382     my @lines = split(/\n/, $raw);
2383     foreach my $line (@lines) {
2384         next if $line =~ m/^\s*$/;
2385
2386         if ($line =~ m/^\[PENDING\]\s*$/i) {
2387             $section = 'pending';
2388             if (defined($descr)) {
2389                 $descr =~ s/\s+$//;
2390                 $conf->{description} = $descr;
2391             }
2392             $descr = undef;
2393             $conf = $res->{$section} = {};
2394             next;
2395
2396         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2397             $section = $1;
2398             if (defined($descr)) {
2399                 $descr =~ s/\s+$//;
2400                 $conf->{description} = $descr;
2401             }
2402             $descr = undef;
2403             $conf = $res->{snapshots}->{$section} = {};
2404             next;
2405         }
2406
2407         if ($line =~ m/^\#(.*)\s*$/) {
2408             $descr = '' if !defined($descr);
2409             $descr .= PVE::Tools::decode_text($1) . "\n";
2410             next;
2411         }
2412
2413         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2414             $descr = '' if !defined($descr);
2415             $descr .= PVE::Tools::decode_text($2);
2416         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2417             $conf->{snapstate} = $1;
2418         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2419             my $key = $1;
2420             my $value = $2;
2421             $conf->{$key} = $value;
2422         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2423             my $value = $1;
2424             if ($section eq 'pending') {
2425                 $conf->{delete} = $value; # we parse this later
2426             } else {
2427                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2428             }
2429         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2430             my $key = $1;
2431             my $value = $2;
2432             eval { $value = check_type($key, $value); };
2433             if ($@) {
2434                 warn "vm $vmid - unable to parse value of '$key' - $@";
2435             } else {
2436                 $key = 'ide2' if $key eq 'cdrom';
2437                 my $fmt = $confdesc->{$key}->{format};
2438                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2439                     my $v = parse_drive($key, $value);
2440                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2441                         $v->{file} = $volid;
2442                         $value = print_drive($vmid, $v);
2443                     } else {
2444                         warn "vm $vmid - unable to parse value of '$key'\n";
2445                         next;
2446                     }
2447                 }
2448
2449                 $conf->{$key} = $value;
2450             }
2451         }
2452     }
2453
2454     if (defined($descr)) {
2455         $descr =~ s/\s+$//;
2456         $conf->{description} = $descr;
2457     }
2458     delete $res->{snapstate}; # just to be sure
2459
2460     return $res;
2461 }
2462
2463 sub write_vm_config {
2464     my ($filename, $conf) = @_;
2465
2466     delete $conf->{snapstate}; # just to be sure
2467
2468     if ($conf->{cdrom}) {
2469         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2470         $conf->{ide2} = $conf->{cdrom};
2471         delete $conf->{cdrom};
2472     }
2473
2474     # we do not use 'smp' any longer
2475     if ($conf->{sockets}) {
2476         delete $conf->{smp};
2477     } elsif ($conf->{smp}) {
2478         $conf->{sockets} = $conf->{smp};
2479         delete $conf->{cores};
2480         delete $conf->{smp};
2481     }
2482
2483     my $used_volids = {};
2484
2485     my $cleanup_config = sub {
2486         my ($cref, $pending, $snapname) = @_;
2487
2488         foreach my $key (keys %$cref) {
2489             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2490                 $key eq 'snapstate' || $key eq 'pending';
2491             my $value = $cref->{$key};
2492             if ($key eq 'delete') {
2493                 die "propertry 'delete' is only allowed in [PENDING]\n"
2494                     if !$pending;
2495                 # fixme: check syntax?
2496                 next;
2497             }
2498             eval { $value = check_type($key, $value); };
2499             die "unable to parse value of '$key' - $@" if $@;
2500
2501             $cref->{$key} = $value;
2502
2503             if (!$snapname && is_valid_drivename($key)) {
2504                 my $drive = parse_drive($key, $value);
2505                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2506             }
2507         }
2508     };
2509
2510     &$cleanup_config($conf);
2511
2512     &$cleanup_config($conf->{pending}, 1);
2513
2514     foreach my $snapname (keys %{$conf->{snapshots}}) {
2515         die "internal error" if $snapname eq 'pending';
2516         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2517     }
2518
2519     # remove 'unusedX' settings if we re-add a volume
2520     foreach my $key (keys %$conf) {
2521         my $value = $conf->{$key};
2522         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2523             delete $conf->{$key};
2524         }
2525     }
2526
2527     my $generate_raw_config = sub {
2528         my ($conf, $pending) = @_;
2529
2530         my $raw = '';
2531
2532         # add description as comment to top of file
2533         if (defined(my $descr = $conf->{description})) {
2534             if ($descr) {
2535                 foreach my $cl (split(/\n/, $descr)) {
2536                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2537                 }
2538             } else {
2539                 $raw .= "#\n" if $pending;
2540             }
2541         }
2542
2543         foreach my $key (sort keys %$conf) {
2544             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2545             $raw .= "$key: $conf->{$key}\n";
2546         }
2547         return $raw;
2548     };
2549
2550     my $raw = &$generate_raw_config($conf);
2551
2552     if (scalar(keys %{$conf->{pending}})){
2553         $raw .= "\n[PENDING]\n";
2554         $raw .= &$generate_raw_config($conf->{pending}, 1);
2555     }
2556
2557     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2558         $raw .= "\n[$snapname]\n";
2559         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2560     }
2561
2562     return $raw;
2563 }
2564
2565 sub load_defaults {
2566
2567     my $res = {};
2568
2569     # we use static defaults from our JSON schema configuration
2570     foreach my $key (keys %$confdesc) {
2571         if (defined(my $default = $confdesc->{$key}->{default})) {
2572             $res->{$key} = $default;
2573         }
2574     }
2575
2576     return $res;
2577 }
2578
2579 sub config_list {
2580     my $vmlist = PVE::Cluster::get_vmlist();
2581     my $res = {};
2582     return $res if !$vmlist || !$vmlist->{ids};
2583     my $ids = $vmlist->{ids};
2584
2585     foreach my $vmid (keys %$ids) {
2586         my $d = $ids->{$vmid};
2587         next if !$d->{node} || $d->{node} ne $nodename;
2588         next if !$d->{type} || $d->{type} ne 'qemu';
2589         $res->{$vmid}->{exists} = 1;
2590     }
2591     return $res;
2592 }
2593
2594 # test if VM uses local resources (to prevent migration)
2595 sub check_local_resources {
2596     my ($conf, $noerr) = @_;
2597
2598     my $loc_res = 0;
2599
2600     $loc_res = 1 if $conf->{hostusb}; # old syntax
2601     $loc_res = 1 if $conf->{hostpci}; # old syntax
2602
2603     foreach my $k (keys %$conf) {
2604         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2605         # sockets are safe: they will recreated be on the target side post-migrate
2606         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2607         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2608     }
2609
2610     die "VM uses local resources\n" if $loc_res && !$noerr;
2611
2612     return $loc_res;
2613 }
2614
2615 # check if used storages are available on all nodes (use by migrate)
2616 sub check_storage_availability {
2617     my ($storecfg, $conf, $node) = @_;
2618
2619     foreach_drive($conf, sub {
2620         my ($ds, $drive) = @_;
2621
2622         my $volid = $drive->{file};
2623         return if !$volid;
2624
2625         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2626         return if !$sid;
2627
2628         # check if storage is available on both nodes
2629         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2630         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2631    });
2632 }
2633
2634 # list nodes where all VM images are available (used by has_feature API)
2635 sub shared_nodes {
2636     my ($conf, $storecfg) = @_;
2637
2638     my $nodelist = PVE::Cluster::get_nodelist();
2639     my $nodehash = { map { $_ => 1 } @$nodelist };
2640     my $nodename = PVE::INotify::nodename();
2641
2642     foreach_drive($conf, sub {
2643         my ($ds, $drive) = @_;
2644
2645         my $volid = $drive->{file};
2646         return if !$volid;
2647
2648         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2649         if ($storeid) {
2650             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2651             if ($scfg->{disable}) {
2652                 $nodehash = {};
2653             } elsif (my $avail = $scfg->{nodes}) {
2654                 foreach my $node (keys %$nodehash) {
2655                     delete $nodehash->{$node} if !$avail->{$node};
2656                 }
2657             } elsif (!$scfg->{shared}) {
2658                 foreach my $node (keys %$nodehash) {
2659                     delete $nodehash->{$node} if $node ne $nodename
2660                 }
2661             }
2662         }
2663     });
2664
2665     return $nodehash
2666 }
2667
2668 sub check_cmdline {
2669     my ($pidfile, $pid) = @_;
2670
2671     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2672     if (defined($fh)) {
2673         my $line = <$fh>;
2674         $fh->close;
2675         return undef if !$line;
2676         my @param = split(/\0/, $line);
2677
2678         my $cmd = $param[0];
2679         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2680
2681         for (my $i = 0; $i < scalar (@param); $i++) {
2682             my $p = $param[$i];
2683             next if !$p;
2684             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2685                 my $p = $param[$i+1];
2686                 return 1 if $p && ($p eq $pidfile);
2687                 return undef;
2688             }
2689         }
2690     }
2691     return undef;
2692 }
2693
2694 sub check_running {
2695     my ($vmid, $nocheck, $node) = @_;
2696
2697     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2698
2699     die "unable to find configuration file for VM $vmid - no such machine\n"
2700         if !$nocheck && ! -f $filename;
2701
2702     my $pidfile = pidfile_name($vmid);
2703
2704     if (my $fd = IO::File->new("<$pidfile")) {
2705         my $st = stat($fd);
2706         my $line = <$fd>;
2707         close($fd);
2708
2709         my $mtime = $st->mtime;
2710         if ($mtime > time()) {
2711             warn "file '$filename' modified in future\n";
2712         }
2713
2714         if ($line =~ m/^(\d+)$/) {
2715             my $pid = $1;
2716             if (check_cmdline($pidfile, $pid)) {
2717                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2718                     return $pid;
2719                 }
2720             }
2721         }
2722     }
2723
2724     return undef;
2725 }
2726
2727 sub vzlist {
2728
2729     my $vzlist = config_list();
2730
2731     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2732
2733     while (defined(my $de = $fd->read)) {
2734         next if $de !~ m/^(\d+)\.pid$/;
2735         my $vmid = $1;
2736         next if !defined($vzlist->{$vmid});
2737         if (my $pid = check_running($vmid)) {
2738             $vzlist->{$vmid}->{pid} = $pid;
2739         }
2740     }
2741
2742     return $vzlist;
2743 }
2744
2745 sub disksize {
2746     my ($storecfg, $conf) = @_;
2747
2748     my $bootdisk = $conf->{bootdisk};
2749     return undef if !$bootdisk;
2750     return undef if !is_valid_drivename($bootdisk);
2751
2752     return undef if !$conf->{$bootdisk};
2753
2754     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2755     return undef if !defined($drive);
2756
2757     return undef if drive_is_cdrom($drive);
2758
2759     my $volid = $drive->{file};
2760     return undef if !$volid;
2761
2762     return $drive->{size};
2763 }
2764
2765 my $last_proc_pid_stat;
2766
2767 # get VM status information
2768 # This must be fast and should not block ($full == false)
2769 # We only query KVM using QMP if $full == true (this can be slow)
2770 sub vmstatus {
2771     my ($opt_vmid, $full) = @_;
2772
2773     my $res = {};
2774
2775     my $storecfg = PVE::Storage::config();
2776
2777     my $list = vzlist();
2778     my $defaults = load_defaults();
2779
2780     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2781
2782     my $cpucount = $cpuinfo->{cpus} || 1;
2783
2784     foreach my $vmid (keys %$list) {
2785         next if $opt_vmid && ($vmid ne $opt_vmid);
2786
2787         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
2788         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2789
2790         my $d = {};
2791         $d->{pid} = $list->{$vmid}->{pid};
2792
2793         # fixme: better status?
2794         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2795
2796         my $size = disksize($storecfg, $conf);
2797         if (defined($size)) {
2798             $d->{disk} = 0; # no info available
2799             $d->{maxdisk} = $size;
2800         } else {
2801             $d->{disk} = 0;
2802             $d->{maxdisk} = 0;
2803         }
2804
2805         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2806             * ($conf->{cores} || $defaults->{cores});
2807         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2808         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2809
2810         $d->{name} = $conf->{name} || "VM $vmid";
2811         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2812             : $defaults->{memory}*(1024*1024);
2813
2814         if ($conf->{balloon}) {
2815             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2816             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2817                 : $defaults->{shares};
2818         }
2819
2820         $d->{uptime} = 0;
2821         $d->{cpu} = 0;
2822         $d->{mem} = 0;
2823
2824         $d->{netout} = 0;
2825         $d->{netin} = 0;
2826
2827         $d->{diskread} = 0;
2828         $d->{diskwrite} = 0;
2829
2830         $d->{template} = PVE::QemuConfig->is_template($conf);
2831
2832         $d->{serial} = 1 if conf_has_serial($conf);
2833
2834         $res->{$vmid} = $d;
2835     }
2836
2837     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2838     foreach my $dev (keys %$netdev) {
2839         next if $dev !~ m/^tap([1-9]\d*)i/;
2840         my $vmid = $1;
2841         my $d = $res->{$vmid};
2842         next if !$d;
2843
2844         $d->{netout} += $netdev->{$dev}->{receive};
2845         $d->{netin} += $netdev->{$dev}->{transmit};
2846
2847         if ($full) {
2848             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2849             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2850         }
2851
2852     }
2853
2854     my $ctime = gettimeofday;
2855
2856     foreach my $vmid (keys %$list) {
2857
2858         my $d = $res->{$vmid};
2859         my $pid = $d->{pid};
2860         next if !$pid;
2861
2862         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2863         next if !$pstat; # not running
2864
2865         my $used = $pstat->{utime} + $pstat->{stime};
2866
2867         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2868
2869         if ($pstat->{vsize}) {
2870             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2871         }
2872
2873         my $old = $last_proc_pid_stat->{$pid};
2874         if (!$old) {
2875             $last_proc_pid_stat->{$pid} = {
2876                 time => $ctime,
2877                 used => $used,
2878                 cpu => 0,
2879             };
2880             next;
2881         }
2882
2883         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2884
2885         if ($dtime > 1000) {
2886             my $dutime = $used -  $old->{used};
2887
2888             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2889             $last_proc_pid_stat->{$pid} = {
2890                 time => $ctime,
2891                 used => $used,
2892                 cpu => $d->{cpu},
2893             };
2894         } else {
2895             $d->{cpu} = $old->{cpu};
2896         }
2897     }
2898
2899     return $res if !$full;
2900
2901     my $qmpclient = PVE::QMPClient->new();
2902
2903     my $ballooncb = sub {
2904         my ($vmid, $resp) = @_;
2905
2906         my $info = $resp->{'return'};
2907         return if !$info->{max_mem};
2908
2909         my $d = $res->{$vmid};
2910
2911         # use memory assigned to VM
2912         $d->{maxmem} = $info->{max_mem};
2913         $d->{balloon} = $info->{actual};
2914
2915         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2916             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2917             $d->{freemem} = $info->{free_mem};
2918         }
2919
2920         $d->{ballooninfo} = $info;
2921     };
2922
2923     my $blockstatscb = sub {
2924         my ($vmid, $resp) = @_;
2925         my $data = $resp->{'return'} || [];
2926         my $totalrdbytes = 0;
2927         my $totalwrbytes = 0;
2928
2929         for my $blockstat (@$data) {
2930             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2931             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2932
2933             $blockstat->{device} =~ s/drive-//;
2934             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2935         }
2936         $res->{$vmid}->{diskread} = $totalrdbytes;
2937         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2938     };
2939
2940     my $statuscb = sub {
2941         my ($vmid, $resp) = @_;
2942
2943         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2944         # this fails if ballon driver is not loaded, so this must be
2945         # the last commnand (following command are aborted if this fails).
2946         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2947
2948         my $status = 'unknown';
2949         if (!defined($status = $resp->{'return'}->{status})) {
2950             warn "unable to get VM status\n";
2951             return;
2952         }
2953
2954         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2955     };
2956
2957     foreach my $vmid (keys %$list) {
2958         next if $opt_vmid && ($vmid ne $opt_vmid);
2959         next if !$res->{$vmid}->{pid}; # not running
2960         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2961     }
2962
2963     $qmpclient->queue_execute(undef, 2);
2964
2965     foreach my $vmid (keys %$list) {
2966         next if $opt_vmid && ($vmid ne $opt_vmid);
2967         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2968     }
2969
2970     return $res;
2971 }
2972
2973 sub foreach_drive {
2974     my ($conf, $func, @param) = @_;
2975
2976     foreach my $ds (valid_drive_names()) {
2977         next if !defined($conf->{$ds});
2978
2979         my $drive = parse_drive($ds, $conf->{$ds});
2980         next if !$drive;
2981
2982         &$func($ds, $drive, @param);
2983     }
2984 }
2985
2986 sub foreach_volid {
2987     my ($conf, $func, @param) = @_;
2988
2989     my $volhash = {};
2990
2991     my $test_volid = sub {
2992         my ($volid, $is_cdrom, $replicate, $shared, $snapname) = @_;
2993
2994         return if !$volid;
2995
2996         $volhash->{$volid}->{cdrom} //= 1;
2997         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
2998
2999         $volhash->{$volid}->{replicate} //= 0;
3000         $volhash->{$volid}->{replicate} = 1 if $replicate;
3001
3002         $volhash->{$volid}->{shared} //= 0;
3003         $volhash->{$volid}->{shared} = 1 if $shared;
3004
3005         $volhash->{$volid}->{referenced_in_config} //= 0;
3006         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
3007
3008         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
3009             if defined($snapname);
3010     };
3011
3012     foreach_drive($conf, sub {
3013         my ($ds, $drive) = @_;
3014         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, undef);
3015     });
3016
3017     foreach my $snapname (keys %{$conf->{snapshots}}) {
3018         my $snap = $conf->{snapshots}->{$snapname};
3019         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
3020         foreach_drive($snap, sub {
3021             my ($ds, $drive) = @_;
3022             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, $snapname);
3023         });
3024     }
3025
3026     foreach my $volid (keys %$volhash) {
3027         &$func($volid, $volhash->{$volid}, @param);
3028     }
3029 }
3030
3031 sub conf_has_serial {
3032     my ($conf) = @_;
3033
3034     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3035         if ($conf->{"serial$i"}) {
3036             return 1;
3037         }
3038     }
3039
3040     return 0;
3041 }
3042
3043 sub vga_conf_has_spice {
3044     my ($vga) = @_;
3045
3046     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
3047
3048     return $1 || 1;
3049 }
3050
3051 sub config_to_command {
3052     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
3053
3054     my $cmd = [];
3055     my $globalFlags = [];
3056     my $machineFlags = [];
3057     my $rtcFlags = [];
3058     my $cpuFlags = [];
3059     my $devices = [];
3060     my $pciaddr = '';
3061     my $bridges = {};
3062     my $kvmver = kvm_user_version();
3063     my $vernum = 0; # unknown
3064     my $ostype = $conf->{ostype};
3065     my $winversion = windows_version($ostype);
3066     my $kvm = $conf->{kvm} // 1;
3067
3068     die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n" if (!$cpuinfo->{hvm} && $kvm);
3069
3070     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
3071         $vernum = $1*1000000+$2*1000;
3072     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
3073         $vernum = $1*1000000+$2*1000+$3;
3074     }
3075
3076     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
3077
3078     my $have_ovz = -f '/proc/vz/vestat';
3079
3080     my $q35 = machine_type_is_q35($conf);
3081     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3082     my $machine_type = $forcemachine || $conf->{machine};
3083     my $use_old_bios_files = undef;
3084     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3085
3086     my $cpuunits = defined($conf->{cpuunits}) ?
3087             $conf->{cpuunits} : $defaults->{cpuunits};
3088
3089     push @$cmd, '/usr/bin/kvm';
3090
3091     push @$cmd, '-id', $vmid;
3092
3093     my $vmname = $conf->{name} || "vm$vmid";
3094
3095     push @$cmd, '-name', $vmname;
3096
3097     my $use_virtio = 0;
3098
3099     my $qmpsocket = qmp_socket($vmid);
3100     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3101     push @$cmd, '-mon', "chardev=qmp,mode=control";
3102
3103
3104     push @$cmd, '-pidfile' , pidfile_name($vmid);
3105
3106     push @$cmd, '-daemonize';
3107
3108     if ($conf->{smbios1}) {
3109         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3110     }
3111
3112     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3113         die "uefi base image not found\n" if ! -f $OVMF_CODE;
3114
3115         my $path;
3116         my $format;
3117         if (my $efidisk = $conf->{efidisk0}) {
3118             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
3119             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3120             $format = $d->{format};
3121             if ($storeid) {
3122                 $path = PVE::Storage::path($storecfg, $d->{file});
3123                 if (!defined($format)) {
3124                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3125                     $format = qemu_img_format($scfg, $volname);
3126                 }
3127             } else {
3128                 $path = $d->{file};
3129                 die "efidisk format must be specified\n"
3130                     if !defined($format);
3131             }
3132         } else {
3133             warn "no efidisk configured! Using temporary efivars disk.\n";
3134             $path = "/tmp/$vmid-ovmf.fd";
3135             PVE::Tools::file_copy($OVMF_VARS, $path, -s $OVMF_VARS);
3136             $format = 'raw';
3137         }
3138
3139         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$OVMF_CODE";
3140         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
3141     }
3142
3143
3144     # add usb controllers
3145     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $q35, $usbdesc->{format}, $MAX_USB_DEVICES);
3146     push @$devices, @usbcontrollers if @usbcontrollers;
3147     my $vga = $conf->{vga};
3148
3149     my $qxlnum = vga_conf_has_spice($vga);
3150     $vga = 'qxl' if $qxlnum;
3151
3152     if (!$vga) {
3153         if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 9)) {
3154             $vga = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3155         } else {
3156             $vga = ($winversion >= 6) ? 'std' : 'cirrus';
3157         }
3158     }
3159
3160     # enable absolute mouse coordinates (needed by vnc)
3161     my $tablet;
3162     if (defined($conf->{tablet})) {
3163         $tablet = $conf->{tablet};
3164     } else {
3165         $tablet = $defaults->{tablet};
3166         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3167         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3168     }
3169
3170     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
3171
3172     my $kvm_off = 0;
3173     my $gpu_passthrough;
3174
3175     # host pci devices
3176     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3177         my $d = parse_hostpci($conf->{"hostpci$i"});
3178         next if !$d;
3179
3180         my $pcie = $d->{pcie};
3181         if($pcie){
3182             die "q35 machine model is not enabled" if !$q35;
3183             $pciaddr = print_pcie_addr("hostpci$i");
3184         }else{
3185             $pciaddr = print_pci_addr("hostpci$i", $bridges);
3186         }
3187
3188         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
3189         my $romfile = $d->{romfile};
3190
3191         my $xvga = '';
3192         if ($d->{'x-vga'}) {
3193             $xvga = ',x-vga=on';
3194             $kvm_off = 1;
3195             $vga = 'none';
3196             $gpu_passthrough = 1;
3197
3198             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3199                 $xvga = "";
3200             }
3201         }
3202         my $pcidevices = $d->{pciid};
3203         my $multifunction = 1 if @$pcidevices > 1;
3204
3205         my $j=0;
3206         foreach my $pcidevice (@$pcidevices) {
3207
3208             my $id = "hostpci$i";
3209             $id .= ".$j" if $multifunction;
3210             my $addr = $pciaddr;
3211             $addr .= ".$j" if $multifunction;
3212             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
3213
3214             if($j == 0){
3215                 $devicestr .= "$rombar$xvga";
3216                 $devicestr .= ",multifunction=on" if $multifunction;
3217                 $devicestr .= ",romfile=/usr/share/kvm/$romfile" if $romfile;
3218             }
3219
3220             push @$devices, '-device', $devicestr;
3221             $j++;
3222         }
3223     }
3224
3225     # usb devices
3226     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES);
3227     push @$devices, @usbdevices if @usbdevices;
3228     # serial devices
3229     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3230         if (my $path = $conf->{"serial$i"}) {
3231             if ($path eq 'socket') {
3232                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3233                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3234                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3235             } else {
3236                 die "no such serial device\n" if ! -c $path;
3237                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3238                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3239             }
3240         }
3241     }
3242
3243     # parallel devices
3244     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3245         if (my $path = $conf->{"parallel$i"}) {
3246             die "no such parallel device\n" if ! -c $path;
3247             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3248             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3249             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3250         }
3251     }
3252
3253
3254     my $sockets = 1;
3255     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3256     $sockets = $conf->{sockets} if  $conf->{sockets};
3257
3258     my $cores = $conf->{cores} || 1;
3259
3260     my $maxcpus = $sockets * $cores;
3261
3262     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3263
3264     my $allowed_vcpus = $cpuinfo->{cpus};
3265
3266     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3267         if ($allowed_vcpus < $maxcpus);
3268
3269     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
3270
3271         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3272         for (my $i = 2; $i <= $vcpus; $i++)  {
3273             my $cpustr = print_cpu_device($conf,$i);
3274             push @$cmd, '-device', $cpustr;
3275         }
3276
3277     } else {
3278
3279         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3280     }
3281     push @$cmd, '-nodefaults';
3282
3283     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3284
3285     my $bootindex_hash = {};
3286     my $i = 1;
3287     foreach my $o (split(//, $bootorder)) {
3288         $bootindex_hash->{$o} = $i*100;
3289         $i++;
3290     }
3291
3292     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3293
3294     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3295
3296     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3297
3298     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
3299
3300     if ($vga && $vga !~ m/^serial\d+$/ && $vga ne 'none'){
3301         my $socket = vnc_socket($vmid);
3302         push @$cmd,  '-vnc', "unix:$socket,x509,password";
3303     } else {
3304         push @$cmd, '-nographic';
3305     }
3306
3307     # time drift fix
3308     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3309
3310     my $useLocaltime = $conf->{localtime};
3311
3312     if ($winversion >= 5) { # windows
3313         $useLocaltime = 1 if !defined($conf->{localtime});
3314
3315         # use time drift fix when acpi is enabled
3316         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3317             $tdf = 1 if !defined($conf->{tdf});
3318         }
3319     }
3320
3321     if ($winversion >= 6) {
3322         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3323         push @$cmd, '-no-hpet';
3324     }
3325
3326     push @$rtcFlags, 'driftfix=slew' if $tdf;
3327
3328     if (!$kvm) {
3329         push @$machineFlags, 'accel=tcg';
3330     }
3331
3332     if ($machine_type) {
3333         push @$machineFlags, "type=${machine_type}";
3334     }
3335
3336     if ($conf->{startdate}) {
3337         push @$rtcFlags, "base=$conf->{startdate}";
3338     } elsif ($useLocaltime) {
3339         push @$rtcFlags, 'base=localtime';
3340     }
3341
3342     my $cpu = $kvm ? "kvm64" : "qemu64";
3343     if (my $cputype = $conf->{cpu}) {
3344         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3345             or die "Cannot parse cpu description: $cputype\n";
3346         $cpu = $cpuconf->{cputype};
3347         $kvm_off = 1 if $cpuconf->{hidden};
3348
3349         if (defined(my $flags = $cpuconf->{flags})) {
3350             push @$cpuFlags, split(";", $flags);
3351         }
3352     }
3353
3354     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
3355
3356     push @$cpuFlags , '-x2apic'
3357         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3358
3359     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3360
3361     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3362
3363     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3364
3365         push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3366         push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3367     }
3368
3369     add_hyperv_enlightenments($cpuFlags, $winversion, $machine_type, $kvmver, $conf->{bios}, $gpu_passthrough) if $kvm;
3370
3371     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm;
3372
3373     push @$cpuFlags, 'kvm=off' if $kvm_off;
3374
3375     my $cpu_vendor = $cpu_vendor_list->{$cpu} ||
3376         die "internal error"; # should not happen
3377
3378     push @$cpuFlags, "vendor=${cpu_vendor}"
3379         if $cpu_vendor ne 'default';
3380
3381     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3382
3383     push @$cmd, '-cpu', $cpu;
3384
3385     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3386
3387     push @$cmd, '-S' if $conf->{freeze};
3388
3389     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3390
3391     # enable sound
3392     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3393     #push @$cmd, '-soundhw', 'es1370';
3394     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3395
3396     if($conf->{agent}) {
3397         my $qgasocket = qmp_socket($vmid, 1);
3398         my $pciaddr = print_pci_addr("qga0", $bridges);
3399         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3400         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3401         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3402     }
3403
3404     my $spice_port;
3405
3406     if ($qxlnum) {
3407         if ($qxlnum > 1) {
3408             if ($winversion){
3409                 for(my $i = 1; $i < $qxlnum; $i++){
3410                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3411                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3412                 }
3413             } else {
3414                 # assume other OS works like Linux
3415                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3416                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3417             }
3418         }
3419
3420         my $pciaddr = print_pci_addr("spice", $bridges);
3421
3422         my $nodename = PVE::INotify::nodename();
3423         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3424         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3425         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3426         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3427         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3428
3429         push @$devices, '-spice', "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3430
3431         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3432         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3433         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3434     }
3435
3436     # enable balloon by default, unless explicitly disabled
3437     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3438         $pciaddr = print_pci_addr("balloon0", $bridges);
3439         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3440     }
3441
3442     if ($conf->{watchdog}) {
3443         my $wdopts = parse_watchdog($conf->{watchdog});
3444         $pciaddr = print_pci_addr("watchdog", $bridges);
3445         my $watchdog = $wdopts->{model} || 'i6300esb';
3446         push @$devices, '-device', "$watchdog$pciaddr";
3447         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3448     }
3449
3450     my $vollist = [];
3451     my $scsicontroller = {};
3452     my $ahcicontroller = {};
3453     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3454
3455     # Add iscsi initiator name if available
3456     if (my $initiator = get_initiator_name()) {
3457         push @$devices, '-iscsi', "initiator-name=$initiator";
3458     }
3459
3460     foreach_drive($conf, sub {
3461         my ($ds, $drive) = @_;
3462
3463         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3464             push @$vollist, $drive->{file};
3465         }
3466
3467         # ignore efidisk here, already added in bios/fw handling code above
3468         return if $drive->{interface} eq 'efidisk';
3469
3470         $use_virtio = 1 if $ds =~ m/^virtio/;
3471
3472         if (drive_is_cdrom ($drive)) {
3473             if ($bootindex_hash->{d}) {
3474                 $drive->{bootindex} = $bootindex_hash->{d};
3475                 $bootindex_hash->{d} += 1;
3476             }
3477         } else {
3478             if ($bootindex_hash->{c}) {
3479                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3480                 $bootindex_hash->{c} += 1;
3481             }
3482         }
3483
3484         if($drive->{interface} eq 'virtio'){
3485            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3486         }
3487
3488         if ($drive->{interface} eq 'scsi') {
3489
3490             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3491
3492             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3493             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3494
3495             my $iothread = '';
3496             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3497                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3498                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3499             } elsif ($drive->{iothread}) {
3500                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3501             }
3502
3503             my $queues = '';
3504             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3505                 $queues = ",num_queues=$drive->{queues}";
3506             }
3507
3508             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3509             $scsicontroller->{$controller}=1;
3510         }
3511
3512         if ($drive->{interface} eq 'sata') {
3513            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3514            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3515            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3516            $ahcicontroller->{$controller}=1;
3517         }
3518
3519         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3520         push @$devices, '-drive',$drive_cmd;
3521         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3522     });
3523
3524     for (my $i = 0; $i < $MAX_NETS; $i++) {
3525          next if !$conf->{"net$i"};
3526          my $d = parse_net($conf->{"net$i"});
3527          next if !$d;
3528
3529          $use_virtio = 1 if $d->{model} eq 'virtio';
3530
3531          if ($bootindex_hash->{n}) {
3532             $d->{bootindex} = $bootindex_hash->{n};
3533             $bootindex_hash->{n} += 1;
3534          }
3535
3536          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3537          push @$devices, '-netdev', $netdevfull;
3538
3539          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3540          push @$devices, '-device', $netdevicefull;
3541     }
3542
3543     if (!$q35) {
3544         # add pci bridges
3545         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3546            $bridges->{1} = 1;
3547            $bridges->{2} = 1;
3548         }
3549
3550         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3551
3552         while (my ($k, $v) = each %$bridges) {
3553             $pciaddr = print_pci_addr("pci.$k");
3554             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3555         }
3556     }
3557
3558     # add custom args
3559     if ($conf->{args}) {
3560         my $aa = PVE::Tools::split_args($conf->{args});
3561         push @$cmd, @$aa;
3562     }
3563
3564     push @$cmd, @$devices;
3565     push @$cmd, '-rtc', join(',', @$rtcFlags)
3566         if scalar(@$rtcFlags);
3567     push @$cmd, '-machine', join(',', @$machineFlags)
3568         if scalar(@$machineFlags);
3569     push @$cmd, '-global', join(',', @$globalFlags)
3570         if scalar(@$globalFlags);
3571
3572     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3573 }
3574
3575 sub vnc_socket {
3576     my ($vmid) = @_;
3577     return "${var_run_tmpdir}/$vmid.vnc";
3578 }
3579
3580 sub spice_port {
3581     my ($vmid) = @_;
3582
3583     my $res = vm_mon_cmd($vmid, 'query-spice');
3584
3585     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3586 }
3587
3588 sub qmp_socket {
3589     my ($vmid, $qga) = @_;
3590     my $sockettype = $qga ? 'qga' : 'qmp';
3591     return "${var_run_tmpdir}/$vmid.$sockettype";
3592 }
3593
3594 sub pidfile_name {
3595     my ($vmid) = @_;
3596     return "${var_run_tmpdir}/$vmid.pid";
3597 }
3598
3599 sub vm_devices_list {
3600     my ($vmid) = @_;
3601
3602     my $res = vm_mon_cmd($vmid, 'query-pci');
3603     my $devices = {};
3604     foreach my $pcibus (@$res) {
3605         foreach my $device (@{$pcibus->{devices}}) {
3606             next if !$device->{'qdev_id'};
3607             if ($device->{'pci_bridge'}) {
3608                 $devices->{$device->{'qdev_id'}} = 1;
3609                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3610                     next if !$bridge_device->{'qdev_id'};
3611                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3612                     $devices->{$device->{'qdev_id'}}++;
3613                 }
3614             } else {
3615                 $devices->{$device->{'qdev_id'}} = 1;
3616             }
3617         }
3618     }
3619
3620     my $resblock = vm_mon_cmd($vmid, 'query-block');
3621     foreach my $block (@$resblock) {
3622         if($block->{device} =~ m/^drive-(\S+)/){
3623                 $devices->{$1} = 1;
3624         }
3625     }
3626
3627     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3628     foreach my $mice (@$resmice) {
3629         if ($mice->{name} eq 'QEMU HID Tablet') {
3630             $devices->{tablet} = 1;
3631             last;
3632         }
3633     }
3634
3635     # for usb devices there is no query-usb
3636     # but we can iterate over the entries in
3637     # qom-list path=/machine/peripheral
3638     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3639     foreach my $per (@$resperipheral) {
3640         if ($per->{name} =~ m/^usb\d+$/) {
3641             $devices->{$per->{name}} = 1;
3642         }
3643     }
3644
3645     return $devices;
3646 }
3647
3648 sub vm_deviceplug {
3649     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3650
3651     my $q35 = machine_type_is_q35($conf);
3652
3653     my $devices_list = vm_devices_list($vmid);
3654     return 1 if defined($devices_list->{$deviceid});
3655
3656     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3657
3658     if ($deviceid eq 'tablet') {
3659
3660         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3661
3662     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3663
3664         die "usb hotplug currently not reliable\n";
3665         # since we can't reliably hot unplug all added usb devices
3666         # and usb passthrough disables live migration
3667         # we disable usb hotplugging for now
3668         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3669
3670     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3671
3672         qemu_iothread_add($vmid, $deviceid, $device);
3673
3674         qemu_driveadd($storecfg, $vmid, $device);
3675         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3676
3677         qemu_deviceadd($vmid, $devicefull);
3678         eval { qemu_deviceaddverify($vmid, $deviceid); };
3679         if (my $err = $@) {
3680             eval { qemu_drivedel($vmid, $deviceid); };
3681             warn $@ if $@;
3682             die $err;
3683         }
3684
3685     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3686
3687
3688         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3689         my $pciaddr = print_pci_addr($deviceid);
3690         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3691
3692         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3693
3694         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3695             qemu_iothread_add($vmid, $deviceid, $device);
3696             $devicefull .= ",iothread=iothread-$deviceid";
3697         }
3698
3699         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3700             $devicefull .= ",num_queues=$device->{queues}";
3701         }
3702
3703         qemu_deviceadd($vmid, $devicefull);
3704         qemu_deviceaddverify($vmid, $deviceid);
3705
3706     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3707
3708         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3709         qemu_driveadd($storecfg, $vmid, $device);
3710
3711         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3712         eval { qemu_deviceadd($vmid, $devicefull); };
3713         if (my $err = $@) {
3714             eval { qemu_drivedel($vmid, $deviceid); };
3715             warn $@ if $@;
3716             die $err;
3717         }
3718
3719     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3720
3721         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3722
3723         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
3724         my $use_old_bios_files = undef;
3725         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3726
3727         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3728         qemu_deviceadd($vmid, $netdevicefull);
3729         eval { qemu_deviceaddverify($vmid, $deviceid); };
3730         if (my $err = $@) {
3731             eval { qemu_netdevdel($vmid, $deviceid); };
3732             warn $@ if $@;
3733             die $err;
3734         }
3735
3736     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3737
3738         my $bridgeid = $2;
3739         my $pciaddr = print_pci_addr($deviceid);
3740         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3741
3742         qemu_deviceadd($vmid, $devicefull);
3743         qemu_deviceaddverify($vmid, $deviceid);
3744
3745     } else {
3746         die "can't hotplug device '$deviceid'\n";
3747     }
3748
3749     return 1;
3750 }
3751
3752 # fixme: this should raise exceptions on error!
3753 sub vm_deviceunplug {
3754     my ($vmid, $conf, $deviceid) = @_;
3755
3756     my $devices_list = vm_devices_list($vmid);
3757     return 1 if !defined($devices_list->{$deviceid});
3758
3759     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3760
3761     if ($deviceid eq 'tablet') {
3762
3763         qemu_devicedel($vmid, $deviceid);
3764
3765     } elsif ($deviceid =~ m/^usb\d+$/) {
3766
3767         die "usb hotplug currently not reliable\n";
3768         # when unplugging usb devices this way,
3769         # there may be remaining usb controllers/hubs
3770         # so we disable it for now
3771         qemu_devicedel($vmid, $deviceid);
3772         qemu_devicedelverify($vmid, $deviceid);
3773
3774     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3775
3776         qemu_devicedel($vmid, $deviceid);
3777         qemu_devicedelverify($vmid, $deviceid);
3778         qemu_drivedel($vmid, $deviceid);
3779         qemu_iothread_del($conf, $vmid, $deviceid);
3780
3781     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3782
3783         qemu_devicedel($vmid, $deviceid);
3784         qemu_devicedelverify($vmid, $deviceid);
3785         qemu_iothread_del($conf, $vmid, $deviceid);
3786
3787     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3788
3789         qemu_devicedel($vmid, $deviceid);
3790         qemu_drivedel($vmid, $deviceid);
3791         qemu_deletescsihw($conf, $vmid, $deviceid);
3792
3793     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3794
3795         qemu_devicedel($vmid, $deviceid);
3796         qemu_devicedelverify($vmid, $deviceid);
3797         qemu_netdevdel($vmid, $deviceid);
3798
3799     } else {
3800         die "can't unplug device '$deviceid'\n";
3801     }
3802
3803     return 1;
3804 }
3805
3806 sub qemu_deviceadd {
3807     my ($vmid, $devicefull) = @_;
3808
3809     $devicefull = "driver=".$devicefull;
3810     my %options =  split(/[=,]/, $devicefull);
3811
3812     vm_mon_cmd($vmid, "device_add" , %options);
3813 }
3814
3815 sub qemu_devicedel {
3816     my ($vmid, $deviceid) = @_;
3817
3818     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3819 }
3820
3821 sub qemu_iothread_add {
3822     my($vmid, $deviceid, $device) = @_;
3823
3824     if ($device->{iothread}) {
3825         my $iothreads = vm_iothreads_list($vmid);
3826         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3827     }
3828 }
3829
3830 sub qemu_iothread_del {
3831     my($conf, $vmid, $deviceid) = @_;
3832
3833     my $device = parse_drive($deviceid, $conf->{$deviceid});
3834     if ($device->{iothread}) {
3835         my $iothreads = vm_iothreads_list($vmid);
3836         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3837     }
3838 }
3839
3840 sub qemu_objectadd {
3841     my($vmid, $objectid, $qomtype) = @_;
3842
3843     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3844
3845     return 1;
3846 }
3847
3848 sub qemu_objectdel {
3849     my($vmid, $objectid) = @_;
3850
3851     vm_mon_cmd($vmid, "object-del", id => $objectid);
3852
3853     return 1;
3854 }
3855
3856 sub qemu_driveadd {
3857     my ($storecfg, $vmid, $device) = @_;
3858
3859     my $drive = print_drive_full($storecfg, $vmid, $device);
3860     $drive =~ s/\\/\\\\/g;
3861     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3862
3863     # If the command succeeds qemu prints: "OK"
3864     return 1 if $ret =~ m/OK/s;
3865
3866     die "adding drive failed: $ret\n";
3867 }
3868
3869 sub qemu_drivedel {
3870     my($vmid, $deviceid) = @_;
3871
3872     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3873     $ret =~ s/^\s+//;
3874
3875     return 1 if $ret eq "";
3876
3877     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3878     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3879
3880     die "deleting drive $deviceid failed : $ret\n";
3881 }
3882
3883 sub qemu_deviceaddverify {
3884     my ($vmid, $deviceid) = @_;
3885
3886     for (my $i = 0; $i <= 5; $i++) {
3887          my $devices_list = vm_devices_list($vmid);
3888          return 1 if defined($devices_list->{$deviceid});
3889          sleep 1;
3890     }
3891
3892     die "error on hotplug device '$deviceid'\n";
3893 }
3894
3895
3896 sub qemu_devicedelverify {
3897     my ($vmid, $deviceid) = @_;
3898
3899     # need to verify that the device is correctly removed as device_del
3900     # is async and empty return is not reliable
3901
3902     for (my $i = 0; $i <= 5; $i++) {
3903          my $devices_list = vm_devices_list($vmid);
3904          return 1 if !defined($devices_list->{$deviceid});
3905          sleep 1;
3906     }
3907
3908     die "error on hot-unplugging device '$deviceid'\n";
3909 }
3910
3911 sub qemu_findorcreatescsihw {
3912     my ($storecfg, $conf, $vmid, $device) = @_;
3913
3914     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3915
3916     my $scsihwid="$controller_prefix$controller";
3917     my $devices_list = vm_devices_list($vmid);
3918
3919     if(!defined($devices_list->{$scsihwid})) {
3920         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3921     }
3922
3923     return 1;
3924 }
3925
3926 sub qemu_deletescsihw {
3927     my ($conf, $vmid, $opt) = @_;
3928
3929     my $device = parse_drive($opt, $conf->{$opt});
3930
3931     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3932         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3933         return 1;
3934     }
3935
3936     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3937
3938     my $devices_list = vm_devices_list($vmid);
3939     foreach my $opt (keys %{$devices_list}) {
3940         if (PVE::QemuServer::is_valid_drivename($opt)) {
3941             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3942             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3943                 return 1;
3944             }
3945         }
3946     }
3947
3948     my $scsihwid="scsihw$controller";
3949
3950     vm_deviceunplug($vmid, $conf, $scsihwid);
3951
3952     return 1;
3953 }
3954
3955 sub qemu_add_pci_bridge {
3956     my ($storecfg, $conf, $vmid, $device) = @_;
3957
3958     my $bridges = {};
3959
3960     my $bridgeid;
3961
3962     print_pci_addr($device, $bridges);
3963
3964     while (my ($k, $v) = each %$bridges) {
3965         $bridgeid = $k;
3966     }
3967     return 1 if !defined($bridgeid) || $bridgeid < 1;
3968
3969     my $bridge = "pci.$bridgeid";
3970     my $devices_list = vm_devices_list($vmid);
3971
3972     if (!defined($devices_list->{$bridge})) {
3973         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3974     }
3975
3976     return 1;
3977 }
3978
3979 sub qemu_set_link_status {
3980     my ($vmid, $device, $up) = @_;
3981
3982     vm_mon_cmd($vmid, "set_link", name => $device,
3983                up => $up ? JSON::true : JSON::false);
3984 }
3985
3986 sub qemu_netdevadd {
3987     my ($vmid, $conf, $device, $deviceid) = @_;
3988
3989     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
3990     my %options =  split(/[=,]/, $netdev);
3991
3992     vm_mon_cmd($vmid, "netdev_add",  %options);
3993     return 1;
3994 }
3995
3996 sub qemu_netdevdel {
3997     my ($vmid, $deviceid) = @_;
3998
3999     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
4000 }
4001
4002 sub qemu_usb_hotplug {
4003     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
4004
4005     return if !$device;
4006
4007     # remove the old one first
4008     vm_deviceunplug($vmid, $conf, $deviceid);
4009
4010     # check if xhci controller is necessary and available
4011     if ($device->{usb3}) {
4012
4013         my $devicelist = vm_devices_list($vmid);
4014
4015         if (!$devicelist->{xhci}) {
4016             my $pciaddr = print_pci_addr("xhci");
4017             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4018         }
4019     }
4020     my $d = parse_usb_device($device->{host});
4021     $d->{usb3} = $device->{usb3};
4022
4023     # add the new one
4024     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d);
4025 }
4026
4027 sub qemu_cpu_hotplug {
4028     my ($vmid, $conf, $vcpus) = @_;
4029
4030     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
4031
4032     my $sockets = 1;
4033     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4034     $sockets = $conf->{sockets} if  $conf->{sockets};
4035     my $cores = $conf->{cores} || 1;
4036     my $maxcpus = $sockets * $cores;
4037
4038     $vcpus = $maxcpus if !$vcpus;
4039
4040     die "you can't add more vcpus than maxcpus\n"
4041         if $vcpus > $maxcpus;
4042
4043     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4044
4045     if ($vcpus < $currentvcpus) {
4046
4047         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4048
4049             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4050                 qemu_devicedel($vmid, "cpu$i");
4051                 my $retry = 0;
4052                 my $currentrunningvcpus = undef;
4053                 while (1) {
4054                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4055                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4056                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4057                     $retry++;
4058                     sleep 1;
4059                 }
4060                 #update conf after each succesfull cpu unplug
4061                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4062                 PVE::QemuConfig->write_config($vmid, $conf);
4063             }
4064         } else {
4065             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4066         }
4067
4068         return;
4069     }
4070
4071     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4072     die "vcpus in running vm does not match its configuration\n"
4073         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4074
4075     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4076
4077         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4078             my $cpustr = print_cpu_device($conf, $i);
4079             qemu_deviceadd($vmid, $cpustr);
4080
4081             my $retry = 0;
4082             my $currentrunningvcpus = undef;
4083             while (1) {
4084                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4085                 last if scalar(@{$currentrunningvcpus}) == $i;
4086                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4087                 sleep 1;
4088                 $retry++;
4089             }
4090             #update conf after each succesfull cpu hotplug
4091             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4092             PVE::QemuConfig->write_config($vmid, $conf);
4093         }
4094     } else {
4095
4096         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4097             vm_mon_cmd($vmid, "cpu-add", id => int($i));
4098         }
4099     }
4100 }
4101
4102 sub qemu_block_set_io_throttle {
4103     my ($vmid, $deviceid,
4104         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4105         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4106         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4107         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4108
4109     return if !check_running($vmid) ;
4110
4111     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4112         bps => int($bps),
4113         bps_rd => int($bps_rd),
4114         bps_wr => int($bps_wr),
4115         iops => int($iops),
4116         iops_rd => int($iops_rd),
4117         iops_wr => int($iops_wr),
4118         bps_max => int($bps_max),
4119         bps_rd_max => int($bps_rd_max),
4120         bps_wr_max => int($bps_wr_max),
4121         iops_max => int($iops_max),
4122         iops_rd_max => int($iops_rd_max),
4123         iops_wr_max => int($iops_wr_max),
4124         bps_max_length => int($bps_max_length),
4125         bps_rd_max_length => int($bps_rd_max_length),
4126         bps_wr_max_length => int($bps_wr_max_length),
4127         iops_max_length => int($iops_max_length),
4128         iops_rd_max_length => int($iops_rd_max_length),
4129         iops_wr_max_length => int($iops_wr_max_length),
4130     );
4131
4132 }
4133
4134 # old code, only used to shutdown old VM after update
4135 sub __read_avail {
4136     my ($fh, $timeout) = @_;
4137
4138     my $sel = new IO::Select;
4139     $sel->add($fh);
4140
4141     my $res = '';
4142     my $buf;
4143
4144     my @ready;
4145     while (scalar (@ready = $sel->can_read($timeout))) {
4146         my $count;
4147         if ($count = $fh->sysread($buf, 8192)) {
4148             if ($buf =~ /^(.*)\(qemu\) $/s) {
4149                 $res .= $1;
4150                 last;
4151             } else {
4152                 $res .= $buf;
4153             }
4154         } else {
4155             if (!defined($count)) {
4156                 die "$!\n";
4157             }
4158             last;
4159         }
4160     }
4161
4162     die "monitor read timeout\n" if !scalar(@ready);
4163
4164     return $res;
4165 }
4166
4167 sub qemu_block_resize {
4168     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4169
4170     my $running = check_running($vmid);
4171
4172     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4173
4174     return if !$running;
4175
4176     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4177
4178 }
4179
4180 sub qemu_volume_snapshot {
4181     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4182
4183     my $running = check_running($vmid);
4184
4185     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4186         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
4187     } else {
4188         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4189     }
4190 }
4191
4192 sub qemu_volume_snapshot_delete {
4193     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4194
4195     my $running = check_running($vmid);
4196
4197     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4198         vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
4199     } else {
4200         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4201     }
4202 }
4203
4204 sub set_migration_caps {
4205     my ($vmid) = @_;
4206
4207     my $cap_ref = [];
4208
4209     my $enabled_cap = {
4210         "auto-converge" => 1,
4211         "xbzrle" => 1,
4212         "x-rdma-pin-all" => 0,
4213         "zero-blocks" => 0,
4214         "compress" => 0
4215     };
4216
4217     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4218
4219     for my $supported_capability (@$supported_capabilities) {
4220         push @$cap_ref, {
4221             capability => $supported_capability->{capability},
4222             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4223         };
4224     }
4225
4226     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4227 }
4228
4229 my $fast_plug_option = {
4230     'lock' => 1,
4231     'name' => 1,
4232     'onboot' => 1,
4233     'shares' => 1,
4234     'startup' => 1,
4235     'description' => 1,
4236     'protection' => 1,
4237     'vmstatestorage' => 1,
4238 };
4239
4240 # hotplug changes in [PENDING]
4241 # $selection hash can be used to only apply specified options, for
4242 # example: { cores => 1 } (only apply changed 'cores')
4243 # $errors ref is used to return error messages
4244 sub vmconfig_hotplug_pending {
4245     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4246
4247     my $defaults = load_defaults();
4248
4249     # commit values which do not have any impact on running VM first
4250     # Note: those option cannot raise errors, we we do not care about
4251     # $selection and always apply them.
4252
4253     my $add_error = sub {
4254         my ($opt, $msg) = @_;
4255         $errors->{$opt} = "hotplug problem - $msg";
4256     };
4257
4258     my $changes = 0;
4259     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4260         if ($fast_plug_option->{$opt}) {
4261             $conf->{$opt} = $conf->{pending}->{$opt};
4262             delete $conf->{pending}->{$opt};
4263             $changes = 1;
4264         }
4265     }
4266
4267     if ($changes) {
4268         PVE::QemuConfig->write_config($vmid, $conf);
4269         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4270     }
4271
4272     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4273
4274     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4275     while (my ($opt, $force) = each %$pending_delete_hash) {
4276         next if $selection && !$selection->{$opt};
4277         eval {
4278             if ($opt eq 'hotplug') {
4279                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4280             } elsif ($opt eq 'tablet') {
4281                 die "skip\n" if !$hotplug_features->{usb};
4282                 if ($defaults->{tablet}) {
4283                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4284                 } else {
4285                     vm_deviceunplug($vmid, $conf, $opt);
4286                 }
4287             } elsif ($opt =~ m/^usb\d+/) {
4288                 die "skip\n";
4289                 # since we cannot reliably hot unplug usb devices
4290                 # we are disabling it
4291                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4292                 vm_deviceunplug($vmid, $conf, $opt);
4293             } elsif ($opt eq 'vcpus') {
4294                 die "skip\n" if !$hotplug_features->{cpu};
4295                 qemu_cpu_hotplug($vmid, $conf, undef);
4296             } elsif ($opt eq 'balloon') {
4297                 # enable balloon device is not hotpluggable
4298                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
4299             } elsif ($fast_plug_option->{$opt}) {
4300                 # do nothing
4301             } elsif ($opt =~ m/^net(\d+)$/) {
4302                 die "skip\n" if !$hotplug_features->{network};
4303                 vm_deviceunplug($vmid, $conf, $opt);
4304             } elsif (is_valid_drivename($opt)) {
4305                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4306                 vm_deviceunplug($vmid, $conf, $opt);
4307                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4308             } elsif ($opt =~ m/^memory$/) {
4309                 die "skip\n" if !$hotplug_features->{memory};
4310                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4311             } elsif ($opt eq 'cpuunits') {
4312                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4313             } elsif ($opt eq 'cpulimit') {
4314                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4315             } else {
4316                 die "skip\n";
4317             }
4318         };
4319         if (my $err = $@) {
4320             &$add_error($opt, $err) if $err ne "skip\n";
4321         } else {
4322             # save new config if hotplug was successful
4323             delete $conf->{$opt};
4324             vmconfig_undelete_pending_option($conf, $opt);
4325             PVE::QemuConfig->write_config($vmid, $conf);
4326             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4327         }
4328     }
4329
4330     my $apply_pending_cloudinit;
4331     $apply_pending_cloudinit = sub {
4332         my ($key, $value) = @_;
4333         $apply_pending_cloudinit = sub {}; # once is enough
4334
4335         my @cloudinit_opts = keys %$confdesc_cloudinit;
4336         foreach my $opt (keys %{$conf->{pending}}) {
4337             next if !grep { $_ eq $opt } @cloudinit_opts;
4338             $conf->{$opt} = delete $conf->{pending}->{$opt};
4339         }
4340
4341         my $new_conf = { %$conf };
4342         $new_conf->{$key} = $value;
4343         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4344     };
4345
4346     foreach my $opt (keys %{$conf->{pending}}) {
4347         next if $selection && !$selection->{$opt};
4348         my $value = $conf->{pending}->{$opt};
4349         eval {
4350             if ($opt eq 'hotplug') {
4351                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4352             } elsif ($opt eq 'tablet') {
4353                 die "skip\n" if !$hotplug_features->{usb};
4354                 if ($value == 1) {
4355                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4356                 } elsif ($value == 0) {
4357                     vm_deviceunplug($vmid, $conf, $opt);
4358                 }
4359             } elsif ($opt =~ m/^usb\d+$/) {
4360                 die "skip\n";
4361                 # since we cannot reliably hot unplug usb devices
4362                 # we are disabling it
4363                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4364                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4365                 die "skip\n" if !$d;
4366                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d);
4367             } elsif ($opt eq 'vcpus') {
4368                 die "skip\n" if !$hotplug_features->{cpu};
4369                 qemu_cpu_hotplug($vmid, $conf, $value);
4370             } elsif ($opt eq 'balloon') {
4371                 # enable/disable balloning device is not hotpluggable
4372                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4373                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4374                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4375
4376                 # allow manual ballooning if shares is set to zero
4377                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4378                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4379                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4380                 }
4381             } elsif ($opt =~ m/^net(\d+)$/) {
4382                 # some changes can be done without hotplug
4383                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4384                                     $vmid, $opt, $value);
4385             } elsif (is_valid_drivename($opt)) {
4386                 # some changes can be done without hotplug
4387                 my $drive = parse_drive($opt, $value);
4388                 if (drive_is_cloudinit($drive)) {
4389                     &$apply_pending_cloudinit($opt, $value);
4390                 }
4391                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4392                                      $vmid, $opt, $value, 1);
4393             } elsif ($opt =~ m/^memory$/) { #dimms
4394                 die "skip\n" if !$hotplug_features->{memory};
4395                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4396             } elsif ($opt eq 'cpuunits') {
4397                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4398             } elsif ($opt eq 'cpulimit') {
4399                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4400                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4401             } else {
4402                 die "skip\n";  # skip non-hot-pluggable options
4403             }
4404         };
4405         if (my $err = $@) {
4406             &$add_error($opt, $err) if $err ne "skip\n";
4407         } else {
4408             # save new config if hotplug was successful
4409             $conf->{$opt} = $value;
4410             delete $conf->{pending}->{$opt};
4411             PVE::QemuConfig->write_config($vmid, $conf);
4412             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4413         }
4414     }
4415 }
4416
4417 sub try_deallocate_drive {
4418     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4419
4420     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4421         my $volid = $drive->{file};
4422         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4423             my $sid = PVE::Storage::parse_volume_id($volid);
4424             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4425
4426             # check if the disk is really unused
4427             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4428                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4429             PVE::Storage::vdisk_free($storecfg, $volid);
4430             return 1;
4431         } else {
4432             # If vm is not owner of this disk remove from config
4433             return 1;
4434         }
4435     }
4436
4437     return undef;
4438 }
4439
4440 sub vmconfig_delete_or_detach_drive {
4441     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4442
4443     my $drive = parse_drive($opt, $conf->{$opt});
4444
4445     my $rpcenv = PVE::RPCEnvironment::get();
4446     my $authuser = $rpcenv->get_user();
4447
4448     if ($force) {
4449         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4450         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4451     } else {
4452         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4453     }
4454 }
4455
4456 sub vmconfig_apply_pending {
4457     my ($vmid, $conf, $storecfg) = @_;
4458
4459     # cold plug
4460
4461     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4462     while (my ($opt, $force) = each %$pending_delete_hash) {
4463         die "internal error" if $opt =~ m/^unused/;
4464         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4465         if (!defined($conf->{$opt})) {
4466             vmconfig_undelete_pending_option($conf, $opt);
4467             PVE::QemuConfig->write_config($vmid, $conf);
4468         } elsif (is_valid_drivename($opt)) {
4469             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4470             vmconfig_undelete_pending_option($conf, $opt);
4471             delete $conf->{$opt};
4472             PVE::QemuConfig->write_config($vmid, $conf);
4473         } else {
4474             vmconfig_undelete_pending_option($conf, $opt);
4475             delete $conf->{$opt};
4476             PVE::QemuConfig->write_config($vmid, $conf);
4477         }
4478     }
4479
4480     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4481
4482     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4483         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4484
4485         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4486             # skip if nothing changed
4487         } elsif (is_valid_drivename($opt)) {
4488             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4489                 if defined($conf->{$opt});
4490             $conf->{$opt} = $conf->{pending}->{$opt};
4491         } else {
4492             $conf->{$opt} = $conf->{pending}->{$opt};
4493         }
4494
4495         delete $conf->{pending}->{$opt};
4496         PVE::QemuConfig->write_config($vmid, $conf);
4497     }
4498 }
4499
4500 my $safe_num_ne = sub {
4501     my ($a, $b) = @_;
4502
4503     return 0 if !defined($a) && !defined($b);
4504     return 1 if !defined($a);
4505     return 1 if !defined($b);
4506
4507     return $a != $b;
4508 };
4509
4510 my $safe_string_ne = sub {
4511     my ($a, $b) = @_;
4512
4513     return 0 if !defined($a) && !defined($b);
4514     return 1 if !defined($a);
4515     return 1 if !defined($b);
4516
4517     return $a ne $b;
4518 };
4519
4520 sub vmconfig_update_net {
4521     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4522
4523     my $newnet = parse_net($value);
4524
4525     if ($conf->{$opt}) {
4526         my $oldnet = parse_net($conf->{$opt});
4527
4528         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4529             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4530             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4531             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4532
4533             # for non online change, we try to hot-unplug
4534             die "skip\n" if !$hotplug;
4535             vm_deviceunplug($vmid, $conf, $opt);
4536         } else {
4537
4538             die "internal error" if $opt !~ m/net(\d+)/;
4539             my $iface = "tap${vmid}i$1";
4540
4541             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4542                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4543                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4544                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4545                 PVE::Network::tap_unplug($iface);
4546                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4547             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4548                 # Rate can be applied on its own but any change above needs to
4549                 # include the rate in tap_plug since OVS resets everything.
4550                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4551             }
4552
4553             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4554                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4555             }
4556
4557             return 1;
4558         }
4559     }
4560
4561     if ($hotplug) {
4562         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4563     } else {
4564         die "skip\n";
4565     }
4566 }
4567
4568 sub vmconfig_update_disk {
4569     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4570
4571     # fixme: do we need force?
4572
4573     my $drive = parse_drive($opt, $value);
4574
4575     if ($conf->{$opt}) {
4576
4577         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4578
4579             my $media = $drive->{media} || 'disk';
4580             my $oldmedia = $old_drive->{media} || 'disk';
4581             die "unable to change media type\n" if $media ne $oldmedia;
4582
4583             if (!drive_is_cdrom($old_drive)) {
4584
4585                 if ($drive->{file} ne $old_drive->{file}) {
4586
4587                     die "skip\n" if !$hotplug;
4588
4589                     # unplug and register as unused
4590                     vm_deviceunplug($vmid, $conf, $opt);
4591                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4592
4593                 } else {
4594                     # update existing disk
4595
4596                     # skip non hotpluggable value
4597                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4598                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4599                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4600                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4601                         die "skip\n";
4602                     }
4603
4604                     # apply throttle
4605                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4606                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4607                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4608                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4609                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4610                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4611                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4612                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4613                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4614                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4615                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4616                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4617                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4618                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4619                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4620                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4621                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4622                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4623
4624                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4625                                                    ($drive->{mbps} || 0)*1024*1024,
4626                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4627                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4628                                                    $drive->{iops} || 0,
4629                                                    $drive->{iops_rd} || 0,
4630                                                    $drive->{iops_wr} || 0,
4631                                                    ($drive->{mbps_max} || 0)*1024*1024,
4632                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4633                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4634                                                    $drive->{iops_max} || 0,
4635                                                    $drive->{iops_rd_max} || 0,
4636                                                    $drive->{iops_wr_max} || 0,
4637                                                    $drive->{bps_max_length} || 1,
4638                                                    $drive->{bps_rd_max_length} || 1,
4639                                                    $drive->{bps_wr_max_length} || 1,
4640                                                    $drive->{iops_max_length} || 1,
4641                                                    $drive->{iops_rd_max_length} || 1,
4642                                                    $drive->{iops_wr_max_length} || 1);
4643
4644                     }
4645
4646                     return 1;
4647                 }
4648
4649             } else { # cdrom
4650
4651                 if ($drive->{file} eq 'none') {
4652                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4653                     if (drive_is_cloudinit($old_drive)) {
4654                         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
4655                     }
4656                 } else {
4657                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4658                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4659                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4660                 }
4661
4662                 return 1;
4663             }
4664         }
4665     }
4666
4667     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4668     # hotplug new disks
4669     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4670     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4671 }
4672
4673 sub vm_start {
4674     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4675         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
4676
4677     PVE::QemuConfig->lock_config($vmid, sub {
4678         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4679
4680         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
4681
4682         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4683
4684         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4685
4686         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4687             vmconfig_apply_pending($vmid, $conf, $storecfg);
4688             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4689         }
4690
4691         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
4692
4693         my $defaults = load_defaults();
4694
4695         # set environment variable useful inside network script
4696         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4697
4698         my $local_volumes = {};
4699
4700         if ($targetstorage) {
4701             foreach_drive($conf, sub {
4702                 my ($ds, $drive) = @_;
4703
4704                 return if drive_is_cdrom($drive);
4705
4706                 my $volid = $drive->{file};
4707
4708                 return if !$volid;
4709
4710                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4711
4712                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4713                 return if $scfg->{shared};
4714                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
4715             });
4716
4717             my $format = undef;
4718
4719             foreach my $opt (sort keys %$local_volumes) {
4720
4721                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
4722                 my $drive = parse_drive($opt, $conf->{$opt});
4723
4724                 #if remote storage is specified, use default format
4725                 if ($targetstorage && $targetstorage ne "1") {
4726                     $storeid = $targetstorage;
4727                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
4728                     $format = $defFormat;
4729                 } else {
4730                     #else we use same format than original
4731                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4732                     $format = qemu_img_format($scfg, $volid);
4733                 }
4734
4735                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
4736                 my $newdrive = $drive;
4737                 $newdrive->{format} = $format;
4738                 $newdrive->{file} = $newvolid;
4739                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
4740                 $local_volumes->{$opt} = $drivestr;
4741                 #pass drive to conf for command line
4742                 $conf->{$opt} = $drivestr;
4743             }
4744         }
4745
4746         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4747
4748         my $migrate_port = 0;
4749         my $migrate_uri;
4750         if ($statefile) {
4751             if ($statefile eq 'tcp') {
4752                 my $localip = "localhost";
4753                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4754                 my $nodename = PVE::INotify::nodename();
4755
4756                 if (!defined($migration_type)) {
4757                     if (defined($datacenterconf->{migration}->{type})) {
4758                         $migration_type = $datacenterconf->{migration}->{type};
4759                     } else {
4760                         $migration_type = 'secure';
4761                     }
4762                 }
4763
4764                 if ($migration_type eq 'insecure') {
4765                     my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4766                     if ($migrate_network_addr) {
4767                         $localip = $migrate_network_addr;
4768                     } else {
4769                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4770                     }
4771
4772                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4773                 }
4774
4775                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4776                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4777                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4778                 push @$cmd, '-incoming', $migrate_uri;
4779                 push @$cmd, '-S';
4780
4781             } elsif ($statefile eq 'unix') {
4782                 # should be default for secure migrations as a ssh TCP forward
4783                 # tunnel is not deterministic reliable ready and fails regurarly
4784                 # to set up in time, so use UNIX socket forwards
4785                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
4786                 unlink $socket_addr;
4787
4788                 $migrate_uri = "unix:$socket_addr";
4789
4790                 push @$cmd, '-incoming', $migrate_uri;
4791                 push @$cmd, '-S';
4792
4793             } else {
4794                 push @$cmd, '-loadstate', $statefile;
4795             }
4796         } elsif ($paused) {
4797             push @$cmd, '-S';
4798         }
4799
4800         # host pci devices
4801         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4802           my $d = parse_hostpci($conf->{"hostpci$i"});
4803           next if !$d;
4804           my $pcidevices = $d->{pciid};
4805           foreach my $pcidevice (@$pcidevices) {
4806                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4807
4808                 my $info = pci_device_info("0000:$pciid");
4809                 die "IOMMU not present\n" if !check_iommu_support();
4810                 die "no pci device info for device '$pciid'\n" if !$info;
4811                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4812                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4813           }
4814         }
4815
4816         PVE::Storage::activate_volumes($storecfg, $vollist);
4817
4818         if (!check_running($vmid, 1)) {
4819             eval {
4820                 run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
4821                     outfunc => sub {}, errfunc => sub {});
4822             };
4823         }
4824
4825         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
4826                                                   : $defaults->{cpuunits};
4827
4828         my $start_timeout = $conf->{hugepages} ? 300 : 30;
4829         my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077);
4830
4831         my %properties = (
4832             Slice => 'qemu.slice',
4833             KillMode => 'none',
4834             CPUShares => $cpuunits
4835         );
4836
4837         if (my $cpulimit = $conf->{cpulimit}) {
4838             $properties{CPUQuota} = int($cpulimit * 100);
4839         }
4840         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
4841
4842         if ($conf->{hugepages}) {
4843
4844             my $code = sub {
4845                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
4846                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
4847
4848                 PVE::QemuServer::Memory::hugepages_mount();
4849                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
4850
4851                 eval  {
4852                     PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4853                     run_command($cmd, %run_params);
4854                 };
4855
4856                 if (my $err = $@) {
4857                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
4858                     die $err;
4859                 }
4860
4861                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
4862             };
4863             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
4864
4865         } else {
4866             eval  {
4867                 PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4868                 run_command($cmd, %run_params);
4869             };
4870         }
4871
4872         if (my $err = $@) {
4873             # deactivate volumes if start fails
4874             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
4875             die "start failed: $err";
4876         }
4877
4878         print "migration listens on $migrate_uri\n" if $migrate_uri;
4879
4880         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
4881             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4882             warn $@ if $@;
4883         }
4884
4885         #start nbd server for storage migration
4886         if ($targetstorage) {
4887             my $nodename = PVE::INotify::nodename();
4888             my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4889             my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
4890             my $pfamily = PVE::Tools::get_host_address_family($nodename);
4891             $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4892
4893             vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${migrate_port}" } } );
4894
4895             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4896
4897             foreach my $opt (sort keys %$local_volumes) {
4898                 my $volid = $local_volumes->{$opt};
4899                 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
4900                 my $migrate_storage_uri = "nbd:${localip}:${migrate_port}:exportname=drive-$opt";
4901                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
4902             }
4903         }
4904
4905         if ($migratedfrom) {
4906             eval {
4907                 set_migration_caps($vmid);
4908             };
4909             warn $@ if $@;
4910
4911             if ($spice_port) {
4912                 print "spice listens on port $spice_port\n";
4913                 if ($spice_ticket) {
4914                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4915                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4916                 }
4917             }
4918
4919         } else {
4920             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4921                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4922                     if $conf->{balloon};
4923             }
4924
4925             foreach my $opt (keys %$conf) {
4926                 next if $opt !~  m/^net\d+$/;
4927                 my $nicconf = parse_net($conf->{$opt});
4928                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4929             }
4930         }
4931
4932         vm_mon_cmd_nocheck($vmid, 'qom-set',
4933                     path => "machine/peripheral/balloon0",
4934                     property => "guest-stats-polling-interval",
4935                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4936
4937     });
4938 }
4939
4940 sub vm_mon_cmd {
4941     my ($vmid, $execute, %params) = @_;
4942
4943     my $cmd = { execute => $execute, arguments => \%params };
4944     vm_qmp_command($vmid, $cmd);
4945 }
4946
4947 sub vm_mon_cmd_nocheck {
4948     my ($vmid, $execute, %params) = @_;
4949
4950     my $cmd = { execute => $execute, arguments => \%params };
4951     vm_qmp_command($vmid, $cmd, 1);
4952 }
4953
4954 sub vm_qmp_command {
4955     my ($vmid, $cmd, $nocheck) = @_;
4956
4957     my $res;
4958
4959     my $timeout;
4960     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4961         $timeout = $cmd->{arguments}->{timeout};
4962         delete $cmd->{arguments}->{timeout};
4963     }
4964
4965     eval {
4966         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4967         my $sname = qmp_socket($vmid);
4968         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4969             my $qmpclient = PVE::QMPClient->new();
4970
4971             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4972         } else {
4973             die "unable to open monitor socket\n";
4974         }
4975     };
4976     if (my $err = $@) {
4977         syslog("err", "VM $vmid qmp command failed - $err");
4978         die $err;
4979     }
4980
4981     return $res;
4982 }
4983
4984 sub vm_human_monitor_command {
4985     my ($vmid, $cmdline) = @_;
4986
4987     my $res;
4988
4989     my $cmd = {
4990         execute => 'human-monitor-command',
4991         arguments => { 'command-line' => $cmdline},
4992     };
4993
4994     return vm_qmp_command($vmid, $cmd);
4995 }
4996
4997 sub vm_commandline {
4998     my ($storecfg, $vmid) = @_;
4999
5000     my $conf = PVE::QemuConfig->load_config($vmid);
5001
5002     my $defaults = load_defaults();
5003
5004     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
5005
5006     return PVE::Tools::cmd2string($cmd);
5007 }
5008
5009 sub vm_reset {
5010     my ($vmid, $skiplock) = @_;
5011
5012     PVE::QemuConfig->lock_config($vmid, sub {
5013
5014         my $conf = PVE::QemuConfig->load_config($vmid);
5015
5016         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5017
5018         vm_mon_cmd($vmid, "system_reset");
5019     });
5020 }
5021
5022 sub get_vm_volumes {
5023     my ($conf) = @_;
5024
5025     my $vollist = [];
5026     foreach_volid($conf, sub {
5027         my ($volid, $attr) = @_;
5028
5029         return if $volid =~ m|^/|;
5030
5031         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5032         return if !$sid;
5033
5034         push @$vollist, $volid;
5035     });
5036
5037     return $vollist;
5038 }
5039
5040 sub vm_stop_cleanup {
5041     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5042
5043     eval {
5044
5045         if (!$keepActive) {
5046             my $vollist = get_vm_volumes($conf);
5047             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5048         }
5049
5050         foreach my $ext (qw(mon qmp pid vnc qga)) {
5051             unlink "/var/run/qemu-server/${vmid}.$ext";
5052         }
5053
5054         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5055     };
5056     warn $@ if $@; # avoid errors - just warn
5057 }
5058
5059 # Note: use $nockeck to skip tests if VM configuration file exists.
5060 # We need that when migration VMs to other nodes (files already moved)
5061 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5062 sub vm_stop {
5063     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5064
5065     $force = 1 if !defined($force) && !$shutdown;
5066
5067     if ($migratedfrom){
5068         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5069         kill 15, $pid if $pid;
5070         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5071         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5072         return;
5073     }
5074
5075     PVE::QemuConfig->lock_config($vmid, sub {
5076
5077         my $pid = check_running($vmid, $nocheck);
5078         return if !$pid;
5079
5080         my $conf;
5081         if (!$nocheck) {
5082             $conf = PVE::QemuConfig->load_config($vmid);
5083             PVE::QemuConfig->check_lock($conf) if !$skiplock;
5084             if (!defined($timeout) && $shutdown && $conf->{startup}) {
5085                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5086                 $timeout = $opts->{down} if $opts->{down};
5087             }
5088         }
5089
5090         $timeout = 60 if !defined($timeout);
5091
5092         eval {
5093             if ($shutdown) {
5094                 if (defined($conf) && $conf->{agent}) {
5095                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
5096                 } else {
5097                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
5098                 }
5099             } else {
5100                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
5101             }
5102         };
5103         my $err = $@;
5104
5105         if (!$err) {
5106             my $count = 0;
5107             while (($count < $timeout) && check_running($vmid, $nocheck)) {
5108                 $count++;
5109                 sleep 1;
5110             }
5111
5112             if ($count >= $timeout) {
5113                 if ($force) {
5114                     warn "VM still running - terminating now with SIGTERM\n";
5115                     kill 15, $pid;
5116                 } else {
5117                     die "VM quit/powerdown failed - got timeout\n";
5118                 }
5119             } else {
5120                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5121                 return;
5122             }
5123         } else {
5124             if ($force) {
5125                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5126                 kill 15, $pid;
5127             } else {
5128                 die "VM quit/powerdown failed\n";
5129             }
5130         }
5131
5132         # wait again
5133         $timeout = 10;
5134
5135         my $count = 0;
5136         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5137             $count++;
5138             sleep 1;
5139         }
5140
5141         if ($count >= $timeout) {
5142             warn "VM still running - terminating now with SIGKILL\n";
5143             kill 9, $pid;
5144             sleep 1;
5145         }
5146
5147         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5148    });
5149 }
5150
5151 sub vm_suspend {
5152     my ($vmid, $skiplock) = @_;
5153
5154     PVE::QemuConfig->lock_config($vmid, sub {
5155
5156         my $conf = PVE::QemuConfig->load_config($vmid);
5157
5158         PVE::QemuConfig->check_lock($conf)
5159             if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5160
5161         vm_mon_cmd($vmid, "stop");
5162     });
5163 }
5164
5165 sub vm_resume {
5166     my ($vmid, $skiplock, $nocheck) = @_;
5167
5168     PVE::QemuConfig->lock_config($vmid, sub {
5169
5170         if (!$nocheck) {
5171
5172             my $conf = PVE::QemuConfig->load_config($vmid);
5173
5174             PVE::QemuConfig->check_lock($conf)
5175                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5176
5177             vm_mon_cmd($vmid, "cont");
5178
5179         } else {
5180             vm_mon_cmd_nocheck($vmid, "cont");
5181         }
5182     });
5183 }
5184
5185 sub vm_sendkey {
5186     my ($vmid, $skiplock, $key) = @_;
5187
5188     PVE::QemuConfig->lock_config($vmid, sub {
5189
5190         my $conf = PVE::QemuConfig->load_config($vmid);
5191
5192         # there is no qmp command, so we use the human monitor command
5193         vm_human_monitor_command($vmid, "sendkey $key");
5194     });
5195 }
5196
5197 sub vm_destroy {
5198     my ($storecfg, $vmid, $skiplock) = @_;
5199
5200     PVE::QemuConfig->lock_config($vmid, sub {
5201
5202         my $conf = PVE::QemuConfig->load_config($vmid);
5203
5204         if (!check_running($vmid)) {
5205             destroy_vm($storecfg, $vmid, undef, $skiplock);
5206         } else {
5207             die "VM $vmid is running - destroy failed\n";
5208         }
5209     });
5210 }
5211
5212 # pci helpers
5213
5214 sub file_write {
5215     my ($filename, $buf) = @_;
5216
5217     my $fh = IO::File->new($filename, "w");
5218     return undef if !$fh;
5219
5220     my $res = print $fh $buf;
5221
5222     $fh->close();
5223
5224     return $res;
5225 }
5226
5227 sub pci_device_info {
5228     my ($name) = @_;
5229
5230     my $res;
5231
5232     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
5233     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
5234
5235     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
5236     return undef if !defined($irq) || $irq !~ m/^\d+$/;
5237
5238     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
5239     return undef if !defined($vendor) || $vendor !~ s/^0x//;
5240
5241     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
5242     return undef if !defined($product) || $product !~ s/^0x//;
5243
5244     $res = {
5245         name => $name,
5246         vendor => $vendor,
5247         product => $product,
5248         domain => $domain,
5249         bus => $bus,
5250         slot => $slot,
5251         func => $func,
5252         irq => $irq,
5253         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
5254     };
5255
5256     return $res;
5257 }
5258
5259 sub pci_dev_reset {
5260     my ($dev) = @_;
5261
5262     my $name = $dev->{name};
5263
5264     my $fn = "$pcisysfs/devices/$name/reset";
5265
5266     return file_write($fn, "1");
5267 }
5268
5269 sub pci_dev_bind_to_vfio {
5270     my ($dev) = @_;
5271
5272     my $name = $dev->{name};
5273
5274     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5275
5276     if (!-d $vfio_basedir) {
5277         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5278     }
5279     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5280
5281     my $testdir = "$vfio_basedir/$name";
5282     return 1 if -d $testdir;
5283
5284     my $data = "$dev->{vendor} $dev->{product}";
5285     return undef if !file_write("$vfio_basedir/new_id", $data);
5286
5287     my $fn = "$pcisysfs/devices/$name/driver/unbind";
5288     if (!file_write($fn, $name)) {
5289         return undef if -f $fn;
5290     }
5291
5292     $fn = "$vfio_basedir/bind";
5293     if (! -d $testdir) {
5294         return undef if !file_write($fn, $name);
5295     }
5296
5297     return -d $testdir;
5298 }
5299
5300 sub pci_dev_group_bind_to_vfio {
5301     my ($pciid) = @_;
5302
5303     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5304
5305     if (!-d $vfio_basedir) {
5306         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5307     }
5308     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5309
5310     # get IOMMU group devices
5311     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
5312       my @devs = grep /^0000:/, readdir($D);
5313     closedir($D);
5314
5315     foreach my $pciid (@devs) {
5316         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
5317
5318         # pci bridges, switches or root ports are not supported
5319         # they have a pci_bus subdirectory so skip them
5320         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
5321
5322         my $info = pci_device_info($1);
5323         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
5324     }
5325
5326     return 1;
5327 }
5328
5329 # vzdump restore implementaion
5330
5331 sub tar_archive_read_firstfile {
5332     my $archive = shift;
5333
5334     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5335
5336     # try to detect archive type first
5337     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5338         die "unable to open file '$archive'\n";
5339     my $firstfile = <$fh>;
5340     kill 15, $pid;
5341     close $fh;
5342
5343     die "ERROR: archive contaions no data\n" if !$firstfile;
5344     chomp $firstfile;
5345
5346     return $firstfile;
5347 }
5348
5349 sub tar_restore_cleanup {
5350     my ($storecfg, $statfile) = @_;
5351
5352     print STDERR "starting cleanup\n";
5353
5354     if (my $fd = IO::File->new($statfile, "r")) {
5355         while (defined(my $line = <$fd>)) {
5356             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5357                 my $volid = $2;
5358                 eval {
5359                     if ($volid =~ m|^/|) {
5360                         unlink $volid || die 'unlink failed\n';
5361                     } else {
5362                         PVE::Storage::vdisk_free($storecfg, $volid);
5363                     }
5364                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5365                 };
5366                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5367             } else {
5368                 print STDERR "unable to parse line in statfile - $line";
5369             }
5370         }
5371         $fd->close();
5372     }
5373 }
5374
5375 sub restore_archive {
5376     my ($archive, $vmid, $user, $opts) = @_;
5377
5378     my $format = $opts->{format};
5379     my $comp;
5380
5381     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5382         $format = 'tar' if !$format;
5383         $comp = 'gzip';
5384     } elsif ($archive =~ m/\.tar$/) {
5385         $format = 'tar' if !$format;
5386     } elsif ($archive =~ m/.tar.lzo$/) {
5387         $format = 'tar' if !$format;
5388         $comp = 'lzop';
5389     } elsif ($archive =~ m/\.vma$/) {
5390         $format = 'vma' if !$format;
5391     } elsif ($archive =~ m/\.vma\.gz$/) {
5392         $format = 'vma' if !$format;
5393         $comp = 'gzip';
5394     } elsif ($archive =~ m/\.vma\.lzo$/) {
5395         $format = 'vma' if !$format;
5396         $comp = 'lzop';
5397     } else {
5398         $format = 'vma' if !$format; # default
5399     }
5400
5401     # try to detect archive format
5402     if ($format eq 'tar') {
5403         return restore_tar_archive($archive, $vmid, $user, $opts);
5404     } else {
5405         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5406     }
5407 }
5408
5409 sub restore_update_config_line {
5410     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5411
5412     return if $line =~ m/^\#qmdump\#/;
5413     return if $line =~ m/^\#vzdump\#/;
5414     return if $line =~ m/^lock:/;
5415     return if $line =~ m/^unused\d+:/;
5416     return if $line =~ m/^parent:/;
5417     return if $line =~ m/^template:/; # restored VM is never a template
5418
5419     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5420     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5421         # try to convert old 1.X settings
5422         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5423         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5424             my ($model, $macaddr) = split(/\=/, $devconfig);
5425             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5426             my $net = {
5427                 model => $model,
5428                 bridge => "vmbr$ind",
5429                 macaddr => $macaddr,
5430             };
5431             my $netstr = print_net($net);
5432
5433             print $outfd "net$cookie->{netcount}: $netstr\n";
5434             $cookie->{netcount}++;
5435         }
5436     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5437         my ($id, $netstr) = ($1, $2);
5438         my $net = parse_net($netstr);
5439         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5440         $netstr = print_net($net);
5441         print $outfd "$id: $netstr\n";
5442     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5443         my $virtdev = $1;
5444         my $value = $3;
5445         my $di = parse_drive($virtdev, $value);
5446         if (defined($di->{backup}) && !$di->{backup}) {
5447             print $outfd "#$line";
5448         } elsif ($map->{$virtdev}) {
5449             delete $di->{format}; # format can change on restore
5450             $di->{file} = $map->{$virtdev};
5451             $value = print_drive($vmid, $di);
5452             print $outfd "$virtdev: $value\n";
5453         } else {
5454             print $outfd $line;
5455         }
5456     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5457         my ($uuid, $uuid_str);
5458         UUID::generate($uuid);
5459         UUID::unparse($uuid, $uuid_str);
5460         my $smbios1 = parse_smbios1($2);
5461         $smbios1->{uuid} = $uuid_str;
5462         print $outfd $1.print_smbios1($smbios1)."\n";
5463     } else {
5464         print $outfd $line;
5465     }
5466 }
5467
5468 sub scan_volids {
5469     my ($cfg, $vmid) = @_;
5470
5471     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5472
5473     my $volid_hash = {};
5474     foreach my $storeid (keys %$info) {
5475         foreach my $item (@{$info->{$storeid}}) {
5476             next if !($item->{volid} && $item->{size});
5477             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5478             $volid_hash->{$item->{volid}} = $item;
5479         }
5480     }
5481
5482     return $volid_hash;
5483 }
5484
5485 sub is_volume_in_use {
5486     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5487
5488     my $path = PVE::Storage::path($storecfg, $volid);
5489
5490     my $scan_config = sub {
5491         my ($cref, $snapname) = @_;
5492
5493         foreach my $key (keys %$cref) {
5494             my $value = $cref->{$key};
5495             if (is_valid_drivename($key)) {
5496                 next if $skip_drive && $key eq $skip_drive;
5497                 my $drive = parse_drive($key, $value);
5498                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5499                 return 1 if $volid eq $drive->{file};
5500                 if ($drive->{file} =~ m!^/!) {
5501                     return 1 if $drive->{file} eq $path;
5502                 } else {
5503                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5504                     next if !$storeid;
5505                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5506                     next if !$scfg;
5507                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5508                 }
5509             }
5510         }
5511
5512         return 0;
5513     };
5514
5515     return 1 if &$scan_config($conf);
5516
5517     undef $skip_drive;
5518
5519     foreach my $snapname (keys %{$conf->{snapshots}}) {
5520         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5521     }
5522
5523     return 0;
5524 }
5525
5526 sub update_disksize {
5527     my ($vmid, $conf, $volid_hash) = @_;
5528
5529     my $changes;
5530
5531     # used and unused disks
5532     my $referenced = {};
5533
5534     # Note: it is allowed to define multiple storages with same path (alias), so
5535     # we need to check both 'volid' and real 'path' (two different volid can point
5536     # to the same path).
5537
5538     my $referencedpath = {};
5539
5540     # update size info
5541     foreach my $opt (keys %$conf) {
5542         if (is_valid_drivename($opt)) {
5543             my $drive = parse_drive($opt, $conf->{$opt});
5544             my $volid = $drive->{file};
5545             next if !$volid;
5546
5547             $referenced->{$volid} = 1;
5548             if ($volid_hash->{$volid} &&
5549                 (my $path = $volid_hash->{$volid}->{path})) {
5550                 $referencedpath->{$path} = 1;
5551             }
5552
5553             next if drive_is_cdrom($drive);
5554             next if !$volid_hash->{$volid};
5555
5556             $drive->{size} = $volid_hash->{$volid}->{size};
5557             my $new = print_drive($vmid, $drive);
5558             if ($new ne $conf->{$opt}) {
5559                 $changes = 1;
5560                 $conf->{$opt} = $new;
5561             }
5562         }
5563     }
5564
5565     # remove 'unusedX' entry if volume is used
5566     foreach my $opt (keys %$conf) {
5567         next if $opt !~ m/^unused\d+$/;
5568         my $volid = $conf->{$opt};
5569         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5570         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
5571             $changes = 1;
5572             delete $conf->{$opt};
5573         }
5574
5575         $referenced->{$volid} = 1;
5576         $referencedpath->{$path} = 1 if $path;
5577     }
5578
5579     foreach my $volid (sort keys %$volid_hash) {
5580         next if $volid =~ m/vm-$vmid-state-/;
5581         next if $referenced->{$volid};
5582         my $path = $volid_hash->{$volid}->{path};
5583         next if !$path; # just to be sure
5584         next if $referencedpath->{$path};
5585         $changes = 1;
5586         PVE::QemuConfig->add_unused_volume($conf, $volid);
5587         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
5588     }
5589
5590     return $changes;
5591 }
5592
5593 sub rescan {
5594     my ($vmid, $nolock) = @_;
5595
5596     my $cfg = PVE::Storage::config();
5597
5598     my $volid_hash = scan_volids($cfg, $vmid);
5599
5600     my $updatefn =  sub {
5601         my ($vmid) = @_;
5602
5603         my $conf = PVE::QemuConfig->load_config($vmid);
5604
5605         PVE::QemuConfig->check_lock($conf);
5606
5607         my $vm_volids = {};
5608         foreach my $volid (keys %$volid_hash) {
5609             my $info = $volid_hash->{$volid};
5610             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5611         }
5612
5613         my $changes = update_disksize($vmid, $conf, $vm_volids);
5614
5615         PVE::QemuConfig->write_config($vmid, $conf) if $changes;
5616     };
5617
5618     if (defined($vmid)) {
5619         if ($nolock) {
5620             &$updatefn($vmid);
5621         } else {
5622             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5623         }
5624     } else {
5625         my $vmlist = config_list();
5626         foreach my $vmid (keys %$vmlist) {
5627             if ($nolock) {
5628                 &$updatefn($vmid);
5629             } else {
5630                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5631             }
5632         }
5633     }
5634 }
5635
5636 sub restore_vma_archive {
5637     my ($archive, $vmid, $user, $opts, $comp) = @_;
5638
5639     my $readfrom = $archive;
5640
5641     my $cfg = PVE::Storage::config();
5642     my $commands = [];
5643     my $bwlimit = $opts->{bwlimit};
5644
5645     my $dbg_cmdstring = '';
5646     my $add_pipe = sub {
5647         my ($cmd) = @_;
5648         push @$commands, $cmd;
5649         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
5650         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
5651         $readfrom = '-';
5652     };
5653
5654     my $input = undef;
5655     if ($archive eq '-') {
5656         $input = '<&STDIN';
5657     } else {
5658         # If we use a backup from a PVE defined storage we also consider that
5659         # storage's rate limit:
5660         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
5661         if (defined($volid)) {
5662             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
5663             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
5664             if ($readlimit) {
5665                 print STDERR "applying read rate limit: $readlimit\n";
5666                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
5667                 $add_pipe->($cstream);
5668             }
5669         }
5670     }
5671
5672     if ($comp) {
5673         my $cmd;
5674         if ($comp eq 'gzip') {
5675             $cmd = ['zcat', $readfrom];
5676         } elsif ($comp eq 'lzop') {
5677             $cmd = ['lzop', '-d', '-c', $readfrom];
5678         } else {
5679             die "unknown compression method '$comp'\n";
5680         }
5681         $add_pipe->($cmd);
5682     }
5683
5684     my $tmpdir = "/var/tmp/vzdumptmp$$";
5685     rmtree $tmpdir;
5686
5687     # disable interrupts (always do cleanups)
5688     local $SIG{INT} =
5689         local $SIG{TERM} =
5690         local $SIG{QUIT} =
5691         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
5692
5693     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5694     POSIX::mkfifo($mapfifo, 0600);
5695     my $fifofh;
5696
5697     my $openfifo = sub {
5698         open($fifofh, '>', $mapfifo) || die $!;
5699     };
5700
5701     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
5702
5703     my $oldtimeout;
5704     my $timeout = 5;
5705
5706     my $devinfo = {};
5707
5708     my $rpcenv = PVE::RPCEnvironment::get();
5709
5710     my $conffile = PVE::QemuConfig->config_file($vmid);
5711     my $tmpfn = "$conffile.$$.tmp";
5712
5713     # Note: $oldconf is undef if VM does not exists
5714     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
5715     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
5716
5717     my %storage_limits;
5718
5719     my $print_devmap = sub {
5720         my $virtdev_hash = {};
5721
5722         my $cfgfn = "$tmpdir/qemu-server.conf";
5723
5724         # we can read the config - that is already extracted
5725         my $fh = IO::File->new($cfgfn, "r") ||
5726             "unable to read qemu-server.conf - $!\n";
5727
5728         my $fwcfgfn = "$tmpdir/qemu-server.fw";
5729         if (-f $fwcfgfn) {
5730             my $pve_firewall_dir = '/etc/pve/firewall';
5731             mkdir $pve_firewall_dir; # make sure the dir exists
5732             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
5733         }
5734
5735         while (defined(my $line = <$fh>)) {
5736             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5737                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5738                 die "archive does not contain data for drive '$virtdev'\n"
5739                     if !$devinfo->{$devname};
5740                 if (defined($opts->{storage})) {
5741                     $storeid = $opts->{storage} || 'local';
5742                 } elsif (!$storeid) {
5743                     $storeid = 'local';
5744                 }
5745                 $format = 'raw' if !$format;
5746                 $devinfo->{$devname}->{devname} = $devname;
5747                 $devinfo->{$devname}->{virtdev} = $virtdev;
5748                 $devinfo->{$devname}->{format} = $format;
5749                 $devinfo->{$devname}->{storeid} = $storeid;
5750
5751                 # check permission on storage
5752                 my $pool = $opts->{pool}; # todo: do we need that?
5753                 if ($user ne 'root@pam') {
5754                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5755                 }
5756
5757                 $storage_limits{$storeid} = $bwlimit;
5758
5759                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5760             }
5761         }
5762
5763         foreach my $key (keys %storage_limits) {
5764             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
5765             next if !$limit;
5766             print STDERR "rate limit for storage $key: $limit KiB/s\n";
5767             $storage_limits{$key} = $limit * 1024;
5768         }
5769
5770         foreach my $devname (keys %$devinfo) {
5771             die "found no device mapping information for device '$devname'\n"
5772                 if !$devinfo->{$devname}->{virtdev};
5773         }
5774
5775         # create empty/temp config
5776         if ($oldconf) {
5777             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5778             foreach_drive($oldconf, sub {
5779                 my ($ds, $drive) = @_;
5780
5781                 return if drive_is_cdrom($drive);
5782
5783                 my $volid = $drive->{file};
5784
5785                 return if !$volid || $volid =~ m|^/|;
5786
5787                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5788                 return if !$path || !$owner || ($owner != $vmid);
5789
5790                 # Note: only delete disk we want to restore
5791                 # other volumes will become unused
5792                 if ($virtdev_hash->{$ds}) {
5793                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
5794                     if (my $err = $@) {
5795                         warn $err;
5796                     }
5797                 }
5798             });
5799
5800             # delete vmstate files
5801             # since after the restore we have no snapshots anymore
5802             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5803                 my $snap = $oldconf->{snapshots}->{$snapname};
5804                 if ($snap->{vmstate}) {
5805                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
5806                     if (my $err = $@) {
5807                         warn $err;
5808                     }
5809                 }
5810             }
5811         }
5812
5813         my $map = {};
5814         foreach my $virtdev (sort keys %$virtdev_hash) {
5815             my $d = $virtdev_hash->{$virtdev};
5816             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5817             my $storeid = $d->{storeid};
5818             my $scfg = PVE::Storage::storage_config($cfg, $storeid);
5819
5820             my $map_opts = '';
5821             if (my $limit = $storage_limits{$storeid}) {
5822                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
5823             }
5824
5825             # test if requested format is supported
5826             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $storeid);
5827             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5828             $d->{format} = $defFormat if !$supported;
5829
5830             my $volid = PVE::Storage::vdisk_alloc($cfg, $storeid, $vmid,
5831                                                   $d->{format}, undef, $alloc_size);
5832             print STDERR "new volume ID is '$volid'\n";
5833             $d->{volid} = $volid;
5834             my $path = PVE::Storage::path($cfg, $volid);
5835
5836             PVE::Storage::activate_volumes($cfg,[$volid]);
5837
5838             my $write_zeros = 1;
5839             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
5840                 $write_zeros = 0;
5841             }
5842
5843             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
5844
5845             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5846             $map->{$virtdev} = $volid;
5847         }
5848
5849         $fh->seek(0, 0) || die "seek failed - $!\n";
5850
5851         my $outfd = new IO::File ($tmpfn, "w") ||
5852             die "unable to write config for VM $vmid\n";
5853
5854         my $cookie = { netcount => 0 };
5855         while (defined(my $line = <$fh>)) {
5856             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5857         }
5858
5859         $fh->close();
5860         $outfd->close();
5861     };
5862
5863     eval {
5864         # enable interrupts
5865         local $SIG{INT} =
5866             local $SIG{TERM} =
5867             local $SIG{QUIT} =
5868             local $SIG{HUP} =
5869             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
5870         local $SIG{ALRM} = sub { die "got timeout\n"; };
5871
5872         $oldtimeout = alarm($timeout);
5873
5874         my $parser = sub {
5875             my $line = shift;
5876
5877             print "$line\n";
5878
5879             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5880                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5881                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5882             } elsif ($line =~ m/^CTIME: /) {
5883                 # we correctly received the vma config, so we can disable
5884                 # the timeout now for disk allocation (set to 10 minutes, so
5885                 # that we always timeout if something goes wrong)
5886                 alarm(600);
5887                 &$print_devmap();
5888                 print $fifofh "done\n";
5889                 my $tmp = $oldtimeout || 0;
5890                 $oldtimeout = undef;
5891                 alarm($tmp);
5892                 close($fifofh);
5893             }
5894         };
5895
5896         print "restore vma archive: $dbg_cmdstring\n";
5897         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
5898     };
5899     my $err = $@;
5900
5901     alarm($oldtimeout) if $oldtimeout;
5902
5903     my $vollist = [];
5904     foreach my $devname (keys %$devinfo) {
5905         my $volid = $devinfo->{$devname}->{volid};
5906         push @$vollist, $volid if $volid;
5907     }
5908
5909     PVE::Storage::deactivate_volumes($cfg, $vollist);
5910
5911     unlink $mapfifo;
5912
5913     if ($err) {
5914         rmtree $tmpdir;
5915         unlink $tmpfn;
5916
5917         foreach my $devname (keys %$devinfo) {
5918             my $volid = $devinfo->{$devname}->{volid};
5919             next if !$volid;
5920             eval {
5921                 if ($volid =~ m|^/|) {
5922                     unlink $volid || die 'unlink failed\n';
5923                 } else {
5924                     PVE::Storage::vdisk_free($cfg, $volid);
5925                 }
5926                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5927             };
5928             print STDERR "unable to cleanup '$volid' - $@" if $@;
5929         }
5930         die $err;
5931     }
5932
5933     rmtree $tmpdir;
5934
5935     rename($tmpfn, $conffile) ||
5936         die "unable to commit configuration file '$conffile'\n";
5937
5938     PVE::Cluster::cfs_update(); # make sure we read new file
5939
5940     eval { rescan($vmid, 1); };
5941     warn $@ if $@;
5942 }
5943
5944 sub restore_tar_archive {
5945     my ($archive, $vmid, $user, $opts) = @_;
5946
5947     if ($archive ne '-') {
5948         my $firstfile = tar_archive_read_firstfile($archive);
5949         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5950             if $firstfile ne 'qemu-server.conf';
5951     }
5952
5953     my $storecfg = PVE::Storage::config();
5954
5955     # destroy existing data - keep empty config
5956     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
5957     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5958
5959     my $tocmd = "/usr/lib/qemu-server/qmextract";
5960
5961     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5962     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5963     $tocmd .= ' --prealloc' if $opts->{prealloc};
5964     $tocmd .= ' --info' if $opts->{info};
5965
5966     # tar option "xf" does not autodetect compression when read from STDIN,
5967     # so we pipe to zcat
5968     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5969         PVE::Tools::shellquote("--to-command=$tocmd");
5970
5971     my $tmpdir = "/var/tmp/vzdumptmp$$";
5972     mkpath $tmpdir;
5973
5974     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5975     local $ENV{VZDUMP_VMID} = $vmid;
5976     local $ENV{VZDUMP_USER} = $user;
5977
5978     my $conffile = PVE::QemuConfig->config_file($vmid);
5979     my $tmpfn = "$conffile.$$.tmp";
5980
5981     # disable interrupts (always do cleanups)
5982     local $SIG{INT} =
5983         local $SIG{TERM} =
5984         local $SIG{QUIT} =
5985         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
5986
5987     eval {
5988         # enable interrupts
5989         local $SIG{INT} =
5990             local $SIG{TERM} =
5991             local $SIG{QUIT} =
5992             local $SIG{HUP} =
5993             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
5994
5995         if ($archive eq '-') {
5996             print "extracting archive from STDIN\n";
5997             run_command($cmd, input => "<&STDIN");
5998         } else {
5999             print "extracting archive '$archive'\n";
6000             run_command($cmd);
6001         }
6002
6003         return if $opts->{info};
6004
6005         # read new mapping
6006         my $map = {};
6007         my $statfile = "$tmpdir/qmrestore.stat";
6008         if (my $fd = IO::File->new($statfile, "r")) {
6009             while (defined (my $line = <$fd>)) {
6010                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6011                     $map->{$1} = $2 if $1;
6012                 } else {
6013                     print STDERR "unable to parse line in statfile - $line\n";
6014                 }
6015             }
6016             $fd->close();
6017         }
6018
6019         my $confsrc = "$tmpdir/qemu-server.conf";
6020
6021         my $srcfd = new IO::File($confsrc, "r") ||
6022             die "unable to open file '$confsrc'\n";
6023
6024         my $outfd = new IO::File ($tmpfn, "w") ||
6025             die "unable to write config for VM $vmid\n";
6026
6027         my $cookie = { netcount => 0 };
6028         while (defined (my $line = <$srcfd>)) {
6029             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6030         }
6031
6032         $srcfd->close();
6033         $outfd->close();
6034     };
6035     my $err = $@;
6036
6037     if ($err) {
6038
6039         unlink $tmpfn;
6040
6041         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6042
6043         die $err;
6044     }
6045
6046     rmtree $tmpdir;
6047
6048     rename $tmpfn, $conffile ||
6049         die "unable to commit configuration file '$conffile'\n";
6050
6051     PVE::Cluster::cfs_update(); # make sure we read new file
6052
6053     eval { rescan($vmid, 1); };
6054     warn $@ if $@;
6055 };
6056
6057 sub foreach_storage_used_by_vm {
6058     my ($conf, $func) = @_;
6059
6060     my $sidhash = {};
6061
6062     foreach_drive($conf, sub {
6063         my ($ds, $drive) = @_;
6064         return if drive_is_cdrom($drive);
6065
6066         my $volid = $drive->{file};
6067
6068         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6069         $sidhash->{$sid} = $sid if $sid;
6070     });
6071
6072     foreach my $sid (sort keys %$sidhash) {
6073         &$func($sid);
6074     }
6075 }
6076
6077 sub do_snapshots_with_qemu {
6078     my ($storecfg, $volid) = @_;
6079
6080     my $storage_name = PVE::Storage::parse_volume_id($volid);
6081
6082     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
6083         && !$storecfg->{ids}->{$storage_name}->{krbd}){
6084         return 1;
6085     }
6086
6087     if ($volid =~ m/\.(qcow2|qed)$/){
6088         return 1;
6089     }
6090
6091     return undef;
6092 }
6093
6094 sub qga_check_running {
6095     my ($vmid) = @_;
6096
6097     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
6098     if ($@) {
6099         warn "Qemu Guest Agent is not running - $@";
6100         return 0;
6101     }
6102     return 1;
6103 }
6104
6105 sub template_create {
6106     my ($vmid, $conf, $disk) = @_;
6107
6108     my $storecfg = PVE::Storage::config();
6109
6110     foreach_drive($conf, sub {
6111         my ($ds, $drive) = @_;
6112
6113         return if drive_is_cdrom($drive);
6114         return if $disk && $ds ne $disk;
6115
6116         my $volid = $drive->{file};
6117         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6118
6119         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6120         $drive->{file} = $voliddst;
6121         $conf->{$ds} = print_drive($vmid, $drive);
6122         PVE::QemuConfig->write_config($vmid, $conf);
6123     });
6124 }
6125
6126 sub qemu_img_convert {
6127     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6128
6129     my $storecfg = PVE::Storage::config();
6130     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6131     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6132
6133     if ($src_storeid && $dst_storeid) {
6134
6135         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6136
6137         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6138         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6139
6140         my $src_format = qemu_img_format($src_scfg, $src_volname);
6141         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6142
6143         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6144         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6145
6146         my $cmd = [];
6147         push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6148         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
6149         push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6150         push @$cmd, '-T', 'none' if $src_scfg->{type} eq 'zfspool';
6151         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path;
6152         if ($is_zero_initialized) {
6153             push @$cmd, "zeroinit:$dst_path";
6154         } else {
6155             push @$cmd, $dst_path;
6156         }
6157
6158         my $parser = sub {
6159             my $line = shift;
6160             if($line =~ m/\((\S+)\/100\%\)/){
6161                 my $percent = $1;
6162                 my $transferred = int($size * $percent / 100);
6163                 my $remaining = $size - $transferred;
6164
6165                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6166             }
6167
6168         };
6169
6170         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6171         my $err = $@;
6172         die "copy failed: $err" if $err;
6173     }
6174 }
6175
6176 sub qemu_img_format {
6177     my ($scfg, $volname) = @_;
6178
6179     if ($scfg->{path} && $volname =~ m/\.($QEMU_FORMAT_RE)$/) {
6180         return $1;
6181     } else {
6182         return "raw";
6183     }
6184 }
6185
6186 sub qemu_drive_mirror {
6187     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga) = @_;
6188
6189     $jobs = {} if !$jobs;
6190
6191     my $qemu_target;
6192     my $format;
6193     $jobs->{"drive-$drive"} = {};
6194
6195     if ($dst_volid =~ /^nbd:/) {
6196         $qemu_target = $dst_volid;
6197         $format = "nbd";
6198     } else {
6199         my $storecfg = PVE::Storage::config();
6200         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6201
6202         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6203
6204         $format = qemu_img_format($dst_scfg, $dst_volname);
6205
6206         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6207
6208         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6209     }
6210
6211     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6212     $opts->{format} = $format if $format;
6213
6214     print "drive mirror is starting for drive-$drive\n";
6215
6216     eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); }; #if a job already run for this device,it's throw an error
6217
6218     if (my $err = $@) {
6219         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6220         die "mirroring error: $err";
6221     }
6222
6223     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6224 }
6225
6226 sub qemu_drive_mirror_monitor {
6227     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6228
6229     eval {
6230         my $err_complete = 0;
6231
6232         while (1) {
6233             die "storage migration timed out\n" if $err_complete > 300;
6234
6235             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6236
6237             my $running_mirror_jobs = {};
6238             foreach my $stat (@$stats) {
6239                 next if $stat->{type} ne 'mirror';
6240                 $running_mirror_jobs->{$stat->{device}} = $stat;
6241             }
6242
6243             my $readycounter = 0;
6244
6245             foreach my $job (keys %$jobs) {
6246
6247                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6248                     print "$job : finished\n";
6249                     delete $jobs->{$job};
6250                     next;
6251                 }
6252
6253                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6254
6255                 my $busy = $running_mirror_jobs->{$job}->{busy};
6256                 my $ready = $running_mirror_jobs->{$job}->{ready};
6257                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6258                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6259                     my $remaining = $total - $transferred;
6260                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6261
6262                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6263                 }
6264
6265                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6266             }
6267
6268             last if scalar(keys %$jobs) == 0;
6269
6270             if ($readycounter == scalar(keys %$jobs)) {
6271                 print "all mirroring jobs are ready \n";
6272                 last if $skipcomplete; #do the complete later
6273
6274                 if ($vmiddst && $vmiddst != $vmid) {
6275                     my $agent_running = $qga && qga_check_running($vmid);
6276                     if ($agent_running) {
6277                         print "freeze filesystem\n";
6278                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6279                     } else {
6280                         print "suspend vm\n";
6281                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6282                     }
6283
6284                     # if we clone a disk for a new target vm, we don't switch the disk
6285                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6286
6287                     if ($agent_running) {
6288                         print "unfreeze filesystem\n";
6289                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6290                     } else {
6291                         print "resume vm\n";
6292                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6293                     }
6294
6295                     last;
6296                 } else {
6297
6298                     foreach my $job (keys %$jobs) {
6299                         # try to switch the disk if source and destination are on the same guest
6300                         print "$job: Completing block job...\n";
6301
6302                         eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
6303                         if ($@ =~ m/cannot be completed/) {
6304                             print "$job: Block job cannot be completed, try again.\n";
6305                             $err_complete++;
6306                         }else {
6307                             print "$job: Completed successfully.\n";
6308                             $jobs->{$job}->{complete} = 1;
6309                         }
6310                     }
6311                 }
6312             }
6313             sleep 1;
6314         }
6315     };
6316     my $err = $@;
6317
6318     if ($err) {
6319         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6320         die "mirroring error: $err";
6321     }
6322
6323 }
6324
6325 sub qemu_blockjobs_cancel {
6326     my ($vmid, $jobs) = @_;
6327
6328     foreach my $job (keys %$jobs) {
6329         print "$job: Cancelling block job\n";
6330         eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
6331         $jobs->{$job}->{cancel} = 1;
6332     }
6333
6334     while (1) {
6335         my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6336
6337         my $running_jobs = {};
6338         foreach my $stat (@$stats) {
6339             $running_jobs->{$stat->{device}} = $stat;
6340         }
6341
6342         foreach my $job (keys %$jobs) {
6343
6344             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6345                 print "$job: Done.\n";
6346                 delete $jobs->{$job};
6347             }
6348         }
6349
6350         last if scalar(keys %$jobs) == 0;
6351
6352         sleep 1;
6353     }
6354 }
6355
6356 sub clone_disk {
6357     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6358         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga) = @_;
6359
6360     my $newvolid;
6361
6362     if (!$full) {
6363         print "create linked clone of drive $drivename ($drive->{file})\n";
6364         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6365         push @$newvollist, $newvolid;
6366     } else {
6367
6368         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6369         $storeid = $storage if $storage;
6370
6371         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
6372         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6373
6374         print "create full clone of drive $drivename ($drive->{file})\n";
6375         my $name = undef;
6376         if (drive_is_cloudinit($drive)) {
6377             $name = "vm-$newvmid-cloudinit";
6378             # cloudinit only supports raw and qcow2 atm:
6379             if ($dst_format eq 'qcow2') {
6380                 $name .= '.qcow2';
6381             } elsif ($dst_format ne 'raw') {
6382                 die "clone: unhandled format for cloudinit image\n";
6383             }
6384         }
6385         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
6386         push @$newvollist, $newvolid;
6387
6388         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
6389
6390         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6391         if (!$running || $snapname) {
6392             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6393         } else {
6394
6395             my $kvmver = get_running_qemu_version ($vmid);
6396             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
6397                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6398                     if $drive->{iothread};
6399             }
6400
6401             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga);
6402         }
6403     }
6404
6405     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6406
6407     my $disk = $drive;
6408     $disk->{format} = undef;
6409     $disk->{file} = $newvolid;
6410     $disk->{size} = $size;
6411
6412     return $disk;
6413 }
6414
6415 # this only works if VM is running
6416 sub get_current_qemu_machine {
6417     my ($vmid) = @_;
6418
6419     my $cmd = { execute => 'query-machines', arguments => {} };
6420     my $res = vm_qmp_command($vmid, $cmd);
6421
6422     my ($current, $default);
6423     foreach my $e (@$res) {
6424         $default = $e->{name} if $e->{'is-default'};
6425         $current = $e->{name} if $e->{'is-current'};
6426     }
6427
6428     # fallback to the default machine if current is not supported by qemu
6429     return $current || $default || 'pc';
6430 }
6431
6432 sub get_running_qemu_version {
6433     my ($vmid) = @_;
6434     my $cmd = { execute => 'query-version', arguments => {} };
6435     my $res = vm_qmp_command($vmid, $cmd);
6436     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
6437 }
6438
6439 sub qemu_machine_feature_enabled {
6440     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6441
6442     my $current_major;
6443     my $current_minor;
6444
6445     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6446
6447         $current_major = $3;
6448         $current_minor = $4;
6449
6450     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6451
6452         $current_major = $1;
6453         $current_minor = $2;
6454     }
6455
6456     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
6457
6458
6459 }
6460
6461 sub qemu_machine_pxe {
6462     my ($vmid, $conf, $machine) = @_;
6463
6464     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6465
6466     foreach my $opt (keys %$conf) {
6467         next if $opt !~ m/^net(\d+)$/;
6468         my $net = PVE::QemuServer::parse_net($conf->{$opt});
6469         next if !$net;
6470         my $romfile = PVE::QemuServer::vm_mon_cmd_nocheck($vmid, 'qom-get', path => $opt, property => 'romfile');
6471         return $machine.".pxe" if $romfile =~ m/pxe/;
6472         last;
6473     }
6474
6475     return $machine;
6476 }
6477
6478 sub qemu_use_old_bios_files {
6479     my ($machine_type) = @_;
6480
6481     return if !$machine_type;
6482
6483     my $use_old_bios_files = undef;
6484
6485     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6486         $machine_type = $1;
6487         $use_old_bios_files = 1;
6488     } else {
6489         my $kvmver = kvm_user_version();
6490         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6491         # load new efi bios files on migration. So this hack is required to allow
6492         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6493         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6494         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
6495     }
6496
6497     return ($use_old_bios_files, $machine_type);
6498 }
6499
6500 sub create_efidisk {
6501     my ($storecfg, $storeid, $vmid, $fmt) = @_;
6502
6503     die "EFI vars default image not found\n" if ! -f $OVMF_VARS;
6504
6505     my $vars_size = PVE::Tools::convert_size(-s $OVMF_VARS, 'b' => 'kb');
6506     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
6507     PVE::Storage::activate_volumes($storecfg, [$volid]);
6508
6509     my $path = PVE::Storage::path($storecfg, $volid);
6510     eval {
6511         run_command(['/usr/bin/qemu-img', 'convert', '-n', '-f', 'raw', '-O', $fmt, $OVMF_VARS, $path]);
6512     };
6513     die "Copying EFI vars image failed: $@" if $@;
6514
6515     return ($volid, $vars_size);
6516 }
6517
6518 sub lspci {
6519
6520     my $devices = {};
6521
6522     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6523             my (undef, $id, $function) = @_;
6524             my $res = { id => $id, function => $function};
6525             push @{$devices->{$id}}, $res;
6526     });
6527
6528     # Entries should be sorted by functions.
6529     foreach my $id (keys %$devices) {
6530         my $dev = $devices->{$id};
6531         $devices->{$id} = [ sort { $a->{function} <=> $b->{function} } @$dev ];
6532     }
6533
6534     return $devices;
6535 }
6536
6537 sub vm_iothreads_list {
6538     my ($vmid) = @_;
6539
6540     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6541
6542     my $iothreads = {};
6543     foreach my $iothread (@$res) {
6544         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6545     }
6546
6547     return $iothreads;
6548 }
6549
6550 sub scsihw_infos {
6551     my ($conf, $drive) = @_;
6552
6553     my $maxdev = 0;
6554
6555     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
6556         $maxdev = 7;
6557     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6558         $maxdev = 1;
6559     } else {
6560         $maxdev = 256;
6561     }
6562
6563     my $controller = int($drive->{index} / $maxdev);
6564     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6565
6566     return ($maxdev, $controller, $controller_prefix);
6567 }
6568
6569 sub add_hyperv_enlightenments {
6570     my ($cpuFlags, $winversion, $machine_type, $kvmver, $bios, $gpu_passthrough) = @_;
6571
6572     return if $winversion < 6;
6573     return if $bios && $bios eq 'ovmf' && $winversion < 8;
6574
6575     push @$cpuFlags , 'hv_vendor_id=proxmox' if $gpu_passthrough;
6576
6577     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
6578         push @$cpuFlags , 'hv_spinlocks=0x1fff';
6579         push @$cpuFlags , 'hv_vapic';
6580         push @$cpuFlags , 'hv_time';
6581     } else {
6582         push @$cpuFlags , 'hv_spinlocks=0xffff';
6583     }
6584
6585     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
6586         push @$cpuFlags , 'hv_reset';
6587         push @$cpuFlags , 'hv_vpindex';
6588         push @$cpuFlags , 'hv_runtime';
6589     }
6590
6591     if ($winversion >= 7) {
6592         push @$cpuFlags , 'hv_relaxed';
6593     }
6594 }
6595
6596 sub windows_version {
6597     my ($ostype) = @_;
6598
6599     return 0 if !$ostype;
6600
6601     my $winversion = 0;
6602
6603     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
6604         $winversion = 5;
6605     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
6606         $winversion = 6;
6607     } elsif ($ostype =~ m/^win(\d+)$/) {
6608         $winversion = $1;
6609     }
6610
6611     return $winversion;
6612 }
6613
6614 sub resolve_dst_disk_format {
6615         my ($storecfg, $storeid, $src_volname, $format) = @_;
6616         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6617
6618         if (!$format) {
6619             # if no target format is specified, use the source disk format as hint
6620             if ($src_volname) {
6621                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6622                 $format = qemu_img_format($scfg, $src_volname);
6623             } else {
6624                 return $defFormat;
6625             }
6626         }
6627
6628         # test if requested format is supported - else use default
6629         my $supported = grep { $_ eq $format } @$validFormats;
6630         $format = $defFormat if !$supported;
6631         return $format;
6632 }
6633
6634 sub resolve_first_disk {
6635     my $conf = shift;
6636     my @disks = PVE::QemuServer::valid_drive_names();
6637     my $firstdisk;
6638     foreach my $ds (reverse @disks) {
6639         next if !$conf->{$ds};
6640         my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
6641         next if PVE::QemuServer::drive_is_cdrom($disk);
6642         $firstdisk = $ds;
6643     }
6644     return $firstdisk;
6645 }
6646
6647 sub generate_smbios1_uuid {
6648     my ($uuid, $uuid_str);
6649     UUID::generate($uuid);
6650     UUID::unparse($uuid, $uuid_str);
6651     return "uuid=$uuid_str";
6652 }
6653
6654 # bash completion helper
6655
6656 sub complete_backup_archives {
6657     my ($cmdname, $pname, $cvalue) = @_;
6658
6659     my $cfg = PVE::Storage::config();
6660
6661     my $storeid;
6662
6663     if ($cvalue =~ m/^([^:]+):/) {
6664         $storeid = $1;
6665     }
6666
6667     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6668
6669     my $res = [];
6670     foreach my $id (keys %$data) {
6671         foreach my $item (@{$data->{$id}}) {
6672             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6673             push @$res, $item->{volid} if defined($item->{volid});
6674         }
6675     }
6676
6677     return $res;
6678 }
6679
6680 my $complete_vmid_full = sub {
6681     my ($running) = @_;
6682
6683     my $idlist = vmstatus();
6684
6685     my $res = [];
6686
6687     foreach my $id (keys %$idlist) {
6688         my $d = $idlist->{$id};
6689         if (defined($running)) {
6690             next if $d->{template};
6691             next if $running && $d->{status} ne 'running';
6692             next if !$running && $d->{status} eq 'running';
6693         }
6694         push @$res, $id;
6695
6696     }
6697     return $res;
6698 };
6699
6700 sub complete_vmid {
6701     return &$complete_vmid_full();
6702 }
6703
6704 sub complete_vmid_stopped {
6705     return &$complete_vmid_full(0);
6706 }
6707
6708 sub complete_vmid_running {
6709     return &$complete_vmid_full(1);
6710 }
6711
6712 sub complete_storage {
6713
6714     my $cfg = PVE::Storage::config();
6715     my $ids = $cfg->{ids};
6716
6717     my $res = [];
6718     foreach my $sid (keys %$ids) {
6719         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6720         next if !$ids->{$sid}->{content}->{images};
6721         push @$res, $sid;
6722     }
6723
6724     return $res;
6725 }
6726
6727 1;