PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach $IPV6RE);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QemuConfig;
  31 use PVE::QMPClient;
  32 use PVE::RPCEnvironment;
  33 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
  34 use PVE::QemuServer::Memory;
  35 use PVE::QemuServer::USB qw(parse_usb_device);
  36 use PVE::QemuServer::Cloudinit;
  37 use Time::HiRes qw(gettimeofday);
  38 use File::Copy qw(copy);
  39 use URI::Escape;
  40
  41 my $OVMF_CODE = '/usr/share/kvm/OVMF_CODE-pure-efi.fd';
  42 my $OVMF_VARS = '/usr/share/kvm/OVMF_VARS-pure-efi.fd';
  43
  44 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  45
  46 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  47
  48 my $QEMU_FORMAT_RE = qr/raw|cow|qcow|qcow2|qed|vmdk|cloop/;
  49
  50 # Note about locking: we use flock on the config file protect
  51 # against concurent actions.
  52 # Aditionaly, we have a 'lock' setting in the config file. This
  53 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  54 # allowed when such lock is set. But you can ignore this kind of
  55 # lock with the --skiplock flag.
  56
  57 cfs_register_file('/qemu-server/',
  58                   \&parse_vm_config,
  59                   \&write_vm_config);
  60
  61 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  62     description => "Some command save/restore state from this location.",
  63     type => 'string',
  64     maxLength => 128,
  65     optional => 1,
  66 });
  67
  68 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  69     description => "The name of the snapshot.",
  70     type => 'string', format => 'pve-configid',
  71     maxLength => 40,
  72 });
  73
  74 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  75     type => 'string',
  76     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  77     description => "The drive's backing file's data format.",
  78     optional => 1,
  79 });
  80
  81 #no warnings 'redefine';
  82
  83 sub cgroups_write {
  84    my ($controller, $vmid, $option, $value) = @_;
  85
  86    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  87    PVE::ProcFSTools::write_proc_entry($path, $value);
  88
  89 }
  90
  91 my $nodename = PVE::INotify::nodename();
  92
  93 mkdir "/etc/pve/nodes/$nodename";
  94 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  95 mkdir $confdir;
  96
  97 my $var_run_tmpdir = "/var/run/qemu-server";
  98 mkdir $var_run_tmpdir;
  99
 100 my $lock_dir = "/var/lock/qemu-server";
 101 mkdir $lock_dir;
 102
 103 my $pcisysfs = "/sys/bus/pci";
 104
 105 my $cpu_vendor_list = {
 106     # Intel CPUs
 107     486 => 'GenuineIntel',
 108     pentium => 'GenuineIntel',
 109     pentium2  => 'GenuineIntel',
 110     pentium3  => 'GenuineIntel',
 111     coreduo => 'GenuineIntel',
 112     core2duo => 'GenuineIntel',
 113     Conroe  => 'GenuineIntel',
 114     Penryn  => 'GenuineIntel',
 115     Nehalem  => 'GenuineIntel',
 116     'Nehalem-IBRS'  => 'GenuineIntel',
 117     Westmere => 'GenuineIntel',
 118     'Westmere-IBRS' => 'GenuineIntel',
 119     SandyBridge => 'GenuineIntel',
 120     'SandyBridge-IBRS' => 'GenuineIntel',
 121     IvyBridge => 'GenuineIntel',
 122     'IvyBridge-IBRS' => 'GenuineIntel',
 123     Haswell => 'GenuineIntel',
 124     'Haswell-IBRS' => 'GenuineIntel',
 125     'Haswell-noTSX' => 'GenuineIntel',
 126     'Haswell-noTSX-IBRS' => 'GenuineIntel',
 127     Broadwell => 'GenuineIntel',
 128     'Broadwell-IBRS' => 'GenuineIntel',
 129     'Broadwell-noTSX' => 'GenuineIntel',
 130     'Broadwell-noTSX-IBRS' => 'GenuineIntel',
 131     'Skylake-Client' => 'GenuineIntel',
 132     'Skylake-Client-IBRS' => 'GenuineIntel',
 133     'Skylake-Server' => 'GenuineIntel',
 134     'Skylake-Server-IBRS' => 'GenuineIntel',
 135
 136     # AMD CPUs
 137     athlon => 'AuthenticAMD',
 138     phenom  => 'AuthenticAMD',
 139     Opteron_G1  => 'AuthenticAMD',
 140     Opteron_G2  => 'AuthenticAMD',
 141     Opteron_G3  => 'AuthenticAMD',
 142     Opteron_G4  => 'AuthenticAMD',
 143     Opteron_G5  => 'AuthenticAMD',
 144     EPYC => 'AuthenticAMD',
 145     'EPYC-IBPB' => 'AuthenticAMD',
 146
 147     # generic types, use vendor from host node
 148     host => 'default',
 149     kvm32 => 'default',
 150     kvm64 => 'default',
 151     qemu32 => 'default',
 152     qemu64 => 'default',
 153     max => 'default',
 154 };
 155
 156 my $cpu_flag = qr/[+-](pcid|spec-ctrl)/;
 157
 158 my $cpu_fmt = {
 159     cputype => {
 160         description => "Emulated CPU type.",
 161         type => 'string',
 162         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 163         default => 'kvm64',
 164         default_key => 1,
 165     },
 166     hidden => {
 167         description => "Do not identify as a KVM virtual machine.",
 168         type => 'boolean',
 169         optional => 1,
 170         default => 0
 171     },
 172     flags => {
 173         description => "List of additional CPU flags separated by ';'."
 174                      . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
 175                      . " Currently supported flags: 'pcid', 'spec-ctrl'.",
 176         format_description => '+FLAG[;-FLAG...]',
 177         type => 'string',
 178         pattern => qr/$cpu_flag(;$cpu_flag)*/,
 179         optional => 1,
 180     },
 181 };
 182
 183 my $watchdog_fmt = {
 184     model => {
 185         default_key => 1,
 186         type => 'string',
 187         enum => [qw(i6300esb ib700)],
 188         description => "Watchdog type to emulate.",
 189         default => 'i6300esb',
 190         optional => 1,
 191     },
 192     action => {
 193         type => 'string',
 194         enum => [qw(reset shutdown poweroff pause debug none)],
 195         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 196         optional => 1,
 197     },
 198 };
 199 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 200
 201 my $confdesc = {
 202     onboot => {
 203         optional => 1,
 204         type => 'boolean',
 205         description => "Specifies whether a VM will be started during system bootup.",
 206         default => 0,
 207     },
 208     autostart => {
 209         optional => 1,
 210         type => 'boolean',
 211         description => "Automatic restart after crash (currently ignored).",
 212         default => 0,
 213     },
 214     hotplug => {
 215         optional => 1,
 216         type => 'string', format => 'pve-hotplug-features',
 217         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 218         default => 'network,disk,usb',
 219     },
 220     reboot => {
 221         optional => 1,
 222         type => 'boolean',
 223         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 224         default => 1,
 225     },
 226     lock => {
 227         optional => 1,
 228         type => 'string',
 229         description => "Lock/unlock the VM.",
 230         enum => [qw(migrate backup snapshot rollback)],
 231     },
 232     cpulimit => {
 233         optional => 1,
 234         type => 'number',
 235         description => "Limit of CPU usage.",
 236         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 237         minimum => 0,
 238         maximum => 128,
 239         default => 0,
 240     },
 241     cpuunits => {
 242         optional => 1,
 243         type => 'integer',
 244         description => "CPU weight for a VM.",
 245         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 246         minimum => 2,
 247         maximum => 262144,
 248         default => 1024,
 249     },
 250     memory => {
 251         optional => 1,
 252         type => 'integer',
 253         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 254         minimum => 16,
 255         default => 512,
 256     },
 257     balloon => {
 258         optional => 1,
 259         type => 'integer',
 260         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 261         minimum => 0,
 262     },
 263     shares => {
 264         optional => 1,
 265         type => 'integer',
 266         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 267         minimum => 0,
 268         maximum => 50000,
 269         default => 1000,
 270     },
 271     keyboard => {
 272         optional => 1,
 273         type => 'string',
 274         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.conf' configuration file.".
 275                        "It should not be necessary to set it.",
 276         enum => PVE::Tools::kvmkeymaplist(),
 277         default => undef,
 278     },
 279     name => {
 280         optional => 1,
 281         type => 'string', format => 'dns-name',
 282         description => "Set a name for the VM. Only used on the configuration web interface.",
 283     },
 284     scsihw => {
 285         optional => 1,
 286         type => 'string',
 287         description => "SCSI controller model",
 288         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 289         default => 'lsi',
 290     },
 291     description => {
 292         optional => 1,
 293         type => 'string',
 294         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 295     },
 296     ostype => {
 297         optional => 1,
 298         type => 'string',
 299         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 300         description => "Specify guest operating system.",
 301         verbose_description => <<EODESC,
 302 Specify guest operating system. This is used to enable special
 303 optimization/features for specific operating systems:
 304
 305 [horizontal]
 306 other;; unspecified OS
 307 wxp;; Microsoft Windows XP
 308 w2k;; Microsoft Windows 2000
 309 w2k3;; Microsoft Windows 2003
 310 w2k8;; Microsoft Windows 2008
 311 wvista;; Microsoft Windows Vista
 312 win7;; Microsoft Windows 7
 313 win8;; Microsoft Windows 8/2012/2012r2
 314 win10;; Microsoft Windows 10/2016
 315 l24;; Linux 2.4 Kernel
 316 l26;; Linux 2.6/3.X Kernel
 317 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 318 EODESC
 319     },
 320     boot => {
 321         optional => 1,
 322         type => 'string',
 323         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 324         pattern => '[acdn]{1,4}',
 325         default => 'cdn',
 326     },
 327     bootdisk => {
 328         optional => 1,
 329         type => 'string', format => 'pve-qm-bootdisk',
 330         description => "Enable booting from specified disk.",
 331         pattern => '(ide|sata|scsi|virtio)\d+',
 332     },
 333     smp => {
 334         optional => 1,
 335         type => 'integer',
 336         description => "The number of CPUs. Please use option -sockets instead.",
 337         minimum => 1,
 338         default => 1,
 339     },
 340     sockets => {
 341         optional => 1,
 342         type => 'integer',
 343         description => "The number of CPU sockets.",
 344         minimum => 1,
 345         default => 1,
 346     },
 347     cores => {
 348         optional => 1,
 349         type => 'integer',
 350         description => "The number of cores per socket.",
 351         minimum => 1,
 352         default => 1,
 353     },
 354     numa => {
 355         optional => 1,
 356         type => 'boolean',
 357         description => "Enable/disable NUMA.",
 358         default => 0,
 359     },
 360     hugepages => {
 361         optional => 1,
 362         type => 'string',
 363         description => "Enable/disable hugepages memory.",
 364         enum => [qw(any 2 1024)],
 365     },
 366     vcpus => {
 367         optional => 1,
 368         type => 'integer',
 369         description => "Number of hotplugged vcpus.",
 370         minimum => 1,
 371         default => 0,
 372     },
 373     acpi => {
 374         optional => 1,
 375         type => 'boolean',
 376         description => "Enable/disable ACPI.",
 377         default => 1,
 378     },
 379     agent => {
 380         optional => 1,
 381         type => 'boolean',
 382         description => "Enable/disable Qemu GuestAgent.",
 383         default => 0,
 384     },
 385     kvm => {
 386         optional => 1,
 387         type => 'boolean',
 388         description => "Enable/disable KVM hardware virtualization.",
 389         default => 1,
 390     },
 391     tdf => {
 392         optional => 1,
 393         type => 'boolean',
 394         description => "Enable/disable time drift fix.",
 395         default => 0,
 396     },
 397     localtime => {
 398         optional => 1,
 399         type => 'boolean',
 400         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 401     },
 402     freeze => {
 403         optional => 1,
 404         type => 'boolean',
 405         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 406     },
 407     vga => {
 408         optional => 1,
 409         type => 'string',
 410         description => "Select the VGA type.",
 411         verbose_description => "Select the VGA type. If you want to use high resolution" .
 412             " modes (>= 1280x1024x16) then you should use the options " .
 413             "'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and " .
 414             "'cirrus' for other OS types. The 'qxl' option enables the SPICE " .
 415             "display sever. For win* OS you can select how many independent " .
 416             "displays you want, Linux guests can add displays them self. " .
 417             "You can also run without any graphic card, using a serial device" .
 418             " as terminal.",
 419         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 420     },
 421     watchdog => {
 422         optional => 1,
 423         type => 'string', format => 'pve-qm-watchdog',
 424         description => "Create a virtual hardware watchdog device.",
 425         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 426             " (by a guest action), the watchdog must be periodically polled " .
 427             "by an agent inside the guest or else the watchdog will reset " .
 428             "the guest (or execute the respective action specified)",
 429     },
 430     startdate => {
 431         optional => 1,
 432         type => 'string',
 433         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 434         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 435         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 436         default => 'now',
 437     },
 438     startup =>  get_standard_option('pve-startup-order'),
 439     template => {
 440         optional => 1,
 441         type => 'boolean',
 442         description => "Enable/disable Template.",
 443         default => 0,
 444     },
 445     args => {
 446         optional => 1,
 447         type => 'string',
 448         description => "Arbitrary arguments passed to kvm.",
 449         verbose_description => <<EODESCR,
 450 Arbitrary arguments passed to kvm, for example:
 451
 452 args: -no-reboot -no-hpet
 453
 454 NOTE: this option is for experts only.
 455 EODESCR
 456     },
 457     tablet => {
 458         optional => 1,
 459         type => 'boolean',
 460         default => 1,
 461         description => "Enable/disable the USB tablet device.",
 462         verbose_description => "Enable/disable the USB tablet device. This device is " .
 463             "usually needed to allow absolute mouse positioning with VNC. " .
 464             "Else the mouse runs out of sync with normal VNC clients. " .
 465             "If you're running lots of console-only guests on one host, " .
 466             "you may consider disabling this to save some context switches. " .
 467             "This is turned off by default if you use spice (-vga=qxl).",
 468     },
 469     migrate_speed => {
 470         optional => 1,
 471         type => 'integer',
 472         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 473         minimum => 0,
 474         default => 0,
 475     },
 476     migrate_downtime => {
 477         optional => 1,
 478         type => 'number',
 479         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 480         minimum => 0,
 481         default => 0.1,
 482     },
 483     cdrom => {
 484         optional => 1,
 485         type => 'string', format => 'pve-qm-ide',
 486         typetext => '<volume>',
 487         description => "This is an alias for option -ide2",
 488     },
 489     cpu => {
 490         optional => 1,
 491         description => "Emulated CPU type.",
 492         type => 'string',
 493         format => $cpu_fmt,
 494     },
 495     parent => get_standard_option('pve-snapshot-name', {
 496         optional => 1,
 497         description => "Parent snapshot name. This is used internally, and should not be modified.",
 498     }),
 499     snaptime => {
 500         optional => 1,
 501         description => "Timestamp for snapshots.",
 502         type => 'integer',
 503         minimum => 0,
 504     },
 505     vmstate => {
 506         optional => 1,
 507         type => 'string', format => 'pve-volume-id',
 508         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 509     },
 510     vmstatestorage => get_standard_option('pve-storage-id', {
 511         description => "Default storage for VM state volumes/files.",
 512         optional => 1,
 513     }),
 514     machine => {
 515         description => "Specific the Qemu machine type.",
 516         type => 'string',
 517         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
 518         maxLength => 40,
 519         optional => 1,
 520     },
 521     smbios1 => {
 522         description => "Specify SMBIOS type 1 fields.",
 523         type => 'string', format => 'pve-qm-smbios1',
 524         maxLength => 256,
 525         optional => 1,
 526     },
 527     protection => {
 528         optional => 1,
 529         type => 'boolean',
 530         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 531         default => 0,
 532     },
 533     bios => {
 534         optional => 1,
 535         type => 'string',
 536         enum => [ qw(seabios ovmf) ],
 537         description => "Select BIOS implementation.",
 538         default => 'seabios',
 539     },
 540 };
 541
 542 my $confdesc_cloudinit = {
 543     citype => {
 544         optional => 1,
 545         type => 'string',
 546         description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
 547         enum => ['configdrive2', 'nocloud'],
 548     },
 549     ciuser => {
 550         optional => 1,
 551         type => 'string',
 552         description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
 553     },
 554     cipassword => {
 555         optional => 1,
 556         type => 'string',
 557         description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
 558     },
 559     searchdomain => {
 560         optional => 1,
 561         type => 'string',
 562         description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 563     },
 564     nameserver => {
 565         optional => 1,
 566         type => 'string', format => 'address-list',
 567         description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 568     },
 569     sshkeys => {
 570         optional => 1,
 571         type => 'string',
 572         format => 'urlencoded',
 573         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 574     },
 575 };
 576
 577 # what about other qemu settings ?
 578 #cpu => 'string',
 579 #machine => 'string',
 580 #fda => 'file',
 581 #fdb => 'file',
 582 #mtdblock => 'file',
 583 #sd => 'file',
 584 #pflash => 'file',
 585 #snapshot => 'bool',
 586 #bootp => 'file',
 587 ##tftp => 'dir',
 588 ##smb => 'dir',
 589 #kernel => 'file',
 590 #append => 'string',
 591 #initrd => 'file',
 592 ##soundhw => 'string',
 593
 594 while (my ($k, $v) = each %$confdesc) {
 595     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 596 }
 597
 598 my $MAX_IDE_DISKS = 4;
 599 my $MAX_SCSI_DISKS = 14;
 600 my $MAX_VIRTIO_DISKS = 16;
 601 my $MAX_SATA_DISKS = 6;
 602 my $MAX_USB_DEVICES = 5;
 603 my $MAX_NETS = 32;
 604 my $MAX_UNUSED_DISKS = 8;
 605 my $MAX_HOSTPCI_DEVICES = 4;
 606 my $MAX_SERIAL_PORTS = 4;
 607 my $MAX_PARALLEL_PORTS = 3;
 608 my $MAX_NUMA = 8;
 609
 610 my $numa_fmt = {
 611     cpus => {
 612         type => "string",
 613         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 614         description => "CPUs accessing this NUMA node.",
 615         format_description => "id[-id];...",
 616     },
 617     memory => {
 618         type => "number",
 619         description => "Amount of memory this NUMA node provides.",
 620         optional => 1,
 621     },
 622     hostnodes => {
 623         type => "string",
 624         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 625         description => "Host NUMA nodes to use.",
 626         format_description => "id[-id];...",
 627         optional => 1,
 628     },
 629     policy => {
 630         type => 'string',
 631         enum => [qw(preferred bind interleave)],
 632         description => "NUMA allocation policy.",
 633         optional => 1,
 634     },
 635 };
 636 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 637 my $numadesc = {
 638     optional => 1,
 639     type => 'string', format => $numa_fmt,
 640     description => "NUMA topology.",
 641 };
 642 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 643
 644 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 645     $confdesc->{"numa$i"} = $numadesc;
 646 }
 647
 648 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 649                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 650                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 651 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 652
 653 my $net_fmt_bridge_descr = <<__EOD__;
 654 Bridge to attach the network device to. The Proxmox VE standard bridge
 655 is called 'vmbr0'.
 656
 657 If you do not specify a bridge, we create a kvm user (NATed) network
 658 device, which provides DHCP and DNS services. The following addresses
 659 are used:
 660
 661  10.0.2.2   Gateway
 662  10.0.2.3   DNS Server
 663  10.0.2.4   SMB Server
 664
 665 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 666 __EOD__
 667
 668 my $net_fmt = {
 669     macaddr => {
 670         type => 'string',
 671         pattern => qr/[0-9a-f]{2}(?::[0-9a-f]{2}){5}/i,
 672         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 673         format_description => "XX:XX:XX:XX:XX:XX",
 674         optional => 1,
 675     },
 676     model => {
 677         type => 'string',
 678         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 679         enum => $nic_model_list,
 680         default_key => 1,
 681     },
 682     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 683     bridge => {
 684         type => 'string',
 685         description => $net_fmt_bridge_descr,
 686         format_description => 'bridge',
 687         optional => 1,
 688     },
 689     queues => {
 690         type => 'integer',
 691         minimum => 0, maximum => 16,
 692         description => 'Number of packet queues to be used on the device.',
 693         optional => 1,
 694     },
 695     rate => {
 696         type => 'number',
 697         minimum => 0,
 698         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 699         optional => 1,
 700     },
 701     tag => {
 702         type => 'integer',
 703         minimum => 1, maximum => 4094,
 704         description => 'VLAN tag to apply to packets on this interface.',
 705         optional => 1,
 706     },
 707     trunks => {
 708         type => 'string',
 709         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 710         description => 'VLAN trunks to pass through this interface.',
 711         format_description => 'vlanid[;vlanid...]',
 712         optional => 1,
 713     },
 714     firewall => {
 715         type => 'boolean',
 716         description => 'Whether this interface should be protected by the firewall.',
 717         optional => 1,
 718     },
 719     link_down => {
 720         type => 'boolean',
 721         description => 'Whether this interface should be disconnected (like pulling the plug).',
 722         optional => 1,
 723     },
 724 };
 725
 726 my $netdesc = {
 727     optional => 1,
 728     type => 'string', format => $net_fmt,
 729     description => "Specify network devices.",
 730 };
 731
 732 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 733
 734 my $ipconfig_fmt = {
 735     ip => {
 736         type => 'string',
 737         format => 'pve-ipv4-config',
 738         format_description => 'IPv4Format/CIDR',
 739         description => 'IPv4 address in CIDR format.',
 740         optional => 1,
 741         default => 'dhcp',
 742     },
 743     gw => {
 744         type => 'string',
 745         format => 'ipv4',
 746         format_description => 'GatewayIPv4',
 747         description => 'Default gateway for IPv4 traffic.',
 748         optional => 1,
 749         requires => 'ip',
 750     },
 751     ip6 => {
 752         type => 'string',
 753         format => 'pve-ipv6-config',
 754         format_description => 'IPv6Format/CIDR',
 755         description => 'IPv6 address in CIDR format.',
 756         optional => 1,
 757         default => 'dhcp',
 758     },
 759     gw6 => {
 760         type => 'string',
 761         format => 'ipv6',
 762         format_description => 'GatewayIPv6',
 763         description => 'Default gateway for IPv6 traffic.',
 764         optional => 1,
 765         requires => 'ip6',
 766     },
 767 };
 768 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 769 my $ipconfigdesc = {
 770     optional => 1,
 771     type => 'string', format => 'pve-qm-ipconfig',
 772     description => <<'EODESCR',
 773 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 774
 775 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 776
 777 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
 778 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 779
 780 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
 781 EODESCR
 782 };
 783 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 784
 785 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 786     $confdesc->{"net$i"} = $netdesc;
 787     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 788 }
 789
 790 foreach my $key (keys %$confdesc_cloudinit) {
 791     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 792 }
 793
 794 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 795 sub verify_volume_id_or_qm_path {
 796     my ($volid, $noerr) = @_;
 797
 798     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 799         return $volid;
 800     }
 801
 802     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 803     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 804     if ($@) {
 805         return undef if $noerr;
 806         die $@;
 807     }
 808     return $volid;
 809 }
 810
 811 my $drivename_hash;
 812
 813 my %drivedesc_base = (
 814     volume => { alias => 'file' },
 815     file => {
 816         type => 'string',
 817         format => 'pve-volume-id-or-qm-path',
 818         default_key => 1,
 819         format_description => 'volume',
 820         description => "The drive's backing volume.",
 821     },
 822     media => {
 823         type => 'string',
 824         enum => [qw(cdrom disk)],
 825         description => "The drive's media type.",
 826         default => 'disk',
 827         optional => 1
 828     },
 829     cyls => {
 830         type => 'integer',
 831         description => "Force the drive's physical geometry to have a specific cylinder count.",
 832         optional => 1
 833     },
 834     heads => {
 835         type => 'integer',
 836         description => "Force the drive's physical geometry to have a specific head count.",
 837         optional => 1
 838     },
 839     secs => {
 840         type => 'integer',
 841         description => "Force the drive's physical geometry to have a specific sector count.",
 842         optional => 1
 843     },
 844     trans => {
 845         type => 'string',
 846         enum => [qw(none lba auto)],
 847         description => "Force disk geometry bios translation mode.",
 848         optional => 1,
 849     },
 850     snapshot => {
 851         type => 'boolean',
 852         description => "Controls qemu's snapshot mode feature."
 853             . " If activated, changes made to the disk are temporary and will"
 854             . " be discarded when the VM is shutdown.",
 855         optional => 1,
 856     },
 857     cache => {
 858         type => 'string',
 859         enum => [qw(none writethrough writeback unsafe directsync)],
 860         description => "The drive's cache mode",
 861         optional => 1,
 862     },
 863     format => get_standard_option('pve-qm-image-format'),
 864     size => {
 865         type => 'string',
 866         format => 'disk-size',
 867         format_description => 'DiskSize',
 868         description => "Disk size. This is purely informational and has no effect.",
 869         optional => 1,
 870     },
 871     backup => {
 872         type => 'boolean',
 873         description => "Whether the drive should be included when making backups.",
 874         optional => 1,
 875     },
 876     replicate => {
 877         type => 'boolean',
 878         description => 'Whether the drive should considered for replication jobs.',
 879         optional => 1,
 880         default => 1,
 881     },
 882     rerror => {
 883         type => 'string',
 884         enum => [qw(ignore report stop)],
 885         description => 'Read error action.',
 886         optional => 1,
 887     },
 888     werror => {
 889         type => 'string',
 890         enum => [qw(enospc ignore report stop)],
 891         description => 'Write error action.',
 892         optional => 1,
 893     },
 894     aio => {
 895         type => 'string',
 896         enum => [qw(native threads)],
 897         description => 'AIO type to use.',
 898         optional => 1,
 899     },
 900     discard => {
 901         type => 'string',
 902         enum => [qw(ignore on)],
 903         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 904         optional => 1,
 905     },
 906     detect_zeroes => {
 907         type => 'boolean',
 908         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 909         optional => 1,
 910     },
 911     serial => {
 912         type => 'string',
 913         format => 'urlencoded',
 914         format_description => 'serial',
 915         maxLength => 20*3, # *3 since it's %xx url enoded
 916         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
 917         optional => 1,
 918     },
 919     shared => {
 920         type => 'boolean',
 921         description => 'Mark this locally-managed volume as available on all nodes',
 922         verbose_description => "Mark this locally-managed volume as available on all nodes.\n\nWARNING: This option does not share the volume automatically, it assumes it is shared already!",
 923         optional => 1,
 924         default => 0,
 925     }
 926 );
 927
 928 my %iothread_fmt = ( iothread => {
 929         type => 'boolean',
 930         description => "Whether to use iothreads for this drive",
 931         optional => 1,
 932 });
 933
 934 my %model_fmt = (
 935     model => {
 936         type => 'string',
 937         format => 'urlencoded',
 938         format_description => 'model',
 939         maxLength => 40*3, # *3 since it's %xx url enoded
 940         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
 941         optional => 1,
 942     },
 943 );
 944
 945 my %queues_fmt = (
 946     queues => {
 947         type => 'integer',
 948         description => "Number of queues.",
 949         minimum => 2,
 950         optional => 1
 951     }
 952 );
 953
 954 my %scsiblock_fmt = (
 955     scsiblock => {
 956         type => 'boolean',
 957         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
 958         optional => 1,
 959         default => 0,
 960     },
 961 );
 962
 963 my $add_throttle_desc = sub {
 964     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
 965     my $d = {
 966         type => $type,
 967         format_description => $unit,
 968         description => "Maximum $what in $longunit.",
 969         optional => 1,
 970     };
 971     $d->{minimum} = $minimum if defined($minimum);
 972     $drivedesc_base{$key} = $d;
 973 };
 974 # throughput: (leaky bucket)
 975 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
 976 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
 977 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
 978 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
 979 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
 980 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
 981 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
 982 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
 983 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
 984
 985 # pools: (pool of IO before throttling starts taking effect)
 986 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
 987 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
 988 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
 989 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
 990 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
 991 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
 992
 993 # burst lengths
 994 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
 995 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
 996 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
 997 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
 998 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
 999 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1000
1001 # legacy support
1002 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
1003 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
1004 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
1005 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
1006
1007 my $ide_fmt = {
1008     %drivedesc_base,
1009     %model_fmt,
1010 };
1011 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
1012
1013 my $idedesc = {
1014     optional => 1,
1015     type => 'string', format => $ide_fmt,
1016     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
1017 };
1018 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
1019
1020 my $scsi_fmt = {
1021     %drivedesc_base,
1022     %iothread_fmt,
1023     %queues_fmt,
1024     %scsiblock_fmt,
1025 };
1026 my $scsidesc = {
1027     optional => 1,
1028     type => 'string', format => $scsi_fmt,
1029     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
1030 };
1031 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
1032
1033 my $sata_fmt = {
1034     %drivedesc_base,
1035 };
1036 my $satadesc = {
1037     optional => 1,
1038     type => 'string', format => $sata_fmt,
1039     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
1040 };
1041 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
1042
1043 my $virtio_fmt = {
1044     %drivedesc_base,
1045     %iothread_fmt,
1046 };
1047 my $virtiodesc = {
1048     optional => 1,
1049     type => 'string', format => $virtio_fmt,
1050     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
1051 };
1052 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
1053
1054 my $alldrive_fmt = {
1055     %drivedesc_base,
1056     %iothread_fmt,
1057     %model_fmt,
1058     %queues_fmt,
1059     %scsiblock_fmt,
1060 };
1061
1062 my $efidisk_fmt = {
1063     volume => { alias => 'file' },
1064     file => {
1065         type => 'string',
1066         format => 'pve-volume-id-or-qm-path',
1067         default_key => 1,
1068         format_description => 'volume',
1069         description => "The drive's backing volume.",
1070     },
1071     format => get_standard_option('pve-qm-image-format'),
1072     size => {
1073         type => 'string',
1074         format => 'disk-size',
1075         format_description => 'DiskSize',
1076         description => "Disk size. This is purely informational and has no effect.",
1077         optional => 1,
1078     },
1079 };
1080
1081 my $efidisk_desc = {
1082     optional => 1,
1083     type => 'string', format => $efidisk_fmt,
1084     description => "Configure a Disk for storing EFI vars",
1085 };
1086
1087 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
1088
1089 my $usb_fmt = {
1090     host => {
1091         default_key => 1,
1092         type => 'string', format => 'pve-qm-usb-device',
1093         format_description => 'HOSTUSBDEVICE|spice',
1094         description => <<EODESCR,
1095 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1096
1097  'bus-port(.port)*' (decimal numbers) or
1098  'vendor_id:product_id' (hexadeciaml numbers) or
1099  'spice'
1100
1101 You can use the 'lsusb -t' command to list existing usb devices.
1102
1103 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1104
1105 The value 'spice' can be used to add a usb redirection devices for spice.
1106 EODESCR
1107     },
1108     usb3 => {
1109         optional => 1,
1110         type => 'boolean',
1111         description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
1112         default => 0,
1113     },
1114 };
1115
1116 my $usbdesc = {
1117     optional => 1,
1118     type => 'string', format => $usb_fmt,
1119     description => "Configure an USB device (n is 0 to 4).",
1120 };
1121 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1122
1123 # NOTE: the match-groups of this regex are used in parse_hostpci
1124 my $PCIRE = qr/([a-f0-9]{2}:[a-f0-9]{2})(?:\.([a-f0-9]))?/;
1125 my $hostpci_fmt = {
1126     host => {
1127         default_key => 1,
1128         type => 'string',
1129         pattern => qr/$PCIRE(;$PCIRE)*/,
1130         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1131         description => <<EODESCR,
1132 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1133 of PCI virtual functions of the host. HOSTPCIID syntax is:
1134
1135 'bus:dev.func' (hexadecimal numbers)
1136
1137 You can us the 'lspci' command to list existing PCI devices.
1138 EODESCR
1139     },
1140     rombar => {
1141         type => 'boolean',
1142         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1143         optional => 1,
1144         default => 1,
1145     },
1146     romfile => {
1147         type => 'string',
1148         pattern => '[^,;]+',
1149         format_description => 'string',
1150         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1151         optional => 1,
1152     },
1153     pcie => {
1154         type => 'boolean',
1155         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1156         optional => 1,
1157         default => 0,
1158     },
1159     'x-vga' => {
1160         type => 'boolean',
1161         description =>  "Enable vfio-vga device support.",
1162         optional => 1,
1163         default => 0,
1164     },
1165 };
1166 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1167
1168 my $hostpcidesc = {
1169         optional => 1,
1170         type => 'string', format => 'pve-qm-hostpci',
1171         description => "Map host PCI devices into guest.",
1172         verbose_description =>  <<EODESCR,
1173 Map host PCI devices into guest.
1174
1175 NOTE: This option allows direct access to host hardware. So it is no longer
1176 possible to migrate such machines - use with special care.
1177
1178 CAUTION: Experimental! User reported problems with this option.
1179 EODESCR
1180 };
1181 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1182
1183 my $serialdesc = {
1184         optional => 1,
1185         type => 'string',
1186         pattern => '(/dev/.+|socket)',
1187         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1188         verbose_description =>  <<EODESCR,
1189 Create a serial device inside the VM (n is 0 to 3), and pass through a
1190 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1191 host side (use 'qm terminal' to open a terminal connection).
1192
1193 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1194
1195 CAUTION: Experimental! User reported problems with this option.
1196 EODESCR
1197 };
1198
1199 my $paralleldesc= {
1200         optional => 1,
1201         type => 'string',
1202         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1203         description =>  "Map host parallel devices (n is 0 to 2).",
1204         verbose_description =>  <<EODESCR,
1205 Map host parallel devices (n is 0 to 2).
1206
1207 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1208
1209 CAUTION: Experimental! User reported problems with this option.
1210 EODESCR
1211 };
1212
1213 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1214     $confdesc->{"parallel$i"} = $paralleldesc;
1215 }
1216
1217 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1218     $confdesc->{"serial$i"} = $serialdesc;
1219 }
1220
1221 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1222     $confdesc->{"hostpci$i"} = $hostpcidesc;
1223 }
1224
1225 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1226     $drivename_hash->{"ide$i"} = 1;
1227     $confdesc->{"ide$i"} = $idedesc;
1228 }
1229
1230 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1231     $drivename_hash->{"sata$i"} = 1;
1232     $confdesc->{"sata$i"} = $satadesc;
1233 }
1234
1235 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1236     $drivename_hash->{"scsi$i"} = 1;
1237     $confdesc->{"scsi$i"} = $scsidesc ;
1238 }
1239
1240 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1241     $drivename_hash->{"virtio$i"} = 1;
1242     $confdesc->{"virtio$i"} = $virtiodesc;
1243 }
1244
1245 $drivename_hash->{efidisk0} = 1;
1246 $confdesc->{efidisk0} = $efidisk_desc;
1247
1248 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1249     $confdesc->{"usb$i"} = $usbdesc;
1250 }
1251
1252 my $unuseddesc = {
1253     optional => 1,
1254     type => 'string', format => 'pve-volume-id',
1255     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1256 };
1257
1258 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1259     $confdesc->{"unused$i"} = $unuseddesc;
1260 }
1261
1262 my $kvm_api_version = 0;
1263
1264 sub kvm_version {
1265
1266     return $kvm_api_version if $kvm_api_version;
1267
1268     my $fh = IO::File->new("</dev/kvm") ||
1269         return 0;
1270
1271     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
1272         $kvm_api_version = $v;
1273     }
1274
1275     $fh->close();
1276
1277     return  $kvm_api_version;
1278 }
1279
1280 my $kvm_user_version;
1281
1282 sub kvm_user_version {
1283
1284     return $kvm_user_version if $kvm_user_version;
1285
1286     $kvm_user_version = 'unknown';
1287
1288     my $code = sub {
1289         my $line = shift;
1290         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1291             $kvm_user_version = $2;
1292         }
1293     };
1294
1295     eval { run_command("kvm -version", outfunc => $code); };
1296     warn $@ if $@;
1297
1298     return $kvm_user_version;
1299
1300 }
1301
1302 my $kernel_has_vhost_net = -c '/dev/vhost-net';
1303
1304 sub valid_drive_names {
1305     # order is important - used to autoselect boot disk
1306     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1307             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1308             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1309             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1310             'efidisk0');
1311 }
1312
1313 sub is_valid_drivename {
1314     my $dev = shift;
1315
1316     return defined($drivename_hash->{$dev});
1317 }
1318
1319 sub option_exists {
1320     my $key = shift;
1321     return defined($confdesc->{$key});
1322 }
1323
1324 sub nic_models {
1325     return $nic_model_list;
1326 }
1327
1328 sub os_list_description {
1329
1330     return {
1331         other => 'Other',
1332         wxp => 'Windows XP',
1333         w2k => 'Windows 2000',
1334         w2k3 =>, 'Windows 2003',
1335         w2k8 => 'Windows 2008',
1336         wvista => 'Windows Vista',
1337         win7 => 'Windows 7',
1338         win8 => 'Windows 8/2012',
1339         win10 => 'Windows 10/2016',
1340         l24 => 'Linux 2.4',
1341         l26 => 'Linux 2.6',
1342     };
1343 }
1344
1345 my $cdrom_path;
1346
1347 sub get_cdrom_path {
1348
1349     return  $cdrom_path if $cdrom_path;
1350
1351     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1352     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1353     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1354 }
1355
1356 sub get_iso_path {
1357     my ($storecfg, $vmid, $cdrom) = @_;
1358
1359     if ($cdrom eq 'cdrom') {
1360         return get_cdrom_path();
1361     } elsif ($cdrom eq 'none') {
1362         return '';
1363     } elsif ($cdrom =~ m|^/|) {
1364         return $cdrom;
1365     } else {
1366         return PVE::Storage::path($storecfg, $cdrom);
1367     }
1368 }
1369
1370 # try to convert old style file names to volume IDs
1371 sub filename_to_volume_id {
1372     my ($vmid, $file, $media) = @_;
1373
1374      if (!($file eq 'none' || $file eq 'cdrom' ||
1375           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1376
1377         return undef if $file =~ m|/|;
1378
1379         if ($media && $media eq 'cdrom') {
1380             $file = "local:iso/$file";
1381         } else {
1382             $file = "local:$vmid/$file";
1383         }
1384     }
1385
1386     return $file;
1387 }
1388
1389 sub verify_media_type {
1390     my ($opt, $vtype, $media) = @_;
1391
1392     return if !$media;
1393
1394     my $etype;
1395     if ($media eq 'disk') {
1396         $etype = 'images';
1397     } elsif ($media eq 'cdrom') {
1398         $etype = 'iso';
1399     } else {
1400         die "internal error";
1401     }
1402
1403     return if ($vtype eq $etype);
1404
1405     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1406 }
1407
1408 sub cleanup_drive_path {
1409     my ($opt, $storecfg, $drive) = @_;
1410
1411     # try to convert filesystem paths to volume IDs
1412
1413     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1414         ($drive->{file} !~ m|^/dev/.+|) &&
1415         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1416         ($drive->{file} !~ m/^\d+$/)) {
1417         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1418         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1419         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1420         verify_media_type($opt, $vtype, $drive->{media});
1421         $drive->{file} = $volid;
1422     }
1423
1424     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1425 }
1426
1427 sub parse_hotplug_features {
1428     my ($data) = @_;
1429
1430     my $res = {};
1431
1432     return $res if $data eq '0';
1433
1434     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1435
1436     foreach my $feature (PVE::Tools::split_list($data)) {
1437         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1438             $res->{$1} = 1;
1439         } else {
1440             die "invalid hotplug feature '$feature'\n";
1441         }
1442     }
1443     return $res;
1444 }
1445
1446 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1447 sub pve_verify_hotplug_features {
1448     my ($value, $noerr) = @_;
1449
1450     return $value if parse_hotplug_features($value);
1451
1452     return undef if $noerr;
1453
1454     die "unable to parse hotplug option\n";
1455 }
1456
1457 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1458 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1459 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1460 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1461 #        [,iothread=on][,serial=serial][,model=model]
1462
1463 sub parse_drive {
1464     my ($key, $data) = @_;
1465
1466     my ($interface, $index);
1467
1468     if ($key =~ m/^([^\d]+)(\d+)$/) {
1469         $interface = $1;
1470         $index = $2;
1471     } else {
1472         return undef;
1473     }
1474
1475     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1476                                      : $confdesc->{$key}->{format};
1477     if (!$desc) {
1478         warn "invalid drive key: $key\n";
1479         return undef;
1480     }
1481     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1482     return undef if !$res;
1483     $res->{interface} = $interface;
1484     $res->{index} = $index;
1485
1486     my $error = 0;
1487     foreach my $opt (qw(bps bps_rd bps_wr)) {
1488         if (my $bps = defined(delete $res->{$opt})) {
1489             if (defined($res->{"m$opt"})) {
1490                 warn "both $opt and m$opt specified\n";
1491                 ++$error;
1492                 next;
1493             }
1494             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1495         }
1496     }
1497
1498     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1499     for my $requirement (
1500         [mbps_max => 'mbps'],
1501         [mbps_rd_max => 'mbps_rd'],
1502         [mbps_wr_max => 'mbps_wr'],
1503         [miops_max => 'miops'],
1504         [miops_rd_max => 'miops_rd'],
1505         [miops_wr_max => 'miops_wr'],
1506         [bps_max_length => 'mbps_max'],
1507         [bps_rd_max_length => 'mbps_rd_max'],
1508         [bps_wr_max_length => 'mbps_wr_max'],
1509         [iops_max_length => 'iops_max'],
1510         [iops_rd_max_length => 'iops_rd_max'],
1511         [iops_wr_max_length => 'iops_wr_max']) {
1512         my ($option, $requires) = @$requirement;
1513         if ($res->{$option} && !$res->{$requires}) {
1514             warn "$option requires $requires\n";
1515             ++$error;
1516         }
1517     }
1518
1519     return undef if $error;
1520
1521     return undef if $res->{mbps_rd} && $res->{mbps};
1522     return undef if $res->{mbps_wr} && $res->{mbps};
1523     return undef if $res->{iops_rd} && $res->{iops};
1524     return undef if $res->{iops_wr} && $res->{iops};
1525
1526     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1527         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1528         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1529         return undef if $res->{interface} eq 'virtio';
1530     }
1531
1532     if (my $size = $res->{size}) {
1533         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1534     }
1535
1536     return $res;
1537 }
1538
1539 sub print_drive {
1540     my ($vmid, $drive) = @_;
1541     my $data = { %$drive };
1542     delete $data->{$_} for qw(index interface);
1543     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1544 }
1545
1546 sub scsi_inquiry {
1547     my($fh, $noerr) = @_;
1548
1549     my $SG_IO = 0x2285;
1550     my $SG_GET_VERSION_NUM = 0x2282;
1551
1552     my $versionbuf = "\x00" x 8;
1553     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1554     if (!$ret) {
1555         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1556         return undef;
1557     }
1558     my $version = unpack("I", $versionbuf);
1559     if ($version < 30000) {
1560         die "scsi generic interface too old\n"  if !$noerr;
1561         return undef;
1562     }
1563
1564     my $buf = "\x00" x 36;
1565     my $sensebuf = "\x00" x 8;
1566     my $cmd = pack("C x3 C x1", 0x12, 36);
1567
1568     # see /usr/include/scsi/sg.h
1569     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1570
1571     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1572                       length($sensebuf), 0, length($buf), $buf,
1573                       $cmd, $sensebuf, 6000);
1574
1575     $ret = ioctl($fh, $SG_IO, $packet);
1576     if (!$ret) {
1577         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1578         return undef;
1579     }
1580
1581     my @res = unpack($sg_io_hdr_t, $packet);
1582     if ($res[17] || $res[18]) {
1583         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1584         return undef;
1585     }
1586
1587     my $res = {};
1588     (my $byte0, my $byte1, $res->{vendor},
1589      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1590
1591     $res->{removable} = $byte1 & 128 ? 1 : 0;
1592     $res->{type} = $byte0 & 31;
1593
1594     return $res;
1595 }
1596
1597 sub path_is_scsi {
1598     my ($path) = @_;
1599
1600     my $fh = IO::File->new("+<$path") || return undef;
1601     my $res = scsi_inquiry($fh, 1);
1602     close($fh);
1603
1604     return $res;
1605 }
1606
1607 sub machine_type_is_q35 {
1608     my ($conf) = @_;
1609
1610     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1611 }
1612
1613 sub print_tabletdevice_full {
1614     my ($conf) = @_;
1615
1616     my $q35 = machine_type_is_q35($conf);
1617
1618     # we use uhci for old VMs because tablet driver was buggy in older qemu
1619     my $usbbus = $q35 ? "ehci" : "uhci";
1620
1621     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1622 }
1623
1624 sub print_drivedevice_full {
1625     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1626
1627     my $device = '';
1628     my $maxdev = 0;
1629
1630     if ($drive->{interface} eq 'virtio') {
1631         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1632         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1633         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1634     } elsif ($drive->{interface} eq 'scsi') {
1635
1636         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1637         my $unit = $drive->{index} % $maxdev;
1638         my $devicetype = 'hd';
1639         my $path = '';
1640         if (drive_is_cdrom($drive)) {
1641             $devicetype = 'cd';
1642         } else {
1643             if ($drive->{file} =~ m|^/|) {
1644                 $path = $drive->{file};
1645                 if (my $info = path_is_scsi($path)) {
1646                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1647                         $devicetype = 'block';
1648                     } elsif ($info->{type} == 1) { # tape
1649                         $devicetype = 'generic';
1650                     }
1651                 }
1652             } else {
1653                  $path = PVE::Storage::path($storecfg, $drive->{file});
1654             }
1655
1656             if($path =~ m/^iscsi\:\/\//){
1657                 $devicetype = 'generic';
1658             }
1659         }
1660
1661         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1662             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1663         } else {
1664             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1665         }
1666
1667     } elsif ($drive->{interface} eq 'ide'){
1668         $maxdev = 2;
1669         my $controller = int($drive->{index} / $maxdev);
1670         my $unit = $drive->{index} % $maxdev;
1671         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1672
1673         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1674         if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
1675             $model = URI::Escape::uri_unescape($model);
1676             $device .= ",model=$model";
1677         }
1678     } elsif ($drive->{interface} eq 'sata'){
1679         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1680         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1681         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1682     } elsif ($drive->{interface} eq 'usb') {
1683         die "implement me";
1684         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1685     } else {
1686         die "unsupported interface type";
1687     }
1688
1689     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1690
1691     return $device;
1692 }
1693
1694 sub get_initiator_name {
1695     my $initiator;
1696
1697     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1698     while (defined(my $line = <$fh>)) {
1699         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1700         $initiator = $1;
1701         last;
1702     }
1703     $fh->close();
1704
1705     return $initiator;
1706 }
1707
1708 sub print_drive_full {
1709     my ($storecfg, $vmid, $drive) = @_;
1710
1711     my $path;
1712     my $volid = $drive->{file};
1713     my $format;
1714
1715     if (drive_is_cdrom($drive)) {
1716         $path = get_iso_path($storecfg, $vmid, $volid);
1717     } else {
1718         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1719         if ($storeid) {
1720             $path = PVE::Storage::path($storecfg, $volid);
1721             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1722             $format = qemu_img_format($scfg, $volname);
1723         } else {
1724             $path = $volid;
1725             $format = "raw";
1726         }
1727    }
1728
1729     my $opts = '';
1730     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1731     foreach my $o (@qemu_drive_options) {
1732         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1733     }
1734
1735     # snapshot only accepts on|off
1736     if (defined($drive->{snapshot})) {
1737         my $v = $drive->{snapshot} ? 'on' : 'off';
1738         $opts .= ",snapshot=$v";
1739     }
1740
1741     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1742         my ($dir, $qmpname) = @$type;
1743         if (my $v = $drive->{"mbps$dir"}) {
1744             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1745         }
1746         if (my $v = $drive->{"mbps${dir}_max"}) {
1747             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1748         }
1749         if (my $v = $drive->{"bps${dir}_max_length"}) {
1750             $opts .= ",throttling.bps$qmpname-max-length=$v";
1751         }
1752         if (my $v = $drive->{"iops${dir}"}) {
1753             $opts .= ",throttling.iops$qmpname=$v";
1754         }
1755         if (my $v = $drive->{"iops${dir}_max"}) {
1756             $opts .= ",throttling.iops$qmpname-max=$v";
1757         }
1758         if (my $v = $drive->{"iops${dir}_max_length"}) {
1759             $opts .= ",throttling.iops$qmpname-max-length=$v";
1760         }
1761     }
1762
1763     if (my $serial = $drive->{serial}) {
1764         $serial = URI::Escape::uri_unescape($serial);
1765         $opts .= ",serial=$serial";
1766     }
1767
1768     $opts .= ",format=$format" if $format && !$drive->{format};
1769
1770     my $cache_direct = 0;
1771
1772     if (my $cache = $drive->{cache}) {
1773         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1774     } elsif (!drive_is_cdrom($drive)) {
1775         $opts .= ",cache=none";
1776         $cache_direct = 1;
1777     }
1778
1779     # aio native works only with O_DIRECT
1780     if (!$drive->{aio}) {
1781         if($cache_direct) {
1782             $opts .= ",aio=native";
1783         } else {
1784             $opts .= ",aio=threads";
1785         }
1786     }
1787
1788     if (!drive_is_cdrom($drive)) {
1789         my $detectzeroes;
1790         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1791             $detectzeroes = 'off';
1792         } elsif ($drive->{discard}) {
1793             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1794         } else {
1795             # This used to be our default with discard not being specified:
1796             $detectzeroes = 'on';
1797         }
1798         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1799     }
1800
1801     my $pathinfo = $path ? "file=$path," : '';
1802
1803     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1804 }
1805
1806 sub print_netdevice_full {
1807     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1808
1809     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1810
1811     my $device = $net->{model};
1812     if ($net->{model} eq 'virtio') {
1813          $device = 'virtio-net-pci';
1814      };
1815
1816     my $pciaddr = print_pci_addr("$netid", $bridges);
1817     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1818     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1819         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1820         my $vectors = $net->{queues} * 2 + 2;
1821         $tmpstr .= ",vectors=$vectors,mq=on";
1822     }
1823     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1824
1825     if ($use_old_bios_files) {
1826         my $romfile;
1827         if ($device eq 'virtio-net-pci') {
1828             $romfile = 'pxe-virtio.rom';
1829         } elsif ($device eq 'e1000') {
1830             $romfile = 'pxe-e1000.rom';
1831         } elsif ($device eq 'ne2k') {
1832             $romfile = 'pxe-ne2k_pci.rom';
1833         } elsif ($device eq 'pcnet') {
1834             $romfile = 'pxe-pcnet.rom';
1835         } elsif ($device eq 'rtl8139') {
1836             $romfile = 'pxe-rtl8139.rom';
1837         }
1838         $tmpstr .= ",romfile=$romfile" if $romfile;
1839     }
1840
1841     return $tmpstr;
1842 }
1843
1844 sub print_netdev_full {
1845     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1846
1847     my $i = '';
1848     if ($netid =~ m/^net(\d+)$/) {
1849         $i = int($1);
1850     }
1851
1852     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1853
1854     my $ifname = "tap${vmid}i$i";
1855
1856     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1857     die "interface name '$ifname' is too long (max 15 character)\n"
1858         if length($ifname) >= 16;
1859
1860     my $vhostparam = '';
1861     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1862
1863     my $vmname = $conf->{name} || "vm$vmid";
1864
1865     my $netdev = "";
1866     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1867
1868     if ($net->{bridge}) {
1869         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1870     } else {
1871         $netdev = "type=user,id=$netid,hostname=$vmname";
1872     }
1873
1874     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1875
1876     return $netdev;
1877 }
1878
1879
1880 sub print_cpu_device {
1881     my ($conf, $id) = @_;
1882
1883     my $kvm = $conf->{kvm} // 1;
1884     my $cpu = $kvm ? "kvm64" : "qemu64";
1885     if (my $cputype = $conf->{cpu}) {
1886         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
1887             or die "Cannot parse cpu description: $cputype\n";
1888         $cpu = $cpuconf->{cputype};
1889     }
1890
1891     my $cores = $conf->{cores} || 1;
1892
1893     my $current_core = ($id - 1) % $cores;
1894     my $current_socket = int(($id - 1 - $current_core)/$cores);
1895
1896     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
1897 }
1898
1899 sub drive_is_cloudinit {
1900     my ($drive) = @_;
1901     return $drive->{file} =~ m@[:/]vm-\d+-cloudinit(?:\.$QEMU_FORMAT_RE)?$@;
1902 }
1903
1904 sub drive_is_cdrom {
1905     my ($drive, $exclude_cloudinit) = @_;
1906
1907     return 0 if $exclude_cloudinit && drive_is_cloudinit($drive);
1908
1909     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1910
1911 }
1912
1913 sub parse_number_sets {
1914     my ($set) = @_;
1915     my $res = [];
1916     foreach my $part (split(/;/, $set)) {
1917         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1918             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1919             push @$res, [ $1, $2 ];
1920         } else {
1921             die "invalid range: $part\n";
1922         }
1923     }
1924     return $res;
1925 }
1926
1927 sub parse_numa {
1928     my ($data) = @_;
1929
1930     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
1931     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1932     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1933     return $res;
1934 }
1935
1936 sub parse_hostpci {
1937     my ($value) = @_;
1938
1939     return undef if !$value;
1940
1941     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
1942
1943     my @idlist = split(/;/, $res->{host});
1944     delete $res->{host};
1945     foreach my $id (@idlist) {
1946         if ($id =~ /^$PCIRE$/) {
1947             if (defined($2)) {
1948                 push @{$res->{pciid}}, { id => $1, function => $2 };
1949             } else {
1950                 my $pcidevices = lspci($1);
1951                 $res->{pciid} = $pcidevices->{$1};
1952             }
1953         } else {
1954             # should have been caught by parse_property_string already
1955             die "failed to parse PCI id: $id\n";
1956         }
1957     }
1958     return $res;
1959 }
1960
1961 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1962 sub parse_net {
1963     my ($data) = @_;
1964
1965     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
1966     if ($@) {
1967         warn $@;
1968         return undef;
1969     }
1970     if (!defined($res->{macaddr})) {
1971         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1972         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1973     }
1974     return $res;
1975 }
1976
1977 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1978 sub parse_ipconfig {
1979     my ($data) = @_;
1980
1981     my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
1982     if ($@) {
1983         warn $@;
1984         return undef;
1985     }
1986
1987     if ($res->{gw} && !$res->{ip}) {
1988         warn 'gateway specified without specifying an IP address';
1989         return undef;
1990     }
1991     if ($res->{gw6} && !$res->{ip6}) {
1992         warn 'IPv6 gateway specified without specifying an IPv6 address';
1993         return undef;
1994     }
1995     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1996         warn 'gateway specified together with DHCP';
1997         return undef;
1998     }
1999     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
2000         # gw6 + auto/dhcp
2001         warn "IPv6 gateway specified together with $res->{ip6} address";
2002         return undef;
2003     }
2004
2005     if (!$res->{ip} && !$res->{ip6}) {
2006         return { ip => 'dhcp', ip6 => 'dhcp' };
2007     }
2008
2009     return $res;
2010 }
2011
2012 sub print_net {
2013     my $net = shift;
2014
2015     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2016 }
2017
2018 sub add_random_macs {
2019     my ($settings) = @_;
2020
2021     foreach my $opt (keys %$settings) {
2022         next if $opt !~ m/^net(\d+)$/;
2023         my $net = parse_net($settings->{$opt});
2024         next if !$net;
2025         $settings->{$opt} = print_net($net);
2026     }
2027 }
2028
2029 sub vm_is_volid_owner {
2030     my ($storecfg, $vmid, $volid) = @_;
2031
2032     if ($volid !~  m|^/|) {
2033         my ($path, $owner);
2034         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2035         if ($owner && ($owner == $vmid)) {
2036             return 1;
2037         }
2038     }
2039
2040     return undef;
2041 }
2042
2043 sub split_flagged_list {
2044     my $text = shift || '';
2045     $text =~ s/[,;]/ /g;
2046     $text =~ s/^\s+//;
2047     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
2048 }
2049
2050 sub join_flagged_list {
2051     my ($how, $lst) = @_;
2052     join $how, map { $lst->{$_} . $_ } keys %$lst;
2053 }
2054
2055 sub vmconfig_delete_pending_option {
2056     my ($conf, $key, $force) = @_;
2057
2058     delete $conf->{pending}->{$key};
2059     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2060     $pending_delete_hash->{$key} = $force ? '!' : '';
2061     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2062 }
2063
2064 sub vmconfig_undelete_pending_option {
2065     my ($conf, $key) = @_;
2066
2067     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2068     delete $pending_delete_hash->{$key};
2069
2070     if (%$pending_delete_hash) {
2071         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2072     } else {
2073         delete $conf->{pending}->{delete};
2074     }
2075 }
2076
2077 sub vmconfig_register_unused_drive {
2078     my ($storecfg, $vmid, $conf, $drive) = @_;
2079
2080     if (drive_is_cloudinit($drive)) {
2081         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2082         warn $@ if $@;
2083     } elsif (!drive_is_cdrom($drive)) {
2084         my $volid = $drive->{file};
2085         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2086             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2087         }
2088     }
2089 }
2090
2091 sub vmconfig_cleanup_pending {
2092     my ($conf) = @_;
2093
2094     # remove pending changes when nothing changed
2095     my $changes;
2096     foreach my $opt (keys %{$conf->{pending}}) {
2097         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
2098             $changes = 1;
2099             delete $conf->{pending}->{$opt};
2100         }
2101     }
2102
2103     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
2104     my $pending_delete_hash = {};
2105     while (my ($opt, $force) = each %$current_delete_hash) {
2106         if (defined($conf->{$opt})) {
2107             $pending_delete_hash->{$opt} = $force;
2108         } else {
2109             $changes = 1;
2110         }
2111     }
2112
2113     if (%$pending_delete_hash) {
2114         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2115     } else {
2116         delete $conf->{pending}->{delete};
2117     }
2118
2119     return $changes;
2120 }
2121
2122 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
2123 my $smbios1_fmt = {
2124     uuid => {
2125         type => 'string',
2126         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2127         format_description => 'UUID',
2128         description => "Set SMBIOS1 UUID.",
2129         optional => 1,
2130     },
2131     version => {
2132         type => 'string',
2133         pattern => '\S+',
2134         format_description => 'string',
2135         description => "Set SMBIOS1 version.",
2136         optional => 1,
2137     },
2138     serial => {
2139         type => 'string',
2140         pattern => '\S+',
2141         format_description => 'string',
2142         description => "Set SMBIOS1 serial number.",
2143         optional => 1,
2144     },
2145     manufacturer => {
2146         type => 'string',
2147         pattern => '\S+',
2148         format_description => 'string',
2149         description => "Set SMBIOS1 manufacturer.",
2150         optional => 1,
2151     },
2152     product => {
2153         type => 'string',
2154         pattern => '\S+',
2155         format_description => 'string',
2156         description => "Set SMBIOS1 product ID.",
2157         optional => 1,
2158     },
2159     sku => {
2160         type => 'string',
2161         pattern => '\S+',
2162         format_description => 'string',
2163         description => "Set SMBIOS1 SKU string.",
2164         optional => 1,
2165     },
2166     family => {
2167         type => 'string',
2168         pattern => '\S+',
2169         format_description => 'string',
2170         description => "Set SMBIOS1 family string.",
2171         optional => 1,
2172     },
2173 };
2174
2175 sub parse_smbios1 {
2176     my ($data) = @_;
2177
2178     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
2179     warn $@ if $@;
2180     return $res;
2181 }
2182
2183 sub print_smbios1 {
2184     my ($smbios1) = @_;
2185     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2186 }
2187
2188 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2189
2190 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2191 sub verify_bootdisk {
2192     my ($value, $noerr) = @_;
2193
2194     return $value if is_valid_drivename($value);
2195
2196     return undef if $noerr;
2197
2198     die "invalid boot disk '$value'\n";
2199 }
2200
2201 sub parse_watchdog {
2202     my ($value) = @_;
2203
2204     return undef if !$value;
2205
2206     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2207     warn $@ if $@;
2208     return $res;
2209 }
2210
2211 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2212 sub verify_usb_device {
2213     my ($value, $noerr) = @_;
2214
2215     return $value if parse_usb_device($value);
2216
2217     return undef if $noerr;
2218
2219     die "unable to parse usb device\n";
2220 }
2221
2222 # add JSON properties for create and set function
2223 sub json_config_properties {
2224     my $prop = shift;
2225
2226     foreach my $opt (keys %$confdesc) {
2227         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
2228         $prop->{$opt} = $confdesc->{$opt};
2229     }
2230
2231     return $prop;
2232 }
2233
2234 # return copy of $confdesc_cloudinit to generate documentation
2235 sub cloudinit_config_properties {
2236
2237     return dclone($confdesc_cloudinit);
2238 }
2239
2240 sub check_type {
2241     my ($key, $value) = @_;
2242
2243     die "unknown setting '$key'\n" if !$confdesc->{$key};
2244
2245     my $type = $confdesc->{$key}->{type};
2246
2247     if (!defined($value)) {
2248         die "got undefined value\n";
2249     }
2250
2251     if ($value =~ m/[\n\r]/) {
2252         die "property contains a line feed\n";
2253     }
2254
2255     if ($type eq 'boolean') {
2256         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2257         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2258         die "type check ('boolean') failed - got '$value'\n";
2259     } elsif ($type eq 'integer') {
2260         return int($1) if $value =~ m/^(\d+)$/;
2261         die "type check ('integer') failed - got '$value'\n";
2262     } elsif ($type eq 'number') {
2263         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2264         die "type check ('number') failed - got '$value'\n";
2265     } elsif ($type eq 'string') {
2266         if (my $fmt = $confdesc->{$key}->{format}) {
2267             PVE::JSONSchema::check_format($fmt, $value);
2268             return $value;
2269         }
2270         $value =~ s/^\"(.*)\"$/$1/;
2271         return $value;
2272     } else {
2273         die "internal error"
2274     }
2275 }
2276
2277 sub check_iommu_support{
2278     #fixme : need to check IOMMU support
2279     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
2280
2281     my $iommu=1;
2282     return $iommu;
2283
2284 }
2285
2286 sub touch_config {
2287     my ($vmid) = @_;
2288
2289     my $conf = PVE::QemuConfig->config_file($vmid);
2290     utime undef, undef, $conf;
2291 }
2292
2293 sub destroy_vm {
2294     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2295
2296     my $conffile = PVE::QemuConfig->config_file($vmid);
2297
2298     my $conf = PVE::QemuConfig->load_config($vmid);
2299
2300     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2301
2302     if ($conf->{template}) {
2303         # check if any base image is still used by a linked clone
2304         foreach_drive($conf, sub {
2305                 my ($ds, $drive) = @_;
2306
2307                 return if drive_is_cdrom($drive);
2308
2309                 my $volid = $drive->{file};
2310
2311                 return if !$volid || $volid =~ m|^/|;
2312
2313                 die "base volume '$volid' is still in use by linked cloned\n"
2314                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2315
2316         });
2317     }
2318
2319     # only remove disks owned by this VM
2320     foreach_drive($conf, sub {
2321         my ($ds, $drive) = @_;
2322
2323         return if drive_is_cdrom($drive, 1);
2324
2325         my $volid = $drive->{file};
2326
2327         return if !$volid || $volid =~ m|^/|;
2328
2329         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2330         return if !$path || !$owner || ($owner != $vmid);
2331
2332         eval {
2333             PVE::Storage::vdisk_free($storecfg, $volid);
2334         };
2335         warn "Could not remove disk '$volid', check manually: $@" if $@;
2336
2337     });
2338
2339     if ($keep_empty_config) {
2340         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2341     } else {
2342         unlink $conffile;
2343     }
2344
2345     # also remove unused disk
2346     eval {
2347         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2348
2349         eval {
2350             PVE::Storage::foreach_volid($dl, sub {
2351                 my ($volid, $sid, $volname, $d) = @_;
2352                 PVE::Storage::vdisk_free($storecfg, $volid);
2353             });
2354         };
2355         warn $@ if $@;
2356
2357     };
2358     warn $@ if $@;
2359 }
2360
2361 sub parse_vm_config {
2362     my ($filename, $raw) = @_;
2363
2364     return undef if !defined($raw);
2365
2366     my $res = {
2367         digest => Digest::SHA::sha1_hex($raw),
2368         snapshots => {},
2369         pending => {},
2370     };
2371
2372     $filename =~ m|/qemu-server/(\d+)\.conf$|
2373         || die "got strange filename '$filename'";
2374
2375     my $vmid = $1;
2376
2377     my $conf = $res;
2378     my $descr;
2379     my $section = '';
2380
2381     my @lines = split(/\n/, $raw);
2382     foreach my $line (@lines) {
2383         next if $line =~ m/^\s*$/;
2384
2385         if ($line =~ m/^\[PENDING\]\s*$/i) {
2386             $section = 'pending';
2387             if (defined($descr)) {
2388                 $descr =~ s/\s+$//;
2389                 $conf->{description} = $descr;
2390             }
2391             $descr = undef;
2392             $conf = $res->{$section} = {};
2393             next;
2394
2395         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2396             $section = $1;
2397             if (defined($descr)) {
2398                 $descr =~ s/\s+$//;
2399                 $conf->{description} = $descr;
2400             }
2401             $descr = undef;
2402             $conf = $res->{snapshots}->{$section} = {};
2403             next;
2404         }
2405
2406         if ($line =~ m/^\#(.*)\s*$/) {
2407             $descr = '' if !defined($descr);
2408             $descr .= PVE::Tools::decode_text($1) . "\n";
2409             next;
2410         }
2411
2412         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2413             $descr = '' if !defined($descr);
2414             $descr .= PVE::Tools::decode_text($2);
2415         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2416             $conf->{snapstate} = $1;
2417         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2418             my $key = $1;
2419             my $value = $2;
2420             $conf->{$key} = $value;
2421         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2422             my $value = $1;
2423             if ($section eq 'pending') {
2424                 $conf->{delete} = $value; # we parse this later
2425             } else {
2426                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2427             }
2428         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2429             my $key = $1;
2430             my $value = $2;
2431             eval { $value = check_type($key, $value); };
2432             if ($@) {
2433                 warn "vm $vmid - unable to parse value of '$key' - $@";
2434             } else {
2435                 $key = 'ide2' if $key eq 'cdrom';
2436                 my $fmt = $confdesc->{$key}->{format};
2437                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2438                     my $v = parse_drive($key, $value);
2439                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2440                         $v->{file} = $volid;
2441                         $value = print_drive($vmid, $v);
2442                     } else {
2443                         warn "vm $vmid - unable to parse value of '$key'\n";
2444                         next;
2445                     }
2446                 }
2447
2448                 $conf->{$key} = $value;
2449             }
2450         }
2451     }
2452
2453     if (defined($descr)) {
2454         $descr =~ s/\s+$//;
2455         $conf->{description} = $descr;
2456     }
2457     delete $res->{snapstate}; # just to be sure
2458
2459     return $res;
2460 }
2461
2462 sub write_vm_config {
2463     my ($filename, $conf) = @_;
2464
2465     delete $conf->{snapstate}; # just to be sure
2466
2467     if ($conf->{cdrom}) {
2468         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2469         $conf->{ide2} = $conf->{cdrom};
2470         delete $conf->{cdrom};
2471     }
2472
2473     # we do not use 'smp' any longer
2474     if ($conf->{sockets}) {
2475         delete $conf->{smp};
2476     } elsif ($conf->{smp}) {
2477         $conf->{sockets} = $conf->{smp};
2478         delete $conf->{cores};
2479         delete $conf->{smp};
2480     }
2481
2482     my $used_volids = {};
2483
2484     my $cleanup_config = sub {
2485         my ($cref, $pending, $snapname) = @_;
2486
2487         foreach my $key (keys %$cref) {
2488             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2489                 $key eq 'snapstate' || $key eq 'pending';
2490             my $value = $cref->{$key};
2491             if ($key eq 'delete') {
2492                 die "propertry 'delete' is only allowed in [PENDING]\n"
2493                     if !$pending;
2494                 # fixme: check syntax?
2495                 next;
2496             }
2497             eval { $value = check_type($key, $value); };
2498             die "unable to parse value of '$key' - $@" if $@;
2499
2500             $cref->{$key} = $value;
2501
2502             if (!$snapname && is_valid_drivename($key)) {
2503                 my $drive = parse_drive($key, $value);
2504                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2505             }
2506         }
2507     };
2508
2509     &$cleanup_config($conf);
2510
2511     &$cleanup_config($conf->{pending}, 1);
2512
2513     foreach my $snapname (keys %{$conf->{snapshots}}) {
2514         die "internal error" if $snapname eq 'pending';
2515         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2516     }
2517
2518     # remove 'unusedX' settings if we re-add a volume
2519     foreach my $key (keys %$conf) {
2520         my $value = $conf->{$key};
2521         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2522             delete $conf->{$key};
2523         }
2524     }
2525
2526     my $generate_raw_config = sub {
2527         my ($conf, $pending) = @_;
2528
2529         my $raw = '';
2530
2531         # add description as comment to top of file
2532         if (defined(my $descr = $conf->{description})) {
2533             if ($descr) {
2534                 foreach my $cl (split(/\n/, $descr)) {
2535                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2536                 }
2537             } else {
2538                 $raw .= "#\n" if $pending;
2539             }
2540         }
2541
2542         foreach my $key (sort keys %$conf) {
2543             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2544             $raw .= "$key: $conf->{$key}\n";
2545         }
2546         return $raw;
2547     };
2548
2549     my $raw = &$generate_raw_config($conf);
2550
2551     if (scalar(keys %{$conf->{pending}})){
2552         $raw .= "\n[PENDING]\n";
2553         $raw .= &$generate_raw_config($conf->{pending}, 1);
2554     }
2555
2556     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2557         $raw .= "\n[$snapname]\n";
2558         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2559     }
2560
2561     return $raw;
2562 }
2563
2564 sub load_defaults {
2565
2566     my $res = {};
2567
2568     # we use static defaults from our JSON schema configuration
2569     foreach my $key (keys %$confdesc) {
2570         if (defined(my $default = $confdesc->{$key}->{default})) {
2571             $res->{$key} = $default;
2572         }
2573     }
2574
2575     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2576     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2577
2578     return $res;
2579 }
2580
2581 sub config_list {
2582     my $vmlist = PVE::Cluster::get_vmlist();
2583     my $res = {};
2584     return $res if !$vmlist || !$vmlist->{ids};
2585     my $ids = $vmlist->{ids};
2586
2587     foreach my $vmid (keys %$ids) {
2588         my $d = $ids->{$vmid};
2589         next if !$d->{node} || $d->{node} ne $nodename;
2590         next if !$d->{type} || $d->{type} ne 'qemu';
2591         $res->{$vmid}->{exists} = 1;
2592     }
2593     return $res;
2594 }
2595
2596 # test if VM uses local resources (to prevent migration)
2597 sub check_local_resources {
2598     my ($conf, $noerr) = @_;
2599
2600     my $loc_res = 0;
2601
2602     $loc_res = 1 if $conf->{hostusb}; # old syntax
2603     $loc_res = 1 if $conf->{hostpci}; # old syntax
2604
2605     foreach my $k (keys %$conf) {
2606         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2607         # sockets are safe: they will recreated be on the target side post-migrate
2608         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2609         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2610     }
2611
2612     die "VM uses local resources\n" if $loc_res && !$noerr;
2613
2614     return $loc_res;
2615 }
2616
2617 # check if used storages are available on all nodes (use by migrate)
2618 sub check_storage_availability {
2619     my ($storecfg, $conf, $node) = @_;
2620
2621     foreach_drive($conf, sub {
2622         my ($ds, $drive) = @_;
2623
2624         my $volid = $drive->{file};
2625         return if !$volid;
2626
2627         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2628         return if !$sid;
2629
2630         # check if storage is available on both nodes
2631         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2632         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2633    });
2634 }
2635
2636 # list nodes where all VM images are available (used by has_feature API)
2637 sub shared_nodes {
2638     my ($conf, $storecfg) = @_;
2639
2640     my $nodelist = PVE::Cluster::get_nodelist();
2641     my $nodehash = { map { $_ => 1 } @$nodelist };
2642     my $nodename = PVE::INotify::nodename();
2643
2644     foreach_drive($conf, sub {
2645         my ($ds, $drive) = @_;
2646
2647         my $volid = $drive->{file};
2648         return if !$volid;
2649
2650         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2651         if ($storeid) {
2652             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2653             if ($scfg->{disable}) {
2654                 $nodehash = {};
2655             } elsif (my $avail = $scfg->{nodes}) {
2656                 foreach my $node (keys %$nodehash) {
2657                     delete $nodehash->{$node} if !$avail->{$node};
2658                 }
2659             } elsif (!$scfg->{shared}) {
2660                 foreach my $node (keys %$nodehash) {
2661                     delete $nodehash->{$node} if $node ne $nodename
2662                 }
2663             }
2664         }
2665     });
2666
2667     return $nodehash
2668 }
2669
2670 sub check_cmdline {
2671     my ($pidfile, $pid) = @_;
2672
2673     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2674     if (defined($fh)) {
2675         my $line = <$fh>;
2676         $fh->close;
2677         return undef if !$line;
2678         my @param = split(/\0/, $line);
2679
2680         my $cmd = $param[0];
2681         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2682
2683         for (my $i = 0; $i < scalar (@param); $i++) {
2684             my $p = $param[$i];
2685             next if !$p;
2686             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2687                 my $p = $param[$i+1];
2688                 return 1 if $p && ($p eq $pidfile);
2689                 return undef;
2690             }
2691         }
2692     }
2693     return undef;
2694 }
2695
2696 sub check_running {
2697     my ($vmid, $nocheck, $node) = @_;
2698
2699     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2700
2701     die "unable to find configuration file for VM $vmid - no such machine\n"
2702         if !$nocheck && ! -f $filename;
2703
2704     my $pidfile = pidfile_name($vmid);
2705
2706     if (my $fd = IO::File->new("<$pidfile")) {
2707         my $st = stat($fd);
2708         my $line = <$fd>;
2709         close($fd);
2710
2711         my $mtime = $st->mtime;
2712         if ($mtime > time()) {
2713             warn "file '$filename' modified in future\n";
2714         }
2715
2716         if ($line =~ m/^(\d+)$/) {
2717             my $pid = $1;
2718             if (check_cmdline($pidfile, $pid)) {
2719                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2720                     return $pid;
2721                 }
2722             }
2723         }
2724     }
2725
2726     return undef;
2727 }
2728
2729 sub vzlist {
2730
2731     my $vzlist = config_list();
2732
2733     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2734
2735     while (defined(my $de = $fd->read)) {
2736         next if $de !~ m/^(\d+)\.pid$/;
2737         my $vmid = $1;
2738         next if !defined($vzlist->{$vmid});
2739         if (my $pid = check_running($vmid)) {
2740             $vzlist->{$vmid}->{pid} = $pid;
2741         }
2742     }
2743
2744     return $vzlist;
2745 }
2746
2747 sub disksize {
2748     my ($storecfg, $conf) = @_;
2749
2750     my $bootdisk = $conf->{bootdisk};
2751     return undef if !$bootdisk;
2752     return undef if !is_valid_drivename($bootdisk);
2753
2754     return undef if !$conf->{$bootdisk};
2755
2756     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2757     return undef if !defined($drive);
2758
2759     return undef if drive_is_cdrom($drive);
2760
2761     my $volid = $drive->{file};
2762     return undef if !$volid;
2763
2764     return $drive->{size};
2765 }
2766
2767 my $last_proc_pid_stat;
2768
2769 # get VM status information
2770 # This must be fast and should not block ($full == false)
2771 # We only query KVM using QMP if $full == true (this can be slow)
2772 sub vmstatus {
2773     my ($opt_vmid, $full) = @_;
2774
2775     my $res = {};
2776
2777     my $storecfg = PVE::Storage::config();
2778
2779     my $list = vzlist();
2780     my $defaults = load_defaults();
2781
2782     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2783
2784     my $cpucount = $cpuinfo->{cpus} || 1;
2785
2786     foreach my $vmid (keys %$list) {
2787         next if $opt_vmid && ($vmid ne $opt_vmid);
2788
2789         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
2790         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2791
2792         my $d = {};
2793         $d->{pid} = $list->{$vmid}->{pid};
2794
2795         # fixme: better status?
2796         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2797
2798         my $size = disksize($storecfg, $conf);
2799         if (defined($size)) {
2800             $d->{disk} = 0; # no info available
2801             $d->{maxdisk} = $size;
2802         } else {
2803             $d->{disk} = 0;
2804             $d->{maxdisk} = 0;
2805         }
2806
2807         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2808             * ($conf->{cores} || $defaults->{cores});
2809         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2810         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2811
2812         $d->{name} = $conf->{name} || "VM $vmid";
2813         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2814             : $defaults->{memory}*(1024*1024);
2815
2816         if ($conf->{balloon}) {
2817             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2818             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2819                 : $defaults->{shares};
2820         }
2821
2822         $d->{uptime} = 0;
2823         $d->{cpu} = 0;
2824         $d->{mem} = 0;
2825
2826         $d->{netout} = 0;
2827         $d->{netin} = 0;
2828
2829         $d->{diskread} = 0;
2830         $d->{diskwrite} = 0;
2831
2832         $d->{template} = PVE::QemuConfig->is_template($conf);
2833
2834         $d->{serial} = 1 if conf_has_serial($conf);
2835
2836         $res->{$vmid} = $d;
2837     }
2838
2839     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2840     foreach my $dev (keys %$netdev) {
2841         next if $dev !~ m/^tap([1-9]\d*)i/;
2842         my $vmid = $1;
2843         my $d = $res->{$vmid};
2844         next if !$d;
2845
2846         $d->{netout} += $netdev->{$dev}->{receive};
2847         $d->{netin} += $netdev->{$dev}->{transmit};
2848
2849         if ($full) {
2850             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2851             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2852         }
2853
2854     }
2855
2856     my $ctime = gettimeofday;
2857
2858     foreach my $vmid (keys %$list) {
2859
2860         my $d = $res->{$vmid};
2861         my $pid = $d->{pid};
2862         next if !$pid;
2863
2864         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2865         next if !$pstat; # not running
2866
2867         my $used = $pstat->{utime} + $pstat->{stime};
2868
2869         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2870
2871         if ($pstat->{vsize}) {
2872             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2873         }
2874
2875         my $old = $last_proc_pid_stat->{$pid};
2876         if (!$old) {
2877             $last_proc_pid_stat->{$pid} = {
2878                 time => $ctime,
2879                 used => $used,
2880                 cpu => 0,
2881             };
2882             next;
2883         }
2884
2885         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2886
2887         if ($dtime > 1000) {
2888             my $dutime = $used -  $old->{used};
2889
2890             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2891             $last_proc_pid_stat->{$pid} = {
2892                 time => $ctime,
2893                 used => $used,
2894                 cpu => $d->{cpu},
2895             };
2896         } else {
2897             $d->{cpu} = $old->{cpu};
2898         }
2899     }
2900
2901     return $res if !$full;
2902
2903     my $qmpclient = PVE::QMPClient->new();
2904
2905     my $ballooncb = sub {
2906         my ($vmid, $resp) = @_;
2907
2908         my $info = $resp->{'return'};
2909         return if !$info->{max_mem};
2910
2911         my $d = $res->{$vmid};
2912
2913         # use memory assigned to VM
2914         $d->{maxmem} = $info->{max_mem};
2915         $d->{balloon} = $info->{actual};
2916
2917         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2918             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2919             $d->{freemem} = $info->{free_mem};
2920         }
2921
2922         $d->{ballooninfo} = $info;
2923     };
2924
2925     my $blockstatscb = sub {
2926         my ($vmid, $resp) = @_;
2927         my $data = $resp->{'return'} || [];
2928         my $totalrdbytes = 0;
2929         my $totalwrbytes = 0;
2930
2931         for my $blockstat (@$data) {
2932             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2933             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2934
2935             $blockstat->{device} =~ s/drive-//;
2936             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2937         }
2938         $res->{$vmid}->{diskread} = $totalrdbytes;
2939         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2940     };
2941
2942     my $statuscb = sub {
2943         my ($vmid, $resp) = @_;
2944
2945         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2946         # this fails if ballon driver is not loaded, so this must be
2947         # the last commnand (following command are aborted if this fails).
2948         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2949
2950         my $status = 'unknown';
2951         if (!defined($status = $resp->{'return'}->{status})) {
2952             warn "unable to get VM status\n";
2953             return;
2954         }
2955
2956         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2957     };
2958
2959     foreach my $vmid (keys %$list) {
2960         next if $opt_vmid && ($vmid ne $opt_vmid);
2961         next if !$res->{$vmid}->{pid}; # not running
2962         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2963     }
2964
2965     $qmpclient->queue_execute(undef, 2);
2966
2967     foreach my $vmid (keys %$list) {
2968         next if $opt_vmid && ($vmid ne $opt_vmid);
2969         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2970     }
2971
2972     return $res;
2973 }
2974
2975 sub foreach_drive {
2976     my ($conf, $func, @param) = @_;
2977
2978     foreach my $ds (valid_drive_names()) {
2979         next if !defined($conf->{$ds});
2980
2981         my $drive = parse_drive($ds, $conf->{$ds});
2982         next if !$drive;
2983
2984         &$func($ds, $drive, @param);
2985     }
2986 }
2987
2988 sub foreach_volid {
2989     my ($conf, $func, @param) = @_;
2990
2991     my $volhash = {};
2992
2993     my $test_volid = sub {
2994         my ($volid, $is_cdrom, $replicate, $shared, $snapname) = @_;
2995
2996         return if !$volid;
2997
2998         $volhash->{$volid}->{cdrom} //= 1;
2999         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
3000
3001         $volhash->{$volid}->{replicate} //= 0;
3002         $volhash->{$volid}->{replicate} = 1 if $replicate;
3003
3004         $volhash->{$volid}->{shared} //= 0;
3005         $volhash->{$volid}->{shared} = 1 if $shared;
3006
3007         $volhash->{$volid}->{referenced_in_config} //= 0;
3008         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
3009
3010         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
3011             if defined($snapname);
3012     };
3013
3014     foreach_drive($conf, sub {
3015         my ($ds, $drive) = @_;
3016         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, undef);
3017     });
3018
3019     foreach my $snapname (keys %{$conf->{snapshots}}) {
3020         my $snap = $conf->{snapshots}->{$snapname};
3021         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
3022         foreach_drive($snap, sub {
3023             my ($ds, $drive) = @_;
3024             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, $snapname);
3025         });
3026     }
3027
3028     foreach my $volid (keys %$volhash) {
3029         &$func($volid, $volhash->{$volid}, @param);
3030     }
3031 }
3032
3033 sub conf_has_serial {
3034     my ($conf) = @_;
3035
3036     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3037         if ($conf->{"serial$i"}) {
3038             return 1;
3039         }
3040     }
3041
3042     return 0;
3043 }
3044
3045 sub vga_conf_has_spice {
3046     my ($vga) = @_;
3047
3048     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
3049
3050     return $1 || 1;
3051 }
3052
3053 sub config_to_command {
3054     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
3055
3056     my $cmd = [];
3057     my $globalFlags = [];
3058     my $machineFlags = [];
3059     my $rtcFlags = [];
3060     my $cpuFlags = [];
3061     my $devices = [];
3062     my $pciaddr = '';
3063     my $bridges = {};
3064     my $kvmver = kvm_user_version();
3065     my $vernum = 0; # unknown
3066     my $ostype = $conf->{ostype};
3067     my $winversion = windows_version($ostype);
3068     my $kvm = $conf->{kvm} // 1;
3069
3070     die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n" if (!$cpuinfo->{hvm} && $kvm);
3071
3072     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
3073         $vernum = $1*1000000+$2*1000;
3074     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
3075         $vernum = $1*1000000+$2*1000+$3;
3076     }
3077
3078     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
3079
3080     my $have_ovz = -f '/proc/vz/vestat';
3081
3082     my $q35 = machine_type_is_q35($conf);
3083     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3084     my $machine_type = $forcemachine || $conf->{machine};
3085     my $use_old_bios_files = undef;
3086     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3087
3088     my $cpuunits = defined($conf->{cpuunits}) ?
3089             $conf->{cpuunits} : $defaults->{cpuunits};
3090
3091     push @$cmd, '/usr/bin/kvm';
3092
3093     push @$cmd, '-id', $vmid;
3094
3095     my $vmname = $conf->{name} || "vm$vmid";
3096
3097     push @$cmd, '-name', $vmname;
3098
3099     my $use_virtio = 0;
3100
3101     my $qmpsocket = qmp_socket($vmid);
3102     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3103     push @$cmd, '-mon', "chardev=qmp,mode=control";
3104
3105
3106     push @$cmd, '-pidfile' , pidfile_name($vmid);
3107
3108     push @$cmd, '-daemonize';
3109
3110     if ($conf->{smbios1}) {
3111         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3112     }
3113
3114     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3115         die "uefi base image not found\n" if ! -f $OVMF_CODE;
3116
3117         my $path;
3118         my $format;
3119         if (my $efidisk = $conf->{efidisk0}) {
3120             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
3121             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3122             $format = $d->{format};
3123             if ($storeid) {
3124                 $path = PVE::Storage::path($storecfg, $d->{file});
3125                 if (!defined($format)) {
3126                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3127                     $format = qemu_img_format($scfg, $volname);
3128                 }
3129             } else {
3130                 $path = $d->{file};
3131                 die "efidisk format must be specified\n"
3132                     if !defined($format);
3133             }
3134         } else {
3135             warn "no efidisk configured! Using temporary efivars disk.\n";
3136             $path = "/tmp/$vmid-ovmf.fd";
3137             PVE::Tools::file_copy($OVMF_VARS, $path, -s $OVMF_VARS);
3138             $format = 'raw';
3139         }
3140
3141         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$OVMF_CODE";
3142         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
3143     }
3144
3145
3146     # add usb controllers
3147     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $q35, $usbdesc->{format}, $MAX_USB_DEVICES);
3148     push @$devices, @usbcontrollers if @usbcontrollers;
3149     my $vga = $conf->{vga};
3150
3151     my $qxlnum = vga_conf_has_spice($vga);
3152     $vga = 'qxl' if $qxlnum;
3153
3154     if (!$vga) {
3155         if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 9)) {
3156             $vga = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3157         } else {
3158             $vga = ($winversion >= 6) ? 'std' : 'cirrus';
3159         }
3160     }
3161
3162     # enable absolute mouse coordinates (needed by vnc)
3163     my $tablet;
3164     if (defined($conf->{tablet})) {
3165         $tablet = $conf->{tablet};
3166     } else {
3167         $tablet = $defaults->{tablet};
3168         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3169         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3170     }
3171
3172     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
3173
3174     my $kvm_off = 0;
3175     my $gpu_passthrough;
3176
3177     # host pci devices
3178     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3179         my $d = parse_hostpci($conf->{"hostpci$i"});
3180         next if !$d;
3181
3182         my $pcie = $d->{pcie};
3183         if($pcie){
3184             die "q35 machine model is not enabled" if !$q35;
3185             $pciaddr = print_pcie_addr("hostpci$i");
3186         }else{
3187             $pciaddr = print_pci_addr("hostpci$i", $bridges);
3188         }
3189
3190         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
3191         my $romfile = $d->{romfile};
3192
3193         my $xvga = '';
3194         if ($d->{'x-vga'}) {
3195             $xvga = ',x-vga=on';
3196             $kvm_off = 1;
3197             $vga = 'none';
3198             $gpu_passthrough = 1;
3199
3200             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3201                 $xvga = "";
3202             }
3203         }
3204         my $pcidevices = $d->{pciid};
3205         my $multifunction = 1 if @$pcidevices > 1;
3206
3207         my $j=0;
3208         foreach my $pcidevice (@$pcidevices) {
3209
3210             my $id = "hostpci$i";
3211             $id .= ".$j" if $multifunction;
3212             my $addr = $pciaddr;
3213             $addr .= ".$j" if $multifunction;
3214             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
3215
3216             if($j == 0){
3217                 $devicestr .= "$rombar$xvga";
3218                 $devicestr .= ",multifunction=on" if $multifunction;
3219                 $devicestr .= ",romfile=/usr/share/kvm/$romfile" if $romfile;
3220             }
3221
3222             push @$devices, '-device', $devicestr;
3223             $j++;
3224         }
3225     }
3226
3227     # usb devices
3228     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES);
3229     push @$devices, @usbdevices if @usbdevices;
3230     # serial devices
3231     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3232         if (my $path = $conf->{"serial$i"}) {
3233             if ($path eq 'socket') {
3234                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3235                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3236                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3237             } else {
3238                 die "no such serial device\n" if ! -c $path;
3239                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3240                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3241             }
3242         }
3243     }
3244
3245     # parallel devices
3246     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3247         if (my $path = $conf->{"parallel$i"}) {
3248             die "no such parallel device\n" if ! -c $path;
3249             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3250             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3251             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3252         }
3253     }
3254
3255
3256     my $sockets = 1;
3257     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3258     $sockets = $conf->{sockets} if  $conf->{sockets};
3259
3260     my $cores = $conf->{cores} || 1;
3261
3262     my $maxcpus = $sockets * $cores;
3263
3264     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3265
3266     my $allowed_vcpus = $cpuinfo->{cpus};
3267
3268     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3269         if ($allowed_vcpus < $maxcpus);
3270
3271     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
3272
3273         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3274         for (my $i = 2; $i <= $vcpus; $i++)  {
3275             my $cpustr = print_cpu_device($conf,$i);
3276             push @$cmd, '-device', $cpustr;
3277         }
3278
3279     } else {
3280
3281         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3282     }
3283     push @$cmd, '-nodefaults';
3284
3285     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3286
3287     my $bootindex_hash = {};
3288     my $i = 1;
3289     foreach my $o (split(//, $bootorder)) {
3290         $bootindex_hash->{$o} = $i*100;
3291         $i++;
3292     }
3293
3294     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3295
3296     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3297
3298     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3299
3300     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
3301
3302     if ($vga && $vga !~ m/^serial\d+$/ && $vga ne 'none'){
3303         my $socket = vnc_socket($vmid);
3304         push @$cmd,  '-vnc', "unix:$socket,x509,password";
3305     } else {
3306         push @$cmd, '-nographic';
3307     }
3308
3309     # time drift fix
3310     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3311
3312     my $useLocaltime = $conf->{localtime};
3313
3314     if ($winversion >= 5) { # windows
3315         $useLocaltime = 1 if !defined($conf->{localtime});
3316
3317         # use time drift fix when acpi is enabled
3318         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3319             $tdf = 1 if !defined($conf->{tdf});
3320         }
3321     }
3322
3323     if ($winversion >= 6) {
3324         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3325         push @$cmd, '-no-hpet';
3326     }
3327
3328     push @$rtcFlags, 'driftfix=slew' if $tdf;
3329
3330     if (!$kvm) {
3331         push @$machineFlags, 'accel=tcg';
3332     }
3333
3334     if ($machine_type) {
3335         push @$machineFlags, "type=${machine_type}";
3336     }
3337
3338     if ($conf->{startdate}) {
3339         push @$rtcFlags, "base=$conf->{startdate}";
3340     } elsif ($useLocaltime) {
3341         push @$rtcFlags, 'base=localtime';
3342     }
3343
3344     my $cpu = $kvm ? "kvm64" : "qemu64";
3345     if (my $cputype = $conf->{cpu}) {
3346         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3347             or die "Cannot parse cpu description: $cputype\n";
3348         $cpu = $cpuconf->{cputype};
3349         $kvm_off = 1 if $cpuconf->{hidden};
3350
3351         if (defined(my $flags = $cpuconf->{flags})) {
3352             push @$cpuFlags, split(";", $flags);
3353         }
3354     }
3355
3356     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
3357
3358     push @$cpuFlags , '-x2apic'
3359         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3360
3361     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3362
3363     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3364
3365     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3366
3367         push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3368         push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3369     }
3370
3371     add_hyperv_enlightenments($cpuFlags, $winversion, $machine_type, $kvmver, $conf->{bios}, $gpu_passthrough) if $kvm;
3372
3373     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm;
3374
3375     push @$cpuFlags, 'kvm=off' if $kvm_off;
3376
3377     my $cpu_vendor = $cpu_vendor_list->{$cpu} ||
3378         die "internal error"; # should not happen
3379
3380     push @$cpuFlags, "vendor=${cpu_vendor}"
3381         if $cpu_vendor ne 'default';
3382
3383     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3384
3385     push @$cmd, '-cpu', $cpu;
3386
3387     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3388
3389     push @$cmd, '-S' if $conf->{freeze};
3390
3391     # set keyboard layout
3392     my $kb = $conf->{keyboard} || $defaults->{keyboard};
3393     push @$cmd, '-k', $kb if $kb;
3394
3395     # enable sound
3396     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3397     #push @$cmd, '-soundhw', 'es1370';
3398     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3399
3400     if($conf->{agent}) {
3401         my $qgasocket = qmp_socket($vmid, 1);
3402         my $pciaddr = print_pci_addr("qga0", $bridges);
3403         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3404         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3405         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3406     }
3407
3408     my $spice_port;
3409
3410     if ($qxlnum) {
3411         if ($qxlnum > 1) {
3412             if ($winversion){
3413                 for(my $i = 1; $i < $qxlnum; $i++){
3414                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3415                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3416                 }
3417             } else {
3418                 # assume other OS works like Linux
3419                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3420                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3421             }
3422         }
3423
3424         my $pciaddr = print_pci_addr("spice", $bridges);
3425
3426         my $nodename = PVE::INotify::nodename();
3427         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3428         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3429         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3430         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3431         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3432
3433         push @$devices, '-spice', "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3434
3435         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3436         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3437         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3438     }
3439
3440     # enable balloon by default, unless explicitly disabled
3441     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3442         $pciaddr = print_pci_addr("balloon0", $bridges);
3443         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3444     }
3445
3446     if ($conf->{watchdog}) {
3447         my $wdopts = parse_watchdog($conf->{watchdog});
3448         $pciaddr = print_pci_addr("watchdog", $bridges);
3449         my $watchdog = $wdopts->{model} || 'i6300esb';
3450         push @$devices, '-device', "$watchdog$pciaddr";
3451         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3452     }
3453
3454     my $vollist = [];
3455     my $scsicontroller = {};
3456     my $ahcicontroller = {};
3457     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3458
3459     # Add iscsi initiator name if available
3460     if (my $initiator = get_initiator_name()) {
3461         push @$devices, '-iscsi', "initiator-name=$initiator";
3462     }
3463
3464     foreach_drive($conf, sub {
3465         my ($ds, $drive) = @_;
3466
3467         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3468             push @$vollist, $drive->{file};
3469         }
3470
3471         # ignore efidisk here, already added in bios/fw handling code above
3472         return if $drive->{interface} eq 'efidisk';
3473
3474         $use_virtio = 1 if $ds =~ m/^virtio/;
3475
3476         if (drive_is_cdrom ($drive)) {
3477             if ($bootindex_hash->{d}) {
3478                 $drive->{bootindex} = $bootindex_hash->{d};
3479                 $bootindex_hash->{d} += 1;
3480             }
3481         } else {
3482             if ($bootindex_hash->{c}) {
3483                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3484                 $bootindex_hash->{c} += 1;
3485             }
3486         }
3487
3488         if($drive->{interface} eq 'virtio'){
3489            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3490         }
3491
3492         if ($drive->{interface} eq 'scsi') {
3493
3494             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3495
3496             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3497             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3498
3499             my $iothread = '';
3500             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3501                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3502                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3503             } elsif ($drive->{iothread}) {
3504                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3505             }
3506
3507             my $queues = '';
3508             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3509                 $queues = ",num_queues=$drive->{queues}";
3510             }
3511
3512             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3513             $scsicontroller->{$controller}=1;
3514         }
3515
3516         if ($drive->{interface} eq 'sata') {
3517            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3518            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3519            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3520            $ahcicontroller->{$controller}=1;
3521         }
3522
3523         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3524         push @$devices, '-drive',$drive_cmd;
3525         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3526     });
3527
3528     for (my $i = 0; $i < $MAX_NETS; $i++) {
3529          next if !$conf->{"net$i"};
3530          my $d = parse_net($conf->{"net$i"});
3531          next if !$d;
3532
3533          $use_virtio = 1 if $d->{model} eq 'virtio';
3534
3535          if ($bootindex_hash->{n}) {
3536             $d->{bootindex} = $bootindex_hash->{n};
3537             $bootindex_hash->{n} += 1;
3538          }
3539
3540          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3541          push @$devices, '-netdev', $netdevfull;
3542
3543          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3544          push @$devices, '-device', $netdevicefull;
3545     }
3546
3547     if (!$q35) {
3548         # add pci bridges
3549         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3550            $bridges->{1} = 1;
3551            $bridges->{2} = 1;
3552         }
3553
3554         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3555
3556         while (my ($k, $v) = each %$bridges) {
3557             $pciaddr = print_pci_addr("pci.$k");
3558             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3559         }
3560     }
3561
3562     # add custom args
3563     if ($conf->{args}) {
3564         my $aa = PVE::Tools::split_args($conf->{args});
3565         push @$cmd, @$aa;
3566     }
3567
3568     push @$cmd, @$devices;
3569     push @$cmd, '-rtc', join(',', @$rtcFlags)
3570         if scalar(@$rtcFlags);
3571     push @$cmd, '-machine', join(',', @$machineFlags)
3572         if scalar(@$machineFlags);
3573     push @$cmd, '-global', join(',', @$globalFlags)
3574         if scalar(@$globalFlags);
3575
3576     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3577 }
3578
3579 sub vnc_socket {
3580     my ($vmid) = @_;
3581     return "${var_run_tmpdir}/$vmid.vnc";
3582 }
3583
3584 sub spice_port {
3585     my ($vmid) = @_;
3586
3587     my $res = vm_mon_cmd($vmid, 'query-spice');
3588
3589     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3590 }
3591
3592 sub qmp_socket {
3593     my ($vmid, $qga) = @_;
3594     my $sockettype = $qga ? 'qga' : 'qmp';
3595     return "${var_run_tmpdir}/$vmid.$sockettype";
3596 }
3597
3598 sub pidfile_name {
3599     my ($vmid) = @_;
3600     return "${var_run_tmpdir}/$vmid.pid";
3601 }
3602
3603 sub vm_devices_list {
3604     my ($vmid) = @_;
3605
3606     my $res = vm_mon_cmd($vmid, 'query-pci');
3607     my $devices = {};
3608     foreach my $pcibus (@$res) {
3609         foreach my $device (@{$pcibus->{devices}}) {
3610             next if !$device->{'qdev_id'};
3611             if ($device->{'pci_bridge'}) {
3612                 $devices->{$device->{'qdev_id'}} = 1;
3613                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3614                     next if !$bridge_device->{'qdev_id'};
3615                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3616                     $devices->{$device->{'qdev_id'}}++;
3617                 }
3618             } else {
3619                 $devices->{$device->{'qdev_id'}} = 1;
3620             }
3621         }
3622     }
3623
3624     my $resblock = vm_mon_cmd($vmid, 'query-block');
3625     foreach my $block (@$resblock) {
3626         if($block->{device} =~ m/^drive-(\S+)/){
3627                 $devices->{$1} = 1;
3628         }
3629     }
3630
3631     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3632     foreach my $mice (@$resmice) {
3633         if ($mice->{name} eq 'QEMU HID Tablet') {
3634             $devices->{tablet} = 1;
3635             last;
3636         }
3637     }
3638
3639     # for usb devices there is no query-usb
3640     # but we can iterate over the entries in
3641     # qom-list path=/machine/peripheral
3642     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3643     foreach my $per (@$resperipheral) {
3644         if ($per->{name} =~ m/^usb\d+$/) {
3645             $devices->{$per->{name}} = 1;
3646         }
3647     }
3648
3649     return $devices;
3650 }
3651
3652 sub vm_deviceplug {
3653     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3654
3655     my $q35 = machine_type_is_q35($conf);
3656
3657     my $devices_list = vm_devices_list($vmid);
3658     return 1 if defined($devices_list->{$deviceid});
3659
3660     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3661
3662     if ($deviceid eq 'tablet') {
3663
3664         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3665
3666     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3667
3668         die "usb hotplug currently not reliable\n";
3669         # since we can't reliably hot unplug all added usb devices
3670         # and usb passthrough disables live migration
3671         # we disable usb hotplugging for now
3672         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3673
3674     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3675
3676         qemu_iothread_add($vmid, $deviceid, $device);
3677
3678         qemu_driveadd($storecfg, $vmid, $device);
3679         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3680
3681         qemu_deviceadd($vmid, $devicefull);
3682         eval { qemu_deviceaddverify($vmid, $deviceid); };
3683         if (my $err = $@) {
3684             eval { qemu_drivedel($vmid, $deviceid); };
3685             warn $@ if $@;
3686             die $err;
3687         }
3688
3689     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3690
3691
3692         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3693         my $pciaddr = print_pci_addr($deviceid);
3694         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3695
3696         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3697
3698         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3699             qemu_iothread_add($vmid, $deviceid, $device);
3700             $devicefull .= ",iothread=iothread-$deviceid";
3701         }
3702
3703         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3704             $devicefull .= ",num_queues=$device->{queues}";
3705         }
3706
3707         qemu_deviceadd($vmid, $devicefull);
3708         qemu_deviceaddverify($vmid, $deviceid);
3709
3710     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3711
3712         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3713         qemu_driveadd($storecfg, $vmid, $device);
3714
3715         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3716         eval { qemu_deviceadd($vmid, $devicefull); };
3717         if (my $err = $@) {
3718             eval { qemu_drivedel($vmid, $deviceid); };
3719             warn $@ if $@;
3720             die $err;
3721         }
3722
3723     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3724
3725         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3726
3727         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
3728         my $use_old_bios_files = undef;
3729         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3730
3731         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3732         qemu_deviceadd($vmid, $netdevicefull);
3733         eval { qemu_deviceaddverify($vmid, $deviceid); };
3734         if (my $err = $@) {
3735             eval { qemu_netdevdel($vmid, $deviceid); };
3736             warn $@ if $@;
3737             die $err;
3738         }
3739
3740     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3741
3742         my $bridgeid = $2;
3743         my $pciaddr = print_pci_addr($deviceid);
3744         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3745
3746         qemu_deviceadd($vmid, $devicefull);
3747         qemu_deviceaddverify($vmid, $deviceid);
3748
3749     } else {
3750         die "can't hotplug device '$deviceid'\n";
3751     }
3752
3753     return 1;
3754 }
3755
3756 # fixme: this should raise exceptions on error!
3757 sub vm_deviceunplug {
3758     my ($vmid, $conf, $deviceid) = @_;
3759
3760     my $devices_list = vm_devices_list($vmid);
3761     return 1 if !defined($devices_list->{$deviceid});
3762
3763     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3764
3765     if ($deviceid eq 'tablet') {
3766
3767         qemu_devicedel($vmid, $deviceid);
3768
3769     } elsif ($deviceid =~ m/^usb\d+$/) {
3770
3771         die "usb hotplug currently not reliable\n";
3772         # when unplugging usb devices this way,
3773         # there may be remaining usb controllers/hubs
3774         # so we disable it for now
3775         qemu_devicedel($vmid, $deviceid);
3776         qemu_devicedelverify($vmid, $deviceid);
3777
3778     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3779
3780         qemu_devicedel($vmid, $deviceid);
3781         qemu_devicedelverify($vmid, $deviceid);
3782         qemu_drivedel($vmid, $deviceid);
3783         qemu_iothread_del($conf, $vmid, $deviceid);
3784
3785     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3786
3787         qemu_devicedel($vmid, $deviceid);
3788         qemu_devicedelverify($vmid, $deviceid);
3789         qemu_iothread_del($conf, $vmid, $deviceid);
3790
3791     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3792
3793         qemu_devicedel($vmid, $deviceid);
3794         qemu_drivedel($vmid, $deviceid);
3795         qemu_deletescsihw($conf, $vmid, $deviceid);
3796
3797     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3798
3799         qemu_devicedel($vmid, $deviceid);
3800         qemu_devicedelverify($vmid, $deviceid);
3801         qemu_netdevdel($vmid, $deviceid);
3802
3803     } else {
3804         die "can't unplug device '$deviceid'\n";
3805     }
3806
3807     return 1;
3808 }
3809
3810 sub qemu_deviceadd {
3811     my ($vmid, $devicefull) = @_;
3812
3813     $devicefull = "driver=".$devicefull;
3814     my %options =  split(/[=,]/, $devicefull);
3815
3816     vm_mon_cmd($vmid, "device_add" , %options);
3817 }
3818
3819 sub qemu_devicedel {
3820     my ($vmid, $deviceid) = @_;
3821
3822     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3823 }
3824
3825 sub qemu_iothread_add {
3826     my($vmid, $deviceid, $device) = @_;
3827
3828     if ($device->{iothread}) {
3829         my $iothreads = vm_iothreads_list($vmid);
3830         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3831     }
3832 }
3833
3834 sub qemu_iothread_del {
3835     my($conf, $vmid, $deviceid) = @_;
3836
3837     my $device = parse_drive($deviceid, $conf->{$deviceid});
3838     if ($device->{iothread}) {
3839         my $iothreads = vm_iothreads_list($vmid);
3840         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3841     }
3842 }
3843
3844 sub qemu_objectadd {
3845     my($vmid, $objectid, $qomtype) = @_;
3846
3847     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3848
3849     return 1;
3850 }
3851
3852 sub qemu_objectdel {
3853     my($vmid, $objectid) = @_;
3854
3855     vm_mon_cmd($vmid, "object-del", id => $objectid);
3856
3857     return 1;
3858 }
3859
3860 sub qemu_driveadd {
3861     my ($storecfg, $vmid, $device) = @_;
3862
3863     my $drive = print_drive_full($storecfg, $vmid, $device);
3864     $drive =~ s/\\/\\\\/g;
3865     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3866
3867     # If the command succeeds qemu prints: "OK"
3868     return 1 if $ret =~ m/OK/s;
3869
3870     die "adding drive failed: $ret\n";
3871 }
3872
3873 sub qemu_drivedel {
3874     my($vmid, $deviceid) = @_;
3875
3876     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3877     $ret =~ s/^\s+//;
3878
3879     return 1 if $ret eq "";
3880
3881     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3882     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3883
3884     die "deleting drive $deviceid failed : $ret\n";
3885 }
3886
3887 sub qemu_deviceaddverify {
3888     my ($vmid, $deviceid) = @_;
3889
3890     for (my $i = 0; $i <= 5; $i++) {
3891          my $devices_list = vm_devices_list($vmid);
3892          return 1 if defined($devices_list->{$deviceid});
3893          sleep 1;
3894     }
3895
3896     die "error on hotplug device '$deviceid'\n";
3897 }
3898
3899
3900 sub qemu_devicedelverify {
3901     my ($vmid, $deviceid) = @_;
3902
3903     # need to verify that the device is correctly removed as device_del
3904     # is async and empty return is not reliable
3905
3906     for (my $i = 0; $i <= 5; $i++) {
3907          my $devices_list = vm_devices_list($vmid);
3908          return 1 if !defined($devices_list->{$deviceid});
3909          sleep 1;
3910     }
3911
3912     die "error on hot-unplugging device '$deviceid'\n";
3913 }
3914
3915 sub qemu_findorcreatescsihw {
3916     my ($storecfg, $conf, $vmid, $device) = @_;
3917
3918     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3919
3920     my $scsihwid="$controller_prefix$controller";
3921     my $devices_list = vm_devices_list($vmid);
3922
3923     if(!defined($devices_list->{$scsihwid})) {
3924         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3925     }
3926
3927     return 1;
3928 }
3929
3930 sub qemu_deletescsihw {
3931     my ($conf, $vmid, $opt) = @_;
3932
3933     my $device = parse_drive($opt, $conf->{$opt});
3934
3935     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3936         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3937         return 1;
3938     }
3939
3940     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3941
3942     my $devices_list = vm_devices_list($vmid);
3943     foreach my $opt (keys %{$devices_list}) {
3944         if (PVE::QemuServer::is_valid_drivename($opt)) {
3945             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3946             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3947                 return 1;
3948             }
3949         }
3950     }
3951
3952     my $scsihwid="scsihw$controller";
3953
3954     vm_deviceunplug($vmid, $conf, $scsihwid);
3955
3956     return 1;
3957 }
3958
3959 sub qemu_add_pci_bridge {
3960     my ($storecfg, $conf, $vmid, $device) = @_;
3961
3962     my $bridges = {};
3963
3964     my $bridgeid;
3965
3966     print_pci_addr($device, $bridges);
3967
3968     while (my ($k, $v) = each %$bridges) {
3969         $bridgeid = $k;
3970     }
3971     return 1 if !defined($bridgeid) || $bridgeid < 1;
3972
3973     my $bridge = "pci.$bridgeid";
3974     my $devices_list = vm_devices_list($vmid);
3975
3976     if (!defined($devices_list->{$bridge})) {
3977         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3978     }
3979
3980     return 1;
3981 }
3982
3983 sub qemu_set_link_status {
3984     my ($vmid, $device, $up) = @_;
3985
3986     vm_mon_cmd($vmid, "set_link", name => $device,
3987                up => $up ? JSON::true : JSON::false);
3988 }
3989
3990 sub qemu_netdevadd {
3991     my ($vmid, $conf, $device, $deviceid) = @_;
3992
3993     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
3994     my %options =  split(/[=,]/, $netdev);
3995
3996     vm_mon_cmd($vmid, "netdev_add",  %options);
3997     return 1;
3998 }
3999
4000 sub qemu_netdevdel {
4001     my ($vmid, $deviceid) = @_;
4002
4003     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
4004 }
4005
4006 sub qemu_usb_hotplug {
4007     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
4008
4009     return if !$device;
4010
4011     # remove the old one first
4012     vm_deviceunplug($vmid, $conf, $deviceid);
4013
4014     # check if xhci controller is necessary and available
4015     if ($device->{usb3}) {
4016
4017         my $devicelist = vm_devices_list($vmid);
4018
4019         if (!$devicelist->{xhci}) {
4020             my $pciaddr = print_pci_addr("xhci");
4021             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4022         }
4023     }
4024     my $d = parse_usb_device($device->{host});
4025     $d->{usb3} = $device->{usb3};
4026
4027     # add the new one
4028     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d);
4029 }
4030
4031 sub qemu_cpu_hotplug {
4032     my ($vmid, $conf, $vcpus) = @_;
4033
4034     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
4035
4036     my $sockets = 1;
4037     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4038     $sockets = $conf->{sockets} if  $conf->{sockets};
4039     my $cores = $conf->{cores} || 1;
4040     my $maxcpus = $sockets * $cores;
4041
4042     $vcpus = $maxcpus if !$vcpus;
4043
4044     die "you can't add more vcpus than maxcpus\n"
4045         if $vcpus > $maxcpus;
4046
4047     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4048
4049     if ($vcpus < $currentvcpus) {
4050
4051         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4052
4053             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4054                 qemu_devicedel($vmid, "cpu$i");
4055                 my $retry = 0;
4056                 my $currentrunningvcpus = undef;
4057                 while (1) {
4058                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4059                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4060                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4061                     $retry++;
4062                     sleep 1;
4063                 }
4064                 #update conf after each succesfull cpu unplug
4065                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4066                 PVE::QemuConfig->write_config($vmid, $conf);
4067             }
4068         } else {
4069             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4070         }
4071
4072         return;
4073     }
4074
4075     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4076     die "vcpus in running vm does not match its configuration\n"
4077         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4078
4079     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4080
4081         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4082             my $cpustr = print_cpu_device($conf, $i);
4083             qemu_deviceadd($vmid, $cpustr);
4084
4085             my $retry = 0;
4086             my $currentrunningvcpus = undef;
4087             while (1) {
4088                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4089                 last if scalar(@{$currentrunningvcpus}) == $i;
4090                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4091                 sleep 1;
4092                 $retry++;
4093             }
4094             #update conf after each succesfull cpu hotplug
4095             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4096             PVE::QemuConfig->write_config($vmid, $conf);
4097         }
4098     } else {
4099
4100         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4101             vm_mon_cmd($vmid, "cpu-add", id => int($i));
4102         }
4103     }
4104 }
4105
4106 sub qemu_block_set_io_throttle {
4107     my ($vmid, $deviceid,
4108         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4109         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4110         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4111         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4112
4113     return if !check_running($vmid) ;
4114
4115     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4116         bps => int($bps),
4117         bps_rd => int($bps_rd),
4118         bps_wr => int($bps_wr),
4119         iops => int($iops),
4120         iops_rd => int($iops_rd),
4121         iops_wr => int($iops_wr),
4122         bps_max => int($bps_max),
4123         bps_rd_max => int($bps_rd_max),
4124         bps_wr_max => int($bps_wr_max),
4125         iops_max => int($iops_max),
4126         iops_rd_max => int($iops_rd_max),
4127         iops_wr_max => int($iops_wr_max),
4128         bps_max_length => int($bps_max_length),
4129         bps_rd_max_length => int($bps_rd_max_length),
4130         bps_wr_max_length => int($bps_wr_max_length),
4131         iops_max_length => int($iops_max_length),
4132         iops_rd_max_length => int($iops_rd_max_length),
4133         iops_wr_max_length => int($iops_wr_max_length),
4134     );
4135
4136 }
4137
4138 # old code, only used to shutdown old VM after update
4139 sub __read_avail {
4140     my ($fh, $timeout) = @_;
4141
4142     my $sel = new IO::Select;
4143     $sel->add($fh);
4144
4145     my $res = '';
4146     my $buf;
4147
4148     my @ready;
4149     while (scalar (@ready = $sel->can_read($timeout))) {
4150         my $count;
4151         if ($count = $fh->sysread($buf, 8192)) {
4152             if ($buf =~ /^(.*)\(qemu\) $/s) {
4153                 $res .= $1;
4154                 last;
4155             } else {
4156                 $res .= $buf;
4157             }
4158         } else {
4159             if (!defined($count)) {
4160                 die "$!\n";
4161             }
4162             last;
4163         }
4164     }
4165
4166     die "monitor read timeout\n" if !scalar(@ready);
4167
4168     return $res;
4169 }
4170
4171 sub qemu_block_resize {
4172     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4173
4174     my $running = check_running($vmid);
4175
4176     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4177
4178     return if !$running;
4179
4180     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4181
4182 }
4183
4184 sub qemu_volume_snapshot {
4185     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4186
4187     my $running = check_running($vmid);
4188
4189     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4190         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
4191     } else {
4192         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4193     }
4194 }
4195
4196 sub qemu_volume_snapshot_delete {
4197     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4198
4199     my $running = check_running($vmid);
4200
4201     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4202         vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
4203     } else {
4204         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4205     }
4206 }
4207
4208 sub set_migration_caps {
4209     my ($vmid) = @_;
4210
4211     my $cap_ref = [];
4212
4213     my $enabled_cap = {
4214         "auto-converge" => 1,
4215         "xbzrle" => 1,
4216         "x-rdma-pin-all" => 0,
4217         "zero-blocks" => 0,
4218         "compress" => 0
4219     };
4220
4221     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4222
4223     for my $supported_capability (@$supported_capabilities) {
4224         push @$cap_ref, {
4225             capability => $supported_capability->{capability},
4226             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4227         };
4228     }
4229
4230     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4231 }
4232
4233 my $fast_plug_option = {
4234     'lock' => 1,
4235     'name' => 1,
4236     'onboot' => 1,
4237     'shares' => 1,
4238     'startup' => 1,
4239     'description' => 1,
4240     'protection' => 1,
4241     'vmstatestorage' => 1,
4242 };
4243
4244 # hotplug changes in [PENDING]
4245 # $selection hash can be used to only apply specified options, for
4246 # example: { cores => 1 } (only apply changed 'cores')
4247 # $errors ref is used to return error messages
4248 sub vmconfig_hotplug_pending {
4249     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4250
4251     my $defaults = load_defaults();
4252
4253     # commit values which do not have any impact on running VM first
4254     # Note: those option cannot raise errors, we we do not care about
4255     # $selection and always apply them.
4256
4257     my $add_error = sub {
4258         my ($opt, $msg) = @_;
4259         $errors->{$opt} = "hotplug problem - $msg";
4260     };
4261
4262     my $changes = 0;
4263     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4264         if ($fast_plug_option->{$opt}) {
4265             $conf->{$opt} = $conf->{pending}->{$opt};
4266             delete $conf->{pending}->{$opt};
4267             $changes = 1;
4268         }
4269     }
4270
4271     if ($changes) {
4272         PVE::QemuConfig->write_config($vmid, $conf);
4273         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4274     }
4275
4276     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4277
4278     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4279     while (my ($opt, $force) = each %$pending_delete_hash) {
4280         next if $selection && !$selection->{$opt};
4281         eval {
4282             if ($opt eq 'hotplug') {
4283                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4284             } elsif ($opt eq 'tablet') {
4285                 die "skip\n" if !$hotplug_features->{usb};
4286                 if ($defaults->{tablet}) {
4287                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4288                 } else {
4289                     vm_deviceunplug($vmid, $conf, $opt);
4290                 }
4291             } elsif ($opt =~ m/^usb\d+/) {
4292                 die "skip\n";
4293                 # since we cannot reliably hot unplug usb devices
4294                 # we are disabling it
4295                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4296                 vm_deviceunplug($vmid, $conf, $opt);
4297             } elsif ($opt eq 'vcpus') {
4298                 die "skip\n" if !$hotplug_features->{cpu};
4299                 qemu_cpu_hotplug($vmid, $conf, undef);
4300             } elsif ($opt eq 'balloon') {
4301                 # enable balloon device is not hotpluggable
4302                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
4303             } elsif ($fast_plug_option->{$opt}) {
4304                 # do nothing
4305             } elsif ($opt =~ m/^net(\d+)$/) {
4306                 die "skip\n" if !$hotplug_features->{network};
4307                 vm_deviceunplug($vmid, $conf, $opt);
4308             } elsif (is_valid_drivename($opt)) {
4309                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4310                 vm_deviceunplug($vmid, $conf, $opt);
4311                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4312             } elsif ($opt =~ m/^memory$/) {
4313                 die "skip\n" if !$hotplug_features->{memory};
4314                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4315             } elsif ($opt eq 'cpuunits') {
4316                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4317             } elsif ($opt eq 'cpulimit') {
4318                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4319             } else {
4320                 die "skip\n";
4321             }
4322         };
4323         if (my $err = $@) {
4324             &$add_error($opt, $err) if $err ne "skip\n";
4325         } else {
4326             # save new config if hotplug was successful
4327             delete $conf->{$opt};
4328             vmconfig_undelete_pending_option($conf, $opt);
4329             PVE::QemuConfig->write_config($vmid, $conf);
4330             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4331         }
4332     }
4333
4334     my $apply_pending_cloudinit;
4335     $apply_pending_cloudinit = sub {
4336         my ($key, $value) = @_;
4337         $apply_pending_cloudinit = sub {}; # once is enough
4338
4339         my @cloudinit_opts = keys %$confdesc_cloudinit;
4340         foreach my $opt (keys %{$conf->{pending}}) {
4341             next if !grep { $_ eq $opt } @cloudinit_opts;
4342             $conf->{$opt} = delete $conf->{pending}->{$opt};
4343         }
4344
4345         my $new_conf = { %$conf };
4346         $new_conf->{$key} = $value;
4347         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4348     };
4349
4350     foreach my $opt (keys %{$conf->{pending}}) {
4351         next if $selection && !$selection->{$opt};
4352         my $value = $conf->{pending}->{$opt};
4353         eval {
4354             if ($opt eq 'hotplug') {
4355                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4356             } elsif ($opt eq 'tablet') {
4357                 die "skip\n" if !$hotplug_features->{usb};
4358                 if ($value == 1) {
4359                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4360                 } elsif ($value == 0) {
4361                     vm_deviceunplug($vmid, $conf, $opt);
4362                 }
4363             } elsif ($opt =~ m/^usb\d+$/) {
4364                 die "skip\n";
4365                 # since we cannot reliably hot unplug usb devices
4366                 # we are disabling it
4367                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4368                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4369                 die "skip\n" if !$d;
4370                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d);
4371             } elsif ($opt eq 'vcpus') {
4372                 die "skip\n" if !$hotplug_features->{cpu};
4373                 qemu_cpu_hotplug($vmid, $conf, $value);
4374             } elsif ($opt eq 'balloon') {
4375                 # enable/disable balloning device is not hotpluggable
4376                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4377                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4378                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4379
4380                 # allow manual ballooning if shares is set to zero
4381                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4382                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4383                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4384                 }
4385             } elsif ($opt =~ m/^net(\d+)$/) {
4386                 # some changes can be done without hotplug
4387                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4388                                     $vmid, $opt, $value);
4389             } elsif (is_valid_drivename($opt)) {
4390                 # some changes can be done without hotplug
4391                 my $drive = parse_drive($opt, $value);
4392                 if (drive_is_cloudinit($drive)) {
4393                     &$apply_pending_cloudinit($opt, $value);
4394                 }
4395                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4396                                      $vmid, $opt, $value, 1);
4397             } elsif ($opt =~ m/^memory$/) { #dimms
4398                 die "skip\n" if !$hotplug_features->{memory};
4399                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4400             } elsif ($opt eq 'cpuunits') {
4401                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4402             } elsif ($opt eq 'cpulimit') {
4403                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4404                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4405             } else {
4406                 die "skip\n";  # skip non-hot-pluggable options
4407             }
4408         };
4409         if (my $err = $@) {
4410             &$add_error($opt, $err) if $err ne "skip\n";
4411         } else {
4412             # save new config if hotplug was successful
4413             $conf->{$opt} = $value;
4414             delete $conf->{pending}->{$opt};
4415             PVE::QemuConfig->write_config($vmid, $conf);
4416             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4417         }
4418     }
4419 }
4420
4421 sub try_deallocate_drive {
4422     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4423
4424     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4425         my $volid = $drive->{file};
4426         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4427             my $sid = PVE::Storage::parse_volume_id($volid);
4428             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4429
4430             # check if the disk is really unused
4431             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4432                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4433             PVE::Storage::vdisk_free($storecfg, $volid);
4434             return 1;
4435         } else {
4436             # If vm is not owner of this disk remove from config
4437             return 1;
4438         }
4439     }
4440
4441     return undef;
4442 }
4443
4444 sub vmconfig_delete_or_detach_drive {
4445     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4446
4447     my $drive = parse_drive($opt, $conf->{$opt});
4448
4449     my $rpcenv = PVE::RPCEnvironment::get();
4450     my $authuser = $rpcenv->get_user();
4451
4452     if ($force) {
4453         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4454         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4455     } else {
4456         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4457     }
4458 }
4459
4460 sub vmconfig_apply_pending {
4461     my ($vmid, $conf, $storecfg) = @_;
4462
4463     # cold plug
4464
4465     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4466     while (my ($opt, $force) = each %$pending_delete_hash) {
4467         die "internal error" if $opt =~ m/^unused/;
4468         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4469         if (!defined($conf->{$opt})) {
4470             vmconfig_undelete_pending_option($conf, $opt);
4471             PVE::QemuConfig->write_config($vmid, $conf);
4472         } elsif (is_valid_drivename($opt)) {
4473             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4474             vmconfig_undelete_pending_option($conf, $opt);
4475             delete $conf->{$opt};
4476             PVE::QemuConfig->write_config($vmid, $conf);
4477         } else {
4478             vmconfig_undelete_pending_option($conf, $opt);
4479             delete $conf->{$opt};
4480             PVE::QemuConfig->write_config($vmid, $conf);
4481         }
4482     }
4483
4484     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4485
4486     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4487         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4488
4489         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4490             # skip if nothing changed
4491         } elsif (is_valid_drivename($opt)) {
4492             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4493                 if defined($conf->{$opt});
4494             $conf->{$opt} = $conf->{pending}->{$opt};
4495         } else {
4496             $conf->{$opt} = $conf->{pending}->{$opt};
4497         }
4498
4499         delete $conf->{pending}->{$opt};
4500         PVE::QemuConfig->write_config($vmid, $conf);
4501     }
4502 }
4503
4504 my $safe_num_ne = sub {
4505     my ($a, $b) = @_;
4506
4507     return 0 if !defined($a) && !defined($b);
4508     return 1 if !defined($a);
4509     return 1 if !defined($b);
4510
4511     return $a != $b;
4512 };
4513
4514 my $safe_string_ne = sub {
4515     my ($a, $b) = @_;
4516
4517     return 0 if !defined($a) && !defined($b);
4518     return 1 if !defined($a);
4519     return 1 if !defined($b);
4520
4521     return $a ne $b;
4522 };
4523
4524 sub vmconfig_update_net {
4525     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4526
4527     my $newnet = parse_net($value);
4528
4529     if ($conf->{$opt}) {
4530         my $oldnet = parse_net($conf->{$opt});
4531
4532         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4533             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4534             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4535             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4536
4537             # for non online change, we try to hot-unplug
4538             die "skip\n" if !$hotplug;
4539             vm_deviceunplug($vmid, $conf, $opt);
4540         } else {
4541
4542             die "internal error" if $opt !~ m/net(\d+)/;
4543             my $iface = "tap${vmid}i$1";
4544
4545             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4546                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4547                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4548                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4549                 PVE::Network::tap_unplug($iface);
4550                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4551             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4552                 # Rate can be applied on its own but any change above needs to
4553                 # include the rate in tap_plug since OVS resets everything.
4554                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4555             }
4556
4557             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4558                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4559             }
4560
4561             return 1;
4562         }
4563     }
4564
4565     if ($hotplug) {
4566         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4567     } else {
4568         die "skip\n";
4569     }
4570 }
4571
4572 sub vmconfig_update_disk {
4573     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4574
4575     # fixme: do we need force?
4576
4577     my $drive = parse_drive($opt, $value);
4578
4579     if ($conf->{$opt}) {
4580
4581         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4582
4583             my $media = $drive->{media} || 'disk';
4584             my $oldmedia = $old_drive->{media} || 'disk';
4585             die "unable to change media type\n" if $media ne $oldmedia;
4586
4587             if (!drive_is_cdrom($old_drive)) {
4588
4589                 if ($drive->{file} ne $old_drive->{file}) {
4590
4591                     die "skip\n" if !$hotplug;
4592
4593                     # unplug and register as unused
4594                     vm_deviceunplug($vmid, $conf, $opt);
4595                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4596
4597                 } else {
4598                     # update existing disk
4599
4600                     # skip non hotpluggable value
4601                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4602                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4603                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4604                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4605                         die "skip\n";
4606                     }
4607
4608                     # apply throttle
4609                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4610                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4611                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4612                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4613                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4614                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4615                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4616                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4617                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4618                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4619                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4620                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4621                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4622                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4623                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4624                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4625                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4626                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4627
4628                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4629                                                    ($drive->{mbps} || 0)*1024*1024,
4630                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4631                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4632                                                    $drive->{iops} || 0,
4633                                                    $drive->{iops_rd} || 0,
4634                                                    $drive->{iops_wr} || 0,
4635                                                    ($drive->{mbps_max} || 0)*1024*1024,
4636                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4637                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4638                                                    $drive->{iops_max} || 0,
4639                                                    $drive->{iops_rd_max} || 0,
4640                                                    $drive->{iops_wr_max} || 0,
4641                                                    $drive->{bps_max_length} || 1,
4642                                                    $drive->{bps_rd_max_length} || 1,
4643                                                    $drive->{bps_wr_max_length} || 1,
4644                                                    $drive->{iops_max_length} || 1,
4645                                                    $drive->{iops_rd_max_length} || 1,
4646                                                    $drive->{iops_wr_max_length} || 1);
4647
4648                     }
4649
4650                     return 1;
4651                 }
4652
4653             } else { # cdrom
4654
4655                 if ($drive->{file} eq 'none') {
4656                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4657                     if (drive_is_cloudinit($old_drive)) {
4658                         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
4659                     }
4660                 } else {
4661                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4662                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4663                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4664                 }
4665
4666                 return 1;
4667             }
4668         }
4669     }
4670
4671     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4672     # hotplug new disks
4673     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4674     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4675 }
4676
4677 sub vm_start {
4678     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4679         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
4680
4681     PVE::QemuConfig->lock_config($vmid, sub {
4682         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4683
4684         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
4685
4686         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4687
4688         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4689
4690         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4691             vmconfig_apply_pending($vmid, $conf, $storecfg);
4692             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4693         }
4694
4695         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
4696
4697         my $defaults = load_defaults();
4698
4699         # set environment variable useful inside network script
4700         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4701
4702         my $local_volumes = {};
4703
4704         if ($targetstorage) {
4705             foreach_drive($conf, sub {
4706                 my ($ds, $drive) = @_;
4707
4708                 return if drive_is_cdrom($drive);
4709
4710                 my $volid = $drive->{file};
4711
4712                 return if !$volid;
4713
4714                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4715
4716                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4717                 return if $scfg->{shared};
4718                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
4719             });
4720
4721             my $format = undef;
4722
4723             foreach my $opt (sort keys %$local_volumes) {
4724
4725                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
4726                 my $drive = parse_drive($opt, $conf->{$opt});
4727
4728                 #if remote storage is specified, use default format
4729                 if ($targetstorage && $targetstorage ne "1") {
4730                     $storeid = $targetstorage;
4731                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
4732                     $format = $defFormat;
4733                 } else {
4734                     #else we use same format than original
4735                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4736                     $format = qemu_img_format($scfg, $volid);
4737                 }
4738
4739                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
4740                 my $newdrive = $drive;
4741                 $newdrive->{format} = $format;
4742                 $newdrive->{file} = $newvolid;
4743                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
4744                 $local_volumes->{$opt} = $drivestr;
4745                 #pass drive to conf for command line
4746                 $conf->{$opt} = $drivestr;
4747             }
4748         }
4749
4750         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4751
4752         my $migrate_port = 0;
4753         my $migrate_uri;
4754         if ($statefile) {
4755             if ($statefile eq 'tcp') {
4756                 my $localip = "localhost";
4757                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4758                 my $nodename = PVE::INotify::nodename();
4759
4760                 if (!defined($migration_type)) {
4761                     if (defined($datacenterconf->{migration}->{type})) {
4762                         $migration_type = $datacenterconf->{migration}->{type};
4763                     } else {
4764                         $migration_type = 'secure';
4765                     }
4766                 }
4767
4768                 if ($migration_type eq 'insecure') {
4769                     my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4770                     if ($migrate_network_addr) {
4771                         $localip = $migrate_network_addr;
4772                     } else {
4773                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4774                     }
4775
4776                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4777                 }
4778
4779                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4780                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4781                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4782                 push @$cmd, '-incoming', $migrate_uri;
4783                 push @$cmd, '-S';
4784
4785             } elsif ($statefile eq 'unix') {
4786                 # should be default for secure migrations as a ssh TCP forward
4787                 # tunnel is not deterministic reliable ready and fails regurarly
4788                 # to set up in time, so use UNIX socket forwards
4789                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
4790                 unlink $socket_addr;
4791
4792                 $migrate_uri = "unix:$socket_addr";
4793
4794                 push @$cmd, '-incoming', $migrate_uri;
4795                 push @$cmd, '-S';
4796
4797             } else {
4798                 push @$cmd, '-loadstate', $statefile;
4799             }
4800         } elsif ($paused) {
4801             push @$cmd, '-S';
4802         }
4803
4804         # host pci devices
4805         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4806           my $d = parse_hostpci($conf->{"hostpci$i"});
4807           next if !$d;
4808           my $pcidevices = $d->{pciid};
4809           foreach my $pcidevice (@$pcidevices) {
4810                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4811
4812                 my $info = pci_device_info("0000:$pciid");
4813                 die "IOMMU not present\n" if !check_iommu_support();
4814                 die "no pci device info for device '$pciid'\n" if !$info;
4815                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4816                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4817           }
4818         }
4819
4820         PVE::Storage::activate_volumes($storecfg, $vollist);
4821
4822         if (!check_running($vmid, 1)) {
4823             eval {
4824                 run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
4825                     outfunc => sub {}, errfunc => sub {});
4826             };
4827         }
4828
4829         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
4830                                                   : $defaults->{cpuunits};
4831
4832         my $start_timeout = $conf->{hugepages} ? 300 : 30;
4833         my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077);
4834
4835         my %properties = (
4836             Slice => 'qemu.slice',
4837             KillMode => 'none',
4838             CPUShares => $cpuunits
4839         );
4840
4841         if (my $cpulimit = $conf->{cpulimit}) {
4842             $properties{CPUQuota} = int($cpulimit * 100);
4843         }
4844         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
4845
4846         if ($conf->{hugepages}) {
4847
4848             my $code = sub {
4849                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
4850                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
4851
4852                 PVE::QemuServer::Memory::hugepages_mount();
4853                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
4854
4855                 eval  {
4856                     PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4857                     run_command($cmd, %run_params);
4858                 };
4859
4860                 if (my $err = $@) {
4861                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
4862                     die $err;
4863                 }
4864
4865                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
4866             };
4867             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
4868
4869         } else {
4870             eval  {
4871                 PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4872                 run_command($cmd, %run_params);
4873             };
4874         }
4875
4876         if (my $err = $@) {
4877             # deactivate volumes if start fails
4878             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
4879             die "start failed: $err";
4880         }
4881
4882         print "migration listens on $migrate_uri\n" if $migrate_uri;
4883
4884         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
4885             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4886             warn $@ if $@;
4887         }
4888
4889         #start nbd server for storage migration
4890         if ($targetstorage) {
4891             my $nodename = PVE::INotify::nodename();
4892             my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4893             my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
4894             my $pfamily = PVE::Tools::get_host_address_family($nodename);
4895             $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4896
4897             vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${migrate_port}" } } );
4898
4899             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4900
4901             foreach my $opt (sort keys %$local_volumes) {
4902                 my $volid = $local_volumes->{$opt};
4903                 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
4904                 my $migrate_storage_uri = "nbd:${localip}:${migrate_port}:exportname=drive-$opt";
4905                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
4906             }
4907         }
4908
4909         if ($migratedfrom) {
4910             eval {
4911                 set_migration_caps($vmid);
4912             };
4913             warn $@ if $@;
4914
4915             if ($spice_port) {
4916                 print "spice listens on port $spice_port\n";
4917                 if ($spice_ticket) {
4918                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4919                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4920                 }
4921             }
4922
4923         } else {
4924             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4925                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4926                     if $conf->{balloon};
4927             }
4928
4929             foreach my $opt (keys %$conf) {
4930                 next if $opt !~  m/^net\d+$/;
4931                 my $nicconf = parse_net($conf->{$opt});
4932                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4933             }
4934         }
4935
4936         vm_mon_cmd_nocheck($vmid, 'qom-set',
4937                     path => "machine/peripheral/balloon0",
4938                     property => "guest-stats-polling-interval",
4939                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4940
4941     });
4942 }
4943
4944 sub vm_mon_cmd {
4945     my ($vmid, $execute, %params) = @_;
4946
4947     my $cmd = { execute => $execute, arguments => \%params };
4948     vm_qmp_command($vmid, $cmd);
4949 }
4950
4951 sub vm_mon_cmd_nocheck {
4952     my ($vmid, $execute, %params) = @_;
4953
4954     my $cmd = { execute => $execute, arguments => \%params };
4955     vm_qmp_command($vmid, $cmd, 1);
4956 }
4957
4958 sub vm_qmp_command {
4959     my ($vmid, $cmd, $nocheck) = @_;
4960
4961     my $res;
4962
4963     my $timeout;
4964     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4965         $timeout = $cmd->{arguments}->{timeout};
4966         delete $cmd->{arguments}->{timeout};
4967     }
4968
4969     eval {
4970         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4971         my $sname = qmp_socket($vmid);
4972         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4973             my $qmpclient = PVE::QMPClient->new();
4974
4975             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4976         } else {
4977             die "unable to open monitor socket\n";
4978         }
4979     };
4980     if (my $err = $@) {
4981         syslog("err", "VM $vmid qmp command failed - $err");
4982         die $err;
4983     }
4984
4985     return $res;
4986 }
4987
4988 sub vm_human_monitor_command {
4989     my ($vmid, $cmdline) = @_;
4990
4991     my $res;
4992
4993     my $cmd = {
4994         execute => 'human-monitor-command',
4995         arguments => { 'command-line' => $cmdline},
4996     };
4997
4998     return vm_qmp_command($vmid, $cmd);
4999 }
5000
5001 sub vm_commandline {
5002     my ($storecfg, $vmid) = @_;
5003
5004     my $conf = PVE::QemuConfig->load_config($vmid);
5005
5006     my $defaults = load_defaults();
5007
5008     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
5009
5010     return PVE::Tools::cmd2string($cmd);
5011 }
5012
5013 sub vm_reset {
5014     my ($vmid, $skiplock) = @_;
5015
5016     PVE::QemuConfig->lock_config($vmid, sub {
5017
5018         my $conf = PVE::QemuConfig->load_config($vmid);
5019
5020         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5021
5022         vm_mon_cmd($vmid, "system_reset");
5023     });
5024 }
5025
5026 sub get_vm_volumes {
5027     my ($conf) = @_;
5028
5029     my $vollist = [];
5030     foreach_volid($conf, sub {
5031         my ($volid, $attr) = @_;
5032
5033         return if $volid =~ m|^/|;
5034
5035         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5036         return if !$sid;
5037
5038         push @$vollist, $volid;
5039     });
5040
5041     return $vollist;
5042 }
5043
5044 sub vm_stop_cleanup {
5045     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5046
5047     eval {
5048
5049         if (!$keepActive) {
5050             my $vollist = get_vm_volumes($conf);
5051             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5052         }
5053
5054         foreach my $ext (qw(mon qmp pid vnc qga)) {
5055             unlink "/var/run/qemu-server/${vmid}.$ext";
5056         }
5057
5058         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5059     };
5060     warn $@ if $@; # avoid errors - just warn
5061 }
5062
5063 # Note: use $nockeck to skip tests if VM configuration file exists.
5064 # We need that when migration VMs to other nodes (files already moved)
5065 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5066 sub vm_stop {
5067     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5068
5069     $force = 1 if !defined($force) && !$shutdown;
5070
5071     if ($migratedfrom){
5072         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5073         kill 15, $pid if $pid;
5074         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5075         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5076         return;
5077     }
5078
5079     PVE::QemuConfig->lock_config($vmid, sub {
5080
5081         my $pid = check_running($vmid, $nocheck);
5082         return if !$pid;
5083
5084         my $conf;
5085         if (!$nocheck) {
5086             $conf = PVE::QemuConfig->load_config($vmid);
5087             PVE::QemuConfig->check_lock($conf) if !$skiplock;
5088             if (!defined($timeout) && $shutdown && $conf->{startup}) {
5089                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5090                 $timeout = $opts->{down} if $opts->{down};
5091             }
5092         }
5093
5094         $timeout = 60 if !defined($timeout);
5095
5096         eval {
5097             if ($shutdown) {
5098                 if (defined($conf) && $conf->{agent}) {
5099                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
5100                 } else {
5101                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
5102                 }
5103             } else {
5104                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
5105             }
5106         };
5107         my $err = $@;
5108
5109         if (!$err) {
5110             my $count = 0;
5111             while (($count < $timeout) && check_running($vmid, $nocheck)) {
5112                 $count++;
5113                 sleep 1;
5114             }
5115
5116             if ($count >= $timeout) {
5117                 if ($force) {
5118                     warn "VM still running - terminating now with SIGTERM\n";
5119                     kill 15, $pid;
5120                 } else {
5121                     die "VM quit/powerdown failed - got timeout\n";
5122                 }
5123             } else {
5124                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5125                 return;
5126             }
5127         } else {
5128             if ($force) {
5129                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5130                 kill 15, $pid;
5131             } else {
5132                 die "VM quit/powerdown failed\n";
5133             }
5134         }
5135
5136         # wait again
5137         $timeout = 10;
5138
5139         my $count = 0;
5140         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5141             $count++;
5142             sleep 1;
5143         }
5144
5145         if ($count >= $timeout) {
5146             warn "VM still running - terminating now with SIGKILL\n";
5147             kill 9, $pid;
5148             sleep 1;
5149         }
5150
5151         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5152    });
5153 }
5154
5155 sub vm_suspend {
5156     my ($vmid, $skiplock) = @_;
5157
5158     PVE::QemuConfig->lock_config($vmid, sub {
5159
5160         my $conf = PVE::QemuConfig->load_config($vmid);
5161
5162         PVE::QemuConfig->check_lock($conf)
5163             if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5164
5165         vm_mon_cmd($vmid, "stop");
5166     });
5167 }
5168
5169 sub vm_resume {
5170     my ($vmid, $skiplock, $nocheck) = @_;
5171
5172     PVE::QemuConfig->lock_config($vmid, sub {
5173
5174         if (!$nocheck) {
5175
5176             my $conf = PVE::QemuConfig->load_config($vmid);
5177
5178             PVE::QemuConfig->check_lock($conf)
5179                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5180
5181             vm_mon_cmd($vmid, "cont");
5182
5183         } else {
5184             vm_mon_cmd_nocheck($vmid, "cont");
5185         }
5186     });
5187 }
5188
5189 sub vm_sendkey {
5190     my ($vmid, $skiplock, $key) = @_;
5191
5192     PVE::QemuConfig->lock_config($vmid, sub {
5193
5194         my $conf = PVE::QemuConfig->load_config($vmid);
5195
5196         # there is no qmp command, so we use the human monitor command
5197         vm_human_monitor_command($vmid, "sendkey $key");
5198     });
5199 }
5200
5201 sub vm_destroy {
5202     my ($storecfg, $vmid, $skiplock) = @_;
5203
5204     PVE::QemuConfig->lock_config($vmid, sub {
5205
5206         my $conf = PVE::QemuConfig->load_config($vmid);
5207
5208         if (!check_running($vmid)) {
5209             destroy_vm($storecfg, $vmid, undef, $skiplock);
5210         } else {
5211             die "VM $vmid is running - destroy failed\n";
5212         }
5213     });
5214 }
5215
5216 # pci helpers
5217
5218 sub file_write {
5219     my ($filename, $buf) = @_;
5220
5221     my $fh = IO::File->new($filename, "w");
5222     return undef if !$fh;
5223
5224     my $res = print $fh $buf;
5225
5226     $fh->close();
5227
5228     return $res;
5229 }
5230
5231 sub pci_device_info {
5232     my ($name) = @_;
5233
5234     my $res;
5235
5236     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
5237     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
5238
5239     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
5240     return undef if !defined($irq) || $irq !~ m/^\d+$/;
5241
5242     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
5243     return undef if !defined($vendor) || $vendor !~ s/^0x//;
5244
5245     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
5246     return undef if !defined($product) || $product !~ s/^0x//;
5247
5248     $res = {
5249         name => $name,
5250         vendor => $vendor,
5251         product => $product,
5252         domain => $domain,
5253         bus => $bus,
5254         slot => $slot,
5255         func => $func,
5256         irq => $irq,
5257         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
5258     };
5259
5260     return $res;
5261 }
5262
5263 sub pci_dev_reset {
5264     my ($dev) = @_;
5265
5266     my $name = $dev->{name};
5267
5268     my $fn = "$pcisysfs/devices/$name/reset";
5269
5270     return file_write($fn, "1");
5271 }
5272
5273 sub pci_dev_bind_to_vfio {
5274     my ($dev) = @_;
5275
5276     my $name = $dev->{name};
5277
5278     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5279
5280     if (!-d $vfio_basedir) {
5281         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5282     }
5283     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5284
5285     my $testdir = "$vfio_basedir/$name";
5286     return 1 if -d $testdir;
5287
5288     my $data = "$dev->{vendor} $dev->{product}";
5289     return undef if !file_write("$vfio_basedir/new_id", $data);
5290
5291     my $fn = "$pcisysfs/devices/$name/driver/unbind";
5292     if (!file_write($fn, $name)) {
5293         return undef if -f $fn;
5294     }
5295
5296     $fn = "$vfio_basedir/bind";
5297     if (! -d $testdir) {
5298         return undef if !file_write($fn, $name);
5299     }
5300
5301     return -d $testdir;
5302 }
5303
5304 sub pci_dev_group_bind_to_vfio {
5305     my ($pciid) = @_;
5306
5307     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5308
5309     if (!-d $vfio_basedir) {
5310         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5311     }
5312     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5313
5314     # get IOMMU group devices
5315     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
5316       my @devs = grep /^0000:/, readdir($D);
5317     closedir($D);
5318
5319     foreach my $pciid (@devs) {
5320         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
5321
5322         # pci bridges, switches or root ports are not supported
5323         # they have a pci_bus subdirectory so skip them
5324         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
5325
5326         my $info = pci_device_info($1);
5327         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
5328     }
5329
5330     return 1;
5331 }
5332
5333 # vzdump restore implementaion
5334
5335 sub tar_archive_read_firstfile {
5336     my $archive = shift;
5337
5338     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5339
5340     # try to detect archive type first
5341     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5342         die "unable to open file '$archive'\n";
5343     my $firstfile = <$fh>;
5344     kill 15, $pid;
5345     close $fh;
5346
5347     die "ERROR: archive contaions no data\n" if !$firstfile;
5348     chomp $firstfile;
5349
5350     return $firstfile;
5351 }
5352
5353 sub tar_restore_cleanup {
5354     my ($storecfg, $statfile) = @_;
5355
5356     print STDERR "starting cleanup\n";
5357
5358     if (my $fd = IO::File->new($statfile, "r")) {
5359         while (defined(my $line = <$fd>)) {
5360             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5361                 my $volid = $2;
5362                 eval {
5363                     if ($volid =~ m|^/|) {
5364                         unlink $volid || die 'unlink failed\n';
5365                     } else {
5366                         PVE::Storage::vdisk_free($storecfg, $volid);
5367                     }
5368                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5369                 };
5370                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5371             } else {
5372                 print STDERR "unable to parse line in statfile - $line";
5373             }
5374         }
5375         $fd->close();
5376     }
5377 }
5378
5379 sub restore_archive {
5380     my ($archive, $vmid, $user, $opts) = @_;
5381
5382     my $format = $opts->{format};
5383     my $comp;
5384
5385     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5386         $format = 'tar' if !$format;
5387         $comp = 'gzip';
5388     } elsif ($archive =~ m/\.tar$/) {
5389         $format = 'tar' if !$format;
5390     } elsif ($archive =~ m/.tar.lzo$/) {
5391         $format = 'tar' if !$format;
5392         $comp = 'lzop';
5393     } elsif ($archive =~ m/\.vma$/) {
5394         $format = 'vma' if !$format;
5395     } elsif ($archive =~ m/\.vma\.gz$/) {
5396         $format = 'vma' if !$format;
5397         $comp = 'gzip';
5398     } elsif ($archive =~ m/\.vma\.lzo$/) {
5399         $format = 'vma' if !$format;
5400         $comp = 'lzop';
5401     } else {
5402         $format = 'vma' if !$format; # default
5403     }
5404
5405     # try to detect archive format
5406     if ($format eq 'tar') {
5407         return restore_tar_archive($archive, $vmid, $user, $opts);
5408     } else {
5409         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5410     }
5411 }
5412
5413 sub restore_update_config_line {
5414     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5415
5416     return if $line =~ m/^\#qmdump\#/;
5417     return if $line =~ m/^\#vzdump\#/;
5418     return if $line =~ m/^lock:/;
5419     return if $line =~ m/^unused\d+:/;
5420     return if $line =~ m/^parent:/;
5421     return if $line =~ m/^template:/; # restored VM is never a template
5422
5423     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5424     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5425         # try to convert old 1.X settings
5426         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5427         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5428             my ($model, $macaddr) = split(/\=/, $devconfig);
5429             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5430             my $net = {
5431                 model => $model,
5432                 bridge => "vmbr$ind",
5433                 macaddr => $macaddr,
5434             };
5435             my $netstr = print_net($net);
5436
5437             print $outfd "net$cookie->{netcount}: $netstr\n";
5438             $cookie->{netcount}++;
5439         }
5440     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5441         my ($id, $netstr) = ($1, $2);
5442         my $net = parse_net($netstr);
5443         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5444         $netstr = print_net($net);
5445         print $outfd "$id: $netstr\n";
5446     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5447         my $virtdev = $1;
5448         my $value = $3;
5449         my $di = parse_drive($virtdev, $value);
5450         if (defined($di->{backup}) && !$di->{backup}) {
5451             print $outfd "#$line";
5452         } elsif ($map->{$virtdev}) {
5453             delete $di->{format}; # format can change on restore
5454             $di->{file} = $map->{$virtdev};
5455             $value = print_drive($vmid, $di);
5456             print $outfd "$virtdev: $value\n";
5457         } else {
5458             print $outfd $line;
5459         }
5460     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5461         my ($uuid, $uuid_str);
5462         UUID::generate($uuid);
5463         UUID::unparse($uuid, $uuid_str);
5464         my $smbios1 = parse_smbios1($2);
5465         $smbios1->{uuid} = $uuid_str;
5466         print $outfd $1.print_smbios1($smbios1)."\n";
5467     } else {
5468         print $outfd $line;
5469     }
5470 }
5471
5472 sub scan_volids {
5473     my ($cfg, $vmid) = @_;
5474
5475     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5476
5477     my $volid_hash = {};
5478     foreach my $storeid (keys %$info) {
5479         foreach my $item (@{$info->{$storeid}}) {
5480             next if !($item->{volid} && $item->{size});
5481             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5482             $volid_hash->{$item->{volid}} = $item;
5483         }
5484     }
5485
5486     return $volid_hash;
5487 }
5488
5489 sub is_volume_in_use {
5490     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5491
5492     my $path = PVE::Storage::path($storecfg, $volid);
5493
5494     my $scan_config = sub {
5495         my ($cref, $snapname) = @_;
5496
5497         foreach my $key (keys %$cref) {
5498             my $value = $cref->{$key};
5499             if (is_valid_drivename($key)) {
5500                 next if $skip_drive && $key eq $skip_drive;
5501                 my $drive = parse_drive($key, $value);
5502                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5503                 return 1 if $volid eq $drive->{file};
5504                 if ($drive->{file} =~ m!^/!) {
5505                     return 1 if $drive->{file} eq $path;
5506                 } else {
5507                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5508                     next if !$storeid;
5509                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5510                     next if !$scfg;
5511                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5512                 }
5513             }
5514         }
5515
5516         return 0;
5517     };
5518
5519     return 1 if &$scan_config($conf);
5520
5521     undef $skip_drive;
5522
5523     foreach my $snapname (keys %{$conf->{snapshots}}) {
5524         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5525     }
5526
5527     return 0;
5528 }
5529
5530 sub update_disksize {
5531     my ($vmid, $conf, $volid_hash) = @_;
5532
5533     my $changes;
5534
5535     # used and unused disks
5536     my $referenced = {};
5537
5538     # Note: it is allowed to define multiple storages with same path (alias), so
5539     # we need to check both 'volid' and real 'path' (two different volid can point
5540     # to the same path).
5541
5542     my $referencedpath = {};
5543
5544     # update size info
5545     foreach my $opt (keys %$conf) {
5546         if (is_valid_drivename($opt)) {
5547             my $drive = parse_drive($opt, $conf->{$opt});
5548             my $volid = $drive->{file};
5549             next if !$volid;
5550
5551             $referenced->{$volid} = 1;
5552             if ($volid_hash->{$volid} &&
5553                 (my $path = $volid_hash->{$volid}->{path})) {
5554                 $referencedpath->{$path} = 1;
5555             }
5556
5557             next if drive_is_cdrom($drive);
5558             next if !$volid_hash->{$volid};
5559
5560             $drive->{size} = $volid_hash->{$volid}->{size};
5561             my $new = print_drive($vmid, $drive);
5562             if ($new ne $conf->{$opt}) {
5563                 $changes = 1;
5564                 $conf->{$opt} = $new;
5565             }
5566         }
5567     }
5568
5569     # remove 'unusedX' entry if volume is used
5570     foreach my $opt (keys %$conf) {
5571         next if $opt !~ m/^unused\d+$/;
5572         my $volid = $conf->{$opt};
5573         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5574         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
5575             $changes = 1;
5576             delete $conf->{$opt};
5577         }
5578
5579         $referenced->{$volid} = 1;
5580         $referencedpath->{$path} = 1 if $path;
5581     }
5582
5583     foreach my $volid (sort keys %$volid_hash) {
5584         next if $volid =~ m/vm-$vmid-state-/;
5585         next if $referenced->{$volid};
5586         my $path = $volid_hash->{$volid}->{path};
5587         next if !$path; # just to be sure
5588         next if $referencedpath->{$path};
5589         $changes = 1;
5590         PVE::QemuConfig->add_unused_volume($conf, $volid);
5591         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
5592     }
5593
5594     return $changes;
5595 }
5596
5597 sub rescan {
5598     my ($vmid, $nolock) = @_;
5599
5600     my $cfg = PVE::Storage::config();
5601
5602     my $volid_hash = scan_volids($cfg, $vmid);
5603
5604     my $updatefn =  sub {
5605         my ($vmid) = @_;
5606
5607         my $conf = PVE::QemuConfig->load_config($vmid);
5608
5609         PVE::QemuConfig->check_lock($conf);
5610
5611         my $vm_volids = {};
5612         foreach my $volid (keys %$volid_hash) {
5613             my $info = $volid_hash->{$volid};
5614             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5615         }
5616
5617         my $changes = update_disksize($vmid, $conf, $vm_volids);
5618
5619         PVE::QemuConfig->write_config($vmid, $conf) if $changes;
5620     };
5621
5622     if (defined($vmid)) {
5623         if ($nolock) {
5624             &$updatefn($vmid);
5625         } else {
5626             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5627         }
5628     } else {
5629         my $vmlist = config_list();
5630         foreach my $vmid (keys %$vmlist) {
5631             if ($nolock) {
5632                 &$updatefn($vmid);
5633             } else {
5634                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5635             }
5636         }
5637     }
5638 }
5639
5640 sub restore_vma_archive {
5641     my ($archive, $vmid, $user, $opts, $comp) = @_;
5642
5643     my $input = $archive eq '-' ? "<&STDIN" : undef;
5644     my $readfrom = $archive;
5645
5646     my $uncomp = '';
5647     if ($comp) {
5648         $readfrom = '-';
5649         my $qarchive = PVE::Tools::shellquote($archive);
5650         if ($comp eq 'gzip') {
5651             $uncomp = "zcat $qarchive|";
5652         } elsif ($comp eq 'lzop') {
5653             $uncomp = "lzop -d -c $qarchive|";
5654         } else {
5655             die "unknown compression method '$comp'\n";
5656         }
5657
5658     }
5659
5660     my $tmpdir = "/var/tmp/vzdumptmp$$";
5661     rmtree $tmpdir;
5662
5663     # disable interrupts (always do cleanups)
5664     local $SIG{INT} =
5665         local $SIG{TERM} =
5666         local $SIG{QUIT} =
5667         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
5668
5669     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5670     POSIX::mkfifo($mapfifo, 0600);
5671     my $fifofh;
5672
5673     my $openfifo = sub {
5674         open($fifofh, '>', $mapfifo) || die $!;
5675     };
5676
5677     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5678
5679     my $oldtimeout;
5680     my $timeout = 5;
5681
5682     my $devinfo = {};
5683
5684     my $rpcenv = PVE::RPCEnvironment::get();
5685
5686     my $conffile = PVE::QemuConfig->config_file($vmid);
5687     my $tmpfn = "$conffile.$$.tmp";
5688
5689     # Note: $oldconf is undef if VM does not exists
5690     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
5691     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
5692
5693     my $print_devmap = sub {
5694         my $virtdev_hash = {};
5695
5696         my $cfgfn = "$tmpdir/qemu-server.conf";
5697
5698         # we can read the config - that is already extracted
5699         my $fh = IO::File->new($cfgfn, "r") ||
5700             "unable to read qemu-server.conf - $!\n";
5701
5702         my $fwcfgfn = "$tmpdir/qemu-server.fw";
5703         if (-f $fwcfgfn) {
5704             my $pve_firewall_dir = '/etc/pve/firewall';
5705             mkdir $pve_firewall_dir; # make sure the dir exists
5706             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
5707         }
5708
5709         while (defined(my $line = <$fh>)) {
5710             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5711                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5712                 die "archive does not contain data for drive '$virtdev'\n"
5713                     if !$devinfo->{$devname};
5714                 if (defined($opts->{storage})) {
5715                     $storeid = $opts->{storage} || 'local';
5716                 } elsif (!$storeid) {
5717                     $storeid = 'local';
5718                 }
5719                 $format = 'raw' if !$format;
5720                 $devinfo->{$devname}->{devname} = $devname;
5721                 $devinfo->{$devname}->{virtdev} = $virtdev;
5722                 $devinfo->{$devname}->{format} = $format;
5723                 $devinfo->{$devname}->{storeid} = $storeid;
5724
5725                 # check permission on storage
5726                 my $pool = $opts->{pool}; # todo: do we need that?
5727                 if ($user ne 'root@pam') {
5728                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5729                 }
5730
5731                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5732             }
5733         }
5734
5735         foreach my $devname (keys %$devinfo) {
5736             die "found no device mapping information for device '$devname'\n"
5737                 if !$devinfo->{$devname}->{virtdev};
5738         }
5739
5740         my $cfg = PVE::Storage::config();
5741
5742         # create empty/temp config
5743         if ($oldconf) {
5744             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5745             foreach_drive($oldconf, sub {
5746                 my ($ds, $drive) = @_;
5747
5748                 return if drive_is_cdrom($drive);
5749
5750                 my $volid = $drive->{file};
5751
5752                 return if !$volid || $volid =~ m|^/|;
5753
5754                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5755                 return if !$path || !$owner || ($owner != $vmid);
5756
5757                 # Note: only delete disk we want to restore
5758                 # other volumes will become unused
5759                 if ($virtdev_hash->{$ds}) {
5760                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
5761                     if (my $err = $@) {
5762                         warn $err;
5763                     }
5764                 }
5765             });
5766
5767             # delete vmstate files
5768             # since after the restore we have no snapshots anymore
5769             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5770                 my $snap = $oldconf->{snapshots}->{$snapname};
5771                 if ($snap->{vmstate}) {
5772                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
5773                     if (my $err = $@) {
5774                         warn $err;
5775                     }
5776                 }
5777             }
5778         }
5779
5780         my $map = {};
5781         foreach my $virtdev (sort keys %$virtdev_hash) {
5782             my $d = $virtdev_hash->{$virtdev};
5783             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5784             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5785
5786             # test if requested format is supported
5787             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5788             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5789             $d->{format} = $defFormat if !$supported;
5790
5791             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5792                                                   $d->{format}, undef, $alloc_size);
5793             print STDERR "new volume ID is '$volid'\n";
5794             $d->{volid} = $volid;
5795             my $path = PVE::Storage::path($cfg, $volid);
5796
5797             PVE::Storage::activate_volumes($cfg,[$volid]);
5798
5799             my $write_zeros = 1;
5800             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
5801                 $write_zeros = 0;
5802             }
5803
5804             print $fifofh "format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
5805
5806             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5807             $map->{$virtdev} = $volid;
5808         }
5809
5810         $fh->seek(0, 0) || die "seek failed - $!\n";
5811
5812         my $outfd = new IO::File ($tmpfn, "w") ||
5813             die "unable to write config for VM $vmid\n";
5814
5815         my $cookie = { netcount => 0 };
5816         while (defined(my $line = <$fh>)) {
5817             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5818         }
5819
5820         $fh->close();
5821         $outfd->close();
5822     };
5823
5824     eval {
5825         # enable interrupts
5826         local $SIG{INT} =
5827             local $SIG{TERM} =
5828             local $SIG{QUIT} =
5829             local $SIG{HUP} =
5830             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
5831         local $SIG{ALRM} = sub { die "got timeout\n"; };
5832
5833         $oldtimeout = alarm($timeout);
5834
5835         my $parser = sub {
5836             my $line = shift;
5837
5838             print "$line\n";
5839
5840             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5841                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5842                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5843             } elsif ($line =~ m/^CTIME: /) {
5844                 # we correctly received the vma config, so we can disable
5845                 # the timeout now for disk allocation (set to 10 minutes, so
5846                 # that we always timeout if something goes wrong)
5847                 alarm(600);
5848                 &$print_devmap();
5849                 print $fifofh "done\n";
5850                 my $tmp = $oldtimeout || 0;
5851                 $oldtimeout = undef;
5852                 alarm($tmp);
5853                 close($fifofh);
5854             }
5855         };
5856
5857         print "restore vma archive: $cmd\n";
5858         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5859     };
5860     my $err = $@;
5861
5862     alarm($oldtimeout) if $oldtimeout;
5863
5864     my $vollist = [];
5865     foreach my $devname (keys %$devinfo) {
5866         my $volid = $devinfo->{$devname}->{volid};
5867         push @$vollist, $volid if $volid;
5868     }
5869
5870     my $cfg = PVE::Storage::config();
5871     PVE::Storage::deactivate_volumes($cfg, $vollist);
5872
5873     unlink $mapfifo;
5874
5875     if ($err) {
5876         rmtree $tmpdir;
5877         unlink $tmpfn;
5878
5879         foreach my $devname (keys %$devinfo) {
5880             my $volid = $devinfo->{$devname}->{volid};
5881             next if !$volid;
5882             eval {
5883                 if ($volid =~ m|^/|) {
5884                     unlink $volid || die 'unlink failed\n';
5885                 } else {
5886                     PVE::Storage::vdisk_free($cfg, $volid);
5887                 }
5888                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5889             };
5890             print STDERR "unable to cleanup '$volid' - $@" if $@;
5891         }
5892         die $err;
5893     }
5894
5895     rmtree $tmpdir;
5896
5897     rename($tmpfn, $conffile) ||
5898         die "unable to commit configuration file '$conffile'\n";
5899
5900     PVE::Cluster::cfs_update(); # make sure we read new file
5901
5902     eval { rescan($vmid, 1); };
5903     warn $@ if $@;
5904 }
5905
5906 sub restore_tar_archive {
5907     my ($archive, $vmid, $user, $opts) = @_;
5908
5909     if ($archive ne '-') {
5910         my $firstfile = tar_archive_read_firstfile($archive);
5911         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5912             if $firstfile ne 'qemu-server.conf';
5913     }
5914
5915     my $storecfg = PVE::Storage::config();
5916
5917     # destroy existing data - keep empty config
5918     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
5919     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5920
5921     my $tocmd = "/usr/lib/qemu-server/qmextract";
5922
5923     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5924     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5925     $tocmd .= ' --prealloc' if $opts->{prealloc};
5926     $tocmd .= ' --info' if $opts->{info};
5927
5928     # tar option "xf" does not autodetect compression when read from STDIN,
5929     # so we pipe to zcat
5930     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5931         PVE::Tools::shellquote("--to-command=$tocmd");
5932
5933     my $tmpdir = "/var/tmp/vzdumptmp$$";
5934     mkpath $tmpdir;
5935
5936     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5937     local $ENV{VZDUMP_VMID} = $vmid;
5938     local $ENV{VZDUMP_USER} = $user;
5939
5940     my $conffile = PVE::QemuConfig->config_file($vmid);
5941     my $tmpfn = "$conffile.$$.tmp";
5942
5943     # disable interrupts (always do cleanups)
5944     local $SIG{INT} =
5945         local $SIG{TERM} =
5946         local $SIG{QUIT} =
5947         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
5948
5949     eval {
5950         # enable interrupts
5951         local $SIG{INT} =
5952             local $SIG{TERM} =
5953             local $SIG{QUIT} =
5954             local $SIG{HUP} =
5955             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
5956
5957         if ($archive eq '-') {
5958             print "extracting archive from STDIN\n";
5959             run_command($cmd, input => "<&STDIN");
5960         } else {
5961             print "extracting archive '$archive'\n";
5962             run_command($cmd);
5963         }
5964
5965         return if $opts->{info};
5966
5967         # read new mapping
5968         my $map = {};
5969         my $statfile = "$tmpdir/qmrestore.stat";
5970         if (my $fd = IO::File->new($statfile, "r")) {
5971             while (defined (my $line = <$fd>)) {
5972                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5973                     $map->{$1} = $2 if $1;
5974                 } else {
5975                     print STDERR "unable to parse line in statfile - $line\n";
5976                 }
5977             }
5978             $fd->close();
5979         }
5980
5981         my $confsrc = "$tmpdir/qemu-server.conf";
5982
5983         my $srcfd = new IO::File($confsrc, "r") ||
5984             die "unable to open file '$confsrc'\n";
5985
5986         my $outfd = new IO::File ($tmpfn, "w") ||
5987             die "unable to write config for VM $vmid\n";
5988
5989         my $cookie = { netcount => 0 };
5990         while (defined (my $line = <$srcfd>)) {
5991             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5992         }
5993
5994         $srcfd->close();
5995         $outfd->close();
5996     };
5997     my $err = $@;
5998
5999     if ($err) {
6000
6001         unlink $tmpfn;
6002
6003         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6004
6005         die $err;
6006     }
6007
6008     rmtree $tmpdir;
6009
6010     rename $tmpfn, $conffile ||
6011         die "unable to commit configuration file '$conffile'\n";
6012
6013     PVE::Cluster::cfs_update(); # make sure we read new file
6014
6015     eval { rescan($vmid, 1); };
6016     warn $@ if $@;
6017 };
6018
6019 sub foreach_storage_used_by_vm {
6020     my ($conf, $func) = @_;
6021
6022     my $sidhash = {};
6023
6024     foreach_drive($conf, sub {
6025         my ($ds, $drive) = @_;
6026         return if drive_is_cdrom($drive);
6027
6028         my $volid = $drive->{file};
6029
6030         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6031         $sidhash->{$sid} = $sid if $sid;
6032     });
6033
6034     foreach my $sid (sort keys %$sidhash) {
6035         &$func($sid);
6036     }
6037 }
6038
6039 sub do_snapshots_with_qemu {
6040     my ($storecfg, $volid) = @_;
6041
6042     my $storage_name = PVE::Storage::parse_volume_id($volid);
6043
6044     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
6045         && !$storecfg->{ids}->{$storage_name}->{krbd}){
6046         return 1;
6047     }
6048
6049     if ($volid =~ m/\.(qcow2|qed)$/){
6050         return 1;
6051     }
6052
6053     return undef;
6054 }
6055
6056 sub qga_check_running {
6057     my ($vmid) = @_;
6058
6059     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
6060     if ($@) {
6061         warn "Qemu Guest Agent is not running - $@";
6062         return 0;
6063     }
6064     return 1;
6065 }
6066
6067 sub template_create {
6068     my ($vmid, $conf, $disk) = @_;
6069
6070     my $storecfg = PVE::Storage::config();
6071
6072     foreach_drive($conf, sub {
6073         my ($ds, $drive) = @_;
6074
6075         return if drive_is_cdrom($drive);
6076         return if $disk && $ds ne $disk;
6077
6078         my $volid = $drive->{file};
6079         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6080
6081         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6082         $drive->{file} = $voliddst;
6083         $conf->{$ds} = print_drive($vmid, $drive);
6084         PVE::QemuConfig->write_config($vmid, $conf);
6085     });
6086 }
6087
6088 sub qemu_img_convert {
6089     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6090
6091     my $storecfg = PVE::Storage::config();
6092     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6093     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6094
6095     if ($src_storeid && $dst_storeid) {
6096
6097         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6098
6099         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6100         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6101
6102         my $src_format = qemu_img_format($src_scfg, $src_volname);
6103         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6104
6105         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6106         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6107
6108         my $cmd = [];
6109         push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6110         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
6111         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path;
6112         if ($is_zero_initialized) {
6113             push @$cmd, "zeroinit:$dst_path";
6114         } else {
6115             push @$cmd, $dst_path;
6116         }
6117
6118         my $parser = sub {
6119             my $line = shift;
6120             if($line =~ m/\((\S+)\/100\%\)/){
6121                 my $percent = $1;
6122                 my $transferred = int($size * $percent / 100);
6123                 my $remaining = $size - $transferred;
6124
6125                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6126             }
6127
6128         };
6129
6130         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6131         my $err = $@;
6132         die "copy failed: $err" if $err;
6133     }
6134 }
6135
6136 sub qemu_img_format {
6137     my ($scfg, $volname) = @_;
6138
6139     if ($scfg->{path} && $volname =~ m/\.($QEMU_FORMAT_RE)$/) {
6140         return $1;
6141     } else {
6142         return "raw";
6143     }
6144 }
6145
6146 sub qemu_drive_mirror {
6147     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga) = @_;
6148
6149     $jobs = {} if !$jobs;
6150
6151     my $qemu_target;
6152     my $format;
6153     $jobs->{"drive-$drive"} = {};
6154
6155     if ($dst_volid =~ /^nbd:/) {
6156         $qemu_target = $dst_volid;
6157         $format = "nbd";
6158     } else {
6159         my $storecfg = PVE::Storage::config();
6160         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6161
6162         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6163
6164         $format = qemu_img_format($dst_scfg, $dst_volname);
6165
6166         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6167
6168         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6169     }
6170
6171     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6172     $opts->{format} = $format if $format;
6173
6174     print "drive mirror is starting for drive-$drive\n";
6175
6176     eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); }; #if a job already run for this device,it's throw an error
6177
6178     if (my $err = $@) {
6179         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6180         die "mirroring error: $err";
6181     }
6182
6183     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6184 }
6185
6186 sub qemu_drive_mirror_monitor {
6187     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6188
6189     eval {
6190         my $err_complete = 0;
6191
6192         while (1) {
6193             die "storage migration timed out\n" if $err_complete > 300;
6194
6195             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6196
6197             my $running_mirror_jobs = {};
6198             foreach my $stat (@$stats) {
6199                 next if $stat->{type} ne 'mirror';
6200                 $running_mirror_jobs->{$stat->{device}} = $stat;
6201             }
6202
6203             my $readycounter = 0;
6204
6205             foreach my $job (keys %$jobs) {
6206
6207                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6208                     print "$job : finished\n";
6209                     delete $jobs->{$job};
6210                     next;
6211                 }
6212
6213                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6214
6215                 my $busy = $running_mirror_jobs->{$job}->{busy};
6216                 my $ready = $running_mirror_jobs->{$job}->{ready};
6217                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6218                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6219                     my $remaining = $total - $transferred;
6220                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6221
6222                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6223                 }
6224
6225                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6226             }
6227
6228             last if scalar(keys %$jobs) == 0;
6229
6230             if ($readycounter == scalar(keys %$jobs)) {
6231                 print "all mirroring jobs are ready \n";
6232                 last if $skipcomplete; #do the complete later
6233
6234                 if ($vmiddst && $vmiddst != $vmid) {
6235                     my $agent_running = $qga && qga_check_running($vmid);
6236                     if ($agent_running) {
6237                         print "freeze filesystem\n";
6238                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6239                     } else {
6240                         print "suspend vm\n";
6241                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6242                     }
6243
6244                     # if we clone a disk for a new target vm, we don't switch the disk
6245                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6246
6247                     if ($agent_running) {
6248                         print "unfreeze filesystem\n";
6249                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6250                     } else {
6251                         print "resume vm\n";
6252                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6253                     }
6254
6255                     last;
6256                 } else {
6257
6258                     foreach my $job (keys %$jobs) {
6259                         # try to switch the disk if source and destination are on the same guest
6260                         print "$job: Completing block job...\n";
6261
6262                         eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
6263                         if ($@ =~ m/cannot be completed/) {
6264                             print "$job: Block job cannot be completed, try again.\n";
6265                             $err_complete++;
6266                         }else {
6267                             print "$job: Completed successfully.\n";
6268                             $jobs->{$job}->{complete} = 1;
6269                         }
6270                     }
6271                 }
6272             }
6273             sleep 1;
6274         }
6275     };
6276     my $err = $@;
6277
6278     if ($err) {
6279         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6280         die "mirroring error: $err";
6281     }
6282
6283 }
6284
6285 sub qemu_blockjobs_cancel {
6286     my ($vmid, $jobs) = @_;
6287
6288     foreach my $job (keys %$jobs) {
6289         print "$job: Cancelling block job\n";
6290         eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
6291         $jobs->{$job}->{cancel} = 1;
6292     }
6293
6294     while (1) {
6295         my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6296
6297         my $running_jobs = {};
6298         foreach my $stat (@$stats) {
6299             $running_jobs->{$stat->{device}} = $stat;
6300         }
6301
6302         foreach my $job (keys %$jobs) {
6303
6304             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6305                 print "$job: Done.\n";
6306                 delete $jobs->{$job};
6307             }
6308         }
6309
6310         last if scalar(keys %$jobs) == 0;
6311
6312         sleep 1;
6313     }
6314 }
6315
6316 sub clone_disk {
6317     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6318         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga) = @_;
6319
6320     my $newvolid;
6321
6322     if (!$full) {
6323         print "create linked clone of drive $drivename ($drive->{file})\n";
6324         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6325         push @$newvollist, $newvolid;
6326     } else {
6327
6328         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6329         $storeid = $storage if $storage;
6330
6331         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
6332         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6333
6334         print "create full clone of drive $drivename ($drive->{file})\n";
6335         my $name = undef;
6336         if (drive_is_cloudinit($drive)) {
6337             $name = "vm-$newvmid-cloudinit";
6338             # cloudinit only supports raw and qcow2 atm:
6339             if ($dst_format eq 'qcow2') {
6340                 $name .= '.qcow2';
6341             } elsif ($dst_format ne 'raw') {
6342                 die "clone: unhandled format for cloudinit image\n";
6343             }
6344         }
6345         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
6346         push @$newvollist, $newvolid;
6347
6348         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
6349
6350         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6351         if (!$running || $snapname) {
6352             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6353         } else {
6354
6355             my $kvmver = get_running_qemu_version ($vmid);
6356             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
6357                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6358                     if $drive->{iothread};
6359             }
6360
6361             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga);
6362         }
6363     }
6364
6365     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6366
6367     my $disk = $drive;
6368     $disk->{format} = undef;
6369     $disk->{file} = $newvolid;
6370     $disk->{size} = $size;
6371
6372     return $disk;
6373 }
6374
6375 # this only works if VM is running
6376 sub get_current_qemu_machine {
6377     my ($vmid) = @_;
6378
6379     my $cmd = { execute => 'query-machines', arguments => {} };
6380     my $res = vm_qmp_command($vmid, $cmd);
6381
6382     my ($current, $default);
6383     foreach my $e (@$res) {
6384         $default = $e->{name} if $e->{'is-default'};
6385         $current = $e->{name} if $e->{'is-current'};
6386     }
6387
6388     # fallback to the default machine if current is not supported by qemu
6389     return $current || $default || 'pc';
6390 }
6391
6392 sub get_running_qemu_version {
6393     my ($vmid) = @_;
6394     my $cmd = { execute => 'query-version', arguments => {} };
6395     my $res = vm_qmp_command($vmid, $cmd);
6396     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
6397 }
6398
6399 sub qemu_machine_feature_enabled {
6400     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6401
6402     my $current_major;
6403     my $current_minor;
6404
6405     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6406
6407         $current_major = $3;
6408         $current_minor = $4;
6409
6410     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6411
6412         $current_major = $1;
6413         $current_minor = $2;
6414     }
6415
6416     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
6417
6418
6419 }
6420
6421 sub qemu_machine_pxe {
6422     my ($vmid, $conf, $machine) = @_;
6423
6424     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6425
6426     foreach my $opt (keys %$conf) {
6427         next if $opt !~ m/^net(\d+)$/;
6428         my $net = PVE::QemuServer::parse_net($conf->{$opt});
6429         next if !$net;
6430         my $romfile = PVE::QemuServer::vm_mon_cmd_nocheck($vmid, 'qom-get', path => $opt, property => 'romfile');
6431         return $machine.".pxe" if $romfile =~ m/pxe/;
6432         last;
6433     }
6434
6435     return $machine;
6436 }
6437
6438 sub qemu_use_old_bios_files {
6439     my ($machine_type) = @_;
6440
6441     return if !$machine_type;
6442
6443     my $use_old_bios_files = undef;
6444
6445     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6446         $machine_type = $1;
6447         $use_old_bios_files = 1;
6448     } else {
6449         my $kvmver = kvm_user_version();
6450         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6451         # load new efi bios files on migration. So this hack is required to allow
6452         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6453         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6454         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
6455     }
6456
6457     return ($use_old_bios_files, $machine_type);
6458 }
6459
6460 sub create_efidisk {
6461     my ($storecfg, $storeid, $vmid, $fmt) = @_;
6462
6463     die "EFI vars default image not found\n" if ! -f $OVMF_VARS;
6464
6465     my $vars_size = PVE::Tools::convert_size(-s $OVMF_VARS, 'b' => 'kb');
6466     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
6467     PVE::Storage::activate_volumes($storecfg, [$volid]);
6468
6469     my $path = PVE::Storage::path($storecfg, $volid);
6470     eval {
6471         run_command(['/usr/bin/qemu-img', 'convert', '-n', '-f', 'raw', '-O', $fmt, $OVMF_VARS, $path]);
6472     };
6473     die "Copying EFI vars image failed: $@" if $@;
6474
6475     return ($volid, $vars_size);
6476 }
6477
6478 sub lspci {
6479
6480     my $devices = {};
6481
6482     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6483             my (undef, $id, $function) = @_;
6484             my $res = { id => $id, function => $function};
6485             push @{$devices->{$id}}, $res;
6486     });
6487
6488     # Entries should be sorted by functions.
6489     foreach my $id (keys %$devices) {
6490         my $dev = $devices->{$id};
6491         $devices->{$id} = [ sort { $a->{function} <=> $b->{function} } @$dev ];
6492     }
6493
6494     return $devices;
6495 }
6496
6497 sub vm_iothreads_list {
6498     my ($vmid) = @_;
6499
6500     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6501
6502     my $iothreads = {};
6503     foreach my $iothread (@$res) {
6504         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6505     }
6506
6507     return $iothreads;
6508 }
6509
6510 sub scsihw_infos {
6511     my ($conf, $drive) = @_;
6512
6513     my $maxdev = 0;
6514
6515     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
6516         $maxdev = 7;
6517     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6518         $maxdev = 1;
6519     } else {
6520         $maxdev = 256;
6521     }
6522
6523     my $controller = int($drive->{index} / $maxdev);
6524     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6525
6526     return ($maxdev, $controller, $controller_prefix);
6527 }
6528
6529 sub add_hyperv_enlightenments {
6530     my ($cpuFlags, $winversion, $machine_type, $kvmver, $bios, $gpu_passthrough) = @_;
6531
6532     return if $winversion < 6;
6533     return if $bios && $bios eq 'ovmf' && $winversion < 8;
6534
6535     push @$cpuFlags , 'hv_vendor_id=proxmox' if $gpu_passthrough;
6536
6537     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
6538         push @$cpuFlags , 'hv_spinlocks=0x1fff';
6539         push @$cpuFlags , 'hv_vapic';
6540         push @$cpuFlags , 'hv_time';
6541     } else {
6542         push @$cpuFlags , 'hv_spinlocks=0xffff';
6543     }
6544
6545     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
6546         push @$cpuFlags , 'hv_reset';
6547         push @$cpuFlags , 'hv_vpindex';
6548         push @$cpuFlags , 'hv_runtime';
6549     }
6550
6551     if ($winversion >= 7) {
6552         push @$cpuFlags , 'hv_relaxed';
6553     }
6554 }
6555
6556 sub windows_version {
6557     my ($ostype) = @_;
6558
6559     return 0 if !$ostype;
6560
6561     my $winversion = 0;
6562
6563     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
6564         $winversion = 5;
6565     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
6566         $winversion = 6;
6567     } elsif ($ostype =~ m/^win(\d+)$/) {
6568         $winversion = $1;
6569     }
6570
6571     return $winversion;
6572 }
6573
6574 sub resolve_dst_disk_format {
6575         my ($storecfg, $storeid, $src_volname, $format) = @_;
6576         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6577
6578         if (!$format) {
6579             # if no target format is specified, use the source disk format as hint
6580             if ($src_volname) {
6581                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6582                 $format = qemu_img_format($scfg, $src_volname);
6583             } else {
6584                 return $defFormat;
6585             }
6586         }
6587
6588         # test if requested format is supported - else use default
6589         my $supported = grep { $_ eq $format } @$validFormats;
6590         $format = $defFormat if !$supported;
6591         return $format;
6592 }
6593
6594 sub resolve_first_disk {
6595     my $conf = shift;
6596     my @disks = PVE::QemuServer::valid_drive_names();
6597     my $firstdisk;
6598     foreach my $ds (reverse @disks) {
6599         next if !$conf->{$ds};
6600         my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
6601         next if PVE::QemuServer::drive_is_cdrom($disk);
6602         $firstdisk = $ds;
6603     }
6604     return $firstdisk;
6605 }
6606
6607 sub generate_smbios1_uuid {
6608     my ($uuid, $uuid_str);
6609     UUID::generate($uuid);
6610     UUID::unparse($uuid, $uuid_str);
6611     return "uuid=$uuid_str";
6612 }
6613
6614 # bash completion helper
6615
6616 sub complete_backup_archives {
6617     my ($cmdname, $pname, $cvalue) = @_;
6618
6619     my $cfg = PVE::Storage::config();
6620
6621     my $storeid;
6622
6623     if ($cvalue =~ m/^([^:]+):/) {
6624         $storeid = $1;
6625     }
6626
6627     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6628
6629     my $res = [];
6630     foreach my $id (keys %$data) {
6631         foreach my $item (@{$data->{$id}}) {
6632             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6633             push @$res, $item->{volid} if defined($item->{volid});
6634         }
6635     }
6636
6637     return $res;
6638 }
6639
6640 my $complete_vmid_full = sub {
6641     my ($running) = @_;
6642
6643     my $idlist = vmstatus();
6644
6645     my $res = [];
6646
6647     foreach my $id (keys %$idlist) {
6648         my $d = $idlist->{$id};
6649         if (defined($running)) {
6650             next if $d->{template};
6651             next if $running && $d->{status} ne 'running';
6652             next if !$running && $d->{status} eq 'running';
6653         }
6654         push @$res, $id;
6655
6656     }
6657     return $res;
6658 };
6659
6660 sub complete_vmid {
6661     return &$complete_vmid_full();
6662 }
6663
6664 sub complete_vmid_stopped {
6665     return &$complete_vmid_full(0);
6666 }
6667
6668 sub complete_vmid_running {
6669     return &$complete_vmid_full(1);
6670 }
6671
6672 sub complete_storage {
6673
6674     my $cfg = PVE::Storage::config();
6675     my $ids = $cfg->{ids};
6676
6677     my $res = [];
6678     foreach my $sid (keys %$ids) {
6679         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6680         next if !$ids->{$sid}->{content}->{images};
6681         push @$res, $sid;
6682     }
6683
6684     return $res;
6685 }
6686
6687 1;