PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach $IPV6RE);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QemuConfig;
  31 use PVE::QMPClient;
  32 use PVE::RPCEnvironment;
  33 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
  34 use PVE::QemuServer::Memory;
  35 use PVE::QemuServer::USB qw(parse_usb_device);
  36 use PVE::QemuServer::Cloudinit;
  37 use Time::HiRes qw(gettimeofday);
  38 use File::Copy qw(copy);
  39 use URI::Escape;
  40
  41 my $OVMF_CODE = '/usr/share/kvm/OVMF_CODE-pure-efi.fd';
  42 my $OVMF_VARS = '/usr/share/kvm/OVMF_VARS-pure-efi.fd';
  43
  44 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  45
  46 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  47
  48 my $QEMU_FORMAT_RE = qr/raw|cow|qcow|qcow2|qed|vmdk|cloop/;
  49
  50 # Note about locking: we use flock on the config file protect
  51 # against concurent actions.
  52 # Aditionaly, we have a 'lock' setting in the config file. This
  53 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  54 # allowed when such lock is set. But you can ignore this kind of
  55 # lock with the --skiplock flag.
  56
  57 cfs_register_file('/qemu-server/',
  58                   \&parse_vm_config,
  59                   \&write_vm_config);
  60
  61 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  62     description => "Some command save/restore state from this location.",
  63     type => 'string',
  64     maxLength => 128,
  65     optional => 1,
  66 });
  67
  68 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  69     description => "The name of the snapshot.",
  70     type => 'string', format => 'pve-configid',
  71     maxLength => 40,
  72 });
  73
  74 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  75     type => 'string',
  76     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  77     description => "The drive's backing file's data format.",
  78     optional => 1,
  79 });
  80
  81 #no warnings 'redefine';
  82
  83 sub cgroups_write {
  84    my ($controller, $vmid, $option, $value) = @_;
  85
  86    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  87    PVE::ProcFSTools::write_proc_entry($path, $value);
  88
  89 }
  90
  91 my $nodename = PVE::INotify::nodename();
  92
  93 mkdir "/etc/pve/nodes/$nodename";
  94 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  95 mkdir $confdir;
  96
  97 my $var_run_tmpdir = "/var/run/qemu-server";
  98 mkdir $var_run_tmpdir;
  99
 100 my $lock_dir = "/var/lock/qemu-server";
 101 mkdir $lock_dir;
 102
 103 my $pcisysfs = "/sys/bus/pci";
 104
 105 my $cpu_vendor_list = {
 106     # Intel CPUs
 107     486 => 'GenuineIntel',
 108     pentium => 'GenuineIntel',
 109     pentium2  => 'GenuineIntel',
 110     pentium3  => 'GenuineIntel',
 111     coreduo => 'GenuineIntel',
 112     core2duo => 'GenuineIntel',
 113     Conroe  => 'GenuineIntel',
 114     Penryn  => 'GenuineIntel',
 115     Nehalem  => 'GenuineIntel',
 116     'Nehalem-IBRS'  => 'GenuineIntel',
 117     Westmere => 'GenuineIntel',
 118     'Westmere-IBRS' => 'GenuineIntel',
 119     SandyBridge => 'GenuineIntel',
 120     'SandyBridge-IBRS' => 'GenuineIntel',
 121     IvyBridge => 'GenuineIntel',
 122     'IvyBridge-IBRS' => 'GenuineIntel',
 123     Haswell => 'GenuineIntel',
 124     'Haswell-IBRS' => 'GenuineIntel',
 125     'Haswell-noTSX' => 'GenuineIntel',
 126     'Haswell-noTSX-IBRS' => 'GenuineIntel',
 127     Broadwell => 'GenuineIntel',
 128     'Broadwell-IBRS' => 'GenuineIntel',
 129     'Broadwell-noTSX' => 'GenuineIntel',
 130     'Broadwell-noTSX-IBRS' => 'GenuineIntel',
 131     'Skylake-Client' => 'GenuineIntel',
 132     'Skylake-Client-IBRS' => 'GenuineIntel',
 133     'Skylake-Server' => 'GenuineIntel',
 134     'Skylake-Server-IBRS' => 'GenuineIntel',
 135
 136     # AMD CPUs
 137     athlon => 'AuthenticAMD',
 138     phenom  => 'AuthenticAMD',
 139     Opteron_G1  => 'AuthenticAMD',
 140     Opteron_G2  => 'AuthenticAMD',
 141     Opteron_G3  => 'AuthenticAMD',
 142     Opteron_G4  => 'AuthenticAMD',
 143     Opteron_G5  => 'AuthenticAMD',
 144     EPYC => 'AuthenticAMD',
 145     'EPYC-IBPB' => 'AuthenticAMD',
 146
 147     # generic types, use vendor from host node
 148     host => 'default',
 149     kvm32 => 'default',
 150     kvm64 => 'default',
 151     qemu32 => 'default',
 152     qemu64 => 'default',
 153     max => 'default',
 154 };
 155
 156 my $cpu_flag = qr/[+-](pcid|spec-ctrl)/;
 157
 158 my $cpu_fmt = {
 159     cputype => {
 160         description => "Emulated CPU type.",
 161         type => 'string',
 162         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 163         default => 'kvm64',
 164         default_key => 1,
 165     },
 166     hidden => {
 167         description => "Do not identify as a KVM virtual machine.",
 168         type => 'boolean',
 169         optional => 1,
 170         default => 0
 171     },
 172     flags => {
 173         description => "List of additional CPU flags separated by ';'."
 174                      . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
 175                      . " Currently supported flags: 'pcid', 'spec-ctrl'.",
 176         format_description => '+FLAG[;-FLAG...]',
 177         type => 'string',
 178         pattern => qr/$cpu_flag(;$cpu_flag)*/,
 179         optional => 1,
 180     },
 181 };
 182
 183 my $watchdog_fmt = {
 184     model => {
 185         default_key => 1,
 186         type => 'string',
 187         enum => [qw(i6300esb ib700)],
 188         description => "Watchdog type to emulate.",
 189         default => 'i6300esb',
 190         optional => 1,
 191     },
 192     action => {
 193         type => 'string',
 194         enum => [qw(reset shutdown poweroff pause debug none)],
 195         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 196         optional => 1,
 197     },
 198 };
 199 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 200
 201 my $confdesc = {
 202     onboot => {
 203         optional => 1,
 204         type => 'boolean',
 205         description => "Specifies whether a VM will be started during system bootup.",
 206         default => 0,
 207     },
 208     autostart => {
 209         optional => 1,
 210         type => 'boolean',
 211         description => "Automatic restart after crash (currently ignored).",
 212         default => 0,
 213     },
 214     hotplug => {
 215         optional => 1,
 216         type => 'string', format => 'pve-hotplug-features',
 217         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 218         default => 'network,disk,usb',
 219     },
 220     reboot => {
 221         optional => 1,
 222         type => 'boolean',
 223         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 224         default => 1,
 225     },
 226     lock => {
 227         optional => 1,
 228         type => 'string',
 229         description => "Lock/unlock the VM.",
 230         enum => [qw(migrate backup snapshot rollback)],
 231     },
 232     cpulimit => {
 233         optional => 1,
 234         type => 'number',
 235         description => "Limit of CPU usage.",
 236         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 237         minimum => 0,
 238         maximum => 128,
 239         default => 0,
 240     },
 241     cpuunits => {
 242         optional => 1,
 243         type => 'integer',
 244         description => "CPU weight for a VM.",
 245         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 246         minimum => 2,
 247         maximum => 262144,
 248         default => 1024,
 249     },
 250     memory => {
 251         optional => 1,
 252         type => 'integer',
 253         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 254         minimum => 16,
 255         default => 512,
 256     },
 257     balloon => {
 258         optional => 1,
 259         type => 'integer',
 260         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 261         minimum => 0,
 262     },
 263     shares => {
 264         optional => 1,
 265         type => 'integer',
 266         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 267         minimum => 0,
 268         maximum => 50000,
 269         default => 1000,
 270     },
 271     keyboard => {
 272         optional => 1,
 273         type => 'string',
 274         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.conf' configuration file.".
 275                        "It should not be necessary to set it.",
 276         enum => PVE::Tools::kvmkeymaplist(),
 277         default => undef,
 278     },
 279     name => {
 280         optional => 1,
 281         type => 'string', format => 'dns-name',
 282         description => "Set a name for the VM. Only used on the configuration web interface.",
 283     },
 284     scsihw => {
 285         optional => 1,
 286         type => 'string',
 287         description => "SCSI controller model",
 288         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 289         default => 'lsi',
 290     },
 291     description => {
 292         optional => 1,
 293         type => 'string',
 294         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 295     },
 296     ostype => {
 297         optional => 1,
 298         type => 'string',
 299         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 300         description => "Specify guest operating system.",
 301         verbose_description => <<EODESC,
 302 Specify guest operating system. This is used to enable special
 303 optimization/features for specific operating systems:
 304
 305 [horizontal]
 306 other;; unspecified OS
 307 wxp;; Microsoft Windows XP
 308 w2k;; Microsoft Windows 2000
 309 w2k3;; Microsoft Windows 2003
 310 w2k8;; Microsoft Windows 2008
 311 wvista;; Microsoft Windows Vista
 312 win7;; Microsoft Windows 7
 313 win8;; Microsoft Windows 8/2012/2012r2
 314 win10;; Microsoft Windows 10/2016
 315 l24;; Linux 2.4 Kernel
 316 l26;; Linux 2.6/3.X Kernel
 317 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 318 EODESC
 319     },
 320     boot => {
 321         optional => 1,
 322         type => 'string',
 323         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 324         pattern => '[acdn]{1,4}',
 325         default => 'cdn',
 326     },
 327     bootdisk => {
 328         optional => 1,
 329         type => 'string', format => 'pve-qm-bootdisk',
 330         description => "Enable booting from specified disk.",
 331         pattern => '(ide|sata|scsi|virtio)\d+',
 332     },
 333     smp => {
 334         optional => 1,
 335         type => 'integer',
 336         description => "The number of CPUs. Please use option -sockets instead.",
 337         minimum => 1,
 338         default => 1,
 339     },
 340     sockets => {
 341         optional => 1,
 342         type => 'integer',
 343         description => "The number of CPU sockets.",
 344         minimum => 1,
 345         default => 1,
 346     },
 347     cores => {
 348         optional => 1,
 349         type => 'integer',
 350         description => "The number of cores per socket.",
 351         minimum => 1,
 352         default => 1,
 353     },
 354     numa => {
 355         optional => 1,
 356         type => 'boolean',
 357         description => "Enable/disable NUMA.",
 358         default => 0,
 359     },
 360     hugepages => {
 361         optional => 1,
 362         type => 'string',
 363         description => "Enable/disable hugepages memory.",
 364         enum => [qw(any 2 1024)],
 365     },
 366     vcpus => {
 367         optional => 1,
 368         type => 'integer',
 369         description => "Number of hotplugged vcpus.",
 370         minimum => 1,
 371         default => 0,
 372     },
 373     acpi => {
 374         optional => 1,
 375         type => 'boolean',
 376         description => "Enable/disable ACPI.",
 377         default => 1,
 378     },
 379     agent => {
 380         optional => 1,
 381         type => 'boolean',
 382         description => "Enable/disable Qemu GuestAgent.",
 383         default => 0,
 384     },
 385     kvm => {
 386         optional => 1,
 387         type => 'boolean',
 388         description => "Enable/disable KVM hardware virtualization.",
 389         default => 1,
 390     },
 391     tdf => {
 392         optional => 1,
 393         type => 'boolean',
 394         description => "Enable/disable time drift fix.",
 395         default => 0,
 396     },
 397     localtime => {
 398         optional => 1,
 399         type => 'boolean',
 400         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 401     },
 402     freeze => {
 403         optional => 1,
 404         type => 'boolean',
 405         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 406     },
 407     vga => {
 408         optional => 1,
 409         type => 'string',
 410         description => "Select the VGA type.",
 411         verbose_description => "Select the VGA type. If you want to use high resolution" .
 412             " modes (>= 1280x1024x16) then you should use the options " .
 413             "'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and " .
 414             "'cirrus' for other OS types. The 'qxl' option enables the SPICE " .
 415             "display sever. For win* OS you can select how many independent " .
 416             "displays you want, Linux guests can add displays them self. " .
 417             "You can also run without any graphic card, using a serial device" .
 418             " as terminal.",
 419         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 420     },
 421     watchdog => {
 422         optional => 1,
 423         type => 'string', format => 'pve-qm-watchdog',
 424         description => "Create a virtual hardware watchdog device.",
 425         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 426             " (by a guest action), the watchdog must be periodically polled " .
 427             "by an agent inside the guest or else the watchdog will reset " .
 428             "the guest (or execute the respective action specified)",
 429     },
 430     startdate => {
 431         optional => 1,
 432         type => 'string',
 433         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 434         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 435         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 436         default => 'now',
 437     },
 438     startup =>  get_standard_option('pve-startup-order'),
 439     template => {
 440         optional => 1,
 441         type => 'boolean',
 442         description => "Enable/disable Template.",
 443         default => 0,
 444     },
 445     args => {
 446         optional => 1,
 447         type => 'string',
 448         description => "Arbitrary arguments passed to kvm.",
 449         verbose_description => <<EODESCR,
 450 Arbitrary arguments passed to kvm, for example:
 451
 452 args: -no-reboot -no-hpet
 453
 454 NOTE: this option is for experts only.
 455 EODESCR
 456     },
 457     tablet => {
 458         optional => 1,
 459         type => 'boolean',
 460         default => 1,
 461         description => "Enable/disable the USB tablet device.",
 462         verbose_description => "Enable/disable the USB tablet device. This device is " .
 463             "usually needed to allow absolute mouse positioning with VNC. " .
 464             "Else the mouse runs out of sync with normal VNC clients. " .
 465             "If you're running lots of console-only guests on one host, " .
 466             "you may consider disabling this to save some context switches. " .
 467             "This is turned off by default if you use spice (-vga=qxl).",
 468     },
 469     migrate_speed => {
 470         optional => 1,
 471         type => 'integer',
 472         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 473         minimum => 0,
 474         default => 0,
 475     },
 476     migrate_downtime => {
 477         optional => 1,
 478         type => 'number',
 479         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 480         minimum => 0,
 481         default => 0.1,
 482     },
 483     cdrom => {
 484         optional => 1,
 485         type => 'string', format => 'pve-qm-ide',
 486         typetext => '<volume>',
 487         description => "This is an alias for option -ide2",
 488     },
 489     cpu => {
 490         optional => 1,
 491         description => "Emulated CPU type.",
 492         type => 'string',
 493         format => $cpu_fmt,
 494     },
 495     parent => get_standard_option('pve-snapshot-name', {
 496         optional => 1,
 497         description => "Parent snapshot name. This is used internally, and should not be modified.",
 498     }),
 499     snaptime => {
 500         optional => 1,
 501         description => "Timestamp for snapshots.",
 502         type => 'integer',
 503         minimum => 0,
 504     },
 505     vmstate => {
 506         optional => 1,
 507         type => 'string', format => 'pve-volume-id',
 508         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 509     },
 510     vmstatestorage => get_standard_option('pve-storage-id', {
 511         description => "Default storage for VM state volumes/files.",
 512         optional => 1,
 513     }),
 514     machine => {
 515         description => "Specific the Qemu machine type.",
 516         type => 'string',
 517         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
 518         maxLength => 40,
 519         optional => 1,
 520     },
 521     smbios1 => {
 522         description => "Specify SMBIOS type 1 fields.",
 523         type => 'string', format => 'pve-qm-smbios1',
 524         maxLength => 256,
 525         optional => 1,
 526     },
 527     protection => {
 528         optional => 1,
 529         type => 'boolean',
 530         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 531         default => 0,
 532     },
 533     bios => {
 534         optional => 1,
 535         type => 'string',
 536         enum => [ qw(seabios ovmf) ],
 537         description => "Select BIOS implementation.",
 538         default => 'seabios',
 539     },
 540 };
 541
 542 my $confdesc_cloudinit = {
 543     citype => {
 544         optional => 1,
 545         type => 'string',
 546         description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
 547         enum => ['configdrive2', 'nocloud'],
 548     },
 549     ciuser => {
 550         optional => 1,
 551         type => 'string',
 552         description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
 553     },
 554     cipassword => {
 555         optional => 1,
 556         type => 'string',
 557         description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
 558     },
 559     searchdomain => {
 560         optional => 1,
 561         type => 'string',
 562         description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 563     },
 564     nameserver => {
 565         optional => 1,
 566         type => 'string', format => 'address-list',
 567         description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 568     },
 569     sshkeys => {
 570         optional => 1,
 571         type => 'string',
 572         format => 'urlencoded',
 573         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 574     },
 575 };
 576
 577 # what about other qemu settings ?
 578 #cpu => 'string',
 579 #machine => 'string',
 580 #fda => 'file',
 581 #fdb => 'file',
 582 #mtdblock => 'file',
 583 #sd => 'file',
 584 #pflash => 'file',
 585 #snapshot => 'bool',
 586 #bootp => 'file',
 587 ##tftp => 'dir',
 588 ##smb => 'dir',
 589 #kernel => 'file',
 590 #append => 'string',
 591 #initrd => 'file',
 592 ##soundhw => 'string',
 593
 594 while (my ($k, $v) = each %$confdesc) {
 595     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 596 }
 597
 598 my $MAX_IDE_DISKS = 4;
 599 my $MAX_SCSI_DISKS = 14;
 600 my $MAX_VIRTIO_DISKS = 16;
 601 my $MAX_SATA_DISKS = 6;
 602 my $MAX_USB_DEVICES = 5;
 603 my $MAX_NETS = 32;
 604 my $MAX_UNUSED_DISKS = 8;
 605 my $MAX_HOSTPCI_DEVICES = 4;
 606 my $MAX_SERIAL_PORTS = 4;
 607 my $MAX_PARALLEL_PORTS = 3;
 608 my $MAX_NUMA = 8;
 609
 610 my $numa_fmt = {
 611     cpus => {
 612         type => "string",
 613         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 614         description => "CPUs accessing this NUMA node.",
 615         format_description => "id[-id];...",
 616     },
 617     memory => {
 618         type => "number",
 619         description => "Amount of memory this NUMA node provides.",
 620         optional => 1,
 621     },
 622     hostnodes => {
 623         type => "string",
 624         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 625         description => "Host NUMA nodes to use.",
 626         format_description => "id[-id];...",
 627         optional => 1,
 628     },
 629     policy => {
 630         type => 'string',
 631         enum => [qw(preferred bind interleave)],
 632         description => "NUMA allocation policy.",
 633         optional => 1,
 634     },
 635 };
 636 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 637 my $numadesc = {
 638     optional => 1,
 639     type => 'string', format => $numa_fmt,
 640     description => "NUMA topology.",
 641 };
 642 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 643
 644 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 645     $confdesc->{"numa$i"} = $numadesc;
 646 }
 647
 648 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 649                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 650                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 651 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 652
 653 my $net_fmt_bridge_descr = <<__EOD__;
 654 Bridge to attach the network device to. The Proxmox VE standard bridge
 655 is called 'vmbr0'.
 656
 657 If you do not specify a bridge, we create a kvm user (NATed) network
 658 device, which provides DHCP and DNS services. The following addresses
 659 are used:
 660
 661  10.0.2.2   Gateway
 662  10.0.2.3   DNS Server
 663  10.0.2.4   SMB Server
 664
 665 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 666 __EOD__
 667
 668 my $net_fmt = {
 669     macaddr => {
 670         type => 'string',
 671         pattern => qr/[0-9a-f]{2}(?::[0-9a-f]{2}){5}/i,
 672         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 673         format_description => "XX:XX:XX:XX:XX:XX",
 674         optional => 1,
 675     },
 676     model => {
 677         type => 'string',
 678         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 679         enum => $nic_model_list,
 680         default_key => 1,
 681     },
 682     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 683     bridge => {
 684         type => 'string',
 685         description => $net_fmt_bridge_descr,
 686         format_description => 'bridge',
 687         optional => 1,
 688     },
 689     queues => {
 690         type => 'integer',
 691         minimum => 0, maximum => 16,
 692         description => 'Number of packet queues to be used on the device.',
 693         optional => 1,
 694     },
 695     rate => {
 696         type => 'number',
 697         minimum => 0,
 698         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 699         optional => 1,
 700     },
 701     tag => {
 702         type => 'integer',
 703         minimum => 1, maximum => 4094,
 704         description => 'VLAN tag to apply to packets on this interface.',
 705         optional => 1,
 706     },
 707     trunks => {
 708         type => 'string',
 709         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 710         description => 'VLAN trunks to pass through this interface.',
 711         format_description => 'vlanid[;vlanid...]',
 712         optional => 1,
 713     },
 714     firewall => {
 715         type => 'boolean',
 716         description => 'Whether this interface should be protected by the firewall.',
 717         optional => 1,
 718     },
 719     link_down => {
 720         type => 'boolean',
 721         description => 'Whether this interface should be disconnected (like pulling the plug).',
 722         optional => 1,
 723     },
 724 };
 725
 726 my $netdesc = {
 727     optional => 1,
 728     type => 'string', format => $net_fmt,
 729     description => "Specify network devices.",
 730 };
 731
 732 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 733
 734 my $ipconfig_fmt = {
 735     ip => {
 736         type => 'string',
 737         format => 'pve-ipv4-config',
 738         format_description => 'IPv4Format/CIDR',
 739         description => 'IPv4 address in CIDR format.',
 740         optional => 1,
 741         default => 'dhcp',
 742     },
 743     gw => {
 744         type => 'string',
 745         format => 'ipv4',
 746         format_description => 'GatewayIPv4',
 747         description => 'Default gateway for IPv4 traffic.',
 748         optional => 1,
 749         requires => 'ip',
 750     },
 751     ip6 => {
 752         type => 'string',
 753         format => 'pve-ipv6-config',
 754         format_description => 'IPv6Format/CIDR',
 755         description => 'IPv6 address in CIDR format.',
 756         optional => 1,
 757         default => 'dhcp',
 758     },
 759     gw6 => {
 760         type => 'string',
 761         format => 'ipv6',
 762         format_description => 'GatewayIPv6',
 763         description => 'Default gateway for IPv6 traffic.',
 764         optional => 1,
 765         requires => 'ip6',
 766     },
 767 };
 768 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 769 my $ipconfigdesc = {
 770     optional => 1,
 771     type => 'string', format => 'pve-qm-ipconfig',
 772     description => <<'EODESCR',
 773 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 774
 775 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 776
 777 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
 778 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 779
 780 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
 781 EODESCR
 782 };
 783 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 784
 785 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 786     $confdesc->{"net$i"} = $netdesc;
 787     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 788 }
 789
 790 foreach my $key (keys %$confdesc_cloudinit) {
 791     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 792 }
 793
 794 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 795 sub verify_volume_id_or_qm_path {
 796     my ($volid, $noerr) = @_;
 797
 798     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 799         return $volid;
 800     }
 801
 802     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 803     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 804     if ($@) {
 805         return undef if $noerr;
 806         die $@;
 807     }
 808     return $volid;
 809 }
 810
 811 my $drivename_hash;
 812
 813 my %drivedesc_base = (
 814     volume => { alias => 'file' },
 815     file => {
 816         type => 'string',
 817         format => 'pve-volume-id-or-qm-path',
 818         default_key => 1,
 819         format_description => 'volume',
 820         description => "The drive's backing volume.",
 821     },
 822     media => {
 823         type => 'string',
 824         enum => [qw(cdrom disk)],
 825         description => "The drive's media type.",
 826         default => 'disk',
 827         optional => 1
 828     },
 829     cyls => {
 830         type => 'integer',
 831         description => "Force the drive's physical geometry to have a specific cylinder count.",
 832         optional => 1
 833     },
 834     heads => {
 835         type => 'integer',
 836         description => "Force the drive's physical geometry to have a specific head count.",
 837         optional => 1
 838     },
 839     secs => {
 840         type => 'integer',
 841         description => "Force the drive's physical geometry to have a specific sector count.",
 842         optional => 1
 843     },
 844     trans => {
 845         type => 'string',
 846         enum => [qw(none lba auto)],
 847         description => "Force disk geometry bios translation mode.",
 848         optional => 1,
 849     },
 850     snapshot => {
 851         type => 'boolean',
 852         description => "Controls qemu's snapshot mode feature."
 853             . " If activated, changes made to the disk are temporary and will"
 854             . " be discarded when the VM is shutdown.",
 855         optional => 1,
 856     },
 857     cache => {
 858         type => 'string',
 859         enum => [qw(none writethrough writeback unsafe directsync)],
 860         description => "The drive's cache mode",
 861         optional => 1,
 862     },
 863     format => get_standard_option('pve-qm-image-format'),
 864     size => {
 865         type => 'string',
 866         format => 'disk-size',
 867         format_description => 'DiskSize',
 868         description => "Disk size. This is purely informational and has no effect.",
 869         optional => 1,
 870     },
 871     backup => {
 872         type => 'boolean',
 873         description => "Whether the drive should be included when making backups.",
 874         optional => 1,
 875     },
 876     replicate => {
 877         type => 'boolean',
 878         description => 'Whether the drive should considered for replication jobs.',
 879         optional => 1,
 880         default => 1,
 881     },
 882     rerror => {
 883         type => 'string',
 884         enum => [qw(ignore report stop)],
 885         description => 'Read error action.',
 886         optional => 1,
 887     },
 888     werror => {
 889         type => 'string',
 890         enum => [qw(enospc ignore report stop)],
 891         description => 'Write error action.',
 892         optional => 1,
 893     },
 894     aio => {
 895         type => 'string',
 896         enum => [qw(native threads)],
 897         description => 'AIO type to use.',
 898         optional => 1,
 899     },
 900     discard => {
 901         type => 'string',
 902         enum => [qw(ignore on)],
 903         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 904         optional => 1,
 905     },
 906     detect_zeroes => {
 907         type => 'boolean',
 908         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 909         optional => 1,
 910     },
 911     serial => {
 912         type => 'string',
 913         format => 'urlencoded',
 914         format_description => 'serial',
 915         maxLength => 20*3, # *3 since it's %xx url enoded
 916         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
 917         optional => 1,
 918     },
 919     shared => {
 920         type => 'boolean',
 921         description => 'Mark this locally-managed volume as available on all nodes',
 922         verbose_description => "Mark this locally-managed volume as available on all nodes.\n\nWARNING: This option does not share the volume automatically, it assumes it is shared already!",
 923         optional => 1,
 924         default => 0,
 925     }
 926 );
 927
 928 my %iothread_fmt = ( iothread => {
 929         type => 'boolean',
 930         description => "Whether to use iothreads for this drive",
 931         optional => 1,
 932 });
 933
 934 my %model_fmt = (
 935     model => {
 936         type => 'string',
 937         format => 'urlencoded',
 938         format_description => 'model',
 939         maxLength => 40*3, # *3 since it's %xx url enoded
 940         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
 941         optional => 1,
 942     },
 943 );
 944
 945 my %queues_fmt = (
 946     queues => {
 947         type => 'integer',
 948         description => "Number of queues.",
 949         minimum => 2,
 950         optional => 1
 951     }
 952 );
 953
 954 my %scsiblock_fmt = (
 955     scsiblock => {
 956         type => 'boolean',
 957         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
 958         optional => 1,
 959         default => 0,
 960     },
 961 );
 962
 963 my $add_throttle_desc = sub {
 964     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
 965     my $d = {
 966         type => $type,
 967         format_description => $unit,
 968         description => "Maximum $what in $longunit.",
 969         optional => 1,
 970     };
 971     $d->{minimum} = $minimum if defined($minimum);
 972     $drivedesc_base{$key} = $d;
 973 };
 974 # throughput: (leaky bucket)
 975 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
 976 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
 977 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
 978 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
 979 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
 980 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
 981 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
 982 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
 983 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
 984
 985 # pools: (pool of IO before throttling starts taking effect)
 986 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
 987 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
 988 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
 989 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
 990 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
 991 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
 992
 993 # burst lengths
 994 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
 995 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
 996 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
 997 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
 998 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
 999 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1000
1001 # legacy support
1002 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
1003 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
1004 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
1005 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
1006
1007 my $ide_fmt = {
1008     %drivedesc_base,
1009     %model_fmt,
1010 };
1011 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
1012
1013 my $idedesc = {
1014     optional => 1,
1015     type => 'string', format => $ide_fmt,
1016     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
1017 };
1018 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
1019
1020 my $scsi_fmt = {
1021     %drivedesc_base,
1022     %iothread_fmt,
1023     %queues_fmt,
1024     %scsiblock_fmt,
1025 };
1026 my $scsidesc = {
1027     optional => 1,
1028     type => 'string', format => $scsi_fmt,
1029     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
1030 };
1031 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
1032
1033 my $sata_fmt = {
1034     %drivedesc_base,
1035 };
1036 my $satadesc = {
1037     optional => 1,
1038     type => 'string', format => $sata_fmt,
1039     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
1040 };
1041 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
1042
1043 my $virtio_fmt = {
1044     %drivedesc_base,
1045     %iothread_fmt,
1046 };
1047 my $virtiodesc = {
1048     optional => 1,
1049     type => 'string', format => $virtio_fmt,
1050     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
1051 };
1052 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
1053
1054 my $alldrive_fmt = {
1055     %drivedesc_base,
1056     %iothread_fmt,
1057     %model_fmt,
1058     %queues_fmt,
1059     %scsiblock_fmt,
1060 };
1061
1062 my $efidisk_fmt = {
1063     volume => { alias => 'file' },
1064     file => {
1065         type => 'string',
1066         format => 'pve-volume-id-or-qm-path',
1067         default_key => 1,
1068         format_description => 'volume',
1069         description => "The drive's backing volume.",
1070     },
1071     format => get_standard_option('pve-qm-image-format'),
1072     size => {
1073         type => 'string',
1074         format => 'disk-size',
1075         format_description => 'DiskSize',
1076         description => "Disk size. This is purely informational and has no effect.",
1077         optional => 1,
1078     },
1079 };
1080
1081 my $efidisk_desc = {
1082     optional => 1,
1083     type => 'string', format => $efidisk_fmt,
1084     description => "Configure a Disk for storing EFI vars",
1085 };
1086
1087 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
1088
1089 my $usb_fmt = {
1090     host => {
1091         default_key => 1,
1092         type => 'string', format => 'pve-qm-usb-device',
1093         format_description => 'HOSTUSBDEVICE|spice',
1094         description => <<EODESCR,
1095 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1096
1097  'bus-port(.port)*' (decimal numbers) or
1098  'vendor_id:product_id' (hexadeciaml numbers) or
1099  'spice'
1100
1101 You can use the 'lsusb -t' command to list existing usb devices.
1102
1103 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1104
1105 The value 'spice' can be used to add a usb redirection devices for spice.
1106 EODESCR
1107     },
1108     usb3 => {
1109         optional => 1,
1110         type => 'boolean',
1111         description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
1112         default => 0,
1113     },
1114 };
1115
1116 my $usbdesc = {
1117     optional => 1,
1118     type => 'string', format => $usb_fmt,
1119     description => "Configure an USB device (n is 0 to 4).",
1120 };
1121 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1122
1123 # NOTE: the match-groups of this regex are used in parse_hostpci
1124 my $PCIRE = qr/([a-f0-9]{2}:[a-f0-9]{2})(?:\.([a-f0-9]))?/;
1125 my $hostpci_fmt = {
1126     host => {
1127         default_key => 1,
1128         type => 'string',
1129         pattern => qr/$PCIRE(;$PCIRE)*/,
1130         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1131         description => <<EODESCR,
1132 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1133 of PCI virtual functions of the host. HOSTPCIID syntax is:
1134
1135 'bus:dev.func' (hexadecimal numbers)
1136
1137 You can us the 'lspci' command to list existing PCI devices.
1138 EODESCR
1139     },
1140     rombar => {
1141         type => 'boolean',
1142         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1143         optional => 1,
1144         default => 1,
1145     },
1146     romfile => {
1147         type => 'string',
1148         pattern => '[^,;]+',
1149         format_description => 'string',
1150         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1151         optional => 1,
1152     },
1153     pcie => {
1154         type => 'boolean',
1155         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1156         optional => 1,
1157         default => 0,
1158     },
1159     'x-vga' => {
1160         type => 'boolean',
1161         description =>  "Enable vfio-vga device support.",
1162         optional => 1,
1163         default => 0,
1164     },
1165 };
1166 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1167
1168 my $hostpcidesc = {
1169         optional => 1,
1170         type => 'string', format => 'pve-qm-hostpci',
1171         description => "Map host PCI devices into guest.",
1172         verbose_description =>  <<EODESCR,
1173 Map host PCI devices into guest.
1174
1175 NOTE: This option allows direct access to host hardware. So it is no longer
1176 possible to migrate such machines - use with special care.
1177
1178 CAUTION: Experimental! User reported problems with this option.
1179 EODESCR
1180 };
1181 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1182
1183 my $serialdesc = {
1184         optional => 1,
1185         type => 'string',
1186         pattern => '(/dev/.+|socket)',
1187         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1188         verbose_description =>  <<EODESCR,
1189 Create a serial device inside the VM (n is 0 to 3), and pass through a
1190 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1191 host side (use 'qm terminal' to open a terminal connection).
1192
1193 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1194
1195 CAUTION: Experimental! User reported problems with this option.
1196 EODESCR
1197 };
1198
1199 my $paralleldesc= {
1200         optional => 1,
1201         type => 'string',
1202         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1203         description =>  "Map host parallel devices (n is 0 to 2).",
1204         verbose_description =>  <<EODESCR,
1205 Map host parallel devices (n is 0 to 2).
1206
1207 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1208
1209 CAUTION: Experimental! User reported problems with this option.
1210 EODESCR
1211 };
1212
1213 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1214     $confdesc->{"parallel$i"} = $paralleldesc;
1215 }
1216
1217 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1218     $confdesc->{"serial$i"} = $serialdesc;
1219 }
1220
1221 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1222     $confdesc->{"hostpci$i"} = $hostpcidesc;
1223 }
1224
1225 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1226     $drivename_hash->{"ide$i"} = 1;
1227     $confdesc->{"ide$i"} = $idedesc;
1228 }
1229
1230 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1231     $drivename_hash->{"sata$i"} = 1;
1232     $confdesc->{"sata$i"} = $satadesc;
1233 }
1234
1235 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1236     $drivename_hash->{"scsi$i"} = 1;
1237     $confdesc->{"scsi$i"} = $scsidesc ;
1238 }
1239
1240 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1241     $drivename_hash->{"virtio$i"} = 1;
1242     $confdesc->{"virtio$i"} = $virtiodesc;
1243 }
1244
1245 $drivename_hash->{efidisk0} = 1;
1246 $confdesc->{efidisk0} = $efidisk_desc;
1247
1248 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1249     $confdesc->{"usb$i"} = $usbdesc;
1250 }
1251
1252 my $unuseddesc = {
1253     optional => 1,
1254     type => 'string', format => 'pve-volume-id',
1255     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1256 };
1257
1258 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1259     $confdesc->{"unused$i"} = $unuseddesc;
1260 }
1261
1262 my $kvm_api_version = 0;
1263
1264 sub kvm_version {
1265
1266     return $kvm_api_version if $kvm_api_version;
1267
1268     my $fh = IO::File->new("</dev/kvm") ||
1269         return 0;
1270
1271     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
1272         $kvm_api_version = $v;
1273     }
1274
1275     $fh->close();
1276
1277     return  $kvm_api_version;
1278 }
1279
1280 my $kvm_user_version;
1281
1282 sub kvm_user_version {
1283
1284     return $kvm_user_version if $kvm_user_version;
1285
1286     $kvm_user_version = 'unknown';
1287
1288     my $code = sub {
1289         my $line = shift;
1290         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1291             $kvm_user_version = $2;
1292         }
1293     };
1294
1295     eval { run_command("kvm -version", outfunc => $code); };
1296     warn $@ if $@;
1297
1298     return $kvm_user_version;
1299
1300 }
1301
1302 my $kernel_has_vhost_net = -c '/dev/vhost-net';
1303
1304 sub valid_drive_names {
1305     # order is important - used to autoselect boot disk
1306     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1307             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1308             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1309             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1310             'efidisk0');
1311 }
1312
1313 sub is_valid_drivename {
1314     my $dev = shift;
1315
1316     return defined($drivename_hash->{$dev});
1317 }
1318
1319 sub option_exists {
1320     my $key = shift;
1321     return defined($confdesc->{$key});
1322 }
1323
1324 sub nic_models {
1325     return $nic_model_list;
1326 }
1327
1328 sub os_list_description {
1329
1330     return {
1331         other => 'Other',
1332         wxp => 'Windows XP',
1333         w2k => 'Windows 2000',
1334         w2k3 =>, 'Windows 2003',
1335         w2k8 => 'Windows 2008',
1336         wvista => 'Windows Vista',
1337         win7 => 'Windows 7',
1338         win8 => 'Windows 8/2012',
1339         win10 => 'Windows 10/2016',
1340         l24 => 'Linux 2.4',
1341         l26 => 'Linux 2.6',
1342     };
1343 }
1344
1345 my $cdrom_path;
1346
1347 sub get_cdrom_path {
1348
1349     return  $cdrom_path if $cdrom_path;
1350
1351     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1352     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1353     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1354 }
1355
1356 sub get_iso_path {
1357     my ($storecfg, $vmid, $cdrom) = @_;
1358
1359     if ($cdrom eq 'cdrom') {
1360         return get_cdrom_path();
1361     } elsif ($cdrom eq 'none') {
1362         return '';
1363     } elsif ($cdrom =~ m|^/|) {
1364         return $cdrom;
1365     } else {
1366         return PVE::Storage::path($storecfg, $cdrom);
1367     }
1368 }
1369
1370 # try to convert old style file names to volume IDs
1371 sub filename_to_volume_id {
1372     my ($vmid, $file, $media) = @_;
1373
1374      if (!($file eq 'none' || $file eq 'cdrom' ||
1375           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1376
1377         return undef if $file =~ m|/|;
1378
1379         if ($media && $media eq 'cdrom') {
1380             $file = "local:iso/$file";
1381         } else {
1382             $file = "local:$vmid/$file";
1383         }
1384     }
1385
1386     return $file;
1387 }
1388
1389 sub verify_media_type {
1390     my ($opt, $vtype, $media) = @_;
1391
1392     return if !$media;
1393
1394     my $etype;
1395     if ($media eq 'disk') {
1396         $etype = 'images';
1397     } elsif ($media eq 'cdrom') {
1398         $etype = 'iso';
1399     } else {
1400         die "internal error";
1401     }
1402
1403     return if ($vtype eq $etype);
1404
1405     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1406 }
1407
1408 sub cleanup_drive_path {
1409     my ($opt, $storecfg, $drive) = @_;
1410
1411     # try to convert filesystem paths to volume IDs
1412
1413     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1414         ($drive->{file} !~ m|^/dev/.+|) &&
1415         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1416         ($drive->{file} !~ m/^\d+$/)) {
1417         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1418         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1419         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1420         verify_media_type($opt, $vtype, $drive->{media});
1421         $drive->{file} = $volid;
1422     }
1423
1424     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1425 }
1426
1427 sub parse_hotplug_features {
1428     my ($data) = @_;
1429
1430     my $res = {};
1431
1432     return $res if $data eq '0';
1433
1434     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1435
1436     foreach my $feature (PVE::Tools::split_list($data)) {
1437         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1438             $res->{$1} = 1;
1439         } else {
1440             die "invalid hotplug feature '$feature'\n";
1441         }
1442     }
1443     return $res;
1444 }
1445
1446 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1447 sub pve_verify_hotplug_features {
1448     my ($value, $noerr) = @_;
1449
1450     return $value if parse_hotplug_features($value);
1451
1452     return undef if $noerr;
1453
1454     die "unable to parse hotplug option\n";
1455 }
1456
1457 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1458 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1459 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1460 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1461 #        [,iothread=on][,serial=serial][,model=model]
1462
1463 sub parse_drive {
1464     my ($key, $data) = @_;
1465
1466     my ($interface, $index);
1467
1468     if ($key =~ m/^([^\d]+)(\d+)$/) {
1469         $interface = $1;
1470         $index = $2;
1471     } else {
1472         return undef;
1473     }
1474
1475     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1476                                      : $confdesc->{$key}->{format};
1477     if (!$desc) {
1478         warn "invalid drive key: $key\n";
1479         return undef;
1480     }
1481     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1482     return undef if !$res;
1483     $res->{interface} = $interface;
1484     $res->{index} = $index;
1485
1486     my $error = 0;
1487     foreach my $opt (qw(bps bps_rd bps_wr)) {
1488         if (my $bps = defined(delete $res->{$opt})) {
1489             if (defined($res->{"m$opt"})) {
1490                 warn "both $opt and m$opt specified\n";
1491                 ++$error;
1492                 next;
1493             }
1494             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1495         }
1496     }
1497
1498     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1499     for my $requirement (
1500         [mbps_max => 'mbps'],
1501         [mbps_rd_max => 'mbps_rd'],
1502         [mbps_wr_max => 'mbps_wr'],
1503         [miops_max => 'miops'],
1504         [miops_rd_max => 'miops_rd'],
1505         [miops_wr_max => 'miops_wr'],
1506         [bps_max_length => 'mbps_max'],
1507         [bps_rd_max_length => 'mbps_rd_max'],
1508         [bps_wr_max_length => 'mbps_wr_max'],
1509         [iops_max_length => 'iops_max'],
1510         [iops_rd_max_length => 'iops_rd_max'],
1511         [iops_wr_max_length => 'iops_wr_max']) {
1512         my ($option, $requires) = @$requirement;
1513         if ($res->{$option} && !$res->{$requires}) {
1514             warn "$option requires $requires\n";
1515             ++$error;
1516         }
1517     }
1518
1519     return undef if $error;
1520
1521     return undef if $res->{mbps_rd} && $res->{mbps};
1522     return undef if $res->{mbps_wr} && $res->{mbps};
1523     return undef if $res->{iops_rd} && $res->{iops};
1524     return undef if $res->{iops_wr} && $res->{iops};
1525
1526     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1527         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1528         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1529         return undef if $res->{interface} eq 'virtio';
1530     }
1531
1532     if (my $size = $res->{size}) {
1533         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1534     }
1535
1536     return $res;
1537 }
1538
1539 sub print_drive {
1540     my ($vmid, $drive) = @_;
1541     my $data = { %$drive };
1542     delete $data->{$_} for qw(index interface);
1543     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1544 }
1545
1546 sub scsi_inquiry {
1547     my($fh, $noerr) = @_;
1548
1549     my $SG_IO = 0x2285;
1550     my $SG_GET_VERSION_NUM = 0x2282;
1551
1552     my $versionbuf = "\x00" x 8;
1553     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1554     if (!$ret) {
1555         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1556         return undef;
1557     }
1558     my $version = unpack("I", $versionbuf);
1559     if ($version < 30000) {
1560         die "scsi generic interface too old\n"  if !$noerr;
1561         return undef;
1562     }
1563
1564     my $buf = "\x00" x 36;
1565     my $sensebuf = "\x00" x 8;
1566     my $cmd = pack("C x3 C x1", 0x12, 36);
1567
1568     # see /usr/include/scsi/sg.h
1569     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1570
1571     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1572                       length($sensebuf), 0, length($buf), $buf,
1573                       $cmd, $sensebuf, 6000);
1574
1575     $ret = ioctl($fh, $SG_IO, $packet);
1576     if (!$ret) {
1577         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1578         return undef;
1579     }
1580
1581     my @res = unpack($sg_io_hdr_t, $packet);
1582     if ($res[17] || $res[18]) {
1583         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1584         return undef;
1585     }
1586
1587     my $res = {};
1588     (my $byte0, my $byte1, $res->{vendor},
1589      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1590
1591     $res->{removable} = $byte1 & 128 ? 1 : 0;
1592     $res->{type} = $byte0 & 31;
1593
1594     return $res;
1595 }
1596
1597 sub path_is_scsi {
1598     my ($path) = @_;
1599
1600     my $fh = IO::File->new("+<$path") || return undef;
1601     my $res = scsi_inquiry($fh, 1);
1602     close($fh);
1603
1604     return $res;
1605 }
1606
1607 sub machine_type_is_q35 {
1608     my ($conf) = @_;
1609
1610     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1611 }
1612
1613 sub print_tabletdevice_full {
1614     my ($conf) = @_;
1615
1616     my $q35 = machine_type_is_q35($conf);
1617
1618     # we use uhci for old VMs because tablet driver was buggy in older qemu
1619     my $usbbus = $q35 ? "ehci" : "uhci";
1620
1621     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1622 }
1623
1624 sub print_drivedevice_full {
1625     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1626
1627     my $device = '';
1628     my $maxdev = 0;
1629
1630     if ($drive->{interface} eq 'virtio') {
1631         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1632         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1633         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1634     } elsif ($drive->{interface} eq 'scsi') {
1635
1636         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1637         my $unit = $drive->{index} % $maxdev;
1638         my $devicetype = 'hd';
1639         my $path = '';
1640         if (drive_is_cdrom($drive)) {
1641             $devicetype = 'cd';
1642         } else {
1643             if ($drive->{file} =~ m|^/|) {
1644                 $path = $drive->{file};
1645                 if (my $info = path_is_scsi($path)) {
1646                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1647                         $devicetype = 'block';
1648                     } elsif ($info->{type} == 1) { # tape
1649                         $devicetype = 'generic';
1650                     }
1651                 }
1652             } else {
1653                  $path = PVE::Storage::path($storecfg, $drive->{file});
1654             }
1655
1656             if($path =~ m/^iscsi\:\/\//){
1657                 $devicetype = 'generic';
1658             }
1659         }
1660
1661         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1662             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1663         } else {
1664             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1665         }
1666
1667     } elsif ($drive->{interface} eq 'ide'){
1668         $maxdev = 2;
1669         my $controller = int($drive->{index} / $maxdev);
1670         my $unit = $drive->{index} % $maxdev;
1671         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1672
1673         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1674         if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
1675             $model = URI::Escape::uri_unescape($model);
1676             $device .= ",model=$model";
1677         }
1678     } elsif ($drive->{interface} eq 'sata'){
1679         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1680         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1681         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1682     } elsif ($drive->{interface} eq 'usb') {
1683         die "implement me";
1684         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1685     } else {
1686         die "unsupported interface type";
1687     }
1688
1689     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1690
1691     return $device;
1692 }
1693
1694 sub get_initiator_name {
1695     my $initiator;
1696
1697     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1698     while (defined(my $line = <$fh>)) {
1699         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1700         $initiator = $1;
1701         last;
1702     }
1703     $fh->close();
1704
1705     return $initiator;
1706 }
1707
1708 sub print_drive_full {
1709     my ($storecfg, $vmid, $drive) = @_;
1710
1711     my $path;
1712     my $volid = $drive->{file};
1713     my $format;
1714
1715     if (drive_is_cdrom($drive)) {
1716         $path = get_iso_path($storecfg, $vmid, $volid);
1717     } else {
1718         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1719         if ($storeid) {
1720             $path = PVE::Storage::path($storecfg, $volid);
1721             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1722             $format = qemu_img_format($scfg, $volname);
1723         } else {
1724             $path = $volid;
1725             $format = "raw";
1726         }
1727    }
1728
1729     my $opts = '';
1730     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1731     foreach my $o (@qemu_drive_options) {
1732         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1733     }
1734
1735     # snapshot only accepts on|off
1736     if (defined($drive->{snapshot})) {
1737         my $v = $drive->{snapshot} ? 'on' : 'off';
1738         $opts .= ",snapshot=$v";
1739     }
1740
1741     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1742         my ($dir, $qmpname) = @$type;
1743         if (my $v = $drive->{"mbps$dir"}) {
1744             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1745         }
1746         if (my $v = $drive->{"mbps${dir}_max"}) {
1747             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1748         }
1749         if (my $v = $drive->{"bps${dir}_max_length"}) {
1750             $opts .= ",throttling.bps$qmpname-max-length=$v";
1751         }
1752         if (my $v = $drive->{"iops${dir}"}) {
1753             $opts .= ",throttling.iops$qmpname=$v";
1754         }
1755         if (my $v = $drive->{"iops${dir}_max"}) {
1756             $opts .= ",throttling.iops$qmpname-max=$v";
1757         }
1758         if (my $v = $drive->{"iops${dir}_max_length"}) {
1759             $opts .= ",throttling.iops$qmpname-max-length=$v";
1760         }
1761     }
1762
1763     if (my $serial = $drive->{serial}) {
1764         $serial = URI::Escape::uri_unescape($serial);
1765         $opts .= ",serial=$serial";
1766     }
1767
1768     $opts .= ",format=$format" if $format && !$drive->{format};
1769
1770     my $cache_direct = 0;
1771
1772     if (my $cache = $drive->{cache}) {
1773         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1774     } elsif (!drive_is_cdrom($drive)) {
1775         $opts .= ",cache=none";
1776         $cache_direct = 1;
1777     }
1778
1779     # aio native works only with O_DIRECT
1780     if (!$drive->{aio}) {
1781         if($cache_direct) {
1782             $opts .= ",aio=native";
1783         } else {
1784             $opts .= ",aio=threads";
1785         }
1786     }
1787
1788     if (!drive_is_cdrom($drive)) {
1789         my $detectzeroes;
1790         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1791             $detectzeroes = 'off';
1792         } elsif ($drive->{discard}) {
1793             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1794         } else {
1795             # This used to be our default with discard not being specified:
1796             $detectzeroes = 'on';
1797         }
1798         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1799     }
1800
1801     my $pathinfo = $path ? "file=$path," : '';
1802
1803     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1804 }
1805
1806 sub print_netdevice_full {
1807     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1808
1809     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1810
1811     my $device = $net->{model};
1812     if ($net->{model} eq 'virtio') {
1813          $device = 'virtio-net-pci';
1814      };
1815
1816     my $pciaddr = print_pci_addr("$netid", $bridges);
1817     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1818     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1819         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1820         my $vectors = $net->{queues} * 2 + 2;
1821         $tmpstr .= ",vectors=$vectors,mq=on";
1822     }
1823     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1824
1825     if ($use_old_bios_files) {
1826         my $romfile;
1827         if ($device eq 'virtio-net-pci') {
1828             $romfile = 'pxe-virtio.rom';
1829         } elsif ($device eq 'e1000') {
1830             $romfile = 'pxe-e1000.rom';
1831         } elsif ($device eq 'ne2k') {
1832             $romfile = 'pxe-ne2k_pci.rom';
1833         } elsif ($device eq 'pcnet') {
1834             $romfile = 'pxe-pcnet.rom';
1835         } elsif ($device eq 'rtl8139') {
1836             $romfile = 'pxe-rtl8139.rom';
1837         }
1838         $tmpstr .= ",romfile=$romfile" if $romfile;
1839     }
1840
1841     return $tmpstr;
1842 }
1843
1844 sub print_netdev_full {
1845     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1846
1847     my $i = '';
1848     if ($netid =~ m/^net(\d+)$/) {
1849         $i = int($1);
1850     }
1851
1852     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1853
1854     my $ifname = "tap${vmid}i$i";
1855
1856     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1857     die "interface name '$ifname' is too long (max 15 character)\n"
1858         if length($ifname) >= 16;
1859
1860     my $vhostparam = '';
1861     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1862
1863     my $vmname = $conf->{name} || "vm$vmid";
1864
1865     my $netdev = "";
1866     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1867
1868     if ($net->{bridge}) {
1869         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1870     } else {
1871         $netdev = "type=user,id=$netid,hostname=$vmname";
1872     }
1873
1874     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1875
1876     return $netdev;
1877 }
1878
1879
1880 sub print_cpu_device {
1881     my ($conf, $id) = @_;
1882
1883     my $kvm = $conf->{kvm} // 1;
1884     my $cpu = $kvm ? "kvm64" : "qemu64";
1885     if (my $cputype = $conf->{cpu}) {
1886         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
1887             or die "Cannot parse cpu description: $cputype\n";
1888         $cpu = $cpuconf->{cputype};
1889     }
1890
1891     my $cores = $conf->{cores} || 1;
1892
1893     my $current_core = ($id - 1) % $cores;
1894     my $current_socket = int(($id - 1 - $current_core)/$cores);
1895
1896     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
1897 }
1898
1899 sub drive_is_cloudinit {
1900     my ($drive) = @_;
1901     return $drive->{file} =~ m@[:/]vm-\d+-cloudinit(?:\.$QEMU_FORMAT_RE)?$@;
1902 }
1903
1904 sub drive_is_cdrom {
1905     my ($drive, $exclude_cloudinit) = @_;
1906
1907     return 0 if $exclude_cloudinit && drive_is_cloudinit($drive);
1908
1909     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1910
1911 }
1912
1913 sub parse_number_sets {
1914     my ($set) = @_;
1915     my $res = [];
1916     foreach my $part (split(/;/, $set)) {
1917         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1918             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1919             push @$res, [ $1, $2 ];
1920         } else {
1921             die "invalid range: $part\n";
1922         }
1923     }
1924     return $res;
1925 }
1926
1927 sub parse_numa {
1928     my ($data) = @_;
1929
1930     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
1931     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1932     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1933     return $res;
1934 }
1935
1936 sub parse_hostpci {
1937     my ($value) = @_;
1938
1939     return undef if !$value;
1940
1941     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
1942
1943     my @idlist = split(/;/, $res->{host});
1944     delete $res->{host};
1945     foreach my $id (@idlist) {
1946         if ($id =~ /^$PCIRE$/) {
1947             if (defined($2)) {
1948                 push @{$res->{pciid}}, { id => $1, function => $2 };
1949             } else {
1950                 my $pcidevices = lspci($1);
1951                 $res->{pciid} = $pcidevices->{$1};
1952             }
1953         } else {
1954             # should have been caught by parse_property_string already
1955             die "failed to parse PCI id: $id\n";
1956         }
1957     }
1958     return $res;
1959 }
1960
1961 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1962 sub parse_net {
1963     my ($data) = @_;
1964
1965     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
1966     if ($@) {
1967         warn $@;
1968         return undef;
1969     }
1970     if (!defined($res->{macaddr})) {
1971         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1972         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1973     }
1974     $res->{macaddr} = PVE::Tools::random_ether_addr() if !defined($res->{macaddr});
1975     return $res;
1976 }
1977
1978 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1979 sub parse_ipconfig {
1980     my ($data) = @_;
1981
1982     my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
1983     if ($@) {
1984         warn $@;
1985         return undef;
1986     }
1987
1988     if ($res->{gw} && !$res->{ip}) {
1989         warn 'gateway specified without specifying an IP address';
1990         return undef;
1991     }
1992     if ($res->{gw6} && !$res->{ip6}) {
1993         warn 'IPv6 gateway specified without specifying an IPv6 address';
1994         return undef;
1995     }
1996     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1997         warn 'gateway specified together with DHCP';
1998         return undef;
1999     }
2000     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
2001         # gw6 + auto/dhcp
2002         warn "IPv6 gateway specified together with $res->{ip6} address";
2003         return undef;
2004     }
2005
2006     if (!$res->{ip} && !$res->{ip6}) {
2007         return { ip => 'dhcp', ip6 => 'dhcp' };
2008     }
2009
2010     return $res;
2011 }
2012
2013 sub print_net {
2014     my $net = shift;
2015
2016     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2017 }
2018
2019 sub add_random_macs {
2020     my ($settings) = @_;
2021
2022     foreach my $opt (keys %$settings) {
2023         next if $opt !~ m/^net(\d+)$/;
2024         my $net = parse_net($settings->{$opt});
2025         next if !$net;
2026         $settings->{$opt} = print_net($net);
2027     }
2028 }
2029
2030 sub vm_is_volid_owner {
2031     my ($storecfg, $vmid, $volid) = @_;
2032
2033     if ($volid !~  m|^/|) {
2034         my ($path, $owner);
2035         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2036         if ($owner && ($owner == $vmid)) {
2037             return 1;
2038         }
2039     }
2040
2041     return undef;
2042 }
2043
2044 sub split_flagged_list {
2045     my $text = shift || '';
2046     $text =~ s/[,;]/ /g;
2047     $text =~ s/^\s+//;
2048     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
2049 }
2050
2051 sub join_flagged_list {
2052     my ($how, $lst) = @_;
2053     join $how, map { $lst->{$_} . $_ } keys %$lst;
2054 }
2055
2056 sub vmconfig_delete_pending_option {
2057     my ($conf, $key, $force) = @_;
2058
2059     delete $conf->{pending}->{$key};
2060     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2061     $pending_delete_hash->{$key} = $force ? '!' : '';
2062     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2063 }
2064
2065 sub vmconfig_undelete_pending_option {
2066     my ($conf, $key) = @_;
2067
2068     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2069     delete $pending_delete_hash->{$key};
2070
2071     if (%$pending_delete_hash) {
2072         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2073     } else {
2074         delete $conf->{pending}->{delete};
2075     }
2076 }
2077
2078 sub vmconfig_register_unused_drive {
2079     my ($storecfg, $vmid, $conf, $drive) = @_;
2080
2081     if (drive_is_cloudinit($drive)) {
2082         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2083         warn $@ if $@;
2084     } elsif (!drive_is_cdrom($drive)) {
2085         my $volid = $drive->{file};
2086         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2087             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2088         }
2089     }
2090 }
2091
2092 sub vmconfig_cleanup_pending {
2093     my ($conf) = @_;
2094
2095     # remove pending changes when nothing changed
2096     my $changes;
2097     foreach my $opt (keys %{$conf->{pending}}) {
2098         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
2099             $changes = 1;
2100             delete $conf->{pending}->{$opt};
2101         }
2102     }
2103
2104     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
2105     my $pending_delete_hash = {};
2106     while (my ($opt, $force) = each %$current_delete_hash) {
2107         if (defined($conf->{$opt})) {
2108             $pending_delete_hash->{$opt} = $force;
2109         } else {
2110             $changes = 1;
2111         }
2112     }
2113
2114     if (%$pending_delete_hash) {
2115         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2116     } else {
2117         delete $conf->{pending}->{delete};
2118     }
2119
2120     return $changes;
2121 }
2122
2123 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
2124 my $smbios1_fmt = {
2125     uuid => {
2126         type => 'string',
2127         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2128         format_description => 'UUID',
2129         description => "Set SMBIOS1 UUID.",
2130         optional => 1,
2131     },
2132     version => {
2133         type => 'string',
2134         pattern => '\S+',
2135         format_description => 'string',
2136         description => "Set SMBIOS1 version.",
2137         optional => 1,
2138     },
2139     serial => {
2140         type => 'string',
2141         pattern => '\S+',
2142         format_description => 'string',
2143         description => "Set SMBIOS1 serial number.",
2144         optional => 1,
2145     },
2146     manufacturer => {
2147         type => 'string',
2148         pattern => '\S+',
2149         format_description => 'string',
2150         description => "Set SMBIOS1 manufacturer.",
2151         optional => 1,
2152     },
2153     product => {
2154         type => 'string',
2155         pattern => '\S+',
2156         format_description => 'string',
2157         description => "Set SMBIOS1 product ID.",
2158         optional => 1,
2159     },
2160     sku => {
2161         type => 'string',
2162         pattern => '\S+',
2163         format_description => 'string',
2164         description => "Set SMBIOS1 SKU string.",
2165         optional => 1,
2166     },
2167     family => {
2168         type => 'string',
2169         pattern => '\S+',
2170         format_description => 'string',
2171         description => "Set SMBIOS1 family string.",
2172         optional => 1,
2173     },
2174 };
2175
2176 sub parse_smbios1 {
2177     my ($data) = @_;
2178
2179     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
2180     warn $@ if $@;
2181     return $res;
2182 }
2183
2184 sub print_smbios1 {
2185     my ($smbios1) = @_;
2186     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2187 }
2188
2189 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2190
2191 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2192 sub verify_bootdisk {
2193     my ($value, $noerr) = @_;
2194
2195     return $value if is_valid_drivename($value);
2196
2197     return undef if $noerr;
2198
2199     die "invalid boot disk '$value'\n";
2200 }
2201
2202 sub parse_watchdog {
2203     my ($value) = @_;
2204
2205     return undef if !$value;
2206
2207     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2208     warn $@ if $@;
2209     return $res;
2210 }
2211
2212 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2213 sub verify_usb_device {
2214     my ($value, $noerr) = @_;
2215
2216     return $value if parse_usb_device($value);
2217
2218     return undef if $noerr;
2219
2220     die "unable to parse usb device\n";
2221 }
2222
2223 # add JSON properties for create and set function
2224 sub json_config_properties {
2225     my $prop = shift;
2226
2227     foreach my $opt (keys %$confdesc) {
2228         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
2229         $prop->{$opt} = $confdesc->{$opt};
2230     }
2231
2232     return $prop;
2233 }
2234
2235 # return copy of $confdesc_cloudinit to generate documentation
2236 sub cloudinit_config_properties {
2237
2238     return dclone($confdesc_cloudinit);
2239 }
2240
2241 sub check_type {
2242     my ($key, $value) = @_;
2243
2244     die "unknown setting '$key'\n" if !$confdesc->{$key};
2245
2246     my $type = $confdesc->{$key}->{type};
2247
2248     if (!defined($value)) {
2249         die "got undefined value\n";
2250     }
2251
2252     if ($value =~ m/[\n\r]/) {
2253         die "property contains a line feed\n";
2254     }
2255
2256     if ($type eq 'boolean') {
2257         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2258         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2259         die "type check ('boolean') failed - got '$value'\n";
2260     } elsif ($type eq 'integer') {
2261         return int($1) if $value =~ m/^(\d+)$/;
2262         die "type check ('integer') failed - got '$value'\n";
2263     } elsif ($type eq 'number') {
2264         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2265         die "type check ('number') failed - got '$value'\n";
2266     } elsif ($type eq 'string') {
2267         if (my $fmt = $confdesc->{$key}->{format}) {
2268             PVE::JSONSchema::check_format($fmt, $value);
2269             return $value;
2270         }
2271         $value =~ s/^\"(.*)\"$/$1/;
2272         return $value;
2273     } else {
2274         die "internal error"
2275     }
2276 }
2277
2278 sub check_iommu_support{
2279     #fixme : need to check IOMMU support
2280     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
2281
2282     my $iommu=1;
2283     return $iommu;
2284
2285 }
2286
2287 sub touch_config {
2288     my ($vmid) = @_;
2289
2290     my $conf = PVE::QemuConfig->config_file($vmid);
2291     utime undef, undef, $conf;
2292 }
2293
2294 sub destroy_vm {
2295     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2296
2297     my $conffile = PVE::QemuConfig->config_file($vmid);
2298
2299     my $conf = PVE::QemuConfig->load_config($vmid);
2300
2301     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2302
2303     if ($conf->{template}) {
2304         # check if any base image is still used by a linked clone
2305         foreach_drive($conf, sub {
2306                 my ($ds, $drive) = @_;
2307
2308                 return if drive_is_cdrom($drive);
2309
2310                 my $volid = $drive->{file};
2311
2312                 return if !$volid || $volid =~ m|^/|;
2313
2314                 die "base volume '$volid' is still in use by linked cloned\n"
2315                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2316
2317         });
2318     }
2319
2320     # only remove disks owned by this VM
2321     foreach_drive($conf, sub {
2322         my ($ds, $drive) = @_;
2323
2324         return if drive_is_cdrom($drive, 1);
2325
2326         my $volid = $drive->{file};
2327
2328         return if !$volid || $volid =~ m|^/|;
2329
2330         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2331         return if !$path || !$owner || ($owner != $vmid);
2332
2333         eval {
2334             PVE::Storage::vdisk_free($storecfg, $volid);
2335         };
2336         warn "Could not remove disk '$volid', check manually: $@" if $@;
2337
2338     });
2339
2340     if ($keep_empty_config) {
2341         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2342     } else {
2343         unlink $conffile;
2344     }
2345
2346     # also remove unused disk
2347     eval {
2348         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2349
2350         eval {
2351             PVE::Storage::foreach_volid($dl, sub {
2352                 my ($volid, $sid, $volname, $d) = @_;
2353                 PVE::Storage::vdisk_free($storecfg, $volid);
2354             });
2355         };
2356         warn $@ if $@;
2357
2358     };
2359     warn $@ if $@;
2360 }
2361
2362 sub parse_vm_config {
2363     my ($filename, $raw) = @_;
2364
2365     return undef if !defined($raw);
2366
2367     my $res = {
2368         digest => Digest::SHA::sha1_hex($raw),
2369         snapshots => {},
2370         pending => {},
2371     };
2372
2373     $filename =~ m|/qemu-server/(\d+)\.conf$|
2374         || die "got strange filename '$filename'";
2375
2376     my $vmid = $1;
2377
2378     my $conf = $res;
2379     my $descr;
2380     my $section = '';
2381
2382     my @lines = split(/\n/, $raw);
2383     foreach my $line (@lines) {
2384         next if $line =~ m/^\s*$/;
2385
2386         if ($line =~ m/^\[PENDING\]\s*$/i) {
2387             $section = 'pending';
2388             if (defined($descr)) {
2389                 $descr =~ s/\s+$//;
2390                 $conf->{description} = $descr;
2391             }
2392             $descr = undef;
2393             $conf = $res->{$section} = {};
2394             next;
2395
2396         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2397             $section = $1;
2398             if (defined($descr)) {
2399                 $descr =~ s/\s+$//;
2400                 $conf->{description} = $descr;
2401             }
2402             $descr = undef;
2403             $conf = $res->{snapshots}->{$section} = {};
2404             next;
2405         }
2406
2407         if ($line =~ m/^\#(.*)\s*$/) {
2408             $descr = '' if !defined($descr);
2409             $descr .= PVE::Tools::decode_text($1) . "\n";
2410             next;
2411         }
2412
2413         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2414             $descr = '' if !defined($descr);
2415             $descr .= PVE::Tools::decode_text($2);
2416         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2417             $conf->{snapstate} = $1;
2418         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2419             my $key = $1;
2420             my $value = $2;
2421             $conf->{$key} = $value;
2422         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2423             my $value = $1;
2424             if ($section eq 'pending') {
2425                 $conf->{delete} = $value; # we parse this later
2426             } else {
2427                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2428             }
2429         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2430             my $key = $1;
2431             my $value = $2;
2432             eval { $value = check_type($key, $value); };
2433             if ($@) {
2434                 warn "vm $vmid - unable to parse value of '$key' - $@";
2435             } else {
2436                 $key = 'ide2' if $key eq 'cdrom';
2437                 my $fmt = $confdesc->{$key}->{format};
2438                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2439                     my $v = parse_drive($key, $value);
2440                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2441                         $v->{file} = $volid;
2442                         $value = print_drive($vmid, $v);
2443                     } else {
2444                         warn "vm $vmid - unable to parse value of '$key'\n";
2445                         next;
2446                     }
2447                 }
2448
2449                 $conf->{$key} = $value;
2450             }
2451         }
2452     }
2453
2454     if (defined($descr)) {
2455         $descr =~ s/\s+$//;
2456         $conf->{description} = $descr;
2457     }
2458     delete $res->{snapstate}; # just to be sure
2459
2460     return $res;
2461 }
2462
2463 sub write_vm_config {
2464     my ($filename, $conf) = @_;
2465
2466     delete $conf->{snapstate}; # just to be sure
2467
2468     if ($conf->{cdrom}) {
2469         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2470         $conf->{ide2} = $conf->{cdrom};
2471         delete $conf->{cdrom};
2472     }
2473
2474     # we do not use 'smp' any longer
2475     if ($conf->{sockets}) {
2476         delete $conf->{smp};
2477     } elsif ($conf->{smp}) {
2478         $conf->{sockets} = $conf->{smp};
2479         delete $conf->{cores};
2480         delete $conf->{smp};
2481     }
2482
2483     my $used_volids = {};
2484
2485     my $cleanup_config = sub {
2486         my ($cref, $pending, $snapname) = @_;
2487
2488         foreach my $key (keys %$cref) {
2489             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2490                 $key eq 'snapstate' || $key eq 'pending';
2491             my $value = $cref->{$key};
2492             if ($key eq 'delete') {
2493                 die "propertry 'delete' is only allowed in [PENDING]\n"
2494                     if !$pending;
2495                 # fixme: check syntax?
2496                 next;
2497             }
2498             eval { $value = check_type($key, $value); };
2499             die "unable to parse value of '$key' - $@" if $@;
2500
2501             $cref->{$key} = $value;
2502
2503             if (!$snapname && is_valid_drivename($key)) {
2504                 my $drive = parse_drive($key, $value);
2505                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2506             }
2507         }
2508     };
2509
2510     &$cleanup_config($conf);
2511
2512     &$cleanup_config($conf->{pending}, 1);
2513
2514     foreach my $snapname (keys %{$conf->{snapshots}}) {
2515         die "internal error" if $snapname eq 'pending';
2516         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2517     }
2518
2519     # remove 'unusedX' settings if we re-add a volume
2520     foreach my $key (keys %$conf) {
2521         my $value = $conf->{$key};
2522         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2523             delete $conf->{$key};
2524         }
2525     }
2526
2527     my $generate_raw_config = sub {
2528         my ($conf, $pending) = @_;
2529
2530         my $raw = '';
2531
2532         # add description as comment to top of file
2533         if (defined(my $descr = $conf->{description})) {
2534             if ($descr) {
2535                 foreach my $cl (split(/\n/, $descr)) {
2536                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2537                 }
2538             } else {
2539                 $raw .= "#\n" if $pending;
2540             }
2541         }
2542
2543         foreach my $key (sort keys %$conf) {
2544             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2545             $raw .= "$key: $conf->{$key}\n";
2546         }
2547         return $raw;
2548     };
2549
2550     my $raw = &$generate_raw_config($conf);
2551
2552     if (scalar(keys %{$conf->{pending}})){
2553         $raw .= "\n[PENDING]\n";
2554         $raw .= &$generate_raw_config($conf->{pending}, 1);
2555     }
2556
2557     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2558         $raw .= "\n[$snapname]\n";
2559         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2560     }
2561
2562     return $raw;
2563 }
2564
2565 sub load_defaults {
2566
2567     my $res = {};
2568
2569     # we use static defaults from our JSON schema configuration
2570     foreach my $key (keys %$confdesc) {
2571         if (defined(my $default = $confdesc->{$key}->{default})) {
2572             $res->{$key} = $default;
2573         }
2574     }
2575
2576     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2577     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2578
2579     return $res;
2580 }
2581
2582 sub config_list {
2583     my $vmlist = PVE::Cluster::get_vmlist();
2584     my $res = {};
2585     return $res if !$vmlist || !$vmlist->{ids};
2586     my $ids = $vmlist->{ids};
2587
2588     foreach my $vmid (keys %$ids) {
2589         my $d = $ids->{$vmid};
2590         next if !$d->{node} || $d->{node} ne $nodename;
2591         next if !$d->{type} || $d->{type} ne 'qemu';
2592         $res->{$vmid}->{exists} = 1;
2593     }
2594     return $res;
2595 }
2596
2597 # test if VM uses local resources (to prevent migration)
2598 sub check_local_resources {
2599     my ($conf, $noerr) = @_;
2600
2601     my $loc_res = 0;
2602
2603     $loc_res = 1 if $conf->{hostusb}; # old syntax
2604     $loc_res = 1 if $conf->{hostpci}; # old syntax
2605
2606     foreach my $k (keys %$conf) {
2607         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2608         # sockets are safe: they will recreated be on the target side post-migrate
2609         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2610         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2611     }
2612
2613     die "VM uses local resources\n" if $loc_res && !$noerr;
2614
2615     return $loc_res;
2616 }
2617
2618 # check if used storages are available on all nodes (use by migrate)
2619 sub check_storage_availability {
2620     my ($storecfg, $conf, $node) = @_;
2621
2622     foreach_drive($conf, sub {
2623         my ($ds, $drive) = @_;
2624
2625         my $volid = $drive->{file};
2626         return if !$volid;
2627
2628         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2629         return if !$sid;
2630
2631         # check if storage is available on both nodes
2632         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2633         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2634    });
2635 }
2636
2637 # list nodes where all VM images are available (used by has_feature API)
2638 sub shared_nodes {
2639     my ($conf, $storecfg) = @_;
2640
2641     my $nodelist = PVE::Cluster::get_nodelist();
2642     my $nodehash = { map { $_ => 1 } @$nodelist };
2643     my $nodename = PVE::INotify::nodename();
2644
2645     foreach_drive($conf, sub {
2646         my ($ds, $drive) = @_;
2647
2648         my $volid = $drive->{file};
2649         return if !$volid;
2650
2651         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2652         if ($storeid) {
2653             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2654             if ($scfg->{disable}) {
2655                 $nodehash = {};
2656             } elsif (my $avail = $scfg->{nodes}) {
2657                 foreach my $node (keys %$nodehash) {
2658                     delete $nodehash->{$node} if !$avail->{$node};
2659                 }
2660             } elsif (!$scfg->{shared}) {
2661                 foreach my $node (keys %$nodehash) {
2662                     delete $nodehash->{$node} if $node ne $nodename
2663                 }
2664             }
2665         }
2666     });
2667
2668     return $nodehash
2669 }
2670
2671 sub check_cmdline {
2672     my ($pidfile, $pid) = @_;
2673
2674     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2675     if (defined($fh)) {
2676         my $line = <$fh>;
2677         $fh->close;
2678         return undef if !$line;
2679         my @param = split(/\0/, $line);
2680
2681         my $cmd = $param[0];
2682         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2683
2684         for (my $i = 0; $i < scalar (@param); $i++) {
2685             my $p = $param[$i];
2686             next if !$p;
2687             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2688                 my $p = $param[$i+1];
2689                 return 1 if $p && ($p eq $pidfile);
2690                 return undef;
2691             }
2692         }
2693     }
2694     return undef;
2695 }
2696
2697 sub check_running {
2698     my ($vmid, $nocheck, $node) = @_;
2699
2700     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2701
2702     die "unable to find configuration file for VM $vmid - no such machine\n"
2703         if !$nocheck && ! -f $filename;
2704
2705     my $pidfile = pidfile_name($vmid);
2706
2707     if (my $fd = IO::File->new("<$pidfile")) {
2708         my $st = stat($fd);
2709         my $line = <$fd>;
2710         close($fd);
2711
2712         my $mtime = $st->mtime;
2713         if ($mtime > time()) {
2714             warn "file '$filename' modified in future\n";
2715         }
2716
2717         if ($line =~ m/^(\d+)$/) {
2718             my $pid = $1;
2719             if (check_cmdline($pidfile, $pid)) {
2720                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2721                     return $pid;
2722                 }
2723             }
2724         }
2725     }
2726
2727     return undef;
2728 }
2729
2730 sub vzlist {
2731
2732     my $vzlist = config_list();
2733
2734     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2735
2736     while (defined(my $de = $fd->read)) {
2737         next if $de !~ m/^(\d+)\.pid$/;
2738         my $vmid = $1;
2739         next if !defined($vzlist->{$vmid});
2740         if (my $pid = check_running($vmid)) {
2741             $vzlist->{$vmid}->{pid} = $pid;
2742         }
2743     }
2744
2745     return $vzlist;
2746 }
2747
2748 sub disksize {
2749     my ($storecfg, $conf) = @_;
2750
2751     my $bootdisk = $conf->{bootdisk};
2752     return undef if !$bootdisk;
2753     return undef if !is_valid_drivename($bootdisk);
2754
2755     return undef if !$conf->{$bootdisk};
2756
2757     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2758     return undef if !defined($drive);
2759
2760     return undef if drive_is_cdrom($drive);
2761
2762     my $volid = $drive->{file};
2763     return undef if !$volid;
2764
2765     return $drive->{size};
2766 }
2767
2768 my $last_proc_pid_stat;
2769
2770 # get VM status information
2771 # This must be fast and should not block ($full == false)
2772 # We only query KVM using QMP if $full == true (this can be slow)
2773 sub vmstatus {
2774     my ($opt_vmid, $full) = @_;
2775
2776     my $res = {};
2777
2778     my $storecfg = PVE::Storage::config();
2779
2780     my $list = vzlist();
2781     my $defaults = load_defaults();
2782
2783     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2784
2785     my $cpucount = $cpuinfo->{cpus} || 1;
2786
2787     foreach my $vmid (keys %$list) {
2788         next if $opt_vmid && ($vmid ne $opt_vmid);
2789
2790         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
2791         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2792
2793         my $d = {};
2794         $d->{pid} = $list->{$vmid}->{pid};
2795
2796         # fixme: better status?
2797         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2798
2799         my $size = disksize($storecfg, $conf);
2800         if (defined($size)) {
2801             $d->{disk} = 0; # no info available
2802             $d->{maxdisk} = $size;
2803         } else {
2804             $d->{disk} = 0;
2805             $d->{maxdisk} = 0;
2806         }
2807
2808         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2809             * ($conf->{cores} || $defaults->{cores});
2810         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2811         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2812
2813         $d->{name} = $conf->{name} || "VM $vmid";
2814         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2815             : $defaults->{memory}*(1024*1024);
2816
2817         if ($conf->{balloon}) {
2818             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2819             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2820                 : $defaults->{shares};
2821         }
2822
2823         $d->{uptime} = 0;
2824         $d->{cpu} = 0;
2825         $d->{mem} = 0;
2826
2827         $d->{netout} = 0;
2828         $d->{netin} = 0;
2829
2830         $d->{diskread} = 0;
2831         $d->{diskwrite} = 0;
2832
2833         $d->{template} = PVE::QemuConfig->is_template($conf);
2834
2835         $d->{serial} = 1 if conf_has_serial($conf);
2836
2837         $res->{$vmid} = $d;
2838     }
2839
2840     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2841     foreach my $dev (keys %$netdev) {
2842         next if $dev !~ m/^tap([1-9]\d*)i/;
2843         my $vmid = $1;
2844         my $d = $res->{$vmid};
2845         next if !$d;
2846
2847         $d->{netout} += $netdev->{$dev}->{receive};
2848         $d->{netin} += $netdev->{$dev}->{transmit};
2849
2850         if ($full) {
2851             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2852             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2853         }
2854
2855     }
2856
2857     my $ctime = gettimeofday;
2858
2859     foreach my $vmid (keys %$list) {
2860
2861         my $d = $res->{$vmid};
2862         my $pid = $d->{pid};
2863         next if !$pid;
2864
2865         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2866         next if !$pstat; # not running
2867
2868         my $used = $pstat->{utime} + $pstat->{stime};
2869
2870         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2871
2872         if ($pstat->{vsize}) {
2873             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2874         }
2875
2876         my $old = $last_proc_pid_stat->{$pid};
2877         if (!$old) {
2878             $last_proc_pid_stat->{$pid} = {
2879                 time => $ctime,
2880                 used => $used,
2881                 cpu => 0,
2882             };
2883             next;
2884         }
2885
2886         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2887
2888         if ($dtime > 1000) {
2889             my $dutime = $used -  $old->{used};
2890
2891             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2892             $last_proc_pid_stat->{$pid} = {
2893                 time => $ctime,
2894                 used => $used,
2895                 cpu => $d->{cpu},
2896             };
2897         } else {
2898             $d->{cpu} = $old->{cpu};
2899         }
2900     }
2901
2902     return $res if !$full;
2903
2904     my $qmpclient = PVE::QMPClient->new();
2905
2906     my $ballooncb = sub {
2907         my ($vmid, $resp) = @_;
2908
2909         my $info = $resp->{'return'};
2910         return if !$info->{max_mem};
2911
2912         my $d = $res->{$vmid};
2913
2914         # use memory assigned to VM
2915         $d->{maxmem} = $info->{max_mem};
2916         $d->{balloon} = $info->{actual};
2917
2918         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2919             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2920             $d->{freemem} = $info->{free_mem};
2921         }
2922
2923         $d->{ballooninfo} = $info;
2924     };
2925
2926     my $blockstatscb = sub {
2927         my ($vmid, $resp) = @_;
2928         my $data = $resp->{'return'} || [];
2929         my $totalrdbytes = 0;
2930         my $totalwrbytes = 0;
2931
2932         for my $blockstat (@$data) {
2933             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2934             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2935
2936             $blockstat->{device} =~ s/drive-//;
2937             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2938         }
2939         $res->{$vmid}->{diskread} = $totalrdbytes;
2940         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2941     };
2942
2943     my $statuscb = sub {
2944         my ($vmid, $resp) = @_;
2945
2946         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2947         # this fails if ballon driver is not loaded, so this must be
2948         # the last commnand (following command are aborted if this fails).
2949         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2950
2951         my $status = 'unknown';
2952         if (!defined($status = $resp->{'return'}->{status})) {
2953             warn "unable to get VM status\n";
2954             return;
2955         }
2956
2957         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2958     };
2959
2960     foreach my $vmid (keys %$list) {
2961         next if $opt_vmid && ($vmid ne $opt_vmid);
2962         next if !$res->{$vmid}->{pid}; # not running
2963         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2964     }
2965
2966     $qmpclient->queue_execute(undef, 2);
2967
2968     foreach my $vmid (keys %$list) {
2969         next if $opt_vmid && ($vmid ne $opt_vmid);
2970         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2971     }
2972
2973     return $res;
2974 }
2975
2976 sub foreach_drive {
2977     my ($conf, $func, @param) = @_;
2978
2979     foreach my $ds (valid_drive_names()) {
2980         next if !defined($conf->{$ds});
2981
2982         my $drive = parse_drive($ds, $conf->{$ds});
2983         next if !$drive;
2984
2985         &$func($ds, $drive, @param);
2986     }
2987 }
2988
2989 sub foreach_volid {
2990     my ($conf, $func, @param) = @_;
2991
2992     my $volhash = {};
2993
2994     my $test_volid = sub {
2995         my ($volid, $is_cdrom, $replicate, $shared, $snapname) = @_;
2996
2997         return if !$volid;
2998
2999         $volhash->{$volid}->{cdrom} //= 1;
3000         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
3001
3002         $volhash->{$volid}->{replicate} //= 0;
3003         $volhash->{$volid}->{replicate} = 1 if $replicate;
3004
3005         $volhash->{$volid}->{shared} //= 0;
3006         $volhash->{$volid}->{shared} = 1 if $shared;
3007
3008         $volhash->{$volid}->{referenced_in_config} //= 0;
3009         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
3010
3011         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
3012             if defined($snapname);
3013     };
3014
3015     foreach_drive($conf, sub {
3016         my ($ds, $drive) = @_;
3017         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, undef);
3018     });
3019
3020     foreach my $snapname (keys %{$conf->{snapshots}}) {
3021         my $snap = $conf->{snapshots}->{$snapname};
3022         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
3023         foreach_drive($snap, sub {
3024             my ($ds, $drive) = @_;
3025             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, $snapname);
3026         });
3027     }
3028
3029     foreach my $volid (keys %$volhash) {
3030         &$func($volid, $volhash->{$volid}, @param);
3031     }
3032 }
3033
3034 sub conf_has_serial {
3035     my ($conf) = @_;
3036
3037     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3038         if ($conf->{"serial$i"}) {
3039             return 1;
3040         }
3041     }
3042
3043     return 0;
3044 }
3045
3046 sub vga_conf_has_spice {
3047     my ($vga) = @_;
3048
3049     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
3050
3051     return $1 || 1;
3052 }
3053
3054 sub config_to_command {
3055     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
3056
3057     my $cmd = [];
3058     my $globalFlags = [];
3059     my $machineFlags = [];
3060     my $rtcFlags = [];
3061     my $cpuFlags = [];
3062     my $devices = [];
3063     my $pciaddr = '';
3064     my $bridges = {};
3065     my $kvmver = kvm_user_version();
3066     my $vernum = 0; # unknown
3067     my $ostype = $conf->{ostype};
3068     my $winversion = windows_version($ostype);
3069     my $kvm = $conf->{kvm} // 1;
3070
3071     die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n" if (!$cpuinfo->{hvm} && $kvm);
3072
3073     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
3074         $vernum = $1*1000000+$2*1000;
3075     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
3076         $vernum = $1*1000000+$2*1000+$3;
3077     }
3078
3079     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
3080
3081     my $have_ovz = -f '/proc/vz/vestat';
3082
3083     my $q35 = machine_type_is_q35($conf);
3084     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3085     my $machine_type = $forcemachine || $conf->{machine};
3086     my $use_old_bios_files = undef;
3087     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3088
3089     my $cpuunits = defined($conf->{cpuunits}) ?
3090             $conf->{cpuunits} : $defaults->{cpuunits};
3091
3092     push @$cmd, '/usr/bin/kvm';
3093
3094     push @$cmd, '-id', $vmid;
3095
3096     my $use_virtio = 0;
3097
3098     my $qmpsocket = qmp_socket($vmid);
3099     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3100     push @$cmd, '-mon', "chardev=qmp,mode=control";
3101
3102
3103     push @$cmd, '-pidfile' , pidfile_name($vmid);
3104
3105     push @$cmd, '-daemonize';
3106
3107     if ($conf->{smbios1}) {
3108         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3109     }
3110
3111     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3112         die "uefi base image not found\n" if ! -f $OVMF_CODE;
3113
3114         my $path;
3115         my $format;
3116         if (my $efidisk = $conf->{efidisk0}) {
3117             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
3118             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3119             $format = $d->{format};
3120             if ($storeid) {
3121                 $path = PVE::Storage::path($storecfg, $d->{file});
3122                 if (!defined($format)) {
3123                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3124                     $format = qemu_img_format($scfg, $volname);
3125                 }
3126             } else {
3127                 $path = $d->{file};
3128                 die "efidisk format must be specified\n"
3129                     if !defined($format);
3130             }
3131         } else {
3132             warn "no efidisk configured! Using temporary efivars disk.\n";
3133             $path = "/tmp/$vmid-ovmf.fd";
3134             PVE::Tools::file_copy($OVMF_VARS, $path, -s $OVMF_VARS);
3135             $format = 'raw';
3136         }
3137
3138         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$OVMF_CODE";
3139         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
3140     }
3141
3142
3143     # add usb controllers
3144     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $q35, $usbdesc->{format}, $MAX_USB_DEVICES);
3145     push @$devices, @usbcontrollers if @usbcontrollers;
3146     my $vga = $conf->{vga};
3147
3148     my $qxlnum = vga_conf_has_spice($vga);
3149     $vga = 'qxl' if $qxlnum;
3150
3151     if (!$vga) {
3152         if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 9)) {
3153             $vga = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3154         } else {
3155             $vga = ($winversion >= 6) ? 'std' : 'cirrus';
3156         }
3157     }
3158
3159     # enable absolute mouse coordinates (needed by vnc)
3160     my $tablet;
3161     if (defined($conf->{tablet})) {
3162         $tablet = $conf->{tablet};
3163     } else {
3164         $tablet = $defaults->{tablet};
3165         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3166         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3167     }
3168
3169     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
3170
3171     my $kvm_off = 0;
3172     my $gpu_passthrough;
3173
3174     # host pci devices
3175     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3176         my $d = parse_hostpci($conf->{"hostpci$i"});
3177         next if !$d;
3178
3179         my $pcie = $d->{pcie};
3180         if($pcie){
3181             die "q35 machine model is not enabled" if !$q35;
3182             $pciaddr = print_pcie_addr("hostpci$i");
3183         }else{
3184             $pciaddr = print_pci_addr("hostpci$i", $bridges);
3185         }
3186
3187         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
3188         my $romfile = $d->{romfile};
3189
3190         my $xvga = '';
3191         if ($d->{'x-vga'}) {
3192             $xvga = ',x-vga=on';
3193             $kvm_off = 1;
3194             $vga = 'none';
3195             $gpu_passthrough = 1;
3196
3197             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3198                 $xvga = "";
3199             }
3200         }
3201         my $pcidevices = $d->{pciid};
3202         my $multifunction = 1 if @$pcidevices > 1;
3203
3204         my $j=0;
3205         foreach my $pcidevice (@$pcidevices) {
3206
3207             my $id = "hostpci$i";
3208             $id .= ".$j" if $multifunction;
3209             my $addr = $pciaddr;
3210             $addr .= ".$j" if $multifunction;
3211             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
3212
3213             if($j == 0){
3214                 $devicestr .= "$rombar$xvga";
3215                 $devicestr .= ",multifunction=on" if $multifunction;
3216                 $devicestr .= ",romfile=/usr/share/kvm/$romfile" if $romfile;
3217             }
3218
3219             push @$devices, '-device', $devicestr;
3220             $j++;
3221         }
3222     }
3223
3224     # usb devices
3225     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES);
3226     push @$devices, @usbdevices if @usbdevices;
3227     # serial devices
3228     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3229         if (my $path = $conf->{"serial$i"}) {
3230             if ($path eq 'socket') {
3231                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3232                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3233                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3234             } else {
3235                 die "no such serial device\n" if ! -c $path;
3236                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3237                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3238             }
3239         }
3240     }
3241
3242     # parallel devices
3243     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3244         if (my $path = $conf->{"parallel$i"}) {
3245             die "no such parallel device\n" if ! -c $path;
3246             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3247             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3248             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3249         }
3250     }
3251
3252     my $vmname = $conf->{name} || "vm$vmid";
3253
3254     push @$cmd, '-name', $vmname;
3255
3256     my $sockets = 1;
3257     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3258     $sockets = $conf->{sockets} if  $conf->{sockets};
3259
3260     my $cores = $conf->{cores} || 1;
3261
3262     my $maxcpus = $sockets * $cores;
3263
3264     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3265
3266     my $allowed_vcpus = $cpuinfo->{cpus};
3267
3268     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3269         if ($allowed_vcpus < $maxcpus);
3270
3271     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
3272
3273         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3274         for (my $i = 2; $i <= $vcpus; $i++)  {
3275             my $cpustr = print_cpu_device($conf,$i);
3276             push @$cmd, '-device', $cpustr;
3277         }
3278
3279     } else {
3280
3281         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3282     }
3283     push @$cmd, '-nodefaults';
3284
3285     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3286
3287     my $bootindex_hash = {};
3288     my $i = 1;
3289     foreach my $o (split(//, $bootorder)) {
3290         $bootindex_hash->{$o} = $i*100;
3291         $i++;
3292     }
3293
3294     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3295
3296     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3297
3298     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3299
3300     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
3301
3302     if ($vga && $vga !~ m/^serial\d+$/ && $vga ne 'none'){
3303         my $socket = vnc_socket($vmid);
3304         push @$cmd,  '-vnc', "unix:$socket,x509,password";
3305     } else {
3306         push @$cmd, '-nographic';
3307     }
3308
3309     # time drift fix
3310     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3311
3312     my $useLocaltime = $conf->{localtime};
3313
3314     if ($winversion >= 5) { # windows
3315         $useLocaltime = 1 if !defined($conf->{localtime});
3316
3317         # use time drift fix when acpi is enabled
3318         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3319             $tdf = 1 if !defined($conf->{tdf});
3320         }
3321     }
3322
3323     if ($winversion >= 6) {
3324         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3325         push @$cmd, '-no-hpet';
3326     }
3327
3328     push @$rtcFlags, 'driftfix=slew' if $tdf;
3329
3330     if (!$kvm) {
3331         push @$machineFlags, 'accel=tcg';
3332     }
3333
3334     if ($machine_type) {
3335         push @$machineFlags, "type=${machine_type}";
3336     }
3337
3338     if ($conf->{startdate}) {
3339         push @$rtcFlags, "base=$conf->{startdate}";
3340     } elsif ($useLocaltime) {
3341         push @$rtcFlags, 'base=localtime';
3342     }
3343
3344     my $cpu = $kvm ? "kvm64" : "qemu64";
3345     if (my $cputype = $conf->{cpu}) {
3346         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3347             or die "Cannot parse cpu description: $cputype\n";
3348         $cpu = $cpuconf->{cputype};
3349         $kvm_off = 1 if $cpuconf->{hidden};
3350
3351         if (defined(my $flags = $cpuconf->{flags})) {
3352             push @$cpuFlags, split(";", $flags);
3353         }
3354     }
3355
3356     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
3357
3358     push @$cpuFlags , '-x2apic'
3359         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3360
3361     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3362
3363     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3364
3365     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3366
3367         push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3368         push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3369     }
3370
3371     add_hyperv_enlightenments($cpuFlags, $winversion, $machine_type, $kvmver, $conf->{bios}, $gpu_passthrough) if $kvm;
3372
3373     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm;
3374
3375     push @$cpuFlags, 'kvm=off' if $kvm_off;
3376
3377     my $cpu_vendor = $cpu_vendor_list->{$cpu} ||
3378         die "internal error"; # should not happen
3379
3380     push @$cpuFlags, "vendor=${cpu_vendor}"
3381         if $cpu_vendor ne 'default';
3382
3383     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3384
3385     push @$cmd, '-cpu', $cpu;
3386
3387     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3388
3389     push @$cmd, '-S' if $conf->{freeze};
3390
3391     # set keyboard layout
3392     my $kb = $conf->{keyboard} || $defaults->{keyboard};
3393     push @$cmd, '-k', $kb if $kb;
3394
3395     # enable sound
3396     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3397     #push @$cmd, '-soundhw', 'es1370';
3398     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3399
3400     if($conf->{agent}) {
3401         my $qgasocket = qmp_socket($vmid, 1);
3402         my $pciaddr = print_pci_addr("qga0", $bridges);
3403         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3404         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3405         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3406     }
3407
3408     my $spice_port;
3409
3410     if ($qxlnum) {
3411         if ($qxlnum > 1) {
3412             if ($winversion){
3413                 for(my $i = 1; $i < $qxlnum; $i++){
3414                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3415                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3416                 }
3417             } else {
3418                 # assume other OS works like Linux
3419                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3420                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3421             }
3422         }
3423
3424         my $pciaddr = print_pci_addr("spice", $bridges);
3425
3426         my $nodename = PVE::INotify::nodename();
3427         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3428         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3429         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3430         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3431         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3432
3433         push @$devices, '-spice', "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3434
3435         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3436         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3437         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3438     }
3439
3440     # enable balloon by default, unless explicitly disabled
3441     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3442         $pciaddr = print_pci_addr("balloon0", $bridges);
3443         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3444     }
3445
3446     if ($conf->{watchdog}) {
3447         my $wdopts = parse_watchdog($conf->{watchdog});
3448         $pciaddr = print_pci_addr("watchdog", $bridges);
3449         my $watchdog = $wdopts->{model} || 'i6300esb';
3450         push @$devices, '-device', "$watchdog$pciaddr";
3451         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3452     }
3453
3454     my $vollist = [];
3455     my $scsicontroller = {};
3456     my $ahcicontroller = {};
3457     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3458
3459     # Add iscsi initiator name if available
3460     if (my $initiator = get_initiator_name()) {
3461         push @$devices, '-iscsi', "initiator-name=$initiator";
3462     }
3463
3464     foreach_drive($conf, sub {
3465         my ($ds, $drive) = @_;
3466
3467         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3468             push @$vollist, $drive->{file};
3469         }
3470
3471         # ignore efidisk here, already added in bios/fw handling code above
3472         return if $drive->{interface} eq 'efidisk';
3473
3474         $use_virtio = 1 if $ds =~ m/^virtio/;
3475
3476         if (drive_is_cdrom ($drive)) {
3477             if ($bootindex_hash->{d}) {
3478                 $drive->{bootindex} = $bootindex_hash->{d};
3479                 $bootindex_hash->{d} += 1;
3480             }
3481         } else {
3482             if ($bootindex_hash->{c}) {
3483                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3484                 $bootindex_hash->{c} += 1;
3485             }
3486         }
3487
3488         if($drive->{interface} eq 'virtio'){
3489            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3490         }
3491
3492         if ($drive->{interface} eq 'scsi') {
3493
3494             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3495
3496             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3497             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3498
3499             my $iothread = '';
3500             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3501                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3502                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3503             } elsif ($drive->{iothread}) {
3504                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3505             }
3506
3507             my $queues = '';
3508             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3509                 $queues = ",num_queues=$drive->{queues}";
3510             }
3511
3512             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3513             $scsicontroller->{$controller}=1;
3514         }
3515
3516         if ($drive->{interface} eq 'sata') {
3517            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3518            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3519            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3520            $ahcicontroller->{$controller}=1;
3521         }
3522
3523         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3524         push @$devices, '-drive',$drive_cmd;
3525         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3526     });
3527
3528     for (my $i = 0; $i < $MAX_NETS; $i++) {
3529          next if !$conf->{"net$i"};
3530          my $d = parse_net($conf->{"net$i"});
3531          next if !$d;
3532
3533          $use_virtio = 1 if $d->{model} eq 'virtio';
3534
3535          if ($bootindex_hash->{n}) {
3536             $d->{bootindex} = $bootindex_hash->{n};
3537             $bootindex_hash->{n} += 1;
3538          }
3539
3540          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3541          push @$devices, '-netdev', $netdevfull;
3542
3543          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3544          push @$devices, '-device', $netdevicefull;
3545     }
3546
3547     if (!$q35) {
3548         # add pci bridges
3549         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3550            $bridges->{1} = 1;
3551            $bridges->{2} = 1;
3552         }
3553
3554         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3555
3556         while (my ($k, $v) = each %$bridges) {
3557             $pciaddr = print_pci_addr("pci.$k");
3558             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3559         }
3560     }
3561
3562     # add custom args
3563     if ($conf->{args}) {
3564         my $aa = PVE::Tools::split_args($conf->{args});
3565         push @$cmd, @$aa;
3566     }
3567
3568     push @$cmd, @$devices;
3569     push @$cmd, '-rtc', join(',', @$rtcFlags)
3570         if scalar(@$rtcFlags);
3571     push @$cmd, '-machine', join(',', @$machineFlags)
3572         if scalar(@$machineFlags);
3573     push @$cmd, '-global', join(',', @$globalFlags)
3574         if scalar(@$globalFlags);
3575
3576     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3577 }
3578
3579 sub vnc_socket {
3580     my ($vmid) = @_;
3581     return "${var_run_tmpdir}/$vmid.vnc";
3582 }
3583
3584 sub spice_port {
3585     my ($vmid) = @_;
3586
3587     my $res = vm_mon_cmd($vmid, 'query-spice');
3588
3589     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3590 }
3591
3592 sub qmp_socket {
3593     my ($vmid, $qga) = @_;
3594     my $sockettype = $qga ? 'qga' : 'qmp';
3595     return "${var_run_tmpdir}/$vmid.$sockettype";
3596 }
3597
3598 sub pidfile_name {
3599     my ($vmid) = @_;
3600     return "${var_run_tmpdir}/$vmid.pid";
3601 }
3602
3603 sub vm_devices_list {
3604     my ($vmid) = @_;
3605
3606     my $res = vm_mon_cmd($vmid, 'query-pci');
3607     my $devices = {};
3608     foreach my $pcibus (@$res) {
3609         foreach my $device (@{$pcibus->{devices}}) {
3610             next if !$device->{'qdev_id'};
3611             if ($device->{'pci_bridge'}) {
3612                 $devices->{$device->{'qdev_id'}} = 1;
3613                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3614                     next if !$bridge_device->{'qdev_id'};
3615                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3616                     $devices->{$device->{'qdev_id'}}++;
3617                 }
3618             } else {
3619                 $devices->{$device->{'qdev_id'}} = 1;
3620             }
3621         }
3622     }
3623
3624     my $resblock = vm_mon_cmd($vmid, 'query-block');
3625     foreach my $block (@$resblock) {
3626         if($block->{device} =~ m/^drive-(\S+)/){
3627                 $devices->{$1} = 1;
3628         }
3629     }
3630
3631     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3632     foreach my $mice (@$resmice) {
3633         if ($mice->{name} eq 'QEMU HID Tablet') {
3634             $devices->{tablet} = 1;
3635             last;
3636         }
3637     }
3638
3639     # for usb devices there is no query-usb
3640     # but we can iterate over the entries in
3641     # qom-list path=/machine/peripheral
3642     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3643     foreach my $per (@$resperipheral) {
3644         if ($per->{name} =~ m/^usb\d+$/) {
3645             $devices->{$per->{name}} = 1;
3646         }
3647     }
3648
3649     return $devices;
3650 }
3651
3652 sub vm_deviceplug {
3653     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3654
3655     my $q35 = machine_type_is_q35($conf);
3656
3657     my $devices_list = vm_devices_list($vmid);
3658     return 1 if defined($devices_list->{$deviceid});
3659
3660     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3661
3662     if ($deviceid eq 'tablet') {
3663
3664         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3665
3666     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3667
3668         die "usb hotplug currently not reliable\n";
3669         # since we can't reliably hot unplug all added usb devices
3670         # and usb passthrough disables live migration
3671         # we disable usb hotplugging for now
3672         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3673
3674     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3675
3676         qemu_iothread_add($vmid, $deviceid, $device);
3677
3678         qemu_driveadd($storecfg, $vmid, $device);
3679         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3680
3681         qemu_deviceadd($vmid, $devicefull);
3682         eval { qemu_deviceaddverify($vmid, $deviceid); };
3683         if (my $err = $@) {
3684             eval { qemu_drivedel($vmid, $deviceid); };
3685             warn $@ if $@;
3686             die $err;
3687         }
3688
3689     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3690
3691
3692         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3693         my $pciaddr = print_pci_addr($deviceid);
3694         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3695
3696         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3697
3698         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3699             qemu_iothread_add($vmid, $deviceid, $device);
3700             $devicefull .= ",iothread=iothread-$deviceid";
3701         }
3702
3703         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3704             $devicefull .= ",num_queues=$device->{queues}";
3705         }
3706
3707         qemu_deviceadd($vmid, $devicefull);
3708         qemu_deviceaddverify($vmid, $deviceid);
3709
3710     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3711
3712         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3713         qemu_driveadd($storecfg, $vmid, $device);
3714
3715         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3716         eval { qemu_deviceadd($vmid, $devicefull); };
3717         if (my $err = $@) {
3718             eval { qemu_drivedel($vmid, $deviceid); };
3719             warn $@ if $@;
3720             die $err;
3721         }
3722
3723     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3724
3725         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3726
3727         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
3728         my $use_old_bios_files = undef;
3729         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3730
3731         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3732         qemu_deviceadd($vmid, $netdevicefull);
3733         eval { qemu_deviceaddverify($vmid, $deviceid); };
3734         if (my $err = $@) {
3735             eval { qemu_netdevdel($vmid, $deviceid); };
3736             warn $@ if $@;
3737             die $err;
3738         }
3739
3740     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3741
3742         my $bridgeid = $2;
3743         my $pciaddr = print_pci_addr($deviceid);
3744         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3745
3746         qemu_deviceadd($vmid, $devicefull);
3747         qemu_deviceaddverify($vmid, $deviceid);
3748
3749     } else {
3750         die "can't hotplug device '$deviceid'\n";
3751     }
3752
3753     return 1;
3754 }
3755
3756 # fixme: this should raise exceptions on error!
3757 sub vm_deviceunplug {
3758     my ($vmid, $conf, $deviceid) = @_;
3759
3760     my $devices_list = vm_devices_list($vmid);
3761     return 1 if !defined($devices_list->{$deviceid});
3762
3763     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3764
3765     if ($deviceid eq 'tablet') {
3766
3767         qemu_devicedel($vmid, $deviceid);
3768
3769     } elsif ($deviceid =~ m/^usb\d+$/) {
3770
3771         die "usb hotplug currently not reliable\n";
3772         # when unplugging usb devices this way,
3773         # there may be remaining usb controllers/hubs
3774         # so we disable it for now
3775         qemu_devicedel($vmid, $deviceid);
3776         qemu_devicedelverify($vmid, $deviceid);
3777
3778     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3779
3780         qemu_devicedel($vmid, $deviceid);
3781         qemu_devicedelverify($vmid, $deviceid);
3782         qemu_drivedel($vmid, $deviceid);
3783         qemu_iothread_del($conf, $vmid, $deviceid);
3784
3785     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3786
3787         qemu_devicedel($vmid, $deviceid);
3788         qemu_devicedelverify($vmid, $deviceid);
3789         qemu_iothread_del($conf, $vmid, $deviceid);
3790
3791     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3792
3793         qemu_devicedel($vmid, $deviceid);
3794         qemu_drivedel($vmid, $deviceid);
3795         qemu_deletescsihw($conf, $vmid, $deviceid);
3796
3797     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3798
3799         qemu_devicedel($vmid, $deviceid);
3800         qemu_devicedelverify($vmid, $deviceid);
3801         qemu_netdevdel($vmid, $deviceid);
3802
3803     } else {
3804         die "can't unplug device '$deviceid'\n";
3805     }
3806
3807     return 1;
3808 }
3809
3810 sub qemu_deviceadd {
3811     my ($vmid, $devicefull) = @_;
3812
3813     $devicefull = "driver=".$devicefull;
3814     my %options =  split(/[=,]/, $devicefull);
3815
3816     vm_mon_cmd($vmid, "device_add" , %options);
3817 }
3818
3819 sub qemu_devicedel {
3820     my ($vmid, $deviceid) = @_;
3821
3822     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3823 }
3824
3825 sub qemu_iothread_add {
3826     my($vmid, $deviceid, $device) = @_;
3827
3828     if ($device->{iothread}) {
3829         my $iothreads = vm_iothreads_list($vmid);
3830         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3831     }
3832 }
3833
3834 sub qemu_iothread_del {
3835     my($conf, $vmid, $deviceid) = @_;
3836
3837     my $device = parse_drive($deviceid, $conf->{$deviceid});
3838     if ($device->{iothread}) {
3839         my $iothreads = vm_iothreads_list($vmid);
3840         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3841     }
3842 }
3843
3844 sub qemu_objectadd {
3845     my($vmid, $objectid, $qomtype) = @_;
3846
3847     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3848
3849     return 1;
3850 }
3851
3852 sub qemu_objectdel {
3853     my($vmid, $objectid) = @_;
3854
3855     vm_mon_cmd($vmid, "object-del", id => $objectid);
3856
3857     return 1;
3858 }
3859
3860 sub qemu_driveadd {
3861     my ($storecfg, $vmid, $device) = @_;
3862
3863     my $drive = print_drive_full($storecfg, $vmid, $device);
3864     $drive =~ s/\\/\\\\/g;
3865     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3866
3867     # If the command succeeds qemu prints: "OK"
3868     return 1 if $ret =~ m/OK/s;
3869
3870     die "adding drive failed: $ret\n";
3871 }
3872
3873 sub qemu_drivedel {
3874     my($vmid, $deviceid) = @_;
3875
3876     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3877     $ret =~ s/^\s+//;
3878
3879     return 1 if $ret eq "";
3880
3881     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3882     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3883
3884     die "deleting drive $deviceid failed : $ret\n";
3885 }
3886
3887 sub qemu_deviceaddverify {
3888     my ($vmid, $deviceid) = @_;
3889
3890     for (my $i = 0; $i <= 5; $i++) {
3891          my $devices_list = vm_devices_list($vmid);
3892          return 1 if defined($devices_list->{$deviceid});
3893          sleep 1;
3894     }
3895
3896     die "error on hotplug device '$deviceid'\n";
3897 }
3898
3899
3900 sub qemu_devicedelverify {
3901     my ($vmid, $deviceid) = @_;
3902
3903     # need to verify that the device is correctly removed as device_del
3904     # is async and empty return is not reliable
3905
3906     for (my $i = 0; $i <= 5; $i++) {
3907          my $devices_list = vm_devices_list($vmid);
3908          return 1 if !defined($devices_list->{$deviceid});
3909          sleep 1;
3910     }
3911
3912     die "error on hot-unplugging device '$deviceid'\n";
3913 }
3914
3915 sub qemu_findorcreatescsihw {
3916     my ($storecfg, $conf, $vmid, $device) = @_;
3917
3918     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3919
3920     my $scsihwid="$controller_prefix$controller";
3921     my $devices_list = vm_devices_list($vmid);
3922
3923     if(!defined($devices_list->{$scsihwid})) {
3924         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3925     }
3926
3927     return 1;
3928 }
3929
3930 sub qemu_deletescsihw {
3931     my ($conf, $vmid, $opt) = @_;
3932
3933     my $device = parse_drive($opt, $conf->{$opt});
3934
3935     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3936         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3937         return 1;
3938     }
3939
3940     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3941
3942     my $devices_list = vm_devices_list($vmid);
3943     foreach my $opt (keys %{$devices_list}) {
3944         if (PVE::QemuServer::is_valid_drivename($opt)) {
3945             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3946             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3947                 return 1;
3948             }
3949         }
3950     }
3951
3952     my $scsihwid="scsihw$controller";
3953
3954     vm_deviceunplug($vmid, $conf, $scsihwid);
3955
3956     return 1;
3957 }
3958
3959 sub qemu_add_pci_bridge {
3960     my ($storecfg, $conf, $vmid, $device) = @_;
3961
3962     my $bridges = {};
3963
3964     my $bridgeid;
3965
3966     print_pci_addr($device, $bridges);
3967
3968     while (my ($k, $v) = each %$bridges) {
3969         $bridgeid = $k;
3970     }
3971     return 1 if !defined($bridgeid) || $bridgeid < 1;
3972
3973     my $bridge = "pci.$bridgeid";
3974     my $devices_list = vm_devices_list($vmid);
3975
3976     if (!defined($devices_list->{$bridge})) {
3977         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3978     }
3979
3980     return 1;
3981 }
3982
3983 sub qemu_set_link_status {
3984     my ($vmid, $device, $up) = @_;
3985
3986     vm_mon_cmd($vmid, "set_link", name => $device,
3987                up => $up ? JSON::true : JSON::false);
3988 }
3989
3990 sub qemu_netdevadd {
3991     my ($vmid, $conf, $device, $deviceid) = @_;
3992
3993     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
3994     my %options =  split(/[=,]/, $netdev);
3995
3996     vm_mon_cmd($vmid, "netdev_add",  %options);
3997     return 1;
3998 }
3999
4000 sub qemu_netdevdel {
4001     my ($vmid, $deviceid) = @_;
4002
4003     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
4004 }
4005
4006 sub qemu_usb_hotplug {
4007     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
4008
4009     return if !$device;
4010
4011     # remove the old one first
4012     vm_deviceunplug($vmid, $conf, $deviceid);
4013
4014     # check if xhci controller is necessary and available
4015     if ($device->{usb3}) {
4016
4017         my $devicelist = vm_devices_list($vmid);
4018
4019         if (!$devicelist->{xhci}) {
4020             my $pciaddr = print_pci_addr("xhci");
4021             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4022         }
4023     }
4024     my $d = parse_usb_device($device->{host});
4025     $d->{usb3} = $device->{usb3};
4026
4027     # add the new one
4028     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d);
4029 }
4030
4031 sub qemu_cpu_hotplug {
4032     my ($vmid, $conf, $vcpus) = @_;
4033
4034     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
4035
4036     my $sockets = 1;
4037     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4038     $sockets = $conf->{sockets} if  $conf->{sockets};
4039     my $cores = $conf->{cores} || 1;
4040     my $maxcpus = $sockets * $cores;
4041
4042     $vcpus = $maxcpus if !$vcpus;
4043
4044     die "you can't add more vcpus than maxcpus\n"
4045         if $vcpus > $maxcpus;
4046
4047     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4048
4049     if ($vcpus < $currentvcpus) {
4050
4051         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4052
4053             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4054                 qemu_devicedel($vmid, "cpu$i");
4055                 my $retry = 0;
4056                 my $currentrunningvcpus = undef;
4057                 while (1) {
4058                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4059                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4060                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4061                     $retry++;
4062                     sleep 1;
4063                 }
4064                 #update conf after each succesfull cpu unplug
4065                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4066                 PVE::QemuConfig->write_config($vmid, $conf);
4067             }
4068         } else {
4069             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4070         }
4071
4072         return;
4073     }
4074
4075     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4076     die "vcpus in running vm does not match its configuration\n"
4077         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4078
4079     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4080
4081         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4082             my $cpustr = print_cpu_device($conf, $i);
4083             qemu_deviceadd($vmid, $cpustr);
4084
4085             my $retry = 0;
4086             my $currentrunningvcpus = undef;
4087             while (1) {
4088                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4089                 last if scalar(@{$currentrunningvcpus}) == $i;
4090                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4091                 sleep 1;
4092                 $retry++;
4093             }
4094             #update conf after each succesfull cpu hotplug
4095             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4096             PVE::QemuConfig->write_config($vmid, $conf);
4097         }
4098     } else {
4099
4100         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4101             vm_mon_cmd($vmid, "cpu-add", id => int($i));
4102         }
4103     }
4104 }
4105
4106 sub qemu_block_set_io_throttle {
4107     my ($vmid, $deviceid,
4108         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4109         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4110         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4111         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4112
4113     return if !check_running($vmid) ;
4114
4115     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4116         bps => int($bps),
4117         bps_rd => int($bps_rd),
4118         bps_wr => int($bps_wr),
4119         iops => int($iops),
4120         iops_rd => int($iops_rd),
4121         iops_wr => int($iops_wr),
4122         bps_max => int($bps_max),
4123         bps_rd_max => int($bps_rd_max),
4124         bps_wr_max => int($bps_wr_max),
4125         iops_max => int($iops_max),
4126         iops_rd_max => int($iops_rd_max),
4127         iops_wr_max => int($iops_wr_max),
4128         bps_max_length => int($bps_max_length),
4129         bps_rd_max_length => int($bps_rd_max_length),
4130         bps_wr_max_length => int($bps_wr_max_length),
4131         iops_max_length => int($iops_max_length),
4132         iops_rd_max_length => int($iops_rd_max_length),
4133         iops_wr_max_length => int($iops_wr_max_length),
4134     );
4135
4136 }
4137
4138 # old code, only used to shutdown old VM after update
4139 sub __read_avail {
4140     my ($fh, $timeout) = @_;
4141
4142     my $sel = new IO::Select;
4143     $sel->add($fh);
4144
4145     my $res = '';
4146     my $buf;
4147
4148     my @ready;
4149     while (scalar (@ready = $sel->can_read($timeout))) {
4150         my $count;
4151         if ($count = $fh->sysread($buf, 8192)) {
4152             if ($buf =~ /^(.*)\(qemu\) $/s) {
4153                 $res .= $1;
4154                 last;
4155             } else {
4156                 $res .= $buf;
4157             }
4158         } else {
4159             if (!defined($count)) {
4160                 die "$!\n";
4161             }
4162             last;
4163         }
4164     }
4165
4166     die "monitor read timeout\n" if !scalar(@ready);
4167
4168     return $res;
4169 }
4170
4171 # old code, only used to shutdown old VM after update
4172 sub vm_monitor_command {
4173     my ($vmid, $cmdstr, $nocheck) = @_;
4174
4175     my $res;
4176
4177     eval {
4178         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4179
4180         my $sname = "${var_run_tmpdir}/$vmid.mon";
4181
4182         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
4183             die "unable to connect to VM $vmid socket - $!\n";
4184
4185         my $timeout = 3;
4186
4187         # hack: migrate sometime blocks the monitor (when migrate_downtime
4188         # is set)
4189         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
4190             $timeout = 60*60; # 1 hour
4191         }
4192
4193         # read banner;
4194         my $data = __read_avail($sock, $timeout);
4195
4196         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
4197             die "got unexpected qemu monitor banner\n";
4198         }
4199
4200         my $sel = new IO::Select;
4201         $sel->add($sock);
4202
4203         if (!scalar(my @ready = $sel->can_write($timeout))) {
4204             die "monitor write error - timeout";
4205         }
4206
4207         my $fullcmd = "$cmdstr\r";
4208
4209         # syslog('info', "VM $vmid monitor command: $cmdstr");
4210
4211         my $b;
4212         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
4213             die "monitor write error - $!";
4214         }
4215
4216         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
4217
4218         $timeout = 20;
4219
4220         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
4221             $timeout = 60*60; # 1 hour
4222         } elsif ($cmdstr =~ m/^(eject|change)/) {
4223             $timeout = 60; # note: cdrom mount command is slow
4224         }
4225         if ($res = __read_avail($sock, $timeout)) {
4226
4227             my @lines = split("\r?\n", $res);
4228
4229             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
4230
4231             $res = join("\n", @lines);
4232             $res .= "\n";
4233         }
4234     };
4235
4236     my $err = $@;
4237
4238     if ($err) {
4239         syslog("err", "VM $vmid monitor command failed - $err");
4240         die $err;
4241     }
4242
4243     return $res;
4244 }
4245
4246 sub qemu_block_resize {
4247     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4248
4249     my $running = check_running($vmid);
4250
4251     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4252
4253     return if !$running;
4254
4255     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4256
4257 }
4258
4259 sub qemu_volume_snapshot {
4260     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4261
4262     my $running = check_running($vmid);
4263
4264     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4265         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
4266     } else {
4267         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4268     }
4269 }
4270
4271 sub qemu_volume_snapshot_delete {
4272     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4273
4274     my $running = check_running($vmid);
4275
4276     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4277         vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
4278     } else {
4279         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4280     }
4281 }
4282
4283 sub set_migration_caps {
4284     my ($vmid) = @_;
4285
4286     my $cap_ref = [];
4287
4288     my $enabled_cap = {
4289         "auto-converge" => 1,
4290         "xbzrle" => 1,
4291         "x-rdma-pin-all" => 0,
4292         "zero-blocks" => 0,
4293         "compress" => 0
4294     };
4295
4296     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4297
4298     for my $supported_capability (@$supported_capabilities) {
4299         push @$cap_ref, {
4300             capability => $supported_capability->{capability},
4301             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4302         };
4303     }
4304
4305     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4306 }
4307
4308 my $fast_plug_option = {
4309     'lock' => 1,
4310     'name' => 1,
4311     'onboot' => 1,
4312     'shares' => 1,
4313     'startup' => 1,
4314     'description' => 1,
4315     'protection' => 1,
4316     'vmstatestorage' => 1,
4317 };
4318
4319 # hotplug changes in [PENDING]
4320 # $selection hash can be used to only apply specified options, for
4321 # example: { cores => 1 } (only apply changed 'cores')
4322 # $errors ref is used to return error messages
4323 sub vmconfig_hotplug_pending {
4324     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4325
4326     my $defaults = load_defaults();
4327
4328     # commit values which do not have any impact on running VM first
4329     # Note: those option cannot raise errors, we we do not care about
4330     # $selection and always apply them.
4331
4332     my $add_error = sub {
4333         my ($opt, $msg) = @_;
4334         $errors->{$opt} = "hotplug problem - $msg";
4335     };
4336
4337     my $changes = 0;
4338     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4339         if ($fast_plug_option->{$opt}) {
4340             $conf->{$opt} = $conf->{pending}->{$opt};
4341             delete $conf->{pending}->{$opt};
4342             $changes = 1;
4343         }
4344     }
4345
4346     if ($changes) {
4347         PVE::QemuConfig->write_config($vmid, $conf);
4348         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4349     }
4350
4351     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4352
4353     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4354     while (my ($opt, $force) = each %$pending_delete_hash) {
4355         next if $selection && !$selection->{$opt};
4356         eval {
4357             if ($opt eq 'hotplug') {
4358                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4359             } elsif ($opt eq 'tablet') {
4360                 die "skip\n" if !$hotplug_features->{usb};
4361                 if ($defaults->{tablet}) {
4362                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4363                 } else {
4364                     vm_deviceunplug($vmid, $conf, $opt);
4365                 }
4366             } elsif ($opt =~ m/^usb\d+/) {
4367                 die "skip\n";
4368                 # since we cannot reliably hot unplug usb devices
4369                 # we are disabling it
4370                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4371                 vm_deviceunplug($vmid, $conf, $opt);
4372             } elsif ($opt eq 'vcpus') {
4373                 die "skip\n" if !$hotplug_features->{cpu};
4374                 qemu_cpu_hotplug($vmid, $conf, undef);
4375             } elsif ($opt eq 'balloon') {
4376                 # enable balloon device is not hotpluggable
4377                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
4378             } elsif ($fast_plug_option->{$opt}) {
4379                 # do nothing
4380             } elsif ($opt =~ m/^net(\d+)$/) {
4381                 die "skip\n" if !$hotplug_features->{network};
4382                 vm_deviceunplug($vmid, $conf, $opt);
4383             } elsif (is_valid_drivename($opt)) {
4384                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4385                 vm_deviceunplug($vmid, $conf, $opt);
4386                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4387             } elsif ($opt =~ m/^memory$/) {
4388                 die "skip\n" if !$hotplug_features->{memory};
4389                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4390             } elsif ($opt eq 'cpuunits') {
4391                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4392             } elsif ($opt eq 'cpulimit') {
4393                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4394             } else {
4395                 die "skip\n";
4396             }
4397         };
4398         if (my $err = $@) {
4399             &$add_error($opt, $err) if $err ne "skip\n";
4400         } else {
4401             # save new config if hotplug was successful
4402             delete $conf->{$opt};
4403             vmconfig_undelete_pending_option($conf, $opt);
4404             PVE::QemuConfig->write_config($vmid, $conf);
4405             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4406         }
4407     }
4408
4409     my $apply_pending_cloudinit;
4410     $apply_pending_cloudinit = sub {
4411         my ($key, $value) = @_;
4412         $apply_pending_cloudinit = sub {}; # once is enough
4413
4414         my @cloudinit_opts = keys %$confdesc_cloudinit;
4415         foreach my $opt (keys %{$conf->{pending}}) {
4416             next if !grep { $_ eq $opt } @cloudinit_opts;
4417             $conf->{$opt} = delete $conf->{pending}->{$opt};
4418         }
4419
4420         my $new_conf = { %$conf };
4421         $new_conf->{$key} = $value;
4422         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4423     };
4424
4425     foreach my $opt (keys %{$conf->{pending}}) {
4426         next if $selection && !$selection->{$opt};
4427         my $value = $conf->{pending}->{$opt};
4428         eval {
4429             if ($opt eq 'hotplug') {
4430                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4431             } elsif ($opt eq 'tablet') {
4432                 die "skip\n" if !$hotplug_features->{usb};
4433                 if ($value == 1) {
4434                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4435                 } elsif ($value == 0) {
4436                     vm_deviceunplug($vmid, $conf, $opt);
4437                 }
4438             } elsif ($opt =~ m/^usb\d+$/) {
4439                 die "skip\n";
4440                 # since we cannot reliably hot unplug usb devices
4441                 # we are disabling it
4442                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4443                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4444                 die "skip\n" if !$d;
4445                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d);
4446             } elsif ($opt eq 'vcpus') {
4447                 die "skip\n" if !$hotplug_features->{cpu};
4448                 qemu_cpu_hotplug($vmid, $conf, $value);
4449             } elsif ($opt eq 'balloon') {
4450                 # enable/disable balloning device is not hotpluggable
4451                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4452                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4453                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4454
4455                 # allow manual ballooning if shares is set to zero
4456                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4457                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4458                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4459                 }
4460             } elsif ($opt =~ m/^net(\d+)$/) {
4461                 # some changes can be done without hotplug
4462                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4463                                     $vmid, $opt, $value);
4464             } elsif (is_valid_drivename($opt)) {
4465                 # some changes can be done without hotplug
4466                 my $drive = parse_drive($opt, $value);
4467                 if (drive_is_cloudinit($drive)) {
4468                     &$apply_pending_cloudinit($opt, $value);
4469                 }
4470                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4471                                      $vmid, $opt, $value, 1);
4472             } elsif ($opt =~ m/^memory$/) { #dimms
4473                 die "skip\n" if !$hotplug_features->{memory};
4474                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4475             } elsif ($opt eq 'cpuunits') {
4476                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4477             } elsif ($opt eq 'cpulimit') {
4478                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4479                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4480             } else {
4481                 die "skip\n";  # skip non-hot-pluggable options
4482             }
4483         };
4484         if (my $err = $@) {
4485             &$add_error($opt, $err) if $err ne "skip\n";
4486         } else {
4487             # save new config if hotplug was successful
4488             $conf->{$opt} = $value;
4489             delete $conf->{pending}->{$opt};
4490             PVE::QemuConfig->write_config($vmid, $conf);
4491             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4492         }
4493     }
4494 }
4495
4496 sub try_deallocate_drive {
4497     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4498
4499     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4500         my $volid = $drive->{file};
4501         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4502             my $sid = PVE::Storage::parse_volume_id($volid);
4503             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4504
4505             # check if the disk is really unused
4506             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4507                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4508             PVE::Storage::vdisk_free($storecfg, $volid);
4509             return 1;
4510         } else {
4511             # If vm is not owner of this disk remove from config
4512             return 1;
4513         }
4514     }
4515
4516     return undef;
4517 }
4518
4519 sub vmconfig_delete_or_detach_drive {
4520     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4521
4522     my $drive = parse_drive($opt, $conf->{$opt});
4523
4524     my $rpcenv = PVE::RPCEnvironment::get();
4525     my $authuser = $rpcenv->get_user();
4526
4527     if ($force) {
4528         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4529         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4530     } else {
4531         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4532     }
4533 }
4534
4535 sub vmconfig_apply_pending {
4536     my ($vmid, $conf, $storecfg) = @_;
4537
4538     # cold plug
4539
4540     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4541     while (my ($opt, $force) = each %$pending_delete_hash) {
4542         die "internal error" if $opt =~ m/^unused/;
4543         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4544         if (!defined($conf->{$opt})) {
4545             vmconfig_undelete_pending_option($conf, $opt);
4546             PVE::QemuConfig->write_config($vmid, $conf);
4547         } elsif (is_valid_drivename($opt)) {
4548             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4549             vmconfig_undelete_pending_option($conf, $opt);
4550             delete $conf->{$opt};
4551             PVE::QemuConfig->write_config($vmid, $conf);
4552         } else {
4553             vmconfig_undelete_pending_option($conf, $opt);
4554             delete $conf->{$opt};
4555             PVE::QemuConfig->write_config($vmid, $conf);
4556         }
4557     }
4558
4559     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4560
4561     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4562         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4563
4564         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4565             # skip if nothing changed
4566         } elsif (is_valid_drivename($opt)) {
4567             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4568                 if defined($conf->{$opt});
4569             $conf->{$opt} = $conf->{pending}->{$opt};
4570         } else {
4571             $conf->{$opt} = $conf->{pending}->{$opt};
4572         }
4573
4574         delete $conf->{pending}->{$opt};
4575         PVE::QemuConfig->write_config($vmid, $conf);
4576     }
4577 }
4578
4579 my $safe_num_ne = sub {
4580     my ($a, $b) = @_;
4581
4582     return 0 if !defined($a) && !defined($b);
4583     return 1 if !defined($a);
4584     return 1 if !defined($b);
4585
4586     return $a != $b;
4587 };
4588
4589 my $safe_string_ne = sub {
4590     my ($a, $b) = @_;
4591
4592     return 0 if !defined($a) && !defined($b);
4593     return 1 if !defined($a);
4594     return 1 if !defined($b);
4595
4596     return $a ne $b;
4597 };
4598
4599 sub vmconfig_update_net {
4600     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4601
4602     my $newnet = parse_net($value);
4603
4604     if ($conf->{$opt}) {
4605         my $oldnet = parse_net($conf->{$opt});
4606
4607         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4608             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4609             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4610             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4611
4612             # for non online change, we try to hot-unplug
4613             die "skip\n" if !$hotplug;
4614             vm_deviceunplug($vmid, $conf, $opt);
4615         } else {
4616
4617             die "internal error" if $opt !~ m/net(\d+)/;
4618             my $iface = "tap${vmid}i$1";
4619
4620             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4621                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4622                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4623                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4624                 PVE::Network::tap_unplug($iface);
4625                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4626             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4627                 # Rate can be applied on its own but any change above needs to
4628                 # include the rate in tap_plug since OVS resets everything.
4629                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4630             }
4631
4632             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4633                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4634             }
4635
4636             return 1;
4637         }
4638     }
4639
4640     if ($hotplug) {
4641         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4642     } else {
4643         die "skip\n";
4644     }
4645 }
4646
4647 sub vmconfig_update_disk {
4648     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4649
4650     # fixme: do we need force?
4651
4652     my $drive = parse_drive($opt, $value);
4653
4654     if ($conf->{$opt}) {
4655
4656         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4657
4658             my $media = $drive->{media} || 'disk';
4659             my $oldmedia = $old_drive->{media} || 'disk';
4660             die "unable to change media type\n" if $media ne $oldmedia;
4661
4662             if (!drive_is_cdrom($old_drive)) {
4663
4664                 if ($drive->{file} ne $old_drive->{file}) {
4665
4666                     die "skip\n" if !$hotplug;
4667
4668                     # unplug and register as unused
4669                     vm_deviceunplug($vmid, $conf, $opt);
4670                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4671
4672                 } else {
4673                     # update existing disk
4674
4675                     # skip non hotpluggable value
4676                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4677                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4678                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4679                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4680                         die "skip\n";
4681                     }
4682
4683                     # apply throttle
4684                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4685                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4686                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4687                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4688                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4689                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4690                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4691                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4692                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4693                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4694                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4695                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4696                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4697                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4698                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4699                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4700                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4701                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4702
4703                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4704                                                    ($drive->{mbps} || 0)*1024*1024,
4705                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4706                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4707                                                    $drive->{iops} || 0,
4708                                                    $drive->{iops_rd} || 0,
4709                                                    $drive->{iops_wr} || 0,
4710                                                    ($drive->{mbps_max} || 0)*1024*1024,
4711                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4712                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4713                                                    $drive->{iops_max} || 0,
4714                                                    $drive->{iops_rd_max} || 0,
4715                                                    $drive->{iops_wr_max} || 0,
4716                                                    $drive->{bps_max_length} || 1,
4717                                                    $drive->{bps_rd_max_length} || 1,
4718                                                    $drive->{bps_wr_max_length} || 1,
4719                                                    $drive->{iops_max_length} || 1,
4720                                                    $drive->{iops_rd_max_length} || 1,
4721                                                    $drive->{iops_wr_max_length} || 1);
4722
4723                     }
4724
4725                     return 1;
4726                 }
4727
4728             } else { # cdrom
4729
4730                 if ($drive->{file} eq 'none') {
4731                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4732                     if (drive_is_cloudinit($old_drive)) {
4733                         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
4734                     }
4735                 } else {
4736                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4737                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4738                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4739                 }
4740
4741                 return 1;
4742             }
4743         }
4744     }
4745
4746     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4747     # hotplug new disks
4748     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4749     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4750 }
4751
4752 sub vm_start {
4753     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4754         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
4755
4756     PVE::QemuConfig->lock_config($vmid, sub {
4757         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4758
4759         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
4760
4761         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4762
4763         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4764
4765         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4766             vmconfig_apply_pending($vmid, $conf, $storecfg);
4767             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4768         }
4769
4770         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
4771
4772         my $defaults = load_defaults();
4773
4774         # set environment variable useful inside network script
4775         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4776
4777         my $local_volumes = {};
4778
4779         if ($targetstorage) {
4780             foreach_drive($conf, sub {
4781                 my ($ds, $drive) = @_;
4782
4783                 return if drive_is_cdrom($drive);
4784
4785                 my $volid = $drive->{file};
4786
4787                 return if !$volid;
4788
4789                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4790
4791                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4792                 return if $scfg->{shared};
4793                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
4794             });
4795
4796             my $format = undef;
4797
4798             foreach my $opt (sort keys %$local_volumes) {
4799
4800                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
4801                 my $drive = parse_drive($opt, $conf->{$opt});
4802
4803                 #if remote storage is specified, use default format
4804                 if ($targetstorage && $targetstorage ne "1") {
4805                     $storeid = $targetstorage;
4806                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
4807                     $format = $defFormat;
4808                 } else {
4809                     #else we use same format than original
4810                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4811                     $format = qemu_img_format($scfg, $volid);
4812                 }
4813
4814                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
4815                 my $newdrive = $drive;
4816                 $newdrive->{format} = $format;
4817                 $newdrive->{file} = $newvolid;
4818                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
4819                 $local_volumes->{$opt} = $drivestr;
4820                 #pass drive to conf for command line
4821                 $conf->{$opt} = $drivestr;
4822             }
4823         }
4824
4825         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4826
4827         my $migrate_port = 0;
4828         my $migrate_uri;
4829         if ($statefile) {
4830             if ($statefile eq 'tcp') {
4831                 my $localip = "localhost";
4832                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4833                 my $nodename = PVE::INotify::nodename();
4834
4835                 if (!defined($migration_type)) {
4836                     if (defined($datacenterconf->{migration}->{type})) {
4837                         $migration_type = $datacenterconf->{migration}->{type};
4838                     } else {
4839                         $migration_type = 'secure';
4840                     }
4841                 }
4842
4843                 if ($migration_type eq 'insecure') {
4844                     my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4845                     if ($migrate_network_addr) {
4846                         $localip = $migrate_network_addr;
4847                     } else {
4848                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4849                     }
4850
4851                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4852                 }
4853
4854                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4855                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4856                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4857                 push @$cmd, '-incoming', $migrate_uri;
4858                 push @$cmd, '-S';
4859
4860             } elsif ($statefile eq 'unix') {
4861                 # should be default for secure migrations as a ssh TCP forward
4862                 # tunnel is not deterministic reliable ready and fails regurarly
4863                 # to set up in time, so use UNIX socket forwards
4864                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
4865                 unlink $socket_addr;
4866
4867                 $migrate_uri = "unix:$socket_addr";
4868
4869                 push @$cmd, '-incoming', $migrate_uri;
4870                 push @$cmd, '-S';
4871
4872             } else {
4873                 push @$cmd, '-loadstate', $statefile;
4874             }
4875         } elsif ($paused) {
4876             push @$cmd, '-S';
4877         }
4878
4879         # host pci devices
4880         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4881           my $d = parse_hostpci($conf->{"hostpci$i"});
4882           next if !$d;
4883           my $pcidevices = $d->{pciid};
4884           foreach my $pcidevice (@$pcidevices) {
4885                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4886
4887                 my $info = pci_device_info("0000:$pciid");
4888                 die "IOMMU not present\n" if !check_iommu_support();
4889                 die "no pci device info for device '$pciid'\n" if !$info;
4890                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4891                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4892           }
4893         }
4894
4895         PVE::Storage::activate_volumes($storecfg, $vollist);
4896
4897         if (!check_running($vmid, 1) && -d "/sys/fs/cgroup/systemd/qemu.slice/$vmid.scope") {
4898             my $cmd = [];
4899             push @$cmd, '/bin/systemctl', 'stop', "$vmid.scope";
4900             eval  { run_command($cmd); };
4901         }
4902
4903         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
4904                                                   : $defaults->{cpuunits};
4905
4906         my $start_timeout = $conf->{hugepages} ? 300 : 30;
4907         my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077);
4908
4909         my %properties = (
4910             Slice => 'qemu.slice',
4911             KillMode => 'none',
4912             CPUShares => $cpuunits
4913         );
4914
4915         if (my $cpulimit = $conf->{cpulimit}) {
4916             $properties{CPUQuota} = int($cpulimit * 100);
4917         }
4918         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
4919
4920         if ($conf->{hugepages}) {
4921
4922             my $code = sub {
4923                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
4924                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
4925
4926                 PVE::QemuServer::Memory::hugepages_mount();
4927                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
4928
4929                 eval  {
4930                     PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4931                     run_command($cmd, %run_params);
4932                 };
4933
4934                 if (my $err = $@) {
4935                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
4936                     die $err;
4937                 }
4938
4939                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
4940             };
4941             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
4942
4943         } else {
4944             eval  {
4945                 PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4946                 run_command($cmd, %run_params);
4947             };
4948         }
4949
4950         if (my $err = $@) {
4951             # deactivate volumes if start fails
4952             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
4953             die "start failed: $err";
4954         }
4955
4956         print "migration listens on $migrate_uri\n" if $migrate_uri;
4957
4958         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
4959             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4960             warn $@ if $@;
4961         }
4962
4963         #start nbd server for storage migration
4964         if ($targetstorage) {
4965             my $nodename = PVE::INotify::nodename();
4966             my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4967             my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
4968             my $pfamily = PVE::Tools::get_host_address_family($nodename);
4969             $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4970
4971             vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${migrate_port}" } } );
4972
4973             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4974
4975             foreach my $opt (sort keys %$local_volumes) {
4976                 my $volid = $local_volumes->{$opt};
4977                 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
4978                 my $migrate_storage_uri = "nbd:${localip}:${migrate_port}:exportname=drive-$opt";
4979                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
4980             }
4981         }
4982
4983         if ($migratedfrom) {
4984             eval {
4985                 set_migration_caps($vmid);
4986             };
4987             warn $@ if $@;
4988
4989             if ($spice_port) {
4990                 print "spice listens on port $spice_port\n";
4991                 if ($spice_ticket) {
4992                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4993                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4994                 }
4995             }
4996
4997         } else {
4998             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4999                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5000                     if $conf->{balloon};
5001             }
5002
5003             foreach my $opt (keys %$conf) {
5004                 next if $opt !~  m/^net\d+$/;
5005                 my $nicconf = parse_net($conf->{$opt});
5006                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5007             }
5008         }
5009
5010         vm_mon_cmd_nocheck($vmid, 'qom-set',
5011                     path => "machine/peripheral/balloon0",
5012                     property => "guest-stats-polling-interval",
5013                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5014
5015     });
5016 }
5017
5018 sub vm_mon_cmd {
5019     my ($vmid, $execute, %params) = @_;
5020
5021     my $cmd = { execute => $execute, arguments => \%params };
5022     vm_qmp_command($vmid, $cmd);
5023 }
5024
5025 sub vm_mon_cmd_nocheck {
5026     my ($vmid, $execute, %params) = @_;
5027
5028     my $cmd = { execute => $execute, arguments => \%params };
5029     vm_qmp_command($vmid, $cmd, 1);
5030 }
5031
5032 sub vm_qmp_command {
5033     my ($vmid, $cmd, $nocheck) = @_;
5034
5035     my $res;
5036
5037     my $timeout;
5038     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
5039         $timeout = $cmd->{arguments}->{timeout};
5040         delete $cmd->{arguments}->{timeout};
5041     }
5042
5043     eval {
5044         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
5045         my $sname = qmp_socket($vmid);
5046         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
5047             my $qmpclient = PVE::QMPClient->new();
5048
5049             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
5050         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
5051             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
5052                 if scalar(%{$cmd->{arguments}});
5053             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
5054         } else {
5055             die "unable to open monitor socket\n";
5056         }
5057     };
5058     if (my $err = $@) {
5059         syslog("err", "VM $vmid qmp command failed - $err");
5060         die $err;
5061     }
5062
5063     return $res;
5064 }
5065
5066 sub vm_human_monitor_command {
5067     my ($vmid, $cmdline) = @_;
5068
5069     my $res;
5070
5071     my $cmd = {
5072         execute => 'human-monitor-command',
5073         arguments => { 'command-line' => $cmdline},
5074     };
5075
5076     return vm_qmp_command($vmid, $cmd);
5077 }
5078
5079 sub vm_commandline {
5080     my ($storecfg, $vmid) = @_;
5081
5082     my $conf = PVE::QemuConfig->load_config($vmid);
5083
5084     my $defaults = load_defaults();
5085
5086     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
5087
5088     return PVE::Tools::cmd2string($cmd);
5089 }
5090
5091 sub vm_reset {
5092     my ($vmid, $skiplock) = @_;
5093
5094     PVE::QemuConfig->lock_config($vmid, sub {
5095
5096         my $conf = PVE::QemuConfig->load_config($vmid);
5097
5098         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5099
5100         vm_mon_cmd($vmid, "system_reset");
5101     });
5102 }
5103
5104 sub get_vm_volumes {
5105     my ($conf) = @_;
5106
5107     my $vollist = [];
5108     foreach_volid($conf, sub {
5109         my ($volid, $attr) = @_;
5110
5111         return if $volid =~ m|^/|;
5112
5113         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5114         return if !$sid;
5115
5116         push @$vollist, $volid;
5117     });
5118
5119     return $vollist;
5120 }
5121
5122 sub vm_stop_cleanup {
5123     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5124
5125     eval {
5126
5127         if (!$keepActive) {
5128             my $vollist = get_vm_volumes($conf);
5129             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5130         }
5131
5132         foreach my $ext (qw(mon qmp pid vnc qga)) {
5133             unlink "/var/run/qemu-server/${vmid}.$ext";
5134         }
5135
5136         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5137     };
5138     warn $@ if $@; # avoid errors - just warn
5139 }
5140
5141 # Note: use $nockeck to skip tests if VM configuration file exists.
5142 # We need that when migration VMs to other nodes (files already moved)
5143 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5144 sub vm_stop {
5145     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5146
5147     $force = 1 if !defined($force) && !$shutdown;
5148
5149     if ($migratedfrom){
5150         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5151         kill 15, $pid if $pid;
5152         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5153         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5154         return;
5155     }
5156
5157     PVE::QemuConfig->lock_config($vmid, sub {
5158
5159         my $pid = check_running($vmid, $nocheck);
5160         return if !$pid;
5161
5162         my $conf;
5163         if (!$nocheck) {
5164             $conf = PVE::QemuConfig->load_config($vmid);
5165             PVE::QemuConfig->check_lock($conf) if !$skiplock;
5166             if (!defined($timeout) && $shutdown && $conf->{startup}) {
5167                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5168                 $timeout = $opts->{down} if $opts->{down};
5169             }
5170         }
5171
5172         $timeout = 60 if !defined($timeout);
5173
5174         eval {
5175             if ($shutdown) {
5176                 if (defined($conf) && $conf->{agent}) {
5177                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
5178                 } else {
5179                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
5180                 }
5181             } else {
5182                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
5183             }
5184         };
5185         my $err = $@;
5186
5187         if (!$err) {
5188             my $count = 0;
5189             while (($count < $timeout) && check_running($vmid, $nocheck)) {
5190                 $count++;
5191                 sleep 1;
5192             }
5193
5194             if ($count >= $timeout) {
5195                 if ($force) {
5196                     warn "VM still running - terminating now with SIGTERM\n";
5197                     kill 15, $pid;
5198                 } else {
5199                     die "VM quit/powerdown failed - got timeout\n";
5200                 }
5201             } else {
5202                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5203                 return;
5204             }
5205         } else {
5206             if ($force) {
5207                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5208                 kill 15, $pid;
5209             } else {
5210                 die "VM quit/powerdown failed\n";
5211             }
5212         }
5213
5214         # wait again
5215         $timeout = 10;
5216
5217         my $count = 0;
5218         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5219             $count++;
5220             sleep 1;
5221         }
5222
5223         if ($count >= $timeout) {
5224             warn "VM still running - terminating now with SIGKILL\n";
5225             kill 9, $pid;
5226             sleep 1;
5227         }
5228
5229         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5230    });
5231 }
5232
5233 sub vm_suspend {
5234     my ($vmid, $skiplock) = @_;
5235
5236     PVE::QemuConfig->lock_config($vmid, sub {
5237
5238         my $conf = PVE::QemuConfig->load_config($vmid);
5239
5240         PVE::QemuConfig->check_lock($conf)
5241             if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5242
5243         vm_mon_cmd($vmid, "stop");
5244     });
5245 }
5246
5247 sub vm_resume {
5248     my ($vmid, $skiplock, $nocheck) = @_;
5249
5250     PVE::QemuConfig->lock_config($vmid, sub {
5251
5252         if (!$nocheck) {
5253
5254             my $conf = PVE::QemuConfig->load_config($vmid);
5255
5256             PVE::QemuConfig->check_lock($conf)
5257                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5258
5259             vm_mon_cmd($vmid, "cont");
5260
5261         } else {
5262             vm_mon_cmd_nocheck($vmid, "cont");
5263         }
5264     });
5265 }
5266
5267 sub vm_sendkey {
5268     my ($vmid, $skiplock, $key) = @_;
5269
5270     PVE::QemuConfig->lock_config($vmid, sub {
5271
5272         my $conf = PVE::QemuConfig->load_config($vmid);
5273
5274         # there is no qmp command, so we use the human monitor command
5275         vm_human_monitor_command($vmid, "sendkey $key");
5276     });
5277 }
5278
5279 sub vm_destroy {
5280     my ($storecfg, $vmid, $skiplock) = @_;
5281
5282     PVE::QemuConfig->lock_config($vmid, sub {
5283
5284         my $conf = PVE::QemuConfig->load_config($vmid);
5285
5286         if (!check_running($vmid)) {
5287             destroy_vm($storecfg, $vmid, undef, $skiplock);
5288         } else {
5289             die "VM $vmid is running - destroy failed\n";
5290         }
5291     });
5292 }
5293
5294 # pci helpers
5295
5296 sub file_write {
5297     my ($filename, $buf) = @_;
5298
5299     my $fh = IO::File->new($filename, "w");
5300     return undef if !$fh;
5301
5302     my $res = print $fh $buf;
5303
5304     $fh->close();
5305
5306     return $res;
5307 }
5308
5309 sub pci_device_info {
5310     my ($name) = @_;
5311
5312     my $res;
5313
5314     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
5315     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
5316
5317     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
5318     return undef if !defined($irq) || $irq !~ m/^\d+$/;
5319
5320     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
5321     return undef if !defined($vendor) || $vendor !~ s/^0x//;
5322
5323     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
5324     return undef if !defined($product) || $product !~ s/^0x//;
5325
5326     $res = {
5327         name => $name,
5328         vendor => $vendor,
5329         product => $product,
5330         domain => $domain,
5331         bus => $bus,
5332         slot => $slot,
5333         func => $func,
5334         irq => $irq,
5335         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
5336     };
5337
5338     return $res;
5339 }
5340
5341 sub pci_dev_reset {
5342     my ($dev) = @_;
5343
5344     my $name = $dev->{name};
5345
5346     my $fn = "$pcisysfs/devices/$name/reset";
5347
5348     return file_write($fn, "1");
5349 }
5350
5351 sub pci_dev_bind_to_vfio {
5352     my ($dev) = @_;
5353
5354     my $name = $dev->{name};
5355
5356     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5357
5358     if (!-d $vfio_basedir) {
5359         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5360     }
5361     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5362
5363     my $testdir = "$vfio_basedir/$name";
5364     return 1 if -d $testdir;
5365
5366     my $data = "$dev->{vendor} $dev->{product}";
5367     return undef if !file_write("$vfio_basedir/new_id", $data);
5368
5369     my $fn = "$pcisysfs/devices/$name/driver/unbind";
5370     if (!file_write($fn, $name)) {
5371         return undef if -f $fn;
5372     }
5373
5374     $fn = "$vfio_basedir/bind";
5375     if (! -d $testdir) {
5376         return undef if !file_write($fn, $name);
5377     }
5378
5379     return -d $testdir;
5380 }
5381
5382 sub pci_dev_group_bind_to_vfio {
5383     my ($pciid) = @_;
5384
5385     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5386
5387     if (!-d $vfio_basedir) {
5388         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5389     }
5390     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5391
5392     # get IOMMU group devices
5393     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
5394       my @devs = grep /^0000:/, readdir($D);
5395     closedir($D);
5396
5397     foreach my $pciid (@devs) {
5398         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
5399
5400         # pci bridges, switches or root ports are not supported
5401         # they have a pci_bus subdirectory so skip them
5402         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
5403
5404         my $info = pci_device_info($1);
5405         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
5406     }
5407
5408     return 1;
5409 }
5410
5411 # vzdump restore implementaion
5412
5413 sub tar_archive_read_firstfile {
5414     my $archive = shift;
5415
5416     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5417
5418     # try to detect archive type first
5419     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5420         die "unable to open file '$archive'\n";
5421     my $firstfile = <$fh>;
5422     kill 15, $pid;
5423     close $fh;
5424
5425     die "ERROR: archive contaions no data\n" if !$firstfile;
5426     chomp $firstfile;
5427
5428     return $firstfile;
5429 }
5430
5431 sub tar_restore_cleanup {
5432     my ($storecfg, $statfile) = @_;
5433
5434     print STDERR "starting cleanup\n";
5435
5436     if (my $fd = IO::File->new($statfile, "r")) {
5437         while (defined(my $line = <$fd>)) {
5438             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5439                 my $volid = $2;
5440                 eval {
5441                     if ($volid =~ m|^/|) {
5442                         unlink $volid || die 'unlink failed\n';
5443                     } else {
5444                         PVE::Storage::vdisk_free($storecfg, $volid);
5445                     }
5446                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5447                 };
5448                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5449             } else {
5450                 print STDERR "unable to parse line in statfile - $line";
5451             }
5452         }
5453         $fd->close();
5454     }
5455 }
5456
5457 sub restore_archive {
5458     my ($archive, $vmid, $user, $opts) = @_;
5459
5460     my $format = $opts->{format};
5461     my $comp;
5462
5463     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5464         $format = 'tar' if !$format;
5465         $comp = 'gzip';
5466     } elsif ($archive =~ m/\.tar$/) {
5467         $format = 'tar' if !$format;
5468     } elsif ($archive =~ m/.tar.lzo$/) {
5469         $format = 'tar' if !$format;
5470         $comp = 'lzop';
5471     } elsif ($archive =~ m/\.vma$/) {
5472         $format = 'vma' if !$format;
5473     } elsif ($archive =~ m/\.vma\.gz$/) {
5474         $format = 'vma' if !$format;
5475         $comp = 'gzip';
5476     } elsif ($archive =~ m/\.vma\.lzo$/) {
5477         $format = 'vma' if !$format;
5478         $comp = 'lzop';
5479     } else {
5480         $format = 'vma' if !$format; # default
5481     }
5482
5483     # try to detect archive format
5484     if ($format eq 'tar') {
5485         return restore_tar_archive($archive, $vmid, $user, $opts);
5486     } else {
5487         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5488     }
5489 }
5490
5491 sub restore_update_config_line {
5492     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5493
5494     return if $line =~ m/^\#qmdump\#/;
5495     return if $line =~ m/^\#vzdump\#/;
5496     return if $line =~ m/^lock:/;
5497     return if $line =~ m/^unused\d+:/;
5498     return if $line =~ m/^parent:/;
5499     return if $line =~ m/^template:/; # restored VM is never a template
5500
5501     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5502     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5503         # try to convert old 1.X settings
5504         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5505         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5506             my ($model, $macaddr) = split(/\=/, $devconfig);
5507             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5508             my $net = {
5509                 model => $model,
5510                 bridge => "vmbr$ind",
5511                 macaddr => $macaddr,
5512             };
5513             my $netstr = print_net($net);
5514
5515             print $outfd "net$cookie->{netcount}: $netstr\n";
5516             $cookie->{netcount}++;
5517         }
5518     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5519         my ($id, $netstr) = ($1, $2);
5520         my $net = parse_net($netstr);
5521         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5522         $netstr = print_net($net);
5523         print $outfd "$id: $netstr\n";
5524     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5525         my $virtdev = $1;
5526         my $value = $3;
5527         my $di = parse_drive($virtdev, $value);
5528         if (defined($di->{backup}) && !$di->{backup}) {
5529             print $outfd "#$line";
5530         } elsif ($map->{$virtdev}) {
5531             delete $di->{format}; # format can change on restore
5532             $di->{file} = $map->{$virtdev};
5533             $value = print_drive($vmid, $di);
5534             print $outfd "$virtdev: $value\n";
5535         } else {
5536             print $outfd $line;
5537         }
5538     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5539         my ($uuid, $uuid_str);
5540         UUID::generate($uuid);
5541         UUID::unparse($uuid, $uuid_str);
5542         my $smbios1 = parse_smbios1($2);
5543         $smbios1->{uuid} = $uuid_str;
5544         print $outfd $1.print_smbios1($smbios1)."\n";
5545     } else {
5546         print $outfd $line;
5547     }
5548 }
5549
5550 sub scan_volids {
5551     my ($cfg, $vmid) = @_;
5552
5553     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5554
5555     my $volid_hash = {};
5556     foreach my $storeid (keys %$info) {
5557         foreach my $item (@{$info->{$storeid}}) {
5558             next if !($item->{volid} && $item->{size});
5559             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5560             $volid_hash->{$item->{volid}} = $item;
5561         }
5562     }
5563
5564     return $volid_hash;
5565 }
5566
5567 sub is_volume_in_use {
5568     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5569
5570     my $path = PVE::Storage::path($storecfg, $volid);
5571
5572     my $scan_config = sub {
5573         my ($cref, $snapname) = @_;
5574
5575         foreach my $key (keys %$cref) {
5576             my $value = $cref->{$key};
5577             if (is_valid_drivename($key)) {
5578                 next if $skip_drive && $key eq $skip_drive;
5579                 my $drive = parse_drive($key, $value);
5580                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5581                 return 1 if $volid eq $drive->{file};
5582                 if ($drive->{file} =~ m!^/!) {
5583                     return 1 if $drive->{file} eq $path;
5584                 } else {
5585                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5586                     next if !$storeid;
5587                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5588                     next if !$scfg;
5589                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5590                 }
5591             }
5592         }
5593
5594         return 0;
5595     };
5596
5597     return 1 if &$scan_config($conf);
5598
5599     undef $skip_drive;
5600
5601     foreach my $snapname (keys %{$conf->{snapshots}}) {
5602         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5603     }
5604
5605     return 0;
5606 }
5607
5608 sub update_disksize {
5609     my ($vmid, $conf, $volid_hash) = @_;
5610
5611     my $changes;
5612
5613     # used and unused disks
5614     my $referenced = {};
5615
5616     # Note: it is allowed to define multiple storages with same path (alias), so
5617     # we need to check both 'volid' and real 'path' (two different volid can point
5618     # to the same path).
5619
5620     my $referencedpath = {};
5621
5622     # update size info
5623     foreach my $opt (keys %$conf) {
5624         if (is_valid_drivename($opt)) {
5625             my $drive = parse_drive($opt, $conf->{$opt});
5626             my $volid = $drive->{file};
5627             next if !$volid;
5628
5629             $referenced->{$volid} = 1;
5630             if ($volid_hash->{$volid} &&
5631                 (my $path = $volid_hash->{$volid}->{path})) {
5632                 $referencedpath->{$path} = 1;
5633             }
5634
5635             next if drive_is_cdrom($drive);
5636             next if !$volid_hash->{$volid};
5637
5638             $drive->{size} = $volid_hash->{$volid}->{size};
5639             my $new = print_drive($vmid, $drive);
5640             if ($new ne $conf->{$opt}) {
5641                 $changes = 1;
5642                 $conf->{$opt} = $new;
5643             }
5644         }
5645     }
5646
5647     # remove 'unusedX' entry if volume is used
5648     foreach my $opt (keys %$conf) {
5649         next if $opt !~ m/^unused\d+$/;
5650         my $volid = $conf->{$opt};
5651         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5652         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
5653             $changes = 1;
5654             delete $conf->{$opt};
5655         }
5656
5657         $referenced->{$volid} = 1;
5658         $referencedpath->{$path} = 1 if $path;
5659     }
5660
5661     foreach my $volid (sort keys %$volid_hash) {
5662         next if $volid =~ m/vm-$vmid-state-/;
5663         next if $referenced->{$volid};
5664         my $path = $volid_hash->{$volid}->{path};
5665         next if !$path; # just to be sure
5666         next if $referencedpath->{$path};
5667         $changes = 1;
5668         PVE::QemuConfig->add_unused_volume($conf, $volid);
5669         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
5670     }
5671
5672     return $changes;
5673 }
5674
5675 sub rescan {
5676     my ($vmid, $nolock) = @_;
5677
5678     my $cfg = PVE::Storage::config();
5679
5680     my $volid_hash = scan_volids($cfg, $vmid);
5681
5682     my $updatefn =  sub {
5683         my ($vmid) = @_;
5684
5685         my $conf = PVE::QemuConfig->load_config($vmid);
5686
5687         PVE::QemuConfig->check_lock($conf);
5688
5689         my $vm_volids = {};
5690         foreach my $volid (keys %$volid_hash) {
5691             my $info = $volid_hash->{$volid};
5692             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5693         }
5694
5695         my $changes = update_disksize($vmid, $conf, $vm_volids);
5696
5697         PVE::QemuConfig->write_config($vmid, $conf) if $changes;
5698     };
5699
5700     if (defined($vmid)) {
5701         if ($nolock) {
5702             &$updatefn($vmid);
5703         } else {
5704             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5705         }
5706     } else {
5707         my $vmlist = config_list();
5708         foreach my $vmid (keys %$vmlist) {
5709             if ($nolock) {
5710                 &$updatefn($vmid);
5711             } else {
5712                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5713             }
5714         }
5715     }
5716 }
5717
5718 sub restore_vma_archive {
5719     my ($archive, $vmid, $user, $opts, $comp) = @_;
5720
5721     my $input = $archive eq '-' ? "<&STDIN" : undef;
5722     my $readfrom = $archive;
5723
5724     my $uncomp = '';
5725     if ($comp) {
5726         $readfrom = '-';
5727         my $qarchive = PVE::Tools::shellquote($archive);
5728         if ($comp eq 'gzip') {
5729             $uncomp = "zcat $qarchive|";
5730         } elsif ($comp eq 'lzop') {
5731             $uncomp = "lzop -d -c $qarchive|";
5732         } else {
5733             die "unknown compression method '$comp'\n";
5734         }
5735
5736     }
5737
5738     my $tmpdir = "/var/tmp/vzdumptmp$$";
5739     rmtree $tmpdir;
5740
5741     # disable interrupts (always do cleanups)
5742     local $SIG{INT} =
5743         local $SIG{TERM} =
5744         local $SIG{QUIT} =
5745         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
5746
5747     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5748     POSIX::mkfifo($mapfifo, 0600);
5749     my $fifofh;
5750
5751     my $openfifo = sub {
5752         open($fifofh, '>', $mapfifo) || die $!;
5753     };
5754
5755     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5756
5757     my $oldtimeout;
5758     my $timeout = 5;
5759
5760     my $devinfo = {};
5761
5762     my $rpcenv = PVE::RPCEnvironment::get();
5763
5764     my $conffile = PVE::QemuConfig->config_file($vmid);
5765     my $tmpfn = "$conffile.$$.tmp";
5766
5767     # Note: $oldconf is undef if VM does not exists
5768     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
5769     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
5770
5771     my $print_devmap = sub {
5772         my $virtdev_hash = {};
5773
5774         my $cfgfn = "$tmpdir/qemu-server.conf";
5775
5776         # we can read the config - that is already extracted
5777         my $fh = IO::File->new($cfgfn, "r") ||
5778             "unable to read qemu-server.conf - $!\n";
5779
5780         my $fwcfgfn = "$tmpdir/qemu-server.fw";
5781         if (-f $fwcfgfn) {
5782             my $pve_firewall_dir = '/etc/pve/firewall';
5783             mkdir $pve_firewall_dir; # make sure the dir exists
5784             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
5785         }
5786
5787         while (defined(my $line = <$fh>)) {
5788             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5789                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5790                 die "archive does not contain data for drive '$virtdev'\n"
5791                     if !$devinfo->{$devname};
5792                 if (defined($opts->{storage})) {
5793                     $storeid = $opts->{storage} || 'local';
5794                 } elsif (!$storeid) {
5795                     $storeid = 'local';
5796                 }
5797                 $format = 'raw' if !$format;
5798                 $devinfo->{$devname}->{devname} = $devname;
5799                 $devinfo->{$devname}->{virtdev} = $virtdev;
5800                 $devinfo->{$devname}->{format} = $format;
5801                 $devinfo->{$devname}->{storeid} = $storeid;
5802
5803                 # check permission on storage
5804                 my $pool = $opts->{pool}; # todo: do we need that?
5805                 if ($user ne 'root@pam') {
5806                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5807                 }
5808
5809                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5810             }
5811         }
5812
5813         foreach my $devname (keys %$devinfo) {
5814             die "found no device mapping information for device '$devname'\n"
5815                 if !$devinfo->{$devname}->{virtdev};
5816         }
5817
5818         my $cfg = PVE::Storage::config();
5819
5820         # create empty/temp config
5821         if ($oldconf) {
5822             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5823             foreach_drive($oldconf, sub {
5824                 my ($ds, $drive) = @_;
5825
5826                 return if drive_is_cdrom($drive);
5827
5828                 my $volid = $drive->{file};
5829
5830                 return if !$volid || $volid =~ m|^/|;
5831
5832                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5833                 return if !$path || !$owner || ($owner != $vmid);
5834
5835                 # Note: only delete disk we want to restore
5836                 # other volumes will become unused
5837                 if ($virtdev_hash->{$ds}) {
5838                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
5839                     if (my $err = $@) {
5840                         warn $err;
5841                     }
5842                 }
5843             });
5844
5845             # delete vmstate files
5846             # since after the restore we have no snapshots anymore
5847             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5848                 my $snap = $oldconf->{snapshots}->{$snapname};
5849                 if ($snap->{vmstate}) {
5850                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
5851                     if (my $err = $@) {
5852                         warn $err;
5853                     }
5854                 }
5855             }
5856         }
5857
5858         my $map = {};
5859         foreach my $virtdev (sort keys %$virtdev_hash) {
5860             my $d = $virtdev_hash->{$virtdev};
5861             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5862             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5863
5864             # test if requested format is supported
5865             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5866             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5867             $d->{format} = $defFormat if !$supported;
5868
5869             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5870                                                   $d->{format}, undef, $alloc_size);
5871             print STDERR "new volume ID is '$volid'\n";
5872             $d->{volid} = $volid;
5873             my $path = PVE::Storage::path($cfg, $volid);
5874
5875             PVE::Storage::activate_volumes($cfg,[$volid]);
5876
5877             my $write_zeros = 1;
5878             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
5879                 $write_zeros = 0;
5880             }
5881
5882             print $fifofh "format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
5883
5884             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5885             $map->{$virtdev} = $volid;
5886         }
5887
5888         $fh->seek(0, 0) || die "seek failed - $!\n";
5889
5890         my $outfd = new IO::File ($tmpfn, "w") ||
5891             die "unable to write config for VM $vmid\n";
5892
5893         my $cookie = { netcount => 0 };
5894         while (defined(my $line = <$fh>)) {
5895             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5896         }
5897
5898         $fh->close();
5899         $outfd->close();
5900     };
5901
5902     eval {
5903         # enable interrupts
5904         local $SIG{INT} =
5905             local $SIG{TERM} =
5906             local $SIG{QUIT} =
5907             local $SIG{HUP} =
5908             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
5909         local $SIG{ALRM} = sub { die "got timeout\n"; };
5910
5911         $oldtimeout = alarm($timeout);
5912
5913         my $parser = sub {
5914             my $line = shift;
5915
5916             print "$line\n";
5917
5918             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5919                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5920                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5921             } elsif ($line =~ m/^CTIME: /) {
5922                 # we correctly received the vma config, so we can disable
5923                 # the timeout now for disk allocation (set to 10 minutes, so
5924                 # that we always timeout if something goes wrong)
5925                 alarm(600);
5926                 &$print_devmap();
5927                 print $fifofh "done\n";
5928                 my $tmp = $oldtimeout || 0;
5929                 $oldtimeout = undef;
5930                 alarm($tmp);
5931                 close($fifofh);
5932             }
5933         };
5934
5935         print "restore vma archive: $cmd\n";
5936         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5937     };
5938     my $err = $@;
5939
5940     alarm($oldtimeout) if $oldtimeout;
5941
5942     my $vollist = [];
5943     foreach my $devname (keys %$devinfo) {
5944         my $volid = $devinfo->{$devname}->{volid};
5945         push @$vollist, $volid if $volid;
5946     }
5947
5948     my $cfg = PVE::Storage::config();
5949     PVE::Storage::deactivate_volumes($cfg, $vollist);
5950
5951     unlink $mapfifo;
5952
5953     if ($err) {
5954         rmtree $tmpdir;
5955         unlink $tmpfn;
5956
5957         foreach my $devname (keys %$devinfo) {
5958             my $volid = $devinfo->{$devname}->{volid};
5959             next if !$volid;
5960             eval {
5961                 if ($volid =~ m|^/|) {
5962                     unlink $volid || die 'unlink failed\n';
5963                 } else {
5964                     PVE::Storage::vdisk_free($cfg, $volid);
5965                 }
5966                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5967             };
5968             print STDERR "unable to cleanup '$volid' - $@" if $@;
5969         }
5970         die $err;
5971     }
5972
5973     rmtree $tmpdir;
5974
5975     rename($tmpfn, $conffile) ||
5976         die "unable to commit configuration file '$conffile'\n";
5977
5978     PVE::Cluster::cfs_update(); # make sure we read new file
5979
5980     eval { rescan($vmid, 1); };
5981     warn $@ if $@;
5982 }
5983
5984 sub restore_tar_archive {
5985     my ($archive, $vmid, $user, $opts) = @_;
5986
5987     if ($archive ne '-') {
5988         my $firstfile = tar_archive_read_firstfile($archive);
5989         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5990             if $firstfile ne 'qemu-server.conf';
5991     }
5992
5993     my $storecfg = PVE::Storage::config();
5994
5995     # destroy existing data - keep empty config
5996     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
5997     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5998
5999     my $tocmd = "/usr/lib/qemu-server/qmextract";
6000
6001     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6002     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6003     $tocmd .= ' --prealloc' if $opts->{prealloc};
6004     $tocmd .= ' --info' if $opts->{info};
6005
6006     # tar option "xf" does not autodetect compression when read from STDIN,
6007     # so we pipe to zcat
6008     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6009         PVE::Tools::shellquote("--to-command=$tocmd");
6010
6011     my $tmpdir = "/var/tmp/vzdumptmp$$";
6012     mkpath $tmpdir;
6013
6014     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6015     local $ENV{VZDUMP_VMID} = $vmid;
6016     local $ENV{VZDUMP_USER} = $user;
6017
6018     my $conffile = PVE::QemuConfig->config_file($vmid);
6019     my $tmpfn = "$conffile.$$.tmp";
6020
6021     # disable interrupts (always do cleanups)
6022     local $SIG{INT} =
6023         local $SIG{TERM} =
6024         local $SIG{QUIT} =
6025         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6026
6027     eval {
6028         # enable interrupts
6029         local $SIG{INT} =
6030             local $SIG{TERM} =
6031             local $SIG{QUIT} =
6032             local $SIG{HUP} =
6033             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6034
6035         if ($archive eq '-') {
6036             print "extracting archive from STDIN\n";
6037             run_command($cmd, input => "<&STDIN");
6038         } else {
6039             print "extracting archive '$archive'\n";
6040             run_command($cmd);
6041         }
6042
6043         return if $opts->{info};
6044
6045         # read new mapping
6046         my $map = {};
6047         my $statfile = "$tmpdir/qmrestore.stat";
6048         if (my $fd = IO::File->new($statfile, "r")) {
6049             while (defined (my $line = <$fd>)) {
6050                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6051                     $map->{$1} = $2 if $1;
6052                 } else {
6053                     print STDERR "unable to parse line in statfile - $line\n";
6054                 }
6055             }
6056             $fd->close();
6057         }
6058
6059         my $confsrc = "$tmpdir/qemu-server.conf";
6060
6061         my $srcfd = new IO::File($confsrc, "r") ||
6062             die "unable to open file '$confsrc'\n";
6063
6064         my $outfd = new IO::File ($tmpfn, "w") ||
6065             die "unable to write config for VM $vmid\n";
6066
6067         my $cookie = { netcount => 0 };
6068         while (defined (my $line = <$srcfd>)) {
6069             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6070         }
6071
6072         $srcfd->close();
6073         $outfd->close();
6074     };
6075     my $err = $@;
6076
6077     if ($err) {
6078
6079         unlink $tmpfn;
6080
6081         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6082
6083         die $err;
6084     }
6085
6086     rmtree $tmpdir;
6087
6088     rename $tmpfn, $conffile ||
6089         die "unable to commit configuration file '$conffile'\n";
6090
6091     PVE::Cluster::cfs_update(); # make sure we read new file
6092
6093     eval { rescan($vmid, 1); };
6094     warn $@ if $@;
6095 };
6096
6097 sub foreach_storage_used_by_vm {
6098     my ($conf, $func) = @_;
6099
6100     my $sidhash = {};
6101
6102     foreach_drive($conf, sub {
6103         my ($ds, $drive) = @_;
6104         return if drive_is_cdrom($drive);
6105
6106         my $volid = $drive->{file};
6107
6108         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6109         $sidhash->{$sid} = $sid if $sid;
6110     });
6111
6112     foreach my $sid (sort keys %$sidhash) {
6113         &$func($sid);
6114     }
6115 }
6116
6117 sub do_snapshots_with_qemu {
6118     my ($storecfg, $volid) = @_;
6119
6120     my $storage_name = PVE::Storage::parse_volume_id($volid);
6121
6122     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
6123         && !$storecfg->{ids}->{$storage_name}->{krbd}){
6124         return 1;
6125     }
6126
6127     if ($volid =~ m/\.(qcow2|qed)$/){
6128         return 1;
6129     }
6130
6131     return undef;
6132 }
6133
6134 sub qga_check_running {
6135     my ($vmid) = @_;
6136
6137     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
6138     if ($@) {
6139         warn "Qemu Guest Agent is not running - $@";
6140         return 0;
6141     }
6142     return 1;
6143 }
6144
6145 sub template_create {
6146     my ($vmid, $conf, $disk) = @_;
6147
6148     my $storecfg = PVE::Storage::config();
6149
6150     foreach_drive($conf, sub {
6151         my ($ds, $drive) = @_;
6152
6153         return if drive_is_cdrom($drive);
6154         return if $disk && $ds ne $disk;
6155
6156         my $volid = $drive->{file};
6157         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6158
6159         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6160         $drive->{file} = $voliddst;
6161         $conf->{$ds} = print_drive($vmid, $drive);
6162         PVE::QemuConfig->write_config($vmid, $conf);
6163     });
6164 }
6165
6166 sub qemu_img_convert {
6167     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6168
6169     my $storecfg = PVE::Storage::config();
6170     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6171     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6172
6173     if ($src_storeid && $dst_storeid) {
6174
6175         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6176
6177         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6178         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6179
6180         my $src_format = qemu_img_format($src_scfg, $src_volname);
6181         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6182
6183         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6184         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6185
6186         my $cmd = [];
6187         push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6188         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
6189         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path;
6190         if ($is_zero_initialized) {
6191             push @$cmd, "zeroinit:$dst_path";
6192         } else {
6193             push @$cmd, $dst_path;
6194         }
6195
6196         my $parser = sub {
6197             my $line = shift;
6198             if($line =~ m/\((\S+)\/100\%\)/){
6199                 my $percent = $1;
6200                 my $transferred = int($size * $percent / 100);
6201                 my $remaining = $size - $transferred;
6202
6203                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6204             }
6205
6206         };
6207
6208         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6209         my $err = $@;
6210         die "copy failed: $err" if $err;
6211     }
6212 }
6213
6214 sub qemu_img_format {
6215     my ($scfg, $volname) = @_;
6216
6217     if ($scfg->{path} && $volname =~ m/\.($QEMU_FORMAT_RE)$/) {
6218         return $1;
6219     } else {
6220         return "raw";
6221     }
6222 }
6223
6224 sub qemu_drive_mirror {
6225     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga) = @_;
6226
6227     $jobs = {} if !$jobs;
6228
6229     my $qemu_target;
6230     my $format;
6231     $jobs->{"drive-$drive"} = {};
6232
6233     if ($dst_volid =~ /^nbd:/) {
6234         $qemu_target = $dst_volid;
6235         $format = "nbd";
6236     } else {
6237         my $storecfg = PVE::Storage::config();
6238         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6239
6240         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6241
6242         $format = qemu_img_format($dst_scfg, $dst_volname);
6243
6244         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6245
6246         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6247     }
6248
6249     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6250     $opts->{format} = $format if $format;
6251
6252     print "drive mirror is starting for drive-$drive\n";
6253
6254     eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); }; #if a job already run for this device,it's throw an error
6255
6256     if (my $err = $@) {
6257         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6258         die "mirroring error: $err";
6259     }
6260
6261     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6262 }
6263
6264 sub qemu_drive_mirror_monitor {
6265     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6266
6267     eval {
6268         my $err_complete = 0;
6269
6270         while (1) {
6271             die "storage migration timed out\n" if $err_complete > 300;
6272
6273             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6274
6275             my $running_mirror_jobs = {};
6276             foreach my $stat (@$stats) {
6277                 next if $stat->{type} ne 'mirror';
6278                 $running_mirror_jobs->{$stat->{device}} = $stat;
6279             }
6280
6281             my $readycounter = 0;
6282
6283             foreach my $job (keys %$jobs) {
6284
6285                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6286                     print "$job : finished\n";
6287                     delete $jobs->{$job};
6288                     next;
6289                 }
6290
6291                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6292
6293                 my $busy = $running_mirror_jobs->{$job}->{busy};
6294                 my $ready = $running_mirror_jobs->{$job}->{ready};
6295                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6296                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6297                     my $remaining = $total - $transferred;
6298                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6299
6300                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6301                 }
6302
6303                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6304             }
6305
6306             last if scalar(keys %$jobs) == 0;
6307
6308             if ($readycounter == scalar(keys %$jobs)) {
6309                 print "all mirroring jobs are ready \n";
6310                 last if $skipcomplete; #do the complete later
6311
6312                 if ($vmiddst && $vmiddst != $vmid) {
6313                     my $agent_running = $qga && qga_check_running($vmid);
6314                     if ($agent_running) {
6315                         print "freeze filesystem\n";
6316                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6317                     } else {
6318                         print "suspend vm\n";
6319                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6320                     }
6321
6322                     # if we clone a disk for a new target vm, we don't switch the disk
6323                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6324
6325                     if ($agent_running) {
6326                         print "unfreeze filesystem\n";
6327                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6328                     } else {
6329                         print "resume vm\n";
6330                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6331                     }
6332
6333                     last;
6334                 } else {
6335
6336                     foreach my $job (keys %$jobs) {
6337                         # try to switch the disk if source and destination are on the same guest
6338                         print "$job: Completing block job...\n";
6339
6340                         eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
6341                         if ($@ =~ m/cannot be completed/) {
6342                             print "$job: Block job cannot be completed, try again.\n";
6343                             $err_complete++;
6344                         }else {
6345                             print "$job: Completed successfully.\n";
6346                             $jobs->{$job}->{complete} = 1;
6347                         }
6348                     }
6349                 }
6350             }
6351             sleep 1;
6352         }
6353     };
6354     my $err = $@;
6355
6356     if ($err) {
6357         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6358         die "mirroring error: $err";
6359     }
6360
6361 }
6362
6363 sub qemu_blockjobs_cancel {
6364     my ($vmid, $jobs) = @_;
6365
6366     foreach my $job (keys %$jobs) {
6367         print "$job: Cancelling block job\n";
6368         eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
6369         $jobs->{$job}->{cancel} = 1;
6370     }
6371
6372     while (1) {
6373         my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6374
6375         my $running_jobs = {};
6376         foreach my $stat (@$stats) {
6377             $running_jobs->{$stat->{device}} = $stat;
6378         }
6379
6380         foreach my $job (keys %$jobs) {
6381
6382             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6383                 print "$job: Done.\n";
6384                 delete $jobs->{$job};
6385             }
6386         }
6387
6388         last if scalar(keys %$jobs) == 0;
6389
6390         sleep 1;
6391     }
6392 }
6393
6394 sub clone_disk {
6395     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6396         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga) = @_;
6397
6398     my $newvolid;
6399
6400     if (!$full) {
6401         print "create linked clone of drive $drivename ($drive->{file})\n";
6402         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6403         push @$newvollist, $newvolid;
6404     } else {
6405
6406         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6407         $storeid = $storage if $storage;
6408
6409         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
6410         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6411
6412         print "create full clone of drive $drivename ($drive->{file})\n";
6413         my $name = undef;
6414         if (drive_is_cloudinit($drive)) {
6415             $name = "vm-$newvmid-cloudinit";
6416             # cloudinit only supports raw and qcow2 atm:
6417             if ($dst_format eq 'qcow2') {
6418                 $name .= '.qcow2';
6419             } elsif ($dst_format ne 'raw') {
6420                 die "clone: unhandled format for cloudinit image\n";
6421             }
6422         }
6423         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
6424         push @$newvollist, $newvolid;
6425
6426         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
6427
6428         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6429         if (!$running || $snapname) {
6430             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6431         } else {
6432
6433             my $kvmver = get_running_qemu_version ($vmid);
6434             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
6435                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6436                     if $drive->{iothread};
6437             }
6438
6439             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga);
6440         }
6441     }
6442
6443     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6444
6445     my $disk = $drive;
6446     $disk->{format} = undef;
6447     $disk->{file} = $newvolid;
6448     $disk->{size} = $size;
6449
6450     return $disk;
6451 }
6452
6453 # this only works if VM is running
6454 sub get_current_qemu_machine {
6455     my ($vmid) = @_;
6456
6457     my $cmd = { execute => 'query-machines', arguments => {} };
6458     my $res = vm_qmp_command($vmid, $cmd);
6459
6460     my ($current, $default);
6461     foreach my $e (@$res) {
6462         $default = $e->{name} if $e->{'is-default'};
6463         $current = $e->{name} if $e->{'is-current'};
6464     }
6465
6466     # fallback to the default machine if current is not supported by qemu
6467     return $current || $default || 'pc';
6468 }
6469
6470 sub get_running_qemu_version {
6471     my ($vmid) = @_;
6472     my $cmd = { execute => 'query-version', arguments => {} };
6473     my $res = vm_qmp_command($vmid, $cmd);
6474     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
6475 }
6476
6477 sub qemu_machine_feature_enabled {
6478     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6479
6480     my $current_major;
6481     my $current_minor;
6482
6483     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6484
6485         $current_major = $3;
6486         $current_minor = $4;
6487
6488     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6489
6490         $current_major = $1;
6491         $current_minor = $2;
6492     }
6493
6494     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
6495
6496
6497 }
6498
6499 sub qemu_machine_pxe {
6500     my ($vmid, $conf, $machine) = @_;
6501
6502     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6503
6504     foreach my $opt (keys %$conf) {
6505         next if $opt !~ m/^net(\d+)$/;
6506         my $net = PVE::QemuServer::parse_net($conf->{$opt});
6507         next if !$net;
6508         my $romfile = PVE::QemuServer::vm_mon_cmd_nocheck($vmid, 'qom-get', path => $opt, property => 'romfile');
6509         return $machine.".pxe" if $romfile =~ m/pxe/;
6510         last;
6511     }
6512
6513     return $machine;
6514 }
6515
6516 sub qemu_use_old_bios_files {
6517     my ($machine_type) = @_;
6518
6519     return if !$machine_type;
6520
6521     my $use_old_bios_files = undef;
6522
6523     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6524         $machine_type = $1;
6525         $use_old_bios_files = 1;
6526     } else {
6527         my $kvmver = kvm_user_version();
6528         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6529         # load new efi bios files on migration. So this hack is required to allow
6530         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6531         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6532         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
6533     }
6534
6535     return ($use_old_bios_files, $machine_type);
6536 }
6537
6538 sub create_efidisk {
6539     my ($storecfg, $storeid, $vmid, $fmt) = @_;
6540
6541     die "EFI vars default image not found\n" if ! -f $OVMF_VARS;
6542
6543     my $vars_size = PVE::Tools::convert_size(-s $OVMF_VARS, 'b' => 'kb');
6544     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
6545     PVE::Storage::activate_volumes($storecfg, [$volid]);
6546
6547     my $path = PVE::Storage::path($storecfg, $volid);
6548     eval {
6549         run_command(['/usr/bin/qemu-img', 'convert', '-n', '-f', 'raw', '-O', $fmt, $OVMF_VARS, $path]);
6550     };
6551     die "Copying EFI vars image failed: $@" if $@;
6552
6553     return ($volid, $vars_size);
6554 }
6555
6556 sub lspci {
6557
6558     my $devices = {};
6559
6560     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6561             my (undef, $id, $function) = @_;
6562             my $res = { id => $id, function => $function};
6563             push @{$devices->{$id}}, $res;
6564     });
6565
6566     # Entries should be sorted by functions.
6567     foreach my $id (keys %$devices) {
6568         my $dev = $devices->{$id};
6569         $devices->{$id} = [ sort { $a->{function} <=> $b->{function} } @$dev ];
6570     }
6571
6572     return $devices;
6573 }
6574
6575 sub vm_iothreads_list {
6576     my ($vmid) = @_;
6577
6578     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6579
6580     my $iothreads = {};
6581     foreach my $iothread (@$res) {
6582         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6583     }
6584
6585     return $iothreads;
6586 }
6587
6588 sub scsihw_infos {
6589     my ($conf, $drive) = @_;
6590
6591     my $maxdev = 0;
6592
6593     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
6594         $maxdev = 7;
6595     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6596         $maxdev = 1;
6597     } else {
6598         $maxdev = 256;
6599     }
6600
6601     my $controller = int($drive->{index} / $maxdev);
6602     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6603
6604     return ($maxdev, $controller, $controller_prefix);
6605 }
6606
6607 sub add_hyperv_enlightenments {
6608     my ($cpuFlags, $winversion, $machine_type, $kvmver, $bios, $gpu_passthrough) = @_;
6609
6610     return if $winversion < 6;
6611     return if $bios && $bios eq 'ovmf' && $winversion < 8;
6612
6613     push @$cpuFlags , 'hv_vendor_id=proxmox' if $gpu_passthrough;
6614
6615     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
6616         push @$cpuFlags , 'hv_spinlocks=0x1fff';
6617         push @$cpuFlags , 'hv_vapic';
6618         push @$cpuFlags , 'hv_time';
6619     } else {
6620         push @$cpuFlags , 'hv_spinlocks=0xffff';
6621     }
6622
6623     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
6624         push @$cpuFlags , 'hv_reset';
6625         push @$cpuFlags , 'hv_vpindex';
6626         push @$cpuFlags , 'hv_runtime';
6627     }
6628
6629     if ($winversion >= 7) {
6630         push @$cpuFlags , 'hv_relaxed';
6631     }
6632 }
6633
6634 sub windows_version {
6635     my ($ostype) = @_;
6636
6637     return 0 if !$ostype;
6638
6639     my $winversion = 0;
6640
6641     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
6642         $winversion = 5;
6643     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
6644         $winversion = 6;
6645     } elsif ($ostype =~ m/^win(\d+)$/) {
6646         $winversion = $1;
6647     }
6648
6649     return $winversion;
6650 }
6651
6652 sub resolve_dst_disk_format {
6653         my ($storecfg, $storeid, $src_volname, $format) = @_;
6654         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6655
6656         if (!$format) {
6657             # if no target format is specified, use the source disk format as hint
6658             if ($src_volname) {
6659                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6660                 $format = qemu_img_format($scfg, $src_volname);
6661             } else {
6662                 return $defFormat;
6663             }
6664         }
6665
6666         # test if requested format is supported - else use default
6667         my $supported = grep { $_ eq $format } @$validFormats;
6668         $format = $defFormat if !$supported;
6669         return $format;
6670 }
6671
6672 sub resolve_first_disk {
6673     my $conf = shift;
6674     my @disks = PVE::QemuServer::valid_drive_names();
6675     my $firstdisk;
6676     foreach my $ds (reverse @disks) {
6677         next if !$conf->{$ds};
6678         my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
6679         next if PVE::QemuServer::drive_is_cdrom($disk);
6680         $firstdisk = $ds;
6681     }
6682     return $firstdisk;
6683 }
6684
6685 sub generate_smbios1_uuid {
6686     my ($uuid, $uuid_str);
6687     UUID::generate($uuid);
6688     UUID::unparse($uuid, $uuid_str);
6689     return "uuid=$uuid_str";
6690 }
6691
6692 # bash completion helper
6693
6694 sub complete_backup_archives {
6695     my ($cmdname, $pname, $cvalue) = @_;
6696
6697     my $cfg = PVE::Storage::config();
6698
6699     my $storeid;
6700
6701     if ($cvalue =~ m/^([^:]+):/) {
6702         $storeid = $1;
6703     }
6704
6705     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6706
6707     my $res = [];
6708     foreach my $id (keys %$data) {
6709         foreach my $item (@{$data->{$id}}) {
6710             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6711             push @$res, $item->{volid} if defined($item->{volid});
6712         }
6713     }
6714
6715     return $res;
6716 }
6717
6718 my $complete_vmid_full = sub {
6719     my ($running) = @_;
6720
6721     my $idlist = vmstatus();
6722
6723     my $res = [];
6724
6725     foreach my $id (keys %$idlist) {
6726         my $d = $idlist->{$id};
6727         if (defined($running)) {
6728             next if $d->{template};
6729             next if $running && $d->{status} ne 'running';
6730             next if !$running && $d->{status} eq 'running';
6731         }
6732         push @$res, $id;
6733
6734     }
6735     return $res;
6736 };
6737
6738 sub complete_vmid {
6739     return &$complete_vmid_full();
6740 }
6741
6742 sub complete_vmid_stopped {
6743     return &$complete_vmid_full(0);
6744 }
6745
6746 sub complete_vmid_running {
6747     return &$complete_vmid_full(1);
6748 }
6749
6750 sub complete_storage {
6751
6752     my $cfg = PVE::Storage::config();
6753     my $ids = $cfg->{ids};
6754
6755     my $res = [];
6756     foreach my $sid (keys %$ids) {
6757         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6758         next if !$ids->{$sid}->{content}->{images};
6759         push @$res, $sid;
6760     }
6761
6762     return $res;
6763 }
6764
6765 1;