PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach $IPV6RE);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QemuConfig;
  31 use PVE::QMPClient;
  32 use PVE::RPCEnvironment;
  33 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
  34 use PVE::QemuServer::Memory;
  35 use PVE::QemuServer::USB qw(parse_usb_device);
  36 use PVE::QemuServer::Cloudinit;
  37 use Time::HiRes qw(gettimeofday);
  38 use File::Copy qw(copy);
  39 use URI::Escape;
  40
  41 my $OVMF_CODE = '/usr/share/kvm/OVMF_CODE-pure-efi.fd';
  42 my $OVMF_VARS = '/usr/share/kvm/OVMF_VARS-pure-efi.fd';
  43
  44 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  45
  46 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  47
  48 my $QEMU_FORMAT_RE = qr/raw|cow|qcow|qcow2|qed|vmdk|cloop/;
  49
  50 # Note about locking: we use flock on the config file protect
  51 # against concurent actions.
  52 # Aditionaly, we have a 'lock' setting in the config file. This
  53 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  54 # allowed when such lock is set. But you can ignore this kind of
  55 # lock with the --skiplock flag.
  56
  57 cfs_register_file('/qemu-server/',
  58                   \&parse_vm_config,
  59                   \&write_vm_config);
  60
  61 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  62     description => "Some command save/restore state from this location.",
  63     type => 'string',
  64     maxLength => 128,
  65     optional => 1,
  66 });
  67
  68 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  69     description => "The name of the snapshot.",
  70     type => 'string', format => 'pve-configid',
  71     maxLength => 40,
  72 });
  73
  74 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  75     type => 'string',
  76     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  77     description => "The drive's backing file's data format.",
  78     optional => 1,
  79 });
  80
  81 #no warnings 'redefine';
  82
  83 sub cgroups_write {
  84    my ($controller, $vmid, $option, $value) = @_;
  85
  86    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  87    PVE::ProcFSTools::write_proc_entry($path, $value);
  88
  89 }
  90
  91 my $nodename = PVE::INotify::nodename();
  92
  93 mkdir "/etc/pve/nodes/$nodename";
  94 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  95 mkdir $confdir;
  96
  97 my $var_run_tmpdir = "/var/run/qemu-server";
  98 mkdir $var_run_tmpdir;
  99
 100 my $lock_dir = "/var/lock/qemu-server";
 101 mkdir $lock_dir;
 102
 103 my $pcisysfs = "/sys/bus/pci";
 104
 105 my $cpu_vendor_list = {
 106     # Intel CPUs
 107     486 => 'GenuineIntel',
 108     pentium => 'GenuineIntel',
 109     pentium2  => 'GenuineIntel',
 110     pentium3  => 'GenuineIntel',
 111     coreduo => 'GenuineIntel',
 112     core2duo => 'GenuineIntel',
 113     Conroe  => 'GenuineIntel',
 114     Penryn  => 'GenuineIntel',
 115     Nehalem  => 'GenuineIntel',
 116     'Nehalem-IBRS'  => 'GenuineIntel',
 117     Westmere => 'GenuineIntel',
 118     'Westmere-IBRS' => 'GenuineIntel',
 119     SandyBridge => 'GenuineIntel',
 120     'SandyBridge-IBRS' => 'GenuineIntel',
 121     IvyBridge => 'GenuineIntel',
 122     'IvyBridge-IBRS' => 'GenuineIntel',
 123     Haswell => 'GenuineIntel',
 124     'Haswell-IBRS' => 'GenuineIntel',
 125     'Haswell-noTSX' => 'GenuineIntel',
 126     'Haswell-noTSX-IBRS' => 'GenuineIntel',
 127     Broadwell => 'GenuineIntel',
 128     'Broadwell-IBRS' => 'GenuineIntel',
 129     'Broadwell-noTSX' => 'GenuineIntel',
 130     'Broadwell-noTSX-IBRS' => 'GenuineIntel',
 131     'Skylake-Client' => 'GenuineIntel',
 132     'Skylake-Client-IBRS' => 'GenuineIntel',
 133     'Skylake-Server' => 'GenuineIntel',
 134     'Skylake-Server-IBRS' => 'GenuineIntel',
 135
 136     # AMD CPUs
 137     athlon => 'AuthenticAMD',
 138     phenom  => 'AuthenticAMD',
 139     Opteron_G1  => 'AuthenticAMD',
 140     Opteron_G2  => 'AuthenticAMD',
 141     Opteron_G3  => 'AuthenticAMD',
 142     Opteron_G4  => 'AuthenticAMD',
 143     Opteron_G5  => 'AuthenticAMD',
 144     EPYC => 'AuthenticAMD',
 145     'EPYC-IBPB' => 'AuthenticAMD',
 146
 147     # generic types, use vendor from host node
 148     host => 'default',
 149     kvm32 => 'default',
 150     kvm64 => 'default',
 151     qemu32 => 'default',
 152     qemu64 => 'default',
 153     max => 'default',
 154 };
 155
 156 my $cpu_flag = qr/[+-](pcid|spec-ctrl)/;
 157
 158 my $cpu_fmt = {
 159     cputype => {
 160         description => "Emulated CPU type.",
 161         type => 'string',
 162         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 163         default => 'kvm64',
 164         default_key => 1,
 165     },
 166     hidden => {
 167         description => "Do not identify as a KVM virtual machine.",
 168         type => 'boolean',
 169         optional => 1,
 170         default => 0
 171     },
 172     flags => {
 173         description => "List of additional CPU flags separated by ';'."
 174                      . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
 175                      . " Currently supported flags: 'pcid', 'spec-ctrl'.",
 176         format_description => '+FLAG[;-FLAG...]',
 177         type => 'string',
 178         pattern => qr/$cpu_flag(;$cpu_flag)*/,
 179         optional => 1,
 180     },
 181 };
 182
 183 my $watchdog_fmt = {
 184     model => {
 185         default_key => 1,
 186         type => 'string',
 187         enum => [qw(i6300esb ib700)],
 188         description => "Watchdog type to emulate.",
 189         default => 'i6300esb',
 190         optional => 1,
 191     },
 192     action => {
 193         type => 'string',
 194         enum => [qw(reset shutdown poweroff pause debug none)],
 195         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 196         optional => 1,
 197     },
 198 };
 199 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 200
 201 my $confdesc = {
 202     onboot => {
 203         optional => 1,
 204         type => 'boolean',
 205         description => "Specifies whether a VM will be started during system bootup.",
 206         default => 0,
 207     },
 208     autostart => {
 209         optional => 1,
 210         type => 'boolean',
 211         description => "Automatic restart after crash (currently ignored).",
 212         default => 0,
 213     },
 214     hotplug => {
 215         optional => 1,
 216         type => 'string', format => 'pve-hotplug-features',
 217         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 218         default => 'network,disk,usb',
 219     },
 220     reboot => {
 221         optional => 1,
 222         type => 'boolean',
 223         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 224         default => 1,
 225     },
 226     lock => {
 227         optional => 1,
 228         type => 'string',
 229         description => "Lock/unlock the VM.",
 230         enum => [qw(migrate backup snapshot rollback)],
 231     },
 232     cpulimit => {
 233         optional => 1,
 234         type => 'number',
 235         description => "Limit of CPU usage.",
 236         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 237         minimum => 0,
 238         maximum => 128,
 239         default => 0,
 240     },
 241     cpuunits => {
 242         optional => 1,
 243         type => 'integer',
 244         description => "CPU weight for a VM.",
 245         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 246         minimum => 2,
 247         maximum => 262144,
 248         default => 1024,
 249     },
 250     memory => {
 251         optional => 1,
 252         type => 'integer',
 253         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 254         minimum => 16,
 255         default => 512,
 256     },
 257     balloon => {
 258         optional => 1,
 259         type => 'integer',
 260         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 261         minimum => 0,
 262     },
 263     shares => {
 264         optional => 1,
 265         type => 'integer',
 266         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 267         minimum => 0,
 268         maximum => 50000,
 269         default => 1000,
 270     },
 271     keyboard => {
 272         optional => 1,
 273         type => 'string',
 274         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.conf' configuration file.".
 275                        "It should not be necessary to set it.",
 276         enum => PVE::Tools::kvmkeymaplist(),
 277         default => undef,
 278     },
 279     name => {
 280         optional => 1,
 281         type => 'string', format => 'dns-name',
 282         description => "Set a name for the VM. Only used on the configuration web interface.",
 283     },
 284     scsihw => {
 285         optional => 1,
 286         type => 'string',
 287         description => "SCSI controller model",
 288         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 289         default => 'lsi',
 290     },
 291     description => {
 292         optional => 1,
 293         type => 'string',
 294         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 295     },
 296     ostype => {
 297         optional => 1,
 298         type => 'string',
 299         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 300         description => "Specify guest operating system.",
 301         verbose_description => <<EODESC,
 302 Specify guest operating system. This is used to enable special
 303 optimization/features for specific operating systems:
 304
 305 [horizontal]
 306 other;; unspecified OS
 307 wxp;; Microsoft Windows XP
 308 w2k;; Microsoft Windows 2000
 309 w2k3;; Microsoft Windows 2003
 310 w2k8;; Microsoft Windows 2008
 311 wvista;; Microsoft Windows Vista
 312 win7;; Microsoft Windows 7
 313 win8;; Microsoft Windows 8/2012/2012r2
 314 win10;; Microsoft Windows 10/2016
 315 l24;; Linux 2.4 Kernel
 316 l26;; Linux 2.6/3.X Kernel
 317 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 318 EODESC
 319     },
 320     boot => {
 321         optional => 1,
 322         type => 'string',
 323         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 324         pattern => '[acdn]{1,4}',
 325         default => 'cdn',
 326     },
 327     bootdisk => {
 328         optional => 1,
 329         type => 'string', format => 'pve-qm-bootdisk',
 330         description => "Enable booting from specified disk.",
 331         pattern => '(ide|sata|scsi|virtio)\d+',
 332     },
 333     smp => {
 334         optional => 1,
 335         type => 'integer',
 336         description => "The number of CPUs. Please use option -sockets instead.",
 337         minimum => 1,
 338         default => 1,
 339     },
 340     sockets => {
 341         optional => 1,
 342         type => 'integer',
 343         description => "The number of CPU sockets.",
 344         minimum => 1,
 345         default => 1,
 346     },
 347     cores => {
 348         optional => 1,
 349         type => 'integer',
 350         description => "The number of cores per socket.",
 351         minimum => 1,
 352         default => 1,
 353     },
 354     numa => {
 355         optional => 1,
 356         type => 'boolean',
 357         description => "Enable/disable NUMA.",
 358         default => 0,
 359     },
 360     hugepages => {
 361         optional => 1,
 362         type => 'string',
 363         description => "Enable/disable hugepages memory.",
 364         enum => [qw(any 2 1024)],
 365     },
 366     vcpus => {
 367         optional => 1,
 368         type => 'integer',
 369         description => "Number of hotplugged vcpus.",
 370         minimum => 1,
 371         default => 0,
 372     },
 373     acpi => {
 374         optional => 1,
 375         type => 'boolean',
 376         description => "Enable/disable ACPI.",
 377         default => 1,
 378     },
 379     agent => {
 380         optional => 1,
 381         type => 'boolean',
 382         description => "Enable/disable Qemu GuestAgent.",
 383         default => 0,
 384     },
 385     kvm => {
 386         optional => 1,
 387         type => 'boolean',
 388         description => "Enable/disable KVM hardware virtualization.",
 389         default => 1,
 390     },
 391     tdf => {
 392         optional => 1,
 393         type => 'boolean',
 394         description => "Enable/disable time drift fix.",
 395         default => 0,
 396     },
 397     localtime => {
 398         optional => 1,
 399         type => 'boolean',
 400         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 401     },
 402     freeze => {
 403         optional => 1,
 404         type => 'boolean',
 405         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 406     },
 407     vga => {
 408         optional => 1,
 409         type => 'string',
 410         description => "Select the VGA type.",
 411         verbose_description => "Select the VGA type. If you want to use high resolution" .
 412             " modes (>= 1280x1024x16) then you should use the options " .
 413             "'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and " .
 414             "'cirrus' for other OS types. The 'qxl' option enables the SPICE " .
 415             "display sever. For win* OS you can select how many independent " .
 416             "displays you want, Linux guests can add displays them self. " .
 417             "You can also run without any graphic card, using a serial device" .
 418             " as terminal.",
 419         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 420     },
 421     watchdog => {
 422         optional => 1,
 423         type => 'string', format => 'pve-qm-watchdog',
 424         description => "Create a virtual hardware watchdog device.",
 425         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 426             " (by a guest action), the watchdog must be periodically polled " .
 427             "by an agent inside the guest or else the watchdog will reset " .
 428             "the guest (or execute the respective action specified)",
 429     },
 430     startdate => {
 431         optional => 1,
 432         type => 'string',
 433         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 434         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 435         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 436         default => 'now',
 437     },
 438     startup =>  get_standard_option('pve-startup-order'),
 439     template => {
 440         optional => 1,
 441         type => 'boolean',
 442         description => "Enable/disable Template.",
 443         default => 0,
 444     },
 445     args => {
 446         optional => 1,
 447         type => 'string',
 448         description => "Arbitrary arguments passed to kvm.",
 449         verbose_description => <<EODESCR,
 450 Arbitrary arguments passed to kvm, for example:
 451
 452 args: -no-reboot -no-hpet
 453
 454 NOTE: this option is for experts only.
 455 EODESCR
 456     },
 457     tablet => {
 458         optional => 1,
 459         type => 'boolean',
 460         default => 1,
 461         description => "Enable/disable the USB tablet device.",
 462         verbose_description => "Enable/disable the USB tablet device. This device is " .
 463             "usually needed to allow absolute mouse positioning with VNC. " .
 464             "Else the mouse runs out of sync with normal VNC clients. " .
 465             "If you're running lots of console-only guests on one host, " .
 466             "you may consider disabling this to save some context switches. " .
 467             "This is turned off by default if you use spice (-vga=qxl).",
 468     },
 469     migrate_speed => {
 470         optional => 1,
 471         type => 'integer',
 472         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 473         minimum => 0,
 474         default => 0,
 475     },
 476     migrate_downtime => {
 477         optional => 1,
 478         type => 'number',
 479         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 480         minimum => 0,
 481         default => 0.1,
 482     },
 483     cdrom => {
 484         optional => 1,
 485         type => 'string', format => 'pve-qm-ide',
 486         typetext => '<volume>',
 487         description => "This is an alias for option -ide2",
 488     },
 489     cpu => {
 490         optional => 1,
 491         description => "Emulated CPU type.",
 492         type => 'string',
 493         format => $cpu_fmt,
 494     },
 495     parent => get_standard_option('pve-snapshot-name', {
 496         optional => 1,
 497         description => "Parent snapshot name. This is used internally, and should not be modified.",
 498     }),
 499     snaptime => {
 500         optional => 1,
 501         description => "Timestamp for snapshots.",
 502         type => 'integer',
 503         minimum => 0,
 504     },
 505     vmstate => {
 506         optional => 1,
 507         type => 'string', format => 'pve-volume-id',
 508         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 509     },
 510     vmstatestorage => get_standard_option('pve-storage-id', {
 511         description => "Default storage for VM state volumes/files.",
 512         optional => 1,
 513     }),
 514     machine => {
 515         description => "Specific the Qemu machine type.",
 516         type => 'string',
 517         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
 518         maxLength => 40,
 519         optional => 1,
 520     },
 521     smbios1 => {
 522         description => "Specify SMBIOS type 1 fields.",
 523         type => 'string', format => 'pve-qm-smbios1',
 524         maxLength => 256,
 525         optional => 1,
 526     },
 527     protection => {
 528         optional => 1,
 529         type => 'boolean',
 530         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 531         default => 0,
 532     },
 533     bios => {
 534         optional => 1,
 535         type => 'string',
 536         enum => [ qw(seabios ovmf) ],
 537         description => "Select BIOS implementation.",
 538         default => 'seabios',
 539     },
 540     searchdomain => {
 541         optional => 1,
 542         type => 'string',
 543         description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 544     },
 545     nameserver => {
 546         optional => 1,
 547         type => 'string', format => 'address-list',
 548         description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 549     },
 550     sshkeys => {
 551         optional => 1,
 552         type => 'string',
 553         format => 'urlencoded',
 554         description => "cloud-init : Setup public SSH keys (one key per line, " .
 555                         "OpenSSH format).",
 556     },
 557     hostname => {
 558         optional => 1,
 559         description => "cloud-init: Hostname to use instead of the vm-name + search-domain.",
 560         type => 'string', format => 'dns-name',
 561         maxLength => 255,
 562     },
 563 };
 564
 565 # what about other qemu settings ?
 566 #cpu => 'string',
 567 #machine => 'string',
 568 #fda => 'file',
 569 #fdb => 'file',
 570 #mtdblock => 'file',
 571 #sd => 'file',
 572 #pflash => 'file',
 573 #snapshot => 'bool',
 574 #bootp => 'file',
 575 ##tftp => 'dir',
 576 ##smb => 'dir',
 577 #kernel => 'file',
 578 #append => 'string',
 579 #initrd => 'file',
 580 ##soundhw => 'string',
 581
 582 while (my ($k, $v) = each %$confdesc) {
 583     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 584 }
 585
 586 my $MAX_IDE_DISKS = 4;
 587 my $MAX_SCSI_DISKS = 14;
 588 my $MAX_VIRTIO_DISKS = 16;
 589 my $MAX_SATA_DISKS = 6;
 590 my $MAX_USB_DEVICES = 5;
 591 my $MAX_NETS = 32;
 592 my $MAX_UNUSED_DISKS = 8;
 593 my $MAX_HOSTPCI_DEVICES = 4;
 594 my $MAX_SERIAL_PORTS = 4;
 595 my $MAX_PARALLEL_PORTS = 3;
 596 my $MAX_NUMA = 8;
 597
 598 my $numa_fmt = {
 599     cpus => {
 600         type => "string",
 601         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 602         description => "CPUs accessing this NUMA node.",
 603         format_description => "id[-id];...",
 604     },
 605     memory => {
 606         type => "number",
 607         description => "Amount of memory this NUMA node provides.",
 608         optional => 1,
 609     },
 610     hostnodes => {
 611         type => "string",
 612         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 613         description => "Host NUMA nodes to use.",
 614         format_description => "id[-id];...",
 615         optional => 1,
 616     },
 617     policy => {
 618         type => 'string',
 619         enum => [qw(preferred bind interleave)],
 620         description => "NUMA allocation policy.",
 621         optional => 1,
 622     },
 623 };
 624 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 625 my $numadesc = {
 626     optional => 1,
 627     type => 'string', format => $numa_fmt,
 628     description => "NUMA topology.",
 629 };
 630 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 631
 632 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 633     $confdesc->{"numa$i"} = $numadesc;
 634 }
 635
 636 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 637                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 638                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 639 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 640
 641 my $net_fmt_bridge_descr = <<__EOD__;
 642 Bridge to attach the network device to. The Proxmox VE standard bridge
 643 is called 'vmbr0'.
 644
 645 If you do not specify a bridge, we create a kvm user (NATed) network
 646 device, which provides DHCP and DNS services. The following addresses
 647 are used:
 648
 649  10.0.2.2   Gateway
 650  10.0.2.3   DNS Server
 651  10.0.2.4   SMB Server
 652
 653 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 654 __EOD__
 655
 656 my $net_fmt = {
 657     macaddr => {
 658         type => 'string',
 659         pattern => qr/[0-9a-f]{2}(?::[0-9a-f]{2}){5}/i,
 660         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 661         format_description => "XX:XX:XX:XX:XX:XX",
 662         optional => 1,
 663     },
 664     model => {
 665         type => 'string',
 666         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 667         enum => $nic_model_list,
 668         default_key => 1,
 669     },
 670     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 671     bridge => {
 672         type => 'string',
 673         description => $net_fmt_bridge_descr,
 674         format_description => 'bridge',
 675         optional => 1,
 676     },
 677     queues => {
 678         type => 'integer',
 679         minimum => 0, maximum => 16,
 680         description => 'Number of packet queues to be used on the device.',
 681         optional => 1,
 682     },
 683     rate => {
 684         type => 'number',
 685         minimum => 0,
 686         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 687         optional => 1,
 688     },
 689     tag => {
 690         type => 'integer',
 691         minimum => 1, maximum => 4094,
 692         description => 'VLAN tag to apply to packets on this interface.',
 693         optional => 1,
 694     },
 695     trunks => {
 696         type => 'string',
 697         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 698         description => 'VLAN trunks to pass through this interface.',
 699         format_description => 'vlanid[;vlanid...]',
 700         optional => 1,
 701     },
 702     firewall => {
 703         type => 'boolean',
 704         description => 'Whether this interface should be protected by the firewall.',
 705         optional => 1,
 706     },
 707     link_down => {
 708         type => 'boolean',
 709         description => 'Whether this interface should be disconnected (like pulling the plug).',
 710         optional => 1,
 711     },
 712 };
 713
 714 my $netdesc = {
 715     optional => 1,
 716     type => 'string', format => $net_fmt,
 717     description => "Specify network devices.",
 718 };
 719
 720 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 721
 722 my $ipconfig_fmt = {
 723     ip => {
 724         type => 'string',
 725         format => 'pve-ipv4-config',
 726         format_description => 'IPv4Format/CIDR',
 727         description => 'IPv4 address in CIDR format.',
 728         optional => 1,
 729         default => 'dhcp',
 730     },
 731     gw => {
 732         type => 'string',
 733         format => 'ipv4',
 734         format_description => 'GatewayIPv4',
 735         description => 'Default gateway for IPv4 traffic.',
 736         optional => 1,
 737         requires => 'ip',
 738     },
 739     ip6 => {
 740         type => 'string',
 741         format => 'pve-ipv6-config',
 742         format_description => 'IPv6Format/CIDR',
 743         description => 'IPv6 address in CIDR format.',
 744         optional => 1,
 745         default => 'dhcp',
 746     },
 747     gw6 => {
 748         type => 'string',
 749         format => 'ipv6',
 750         format_description => 'GatewayIPv6',
 751         description => 'Default gateway for IPv6 traffic.',
 752         optional => 1,
 753         requires => 'ip6',
 754     },
 755 };
 756 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 757 my $ipconfigdesc = {
 758     optional => 1,
 759     type => 'string', format => 'pve-qm-ipconfig',
 760     description => <<'EODESCR',
 761 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 762
 763 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 764
 765 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
 766 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 767
 768 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
 769 EODESCR
 770 };
 771 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 772
 773 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 774     $confdesc->{"net$i"} = $netdesc;
 775     $confdesc->{"ipconfig$i"} = $ipconfigdesc;
 776 }
 777
 778 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 779 sub verify_volume_id_or_qm_path {
 780     my ($volid, $noerr) = @_;
 781
 782     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 783         return $volid;
 784     }
 785
 786     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 787     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 788     if ($@) {
 789         return undef if $noerr;
 790         die $@;
 791     }
 792     return $volid;
 793 }
 794
 795 my $drivename_hash;
 796
 797 my %drivedesc_base = (
 798     volume => { alias => 'file' },
 799     file => {
 800         type => 'string',
 801         format => 'pve-volume-id-or-qm-path',
 802         default_key => 1,
 803         format_description => 'volume',
 804         description => "The drive's backing volume.",
 805     },
 806     media => {
 807         type => 'string',
 808         enum => [qw(cdrom disk)],
 809         description => "The drive's media type.",
 810         default => 'disk',
 811         optional => 1
 812     },
 813     cyls => {
 814         type => 'integer',
 815         description => "Force the drive's physical geometry to have a specific cylinder count.",
 816         optional => 1
 817     },
 818     heads => {
 819         type => 'integer',
 820         description => "Force the drive's physical geometry to have a specific head count.",
 821         optional => 1
 822     },
 823     secs => {
 824         type => 'integer',
 825         description => "Force the drive's physical geometry to have a specific sector count.",
 826         optional => 1
 827     },
 828     trans => {
 829         type => 'string',
 830         enum => [qw(none lba auto)],
 831         description => "Force disk geometry bios translation mode.",
 832         optional => 1,
 833     },
 834     snapshot => {
 835         type => 'boolean',
 836         description => "Controls qemu's snapshot mode feature."
 837             . " If activated, changes made to the disk are temporary and will"
 838             . " be discarded when the VM is shutdown.",
 839         optional => 1,
 840     },
 841     cache => {
 842         type => 'string',
 843         enum => [qw(none writethrough writeback unsafe directsync)],
 844         description => "The drive's cache mode",
 845         optional => 1,
 846     },
 847     format => get_standard_option('pve-qm-image-format'),
 848     size => {
 849         type => 'string',
 850         format => 'disk-size',
 851         format_description => 'DiskSize',
 852         description => "Disk size. This is purely informational and has no effect.",
 853         optional => 1,
 854     },
 855     backup => {
 856         type => 'boolean',
 857         description => "Whether the drive should be included when making backups.",
 858         optional => 1,
 859     },
 860     replicate => {
 861         type => 'boolean',
 862         description => 'Whether the drive should considered for replication jobs.',
 863         optional => 1,
 864         default => 1,
 865     },
 866     rerror => {
 867         type => 'string',
 868         enum => [qw(ignore report stop)],
 869         description => 'Read error action.',
 870         optional => 1,
 871     },
 872     werror => {
 873         type => 'string',
 874         enum => [qw(enospc ignore report stop)],
 875         description => 'Write error action.',
 876         optional => 1,
 877     },
 878     aio => {
 879         type => 'string',
 880         enum => [qw(native threads)],
 881         description => 'AIO type to use.',
 882         optional => 1,
 883     },
 884     discard => {
 885         type => 'string',
 886         enum => [qw(ignore on)],
 887         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 888         optional => 1,
 889     },
 890     detect_zeroes => {
 891         type => 'boolean',
 892         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 893         optional => 1,
 894     },
 895     serial => {
 896         type => 'string',
 897         format => 'urlencoded',
 898         format_description => 'serial',
 899         maxLength => 20*3, # *3 since it's %xx url enoded
 900         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
 901         optional => 1,
 902     },
 903     shared => {
 904         type => 'boolean',
 905         description => 'Mark this locally-managed volume as available on all nodes',
 906         verbose_description => "Mark this locally-managed volume as available on all nodes.\n\nWARNING: This option does not share the volume automatically, it assumes it is shared already!",
 907         optional => 1,
 908         default => 0,
 909     }
 910 );
 911
 912 my %iothread_fmt = ( iothread => {
 913         type => 'boolean',
 914         description => "Whether to use iothreads for this drive",
 915         optional => 1,
 916 });
 917
 918 my %model_fmt = (
 919     model => {
 920         type => 'string',
 921         format => 'urlencoded',
 922         format_description => 'model',
 923         maxLength => 40*3, # *3 since it's %xx url enoded
 924         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
 925         optional => 1,
 926     },
 927 );
 928
 929 my %queues_fmt = (
 930     queues => {
 931         type => 'integer',
 932         description => "Number of queues.",
 933         minimum => 2,
 934         optional => 1
 935     }
 936 );
 937
 938 my %scsiblock_fmt = (
 939     scsiblock => {
 940         type => 'boolean',
 941         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
 942         optional => 1,
 943         default => 0,
 944     },
 945 );
 946
 947 my $add_throttle_desc = sub {
 948     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
 949     my $d = {
 950         type => $type,
 951         format_description => $unit,
 952         description => "Maximum $what in $longunit.",
 953         optional => 1,
 954     };
 955     $d->{minimum} = $minimum if defined($minimum);
 956     $drivedesc_base{$key} = $d;
 957 };
 958 # throughput: (leaky bucket)
 959 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
 960 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
 961 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
 962 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
 963 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
 964 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
 965 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
 966 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
 967 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
 968
 969 # pools: (pool of IO before throttling starts taking effect)
 970 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
 971 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
 972 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
 973 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
 974 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
 975 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
 976
 977 # burst lengths
 978 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
 979 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
 980 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
 981 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
 982 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
 983 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
 984
 985 # legacy support
 986 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
 987 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
 988 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
 989 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
 990
 991 my $ide_fmt = {
 992     %drivedesc_base,
 993     %model_fmt,
 994 };
 995 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
 996
 997 my $idedesc = {
 998     optional => 1,
 999     type => 'string', format => $ide_fmt,
1000     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
1001 };
1002 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
1003
1004 my $scsi_fmt = {
1005     %drivedesc_base,
1006     %iothread_fmt,
1007     %queues_fmt,
1008     %scsiblock_fmt,
1009 };
1010 my $scsidesc = {
1011     optional => 1,
1012     type => 'string', format => $scsi_fmt,
1013     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
1014 };
1015 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
1016
1017 my $sata_fmt = {
1018     %drivedesc_base,
1019 };
1020 my $satadesc = {
1021     optional => 1,
1022     type => 'string', format => $sata_fmt,
1023     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
1024 };
1025 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
1026
1027 my $virtio_fmt = {
1028     %drivedesc_base,
1029     %iothread_fmt,
1030 };
1031 my $virtiodesc = {
1032     optional => 1,
1033     type => 'string', format => $virtio_fmt,
1034     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
1035 };
1036 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
1037
1038 my $alldrive_fmt = {
1039     %drivedesc_base,
1040     %iothread_fmt,
1041     %model_fmt,
1042     %queues_fmt,
1043     %scsiblock_fmt,
1044 };
1045
1046 my $efidisk_fmt = {
1047     volume => { alias => 'file' },
1048     file => {
1049         type => 'string',
1050         format => 'pve-volume-id-or-qm-path',
1051         default_key => 1,
1052         format_description => 'volume',
1053         description => "The drive's backing volume.",
1054     },
1055     format => get_standard_option('pve-qm-image-format'),
1056     size => {
1057         type => 'string',
1058         format => 'disk-size',
1059         format_description => 'DiskSize',
1060         description => "Disk size. This is purely informational and has no effect.",
1061         optional => 1,
1062     },
1063 };
1064
1065 my $efidisk_desc = {
1066     optional => 1,
1067     type => 'string', format => $efidisk_fmt,
1068     description => "Configure a Disk for storing EFI vars",
1069 };
1070
1071 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
1072
1073 my $usb_fmt = {
1074     host => {
1075         default_key => 1,
1076         type => 'string', format => 'pve-qm-usb-device',
1077         format_description => 'HOSTUSBDEVICE|spice',
1078         description => <<EODESCR,
1079 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1080
1081  'bus-port(.port)*' (decimal numbers) or
1082  'vendor_id:product_id' (hexadeciaml numbers) or
1083  'spice'
1084
1085 You can use the 'lsusb -t' command to list existing usb devices.
1086
1087 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1088
1089 The value 'spice' can be used to add a usb redirection devices for spice.
1090 EODESCR
1091     },
1092     usb3 => {
1093         optional => 1,
1094         type => 'boolean',
1095         description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
1096         default => 0,
1097     },
1098 };
1099
1100 my $usbdesc = {
1101     optional => 1,
1102     type => 'string', format => $usb_fmt,
1103     description => "Configure an USB device (n is 0 to 4).",
1104 };
1105 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1106
1107 # NOTE: the match-groups of this regex are used in parse_hostpci
1108 my $PCIRE = qr/([a-f0-9]{2}:[a-f0-9]{2})(?:\.([a-f0-9]))?/;
1109 my $hostpci_fmt = {
1110     host => {
1111         default_key => 1,
1112         type => 'string',
1113         pattern => qr/$PCIRE(;$PCIRE)*/,
1114         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1115         description => <<EODESCR,
1116 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1117 of PCI virtual functions of the host. HOSTPCIID syntax is:
1118
1119 'bus:dev.func' (hexadecimal numbers)
1120
1121 You can us the 'lspci' command to list existing PCI devices.
1122 EODESCR
1123     },
1124     rombar => {
1125         type => 'boolean',
1126         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1127         optional => 1,
1128         default => 1,
1129     },
1130     romfile => {
1131         type => 'string',
1132         pattern => '[^,;]+',
1133         format_description => 'string',
1134         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1135         optional => 1,
1136     },
1137     pcie => {
1138         type => 'boolean',
1139         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1140         optional => 1,
1141         default => 0,
1142     },
1143     'x-vga' => {
1144         type => 'boolean',
1145         description =>  "Enable vfio-vga device support.",
1146         optional => 1,
1147         default => 0,
1148     },
1149 };
1150 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1151
1152 my $hostpcidesc = {
1153         optional => 1,
1154         type => 'string', format => 'pve-qm-hostpci',
1155         description => "Map host PCI devices into guest.",
1156         verbose_description =>  <<EODESCR,
1157 Map host PCI devices into guest.
1158
1159 NOTE: This option allows direct access to host hardware. So it is no longer
1160 possible to migrate such machines - use with special care.
1161
1162 CAUTION: Experimental! User reported problems with this option.
1163 EODESCR
1164 };
1165 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1166
1167 my $serialdesc = {
1168         optional => 1,
1169         type => 'string',
1170         pattern => '(/dev/.+|socket)',
1171         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1172         verbose_description =>  <<EODESCR,
1173 Create a serial device inside the VM (n is 0 to 3), and pass through a
1174 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1175 host side (use 'qm terminal' to open a terminal connection).
1176
1177 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1178
1179 CAUTION: Experimental! User reported problems with this option.
1180 EODESCR
1181 };
1182
1183 my $paralleldesc= {
1184         optional => 1,
1185         type => 'string',
1186         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1187         description =>  "Map host parallel devices (n is 0 to 2).",
1188         verbose_description =>  <<EODESCR,
1189 Map host parallel devices (n is 0 to 2).
1190
1191 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1192
1193 CAUTION: Experimental! User reported problems with this option.
1194 EODESCR
1195 };
1196
1197 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1198     $confdesc->{"parallel$i"} = $paralleldesc;
1199 }
1200
1201 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1202     $confdesc->{"serial$i"} = $serialdesc;
1203 }
1204
1205 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1206     $confdesc->{"hostpci$i"} = $hostpcidesc;
1207 }
1208
1209 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1210     $drivename_hash->{"ide$i"} = 1;
1211     $confdesc->{"ide$i"} = $idedesc;
1212 }
1213
1214 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1215     $drivename_hash->{"sata$i"} = 1;
1216     $confdesc->{"sata$i"} = $satadesc;
1217 }
1218
1219 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1220     $drivename_hash->{"scsi$i"} = 1;
1221     $confdesc->{"scsi$i"} = $scsidesc ;
1222 }
1223
1224 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1225     $drivename_hash->{"virtio$i"} = 1;
1226     $confdesc->{"virtio$i"} = $virtiodesc;
1227 }
1228
1229 $drivename_hash->{efidisk0} = 1;
1230 $confdesc->{efidisk0} = $efidisk_desc;
1231
1232 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1233     $confdesc->{"usb$i"} = $usbdesc;
1234 }
1235
1236 my $unuseddesc = {
1237     optional => 1,
1238     type => 'string', format => 'pve-volume-id',
1239     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1240 };
1241
1242 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1243     $confdesc->{"unused$i"} = $unuseddesc;
1244 }
1245
1246 my $kvm_api_version = 0;
1247
1248 sub kvm_version {
1249
1250     return $kvm_api_version if $kvm_api_version;
1251
1252     my $fh = IO::File->new("</dev/kvm") ||
1253         return 0;
1254
1255     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
1256         $kvm_api_version = $v;
1257     }
1258
1259     $fh->close();
1260
1261     return  $kvm_api_version;
1262 }
1263
1264 my $kvm_user_version;
1265
1266 sub kvm_user_version {
1267
1268     return $kvm_user_version if $kvm_user_version;
1269
1270     $kvm_user_version = 'unknown';
1271
1272     my $code = sub {
1273         my $line = shift;
1274         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1275             $kvm_user_version = $2;
1276         }
1277     };
1278
1279     eval { run_command("kvm -version", outfunc => $code); };
1280     warn $@ if $@;
1281
1282     return $kvm_user_version;
1283
1284 }
1285
1286 my $kernel_has_vhost_net = -c '/dev/vhost-net';
1287
1288 sub valid_drive_names {
1289     # order is important - used to autoselect boot disk
1290     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1291             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1292             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1293             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1294             'efidisk0');
1295 }
1296
1297 sub is_valid_drivename {
1298     my $dev = shift;
1299
1300     return defined($drivename_hash->{$dev});
1301 }
1302
1303 sub option_exists {
1304     my $key = shift;
1305     return defined($confdesc->{$key});
1306 }
1307
1308 sub nic_models {
1309     return $nic_model_list;
1310 }
1311
1312 sub os_list_description {
1313
1314     return {
1315         other => 'Other',
1316         wxp => 'Windows XP',
1317         w2k => 'Windows 2000',
1318         w2k3 =>, 'Windows 2003',
1319         w2k8 => 'Windows 2008',
1320         wvista => 'Windows Vista',
1321         win7 => 'Windows 7',
1322         win8 => 'Windows 8/2012',
1323         win10 => 'Windows 10/2016',
1324         l24 => 'Linux 2.4',
1325         l26 => 'Linux 2.6',
1326     };
1327 }
1328
1329 my $cdrom_path;
1330
1331 sub get_cdrom_path {
1332
1333     return  $cdrom_path if $cdrom_path;
1334
1335     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1336     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1337     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1338 }
1339
1340 sub get_iso_path {
1341     my ($storecfg, $vmid, $cdrom) = @_;
1342
1343     if ($cdrom eq 'cdrom') {
1344         return get_cdrom_path();
1345     } elsif ($cdrom eq 'none') {
1346         return '';
1347     } elsif ($cdrom =~ m|^/|) {
1348         return $cdrom;
1349     } else {
1350         return PVE::Storage::path($storecfg, $cdrom);
1351     }
1352 }
1353
1354 # try to convert old style file names to volume IDs
1355 sub filename_to_volume_id {
1356     my ($vmid, $file, $media) = @_;
1357
1358      if (!($file eq 'none' || $file eq 'cdrom' ||
1359           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1360
1361         return undef if $file =~ m|/|;
1362
1363         if ($media && $media eq 'cdrom') {
1364             $file = "local:iso/$file";
1365         } else {
1366             $file = "local:$vmid/$file";
1367         }
1368     }
1369
1370     return $file;
1371 }
1372
1373 sub verify_media_type {
1374     my ($opt, $vtype, $media) = @_;
1375
1376     return if !$media;
1377
1378     my $etype;
1379     if ($media eq 'disk') {
1380         $etype = 'images';
1381     } elsif ($media eq 'cdrom') {
1382         $etype = 'iso';
1383     } else {
1384         die "internal error";
1385     }
1386
1387     return if ($vtype eq $etype);
1388
1389     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1390 }
1391
1392 sub cleanup_drive_path {
1393     my ($opt, $storecfg, $drive) = @_;
1394
1395     # try to convert filesystem paths to volume IDs
1396
1397     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1398         ($drive->{file} !~ m|^/dev/.+|) &&
1399         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1400         ($drive->{file} !~ m/^\d+$/)) {
1401         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1402         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1403         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1404         verify_media_type($opt, $vtype, $drive->{media});
1405         $drive->{file} = $volid;
1406     }
1407
1408     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1409 }
1410
1411 sub parse_hotplug_features {
1412     my ($data) = @_;
1413
1414     my $res = {};
1415
1416     return $res if $data eq '0';
1417
1418     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1419
1420     foreach my $feature (PVE::Tools::split_list($data)) {
1421         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1422             $res->{$1} = 1;
1423         } else {
1424             die "invalid hotplug feature '$feature'\n";
1425         }
1426     }
1427     return $res;
1428 }
1429
1430 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1431 sub pve_verify_hotplug_features {
1432     my ($value, $noerr) = @_;
1433
1434     return $value if parse_hotplug_features($value);
1435
1436     return undef if $noerr;
1437
1438     die "unable to parse hotplug option\n";
1439 }
1440
1441 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1442 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1443 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1444 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1445 #        [,iothread=on][,serial=serial][,model=model]
1446
1447 sub parse_drive {
1448     my ($key, $data) = @_;
1449
1450     my ($interface, $index);
1451
1452     if ($key =~ m/^([^\d]+)(\d+)$/) {
1453         $interface = $1;
1454         $index = $2;
1455     } else {
1456         return undef;
1457     }
1458
1459     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1460                                      : $confdesc->{$key}->{format};
1461     if (!$desc) {
1462         warn "invalid drive key: $key\n";
1463         return undef;
1464     }
1465     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1466     return undef if !$res;
1467     $res->{interface} = $interface;
1468     $res->{index} = $index;
1469
1470     my $error = 0;
1471     foreach my $opt (qw(bps bps_rd bps_wr)) {
1472         if (my $bps = defined(delete $res->{$opt})) {
1473             if (defined($res->{"m$opt"})) {
1474                 warn "both $opt and m$opt specified\n";
1475                 ++$error;
1476                 next;
1477             }
1478             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1479         }
1480     }
1481
1482     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1483     for my $requirement (
1484         [mbps_max => 'mbps'],
1485         [mbps_rd_max => 'mbps_rd'],
1486         [mbps_wr_max => 'mbps_wr'],
1487         [miops_max => 'miops'],
1488         [miops_rd_max => 'miops_rd'],
1489         [miops_wr_max => 'miops_wr'],
1490         [bps_max_length => 'mbps_max'],
1491         [bps_rd_max_length => 'mbps_rd_max'],
1492         [bps_wr_max_length => 'mbps_wr_max'],
1493         [iops_max_length => 'iops_max'],
1494         [iops_rd_max_length => 'iops_rd_max'],
1495         [iops_wr_max_length => 'iops_wr_max']) {
1496         my ($option, $requires) = @$requirement;
1497         if ($res->{$option} && !$res->{$requires}) {
1498             warn "$option requires $requires\n";
1499             ++$error;
1500         }
1501     }
1502
1503     return undef if $error;
1504
1505     return undef if $res->{mbps_rd} && $res->{mbps};
1506     return undef if $res->{mbps_wr} && $res->{mbps};
1507     return undef if $res->{iops_rd} && $res->{iops};
1508     return undef if $res->{iops_wr} && $res->{iops};
1509
1510     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1511         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1512         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1513         return undef if $res->{interface} eq 'virtio';
1514     }
1515
1516     if (my $size = $res->{size}) {
1517         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1518     }
1519
1520     return $res;
1521 }
1522
1523 sub print_drive {
1524     my ($vmid, $drive) = @_;
1525     my $data = { %$drive };
1526     delete $data->{$_} for qw(index interface);
1527     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1528 }
1529
1530 sub scsi_inquiry {
1531     my($fh, $noerr) = @_;
1532
1533     my $SG_IO = 0x2285;
1534     my $SG_GET_VERSION_NUM = 0x2282;
1535
1536     my $versionbuf = "\x00" x 8;
1537     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1538     if (!$ret) {
1539         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1540         return undef;
1541     }
1542     my $version = unpack("I", $versionbuf);
1543     if ($version < 30000) {
1544         die "scsi generic interface too old\n"  if !$noerr;
1545         return undef;
1546     }
1547
1548     my $buf = "\x00" x 36;
1549     my $sensebuf = "\x00" x 8;
1550     my $cmd = pack("C x3 C x1", 0x12, 36);
1551
1552     # see /usr/include/scsi/sg.h
1553     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1554
1555     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1556                       length($sensebuf), 0, length($buf), $buf,
1557                       $cmd, $sensebuf, 6000);
1558
1559     $ret = ioctl($fh, $SG_IO, $packet);
1560     if (!$ret) {
1561         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1562         return undef;
1563     }
1564
1565     my @res = unpack($sg_io_hdr_t, $packet);
1566     if ($res[17] || $res[18]) {
1567         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1568         return undef;
1569     }
1570
1571     my $res = {};
1572     (my $byte0, my $byte1, $res->{vendor},
1573      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1574
1575     $res->{removable} = $byte1 & 128 ? 1 : 0;
1576     $res->{type} = $byte0 & 31;
1577
1578     return $res;
1579 }
1580
1581 sub path_is_scsi {
1582     my ($path) = @_;
1583
1584     my $fh = IO::File->new("+<$path") || return undef;
1585     my $res = scsi_inquiry($fh, 1);
1586     close($fh);
1587
1588     return $res;
1589 }
1590
1591 sub machine_type_is_q35 {
1592     my ($conf) = @_;
1593
1594     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1595 }
1596
1597 sub print_tabletdevice_full {
1598     my ($conf) = @_;
1599
1600     my $q35 = machine_type_is_q35($conf);
1601
1602     # we use uhci for old VMs because tablet driver was buggy in older qemu
1603     my $usbbus = $q35 ? "ehci" : "uhci";
1604
1605     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1606 }
1607
1608 sub print_drivedevice_full {
1609     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1610
1611     my $device = '';
1612     my $maxdev = 0;
1613
1614     if ($drive->{interface} eq 'virtio') {
1615         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1616         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1617         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1618     } elsif ($drive->{interface} eq 'scsi') {
1619
1620         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1621         my $unit = $drive->{index} % $maxdev;
1622         my $devicetype = 'hd';
1623         my $path = '';
1624         if (drive_is_cdrom($drive)) {
1625             $devicetype = 'cd';
1626         } else {
1627             if ($drive->{file} =~ m|^/|) {
1628                 $path = $drive->{file};
1629                 if (my $info = path_is_scsi($path)) {
1630                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1631                         $devicetype = 'block';
1632                     } elsif ($info->{type} == 1) { # tape
1633                         $devicetype = 'generic';
1634                     }
1635                 }
1636             } else {
1637                  $path = PVE::Storage::path($storecfg, $drive->{file});
1638             }
1639
1640             if($path =~ m/^iscsi\:\/\//){
1641                 $devicetype = 'generic';
1642             }
1643         }
1644
1645         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1646             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1647         } else {
1648             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1649         }
1650
1651     } elsif ($drive->{interface} eq 'ide'){
1652         $maxdev = 2;
1653         my $controller = int($drive->{index} / $maxdev);
1654         my $unit = $drive->{index} % $maxdev;
1655         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1656
1657         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1658         if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
1659             $model = URI::Escape::uri_unescape($model);
1660             $device .= ",model=$model";
1661         }
1662     } elsif ($drive->{interface} eq 'sata'){
1663         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1664         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1665         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1666     } elsif ($drive->{interface} eq 'usb') {
1667         die "implement me";
1668         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1669     } else {
1670         die "unsupported interface type";
1671     }
1672
1673     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1674
1675     return $device;
1676 }
1677
1678 sub get_initiator_name {
1679     my $initiator;
1680
1681     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1682     while (defined(my $line = <$fh>)) {
1683         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1684         $initiator = $1;
1685         last;
1686     }
1687     $fh->close();
1688
1689     return $initiator;
1690 }
1691
1692 sub print_drive_full {
1693     my ($storecfg, $vmid, $drive) = @_;
1694
1695     my $path;
1696     my $volid = $drive->{file};
1697     my $format;
1698
1699     if (drive_is_cdrom($drive)) {
1700         $path = get_iso_path($storecfg, $vmid, $volid);
1701     } else {
1702         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1703         if ($storeid) {
1704             $path = PVE::Storage::path($storecfg, $volid);
1705             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1706             $format = qemu_img_format($scfg, $volname);
1707         } else {
1708             $path = $volid;
1709             $format = "raw";
1710         }
1711    }
1712
1713     my $opts = '';
1714     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1715     foreach my $o (@qemu_drive_options) {
1716         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1717     }
1718
1719     # snapshot only accepts on|off
1720     if (defined($drive->{snapshot})) {
1721         my $v = $drive->{snapshot} ? 'on' : 'off';
1722         $opts .= ",snapshot=$v";
1723     }
1724
1725     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1726         my ($dir, $qmpname) = @$type;
1727         if (my $v = $drive->{"mbps$dir"}) {
1728             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1729         }
1730         if (my $v = $drive->{"mbps${dir}_max"}) {
1731             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1732         }
1733         if (my $v = $drive->{"bps${dir}_max_length"}) {
1734             $opts .= ",throttling.bps$qmpname-max-length=$v";
1735         }
1736         if (my $v = $drive->{"iops${dir}"}) {
1737             $opts .= ",throttling.iops$qmpname=$v";
1738         }
1739         if (my $v = $drive->{"iops${dir}_max"}) {
1740             $opts .= ",throttling.iops$qmpname-max=$v";
1741         }
1742         if (my $v = $drive->{"iops${dir}_max_length"}) {
1743             $opts .= ",throttling.iops$qmpname-max-length=$v";
1744         }
1745     }
1746
1747     if (my $serial = $drive->{serial}) {
1748         $serial = URI::Escape::uri_unescape($serial);
1749         $opts .= ",serial=$serial";
1750     }
1751
1752     $opts .= ",format=$format" if $format && !$drive->{format};
1753
1754     my $cache_direct = 0;
1755
1756     if (my $cache = $drive->{cache}) {
1757         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1758     } elsif (!drive_is_cdrom($drive)) {
1759         $opts .= ",cache=none";
1760         $cache_direct = 1;
1761     }
1762
1763     # aio native works only with O_DIRECT
1764     if (!$drive->{aio}) {
1765         if($cache_direct) {
1766             $opts .= ",aio=native";
1767         } else {
1768             $opts .= ",aio=threads";
1769         }
1770     }
1771
1772     if (!drive_is_cdrom($drive)) {
1773         my $detectzeroes;
1774         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1775             $detectzeroes = 'off';
1776         } elsif ($drive->{discard}) {
1777             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1778         } else {
1779             # This used to be our default with discard not being specified:
1780             $detectzeroes = 'on';
1781         }
1782         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1783     }
1784
1785     my $pathinfo = $path ? "file=$path," : '';
1786
1787     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1788 }
1789
1790 sub print_netdevice_full {
1791     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1792
1793     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1794
1795     my $device = $net->{model};
1796     if ($net->{model} eq 'virtio') {
1797          $device = 'virtio-net-pci';
1798      };
1799
1800     my $pciaddr = print_pci_addr("$netid", $bridges);
1801     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1802     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1803         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1804         my $vectors = $net->{queues} * 2 + 2;
1805         $tmpstr .= ",vectors=$vectors,mq=on";
1806     }
1807     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1808
1809     if ($use_old_bios_files) {
1810         my $romfile;
1811         if ($device eq 'virtio-net-pci') {
1812             $romfile = 'pxe-virtio.rom';
1813         } elsif ($device eq 'e1000') {
1814             $romfile = 'pxe-e1000.rom';
1815         } elsif ($device eq 'ne2k') {
1816             $romfile = 'pxe-ne2k_pci.rom';
1817         } elsif ($device eq 'pcnet') {
1818             $romfile = 'pxe-pcnet.rom';
1819         } elsif ($device eq 'rtl8139') {
1820             $romfile = 'pxe-rtl8139.rom';
1821         }
1822         $tmpstr .= ",romfile=$romfile" if $romfile;
1823     }
1824
1825     return $tmpstr;
1826 }
1827
1828 sub print_netdev_full {
1829     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1830
1831     my $i = '';
1832     if ($netid =~ m/^net(\d+)$/) {
1833         $i = int($1);
1834     }
1835
1836     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1837
1838     my $ifname = "tap${vmid}i$i";
1839
1840     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1841     die "interface name '$ifname' is too long (max 15 character)\n"
1842         if length($ifname) >= 16;
1843
1844     my $vhostparam = '';
1845     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1846
1847     my $vmname = $conf->{name} || "vm$vmid";
1848
1849     my $netdev = "";
1850     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1851
1852     if ($net->{bridge}) {
1853         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1854     } else {
1855         $netdev = "type=user,id=$netid,hostname=$vmname";
1856     }
1857
1858     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1859
1860     return $netdev;
1861 }
1862
1863
1864 sub print_cpu_device {
1865     my ($conf, $id) = @_;
1866
1867     my $kvm = $conf->{kvm} // 1;
1868     my $cpu = $kvm ? "kvm64" : "qemu64";
1869     if (my $cputype = $conf->{cpu}) {
1870         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
1871             or die "Cannot parse cpu description: $cputype\n";
1872         $cpu = $cpuconf->{cputype};
1873     }
1874
1875     my $cores = $conf->{cores} || 1;
1876
1877     my $current_core = ($id - 1) % $cores;
1878     my $current_socket = int(($id - 1 - $current_core)/$cores);
1879
1880     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
1881 }
1882
1883 sub drive_is_cdrom {
1884     my ($drive, $exclude_cloudinit) = @_;
1885
1886     return 0 if $exclude_cloudinit && $drive->{file} =~ m@[:/]vm-\d+-cloudinit(?:\.$QEMU_FORMAT_RE)?$@;
1887
1888     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1889
1890 }
1891
1892 sub parse_number_sets {
1893     my ($set) = @_;
1894     my $res = [];
1895     foreach my $part (split(/;/, $set)) {
1896         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1897             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1898             push @$res, [ $1, $2 ];
1899         } else {
1900             die "invalid range: $part\n";
1901         }
1902     }
1903     return $res;
1904 }
1905
1906 sub parse_numa {
1907     my ($data) = @_;
1908
1909     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
1910     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1911     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1912     return $res;
1913 }
1914
1915 sub parse_hostpci {
1916     my ($value) = @_;
1917
1918     return undef if !$value;
1919
1920     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
1921
1922     my @idlist = split(/;/, $res->{host});
1923     delete $res->{host};
1924     foreach my $id (@idlist) {
1925         if ($id =~ /^$PCIRE$/) {
1926             if (defined($2)) {
1927                 push @{$res->{pciid}}, { id => $1, function => $2 };
1928             } else {
1929                 my $pcidevices = lspci($1);
1930                 $res->{pciid} = $pcidevices->{$1};
1931             }
1932         } else {
1933             # should have been caught by parse_property_string already
1934             die "failed to parse PCI id: $id\n";
1935         }
1936     }
1937     return $res;
1938 }
1939
1940 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1941 sub parse_net {
1942     my ($data) = @_;
1943
1944     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
1945     if ($@) {
1946         warn $@;
1947         return undef;
1948     }
1949     if (!defined($res->{macaddr})) {
1950         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1951         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1952     }
1953     $res->{macaddr} = PVE::Tools::random_ether_addr() if !defined($res->{macaddr});
1954     return $res;
1955 }
1956
1957 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1958 sub parse_ipconfig {
1959     my ($data) = @_;
1960
1961     my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
1962     if ($@) {
1963         warn $@;
1964         return undef;
1965     }
1966
1967     if ($res->{gw} && !$res->{ip}) {
1968         warn 'gateway specified without specifying an IP address';
1969         return undef;
1970     }
1971     if ($res->{gw6} && !$res->{ip6}) {
1972         warn 'IPv6 gateway specified without specifying an IPv6 address';
1973         return undef;
1974     }
1975     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1976         warn 'gateway specified together with DHCP';
1977         return undef;
1978     }
1979     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
1980         # gw6 + auto/dhcp
1981         warn "IPv6 gateway specified together with $res->{ip6} address";
1982         return undef;
1983     }
1984
1985     if (!$res->{ip} && !$res->{ip6}) {
1986         return { ip => 'dhcp', ip6 => 'dhcp' };
1987     }
1988
1989     return $res;
1990 }
1991
1992 sub print_net {
1993     my $net = shift;
1994
1995     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1996 }
1997
1998 sub add_random_macs {
1999     my ($settings) = @_;
2000
2001     foreach my $opt (keys %$settings) {
2002         next if $opt !~ m/^net(\d+)$/;
2003         my $net = parse_net($settings->{$opt});
2004         next if !$net;
2005         $settings->{$opt} = print_net($net);
2006     }
2007 }
2008
2009 sub vm_is_volid_owner {
2010     my ($storecfg, $vmid, $volid) = @_;
2011
2012     if ($volid !~  m|^/|) {
2013         my ($path, $owner);
2014         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2015         if ($owner && ($owner == $vmid)) {
2016             return 1;
2017         }
2018     }
2019
2020     return undef;
2021 }
2022
2023 sub split_flagged_list {
2024     my $text = shift || '';
2025     $text =~ s/[,;]/ /g;
2026     $text =~ s/^\s+//;
2027     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
2028 }
2029
2030 sub join_flagged_list {
2031     my ($how, $lst) = @_;
2032     join $how, map { $lst->{$_} . $_ } keys %$lst;
2033 }
2034
2035 sub vmconfig_delete_pending_option {
2036     my ($conf, $key, $force) = @_;
2037
2038     delete $conf->{pending}->{$key};
2039     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2040     $pending_delete_hash->{$key} = $force ? '!' : '';
2041     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2042 }
2043
2044 sub vmconfig_undelete_pending_option {
2045     my ($conf, $key) = @_;
2046
2047     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2048     delete $pending_delete_hash->{$key};
2049
2050     if (%$pending_delete_hash) {
2051         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2052     } else {
2053         delete $conf->{pending}->{delete};
2054     }
2055 }
2056
2057 sub vmconfig_register_unused_drive {
2058     my ($storecfg, $vmid, $conf, $drive) = @_;
2059
2060     if (drive_is_cloudinit($drive)) {
2061         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2062         warn $@ if $@;
2063     } elsif (!drive_is_cdrom($drive)) {
2064         my $volid = $drive->{file};
2065         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2066             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2067         }
2068     }
2069 }
2070
2071 sub vmconfig_cleanup_pending {
2072     my ($conf) = @_;
2073
2074     # remove pending changes when nothing changed
2075     my $changes;
2076     foreach my $opt (keys %{$conf->{pending}}) {
2077         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
2078             $changes = 1;
2079             delete $conf->{pending}->{$opt};
2080         }
2081     }
2082
2083     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
2084     my $pending_delete_hash = {};
2085     while (my ($opt, $force) = each %$current_delete_hash) {
2086         if (defined($conf->{$opt})) {
2087             $pending_delete_hash->{$opt} = $force;
2088         } else {
2089             $changes = 1;
2090         }
2091     }
2092
2093     if (%$pending_delete_hash) {
2094         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2095     } else {
2096         delete $conf->{pending}->{delete};
2097     }
2098
2099     return $changes;
2100 }
2101
2102 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
2103 my $smbios1_fmt = {
2104     uuid => {
2105         type => 'string',
2106         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2107         format_description => 'UUID',
2108         description => "Set SMBIOS1 UUID.",
2109         optional => 1,
2110     },
2111     version => {
2112         type => 'string',
2113         pattern => '\S+',
2114         format_description => 'string',
2115         description => "Set SMBIOS1 version.",
2116         optional => 1,
2117     },
2118     serial => {
2119         type => 'string',
2120         pattern => '\S+',
2121         format_description => 'string',
2122         description => "Set SMBIOS1 serial number.",
2123         optional => 1,
2124     },
2125     manufacturer => {
2126         type => 'string',
2127         pattern => '\S+',
2128         format_description => 'string',
2129         description => "Set SMBIOS1 manufacturer.",
2130         optional => 1,
2131     },
2132     product => {
2133         type => 'string',
2134         pattern => '\S+',
2135         format_description => 'string',
2136         description => "Set SMBIOS1 product ID.",
2137         optional => 1,
2138     },
2139     sku => {
2140         type => 'string',
2141         pattern => '\S+',
2142         format_description => 'string',
2143         description => "Set SMBIOS1 SKU string.",
2144         optional => 1,
2145     },
2146     family => {
2147         type => 'string',
2148         pattern => '\S+',
2149         format_description => 'string',
2150         description => "Set SMBIOS1 family string.",
2151         optional => 1,
2152     },
2153 };
2154
2155 sub parse_smbios1 {
2156     my ($data) = @_;
2157
2158     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
2159     warn $@ if $@;
2160     return $res;
2161 }
2162
2163 sub print_smbios1 {
2164     my ($smbios1) = @_;
2165     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2166 }
2167
2168 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2169
2170 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2171 sub verify_bootdisk {
2172     my ($value, $noerr) = @_;
2173
2174     return $value if is_valid_drivename($value);
2175
2176     return undef if $noerr;
2177
2178     die "invalid boot disk '$value'\n";
2179 }
2180
2181 sub parse_watchdog {
2182     my ($value) = @_;
2183
2184     return undef if !$value;
2185
2186     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2187     warn $@ if $@;
2188     return $res;
2189 }
2190
2191 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2192 sub verify_usb_device {
2193     my ($value, $noerr) = @_;
2194
2195     return $value if parse_usb_device($value);
2196
2197     return undef if $noerr;
2198
2199     die "unable to parse usb device\n";
2200 }
2201
2202 # add JSON properties for create and set function
2203 sub json_config_properties {
2204     my $prop = shift;
2205
2206     foreach my $opt (keys %$confdesc) {
2207         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
2208         $prop->{$opt} = $confdesc->{$opt};
2209     }
2210
2211     return $prop;
2212 }
2213
2214 sub check_type {
2215     my ($key, $value) = @_;
2216
2217     die "unknown setting '$key'\n" if !$confdesc->{$key};
2218
2219     my $type = $confdesc->{$key}->{type};
2220
2221     if (!defined($value)) {
2222         die "got undefined value\n";
2223     }
2224
2225     if ($value =~ m/[\n\r]/) {
2226         die "property contains a line feed\n";
2227     }
2228
2229     if ($type eq 'boolean') {
2230         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2231         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2232         die "type check ('boolean') failed - got '$value'\n";
2233     } elsif ($type eq 'integer') {
2234         return int($1) if $value =~ m/^(\d+)$/;
2235         die "type check ('integer') failed - got '$value'\n";
2236     } elsif ($type eq 'number') {
2237         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2238         die "type check ('number') failed - got '$value'\n";
2239     } elsif ($type eq 'string') {
2240         if (my $fmt = $confdesc->{$key}->{format}) {
2241             PVE::JSONSchema::check_format($fmt, $value);
2242             return $value;
2243         }
2244         $value =~ s/^\"(.*)\"$/$1/;
2245         return $value;
2246     } else {
2247         die "internal error"
2248     }
2249 }
2250
2251 sub check_iommu_support{
2252     #fixme : need to check IOMMU support
2253     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
2254
2255     my $iommu=1;
2256     return $iommu;
2257
2258 }
2259
2260 sub touch_config {
2261     my ($vmid) = @_;
2262
2263     my $conf = PVE::QemuConfig->config_file($vmid);
2264     utime undef, undef, $conf;
2265 }
2266
2267 sub destroy_vm {
2268     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2269
2270     my $conffile = PVE::QemuConfig->config_file($vmid);
2271
2272     my $conf = PVE::QemuConfig->load_config($vmid);
2273
2274     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2275
2276     if ($conf->{template}) {
2277         # check if any base image is still used by a linked clone
2278         foreach_drive($conf, sub {
2279                 my ($ds, $drive) = @_;
2280
2281                 return if drive_is_cdrom($drive);
2282
2283                 my $volid = $drive->{file};
2284
2285                 return if !$volid || $volid =~ m|^/|;
2286
2287                 die "base volume '$volid' is still in use by linked cloned\n"
2288                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2289
2290         });
2291     }
2292
2293     # only remove disks owned by this VM
2294     foreach_drive($conf, sub {
2295         my ($ds, $drive) = @_;
2296
2297         return if drive_is_cdrom($drive, 1);
2298
2299         my $volid = $drive->{file};
2300
2301         return if !$volid || $volid =~ m|^/|;
2302
2303         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2304         return if !$path || !$owner || ($owner != $vmid);
2305
2306         eval {
2307             PVE::Storage::vdisk_free($storecfg, $volid);
2308         };
2309         warn "Could not remove disk '$volid', check manually: $@" if $@;
2310
2311     });
2312
2313     if ($keep_empty_config) {
2314         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2315     } else {
2316         unlink $conffile;
2317     }
2318
2319     # also remove unused disk
2320     eval {
2321         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2322
2323         eval {
2324             PVE::Storage::foreach_volid($dl, sub {
2325                 my ($volid, $sid, $volname, $d) = @_;
2326                 PVE::Storage::vdisk_free($storecfg, $volid);
2327             });
2328         };
2329         warn $@ if $@;
2330
2331     };
2332     warn $@ if $@;
2333 }
2334
2335 sub parse_vm_config {
2336     my ($filename, $raw) = @_;
2337
2338     return undef if !defined($raw);
2339
2340     my $res = {
2341         digest => Digest::SHA::sha1_hex($raw),
2342         snapshots => {},
2343         pending => {},
2344     };
2345
2346     $filename =~ m|/qemu-server/(\d+)\.conf$|
2347         || die "got strange filename '$filename'";
2348
2349     my $vmid = $1;
2350
2351     my $conf = $res;
2352     my $descr;
2353     my $section = '';
2354
2355     my @lines = split(/\n/, $raw);
2356     foreach my $line (@lines) {
2357         next if $line =~ m/^\s*$/;
2358
2359         if ($line =~ m/^\[PENDING\]\s*$/i) {
2360             $section = 'pending';
2361             if (defined($descr)) {
2362                 $descr =~ s/\s+$//;
2363                 $conf->{description} = $descr;
2364             }
2365             $descr = undef;
2366             $conf = $res->{$section} = {};
2367             next;
2368
2369         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2370             $section = $1;
2371             if (defined($descr)) {
2372                 $descr =~ s/\s+$//;
2373                 $conf->{description} = $descr;
2374             }
2375             $descr = undef;
2376             $conf = $res->{snapshots}->{$section} = {};
2377             next;
2378         }
2379
2380         if ($line =~ m/^\#(.*)\s*$/) {
2381             $descr = '' if !defined($descr);
2382             $descr .= PVE::Tools::decode_text($1) . "\n";
2383             next;
2384         }
2385
2386         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2387             $descr = '' if !defined($descr);
2388             $descr .= PVE::Tools::decode_text($2);
2389         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2390             $conf->{snapstate} = $1;
2391         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2392             my $key = $1;
2393             my $value = $2;
2394             $conf->{$key} = $value;
2395         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2396             my $value = $1;
2397             if ($section eq 'pending') {
2398                 $conf->{delete} = $value; # we parse this later
2399             } else {
2400                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2401             }
2402         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2403             my $key = $1;
2404             my $value = $2;
2405             eval { $value = check_type($key, $value); };
2406             if ($@) {
2407                 warn "vm $vmid - unable to parse value of '$key' - $@";
2408             } else {
2409                 $key = 'ide2' if $key eq 'cdrom';
2410                 my $fmt = $confdesc->{$key}->{format};
2411                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2412                     my $v = parse_drive($key, $value);
2413                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2414                         $v->{file} = $volid;
2415                         $value = print_drive($vmid, $v);
2416                     } else {
2417                         warn "vm $vmid - unable to parse value of '$key'\n";
2418                         next;
2419                     }
2420                 }
2421
2422                 $conf->{$key} = $value;
2423             }
2424         }
2425     }
2426
2427     if (defined($descr)) {
2428         $descr =~ s/\s+$//;
2429         $conf->{description} = $descr;
2430     }
2431     delete $res->{snapstate}; # just to be sure
2432
2433     return $res;
2434 }
2435
2436 sub write_vm_config {
2437     my ($filename, $conf) = @_;
2438
2439     delete $conf->{snapstate}; # just to be sure
2440
2441     if ($conf->{cdrom}) {
2442         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2443         $conf->{ide2} = $conf->{cdrom};
2444         delete $conf->{cdrom};
2445     }
2446
2447     # we do not use 'smp' any longer
2448     if ($conf->{sockets}) {
2449         delete $conf->{smp};
2450     } elsif ($conf->{smp}) {
2451         $conf->{sockets} = $conf->{smp};
2452         delete $conf->{cores};
2453         delete $conf->{smp};
2454     }
2455
2456     my $used_volids = {};
2457
2458     my $cleanup_config = sub {
2459         my ($cref, $pending, $snapname) = @_;
2460
2461         foreach my $key (keys %$cref) {
2462             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2463                 $key eq 'snapstate' || $key eq 'pending';
2464             my $value = $cref->{$key};
2465             if ($key eq 'delete') {
2466                 die "propertry 'delete' is only allowed in [PENDING]\n"
2467                     if !$pending;
2468                 # fixme: check syntax?
2469                 next;
2470             }
2471             eval { $value = check_type($key, $value); };
2472             die "unable to parse value of '$key' - $@" if $@;
2473
2474             $cref->{$key} = $value;
2475
2476             if (!$snapname && is_valid_drivename($key)) {
2477                 my $drive = parse_drive($key, $value);
2478                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2479             }
2480         }
2481     };
2482
2483     &$cleanup_config($conf);
2484
2485     &$cleanup_config($conf->{pending}, 1);
2486
2487     foreach my $snapname (keys %{$conf->{snapshots}}) {
2488         die "internal error" if $snapname eq 'pending';
2489         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2490     }
2491
2492     # remove 'unusedX' settings if we re-add a volume
2493     foreach my $key (keys %$conf) {
2494         my $value = $conf->{$key};
2495         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2496             delete $conf->{$key};
2497         }
2498     }
2499
2500     my $generate_raw_config = sub {
2501         my ($conf, $pending) = @_;
2502
2503         my $raw = '';
2504
2505         # add description as comment to top of file
2506         if (defined(my $descr = $conf->{description})) {
2507             if ($descr) {
2508                 foreach my $cl (split(/\n/, $descr)) {
2509                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2510                 }
2511             } else {
2512                 $raw .= "#\n" if $pending;
2513             }
2514         }
2515
2516         foreach my $key (sort keys %$conf) {
2517             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2518             $raw .= "$key: $conf->{$key}\n";
2519         }
2520         return $raw;
2521     };
2522
2523     my $raw = &$generate_raw_config($conf);
2524
2525     if (scalar(keys %{$conf->{pending}})){
2526         $raw .= "\n[PENDING]\n";
2527         $raw .= &$generate_raw_config($conf->{pending}, 1);
2528     }
2529
2530     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2531         $raw .= "\n[$snapname]\n";
2532         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2533     }
2534
2535     return $raw;
2536 }
2537
2538 sub load_defaults {
2539
2540     my $res = {};
2541
2542     # we use static defaults from our JSON schema configuration
2543     foreach my $key (keys %$confdesc) {
2544         if (defined(my $default = $confdesc->{$key}->{default})) {
2545             $res->{$key} = $default;
2546         }
2547     }
2548
2549     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2550     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2551
2552     return $res;
2553 }
2554
2555 sub config_list {
2556     my $vmlist = PVE::Cluster::get_vmlist();
2557     my $res = {};
2558     return $res if !$vmlist || !$vmlist->{ids};
2559     my $ids = $vmlist->{ids};
2560
2561     foreach my $vmid (keys %$ids) {
2562         my $d = $ids->{$vmid};
2563         next if !$d->{node} || $d->{node} ne $nodename;
2564         next if !$d->{type} || $d->{type} ne 'qemu';
2565         $res->{$vmid}->{exists} = 1;
2566     }
2567     return $res;
2568 }
2569
2570 # test if VM uses local resources (to prevent migration)
2571 sub check_local_resources {
2572     my ($conf, $noerr) = @_;
2573
2574     my $loc_res = 0;
2575
2576     $loc_res = 1 if $conf->{hostusb}; # old syntax
2577     $loc_res = 1 if $conf->{hostpci}; # old syntax
2578
2579     foreach my $k (keys %$conf) {
2580         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2581         # sockets are safe: they will recreated be on the target side post-migrate
2582         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2583         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2584     }
2585
2586     die "VM uses local resources\n" if $loc_res && !$noerr;
2587
2588     return $loc_res;
2589 }
2590
2591 # check if used storages are available on all nodes (use by migrate)
2592 sub check_storage_availability {
2593     my ($storecfg, $conf, $node) = @_;
2594
2595     foreach_drive($conf, sub {
2596         my ($ds, $drive) = @_;
2597
2598         my $volid = $drive->{file};
2599         return if !$volid;
2600
2601         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2602         return if !$sid;
2603
2604         # check if storage is available on both nodes
2605         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2606         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2607    });
2608 }
2609
2610 # list nodes where all VM images are available (used by has_feature API)
2611 sub shared_nodes {
2612     my ($conf, $storecfg) = @_;
2613
2614     my $nodelist = PVE::Cluster::get_nodelist();
2615     my $nodehash = { map { $_ => 1 } @$nodelist };
2616     my $nodename = PVE::INotify::nodename();
2617
2618     foreach_drive($conf, sub {
2619         my ($ds, $drive) = @_;
2620
2621         my $volid = $drive->{file};
2622         return if !$volid;
2623
2624         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2625         if ($storeid) {
2626             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2627             if ($scfg->{disable}) {
2628                 $nodehash = {};
2629             } elsif (my $avail = $scfg->{nodes}) {
2630                 foreach my $node (keys %$nodehash) {
2631                     delete $nodehash->{$node} if !$avail->{$node};
2632                 }
2633             } elsif (!$scfg->{shared}) {
2634                 foreach my $node (keys %$nodehash) {
2635                     delete $nodehash->{$node} if $node ne $nodename
2636                 }
2637             }
2638         }
2639     });
2640
2641     return $nodehash
2642 }
2643
2644 sub check_cmdline {
2645     my ($pidfile, $pid) = @_;
2646
2647     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2648     if (defined($fh)) {
2649         my $line = <$fh>;
2650         $fh->close;
2651         return undef if !$line;
2652         my @param = split(/\0/, $line);
2653
2654         my $cmd = $param[0];
2655         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2656
2657         for (my $i = 0; $i < scalar (@param); $i++) {
2658             my $p = $param[$i];
2659             next if !$p;
2660             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2661                 my $p = $param[$i+1];
2662                 return 1 if $p && ($p eq $pidfile);
2663                 return undef;
2664             }
2665         }
2666     }
2667     return undef;
2668 }
2669
2670 sub check_running {
2671     my ($vmid, $nocheck, $node) = @_;
2672
2673     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2674
2675     die "unable to find configuration file for VM $vmid - no such machine\n"
2676         if !$nocheck && ! -f $filename;
2677
2678     my $pidfile = pidfile_name($vmid);
2679
2680     if (my $fd = IO::File->new("<$pidfile")) {
2681         my $st = stat($fd);
2682         my $line = <$fd>;
2683         close($fd);
2684
2685         my $mtime = $st->mtime;
2686         if ($mtime > time()) {
2687             warn "file '$filename' modified in future\n";
2688         }
2689
2690         if ($line =~ m/^(\d+)$/) {
2691             my $pid = $1;
2692             if (check_cmdline($pidfile, $pid)) {
2693                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2694                     return $pid;
2695                 }
2696             }
2697         }
2698     }
2699
2700     return undef;
2701 }
2702
2703 sub vzlist {
2704
2705     my $vzlist = config_list();
2706
2707     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2708
2709     while (defined(my $de = $fd->read)) {
2710         next if $de !~ m/^(\d+)\.pid$/;
2711         my $vmid = $1;
2712         next if !defined($vzlist->{$vmid});
2713         if (my $pid = check_running($vmid)) {
2714             $vzlist->{$vmid}->{pid} = $pid;
2715         }
2716     }
2717
2718     return $vzlist;
2719 }
2720
2721 sub disksize {
2722     my ($storecfg, $conf) = @_;
2723
2724     my $bootdisk = $conf->{bootdisk};
2725     return undef if !$bootdisk;
2726     return undef if !is_valid_drivename($bootdisk);
2727
2728     return undef if !$conf->{$bootdisk};
2729
2730     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2731     return undef if !defined($drive);
2732
2733     return undef if drive_is_cdrom($drive);
2734
2735     my $volid = $drive->{file};
2736     return undef if !$volid;
2737
2738     return $drive->{size};
2739 }
2740
2741 my $last_proc_pid_stat;
2742
2743 # get VM status information
2744 # This must be fast and should not block ($full == false)
2745 # We only query KVM using QMP if $full == true (this can be slow)
2746 sub vmstatus {
2747     my ($opt_vmid, $full) = @_;
2748
2749     my $res = {};
2750
2751     my $storecfg = PVE::Storage::config();
2752
2753     my $list = vzlist();
2754     my $defaults = load_defaults();
2755
2756     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2757
2758     my $cpucount = $cpuinfo->{cpus} || 1;
2759
2760     foreach my $vmid (keys %$list) {
2761         next if $opt_vmid && ($vmid ne $opt_vmid);
2762
2763         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
2764         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2765
2766         my $d = {};
2767         $d->{pid} = $list->{$vmid}->{pid};
2768
2769         # fixme: better status?
2770         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2771
2772         my $size = disksize($storecfg, $conf);
2773         if (defined($size)) {
2774             $d->{disk} = 0; # no info available
2775             $d->{maxdisk} = $size;
2776         } else {
2777             $d->{disk} = 0;
2778             $d->{maxdisk} = 0;
2779         }
2780
2781         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2782             * ($conf->{cores} || $defaults->{cores});
2783         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2784         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2785
2786         $d->{name} = $conf->{name} || "VM $vmid";
2787         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2788             : $defaults->{memory}*(1024*1024);
2789
2790         if ($conf->{balloon}) {
2791             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2792             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2793                 : $defaults->{shares};
2794         }
2795
2796         $d->{uptime} = 0;
2797         $d->{cpu} = 0;
2798         $d->{mem} = 0;
2799
2800         $d->{netout} = 0;
2801         $d->{netin} = 0;
2802
2803         $d->{diskread} = 0;
2804         $d->{diskwrite} = 0;
2805
2806         $d->{template} = PVE::QemuConfig->is_template($conf);
2807
2808         $d->{serial} = 1 if conf_has_serial($conf);
2809
2810         $res->{$vmid} = $d;
2811     }
2812
2813     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2814     foreach my $dev (keys %$netdev) {
2815         next if $dev !~ m/^tap([1-9]\d*)i/;
2816         my $vmid = $1;
2817         my $d = $res->{$vmid};
2818         next if !$d;
2819
2820         $d->{netout} += $netdev->{$dev}->{receive};
2821         $d->{netin} += $netdev->{$dev}->{transmit};
2822
2823         if ($full) {
2824             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2825             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2826         }
2827
2828     }
2829
2830     my $ctime = gettimeofday;
2831
2832     foreach my $vmid (keys %$list) {
2833
2834         my $d = $res->{$vmid};
2835         my $pid = $d->{pid};
2836         next if !$pid;
2837
2838         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2839         next if !$pstat; # not running
2840
2841         my $used = $pstat->{utime} + $pstat->{stime};
2842
2843         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2844
2845         if ($pstat->{vsize}) {
2846             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2847         }
2848
2849         my $old = $last_proc_pid_stat->{$pid};
2850         if (!$old) {
2851             $last_proc_pid_stat->{$pid} = {
2852                 time => $ctime,
2853                 used => $used,
2854                 cpu => 0,
2855             };
2856             next;
2857         }
2858
2859         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2860
2861         if ($dtime > 1000) {
2862             my $dutime = $used -  $old->{used};
2863
2864             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2865             $last_proc_pid_stat->{$pid} = {
2866                 time => $ctime,
2867                 used => $used,
2868                 cpu => $d->{cpu},
2869             };
2870         } else {
2871             $d->{cpu} = $old->{cpu};
2872         }
2873     }
2874
2875     return $res if !$full;
2876
2877     my $qmpclient = PVE::QMPClient->new();
2878
2879     my $ballooncb = sub {
2880         my ($vmid, $resp) = @_;
2881
2882         my $info = $resp->{'return'};
2883         return if !$info->{max_mem};
2884
2885         my $d = $res->{$vmid};
2886
2887         # use memory assigned to VM
2888         $d->{maxmem} = $info->{max_mem};
2889         $d->{balloon} = $info->{actual};
2890
2891         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2892             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2893             $d->{freemem} = $info->{free_mem};
2894         }
2895
2896         $d->{ballooninfo} = $info;
2897     };
2898
2899     my $blockstatscb = sub {
2900         my ($vmid, $resp) = @_;
2901         my $data = $resp->{'return'} || [];
2902         my $totalrdbytes = 0;
2903         my $totalwrbytes = 0;
2904
2905         for my $blockstat (@$data) {
2906             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2907             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2908
2909             $blockstat->{device} =~ s/drive-//;
2910             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2911         }
2912         $res->{$vmid}->{diskread} = $totalrdbytes;
2913         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2914     };
2915
2916     my $statuscb = sub {
2917         my ($vmid, $resp) = @_;
2918
2919         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2920         # this fails if ballon driver is not loaded, so this must be
2921         # the last commnand (following command are aborted if this fails).
2922         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2923
2924         my $status = 'unknown';
2925         if (!defined($status = $resp->{'return'}->{status})) {
2926             warn "unable to get VM status\n";
2927             return;
2928         }
2929
2930         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2931     };
2932
2933     foreach my $vmid (keys %$list) {
2934         next if $opt_vmid && ($vmid ne $opt_vmid);
2935         next if !$res->{$vmid}->{pid}; # not running
2936         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2937     }
2938
2939     $qmpclient->queue_execute(undef, 2);
2940
2941     foreach my $vmid (keys %$list) {
2942         next if $opt_vmid && ($vmid ne $opt_vmid);
2943         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2944     }
2945
2946     return $res;
2947 }
2948
2949 sub foreach_drive {
2950     my ($conf, $func, @param) = @_;
2951
2952     foreach my $ds (valid_drive_names()) {
2953         next if !defined($conf->{$ds});
2954
2955         my $drive = parse_drive($ds, $conf->{$ds});
2956         next if !$drive;
2957
2958         &$func($ds, $drive, @param);
2959     }
2960 }
2961
2962 sub foreach_volid {
2963     my ($conf, $func, @param) = @_;
2964
2965     my $volhash = {};
2966
2967     my $test_volid = sub {
2968         my ($volid, $is_cdrom, $replicate, $shared, $snapname) = @_;
2969
2970         return if !$volid;
2971
2972         $volhash->{$volid}->{cdrom} //= 1;
2973         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
2974
2975         $volhash->{$volid}->{replicate} //= 0;
2976         $volhash->{$volid}->{replicate} = 1 if $replicate;
2977
2978         $volhash->{$volid}->{shared} //= 0;
2979         $volhash->{$volid}->{shared} = 1 if $shared;
2980
2981         $volhash->{$volid}->{referenced_in_config} //= 0;
2982         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
2983
2984         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
2985             if defined($snapname);
2986     };
2987
2988     foreach_drive($conf, sub {
2989         my ($ds, $drive) = @_;
2990         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, undef);
2991     });
2992
2993     foreach my $snapname (keys %{$conf->{snapshots}}) {
2994         my $snap = $conf->{snapshots}->{$snapname};
2995         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
2996         foreach_drive($snap, sub {
2997             my ($ds, $drive) = @_;
2998             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, $snapname);
2999         });
3000     }
3001
3002     foreach my $volid (keys %$volhash) {
3003         &$func($volid, $volhash->{$volid}, @param);
3004     }
3005 }
3006
3007 sub conf_has_serial {
3008     my ($conf) = @_;
3009
3010     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3011         if ($conf->{"serial$i"}) {
3012             return 1;
3013         }
3014     }
3015
3016     return 0;
3017 }
3018
3019 sub vga_conf_has_spice {
3020     my ($vga) = @_;
3021
3022     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
3023
3024     return $1 || 1;
3025 }
3026
3027 sub config_to_command {
3028     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
3029
3030     my $cmd = [];
3031     my $globalFlags = [];
3032     my $machineFlags = [];
3033     my $rtcFlags = [];
3034     my $cpuFlags = [];
3035     my $devices = [];
3036     my $pciaddr = '';
3037     my $bridges = {};
3038     my $kvmver = kvm_user_version();
3039     my $vernum = 0; # unknown
3040     my $ostype = $conf->{ostype};
3041     my $winversion = windows_version($ostype);
3042     my $kvm = $conf->{kvm} // 1;
3043
3044     die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n" if (!$cpuinfo->{hvm} && $kvm);
3045
3046     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
3047         $vernum = $1*1000000+$2*1000;
3048     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
3049         $vernum = $1*1000000+$2*1000+$3;
3050     }
3051
3052     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
3053
3054     my $have_ovz = -f '/proc/vz/vestat';
3055
3056     my $q35 = machine_type_is_q35($conf);
3057     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3058     my $machine_type = $forcemachine || $conf->{machine};
3059     my $use_old_bios_files = undef;
3060     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3061
3062     my $cpuunits = defined($conf->{cpuunits}) ?
3063             $conf->{cpuunits} : $defaults->{cpuunits};
3064
3065     push @$cmd, '/usr/bin/kvm';
3066
3067     push @$cmd, '-id', $vmid;
3068
3069     my $use_virtio = 0;
3070
3071     my $qmpsocket = qmp_socket($vmid);
3072     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3073     push @$cmd, '-mon', "chardev=qmp,mode=control";
3074
3075
3076     push @$cmd, '-pidfile' , pidfile_name($vmid);
3077
3078     push @$cmd, '-daemonize';
3079
3080     if ($conf->{smbios1}) {
3081         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3082     }
3083
3084     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3085         die "uefi base image not found\n" if ! -f $OVMF_CODE;
3086
3087         my $path;
3088         my $format;
3089         if (my $efidisk = $conf->{efidisk0}) {
3090             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
3091             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3092             $format = $d->{format};
3093             if ($storeid) {
3094                 $path = PVE::Storage::path($storecfg, $d->{file});
3095                 if (!defined($format)) {
3096                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3097                     $format = qemu_img_format($scfg, $volname);
3098                 }
3099             } else {
3100                 $path = $d->{file};
3101                 die "efidisk format must be specified\n"
3102                     if !defined($format);
3103             }
3104         } else {
3105             warn "no efidisk configured! Using temporary efivars disk.\n";
3106             $path = "/tmp/$vmid-ovmf.fd";
3107             PVE::Tools::file_copy($OVMF_VARS, $path, -s $OVMF_VARS);
3108             $format = 'raw';
3109         }
3110
3111         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$OVMF_CODE";
3112         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
3113     }
3114
3115
3116     # add usb controllers
3117     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $q35, $usbdesc->{format}, $MAX_USB_DEVICES);
3118     push @$devices, @usbcontrollers if @usbcontrollers;
3119     my $vga = $conf->{vga};
3120
3121     my $qxlnum = vga_conf_has_spice($vga);
3122     $vga = 'qxl' if $qxlnum;
3123
3124     if (!$vga) {
3125         if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 9)) {
3126             $vga = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3127         } else {
3128             $vga = ($winversion >= 6) ? 'std' : 'cirrus';
3129         }
3130     }
3131
3132     # enable absolute mouse coordinates (needed by vnc)
3133     my $tablet;
3134     if (defined($conf->{tablet})) {
3135         $tablet = $conf->{tablet};
3136     } else {
3137         $tablet = $defaults->{tablet};
3138         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3139         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3140     }
3141
3142     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
3143
3144     my $kvm_off = 0;
3145     my $gpu_passthrough;
3146
3147     # host pci devices
3148     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3149         my $d = parse_hostpci($conf->{"hostpci$i"});
3150         next if !$d;
3151
3152         my $pcie = $d->{pcie};
3153         if($pcie){
3154             die "q35 machine model is not enabled" if !$q35;
3155             $pciaddr = print_pcie_addr("hostpci$i");
3156         }else{
3157             $pciaddr = print_pci_addr("hostpci$i", $bridges);
3158         }
3159
3160         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
3161         my $romfile = $d->{romfile};
3162
3163         my $xvga = '';
3164         if ($d->{'x-vga'}) {
3165             $xvga = ',x-vga=on';
3166             $kvm_off = 1;
3167             $vga = 'none';
3168             $gpu_passthrough = 1;
3169
3170             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3171                 $xvga = "";
3172             }
3173         }
3174         my $pcidevices = $d->{pciid};
3175         my $multifunction = 1 if @$pcidevices > 1;
3176
3177         my $j=0;
3178         foreach my $pcidevice (@$pcidevices) {
3179
3180             my $id = "hostpci$i";
3181             $id .= ".$j" if $multifunction;
3182             my $addr = $pciaddr;
3183             $addr .= ".$j" if $multifunction;
3184             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
3185
3186             if($j == 0){
3187                 $devicestr .= "$rombar$xvga";
3188                 $devicestr .= ",multifunction=on" if $multifunction;
3189                 $devicestr .= ",romfile=/usr/share/kvm/$romfile" if $romfile;
3190             }
3191
3192             push @$devices, '-device', $devicestr;
3193             $j++;
3194         }
3195     }
3196
3197     # usb devices
3198     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES);
3199     push @$devices, @usbdevices if @usbdevices;
3200     # serial devices
3201     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3202         if (my $path = $conf->{"serial$i"}) {
3203             if ($path eq 'socket') {
3204                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3205                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3206                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3207             } else {
3208                 die "no such serial device\n" if ! -c $path;
3209                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3210                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3211             }
3212         }
3213     }
3214
3215     # parallel devices
3216     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3217         if (my $path = $conf->{"parallel$i"}) {
3218             die "no such parallel device\n" if ! -c $path;
3219             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3220             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3221             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3222         }
3223     }
3224
3225     my $vmname = $conf->{name} || "vm$vmid";
3226
3227     push @$cmd, '-name', $vmname;
3228
3229     my $sockets = 1;
3230     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3231     $sockets = $conf->{sockets} if  $conf->{sockets};
3232
3233     my $cores = $conf->{cores} || 1;
3234
3235     my $maxcpus = $sockets * $cores;
3236
3237     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3238
3239     my $allowed_vcpus = $cpuinfo->{cpus};
3240
3241     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3242         if ($allowed_vcpus < $maxcpus);
3243
3244     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
3245
3246         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3247         for (my $i = 2; $i <= $vcpus; $i++)  {
3248             my $cpustr = print_cpu_device($conf,$i);
3249             push @$cmd, '-device', $cpustr;
3250         }
3251
3252     } else {
3253
3254         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3255     }
3256     push @$cmd, '-nodefaults';
3257
3258     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3259
3260     my $bootindex_hash = {};
3261     my $i = 1;
3262     foreach my $o (split(//, $bootorder)) {
3263         $bootindex_hash->{$o} = $i*100;
3264         $i++;
3265     }
3266
3267     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3268
3269     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3270
3271     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3272
3273     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
3274
3275     if ($vga && $vga !~ m/^serial\d+$/ && $vga ne 'none'){
3276         my $socket = vnc_socket($vmid);
3277         push @$cmd,  '-vnc', "unix:$socket,x509,password";
3278     } else {
3279         push @$cmd, '-nographic';
3280     }
3281
3282     # time drift fix
3283     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3284
3285     my $useLocaltime = $conf->{localtime};
3286
3287     if ($winversion >= 5) { # windows
3288         $useLocaltime = 1 if !defined($conf->{localtime});
3289
3290         # use time drift fix when acpi is enabled
3291         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3292             $tdf = 1 if !defined($conf->{tdf});
3293         }
3294     }
3295
3296     if ($winversion >= 6) {
3297         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3298         push @$cmd, '-no-hpet';
3299     }
3300
3301     push @$rtcFlags, 'driftfix=slew' if $tdf;
3302
3303     if (!$kvm) {
3304         push @$machineFlags, 'accel=tcg';
3305     }
3306
3307     if ($machine_type) {
3308         push @$machineFlags, "type=${machine_type}";
3309     }
3310
3311     if ($conf->{startdate}) {
3312         push @$rtcFlags, "base=$conf->{startdate}";
3313     } elsif ($useLocaltime) {
3314         push @$rtcFlags, 'base=localtime';
3315     }
3316
3317     my $cpu = $kvm ? "kvm64" : "qemu64";
3318     if (my $cputype = $conf->{cpu}) {
3319         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3320             or die "Cannot parse cpu description: $cputype\n";
3321         $cpu = $cpuconf->{cputype};
3322         $kvm_off = 1 if $cpuconf->{hidden};
3323
3324         if (defined(my $flags = $cpuconf->{flags})) {
3325             push @$cpuFlags, split(";", $flags);
3326         }
3327     }
3328
3329     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
3330
3331     push @$cpuFlags , '-x2apic'
3332         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3333
3334     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3335
3336     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3337
3338     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3339
3340         push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3341         push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3342     }
3343
3344     add_hyperv_enlightenments($cpuFlags, $winversion, $machine_type, $kvmver, $conf->{bios}, $gpu_passthrough) if $kvm;
3345
3346     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm;
3347
3348     push @$cpuFlags, 'kvm=off' if $kvm_off;
3349
3350     my $cpu_vendor = $cpu_vendor_list->{$cpu} ||
3351         die "internal error"; # should not happen
3352
3353     push @$cpuFlags, "vendor=${cpu_vendor}"
3354         if $cpu_vendor ne 'default';
3355
3356     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3357
3358     push @$cmd, '-cpu', $cpu;
3359
3360     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3361
3362     push @$cmd, '-S' if $conf->{freeze};
3363
3364     # set keyboard layout
3365     my $kb = $conf->{keyboard} || $defaults->{keyboard};
3366     push @$cmd, '-k', $kb if $kb;
3367
3368     # enable sound
3369     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3370     #push @$cmd, '-soundhw', 'es1370';
3371     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3372
3373     if($conf->{agent}) {
3374         my $qgasocket = qmp_socket($vmid, 1);
3375         my $pciaddr = print_pci_addr("qga0", $bridges);
3376         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3377         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3378         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3379     }
3380
3381     my $spice_port;
3382
3383     if ($qxlnum) {
3384         if ($qxlnum > 1) {
3385             if ($winversion){
3386                 for(my $i = 1; $i < $qxlnum; $i++){
3387                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3388                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3389                 }
3390             } else {
3391                 # assume other OS works like Linux
3392                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3393                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3394             }
3395         }
3396
3397         my $pciaddr = print_pci_addr("spice", $bridges);
3398
3399         my $nodename = PVE::INotify::nodename();
3400         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3401         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3402         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3403         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3404         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3405
3406         push @$devices, '-spice', "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3407
3408         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3409         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3410         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3411     }
3412
3413     # enable balloon by default, unless explicitly disabled
3414     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3415         $pciaddr = print_pci_addr("balloon0", $bridges);
3416         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3417     }
3418
3419     if ($conf->{watchdog}) {
3420         my $wdopts = parse_watchdog($conf->{watchdog});
3421         $pciaddr = print_pci_addr("watchdog", $bridges);
3422         my $watchdog = $wdopts->{model} || 'i6300esb';
3423         push @$devices, '-device', "$watchdog$pciaddr";
3424         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3425     }
3426
3427     my $vollist = [];
3428     my $scsicontroller = {};
3429     my $ahcicontroller = {};
3430     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3431
3432     # Add iscsi initiator name if available
3433     if (my $initiator = get_initiator_name()) {
3434         push @$devices, '-iscsi', "initiator-name=$initiator";
3435     }
3436
3437     foreach_drive($conf, sub {
3438         my ($ds, $drive) = @_;
3439
3440         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3441             push @$vollist, $drive->{file};
3442         }
3443
3444         # ignore efidisk here, already added in bios/fw handling code above
3445         return if $drive->{interface} eq 'efidisk';
3446
3447         $use_virtio = 1 if $ds =~ m/^virtio/;
3448
3449         if (drive_is_cdrom ($drive)) {
3450             if ($bootindex_hash->{d}) {
3451                 $drive->{bootindex} = $bootindex_hash->{d};
3452                 $bootindex_hash->{d} += 1;
3453             }
3454         } else {
3455             if ($bootindex_hash->{c}) {
3456                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3457                 $bootindex_hash->{c} += 1;
3458             }
3459         }
3460
3461         if($drive->{interface} eq 'virtio'){
3462            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3463         }
3464
3465         if ($drive->{interface} eq 'scsi') {
3466
3467             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3468
3469             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3470             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3471
3472             my $iothread = '';
3473             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3474                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3475                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3476             } elsif ($drive->{iothread}) {
3477                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3478             }
3479
3480             my $queues = '';
3481             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3482                 $queues = ",num_queues=$drive->{queues}";
3483             }
3484
3485             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3486             $scsicontroller->{$controller}=1;
3487         }
3488
3489         if ($drive->{interface} eq 'sata') {
3490            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3491            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3492            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3493            $ahcicontroller->{$controller}=1;
3494         }
3495
3496         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3497         push @$devices, '-drive',$drive_cmd;
3498         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3499     });
3500
3501     for (my $i = 0; $i < $MAX_NETS; $i++) {
3502          next if !$conf->{"net$i"};
3503          my $d = parse_net($conf->{"net$i"});
3504          next if !$d;
3505
3506          $use_virtio = 1 if $d->{model} eq 'virtio';
3507
3508          if ($bootindex_hash->{n}) {
3509             $d->{bootindex} = $bootindex_hash->{n};
3510             $bootindex_hash->{n} += 1;
3511          }
3512
3513          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3514          push @$devices, '-netdev', $netdevfull;
3515
3516          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3517          push @$devices, '-device', $netdevicefull;
3518     }
3519
3520     if (!$q35) {
3521         # add pci bridges
3522         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3523            $bridges->{1} = 1;
3524            $bridges->{2} = 1;
3525         }
3526
3527         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3528
3529         while (my ($k, $v) = each %$bridges) {
3530             $pciaddr = print_pci_addr("pci.$k");
3531             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3532         }
3533     }
3534
3535     # add custom args
3536     if ($conf->{args}) {
3537         my $aa = PVE::Tools::split_args($conf->{args});
3538         push @$cmd, @$aa;
3539     }
3540
3541     push @$cmd, @$devices;
3542     push @$cmd, '-rtc', join(',', @$rtcFlags)
3543         if scalar(@$rtcFlags);
3544     push @$cmd, '-machine', join(',', @$machineFlags)
3545         if scalar(@$machineFlags);
3546     push @$cmd, '-global', join(',', @$globalFlags)
3547         if scalar(@$globalFlags);
3548
3549     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3550 }
3551
3552 sub vnc_socket {
3553     my ($vmid) = @_;
3554     return "${var_run_tmpdir}/$vmid.vnc";
3555 }
3556
3557 sub spice_port {
3558     my ($vmid) = @_;
3559
3560     my $res = vm_mon_cmd($vmid, 'query-spice');
3561
3562     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3563 }
3564
3565 sub qmp_socket {
3566     my ($vmid, $qga) = @_;
3567     my $sockettype = $qga ? 'qga' : 'qmp';
3568     return "${var_run_tmpdir}/$vmid.$sockettype";
3569 }
3570
3571 sub pidfile_name {
3572     my ($vmid) = @_;
3573     return "${var_run_tmpdir}/$vmid.pid";
3574 }
3575
3576 sub vm_devices_list {
3577     my ($vmid) = @_;
3578
3579     my $res = vm_mon_cmd($vmid, 'query-pci');
3580     my $devices = {};
3581     foreach my $pcibus (@$res) {
3582         foreach my $device (@{$pcibus->{devices}}) {
3583             next if !$device->{'qdev_id'};
3584             if ($device->{'pci_bridge'}) {
3585                 $devices->{$device->{'qdev_id'}} = 1;
3586                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3587                     next if !$bridge_device->{'qdev_id'};
3588                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3589                     $devices->{$device->{'qdev_id'}}++;
3590                 }
3591             } else {
3592                 $devices->{$device->{'qdev_id'}} = 1;
3593             }
3594         }
3595     }
3596
3597     my $resblock = vm_mon_cmd($vmid, 'query-block');
3598     foreach my $block (@$resblock) {
3599         if($block->{device} =~ m/^drive-(\S+)/){
3600                 $devices->{$1} = 1;
3601         }
3602     }
3603
3604     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3605     foreach my $mice (@$resmice) {
3606         if ($mice->{name} eq 'QEMU HID Tablet') {
3607             $devices->{tablet} = 1;
3608             last;
3609         }
3610     }
3611
3612     # for usb devices there is no query-usb
3613     # but we can iterate over the entries in
3614     # qom-list path=/machine/peripheral
3615     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3616     foreach my $per (@$resperipheral) {
3617         if ($per->{name} =~ m/^usb\d+$/) {
3618             $devices->{$per->{name}} = 1;
3619         }
3620     }
3621
3622     return $devices;
3623 }
3624
3625 sub vm_deviceplug {
3626     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3627
3628     my $q35 = machine_type_is_q35($conf);
3629
3630     my $devices_list = vm_devices_list($vmid);
3631     return 1 if defined($devices_list->{$deviceid});
3632
3633     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3634
3635     if ($deviceid eq 'tablet') {
3636
3637         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3638
3639     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3640
3641         die "usb hotplug currently not reliable\n";
3642         # since we can't reliably hot unplug all added usb devices
3643         # and usb passthrough disables live migration
3644         # we disable usb hotplugging for now
3645         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3646
3647     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3648
3649         qemu_iothread_add($vmid, $deviceid, $device);
3650
3651         qemu_driveadd($storecfg, $vmid, $device);
3652         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3653
3654         qemu_deviceadd($vmid, $devicefull);
3655         eval { qemu_deviceaddverify($vmid, $deviceid); };
3656         if (my $err = $@) {
3657             eval { qemu_drivedel($vmid, $deviceid); };
3658             warn $@ if $@;
3659             die $err;
3660         }
3661
3662     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3663
3664
3665         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3666         my $pciaddr = print_pci_addr($deviceid);
3667         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3668
3669         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3670
3671         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3672             qemu_iothread_add($vmid, $deviceid, $device);
3673             $devicefull .= ",iothread=iothread-$deviceid";
3674         }
3675
3676         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3677             $devicefull .= ",num_queues=$device->{queues}";
3678         }
3679
3680         qemu_deviceadd($vmid, $devicefull);
3681         qemu_deviceaddverify($vmid, $deviceid);
3682
3683     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3684
3685         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3686         qemu_driveadd($storecfg, $vmid, $device);
3687
3688         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3689         eval { qemu_deviceadd($vmid, $devicefull); };
3690         if (my $err = $@) {
3691             eval { qemu_drivedel($vmid, $deviceid); };
3692             warn $@ if $@;
3693             die $err;
3694         }
3695
3696     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3697
3698         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3699
3700         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
3701         my $use_old_bios_files = undef;
3702         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3703
3704         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3705         qemu_deviceadd($vmid, $netdevicefull);
3706         eval { qemu_deviceaddverify($vmid, $deviceid); };
3707         if (my $err = $@) {
3708             eval { qemu_netdevdel($vmid, $deviceid); };
3709             warn $@ if $@;
3710             die $err;
3711         }
3712
3713     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3714
3715         my $bridgeid = $2;
3716         my $pciaddr = print_pci_addr($deviceid);
3717         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3718
3719         qemu_deviceadd($vmid, $devicefull);
3720         qemu_deviceaddverify($vmid, $deviceid);
3721
3722     } else {
3723         die "can't hotplug device '$deviceid'\n";
3724     }
3725
3726     return 1;
3727 }
3728
3729 # fixme: this should raise exceptions on error!
3730 sub vm_deviceunplug {
3731     my ($vmid, $conf, $deviceid) = @_;
3732
3733     my $devices_list = vm_devices_list($vmid);
3734     return 1 if !defined($devices_list->{$deviceid});
3735
3736     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3737
3738     if ($deviceid eq 'tablet') {
3739
3740         qemu_devicedel($vmid, $deviceid);
3741
3742     } elsif ($deviceid =~ m/^usb\d+$/) {
3743
3744         die "usb hotplug currently not reliable\n";
3745         # when unplugging usb devices this way,
3746         # there may be remaining usb controllers/hubs
3747         # so we disable it for now
3748         qemu_devicedel($vmid, $deviceid);
3749         qemu_devicedelverify($vmid, $deviceid);
3750
3751     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3752
3753         qemu_devicedel($vmid, $deviceid);
3754         qemu_devicedelverify($vmid, $deviceid);
3755         qemu_drivedel($vmid, $deviceid);
3756         qemu_iothread_del($conf, $vmid, $deviceid);
3757
3758     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3759
3760         qemu_devicedel($vmid, $deviceid);
3761         qemu_devicedelverify($vmid, $deviceid);
3762         qemu_iothread_del($conf, $vmid, $deviceid);
3763
3764     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3765
3766         qemu_devicedel($vmid, $deviceid);
3767         qemu_drivedel($vmid, $deviceid);
3768         qemu_deletescsihw($conf, $vmid, $deviceid);
3769
3770     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3771
3772         qemu_devicedel($vmid, $deviceid);
3773         qemu_devicedelverify($vmid, $deviceid);
3774         qemu_netdevdel($vmid, $deviceid);
3775
3776     } else {
3777         die "can't unplug device '$deviceid'\n";
3778     }
3779
3780     return 1;
3781 }
3782
3783 sub qemu_deviceadd {
3784     my ($vmid, $devicefull) = @_;
3785
3786     $devicefull = "driver=".$devicefull;
3787     my %options =  split(/[=,]/, $devicefull);
3788
3789     vm_mon_cmd($vmid, "device_add" , %options);
3790 }
3791
3792 sub qemu_devicedel {
3793     my ($vmid, $deviceid) = @_;
3794
3795     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3796 }
3797
3798 sub qemu_iothread_add {
3799     my($vmid, $deviceid, $device) = @_;
3800
3801     if ($device->{iothread}) {
3802         my $iothreads = vm_iothreads_list($vmid);
3803         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3804     }
3805 }
3806
3807 sub qemu_iothread_del {
3808     my($conf, $vmid, $deviceid) = @_;
3809
3810     my $device = parse_drive($deviceid, $conf->{$deviceid});
3811     if ($device->{iothread}) {
3812         my $iothreads = vm_iothreads_list($vmid);
3813         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3814     }
3815 }
3816
3817 sub qemu_objectadd {
3818     my($vmid, $objectid, $qomtype) = @_;
3819
3820     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3821
3822     return 1;
3823 }
3824
3825 sub qemu_objectdel {
3826     my($vmid, $objectid) = @_;
3827
3828     vm_mon_cmd($vmid, "object-del", id => $objectid);
3829
3830     return 1;
3831 }
3832
3833 sub qemu_driveadd {
3834     my ($storecfg, $vmid, $device) = @_;
3835
3836     my $drive = print_drive_full($storecfg, $vmid, $device);
3837     $drive =~ s/\\/\\\\/g;
3838     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3839
3840     # If the command succeeds qemu prints: "OK"
3841     return 1 if $ret =~ m/OK/s;
3842
3843     die "adding drive failed: $ret\n";
3844 }
3845
3846 sub qemu_drivedel {
3847     my($vmid, $deviceid) = @_;
3848
3849     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3850     $ret =~ s/^\s+//;
3851
3852     return 1 if $ret eq "";
3853
3854     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3855     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3856
3857     die "deleting drive $deviceid failed : $ret\n";
3858 }
3859
3860 sub qemu_deviceaddverify {
3861     my ($vmid, $deviceid) = @_;
3862
3863     for (my $i = 0; $i <= 5; $i++) {
3864          my $devices_list = vm_devices_list($vmid);
3865          return 1 if defined($devices_list->{$deviceid});
3866          sleep 1;
3867     }
3868
3869     die "error on hotplug device '$deviceid'\n";
3870 }
3871
3872
3873 sub qemu_devicedelverify {
3874     my ($vmid, $deviceid) = @_;
3875
3876     # need to verify that the device is correctly removed as device_del
3877     # is async and empty return is not reliable
3878
3879     for (my $i = 0; $i <= 5; $i++) {
3880          my $devices_list = vm_devices_list($vmid);
3881          return 1 if !defined($devices_list->{$deviceid});
3882          sleep 1;
3883     }
3884
3885     die "error on hot-unplugging device '$deviceid'\n";
3886 }
3887
3888 sub qemu_findorcreatescsihw {
3889     my ($storecfg, $conf, $vmid, $device) = @_;
3890
3891     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3892
3893     my $scsihwid="$controller_prefix$controller";
3894     my $devices_list = vm_devices_list($vmid);
3895
3896     if(!defined($devices_list->{$scsihwid})) {
3897         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3898     }
3899
3900     return 1;
3901 }
3902
3903 sub qemu_deletescsihw {
3904     my ($conf, $vmid, $opt) = @_;
3905
3906     my $device = parse_drive($opt, $conf->{$opt});
3907
3908     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3909         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3910         return 1;
3911     }
3912
3913     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3914
3915     my $devices_list = vm_devices_list($vmid);
3916     foreach my $opt (keys %{$devices_list}) {
3917         if (PVE::QemuServer::is_valid_drivename($opt)) {
3918             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3919             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3920                 return 1;
3921             }
3922         }
3923     }
3924
3925     my $scsihwid="scsihw$controller";
3926
3927     vm_deviceunplug($vmid, $conf, $scsihwid);
3928
3929     return 1;
3930 }
3931
3932 sub qemu_add_pci_bridge {
3933     my ($storecfg, $conf, $vmid, $device) = @_;
3934
3935     my $bridges = {};
3936
3937     my $bridgeid;
3938
3939     print_pci_addr($device, $bridges);
3940
3941     while (my ($k, $v) = each %$bridges) {
3942         $bridgeid = $k;
3943     }
3944     return 1 if !defined($bridgeid) || $bridgeid < 1;
3945
3946     my $bridge = "pci.$bridgeid";
3947     my $devices_list = vm_devices_list($vmid);
3948
3949     if (!defined($devices_list->{$bridge})) {
3950         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3951     }
3952
3953     return 1;
3954 }
3955
3956 sub qemu_set_link_status {
3957     my ($vmid, $device, $up) = @_;
3958
3959     vm_mon_cmd($vmid, "set_link", name => $device,
3960                up => $up ? JSON::true : JSON::false);
3961 }
3962
3963 sub qemu_netdevadd {
3964     my ($vmid, $conf, $device, $deviceid) = @_;
3965
3966     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
3967     my %options =  split(/[=,]/, $netdev);
3968
3969     vm_mon_cmd($vmid, "netdev_add",  %options);
3970     return 1;
3971 }
3972
3973 sub qemu_netdevdel {
3974     my ($vmid, $deviceid) = @_;
3975
3976     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3977 }
3978
3979 sub qemu_usb_hotplug {
3980     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3981
3982     return if !$device;
3983
3984     # remove the old one first
3985     vm_deviceunplug($vmid, $conf, $deviceid);
3986
3987     # check if xhci controller is necessary and available
3988     if ($device->{usb3}) {
3989
3990         my $devicelist = vm_devices_list($vmid);
3991
3992         if (!$devicelist->{xhci}) {
3993             my $pciaddr = print_pci_addr("xhci");
3994             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
3995         }
3996     }
3997     my $d = parse_usb_device($device->{host});
3998     $d->{usb3} = $device->{usb3};
3999
4000     # add the new one
4001     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d);
4002 }
4003
4004 sub qemu_cpu_hotplug {
4005     my ($vmid, $conf, $vcpus) = @_;
4006
4007     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
4008
4009     my $sockets = 1;
4010     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4011     $sockets = $conf->{sockets} if  $conf->{sockets};
4012     my $cores = $conf->{cores} || 1;
4013     my $maxcpus = $sockets * $cores;
4014
4015     $vcpus = $maxcpus if !$vcpus;
4016
4017     die "you can't add more vcpus than maxcpus\n"
4018         if $vcpus > $maxcpus;
4019
4020     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4021
4022     if ($vcpus < $currentvcpus) {
4023
4024         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4025
4026             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4027                 qemu_devicedel($vmid, "cpu$i");
4028                 my $retry = 0;
4029                 my $currentrunningvcpus = undef;
4030                 while (1) {
4031                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4032                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4033                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4034                     $retry++;
4035                     sleep 1;
4036                 }
4037                 #update conf after each succesfull cpu unplug
4038                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4039                 PVE::QemuConfig->write_config($vmid, $conf);
4040             }
4041         } else {
4042             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4043         }
4044
4045         return;
4046     }
4047
4048     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4049     die "vcpus in running vm does not match its configuration\n"
4050         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4051
4052     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4053
4054         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4055             my $cpustr = print_cpu_device($conf, $i);
4056             qemu_deviceadd($vmid, $cpustr);
4057
4058             my $retry = 0;
4059             my $currentrunningvcpus = undef;
4060             while (1) {
4061                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4062                 last if scalar(@{$currentrunningvcpus}) == $i;
4063                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4064                 sleep 1;
4065                 $retry++;
4066             }
4067             #update conf after each succesfull cpu hotplug
4068             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4069             PVE::QemuConfig->write_config($vmid, $conf);
4070         }
4071     } else {
4072
4073         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4074             vm_mon_cmd($vmid, "cpu-add", id => int($i));
4075         }
4076     }
4077 }
4078
4079 sub qemu_block_set_io_throttle {
4080     my ($vmid, $deviceid,
4081         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4082         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4083         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4084         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4085
4086     return if !check_running($vmid) ;
4087
4088     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4089         bps => int($bps),
4090         bps_rd => int($bps_rd),
4091         bps_wr => int($bps_wr),
4092         iops => int($iops),
4093         iops_rd => int($iops_rd),
4094         iops_wr => int($iops_wr),
4095         bps_max => int($bps_max),
4096         bps_rd_max => int($bps_rd_max),
4097         bps_wr_max => int($bps_wr_max),
4098         iops_max => int($iops_max),
4099         iops_rd_max => int($iops_rd_max),
4100         iops_wr_max => int($iops_wr_max),
4101         bps_max_length => int($bps_max_length),
4102         bps_rd_max_length => int($bps_rd_max_length),
4103         bps_wr_max_length => int($bps_wr_max_length),
4104         iops_max_length => int($iops_max_length),
4105         iops_rd_max_length => int($iops_rd_max_length),
4106         iops_wr_max_length => int($iops_wr_max_length),
4107     );
4108
4109 }
4110
4111 # old code, only used to shutdown old VM after update
4112 sub __read_avail {
4113     my ($fh, $timeout) = @_;
4114
4115     my $sel = new IO::Select;
4116     $sel->add($fh);
4117
4118     my $res = '';
4119     my $buf;
4120
4121     my @ready;
4122     while (scalar (@ready = $sel->can_read($timeout))) {
4123         my $count;
4124         if ($count = $fh->sysread($buf, 8192)) {
4125             if ($buf =~ /^(.*)\(qemu\) $/s) {
4126                 $res .= $1;
4127                 last;
4128             } else {
4129                 $res .= $buf;
4130             }
4131         } else {
4132             if (!defined($count)) {
4133                 die "$!\n";
4134             }
4135             last;
4136         }
4137     }
4138
4139     die "monitor read timeout\n" if !scalar(@ready);
4140
4141     return $res;
4142 }
4143
4144 # old code, only used to shutdown old VM after update
4145 sub vm_monitor_command {
4146     my ($vmid, $cmdstr, $nocheck) = @_;
4147
4148     my $res;
4149
4150     eval {
4151         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4152
4153         my $sname = "${var_run_tmpdir}/$vmid.mon";
4154
4155         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
4156             die "unable to connect to VM $vmid socket - $!\n";
4157
4158         my $timeout = 3;
4159
4160         # hack: migrate sometime blocks the monitor (when migrate_downtime
4161         # is set)
4162         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
4163             $timeout = 60*60; # 1 hour
4164         }
4165
4166         # read banner;
4167         my $data = __read_avail($sock, $timeout);
4168
4169         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
4170             die "got unexpected qemu monitor banner\n";
4171         }
4172
4173         my $sel = new IO::Select;
4174         $sel->add($sock);
4175
4176         if (!scalar(my @ready = $sel->can_write($timeout))) {
4177             die "monitor write error - timeout";
4178         }
4179
4180         my $fullcmd = "$cmdstr\r";
4181
4182         # syslog('info', "VM $vmid monitor command: $cmdstr");
4183
4184         my $b;
4185         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
4186             die "monitor write error - $!";
4187         }
4188
4189         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
4190
4191         $timeout = 20;
4192
4193         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
4194             $timeout = 60*60; # 1 hour
4195         } elsif ($cmdstr =~ m/^(eject|change)/) {
4196             $timeout = 60; # note: cdrom mount command is slow
4197         }
4198         if ($res = __read_avail($sock, $timeout)) {
4199
4200             my @lines = split("\r?\n", $res);
4201
4202             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
4203
4204             $res = join("\n", @lines);
4205             $res .= "\n";
4206         }
4207     };
4208
4209     my $err = $@;
4210
4211     if ($err) {
4212         syslog("err", "VM $vmid monitor command failed - $err");
4213         die $err;
4214     }
4215
4216     return $res;
4217 }
4218
4219 sub qemu_block_resize {
4220     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4221
4222     my $running = check_running($vmid);
4223
4224     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4225
4226     return if !$running;
4227
4228     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4229
4230 }
4231
4232 sub qemu_volume_snapshot {
4233     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4234
4235     my $running = check_running($vmid);
4236
4237     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4238         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
4239     } else {
4240         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4241     }
4242 }
4243
4244 sub qemu_volume_snapshot_delete {
4245     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4246
4247     my $running = check_running($vmid);
4248
4249     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4250         vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
4251     } else {
4252         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4253     }
4254 }
4255
4256 sub set_migration_caps {
4257     my ($vmid) = @_;
4258
4259     my $cap_ref = [];
4260
4261     my $enabled_cap = {
4262         "auto-converge" => 1,
4263         "xbzrle" => 1,
4264         "x-rdma-pin-all" => 0,
4265         "zero-blocks" => 0,
4266         "compress" => 0
4267     };
4268
4269     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4270
4271     for my $supported_capability (@$supported_capabilities) {
4272         push @$cap_ref, {
4273             capability => $supported_capability->{capability},
4274             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4275         };
4276     }
4277
4278     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4279 }
4280
4281 my $fast_plug_option = {
4282     'lock' => 1,
4283     'name' => 1,
4284     'onboot' => 1,
4285     'shares' => 1,
4286     'startup' => 1,
4287     'description' => 1,
4288     'protection' => 1,
4289     'vmstatestorage' => 1,
4290 };
4291
4292 # hotplug changes in [PENDING]
4293 # $selection hash can be used to only apply specified options, for
4294 # example: { cores => 1 } (only apply changed 'cores')
4295 # $errors ref is used to return error messages
4296 sub vmconfig_hotplug_pending {
4297     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4298
4299     my $defaults = load_defaults();
4300
4301     # commit values which do not have any impact on running VM first
4302     # Note: those option cannot raise errors, we we do not care about
4303     # $selection and always apply them.
4304
4305     my $add_error = sub {
4306         my ($opt, $msg) = @_;
4307         $errors->{$opt} = "hotplug problem - $msg";
4308     };
4309
4310     my $changes = 0;
4311     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4312         if ($fast_plug_option->{$opt}) {
4313             $conf->{$opt} = $conf->{pending}->{$opt};
4314             delete $conf->{pending}->{$opt};
4315             $changes = 1;
4316         }
4317     }
4318
4319     if ($changes) {
4320         PVE::QemuConfig->write_config($vmid, $conf);
4321         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4322     }
4323
4324     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4325
4326     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4327     while (my ($opt, $force) = each %$pending_delete_hash) {
4328         next if $selection && !$selection->{$opt};
4329         eval {
4330             if ($opt eq 'hotplug') {
4331                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4332             } elsif ($opt eq 'tablet') {
4333                 die "skip\n" if !$hotplug_features->{usb};
4334                 if ($defaults->{tablet}) {
4335                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4336                 } else {
4337                     vm_deviceunplug($vmid, $conf, $opt);
4338                 }
4339             } elsif ($opt =~ m/^usb\d+/) {
4340                 die "skip\n";
4341                 # since we cannot reliably hot unplug usb devices
4342                 # we are disabling it
4343                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4344                 vm_deviceunplug($vmid, $conf, $opt);
4345             } elsif ($opt eq 'vcpus') {
4346                 die "skip\n" if !$hotplug_features->{cpu};
4347                 qemu_cpu_hotplug($vmid, $conf, undef);
4348             } elsif ($opt eq 'balloon') {
4349                 # enable balloon device is not hotpluggable
4350                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
4351             } elsif ($fast_plug_option->{$opt}) {
4352                 # do nothing
4353             } elsif ($opt =~ m/^net(\d+)$/) {
4354                 die "skip\n" if !$hotplug_features->{network};
4355                 vm_deviceunplug($vmid, $conf, $opt);
4356             } elsif (is_valid_drivename($opt)) {
4357                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4358                 vm_deviceunplug($vmid, $conf, $opt);
4359                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4360             } elsif ($opt =~ m/^memory$/) {
4361                 die "skip\n" if !$hotplug_features->{memory};
4362                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4363             } elsif ($opt eq 'cpuunits') {
4364                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4365             } elsif ($opt eq 'cpulimit') {
4366                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4367             } else {
4368                 die "skip\n";
4369             }
4370         };
4371         if (my $err = $@) {
4372             &$add_error($opt, $err) if $err ne "skip\n";
4373         } else {
4374             # save new config if hotplug was successful
4375             delete $conf->{$opt};
4376             vmconfig_undelete_pending_option($conf, $opt);
4377             PVE::QemuConfig->write_config($vmid, $conf);
4378             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4379         }
4380     }
4381
4382     foreach my $opt (keys %{$conf->{pending}}) {
4383         next if $selection && !$selection->{$opt};
4384         my $value = $conf->{pending}->{$opt};
4385         eval {
4386             if ($opt eq 'hotplug') {
4387                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4388             } elsif ($opt eq 'tablet') {
4389                 die "skip\n" if !$hotplug_features->{usb};
4390                 if ($value == 1) {
4391                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4392                 } elsif ($value == 0) {
4393                     vm_deviceunplug($vmid, $conf, $opt);
4394                 }
4395             } elsif ($opt =~ m/^usb\d+$/) {
4396                 die "skip\n";
4397                 # since we cannot reliably hot unplug usb devices
4398                 # we are disabling it
4399                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4400                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4401                 die "skip\n" if !$d;
4402                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d);
4403             } elsif ($opt eq 'vcpus') {
4404                 die "skip\n" if !$hotplug_features->{cpu};
4405                 qemu_cpu_hotplug($vmid, $conf, $value);
4406             } elsif ($opt eq 'balloon') {
4407                 # enable/disable balloning device is not hotpluggable
4408                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4409                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4410                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4411
4412                 # allow manual ballooning if shares is set to zero
4413                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4414                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4415                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4416                 }
4417             } elsif ($opt =~ m/^net(\d+)$/) {
4418                 # some changes can be done without hotplug
4419                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4420                                     $vmid, $opt, $value);
4421             } elsif (is_valid_drivename($opt)) {
4422                 # some changes can be done without hotplug
4423                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4424                                      $vmid, $opt, $value, 1);
4425             } elsif ($opt =~ m/^memory$/) { #dimms
4426                 die "skip\n" if !$hotplug_features->{memory};
4427                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4428             } elsif ($opt eq 'cpuunits') {
4429                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4430             } elsif ($opt eq 'cpulimit') {
4431                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4432                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4433             } else {
4434                 die "skip\n";  # skip non-hot-pluggable options
4435             }
4436         };
4437         if (my $err = $@) {
4438             &$add_error($opt, $err) if $err ne "skip\n";
4439         } else {
4440             # save new config if hotplug was successful
4441             $conf->{$opt} = $value;
4442             delete $conf->{pending}->{$opt};
4443             PVE::QemuConfig->write_config($vmid, $conf);
4444             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4445         }
4446     }
4447 }
4448
4449 sub try_deallocate_drive {
4450     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4451
4452     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4453         my $volid = $drive->{file};
4454         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4455             my $sid = PVE::Storage::parse_volume_id($volid);
4456             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4457
4458             # check if the disk is really unused
4459             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4460                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4461             PVE::Storage::vdisk_free($storecfg, $volid);
4462             return 1;
4463         } else {
4464             # If vm is not owner of this disk remove from config
4465             return 1;
4466         }
4467     }
4468
4469     return undef;
4470 }
4471
4472 sub vmconfig_delete_or_detach_drive {
4473     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4474
4475     my $drive = parse_drive($opt, $conf->{$opt});
4476
4477     my $rpcenv = PVE::RPCEnvironment::get();
4478     my $authuser = $rpcenv->get_user();
4479
4480     if ($force) {
4481         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4482         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4483     } else {
4484         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4485     }
4486 }
4487
4488 sub vmconfig_apply_pending {
4489     my ($vmid, $conf, $storecfg) = @_;
4490
4491     # cold plug
4492
4493     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4494     while (my ($opt, $force) = each %$pending_delete_hash) {
4495         die "internal error" if $opt =~ m/^unused/;
4496         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4497         if (!defined($conf->{$opt})) {
4498             vmconfig_undelete_pending_option($conf, $opt);
4499             PVE::QemuConfig->write_config($vmid, $conf);
4500         } elsif (is_valid_drivename($opt)) {
4501             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4502             vmconfig_undelete_pending_option($conf, $opt);
4503             delete $conf->{$opt};
4504             PVE::QemuConfig->write_config($vmid, $conf);
4505         } else {
4506             vmconfig_undelete_pending_option($conf, $opt);
4507             delete $conf->{$opt};
4508             PVE::QemuConfig->write_config($vmid, $conf);
4509         }
4510     }
4511
4512     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4513
4514     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4515         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4516
4517         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4518             # skip if nothing changed
4519         } elsif (is_valid_drivename($opt)) {
4520             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4521                 if defined($conf->{$opt});
4522             $conf->{$opt} = $conf->{pending}->{$opt};
4523         } else {
4524             $conf->{$opt} = $conf->{pending}->{$opt};
4525         }
4526
4527         delete $conf->{pending}->{$opt};
4528         PVE::QemuConfig->write_config($vmid, $conf);
4529     }
4530 }
4531
4532 my $safe_num_ne = sub {
4533     my ($a, $b) = @_;
4534
4535     return 0 if !defined($a) && !defined($b);
4536     return 1 if !defined($a);
4537     return 1 if !defined($b);
4538
4539     return $a != $b;
4540 };
4541
4542 my $safe_string_ne = sub {
4543     my ($a, $b) = @_;
4544
4545     return 0 if !defined($a) && !defined($b);
4546     return 1 if !defined($a);
4547     return 1 if !defined($b);
4548
4549     return $a ne $b;
4550 };
4551
4552 sub vmconfig_update_net {
4553     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4554
4555     my $newnet = parse_net($value);
4556
4557     if ($conf->{$opt}) {
4558         my $oldnet = parse_net($conf->{$opt});
4559
4560         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4561             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4562             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4563             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4564
4565             # for non online change, we try to hot-unplug
4566             die "skip\n" if !$hotplug;
4567             vm_deviceunplug($vmid, $conf, $opt);
4568         } else {
4569
4570             die "internal error" if $opt !~ m/net(\d+)/;
4571             my $iface = "tap${vmid}i$1";
4572
4573             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4574                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4575                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4576                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4577                 PVE::Network::tap_unplug($iface);
4578                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4579             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4580                 # Rate can be applied on its own but any change above needs to
4581                 # include the rate in tap_plug since OVS resets everything.
4582                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4583             }
4584
4585             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4586                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4587             }
4588
4589             return 1;
4590         }
4591     }
4592
4593     if ($hotplug) {
4594         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4595     } else {
4596         die "skip\n";
4597     }
4598 }
4599
4600 sub vmconfig_update_disk {
4601     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4602
4603     # fixme: do we need force?
4604
4605     my $drive = parse_drive($opt, $value);
4606
4607     if ($conf->{$opt}) {
4608
4609         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4610
4611             my $media = $drive->{media} || 'disk';
4612             my $oldmedia = $old_drive->{media} || 'disk';
4613             die "unable to change media type\n" if $media ne $oldmedia;
4614
4615             if (!drive_is_cdrom($old_drive)) {
4616
4617                 if ($drive->{file} ne $old_drive->{file}) {
4618
4619                     die "skip\n" if !$hotplug;
4620
4621                     # unplug and register as unused
4622                     vm_deviceunplug($vmid, $conf, $opt);
4623                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4624
4625                 } else {
4626                     # update existing disk
4627
4628                     # skip non hotpluggable value
4629                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4630                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4631                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4632                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4633                         die "skip\n";
4634                     }
4635
4636                     # apply throttle
4637                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4638                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4639                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4640                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4641                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4642                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4643                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4644                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4645                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4646                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4647                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4648                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4649                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4650                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4651                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4652                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4653                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4654                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4655
4656                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4657                                                    ($drive->{mbps} || 0)*1024*1024,
4658                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4659                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4660                                                    $drive->{iops} || 0,
4661                                                    $drive->{iops_rd} || 0,
4662                                                    $drive->{iops_wr} || 0,
4663                                                    ($drive->{mbps_max} || 0)*1024*1024,
4664                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4665                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4666                                                    $drive->{iops_max} || 0,
4667                                                    $drive->{iops_rd_max} || 0,
4668                                                    $drive->{iops_wr_max} || 0,
4669                                                    $drive->{bps_max_length} || 1,
4670                                                    $drive->{bps_rd_max_length} || 1,
4671                                                    $drive->{bps_wr_max_length} || 1,
4672                                                    $drive->{iops_max_length} || 1,
4673                                                    $drive->{iops_rd_max_length} || 1,
4674                                                    $drive->{iops_wr_max_length} || 1);
4675
4676                     }
4677
4678                     return 1;
4679                 }
4680
4681             } else { # cdrom
4682
4683                 if ($drive->{file} eq 'none') {
4684                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4685                     if (drive_is_cloudinit($old_drive)) {
4686                         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
4687                     }
4688                 } else {
4689                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4690                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4691                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4692                 }
4693
4694                 return 1;
4695             }
4696         }
4697     }
4698
4699     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4700     # hotplug new disks
4701     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4702     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4703 }
4704
4705 sub vm_start {
4706     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4707         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
4708
4709     PVE::QemuConfig->lock_config($vmid, sub {
4710         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4711
4712         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
4713
4714         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4715
4716         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4717
4718         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4719             vmconfig_apply_pending($vmid, $conf, $storecfg);
4720             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4721         }
4722
4723         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
4724
4725         my $defaults = load_defaults();
4726
4727         # set environment variable useful inside network script
4728         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4729
4730         my $local_volumes = {};
4731
4732         if ($targetstorage) {
4733             foreach_drive($conf, sub {
4734                 my ($ds, $drive) = @_;
4735
4736                 return if drive_is_cdrom($drive);
4737
4738                 my $volid = $drive->{file};
4739
4740                 return if !$volid;
4741
4742                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4743
4744                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4745                 return if $scfg->{shared};
4746                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
4747             });
4748
4749             my $format = undef;
4750
4751             foreach my $opt (sort keys %$local_volumes) {
4752
4753                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
4754                 my $drive = parse_drive($opt, $conf->{$opt});
4755
4756                 #if remote storage is specified, use default format
4757                 if ($targetstorage && $targetstorage ne "1") {
4758                     $storeid = $targetstorage;
4759                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
4760                     $format = $defFormat;
4761                 } else {
4762                     #else we use same format than original
4763                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4764                     $format = qemu_img_format($scfg, $volid);
4765                 }
4766
4767                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
4768                 my $newdrive = $drive;
4769                 $newdrive->{format} = $format;
4770                 $newdrive->{file} = $newvolid;
4771                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
4772                 $local_volumes->{$opt} = $drivestr;
4773                 #pass drive to conf for command line
4774                 $conf->{$opt} = $drivestr;
4775             }
4776         }
4777
4778         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4779
4780         my $migrate_port = 0;
4781         my $migrate_uri;
4782         if ($statefile) {
4783             if ($statefile eq 'tcp') {
4784                 my $localip = "localhost";
4785                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4786                 my $nodename = PVE::INotify::nodename();
4787
4788                 if (!defined($migration_type)) {
4789                     if (defined($datacenterconf->{migration}->{type})) {
4790                         $migration_type = $datacenterconf->{migration}->{type};
4791                     } else {
4792                         $migration_type = 'secure';
4793                     }
4794                 }
4795
4796                 if ($migration_type eq 'insecure') {
4797                     my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4798                     if ($migrate_network_addr) {
4799                         $localip = $migrate_network_addr;
4800                     } else {
4801                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4802                     }
4803
4804                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4805                 }
4806
4807                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4808                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4809                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4810                 push @$cmd, '-incoming', $migrate_uri;
4811                 push @$cmd, '-S';
4812
4813             } elsif ($statefile eq 'unix') {
4814                 # should be default for secure migrations as a ssh TCP forward
4815                 # tunnel is not deterministic reliable ready and fails regurarly
4816                 # to set up in time, so use UNIX socket forwards
4817                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
4818                 unlink $socket_addr;
4819
4820                 $migrate_uri = "unix:$socket_addr";
4821
4822                 push @$cmd, '-incoming', $migrate_uri;
4823                 push @$cmd, '-S';
4824
4825             } else {
4826                 push @$cmd, '-loadstate', $statefile;
4827             }
4828         } elsif ($paused) {
4829             push @$cmd, '-S';
4830         }
4831
4832         # host pci devices
4833         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4834           my $d = parse_hostpci($conf->{"hostpci$i"});
4835           next if !$d;
4836           my $pcidevices = $d->{pciid};
4837           foreach my $pcidevice (@$pcidevices) {
4838                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4839
4840                 my $info = pci_device_info("0000:$pciid");
4841                 die "IOMMU not present\n" if !check_iommu_support();
4842                 die "no pci device info for device '$pciid'\n" if !$info;
4843                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4844                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4845           }
4846         }
4847
4848         PVE::Storage::activate_volumes($storecfg, $vollist);
4849
4850         if (!check_running($vmid, 1) && -d "/sys/fs/cgroup/systemd/qemu.slice/$vmid.scope") {
4851             my $cmd = [];
4852             push @$cmd, '/bin/systemctl', 'stop', "$vmid.scope";
4853             eval  { run_command($cmd); };
4854         }
4855
4856         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
4857                                                   : $defaults->{cpuunits};
4858
4859         my $start_timeout = $conf->{hugepages} ? 300 : 30;
4860         my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077);
4861
4862         my %properties = (
4863             Slice => 'qemu.slice',
4864             KillMode => 'none',
4865             CPUShares => $cpuunits
4866         );
4867
4868         if (my $cpulimit = $conf->{cpulimit}) {
4869             $properties{CPUQuota} = int($cpulimit * 100);
4870         }
4871         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
4872
4873         if ($conf->{hugepages}) {
4874
4875             my $code = sub {
4876                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
4877                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
4878
4879                 PVE::QemuServer::Memory::hugepages_mount();
4880                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
4881
4882                 eval  {
4883                     PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4884                     run_command($cmd, %run_params);
4885                 };
4886
4887                 if (my $err = $@) {
4888                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
4889                     die $err;
4890                 }
4891
4892                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
4893             };
4894             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
4895
4896         } else {
4897             eval  {
4898                 PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4899                 run_command($cmd, %run_params);
4900             };
4901         }
4902
4903         if (my $err = $@) {
4904             # deactivate volumes if start fails
4905             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
4906             die "start failed: $err";
4907         }
4908
4909         print "migration listens on $migrate_uri\n" if $migrate_uri;
4910
4911         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
4912             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4913             warn $@ if $@;
4914         }
4915
4916         #start nbd server for storage migration
4917         if ($targetstorage) {
4918             my $nodename = PVE::INotify::nodename();
4919             my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4920             my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
4921             my $pfamily = PVE::Tools::get_host_address_family($nodename);
4922             $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4923
4924             vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${migrate_port}" } } );
4925
4926             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4927
4928             foreach my $opt (sort keys %$local_volumes) {
4929                 my $volid = $local_volumes->{$opt};
4930                 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
4931                 my $migrate_storage_uri = "nbd:${localip}:${migrate_port}:exportname=drive-$opt";
4932                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
4933             }
4934         }
4935
4936         if ($migratedfrom) {
4937             eval {
4938                 set_migration_caps($vmid);
4939             };
4940             warn $@ if $@;
4941
4942             if ($spice_port) {
4943                 print "spice listens on port $spice_port\n";
4944                 if ($spice_ticket) {
4945                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4946                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4947                 }
4948             }
4949
4950         } else {
4951             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4952                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4953                     if $conf->{balloon};
4954             }
4955
4956             foreach my $opt (keys %$conf) {
4957                 next if $opt !~  m/^net\d+$/;
4958                 my $nicconf = parse_net($conf->{$opt});
4959                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4960             }
4961         }
4962
4963         vm_mon_cmd_nocheck($vmid, 'qom-set',
4964                     path => "machine/peripheral/balloon0",
4965                     property => "guest-stats-polling-interval",
4966                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4967
4968     });
4969 }
4970
4971 sub vm_mon_cmd {
4972     my ($vmid, $execute, %params) = @_;
4973
4974     my $cmd = { execute => $execute, arguments => \%params };
4975     vm_qmp_command($vmid, $cmd);
4976 }
4977
4978 sub vm_mon_cmd_nocheck {
4979     my ($vmid, $execute, %params) = @_;
4980
4981     my $cmd = { execute => $execute, arguments => \%params };
4982     vm_qmp_command($vmid, $cmd, 1);
4983 }
4984
4985 sub vm_qmp_command {
4986     my ($vmid, $cmd, $nocheck) = @_;
4987
4988     my $res;
4989
4990     my $timeout;
4991     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4992         $timeout = $cmd->{arguments}->{timeout};
4993         delete $cmd->{arguments}->{timeout};
4994     }
4995
4996     eval {
4997         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4998         my $sname = qmp_socket($vmid);
4999         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
5000             my $qmpclient = PVE::QMPClient->new();
5001
5002             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
5003         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
5004             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
5005                 if scalar(%{$cmd->{arguments}});
5006             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
5007         } else {
5008             die "unable to open monitor socket\n";
5009         }
5010     };
5011     if (my $err = $@) {
5012         syslog("err", "VM $vmid qmp command failed - $err");
5013         die $err;
5014     }
5015
5016     return $res;
5017 }
5018
5019 sub vm_human_monitor_command {
5020     my ($vmid, $cmdline) = @_;
5021
5022     my $res;
5023
5024     my $cmd = {
5025         execute => 'human-monitor-command',
5026         arguments => { 'command-line' => $cmdline},
5027     };
5028
5029     return vm_qmp_command($vmid, $cmd);
5030 }
5031
5032 sub vm_commandline {
5033     my ($storecfg, $vmid) = @_;
5034
5035     my $conf = PVE::QemuConfig->load_config($vmid);
5036
5037     my $defaults = load_defaults();
5038
5039     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
5040
5041     return PVE::Tools::cmd2string($cmd);
5042 }
5043
5044 sub vm_reset {
5045     my ($vmid, $skiplock) = @_;
5046
5047     PVE::QemuConfig->lock_config($vmid, sub {
5048
5049         my $conf = PVE::QemuConfig->load_config($vmid);
5050
5051         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5052
5053         vm_mon_cmd($vmid, "system_reset");
5054     });
5055 }
5056
5057 sub get_vm_volumes {
5058     my ($conf) = @_;
5059
5060     my $vollist = [];
5061     foreach_volid($conf, sub {
5062         my ($volid, $attr) = @_;
5063
5064         return if $volid =~ m|^/|;
5065
5066         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5067         return if !$sid;
5068
5069         push @$vollist, $volid;
5070     });
5071
5072     return $vollist;
5073 }
5074
5075 sub vm_stop_cleanup {
5076     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5077
5078     eval {
5079
5080         if (!$keepActive) {
5081             my $vollist = get_vm_volumes($conf);
5082             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5083         }
5084
5085         foreach my $ext (qw(mon qmp pid vnc qga)) {
5086             unlink "/var/run/qemu-server/${vmid}.$ext";
5087         }
5088
5089         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5090     };
5091     warn $@ if $@; # avoid errors - just warn
5092 }
5093
5094 # Note: use $nockeck to skip tests if VM configuration file exists.
5095 # We need that when migration VMs to other nodes (files already moved)
5096 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5097 sub vm_stop {
5098     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5099
5100     $force = 1 if !defined($force) && !$shutdown;
5101
5102     if ($migratedfrom){
5103         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5104         kill 15, $pid if $pid;
5105         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5106         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5107         return;
5108     }
5109
5110     PVE::QemuConfig->lock_config($vmid, sub {
5111
5112         my $pid = check_running($vmid, $nocheck);
5113         return if !$pid;
5114
5115         my $conf;
5116         if (!$nocheck) {
5117             $conf = PVE::QemuConfig->load_config($vmid);
5118             PVE::QemuConfig->check_lock($conf) if !$skiplock;
5119             if (!defined($timeout) && $shutdown && $conf->{startup}) {
5120                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5121                 $timeout = $opts->{down} if $opts->{down};
5122             }
5123         }
5124
5125         $timeout = 60 if !defined($timeout);
5126
5127         eval {
5128             if ($shutdown) {
5129                 if (defined($conf) && $conf->{agent}) {
5130                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
5131                 } else {
5132                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
5133                 }
5134             } else {
5135                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
5136             }
5137         };
5138         my $err = $@;
5139
5140         if (!$err) {
5141             my $count = 0;
5142             while (($count < $timeout) && check_running($vmid, $nocheck)) {
5143                 $count++;
5144                 sleep 1;
5145             }
5146
5147             if ($count >= $timeout) {
5148                 if ($force) {
5149                     warn "VM still running - terminating now with SIGTERM\n";
5150                     kill 15, $pid;
5151                 } else {
5152                     die "VM quit/powerdown failed - got timeout\n";
5153                 }
5154             } else {
5155                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5156                 return;
5157             }
5158         } else {
5159             if ($force) {
5160                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5161                 kill 15, $pid;
5162             } else {
5163                 die "VM quit/powerdown failed\n";
5164             }
5165         }
5166
5167         # wait again
5168         $timeout = 10;
5169
5170         my $count = 0;
5171         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5172             $count++;
5173             sleep 1;
5174         }
5175
5176         if ($count >= $timeout) {
5177             warn "VM still running - terminating now with SIGKILL\n";
5178             kill 9, $pid;
5179             sleep 1;
5180         }
5181
5182         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5183    });
5184 }
5185
5186 sub vm_suspend {
5187     my ($vmid, $skiplock) = @_;
5188
5189     PVE::QemuConfig->lock_config($vmid, sub {
5190
5191         my $conf = PVE::QemuConfig->load_config($vmid);
5192
5193         PVE::QemuConfig->check_lock($conf)
5194             if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5195
5196         vm_mon_cmd($vmid, "stop");
5197     });
5198 }
5199
5200 sub vm_resume {
5201     my ($vmid, $skiplock, $nocheck) = @_;
5202
5203     PVE::QemuConfig->lock_config($vmid, sub {
5204
5205         if (!$nocheck) {
5206
5207             my $conf = PVE::QemuConfig->load_config($vmid);
5208
5209             PVE::QemuConfig->check_lock($conf)
5210                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5211
5212             vm_mon_cmd($vmid, "cont");
5213
5214         } else {
5215             vm_mon_cmd_nocheck($vmid, "cont");
5216         }
5217     });
5218 }
5219
5220 sub vm_sendkey {
5221     my ($vmid, $skiplock, $key) = @_;
5222
5223     PVE::QemuConfig->lock_config($vmid, sub {
5224
5225         my $conf = PVE::QemuConfig->load_config($vmid);
5226
5227         # there is no qmp command, so we use the human monitor command
5228         vm_human_monitor_command($vmid, "sendkey $key");
5229     });
5230 }
5231
5232 sub vm_destroy {
5233     my ($storecfg, $vmid, $skiplock) = @_;
5234
5235     PVE::QemuConfig->lock_config($vmid, sub {
5236
5237         my $conf = PVE::QemuConfig->load_config($vmid);
5238
5239         if (!check_running($vmid)) {
5240             destroy_vm($storecfg, $vmid, undef, $skiplock);
5241         } else {
5242             die "VM $vmid is running - destroy failed\n";
5243         }
5244     });
5245 }
5246
5247 # pci helpers
5248
5249 sub file_write {
5250     my ($filename, $buf) = @_;
5251
5252     my $fh = IO::File->new($filename, "w");
5253     return undef if !$fh;
5254
5255     my $res = print $fh $buf;
5256
5257     $fh->close();
5258
5259     return $res;
5260 }
5261
5262 sub pci_device_info {
5263     my ($name) = @_;
5264
5265     my $res;
5266
5267     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
5268     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
5269
5270     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
5271     return undef if !defined($irq) || $irq !~ m/^\d+$/;
5272
5273     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
5274     return undef if !defined($vendor) || $vendor !~ s/^0x//;
5275
5276     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
5277     return undef if !defined($product) || $product !~ s/^0x//;
5278
5279     $res = {
5280         name => $name,
5281         vendor => $vendor,
5282         product => $product,
5283         domain => $domain,
5284         bus => $bus,
5285         slot => $slot,
5286         func => $func,
5287         irq => $irq,
5288         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
5289     };
5290
5291     return $res;
5292 }
5293
5294 sub pci_dev_reset {
5295     my ($dev) = @_;
5296
5297     my $name = $dev->{name};
5298
5299     my $fn = "$pcisysfs/devices/$name/reset";
5300
5301     return file_write($fn, "1");
5302 }
5303
5304 sub pci_dev_bind_to_vfio {
5305     my ($dev) = @_;
5306
5307     my $name = $dev->{name};
5308
5309     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5310
5311     if (!-d $vfio_basedir) {
5312         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5313     }
5314     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5315
5316     my $testdir = "$vfio_basedir/$name";
5317     return 1 if -d $testdir;
5318
5319     my $data = "$dev->{vendor} $dev->{product}";
5320     return undef if !file_write("$vfio_basedir/new_id", $data);
5321
5322     my $fn = "$pcisysfs/devices/$name/driver/unbind";
5323     if (!file_write($fn, $name)) {
5324         return undef if -f $fn;
5325     }
5326
5327     $fn = "$vfio_basedir/bind";
5328     if (! -d $testdir) {
5329         return undef if !file_write($fn, $name);
5330     }
5331
5332     return -d $testdir;
5333 }
5334
5335 sub pci_dev_group_bind_to_vfio {
5336     my ($pciid) = @_;
5337
5338     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5339
5340     if (!-d $vfio_basedir) {
5341         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5342     }
5343     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5344
5345     # get IOMMU group devices
5346     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
5347       my @devs = grep /^0000:/, readdir($D);
5348     closedir($D);
5349
5350     foreach my $pciid (@devs) {
5351         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
5352
5353         # pci bridges, switches or root ports are not supported
5354         # they have a pci_bus subdirectory so skip them
5355         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
5356
5357         my $info = pci_device_info($1);
5358         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
5359     }
5360
5361     return 1;
5362 }
5363
5364 # vzdump restore implementaion
5365
5366 sub tar_archive_read_firstfile {
5367     my $archive = shift;
5368
5369     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5370
5371     # try to detect archive type first
5372     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5373         die "unable to open file '$archive'\n";
5374     my $firstfile = <$fh>;
5375     kill 15, $pid;
5376     close $fh;
5377
5378     die "ERROR: archive contaions no data\n" if !$firstfile;
5379     chomp $firstfile;
5380
5381     return $firstfile;
5382 }
5383
5384 sub tar_restore_cleanup {
5385     my ($storecfg, $statfile) = @_;
5386
5387     print STDERR "starting cleanup\n";
5388
5389     if (my $fd = IO::File->new($statfile, "r")) {
5390         while (defined(my $line = <$fd>)) {
5391             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5392                 my $volid = $2;
5393                 eval {
5394                     if ($volid =~ m|^/|) {
5395                         unlink $volid || die 'unlink failed\n';
5396                     } else {
5397                         PVE::Storage::vdisk_free($storecfg, $volid);
5398                     }
5399                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5400                 };
5401                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5402             } else {
5403                 print STDERR "unable to parse line in statfile - $line";
5404             }
5405         }
5406         $fd->close();
5407     }
5408 }
5409
5410 sub restore_archive {
5411     my ($archive, $vmid, $user, $opts) = @_;
5412
5413     my $format = $opts->{format};
5414     my $comp;
5415
5416     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5417         $format = 'tar' if !$format;
5418         $comp = 'gzip';
5419     } elsif ($archive =~ m/\.tar$/) {
5420         $format = 'tar' if !$format;
5421     } elsif ($archive =~ m/.tar.lzo$/) {
5422         $format = 'tar' if !$format;
5423         $comp = 'lzop';
5424     } elsif ($archive =~ m/\.vma$/) {
5425         $format = 'vma' if !$format;
5426     } elsif ($archive =~ m/\.vma\.gz$/) {
5427         $format = 'vma' if !$format;
5428         $comp = 'gzip';
5429     } elsif ($archive =~ m/\.vma\.lzo$/) {
5430         $format = 'vma' if !$format;
5431         $comp = 'lzop';
5432     } else {
5433         $format = 'vma' if !$format; # default
5434     }
5435
5436     # try to detect archive format
5437     if ($format eq 'tar') {
5438         return restore_tar_archive($archive, $vmid, $user, $opts);
5439     } else {
5440         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5441     }
5442 }
5443
5444 sub restore_update_config_line {
5445     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5446
5447     return if $line =~ m/^\#qmdump\#/;
5448     return if $line =~ m/^\#vzdump\#/;
5449     return if $line =~ m/^lock:/;
5450     return if $line =~ m/^unused\d+:/;
5451     return if $line =~ m/^parent:/;
5452     return if $line =~ m/^template:/; # restored VM is never a template
5453
5454     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5455     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5456         # try to convert old 1.X settings
5457         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5458         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5459             my ($model, $macaddr) = split(/\=/, $devconfig);
5460             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5461             my $net = {
5462                 model => $model,
5463                 bridge => "vmbr$ind",
5464                 macaddr => $macaddr,
5465             };
5466             my $netstr = print_net($net);
5467
5468             print $outfd "net$cookie->{netcount}: $netstr\n";
5469             $cookie->{netcount}++;
5470         }
5471     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5472         my ($id, $netstr) = ($1, $2);
5473         my $net = parse_net($netstr);
5474         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5475         $netstr = print_net($net);
5476         print $outfd "$id: $netstr\n";
5477     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5478         my $virtdev = $1;
5479         my $value = $3;
5480         my $di = parse_drive($virtdev, $value);
5481         if (defined($di->{backup}) && !$di->{backup}) {
5482             print $outfd "#$line";
5483         } elsif ($map->{$virtdev}) {
5484             delete $di->{format}; # format can change on restore
5485             $di->{file} = $map->{$virtdev};
5486             $value = print_drive($vmid, $di);
5487             print $outfd "$virtdev: $value\n";
5488         } else {
5489             print $outfd $line;
5490         }
5491     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5492         my ($uuid, $uuid_str);
5493         UUID::generate($uuid);
5494         UUID::unparse($uuid, $uuid_str);
5495         my $smbios1 = parse_smbios1($2);
5496         $smbios1->{uuid} = $uuid_str;
5497         print $outfd $1.print_smbios1($smbios1)."\n";
5498     } else {
5499         print $outfd $line;
5500     }
5501 }
5502
5503 sub scan_volids {
5504     my ($cfg, $vmid) = @_;
5505
5506     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5507
5508     my $volid_hash = {};
5509     foreach my $storeid (keys %$info) {
5510         foreach my $item (@{$info->{$storeid}}) {
5511             next if !($item->{volid} && $item->{size});
5512             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5513             $volid_hash->{$item->{volid}} = $item;
5514         }
5515     }
5516
5517     return $volid_hash;
5518 }
5519
5520 sub is_volume_in_use {
5521     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5522
5523     my $path = PVE::Storage::path($storecfg, $volid);
5524
5525     my $scan_config = sub {
5526         my ($cref, $snapname) = @_;
5527
5528         foreach my $key (keys %$cref) {
5529             my $value = $cref->{$key};
5530             if (is_valid_drivename($key)) {
5531                 next if $skip_drive && $key eq $skip_drive;
5532                 my $drive = parse_drive($key, $value);
5533                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5534                 return 1 if $volid eq $drive->{file};
5535                 if ($drive->{file} =~ m!^/!) {
5536                     return 1 if $drive->{file} eq $path;
5537                 } else {
5538                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5539                     next if !$storeid;
5540                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5541                     next if !$scfg;
5542                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5543                 }
5544             }
5545         }
5546
5547         return 0;
5548     };
5549
5550     return 1 if &$scan_config($conf);
5551
5552     undef $skip_drive;
5553
5554     foreach my $snapname (keys %{$conf->{snapshots}}) {
5555         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5556     }
5557
5558     return 0;
5559 }
5560
5561 sub update_disksize {
5562     my ($vmid, $conf, $volid_hash) = @_;
5563
5564     my $changes;
5565
5566     # used and unused disks
5567     my $referenced = {};
5568
5569     # Note: it is allowed to define multiple storages with same path (alias), so
5570     # we need to check both 'volid' and real 'path' (two different volid can point
5571     # to the same path).
5572
5573     my $referencedpath = {};
5574
5575     # update size info
5576     foreach my $opt (keys %$conf) {
5577         if (is_valid_drivename($opt)) {
5578             my $drive = parse_drive($opt, $conf->{$opt});
5579             my $volid = $drive->{file};
5580             next if !$volid;
5581
5582             $referenced->{$volid} = 1;
5583             if ($volid_hash->{$volid} &&
5584                 (my $path = $volid_hash->{$volid}->{path})) {
5585                 $referencedpath->{$path} = 1;
5586             }
5587
5588             next if drive_is_cdrom($drive);
5589             next if !$volid_hash->{$volid};
5590
5591             $drive->{size} = $volid_hash->{$volid}->{size};
5592             my $new = print_drive($vmid, $drive);
5593             if ($new ne $conf->{$opt}) {
5594                 $changes = 1;
5595                 $conf->{$opt} = $new;
5596             }
5597         }
5598     }
5599
5600     # remove 'unusedX' entry if volume is used
5601     foreach my $opt (keys %$conf) {
5602         next if $opt !~ m/^unused\d+$/;
5603         my $volid = $conf->{$opt};
5604         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5605         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
5606             $changes = 1;
5607             delete $conf->{$opt};
5608         }
5609
5610         $referenced->{$volid} = 1;
5611         $referencedpath->{$path} = 1 if $path;
5612     }
5613
5614     foreach my $volid (sort keys %$volid_hash) {
5615         next if $volid =~ m/vm-$vmid-state-/;
5616         next if $referenced->{$volid};
5617         my $path = $volid_hash->{$volid}->{path};
5618         next if !$path; # just to be sure
5619         next if $referencedpath->{$path};
5620         $changes = 1;
5621         PVE::QemuConfig->add_unused_volume($conf, $volid);
5622         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
5623     }
5624
5625     return $changes;
5626 }
5627
5628 sub rescan {
5629     my ($vmid, $nolock) = @_;
5630
5631     my $cfg = PVE::Storage::config();
5632
5633     my $volid_hash = scan_volids($cfg, $vmid);
5634
5635     my $updatefn =  sub {
5636         my ($vmid) = @_;
5637
5638         my $conf = PVE::QemuConfig->load_config($vmid);
5639
5640         PVE::QemuConfig->check_lock($conf);
5641
5642         my $vm_volids = {};
5643         foreach my $volid (keys %$volid_hash) {
5644             my $info = $volid_hash->{$volid};
5645             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5646         }
5647
5648         my $changes = update_disksize($vmid, $conf, $vm_volids);
5649
5650         PVE::QemuConfig->write_config($vmid, $conf) if $changes;
5651     };
5652
5653     if (defined($vmid)) {
5654         if ($nolock) {
5655             &$updatefn($vmid);
5656         } else {
5657             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5658         }
5659     } else {
5660         my $vmlist = config_list();
5661         foreach my $vmid (keys %$vmlist) {
5662             if ($nolock) {
5663                 &$updatefn($vmid);
5664             } else {
5665                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5666             }
5667         }
5668     }
5669 }
5670
5671 sub restore_vma_archive {
5672     my ($archive, $vmid, $user, $opts, $comp) = @_;
5673
5674     my $input = $archive eq '-' ? "<&STDIN" : undef;
5675     my $readfrom = $archive;
5676
5677     my $uncomp = '';
5678     if ($comp) {
5679         $readfrom = '-';
5680         my $qarchive = PVE::Tools::shellquote($archive);
5681         if ($comp eq 'gzip') {
5682             $uncomp = "zcat $qarchive|";
5683         } elsif ($comp eq 'lzop') {
5684             $uncomp = "lzop -d -c $qarchive|";
5685         } else {
5686             die "unknown compression method '$comp'\n";
5687         }
5688
5689     }
5690
5691     my $tmpdir = "/var/tmp/vzdumptmp$$";
5692     rmtree $tmpdir;
5693
5694     # disable interrupts (always do cleanups)
5695     local $SIG{INT} =
5696         local $SIG{TERM} =
5697         local $SIG{QUIT} =
5698         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
5699
5700     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5701     POSIX::mkfifo($mapfifo, 0600);
5702     my $fifofh;
5703
5704     my $openfifo = sub {
5705         open($fifofh, '>', $mapfifo) || die $!;
5706     };
5707
5708     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5709
5710     my $oldtimeout;
5711     my $timeout = 5;
5712
5713     my $devinfo = {};
5714
5715     my $rpcenv = PVE::RPCEnvironment::get();
5716
5717     my $conffile = PVE::QemuConfig->config_file($vmid);
5718     my $tmpfn = "$conffile.$$.tmp";
5719
5720     # Note: $oldconf is undef if VM does not exists
5721     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
5722     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
5723
5724     my $print_devmap = sub {
5725         my $virtdev_hash = {};
5726
5727         my $cfgfn = "$tmpdir/qemu-server.conf";
5728
5729         # we can read the config - that is already extracted
5730         my $fh = IO::File->new($cfgfn, "r") ||
5731             "unable to read qemu-server.conf - $!\n";
5732
5733         my $fwcfgfn = "$tmpdir/qemu-server.fw";
5734         if (-f $fwcfgfn) {
5735             my $pve_firewall_dir = '/etc/pve/firewall';
5736             mkdir $pve_firewall_dir; # make sure the dir exists
5737             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
5738         }
5739
5740         while (defined(my $line = <$fh>)) {
5741             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5742                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5743                 die "archive does not contain data for drive '$virtdev'\n"
5744                     if !$devinfo->{$devname};
5745                 if (defined($opts->{storage})) {
5746                     $storeid = $opts->{storage} || 'local';
5747                 } elsif (!$storeid) {
5748                     $storeid = 'local';
5749                 }
5750                 $format = 'raw' if !$format;
5751                 $devinfo->{$devname}->{devname} = $devname;
5752                 $devinfo->{$devname}->{virtdev} = $virtdev;
5753                 $devinfo->{$devname}->{format} = $format;
5754                 $devinfo->{$devname}->{storeid} = $storeid;
5755
5756                 # check permission on storage
5757                 my $pool = $opts->{pool}; # todo: do we need that?
5758                 if ($user ne 'root@pam') {
5759                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5760                 }
5761
5762                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5763             }
5764         }
5765
5766         foreach my $devname (keys %$devinfo) {
5767             die "found no device mapping information for device '$devname'\n"
5768                 if !$devinfo->{$devname}->{virtdev};
5769         }
5770
5771         my $cfg = PVE::Storage::config();
5772
5773         # create empty/temp config
5774         if ($oldconf) {
5775             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5776             foreach_drive($oldconf, sub {
5777                 my ($ds, $drive) = @_;
5778
5779                 return if drive_is_cdrom($drive);
5780
5781                 my $volid = $drive->{file};
5782
5783                 return if !$volid || $volid =~ m|^/|;
5784
5785                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5786                 return if !$path || !$owner || ($owner != $vmid);
5787
5788                 # Note: only delete disk we want to restore
5789                 # other volumes will become unused
5790                 if ($virtdev_hash->{$ds}) {
5791                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
5792                     if (my $err = $@) {
5793                         warn $err;
5794                     }
5795                 }
5796             });
5797
5798             # delete vmstate files
5799             # since after the restore we have no snapshots anymore
5800             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5801                 my $snap = $oldconf->{snapshots}->{$snapname};
5802                 if ($snap->{vmstate}) {
5803                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
5804                     if (my $err = $@) {
5805                         warn $err;
5806                     }
5807                 }
5808             }
5809         }
5810
5811         my $map = {};
5812         foreach my $virtdev (sort keys %$virtdev_hash) {
5813             my $d = $virtdev_hash->{$virtdev};
5814             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5815             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5816
5817             # test if requested format is supported
5818             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5819             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5820             $d->{format} = $defFormat if !$supported;
5821
5822             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5823                                                   $d->{format}, undef, $alloc_size);
5824             print STDERR "new volume ID is '$volid'\n";
5825             $d->{volid} = $volid;
5826             my $path = PVE::Storage::path($cfg, $volid);
5827
5828             PVE::Storage::activate_volumes($cfg,[$volid]);
5829
5830             my $write_zeros = 1;
5831             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
5832                 $write_zeros = 0;
5833             }
5834
5835             print $fifofh "format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
5836
5837             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5838             $map->{$virtdev} = $volid;
5839         }
5840
5841         $fh->seek(0, 0) || die "seek failed - $!\n";
5842
5843         my $outfd = new IO::File ($tmpfn, "w") ||
5844             die "unable to write config for VM $vmid\n";
5845
5846         my $cookie = { netcount => 0 };
5847         while (defined(my $line = <$fh>)) {
5848             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5849         }
5850
5851         $fh->close();
5852         $outfd->close();
5853     };
5854
5855     eval {
5856         # enable interrupts
5857         local $SIG{INT} =
5858             local $SIG{TERM} =
5859             local $SIG{QUIT} =
5860             local $SIG{HUP} =
5861             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
5862         local $SIG{ALRM} = sub { die "got timeout\n"; };
5863
5864         $oldtimeout = alarm($timeout);
5865
5866         my $parser = sub {
5867             my $line = shift;
5868
5869             print "$line\n";
5870
5871             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5872                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5873                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5874             } elsif ($line =~ m/^CTIME: /) {
5875                 # we correctly received the vma config, so we can disable
5876                 # the timeout now for disk allocation (set to 10 minutes, so
5877                 # that we always timeout if something goes wrong)
5878                 alarm(600);
5879                 &$print_devmap();
5880                 print $fifofh "done\n";
5881                 my $tmp = $oldtimeout || 0;
5882                 $oldtimeout = undef;
5883                 alarm($tmp);
5884                 close($fifofh);
5885             }
5886         };
5887
5888         print "restore vma archive: $cmd\n";
5889         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5890     };
5891     my $err = $@;
5892
5893     alarm($oldtimeout) if $oldtimeout;
5894
5895     my $vollist = [];
5896     foreach my $devname (keys %$devinfo) {
5897         my $volid = $devinfo->{$devname}->{volid};
5898         push @$vollist, $volid if $volid;
5899     }
5900
5901     my $cfg = PVE::Storage::config();
5902     PVE::Storage::deactivate_volumes($cfg, $vollist);
5903
5904     unlink $mapfifo;
5905
5906     if ($err) {
5907         rmtree $tmpdir;
5908         unlink $tmpfn;
5909
5910         foreach my $devname (keys %$devinfo) {
5911             my $volid = $devinfo->{$devname}->{volid};
5912             next if !$volid;
5913             eval {
5914                 if ($volid =~ m|^/|) {
5915                     unlink $volid || die 'unlink failed\n';
5916                 } else {
5917                     PVE::Storage::vdisk_free($cfg, $volid);
5918                 }
5919                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5920             };
5921             print STDERR "unable to cleanup '$volid' - $@" if $@;
5922         }
5923         die $err;
5924     }
5925
5926     rmtree $tmpdir;
5927
5928     rename($tmpfn, $conffile) ||
5929         die "unable to commit configuration file '$conffile'\n";
5930
5931     PVE::Cluster::cfs_update(); # make sure we read new file
5932
5933     eval { rescan($vmid, 1); };
5934     warn $@ if $@;
5935 }
5936
5937 sub restore_tar_archive {
5938     my ($archive, $vmid, $user, $opts) = @_;
5939
5940     if ($archive ne '-') {
5941         my $firstfile = tar_archive_read_firstfile($archive);
5942         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5943             if $firstfile ne 'qemu-server.conf';
5944     }
5945
5946     my $storecfg = PVE::Storage::config();
5947
5948     # destroy existing data - keep empty config
5949     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
5950     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5951
5952     my $tocmd = "/usr/lib/qemu-server/qmextract";
5953
5954     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5955     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5956     $tocmd .= ' --prealloc' if $opts->{prealloc};
5957     $tocmd .= ' --info' if $opts->{info};
5958
5959     # tar option "xf" does not autodetect compression when read from STDIN,
5960     # so we pipe to zcat
5961     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5962         PVE::Tools::shellquote("--to-command=$tocmd");
5963
5964     my $tmpdir = "/var/tmp/vzdumptmp$$";
5965     mkpath $tmpdir;
5966
5967     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5968     local $ENV{VZDUMP_VMID} = $vmid;
5969     local $ENV{VZDUMP_USER} = $user;
5970
5971     my $conffile = PVE::QemuConfig->config_file($vmid);
5972     my $tmpfn = "$conffile.$$.tmp";
5973
5974     # disable interrupts (always do cleanups)
5975     local $SIG{INT} =
5976         local $SIG{TERM} =
5977         local $SIG{QUIT} =
5978         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
5979
5980     eval {
5981         # enable interrupts
5982         local $SIG{INT} =
5983             local $SIG{TERM} =
5984             local $SIG{QUIT} =
5985             local $SIG{HUP} =
5986             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
5987
5988         if ($archive eq '-') {
5989             print "extracting archive from STDIN\n";
5990             run_command($cmd, input => "<&STDIN");
5991         } else {
5992             print "extracting archive '$archive'\n";
5993             run_command($cmd);
5994         }
5995
5996         return if $opts->{info};
5997
5998         # read new mapping
5999         my $map = {};
6000         my $statfile = "$tmpdir/qmrestore.stat";
6001         if (my $fd = IO::File->new($statfile, "r")) {
6002             while (defined (my $line = <$fd>)) {
6003                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6004                     $map->{$1} = $2 if $1;
6005                 } else {
6006                     print STDERR "unable to parse line in statfile - $line\n";
6007                 }
6008             }
6009             $fd->close();
6010         }
6011
6012         my $confsrc = "$tmpdir/qemu-server.conf";
6013
6014         my $srcfd = new IO::File($confsrc, "r") ||
6015             die "unable to open file '$confsrc'\n";
6016
6017         my $outfd = new IO::File ($tmpfn, "w") ||
6018             die "unable to write config for VM $vmid\n";
6019
6020         my $cookie = { netcount => 0 };
6021         while (defined (my $line = <$srcfd>)) {
6022             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6023         }
6024
6025         $srcfd->close();
6026         $outfd->close();
6027     };
6028     my $err = $@;
6029
6030     if ($err) {
6031
6032         unlink $tmpfn;
6033
6034         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6035
6036         die $err;
6037     }
6038
6039     rmtree $tmpdir;
6040
6041     rename $tmpfn, $conffile ||
6042         die "unable to commit configuration file '$conffile'\n";
6043
6044     PVE::Cluster::cfs_update(); # make sure we read new file
6045
6046     eval { rescan($vmid, 1); };
6047     warn $@ if $@;
6048 };
6049
6050 sub foreach_storage_used_by_vm {
6051     my ($conf, $func) = @_;
6052
6053     my $sidhash = {};
6054
6055     foreach_drive($conf, sub {
6056         my ($ds, $drive) = @_;
6057         return if drive_is_cdrom($drive);
6058
6059         my $volid = $drive->{file};
6060
6061         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6062         $sidhash->{$sid} = $sid if $sid;
6063     });
6064
6065     foreach my $sid (sort keys %$sidhash) {
6066         &$func($sid);
6067     }
6068 }
6069
6070 sub do_snapshots_with_qemu {
6071     my ($storecfg, $volid) = @_;
6072
6073     my $storage_name = PVE::Storage::parse_volume_id($volid);
6074
6075     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
6076         && !$storecfg->{ids}->{$storage_name}->{krbd}){
6077         return 1;
6078     }
6079
6080     if ($volid =~ m/\.(qcow2|qed)$/){
6081         return 1;
6082     }
6083
6084     return undef;
6085 }
6086
6087 sub qga_check_running {
6088     my ($vmid) = @_;
6089
6090     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
6091     if ($@) {
6092         warn "Qemu Guest Agent is not running - $@";
6093         return 0;
6094     }
6095     return 1;
6096 }
6097
6098 sub template_create {
6099     my ($vmid, $conf, $disk) = @_;
6100
6101     my $storecfg = PVE::Storage::config();
6102
6103     foreach_drive($conf, sub {
6104         my ($ds, $drive) = @_;
6105
6106         return if drive_is_cdrom($drive);
6107         return if $disk && $ds ne $disk;
6108
6109         my $volid = $drive->{file};
6110         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6111
6112         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6113         $drive->{file} = $voliddst;
6114         $conf->{$ds} = print_drive($vmid, $drive);
6115         PVE::QemuConfig->write_config($vmid, $conf);
6116     });
6117 }
6118
6119 sub qemu_img_convert {
6120     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6121
6122     my $storecfg = PVE::Storage::config();
6123     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6124     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6125
6126     if ($src_storeid && $dst_storeid) {
6127
6128         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6129
6130         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6131         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6132
6133         my $src_format = qemu_img_format($src_scfg, $src_volname);
6134         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6135
6136         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6137         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6138
6139         my $cmd = [];
6140         push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6141         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
6142         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path;
6143         if ($is_zero_initialized) {
6144             push @$cmd, "zeroinit:$dst_path";
6145         } else {
6146             push @$cmd, $dst_path;
6147         }
6148
6149         my $parser = sub {
6150             my $line = shift;
6151             if($line =~ m/\((\S+)\/100\%\)/){
6152                 my $percent = $1;
6153                 my $transferred = int($size * $percent / 100);
6154                 my $remaining = $size - $transferred;
6155
6156                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6157             }
6158
6159         };
6160
6161         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6162         my $err = $@;
6163         die "copy failed: $err" if $err;
6164     }
6165 }
6166
6167 sub qemu_img_format {
6168     my ($scfg, $volname) = @_;
6169
6170     if ($scfg->{path} && $volname =~ m/\.($QEMU_FORMAT_RE)$/) {
6171         return $1;
6172     } else {
6173         return "raw";
6174     }
6175 }
6176
6177 sub qemu_drive_mirror {
6178     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga) = @_;
6179
6180     $jobs = {} if !$jobs;
6181
6182     my $qemu_target;
6183     my $format;
6184     $jobs->{"drive-$drive"} = {};
6185
6186     if ($dst_volid =~ /^nbd:/) {
6187         $qemu_target = $dst_volid;
6188         $format = "nbd";
6189     } else {
6190         my $storecfg = PVE::Storage::config();
6191         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6192
6193         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6194
6195         $format = qemu_img_format($dst_scfg, $dst_volname);
6196
6197         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6198
6199         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6200     }
6201
6202     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6203     $opts->{format} = $format if $format;
6204
6205     print "drive mirror is starting for drive-$drive\n";
6206
6207     eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); }; #if a job already run for this device,it's throw an error
6208
6209     if (my $err = $@) {
6210         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6211         die "mirroring error: $err";
6212     }
6213
6214     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6215 }
6216
6217 sub qemu_drive_mirror_monitor {
6218     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6219
6220     eval {
6221         my $err_complete = 0;
6222
6223         while (1) {
6224             die "storage migration timed out\n" if $err_complete > 300;
6225
6226             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6227
6228             my $running_mirror_jobs = {};
6229             foreach my $stat (@$stats) {
6230                 next if $stat->{type} ne 'mirror';
6231                 $running_mirror_jobs->{$stat->{device}} = $stat;
6232             }
6233
6234             my $readycounter = 0;
6235
6236             foreach my $job (keys %$jobs) {
6237
6238                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6239                     print "$job : finished\n";
6240                     delete $jobs->{$job};
6241                     next;
6242                 }
6243
6244                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6245
6246                 my $busy = $running_mirror_jobs->{$job}->{busy};
6247                 my $ready = $running_mirror_jobs->{$job}->{ready};
6248                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6249                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6250                     my $remaining = $total - $transferred;
6251                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6252
6253                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6254                 }
6255
6256                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6257             }
6258
6259             last if scalar(keys %$jobs) == 0;
6260
6261             if ($readycounter == scalar(keys %$jobs)) {
6262                 print "all mirroring jobs are ready \n";
6263                 last if $skipcomplete; #do the complete later
6264
6265                 if ($vmiddst && $vmiddst != $vmid) {
6266                     my $agent_running = $qga && qga_check_running($vmid);
6267                     if ($agent_running) {
6268                         print "freeze filesystem\n";
6269                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6270                     } else {
6271                         print "suspend vm\n";
6272                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6273                     }
6274
6275                     # if we clone a disk for a new target vm, we don't switch the disk
6276                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6277
6278                     if ($agent_running) {
6279                         print "unfreeze filesystem\n";
6280                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6281                     } else {
6282                         print "resume vm\n";
6283                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6284                     }
6285
6286                     last;
6287                 } else {
6288
6289                     foreach my $job (keys %$jobs) {
6290                         # try to switch the disk if source and destination are on the same guest
6291                         print "$job: Completing block job...\n";
6292
6293                         eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
6294                         if ($@ =~ m/cannot be completed/) {
6295                             print "$job: Block job cannot be completed, try again.\n";
6296                             $err_complete++;
6297                         }else {
6298                             print "$job: Completed successfully.\n";
6299                             $jobs->{$job}->{complete} = 1;
6300                         }
6301                     }
6302                 }
6303             }
6304             sleep 1;
6305         }
6306     };
6307     my $err = $@;
6308
6309     if ($err) {
6310         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6311         die "mirroring error: $err";
6312     }
6313
6314 }
6315
6316 sub qemu_blockjobs_cancel {
6317     my ($vmid, $jobs) = @_;
6318
6319     foreach my $job (keys %$jobs) {
6320         print "$job: Cancelling block job\n";
6321         eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
6322         $jobs->{$job}->{cancel} = 1;
6323     }
6324
6325     while (1) {
6326         my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6327
6328         my $running_jobs = {};
6329         foreach my $stat (@$stats) {
6330             $running_jobs->{$stat->{device}} = $stat;
6331         }
6332
6333         foreach my $job (keys %$jobs) {
6334
6335             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6336                 print "$job: Done.\n";
6337                 delete $jobs->{$job};
6338             }
6339         }
6340
6341         last if scalar(keys %$jobs) == 0;
6342
6343         sleep 1;
6344     }
6345 }
6346
6347 sub clone_disk {
6348     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6349         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga) = @_;
6350
6351     my $newvolid;
6352
6353     if (!$full) {
6354         print "create linked clone of drive $drivename ($drive->{file})\n";
6355         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6356         push @$newvollist, $newvolid;
6357     } else {
6358
6359         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6360         $storeid = $storage if $storage;
6361
6362         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
6363         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6364
6365         print "create full clone of drive $drivename ($drive->{file})\n";
6366         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, undef, ($size/1024));
6367         push @$newvollist, $newvolid;
6368
6369         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
6370
6371         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6372         if (!$running || $snapname) {
6373             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6374         } else {
6375
6376             my $kvmver = get_running_qemu_version ($vmid);
6377             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
6378                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6379                     if $drive->{iothread};
6380             }
6381
6382             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga);
6383         }
6384     }
6385
6386     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6387
6388     my $disk = $drive;
6389     $disk->{format} = undef;
6390     $disk->{file} = $newvolid;
6391     $disk->{size} = $size;
6392
6393     return $disk;
6394 }
6395
6396 # this only works if VM is running
6397 sub get_current_qemu_machine {
6398     my ($vmid) = @_;
6399
6400     my $cmd = { execute => 'query-machines', arguments => {} };
6401     my $res = vm_qmp_command($vmid, $cmd);
6402
6403     my ($current, $default);
6404     foreach my $e (@$res) {
6405         $default = $e->{name} if $e->{'is-default'};
6406         $current = $e->{name} if $e->{'is-current'};
6407     }
6408
6409     # fallback to the default machine if current is not supported by qemu
6410     return $current || $default || 'pc';
6411 }
6412
6413 sub get_running_qemu_version {
6414     my ($vmid) = @_;
6415     my $cmd = { execute => 'query-version', arguments => {} };
6416     my $res = vm_qmp_command($vmid, $cmd);
6417     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
6418 }
6419
6420 sub qemu_machine_feature_enabled {
6421     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6422
6423     my $current_major;
6424     my $current_minor;
6425
6426     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6427
6428         $current_major = $3;
6429         $current_minor = $4;
6430
6431     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6432
6433         $current_major = $1;
6434         $current_minor = $2;
6435     }
6436
6437     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
6438
6439
6440 }
6441
6442 sub qemu_machine_pxe {
6443     my ($vmid, $conf, $machine) = @_;
6444
6445     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6446
6447     foreach my $opt (keys %$conf) {
6448         next if $opt !~ m/^net(\d+)$/;
6449         my $net = PVE::QemuServer::parse_net($conf->{$opt});
6450         next if !$net;
6451         my $romfile = PVE::QemuServer::vm_mon_cmd_nocheck($vmid, 'qom-get', path => $opt, property => 'romfile');
6452         return $machine.".pxe" if $romfile =~ m/pxe/;
6453         last;
6454     }
6455
6456     return $machine;
6457 }
6458
6459 sub qemu_use_old_bios_files {
6460     my ($machine_type) = @_;
6461
6462     return if !$machine_type;
6463
6464     my $use_old_bios_files = undef;
6465
6466     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6467         $machine_type = $1;
6468         $use_old_bios_files = 1;
6469     } else {
6470         my $kvmver = kvm_user_version();
6471         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6472         # load new efi bios files on migration. So this hack is required to allow
6473         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6474         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6475         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
6476     }
6477
6478     return ($use_old_bios_files, $machine_type);
6479 }
6480
6481 sub create_efidisk {
6482     my ($storecfg, $storeid, $vmid, $fmt) = @_;
6483
6484     die "EFI vars default image not found\n" if ! -f $OVMF_VARS;
6485
6486     my $vars_size = PVE::Tools::convert_size(-s $OVMF_VARS, 'b' => 'kb');
6487     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
6488     PVE::Storage::activate_volumes($storecfg, [$volid]);
6489
6490     my $path = PVE::Storage::path($storecfg, $volid);
6491     eval {
6492         run_command(['/usr/bin/qemu-img', 'convert', '-n', '-f', 'raw', '-O', $fmt, $OVMF_VARS, $path]);
6493     };
6494     die "Copying EFI vars image failed: $@" if $@;
6495
6496     return ($volid, $vars_size);
6497 }
6498
6499 sub lspci {
6500
6501     my $devices = {};
6502
6503     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6504             my (undef, $id, $function) = @_;
6505             my $res = { id => $id, function => $function};
6506             push @{$devices->{$id}}, $res;
6507     });
6508
6509     # Entries should be sorted by functions.
6510     foreach my $id (keys %$devices) {
6511         my $dev = $devices->{$id};
6512         $devices->{$id} = [ sort { $a->{function} <=> $b->{function} } @$dev ];
6513     }
6514
6515     return $devices;
6516 }
6517
6518 sub vm_iothreads_list {
6519     my ($vmid) = @_;
6520
6521     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6522
6523     my $iothreads = {};
6524     foreach my $iothread (@$res) {
6525         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6526     }
6527
6528     return $iothreads;
6529 }
6530
6531 sub scsihw_infos {
6532     my ($conf, $drive) = @_;
6533
6534     my $maxdev = 0;
6535
6536     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
6537         $maxdev = 7;
6538     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6539         $maxdev = 1;
6540     } else {
6541         $maxdev = 256;
6542     }
6543
6544     my $controller = int($drive->{index} / $maxdev);
6545     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6546
6547     return ($maxdev, $controller, $controller_prefix);
6548 }
6549
6550 sub add_hyperv_enlightenments {
6551     my ($cpuFlags, $winversion, $machine_type, $kvmver, $bios, $gpu_passthrough) = @_;
6552
6553     return if $winversion < 6;
6554     return if $bios && $bios eq 'ovmf' && $winversion < 8;
6555
6556     push @$cpuFlags , 'hv_vendor_id=proxmox' if $gpu_passthrough;
6557
6558     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
6559         push @$cpuFlags , 'hv_spinlocks=0x1fff';
6560         push @$cpuFlags , 'hv_vapic';
6561         push @$cpuFlags , 'hv_time';
6562     } else {
6563         push @$cpuFlags , 'hv_spinlocks=0xffff';
6564     }
6565
6566     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
6567         push @$cpuFlags , 'hv_reset';
6568         push @$cpuFlags , 'hv_vpindex';
6569         push @$cpuFlags , 'hv_runtime';
6570     }
6571
6572     if ($winversion >= 7) {
6573         push @$cpuFlags , 'hv_relaxed';
6574     }
6575 }
6576
6577 sub windows_version {
6578     my ($ostype) = @_;
6579
6580     return 0 if !$ostype;
6581
6582     my $winversion = 0;
6583
6584     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
6585         $winversion = 5;
6586     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
6587         $winversion = 6;
6588     } elsif ($ostype =~ m/^win(\d+)$/) {
6589         $winversion = $1;
6590     }
6591
6592     return $winversion;
6593 }
6594
6595 sub resolve_dst_disk_format {
6596         my ($storecfg, $storeid, $src_volname, $format) = @_;
6597         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6598
6599         if (!$format) {
6600             # if no target format is specified, use the source disk format as hint
6601             if ($src_volname) {
6602                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6603                 $format = qemu_img_format($scfg, $src_volname);
6604             } else {
6605                 return $defFormat;
6606             }
6607         }
6608
6609         # test if requested format is supported - else use default
6610         my $supported = grep { $_ eq $format } @$validFormats;
6611         $format = $defFormat if !$supported;
6612         return $format;
6613 }
6614
6615 sub resolve_first_disk {
6616     my $conf = shift;
6617     my @disks = PVE::QemuServer::valid_drive_names();
6618     my $firstdisk;
6619     foreach my $ds (reverse @disks) {
6620         next if !$conf->{$ds};
6621         my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
6622         next if PVE::QemuServer::drive_is_cdrom($disk);
6623         $firstdisk = $ds;
6624     }
6625     return $firstdisk;
6626 }
6627
6628 sub generate_smbios1_uuid {
6629     my ($uuid, $uuid_str);
6630     UUID::generate($uuid);
6631     UUID::unparse($uuid, $uuid_str);
6632     return "uuid=$uuid_str";
6633 }
6634
6635 # bash completion helper
6636
6637 sub complete_backup_archives {
6638     my ($cmdname, $pname, $cvalue) = @_;
6639
6640     my $cfg = PVE::Storage::config();
6641
6642     my $storeid;
6643
6644     if ($cvalue =~ m/^([^:]+):/) {
6645         $storeid = $1;
6646     }
6647
6648     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6649
6650     my $res = [];
6651     foreach my $id (keys %$data) {
6652         foreach my $item (@{$data->{$id}}) {
6653             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6654             push @$res, $item->{volid} if defined($item->{volid});
6655         }
6656     }
6657
6658     return $res;
6659 }
6660
6661 my $complete_vmid_full = sub {
6662     my ($running) = @_;
6663
6664     my $idlist = vmstatus();
6665
6666     my $res = [];
6667
6668     foreach my $id (keys %$idlist) {
6669         my $d = $idlist->{$id};
6670         if (defined($running)) {
6671             next if $d->{template};
6672             next if $running && $d->{status} ne 'running';
6673             next if !$running && $d->{status} eq 'running';
6674         }
6675         push @$res, $id;
6676
6677     }
6678     return $res;
6679 };
6680
6681 sub complete_vmid {
6682     return &$complete_vmid_full();
6683 }
6684
6685 sub complete_vmid_stopped {
6686     return &$complete_vmid_full(0);
6687 }
6688
6689 sub complete_vmid_running {
6690     return &$complete_vmid_full(1);
6691 }
6692
6693 sub complete_storage {
6694
6695     my $cfg = PVE::Storage::config();
6696     my $ids = $cfg->{ids};
6697
6698     my $res = [];
6699     foreach my $sid (keys %$ids) {
6700         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6701         next if !$ids->{$sid}->{content}->{images};
6702         push @$res, $sid;
6703     }
6704
6705     return $res;
6706 }
6707
6708 1;