PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QemuConfig;
  31 use PVE::QMPClient;
  32 use PVE::RPCEnvironment;
  33 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
  34 use PVE::QemuServer::Memory;
  35 use PVE::QemuServer::USB qw(parse_usb_device);
  36 use Time::HiRes qw(gettimeofday);
  37 use File::Copy qw(copy);
  38 use URI::Escape;
  39
  40 my $OVMF_CODE = '/usr/share/kvm/OVMF_CODE-pure-efi.fd';
  41 my $OVMF_VARS = '/usr/share/kvm/OVMF_VARS-pure-efi.fd';
  42
  43 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  44
  45 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  46
  47 # Note about locking: we use flock on the config file protect
  48 # against concurent actions.
  49 # Aditionaly, we have a 'lock' setting in the config file. This
  50 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  51 # allowed when such lock is set. But you can ignore this kind of
  52 # lock with the --skiplock flag.
  53
  54 cfs_register_file('/qemu-server/',
  55                   \&parse_vm_config,
  56                   \&write_vm_config);
  57
  58 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  59     description => "Some command save/restore state from this location.",
  60     type => 'string',
  61     maxLength => 128,
  62     optional => 1,
  63 });
  64
  65 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  66     description => "The name of the snapshot.",
  67     type => 'string', format => 'pve-configid',
  68     maxLength => 40,
  69 });
  70
  71 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  72     type => 'string',
  73     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  74     description => "The drive's backing file's data format.",
  75     optional => 1,
  76 });
  77
  78 #no warnings 'redefine';
  79
  80 sub cgroups_write {
  81    my ($controller, $vmid, $option, $value) = @_;
  82
  83    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  84    PVE::ProcFSTools::write_proc_entry($path, $value);
  85
  86 }
  87
  88 my $nodename = PVE::INotify::nodename();
  89
  90 mkdir "/etc/pve/nodes/$nodename";
  91 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  92 mkdir $confdir;
  93
  94 my $var_run_tmpdir = "/var/run/qemu-server";
  95 mkdir $var_run_tmpdir;
  96
  97 my $lock_dir = "/var/lock/qemu-server";
  98 mkdir $lock_dir;
  99
 100 my $pcisysfs = "/sys/bus/pci";
 101
 102 my $cpu_vendor_list = {
 103     # Intel CPUs
 104     486 => 'GenuineIntel',
 105     pentium => 'GenuineIntel',
 106     pentium2  => 'GenuineIntel',
 107     pentium3  => 'GenuineIntel',
 108     coreduo => 'GenuineIntel',
 109     core2duo => 'GenuineIntel',
 110     Conroe  => 'GenuineIntel',
 111     Penryn  => 'GenuineIntel',
 112     Nehalem  => 'GenuineIntel',
 113     Westmere => 'GenuineIntel',
 114     SandyBridge => 'GenuineIntel',
 115     IvyBridge => 'GenuineIntel',
 116     Haswell => 'GenuineIntel',
 117     'Haswell-noTSX' => 'GenuineIntel',
 118     Broadwell => 'GenuineIntel',
 119     'Broadwell-noTSX' => 'GenuineIntel',
 120     'Skylake-Client' => 'GenuineIntel',
 121
 122     # AMD CPUs
 123     athlon => 'AuthenticAMD',
 124     phenom  => 'AuthenticAMD',
 125     Opteron_G1  => 'AuthenticAMD',
 126     Opteron_G2  => 'AuthenticAMD',
 127     Opteron_G3  => 'AuthenticAMD',
 128     Opteron_G4  => 'AuthenticAMD',
 129     Opteron_G5  => 'AuthenticAMD',
 130
 131     # generic types, use vendor from host node
 132     host => 'default',
 133     kvm32 => 'default',
 134     kvm64 => 'default',
 135     qemu32 => 'default',
 136     qemu64 => 'default',
 137 };
 138
 139 my $cpu_fmt = {
 140     cputype => {
 141         description => "Emulated CPU type.",
 142         type => 'string',
 143         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 144         default => 'kvm64',
 145         default_key => 1,
 146     },
 147     hidden => {
 148         description => "Do not identify as a KVM virtual machine.",
 149         type => 'boolean',
 150         optional => 1,
 151         default => 0
 152     },
 153 };
 154
 155 my $watchdog_fmt = {
 156     model => {
 157         default_key => 1,
 158         type => 'string',
 159         enum => [qw(i6300esb ib700)],
 160         description => "Watchdog type to emulate.",
 161         default => 'i6300esb',
 162         optional => 1,
 163     },
 164     action => {
 165         type => 'string',
 166         enum => [qw(reset shutdown poweroff pause debug none)],
 167         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 168         optional => 1,
 169     },
 170 };
 171 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 172
 173 my $confdesc = {
 174     onboot => {
 175         optional => 1,
 176         type => 'boolean',
 177         description => "Specifies whether a VM will be started during system bootup.",
 178         default => 0,
 179     },
 180     autostart => {
 181         optional => 1,
 182         type => 'boolean',
 183         description => "Automatic restart after crash (currently ignored).",
 184         default => 0,
 185     },
 186     hotplug => {
 187         optional => 1,
 188         type => 'string', format => 'pve-hotplug-features',
 189         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 190         default => 'network,disk,usb',
 191     },
 192     reboot => {
 193         optional => 1,
 194         type => 'boolean',
 195         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 196         default => 1,
 197     },
 198     lock => {
 199         optional => 1,
 200         type => 'string',
 201         description => "Lock/unlock the VM.",
 202         enum => [qw(migrate backup snapshot rollback)],
 203     },
 204     cpulimit => {
 205         optional => 1,
 206         type => 'number',
 207         description => "Limit of CPU usage.",
 208         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 209         minimum => 0,
 210         maximum => 128,
 211         default => 0,
 212     },
 213     cpuunits => {
 214         optional => 1,
 215         type => 'integer',
 216         description => "CPU weight for a VM.",
 217         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 218         minimum => 0,
 219         maximum => 500000,
 220         default => 1024,
 221     },
 222     memory => {
 223         optional => 1,
 224         type => 'integer',
 225         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 226         minimum => 16,
 227         default => 512,
 228     },
 229     balloon => {
 230         optional => 1,
 231         type => 'integer',
 232         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 233         minimum => 0,
 234     },
 235     shares => {
 236         optional => 1,
 237         type => 'integer',
 238         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 239         minimum => 0,
 240         maximum => 50000,
 241         default => 1000,
 242     },
 243     keyboard => {
 244         optional => 1,
 245         type => 'string',
 246         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.conf' configuration file.",
 247         enum => PVE::Tools::kvmkeymaplist(),
 248         default => 'en-us',
 249     },
 250     name => {
 251         optional => 1,
 252         type => 'string', format => 'dns-name',
 253         description => "Set a name for the VM. Only used on the configuration web interface.",
 254     },
 255     scsihw => {
 256         optional => 1,
 257         type => 'string',
 258         description => "SCSI controller model",
 259         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 260         default => 'lsi',
 261     },
 262     description => {
 263         optional => 1,
 264         type => 'string',
 265         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 266     },
 267     ostype => {
 268         optional => 1,
 269         type => 'string',
 270         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 271         description => "Specify guest operating system.",
 272         verbose_description => <<EODESC,
 273 Specify guest operating system. This is used to enable special
 274 optimization/features for specific operating systems:
 275
 276 [horizontal]
 277 other;; unspecified OS
 278 wxp;; Microsoft Windows XP
 279 w2k;; Microsoft Windows 2000
 280 w2k3;; Microsoft Windows 2003
 281 w2k8;; Microsoft Windows 2008
 282 wvista;; Microsoft Windows Vista
 283 win7;; Microsoft Windows 7
 284 win8;; Microsoft Windows 8/2012
 285 l24;; Linux 2.4 Kernel
 286 l26;; Linux 2.6/3.X Kernel
 287 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 288 EODESC
 289     },
 290     boot => {
 291         optional => 1,
 292         type => 'string',
 293         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 294         pattern => '[acdn]{1,4}',
 295         default => 'cdn',
 296     },
 297     bootdisk => {
 298         optional => 1,
 299         type => 'string', format => 'pve-qm-bootdisk',
 300         description => "Enable booting from specified disk.",
 301         pattern => '(ide|sata|scsi|virtio)\d+',
 302     },
 303     smp => {
 304         optional => 1,
 305         type => 'integer',
 306         description => "The number of CPUs. Please use option -sockets instead.",
 307         minimum => 1,
 308         default => 1,
 309     },
 310     sockets => {
 311         optional => 1,
 312         type => 'integer',
 313         description => "The number of CPU sockets.",
 314         minimum => 1,
 315         default => 1,
 316     },
 317     cores => {
 318         optional => 1,
 319         type => 'integer',
 320         description => "The number of cores per socket.",
 321         minimum => 1,
 322         default => 1,
 323     },
 324     numa => {
 325         optional => 1,
 326         type => 'boolean',
 327         description => "Enable/disable NUMA.",
 328         default => 0,
 329     },
 330     hugepages => {
 331         optional => 1,
 332         type => 'string',
 333         description => "Enable/disable hugepages memory.",
 334         enum => [qw(any 2 1024)],
 335     },
 336     vcpus => {
 337         optional => 1,
 338         type => 'integer',
 339         description => "Number of hotplugged vcpus.",
 340         minimum => 1,
 341         default => 0,
 342     },
 343     acpi => {
 344         optional => 1,
 345         type => 'boolean',
 346         description => "Enable/disable ACPI.",
 347         default => 1,
 348     },
 349     agent => {
 350         optional => 1,
 351         type => 'boolean',
 352         description => "Enable/disable Qemu GuestAgent.",
 353         default => 0,
 354     },
 355     kvm => {
 356         optional => 1,
 357         type => 'boolean',
 358         description => "Enable/disable KVM hardware virtualization.",
 359         default => 1,
 360     },
 361     tdf => {
 362         optional => 1,
 363         type => 'boolean',
 364         description => "Enable/disable time drift fix.",
 365         default => 0,
 366     },
 367     localtime => {
 368         optional => 1,
 369         type => 'boolean',
 370         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 371     },
 372     freeze => {
 373         optional => 1,
 374         type => 'boolean',
 375         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 376     },
 377     vga => {
 378         optional => 1,
 379         type => 'string',
 380         description => "Select the VGA type.",
 381         verbose_description => "Select the VGA type. If you want to use high resolution" .
 382             " modes (>= 1280x1024x16) then you should use the options " .
 383             "'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and " .
 384             "'cirrus' for other OS types. The 'qxl' option enables the SPICE " .
 385             "display sever. For win* OS you can select how many independent " .
 386             "displays you want, Linux guests can add displays them self. " .
 387             "You can also run without any graphic card, using a serial device" .
 388             " as terminal.",
 389         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 390     },
 391     watchdog => {
 392         optional => 1,
 393         type => 'string', format => 'pve-qm-watchdog',
 394         description => "Create a virtual hardware watchdog device.",
 395         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 396             " (by a guest action), the watchdog must be periodically polled " .
 397             "by an agent inside the guest or else the watchdog will reset " .
 398             "the guest (or execute the respective action specified)",
 399     },
 400     startdate => {
 401         optional => 1,
 402         type => 'string',
 403         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 404         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 405         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 406         default => 'now',
 407     },
 408     startup =>  get_standard_option('pve-startup-order'),
 409     template => {
 410         optional => 1,
 411         type => 'boolean',
 412         description => "Enable/disable Template.",
 413         default => 0,
 414     },
 415     args => {
 416         optional => 1,
 417         type => 'string',
 418         description => "Arbitrary arguments passed to kvm.",
 419         verbose_description => <<EODESCR,
 420 Arbitrary arguments passed to kvm, for example:
 421
 422 args: -no-reboot -no-hpet
 423
 424 NOTE: this option is for experts only.
 425 EODESCR
 426     },
 427     tablet => {
 428         optional => 1,
 429         type => 'boolean',
 430         default => 1,
 431         description => "Enable/disable the USB tablet device.",
 432         verbose_description => "Enable/disable the USB tablet device. This device is " .
 433             "usually needed to allow absolute mouse positioning with VNC. " .
 434             "Else the mouse runs out of sync with normal VNC clients. " .
 435             "If you're running lots of console-only guests on one host, " .
 436             "you may consider disabling this to save some context switches. " .
 437             "This is turned off by default if you use spice (-vga=qxl).",
 438     },
 439     migrate_speed => {
 440         optional => 1,
 441         type => 'integer',
 442         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 443         minimum => 0,
 444         default => 0,
 445     },
 446     migrate_downtime => {
 447         optional => 1,
 448         type => 'number',
 449         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 450         minimum => 0,
 451         default => 0.1,
 452     },
 453     cdrom => {
 454         optional => 1,
 455         type => 'string', format => 'pve-qm-ide',
 456         typetext => '<volume>',
 457         description => "This is an alias for option -ide2",
 458     },
 459     cpu => {
 460         optional => 1,
 461         description => "Emulated CPU type.",
 462         type => 'string',
 463         format => $cpu_fmt,
 464     },
 465     parent => get_standard_option('pve-snapshot-name', {
 466         optional => 1,
 467         description => "Parent snapshot name. This is used internally, and should not be modified.",
 468     }),
 469     snaptime => {
 470         optional => 1,
 471         description => "Timestamp for snapshots.",
 472         type => 'integer',
 473         minimum => 0,
 474     },
 475     vmstate => {
 476         optional => 1,
 477         type => 'string', format => 'pve-volume-id',
 478         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 479     },
 480     vmstatestorage => get_standard_option('pve-storage-id', {
 481         description => "Default storage for VM state volumes/files.",
 482         optional => 1,
 483     }),
 484     machine => {
 485         description => "Specific the Qemu machine type.",
 486         type => 'string',
 487         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
 488         maxLength => 40,
 489         optional => 1,
 490     },
 491     smbios1 => {
 492         description => "Specify SMBIOS type 1 fields.",
 493         type => 'string', format => 'pve-qm-smbios1',
 494         maxLength => 256,
 495         optional => 1,
 496     },
 497     protection => {
 498         optional => 1,
 499         type => 'boolean',
 500         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 501         default => 0,
 502     },
 503     bios => {
 504         optional => 1,
 505         type => 'string',
 506         enum => [ qw(seabios ovmf) ],
 507         description => "Select BIOS implementation.",
 508         default => 'seabios',
 509     },
 510 };
 511
 512 # what about other qemu settings ?
 513 #cpu => 'string',
 514 #machine => 'string',
 515 #fda => 'file',
 516 #fdb => 'file',
 517 #mtdblock => 'file',
 518 #sd => 'file',
 519 #pflash => 'file',
 520 #snapshot => 'bool',
 521 #bootp => 'file',
 522 ##tftp => 'dir',
 523 ##smb => 'dir',
 524 #kernel => 'file',
 525 #append => 'string',
 526 #initrd => 'file',
 527 ##soundhw => 'string',
 528
 529 while (my ($k, $v) = each %$confdesc) {
 530     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 531 }
 532
 533 my $MAX_IDE_DISKS = 4;
 534 my $MAX_SCSI_DISKS = 14;
 535 my $MAX_VIRTIO_DISKS = 16;
 536 my $MAX_SATA_DISKS = 6;
 537 my $MAX_USB_DEVICES = 5;
 538 my $MAX_NETS = 32;
 539 my $MAX_UNUSED_DISKS = 8;
 540 my $MAX_HOSTPCI_DEVICES = 4;
 541 my $MAX_SERIAL_PORTS = 4;
 542 my $MAX_PARALLEL_PORTS = 3;
 543 my $MAX_NUMA = 8;
 544
 545 my $numa_fmt = {
 546     cpus => {
 547         type => "string",
 548         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 549         description => "CPUs accessing this NUMA node.",
 550         format_description => "id[-id];...",
 551     },
 552     memory => {
 553         type => "number",
 554         description => "Amount of memory this NUMA node provides.",
 555         optional => 1,
 556     },
 557     hostnodes => {
 558         type => "string",
 559         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 560         description => "Host NUMA nodes to use.",
 561         format_description => "id[-id];...",
 562         optional => 1,
 563     },
 564     policy => {
 565         type => 'string',
 566         enum => [qw(preferred bind interleave)],
 567         description => "NUMA allocation policy.",
 568         optional => 1,
 569     },
 570 };
 571 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 572 my $numadesc = {
 573     optional => 1,
 574     type => 'string', format => $numa_fmt,
 575     description => "NUMA topology.",
 576 };
 577 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 578
 579 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 580     $confdesc->{"numa$i"} = $numadesc;
 581 }
 582
 583 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 584                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 585                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 586 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 587
 588 my $net_fmt_bridge_descr = <<__EOD__;
 589 Bridge to attach the network device to. The Proxmox VE standard bridge
 590 is called 'vmbr0'.
 591
 592 If you do not specify a bridge, we create a kvm user (NATed) network
 593 device, which provides DHCP and DNS services. The following addresses
 594 are used:
 595
 596  10.0.2.2   Gateway
 597  10.0.2.3   DNS Server
 598  10.0.2.4   SMB Server
 599
 600 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 601 __EOD__
 602
 603 my $net_fmt = {
 604     macaddr => {
 605         type => 'string',
 606         pattern => qr/[0-9a-f]{2}(?::[0-9a-f]{2}){5}/i,
 607         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 608         format_description => "XX:XX:XX:XX:XX:XX",
 609         optional => 1,
 610     },
 611     model => {
 612         type => 'string',
 613         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 614         enum => $nic_model_list,
 615         default_key => 1,
 616     },
 617     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 618     bridge => {
 619         type => 'string',
 620         description => $net_fmt_bridge_descr,
 621         format_description => 'bridge',
 622         optional => 1,
 623     },
 624     queues => {
 625         type => 'integer',
 626         minimum => 0, maximum => 16,
 627         description => 'Number of packet queues to be used on the device.',
 628         optional => 1,
 629     },
 630     rate => {
 631         type => 'number',
 632         minimum => 0,
 633         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 634         optional => 1,
 635     },
 636     tag => {
 637         type => 'integer',
 638         minimum => 1, maximum => 4094,
 639         description => 'VLAN tag to apply to packets on this interface.',
 640         optional => 1,
 641     },
 642     trunks => {
 643         type => 'string',
 644         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 645         description => 'VLAN trunks to pass through this interface.',
 646         format_description => 'vlanid[;vlanid...]',
 647         optional => 1,
 648     },
 649     firewall => {
 650         type => 'boolean',
 651         description => 'Whether this interface should be protected by the firewall.',
 652         optional => 1,
 653     },
 654     link_down => {
 655         type => 'boolean',
 656         description => 'Whether this interface should be disconnected (like pulling the plug).',
 657         optional => 1,
 658     },
 659 };
 660
 661 my $netdesc = {
 662     optional => 1,
 663     type => 'string', format => $net_fmt,
 664     description => "Specify network devices.",
 665 };
 666
 667 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 668
 669 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 670     $confdesc->{"net$i"} = $netdesc;
 671 }
 672
 673 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 674 sub verify_volume_id_or_qm_path {
 675     my ($volid, $noerr) = @_;
 676
 677     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 678         return $volid;
 679     }
 680
 681     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 682     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 683     if ($@) {
 684         return undef if $noerr;
 685         die $@;
 686     }
 687     return $volid;
 688 }
 689
 690 my $drivename_hash;
 691
 692 my %drivedesc_base = (
 693     volume => { alias => 'file' },
 694     file => {
 695         type => 'string',
 696         format => 'pve-volume-id-or-qm-path',
 697         default_key => 1,
 698         format_description => 'volume',
 699         description => "The drive's backing volume.",
 700     },
 701     media => {
 702         type => 'string',
 703         enum => [qw(cdrom disk)],
 704         description => "The drive's media type.",
 705         default => 'disk',
 706         optional => 1
 707     },
 708     cyls => {
 709         type => 'integer',
 710         description => "Force the drive's physical geometry to have a specific cylinder count.",
 711         optional => 1
 712     },
 713     heads => {
 714         type => 'integer',
 715         description => "Force the drive's physical geometry to have a specific head count.",
 716         optional => 1
 717     },
 718     secs => {
 719         type => 'integer',
 720         description => "Force the drive's physical geometry to have a specific sector count.",
 721         optional => 1
 722     },
 723     trans => {
 724         type => 'string',
 725         enum => [qw(none lba auto)],
 726         description => "Force disk geometry bios translation mode.",
 727         optional => 1,
 728     },
 729     snapshot => {
 730         type => 'boolean',
 731         description => "Whether the drive should be included when making snapshots.",
 732         optional => 1,
 733     },
 734     cache => {
 735         type => 'string',
 736         enum => [qw(none writethrough writeback unsafe directsync)],
 737         description => "The drive's cache mode",
 738         optional => 1,
 739     },
 740     format => get_standard_option('pve-qm-image-format'),
 741     size => {
 742         type => 'string',
 743         format => 'disk-size',
 744         format_description => 'DiskSize',
 745         description => "Disk size. This is purely informational and has no effect.",
 746         optional => 1,
 747     },
 748     backup => {
 749         type => 'boolean',
 750         description => "Whether the drive should be included when making backups.",
 751         optional => 1,
 752     },
 753     replicate => {
 754         type => 'boolean',
 755         description => 'Whether the drive should considered for replication jobs.',
 756         optional => 1,
 757         default => 1,
 758     },
 759     rerror => {
 760         type => 'string',
 761         enum => [qw(ignore report stop)],
 762         description => 'Read error action.',
 763         optional => 1,
 764     },
 765     werror => {
 766         type => 'string',
 767         enum => [qw(enospc ignore report stop)],
 768         description => 'Write error action.',
 769         optional => 1,
 770     },
 771     aio => {
 772         type => 'string',
 773         enum => [qw(native threads)],
 774         description => 'AIO type to use.',
 775         optional => 1,
 776     },
 777     discard => {
 778         type => 'string',
 779         enum => [qw(ignore on)],
 780         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 781         optional => 1,
 782     },
 783     detect_zeroes => {
 784         type => 'boolean',
 785         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 786         optional => 1,
 787     },
 788     serial => {
 789         type => 'string',
 790         format => 'urlencoded',
 791         format_description => 'serial',
 792         maxLength => 20*3, # *3 since it's %xx url enoded
 793         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
 794         optional => 1,
 795     }
 796 );
 797
 798 my %iothread_fmt = ( iothread => {
 799         type => 'boolean',
 800         description => "Whether to use iothreads for this drive",
 801         optional => 1,
 802 });
 803
 804 my %model_fmt = (
 805     model => {
 806         type => 'string',
 807         format => 'urlencoded',
 808         format_description => 'model',
 809         maxLength => 40*3, # *3 since it's %xx url enoded
 810         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
 811         optional => 1,
 812     },
 813 );
 814
 815 my %queues_fmt = (
 816     queues => {
 817         type => 'integer',
 818         description => "Number of queues.",
 819         minimum => 2,
 820         optional => 1
 821     }
 822 );
 823
 824 my %scsiblock_fmt = (
 825     scsiblock => {
 826         type => 'boolean',
 827         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
 828         optional => 1,
 829         default => 0,
 830     },
 831 );
 832
 833 my $add_throttle_desc = sub {
 834     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
 835     my $d = {
 836         type => $type,
 837         format_description => $unit,
 838         description => "Maximum $what in $longunit.",
 839         optional => 1,
 840     };
 841     $d->{minimum} = $minimum if defined($minimum);
 842     $drivedesc_base{$key} = $d;
 843 };
 844 # throughput: (leaky bucket)
 845 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
 846 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
 847 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
 848 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
 849 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
 850 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
 851 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
 852 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
 853 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
 854
 855 # pools: (pool of IO before throttling starts taking effect)
 856 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
 857 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
 858 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
 859 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
 860 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
 861 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
 862
 863 # burst lengths
 864 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
 865 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
 866 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
 867 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
 868 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
 869 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
 870
 871 # legacy support
 872 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
 873 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
 874 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
 875 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
 876
 877 my $ide_fmt = {
 878     %drivedesc_base,
 879     %model_fmt,
 880 };
 881 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
 882
 883 my $idedesc = {
 884     optional => 1,
 885     type => 'string', format => $ide_fmt,
 886     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
 887 };
 888 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
 889
 890 my $scsi_fmt = {
 891     %drivedesc_base,
 892     %iothread_fmt,
 893     %queues_fmt,
 894     %scsiblock_fmt,
 895 };
 896 my $scsidesc = {
 897     optional => 1,
 898     type => 'string', format => $scsi_fmt,
 899     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
 900 };
 901 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 902
 903 my $sata_fmt = {
 904     %drivedesc_base,
 905 };
 906 my $satadesc = {
 907     optional => 1,
 908     type => 'string', format => $sata_fmt,
 909     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
 910 };
 911 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
 912
 913 my $virtio_fmt = {
 914     %drivedesc_base,
 915     %iothread_fmt,
 916 };
 917 my $virtiodesc = {
 918     optional => 1,
 919     type => 'string', format => $virtio_fmt,
 920     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
 921 };
 922 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
 923
 924 my $alldrive_fmt = {
 925     %drivedesc_base,
 926     %iothread_fmt,
 927     %model_fmt,
 928     %queues_fmt,
 929     %scsiblock_fmt,
 930 };
 931
 932 my $efidisk_fmt = {
 933     volume => { alias => 'file' },
 934     file => {
 935         type => 'string',
 936         format => 'pve-volume-id-or-qm-path',
 937         default_key => 1,
 938         format_description => 'volume',
 939         description => "The drive's backing volume.",
 940     },
 941     format => get_standard_option('pve-qm-image-format'),
 942     size => {
 943         type => 'string',
 944         format => 'disk-size',
 945         format_description => 'DiskSize',
 946         description => "Disk size. This is purely informational and has no effect.",
 947         optional => 1,
 948     },
 949 };
 950
 951 my $efidisk_desc = {
 952     optional => 1,
 953     type => 'string', format => $efidisk_fmt,
 954     description => "Configure a Disk for storing EFI vars",
 955 };
 956
 957 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
 958
 959 my $usb_fmt = {
 960     host => {
 961         default_key => 1,
 962         type => 'string', format => 'pve-qm-usb-device',
 963         format_description => 'HOSTUSBDEVICE|spice',
 964         description => <<EODESCR,
 965 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
 966
 967  'bus-port(.port)*' (decimal numbers) or
 968  'vendor_id:product_id' (hexadeciaml numbers) or
 969  'spice'
 970
 971 You can use the 'lsusb -t' command to list existing usb devices.
 972
 973 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 974
 975 The value 'spice' can be used to add a usb redirection devices for spice.
 976 EODESCR
 977     },
 978     usb3 => {
 979         optional => 1,
 980         type => 'boolean',
 981         description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
 982         default => 0,
 983     },
 984 };
 985
 986 my $usbdesc = {
 987     optional => 1,
 988     type => 'string', format => $usb_fmt,
 989     description => "Configure an USB device (n is 0 to 4).",
 990 };
 991 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
 992
 993 # NOTE: the match-groups of this regex are used in parse_hostpci
 994 my $PCIRE = qr/([a-f0-9]{2}:[a-f0-9]{2})(?:\.([a-f0-9]))?/;
 995 my $hostpci_fmt = {
 996     host => {
 997         default_key => 1,
 998         type => 'string',
 999         pattern => qr/$PCIRE(;$PCIRE)*/,
1000         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1001         description => <<EODESCR,
1002 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1003 of PCI virtual functions of the host. HOSTPCIID syntax is:
1004
1005 'bus:dev.func' (hexadecimal numbers)
1006
1007 You can us the 'lspci' command to list existing PCI devices.
1008 EODESCR
1009     },
1010     rombar => {
1011         type => 'boolean',
1012         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1013         optional => 1,
1014         default => 1,
1015     },
1016     romfile => {
1017         type => 'string',
1018         pattern => '[^,;]+',
1019         format_description => 'string',
1020         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1021         optional => 1,
1022     },
1023     pcie => {
1024         type => 'boolean',
1025         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1026         optional => 1,
1027         default => 0,
1028     },
1029     'x-vga' => {
1030         type => 'boolean',
1031         description =>  "Enable vfio-vga device support.",
1032         optional => 1,
1033         default => 0,
1034     },
1035 };
1036 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1037
1038 my $hostpcidesc = {
1039         optional => 1,
1040         type => 'string', format => 'pve-qm-hostpci',
1041         description => "Map host PCI devices into guest.",
1042         verbose_description =>  <<EODESCR,
1043 Map host PCI devices into guest.
1044
1045 NOTE: This option allows direct access to host hardware. So it is no longer
1046 possible to migrate such machines - use with special care.
1047
1048 CAUTION: Experimental! User reported problems with this option.
1049 EODESCR
1050 };
1051 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1052
1053 my $serialdesc = {
1054         optional => 1,
1055         type => 'string',
1056         pattern => '(/dev/.+|socket)',
1057         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1058         verbose_description =>  <<EODESCR,
1059 Create a serial device inside the VM (n is 0 to 3), and pass through a
1060 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1061 host side (use 'qm terminal' to open a terminal connection).
1062
1063 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1064
1065 CAUTION: Experimental! User reported problems with this option.
1066 EODESCR
1067 };
1068
1069 my $paralleldesc= {
1070         optional => 1,
1071         type => 'string',
1072         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1073         description =>  "Map host parallel devices (n is 0 to 2).",
1074         verbose_description =>  <<EODESCR,
1075 Map host parallel devices (n is 0 to 2).
1076
1077 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1078
1079 CAUTION: Experimental! User reported problems with this option.
1080 EODESCR
1081 };
1082
1083 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1084     $confdesc->{"parallel$i"} = $paralleldesc;
1085 }
1086
1087 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1088     $confdesc->{"serial$i"} = $serialdesc;
1089 }
1090
1091 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1092     $confdesc->{"hostpci$i"} = $hostpcidesc;
1093 }
1094
1095 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1096     $drivename_hash->{"ide$i"} = 1;
1097     $confdesc->{"ide$i"} = $idedesc;
1098 }
1099
1100 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1101     $drivename_hash->{"sata$i"} = 1;
1102     $confdesc->{"sata$i"} = $satadesc;
1103 }
1104
1105 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1106     $drivename_hash->{"scsi$i"} = 1;
1107     $confdesc->{"scsi$i"} = $scsidesc ;
1108 }
1109
1110 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1111     $drivename_hash->{"virtio$i"} = 1;
1112     $confdesc->{"virtio$i"} = $virtiodesc;
1113 }
1114
1115 $drivename_hash->{efidisk0} = 1;
1116 $confdesc->{efidisk0} = $efidisk_desc;
1117
1118 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1119     $confdesc->{"usb$i"} = $usbdesc;
1120 }
1121
1122 my $unuseddesc = {
1123     optional => 1,
1124     type => 'string', format => 'pve-volume-id',
1125     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1126 };
1127
1128 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1129     $confdesc->{"unused$i"} = $unuseddesc;
1130 }
1131
1132 my $kvm_api_version = 0;
1133
1134 sub kvm_version {
1135
1136     return $kvm_api_version if $kvm_api_version;
1137
1138     my $fh = IO::File->new("</dev/kvm") ||
1139         return 0;
1140
1141     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
1142         $kvm_api_version = $v;
1143     }
1144
1145     $fh->close();
1146
1147     return  $kvm_api_version;
1148 }
1149
1150 my $kvm_user_version;
1151
1152 sub kvm_user_version {
1153
1154     return $kvm_user_version if $kvm_user_version;
1155
1156     $kvm_user_version = 'unknown';
1157
1158     my $code = sub {
1159         my $line = shift;
1160         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1161             $kvm_user_version = $2;
1162         }
1163     };
1164
1165     eval { run_command("kvm -version", outfunc => $code); };
1166     warn $@ if $@;
1167
1168     return $kvm_user_version;
1169
1170 }
1171
1172 my $kernel_has_vhost_net = -c '/dev/vhost-net';
1173
1174 sub valid_drive_names {
1175     # order is important - used to autoselect boot disk
1176     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1177             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1178             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1179             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1180             'efidisk0');
1181 }
1182
1183 sub is_valid_drivename {
1184     my $dev = shift;
1185
1186     return defined($drivename_hash->{$dev});
1187 }
1188
1189 sub option_exists {
1190     my $key = shift;
1191     return defined($confdesc->{$key});
1192 }
1193
1194 sub nic_models {
1195     return $nic_model_list;
1196 }
1197
1198 sub os_list_description {
1199
1200     return {
1201         other => 'Other',
1202         wxp => 'Windows XP',
1203         w2k => 'Windows 2000',
1204         w2k3 =>, 'Windows 2003',
1205         w2k8 => 'Windows 2008',
1206         wvista => 'Windows Vista',
1207         win7 => 'Windows 7',
1208         win8 => 'Windows 8/2012',
1209         win10 => 'Windows 10/2016',
1210         l24 => 'Linux 2.4',
1211         l26 => 'Linux 2.6',
1212     };
1213 }
1214
1215 my $cdrom_path;
1216
1217 sub get_cdrom_path {
1218
1219     return  $cdrom_path if $cdrom_path;
1220
1221     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1222     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1223     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1224 }
1225
1226 sub get_iso_path {
1227     my ($storecfg, $vmid, $cdrom) = @_;
1228
1229     if ($cdrom eq 'cdrom') {
1230         return get_cdrom_path();
1231     } elsif ($cdrom eq 'none') {
1232         return '';
1233     } elsif ($cdrom =~ m|^/|) {
1234         return $cdrom;
1235     } else {
1236         return PVE::Storage::path($storecfg, $cdrom);
1237     }
1238 }
1239
1240 # try to convert old style file names to volume IDs
1241 sub filename_to_volume_id {
1242     my ($vmid, $file, $media) = @_;
1243
1244     if (!($file eq 'none' || $file eq 'cdrom' ||
1245           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1246
1247         return undef if $file =~ m|/|;
1248
1249         if ($media && $media eq 'cdrom') {
1250             $file = "local:iso/$file";
1251         } else {
1252             $file = "local:$vmid/$file";
1253         }
1254     }
1255
1256     return $file;
1257 }
1258
1259 sub verify_media_type {
1260     my ($opt, $vtype, $media) = @_;
1261
1262     return if !$media;
1263
1264     my $etype;
1265     if ($media eq 'disk') {
1266         $etype = 'images';
1267     } elsif ($media eq 'cdrom') {
1268         $etype = 'iso';
1269     } else {
1270         die "internal error";
1271     }
1272
1273     return if ($vtype eq $etype);
1274
1275     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1276 }
1277
1278 sub cleanup_drive_path {
1279     my ($opt, $storecfg, $drive) = @_;
1280
1281     # try to convert filesystem paths to volume IDs
1282
1283     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1284         ($drive->{file} !~ m|^/dev/.+|) &&
1285         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1286         ($drive->{file} !~ m/^\d+$/)) {
1287         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1288         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1289         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1290         verify_media_type($opt, $vtype, $drive->{media});
1291         $drive->{file} = $volid;
1292     }
1293
1294     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1295 }
1296
1297 sub parse_hotplug_features {
1298     my ($data) = @_;
1299
1300     my $res = {};
1301
1302     return $res if $data eq '0';
1303
1304     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1305
1306     foreach my $feature (PVE::Tools::split_list($data)) {
1307         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1308             $res->{$1} = 1;
1309         } else {
1310             die "invalid hotplug feature '$feature'\n";
1311         }
1312     }
1313     return $res;
1314 }
1315
1316 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1317 sub pve_verify_hotplug_features {
1318     my ($value, $noerr) = @_;
1319
1320     return $value if parse_hotplug_features($value);
1321
1322     return undef if $noerr;
1323
1324     die "unable to parse hotplug option\n";
1325 }
1326
1327 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1328 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1329 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1330 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1331 #        [,iothread=on][,serial=serial][,model=model]
1332
1333 sub parse_drive {
1334     my ($key, $data) = @_;
1335
1336     my ($interface, $index);
1337
1338     if ($key =~ m/^([^\d]+)(\d+)$/) {
1339         $interface = $1;
1340         $index = $2;
1341     } else {
1342         return undef;
1343     }
1344
1345     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1346                                      : $confdesc->{$key}->{format};
1347     if (!$desc) {
1348         warn "invalid drive key: $key\n";
1349         return undef;
1350     }
1351     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1352     return undef if !$res;
1353     $res->{interface} = $interface;
1354     $res->{index} = $index;
1355
1356     my $error = 0;
1357     foreach my $opt (qw(bps bps_rd bps_wr)) {
1358         if (my $bps = defined(delete $res->{$opt})) {
1359             if (defined($res->{"m$opt"})) {
1360                 warn "both $opt and m$opt specified\n";
1361                 ++$error;
1362                 next;
1363             }
1364             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1365         }
1366     }
1367
1368     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1369     for my $requirement (
1370         [mbps_max => 'mbps'],
1371         [mbps_rd_max => 'mbps_rd'],
1372         [mbps_wr_max => 'mbps_wr'],
1373         [miops_max => 'miops'],
1374         [miops_rd_max => 'miops_rd'],
1375         [miops_wr_max => 'miops_wr'],
1376         [bps_max_length => 'mbps_max'],
1377         [bps_rd_max_length => 'mbps_rd_max'],
1378         [bps_wr_max_length => 'mbps_wr_max'],
1379         [iops_max_length => 'iops_max'],
1380         [iops_rd_max_length => 'iops_rd_max'],
1381         [iops_wr_max_length => 'iops_wr_max']) {
1382         my ($option, $requires) = @$requirement;
1383         if ($res->{$option} && !$res->{$requires}) {
1384             warn "$option requires $requires\n";
1385             ++$error;
1386         }
1387     }
1388
1389     return undef if $error;
1390
1391     return undef if $res->{mbps_rd} && $res->{mbps};
1392     return undef if $res->{mbps_wr} && $res->{mbps};
1393     return undef if $res->{iops_rd} && $res->{iops};
1394     return undef if $res->{iops_wr} && $res->{iops};
1395
1396     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1397         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1398         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1399         return undef if $res->{interface} eq 'virtio';
1400     }
1401
1402     if (my $size = $res->{size}) {
1403         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1404     }
1405
1406     return $res;
1407 }
1408
1409 sub print_drive {
1410     my ($vmid, $drive) = @_;
1411     my $data = { %$drive };
1412     delete $data->{$_} for qw(index interface);
1413     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1414 }
1415
1416 sub scsi_inquiry {
1417     my($fh, $noerr) = @_;
1418
1419     my $SG_IO = 0x2285;
1420     my $SG_GET_VERSION_NUM = 0x2282;
1421
1422     my $versionbuf = "\x00" x 8;
1423     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1424     if (!$ret) {
1425         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1426         return undef;
1427     }
1428     my $version = unpack("I", $versionbuf);
1429     if ($version < 30000) {
1430         die "scsi generic interface too old\n"  if !$noerr;
1431         return undef;
1432     }
1433
1434     my $buf = "\x00" x 36;
1435     my $sensebuf = "\x00" x 8;
1436     my $cmd = pack("C x3 C x1", 0x12, 36);
1437
1438     # see /usr/include/scsi/sg.h
1439     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1440
1441     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1442                       length($sensebuf), 0, length($buf), $buf,
1443                       $cmd, $sensebuf, 6000);
1444
1445     $ret = ioctl($fh, $SG_IO, $packet);
1446     if (!$ret) {
1447         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1448         return undef;
1449     }
1450
1451     my @res = unpack($sg_io_hdr_t, $packet);
1452     if ($res[17] || $res[18]) {
1453         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1454         return undef;
1455     }
1456
1457     my $res = {};
1458     (my $byte0, my $byte1, $res->{vendor},
1459      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1460
1461     $res->{removable} = $byte1 & 128 ? 1 : 0;
1462     $res->{type} = $byte0 & 31;
1463
1464     return $res;
1465 }
1466
1467 sub path_is_scsi {
1468     my ($path) = @_;
1469
1470     my $fh = IO::File->new("+<$path") || return undef;
1471     my $res = scsi_inquiry($fh, 1);
1472     close($fh);
1473
1474     return $res;
1475 }
1476
1477 sub machine_type_is_q35 {
1478     my ($conf) = @_;
1479
1480     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1481 }
1482
1483 sub print_tabletdevice_full {
1484     my ($conf) = @_;
1485
1486     my $q35 = machine_type_is_q35($conf);
1487
1488     # we use uhci for old VMs because tablet driver was buggy in older qemu
1489     my $usbbus = $q35 ? "ehci" : "uhci";
1490
1491     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1492 }
1493
1494 sub print_drivedevice_full {
1495     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1496
1497     my $device = '';
1498     my $maxdev = 0;
1499
1500     if ($drive->{interface} eq 'virtio') {
1501         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1502         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1503         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1504     } elsif ($drive->{interface} eq 'scsi') {
1505
1506         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1507         my $unit = $drive->{index} % $maxdev;
1508         my $devicetype = 'hd';
1509         my $path = '';
1510         if (drive_is_cdrom($drive)) {
1511             $devicetype = 'cd';
1512         } else {
1513             if ($drive->{file} =~ m|^/|) {
1514                 $path = $drive->{file};
1515                 if (my $info = path_is_scsi($path)) {
1516                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1517                         $devicetype = 'block';
1518                     } elsif ($info->{type} == 1) { # tape
1519                         $devicetype = 'generic';
1520                     }
1521                 }
1522             } else {
1523                  $path = PVE::Storage::path($storecfg, $drive->{file});
1524             }
1525
1526             if($path =~ m/^iscsi\:\/\//){
1527                 $devicetype = 'generic';
1528             }
1529         }
1530
1531         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1532             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1533         } else {
1534             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1535         }
1536
1537     } elsif ($drive->{interface} eq 'ide'){
1538         $maxdev = 2;
1539         my $controller = int($drive->{index} / $maxdev);
1540         my $unit = $drive->{index} % $maxdev;
1541         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1542
1543         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1544         if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
1545             $model = URI::Escape::uri_unescape($model);
1546             $device .= ",model=$model";
1547         }
1548     } elsif ($drive->{interface} eq 'sata'){
1549         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1550         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1551         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1552     } elsif ($drive->{interface} eq 'usb') {
1553         die "implement me";
1554         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1555     } else {
1556         die "unsupported interface type";
1557     }
1558
1559     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1560
1561     return $device;
1562 }
1563
1564 sub get_initiator_name {
1565     my $initiator;
1566
1567     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1568     while (defined(my $line = <$fh>)) {
1569         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1570         $initiator = $1;
1571         last;
1572     }
1573     $fh->close();
1574
1575     return $initiator;
1576 }
1577
1578 sub print_drive_full {
1579     my ($storecfg, $vmid, $drive) = @_;
1580
1581     my $path;
1582     my $volid = $drive->{file};
1583     my $format;
1584
1585     if (drive_is_cdrom($drive)) {
1586         $path = get_iso_path($storecfg, $vmid, $volid);
1587     } else {
1588         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1589         if ($storeid) {
1590             $path = PVE::Storage::path($storecfg, $volid);
1591             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1592             $format = qemu_img_format($scfg, $volname);
1593         } else {
1594             $path = $volid;
1595             $format = "raw";
1596         }
1597    }
1598
1599     my $opts = '';
1600     my @qemu_drive_options = qw(heads secs cyls trans media format cache snapshot rerror werror aio discard);
1601     foreach my $o (@qemu_drive_options) {
1602         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1603     }
1604     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1605         my ($dir, $qmpname) = @$type;
1606         if (my $v = $drive->{"mbps$dir"}) {
1607             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1608         }
1609         if (my $v = $drive->{"mbps${dir}_max"}) {
1610             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1611         }
1612         if (my $v = $drive->{"bps${dir}_max_length"}) {
1613             $opts .= ",throttling.bps$qmpname-max-length=$v";
1614         }
1615         if (my $v = $drive->{"iops${dir}"}) {
1616             $opts .= ",throttling.iops$qmpname=$v";
1617         }
1618         if (my $v = $drive->{"iops${dir}_max"}) {
1619             $opts .= ",throttling.iops$qmpname-max=$v";
1620         }
1621         if (my $v = $drive->{"iops${dir}_max_length"}) {
1622             $opts .= ",throttling.iops$qmpname-max-length=$v";
1623         }
1624     }
1625
1626     if (my $serial = $drive->{serial}) {
1627         $serial = URI::Escape::uri_unescape($serial);
1628         $opts .= ",serial=$serial";
1629     }
1630
1631     $opts .= ",format=$format" if $format && !$drive->{format};
1632
1633     my $cache_direct = 0;
1634
1635     if (my $cache = $drive->{cache}) {
1636         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1637     } elsif (!drive_is_cdrom($drive)) {
1638         $opts .= ",cache=none";
1639         $cache_direct = 1;
1640     }
1641
1642     # aio native works only with O_DIRECT
1643     if (!$drive->{aio}) {
1644         if($cache_direct) {
1645             $opts .= ",aio=native";
1646         } else {
1647             $opts .= ",aio=threads";
1648         }
1649     }
1650
1651     if (!drive_is_cdrom($drive)) {
1652         my $detectzeroes;
1653         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1654             $detectzeroes = 'off';
1655         } elsif ($drive->{discard}) {
1656             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1657         } else {
1658             # This used to be our default with discard not being specified:
1659             $detectzeroes = 'on';
1660         }
1661         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1662     }
1663
1664     my $pathinfo = $path ? "file=$path," : '';
1665
1666     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1667 }
1668
1669 sub print_netdevice_full {
1670     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1671
1672     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1673
1674     my $device = $net->{model};
1675     if ($net->{model} eq 'virtio') {
1676          $device = 'virtio-net-pci';
1677      };
1678
1679     my $pciaddr = print_pci_addr("$netid", $bridges);
1680     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1681     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1682         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1683         my $vectors = $net->{queues} * 2 + 2;
1684         $tmpstr .= ",vectors=$vectors,mq=on";
1685     }
1686     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1687
1688     if ($use_old_bios_files) {
1689         my $romfile;
1690         if ($device eq 'virtio-net-pci') {
1691             $romfile = 'pxe-virtio.rom';
1692         } elsif ($device eq 'e1000') {
1693             $romfile = 'pxe-e1000.rom';
1694         } elsif ($device eq 'ne2k') {
1695             $romfile = 'pxe-ne2k_pci.rom';
1696         } elsif ($device eq 'pcnet') {
1697             $romfile = 'pxe-pcnet.rom';
1698         } elsif ($device eq 'rtl8139') {
1699             $romfile = 'pxe-rtl8139.rom';
1700         }
1701         $tmpstr .= ",romfile=$romfile" if $romfile;
1702     }
1703
1704     return $tmpstr;
1705 }
1706
1707 sub print_netdev_full {
1708     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1709
1710     my $i = '';
1711     if ($netid =~ m/^net(\d+)$/) {
1712         $i = int($1);
1713     }
1714
1715     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1716
1717     my $ifname = "tap${vmid}i$i";
1718
1719     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1720     die "interface name '$ifname' is too long (max 15 character)\n"
1721         if length($ifname) >= 16;
1722
1723     my $vhostparam = '';
1724     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1725
1726     my $vmname = $conf->{name} || "vm$vmid";
1727
1728     my $netdev = "";
1729     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1730
1731     if ($net->{bridge}) {
1732         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1733     } else {
1734         $netdev = "type=user,id=$netid,hostname=$vmname";
1735     }
1736
1737     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1738
1739     return $netdev;
1740 }
1741
1742
1743 sub print_cpu_device {
1744     my ($conf, $id) = @_;
1745
1746     my $kvm = $conf->{kvm} // 1;
1747     my $cpu = $kvm ? "kvm64" : "qemu64";
1748     if (my $cputype = $conf->{cpu}) {
1749         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
1750             or die "Cannot parse cpu description: $cputype\n";
1751         $cpu = $cpuconf->{cputype};
1752     }
1753
1754     my $cores = $conf->{cores} || 1;
1755
1756     my $current_core = ($id - 1) % $cores;
1757     my $current_socket = int(($id - 1 - $current_core)/$cores);
1758
1759     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
1760 }
1761
1762 sub drive_is_cdrom {
1763     my ($drive) = @_;
1764
1765     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1766
1767 }
1768
1769 sub parse_number_sets {
1770     my ($set) = @_;
1771     my $res = [];
1772     foreach my $part (split(/;/, $set)) {
1773         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1774             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1775             push @$res, [ $1, $2 ];
1776         } else {
1777             die "invalid range: $part\n";
1778         }
1779     }
1780     return $res;
1781 }
1782
1783 sub parse_numa {
1784     my ($data) = @_;
1785
1786     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
1787     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1788     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1789     return $res;
1790 }
1791
1792 sub parse_hostpci {
1793     my ($value) = @_;
1794
1795     return undef if !$value;
1796
1797     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
1798
1799     my @idlist = split(/;/, $res->{host});
1800     delete $res->{host};
1801     foreach my $id (@idlist) {
1802         if ($id =~ /^$PCIRE$/) {
1803             if (defined($2)) {
1804                 push @{$res->{pciid}}, { id => $1, function => $2 };
1805             } else {
1806                 my $pcidevices = lspci($1);
1807                 $res->{pciid} = $pcidevices->{$1};
1808             }
1809         } else {
1810             # should have been caught by parse_property_string already
1811             die "failed to parse PCI id: $id\n";
1812         }
1813     }
1814     return $res;
1815 }
1816
1817 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1818 sub parse_net {
1819     my ($data) = @_;
1820
1821     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
1822     if ($@) {
1823         warn $@;
1824         return undef;
1825     }
1826     if (!defined($res->{macaddr})) {
1827         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1828         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1829     }
1830     return $res;
1831 }
1832
1833 sub print_net {
1834     my $net = shift;
1835
1836     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1837 }
1838
1839 sub add_random_macs {
1840     my ($settings) = @_;
1841
1842     foreach my $opt (keys %$settings) {
1843         next if $opt !~ m/^net(\d+)$/;
1844         my $net = parse_net($settings->{$opt});
1845         next if !$net;
1846         $settings->{$opt} = print_net($net);
1847     }
1848 }
1849
1850 sub vm_is_volid_owner {
1851     my ($storecfg, $vmid, $volid) = @_;
1852
1853     if ($volid !~  m|^/|) {
1854         my ($path, $owner);
1855         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1856         if ($owner && ($owner == $vmid)) {
1857             return 1;
1858         }
1859     }
1860
1861     return undef;
1862 }
1863
1864 sub split_flagged_list {
1865     my $text = shift || '';
1866     $text =~ s/[,;]/ /g;
1867     $text =~ s/^\s+//;
1868     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
1869 }
1870
1871 sub join_flagged_list {
1872     my ($how, $lst) = @_;
1873     join $how, map { $lst->{$_} . $_ } keys %$lst;
1874 }
1875
1876 sub vmconfig_delete_pending_option {
1877     my ($conf, $key, $force) = @_;
1878
1879     delete $conf->{pending}->{$key};
1880     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1881     $pending_delete_hash->{$key} = $force ? '!' : '';
1882     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1883 }
1884
1885 sub vmconfig_undelete_pending_option {
1886     my ($conf, $key) = @_;
1887
1888     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1889     delete $pending_delete_hash->{$key};
1890
1891     if (%$pending_delete_hash) {
1892         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1893     } else {
1894         delete $conf->{pending}->{delete};
1895     }
1896 }
1897
1898 sub vmconfig_register_unused_drive {
1899     my ($storecfg, $vmid, $conf, $drive) = @_;
1900
1901     if (!drive_is_cdrom($drive)) {
1902         my $volid = $drive->{file};
1903         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1904             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1905         }
1906     }
1907 }
1908
1909 sub vmconfig_cleanup_pending {
1910     my ($conf) = @_;
1911
1912     # remove pending changes when nothing changed
1913     my $changes;
1914     foreach my $opt (keys %{$conf->{pending}}) {
1915         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
1916             $changes = 1;
1917             delete $conf->{pending}->{$opt};
1918         }
1919     }
1920
1921     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
1922     my $pending_delete_hash = {};
1923     while (my ($opt, $force) = each %$current_delete_hash) {
1924         if (defined($conf->{$opt})) {
1925             $pending_delete_hash->{$opt} = $force;
1926         } else {
1927             $changes = 1;
1928         }
1929     }
1930
1931     if (%$pending_delete_hash) {
1932         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1933     } else {
1934         delete $conf->{pending}->{delete};
1935     }
1936
1937     return $changes;
1938 }
1939
1940 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
1941 my $smbios1_fmt = {
1942     uuid => {
1943         type => 'string',
1944         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1945         format_description => 'UUID',
1946         description => "Set SMBIOS1 UUID.",
1947         optional => 1,
1948     },
1949     version => {
1950         type => 'string',
1951         pattern => '\S+',
1952         format_description => 'string',
1953         description => "Set SMBIOS1 version.",
1954         optional => 1,
1955     },
1956     serial => {
1957         type => 'string',
1958         pattern => '\S+',
1959         format_description => 'string',
1960         description => "Set SMBIOS1 serial number.",
1961         optional => 1,
1962     },
1963     manufacturer => {
1964         type => 'string',
1965         pattern => '\S+',
1966         format_description => 'string',
1967         description => "Set SMBIOS1 manufacturer.",
1968         optional => 1,
1969     },
1970     product => {
1971         type => 'string',
1972         pattern => '\S+',
1973         format_description => 'string',
1974         description => "Set SMBIOS1 product ID.",
1975         optional => 1,
1976     },
1977     sku => {
1978         type => 'string',
1979         pattern => '\S+',
1980         format_description => 'string',
1981         description => "Set SMBIOS1 SKU string.",
1982         optional => 1,
1983     },
1984     family => {
1985         type => 'string',
1986         pattern => '\S+',
1987         format_description => 'string',
1988         description => "Set SMBIOS1 family string.",
1989         optional => 1,
1990     },
1991 };
1992
1993 sub parse_smbios1 {
1994     my ($data) = @_;
1995
1996     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
1997     warn $@ if $@;
1998     return $res;
1999 }
2000
2001 sub print_smbios1 {
2002     my ($smbios1) = @_;
2003     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2004 }
2005
2006 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2007
2008 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2009 sub verify_bootdisk {
2010     my ($value, $noerr) = @_;
2011
2012     return $value if is_valid_drivename($value);
2013
2014     return undef if $noerr;
2015
2016     die "invalid boot disk '$value'\n";
2017 }
2018
2019 sub parse_watchdog {
2020     my ($value) = @_;
2021
2022     return undef if !$value;
2023
2024     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2025     warn $@ if $@;
2026     return $res;
2027 }
2028
2029 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2030 sub verify_usb_device {
2031     my ($value, $noerr) = @_;
2032
2033     return $value if parse_usb_device($value);
2034
2035     return undef if $noerr;
2036
2037     die "unable to parse usb device\n";
2038 }
2039
2040 # add JSON properties for create and set function
2041 sub json_config_properties {
2042     my $prop = shift;
2043
2044     foreach my $opt (keys %$confdesc) {
2045         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
2046         $prop->{$opt} = $confdesc->{$opt};
2047     }
2048
2049     return $prop;
2050 }
2051
2052 sub check_type {
2053     my ($key, $value) = @_;
2054
2055     die "unknown setting '$key'\n" if !$confdesc->{$key};
2056
2057     my $type = $confdesc->{$key}->{type};
2058
2059     if (!defined($value)) {
2060         die "got undefined value\n";
2061     }
2062
2063     if ($value =~ m/[\n\r]/) {
2064         die "property contains a line feed\n";
2065     }
2066
2067     if ($type eq 'boolean') {
2068         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2069         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2070         die "type check ('boolean') failed - got '$value'\n";
2071     } elsif ($type eq 'integer') {
2072         return int($1) if $value =~ m/^(\d+)$/;
2073         die "type check ('integer') failed - got '$value'\n";
2074     } elsif ($type eq 'number') {
2075         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2076         die "type check ('number') failed - got '$value'\n";
2077     } elsif ($type eq 'string') {
2078         if (my $fmt = $confdesc->{$key}->{format}) {
2079             PVE::JSONSchema::check_format($fmt, $value);
2080             return $value;
2081         }
2082         $value =~ s/^\"(.*)\"$/$1/;
2083         return $value;
2084     } else {
2085         die "internal error"
2086     }
2087 }
2088
2089 sub check_iommu_support{
2090     #fixme : need to check IOMMU support
2091     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
2092
2093     my $iommu=1;
2094     return $iommu;
2095
2096 }
2097
2098 sub touch_config {
2099     my ($vmid) = @_;
2100
2101     my $conf = PVE::QemuConfig->config_file($vmid);
2102     utime undef, undef, $conf;
2103 }
2104
2105 sub destroy_vm {
2106     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2107
2108     my $conffile = PVE::QemuConfig->config_file($vmid);
2109
2110     my $conf = PVE::QemuConfig->load_config($vmid);
2111
2112     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2113
2114     if ($conf->{template}) {
2115         # check if any base image is still used by a linked clone
2116         foreach_drive($conf, sub {
2117                 my ($ds, $drive) = @_;
2118
2119                 return if drive_is_cdrom($drive);
2120
2121                 my $volid = $drive->{file};
2122
2123                 return if !$volid || $volid =~ m|^/|;
2124
2125                 die "base volume '$volid' is still in use by linked cloned\n"
2126                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2127
2128         });
2129     }
2130
2131     # only remove disks owned by this VM
2132     foreach_drive($conf, sub {
2133         my ($ds, $drive) = @_;
2134
2135         return if drive_is_cdrom($drive);
2136
2137         my $volid = $drive->{file};
2138
2139         return if !$volid || $volid =~ m|^/|;
2140
2141         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2142         return if !$path || !$owner || ($owner != $vmid);
2143
2144         eval {
2145             PVE::Storage::vdisk_free($storecfg, $volid);
2146         };
2147         warn "Could not remove disk '$volid', check manually: $@" if $@;
2148
2149     });
2150
2151     if ($keep_empty_config) {
2152         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2153     } else {
2154         unlink $conffile;
2155     }
2156
2157     # also remove unused disk
2158     eval {
2159         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2160
2161         eval {
2162             PVE::Storage::foreach_volid($dl, sub {
2163                 my ($volid, $sid, $volname, $d) = @_;
2164                 PVE::Storage::vdisk_free($storecfg, $volid);
2165             });
2166         };
2167         warn $@ if $@;
2168
2169     };
2170     warn $@ if $@;
2171 }
2172
2173 sub parse_vm_config {
2174     my ($filename, $raw) = @_;
2175
2176     return undef if !defined($raw);
2177
2178     my $res = {
2179         digest => Digest::SHA::sha1_hex($raw),
2180         snapshots => {},
2181         pending => {},
2182     };
2183
2184     $filename =~ m|/qemu-server/(\d+)\.conf$|
2185         || die "got strange filename '$filename'";
2186
2187     my $vmid = $1;
2188
2189     my $conf = $res;
2190     my $descr;
2191     my $section = '';
2192
2193     my @lines = split(/\n/, $raw);
2194     foreach my $line (@lines) {
2195         next if $line =~ m/^\s*$/;
2196
2197         if ($line =~ m/^\[PENDING\]\s*$/i) {
2198             $section = 'pending';
2199             if (defined($descr)) {
2200                 $descr =~ s/\s+$//;
2201                 $conf->{description} = $descr;
2202             }
2203             $descr = undef;
2204             $conf = $res->{$section} = {};
2205             next;
2206
2207         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2208             $section = $1;
2209             if (defined($descr)) {
2210                 $descr =~ s/\s+$//;
2211                 $conf->{description} = $descr;
2212             }
2213             $descr = undef;
2214             $conf = $res->{snapshots}->{$section} = {};
2215             next;
2216         }
2217
2218         if ($line =~ m/^\#(.*)\s*$/) {
2219             $descr = '' if !defined($descr);
2220             $descr .= PVE::Tools::decode_text($1) . "\n";
2221             next;
2222         }
2223
2224         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2225             $descr = '' if !defined($descr);
2226             $descr .= PVE::Tools::decode_text($2);
2227         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2228             $conf->{snapstate} = $1;
2229         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2230             my $key = $1;
2231             my $value = $2;
2232             $conf->{$key} = $value;
2233         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2234             my $value = $1;
2235             if ($section eq 'pending') {
2236                 $conf->{delete} = $value; # we parse this later
2237             } else {
2238                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2239             }
2240         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
2241             my $key = $1;
2242             my $value = $2;
2243             eval { $value = check_type($key, $value); };
2244             if ($@) {
2245                 warn "vm $vmid - unable to parse value of '$key' - $@";
2246             } else {
2247                 $key = 'ide2' if $key eq 'cdrom';
2248                 my $fmt = $confdesc->{$key}->{format};
2249                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2250                     my $v = parse_drive($key, $value);
2251                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2252                         $v->{file} = $volid;
2253                         $value = print_drive($vmid, $v);
2254                     } else {
2255                         warn "vm $vmid - unable to parse value of '$key'\n";
2256                         next;
2257                     }
2258                 }
2259
2260                 $conf->{$key} = $value;
2261             }
2262         }
2263     }
2264
2265     if (defined($descr)) {
2266         $descr =~ s/\s+$//;
2267         $conf->{description} = $descr;
2268     }
2269     delete $res->{snapstate}; # just to be sure
2270
2271     return $res;
2272 }
2273
2274 sub write_vm_config {
2275     my ($filename, $conf) = @_;
2276
2277     delete $conf->{snapstate}; # just to be sure
2278
2279     if ($conf->{cdrom}) {
2280         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2281         $conf->{ide2} = $conf->{cdrom};
2282         delete $conf->{cdrom};
2283     }
2284
2285     # we do not use 'smp' any longer
2286     if ($conf->{sockets}) {
2287         delete $conf->{smp};
2288     } elsif ($conf->{smp}) {
2289         $conf->{sockets} = $conf->{smp};
2290         delete $conf->{cores};
2291         delete $conf->{smp};
2292     }
2293
2294     my $used_volids = {};
2295
2296     my $cleanup_config = sub {
2297         my ($cref, $pending, $snapname) = @_;
2298
2299         foreach my $key (keys %$cref) {
2300             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2301                 $key eq 'snapstate' || $key eq 'pending';
2302             my $value = $cref->{$key};
2303             if ($key eq 'delete') {
2304                 die "propertry 'delete' is only allowed in [PENDING]\n"
2305                     if !$pending;
2306                 # fixme: check syntax?
2307                 next;
2308             }
2309             eval { $value = check_type($key, $value); };
2310             die "unable to parse value of '$key' - $@" if $@;
2311
2312             $cref->{$key} = $value;
2313
2314             if (!$snapname && is_valid_drivename($key)) {
2315                 my $drive = parse_drive($key, $value);
2316                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2317             }
2318         }
2319     };
2320
2321     &$cleanup_config($conf);
2322
2323     &$cleanup_config($conf->{pending}, 1);
2324
2325     foreach my $snapname (keys %{$conf->{snapshots}}) {
2326         die "internal error" if $snapname eq 'pending';
2327         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2328     }
2329
2330     # remove 'unusedX' settings if we re-add a volume
2331     foreach my $key (keys %$conf) {
2332         my $value = $conf->{$key};
2333         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2334             delete $conf->{$key};
2335         }
2336     }
2337
2338     my $generate_raw_config = sub {
2339         my ($conf, $pending) = @_;
2340
2341         my $raw = '';
2342
2343         # add description as comment to top of file
2344         if (defined(my $descr = $conf->{description})) {
2345             if ($descr) {
2346                 foreach my $cl (split(/\n/, $descr)) {
2347                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2348                 }
2349             } else {
2350                 $raw .= "#\n" if $pending;
2351             }
2352         }
2353
2354         foreach my $key (sort keys %$conf) {
2355             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2356             $raw .= "$key: $conf->{$key}\n";
2357         }
2358         return $raw;
2359     };
2360
2361     my $raw = &$generate_raw_config($conf);
2362
2363     if (scalar(keys %{$conf->{pending}})){
2364         $raw .= "\n[PENDING]\n";
2365         $raw .= &$generate_raw_config($conf->{pending}, 1);
2366     }
2367
2368     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2369         $raw .= "\n[$snapname]\n";
2370         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2371     }
2372
2373     return $raw;
2374 }
2375
2376 sub load_defaults {
2377
2378     my $res = {};
2379
2380     # we use static defaults from our JSON schema configuration
2381     foreach my $key (keys %$confdesc) {
2382         if (defined(my $default = $confdesc->{$key}->{default})) {
2383             $res->{$key} = $default;
2384         }
2385     }
2386
2387     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2388     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2389
2390     return $res;
2391 }
2392
2393 sub config_list {
2394     my $vmlist = PVE::Cluster::get_vmlist();
2395     my $res = {};
2396     return $res if !$vmlist || !$vmlist->{ids};
2397     my $ids = $vmlist->{ids};
2398
2399     foreach my $vmid (keys %$ids) {
2400         my $d = $ids->{$vmid};
2401         next if !$d->{node} || $d->{node} ne $nodename;
2402         next if !$d->{type} || $d->{type} ne 'qemu';
2403         $res->{$vmid}->{exists} = 1;
2404     }
2405     return $res;
2406 }
2407
2408 # test if VM uses local resources (to prevent migration)
2409 sub check_local_resources {
2410     my ($conf, $noerr) = @_;
2411
2412     my $loc_res = 0;
2413
2414     $loc_res = 1 if $conf->{hostusb}; # old syntax
2415     $loc_res = 1 if $conf->{hostpci}; # old syntax
2416
2417     foreach my $k (keys %$conf) {
2418         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2419         # sockets are safe: they will recreated be on the target side post-migrate
2420         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2421         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2422     }
2423
2424     die "VM uses local resources\n" if $loc_res && !$noerr;
2425
2426     return $loc_res;
2427 }
2428
2429 # check if used storages are available on all nodes (use by migrate)
2430 sub check_storage_availability {
2431     my ($storecfg, $conf, $node) = @_;
2432
2433     foreach_drive($conf, sub {
2434         my ($ds, $drive) = @_;
2435
2436         my $volid = $drive->{file};
2437         return if !$volid;
2438
2439         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2440         return if !$sid;
2441
2442         # check if storage is available on both nodes
2443         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2444         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2445    });
2446 }
2447
2448 # list nodes where all VM images are available (used by has_feature API)
2449 sub shared_nodes {
2450     my ($conf, $storecfg) = @_;
2451
2452     my $nodelist = PVE::Cluster::get_nodelist();
2453     my $nodehash = { map { $_ => 1 } @$nodelist };
2454     my $nodename = PVE::INotify::nodename();
2455
2456     foreach_drive($conf, sub {
2457         my ($ds, $drive) = @_;
2458
2459         my $volid = $drive->{file};
2460         return if !$volid;
2461
2462         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2463         if ($storeid) {
2464             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2465             if ($scfg->{disable}) {
2466                 $nodehash = {};
2467             } elsif (my $avail = $scfg->{nodes}) {
2468                 foreach my $node (keys %$nodehash) {
2469                     delete $nodehash->{$node} if !$avail->{$node};
2470                 }
2471             } elsif (!$scfg->{shared}) {
2472                 foreach my $node (keys %$nodehash) {
2473                     delete $nodehash->{$node} if $node ne $nodename
2474                 }
2475             }
2476         }
2477     });
2478
2479     return $nodehash
2480 }
2481
2482 sub check_cmdline {
2483     my ($pidfile, $pid) = @_;
2484
2485     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2486     if (defined($fh)) {
2487         my $line = <$fh>;
2488         $fh->close;
2489         return undef if !$line;
2490         my @param = split(/\0/, $line);
2491
2492         my $cmd = $param[0];
2493         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2494
2495         for (my $i = 0; $i < scalar (@param); $i++) {
2496             my $p = $param[$i];
2497             next if !$p;
2498             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2499                 my $p = $param[$i+1];
2500                 return 1 if $p && ($p eq $pidfile);
2501                 return undef;
2502             }
2503         }
2504     }
2505     return undef;
2506 }
2507
2508 sub check_running {
2509     my ($vmid, $nocheck, $node) = @_;
2510
2511     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2512
2513     die "unable to find configuration file for VM $vmid - no such machine\n"
2514         if !$nocheck && ! -f $filename;
2515
2516     my $pidfile = pidfile_name($vmid);
2517
2518     if (my $fd = IO::File->new("<$pidfile")) {
2519         my $st = stat($fd);
2520         my $line = <$fd>;
2521         close($fd);
2522
2523         my $mtime = $st->mtime;
2524         if ($mtime > time()) {
2525             warn "file '$filename' modified in future\n";
2526         }
2527
2528         if ($line =~ m/^(\d+)$/) {
2529             my $pid = $1;
2530             if (check_cmdline($pidfile, $pid)) {
2531                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2532                     return $pid;
2533                 }
2534             }
2535         }
2536     }
2537
2538     return undef;
2539 }
2540
2541 sub vzlist {
2542
2543     my $vzlist = config_list();
2544
2545     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2546
2547     while (defined(my $de = $fd->read)) {
2548         next if $de !~ m/^(\d+)\.pid$/;
2549         my $vmid = $1;
2550         next if !defined($vzlist->{$vmid});
2551         if (my $pid = check_running($vmid)) {
2552             $vzlist->{$vmid}->{pid} = $pid;
2553         }
2554     }
2555
2556     return $vzlist;
2557 }
2558
2559 sub disksize {
2560     my ($storecfg, $conf) = @_;
2561
2562     my $bootdisk = $conf->{bootdisk};
2563     return undef if !$bootdisk;
2564     return undef if !is_valid_drivename($bootdisk);
2565
2566     return undef if !$conf->{$bootdisk};
2567
2568     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2569     return undef if !defined($drive);
2570
2571     return undef if drive_is_cdrom($drive);
2572
2573     my $volid = $drive->{file};
2574     return undef if !$volid;
2575
2576     return $drive->{size};
2577 }
2578
2579 my $last_proc_pid_stat;
2580
2581 # get VM status information
2582 # This must be fast and should not block ($full == false)
2583 # We only query KVM using QMP if $full == true (this can be slow)
2584 sub vmstatus {
2585     my ($opt_vmid, $full) = @_;
2586
2587     my $res = {};
2588
2589     my $storecfg = PVE::Storage::config();
2590
2591     my $list = vzlist();
2592     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2593
2594     my $cpucount = $cpuinfo->{cpus} || 1;
2595
2596     foreach my $vmid (keys %$list) {
2597         next if $opt_vmid && ($vmid ne $opt_vmid);
2598
2599         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
2600         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2601
2602         my $d = {};
2603         $d->{pid} = $list->{$vmid}->{pid};
2604
2605         # fixme: better status?
2606         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2607
2608         my $size = disksize($storecfg, $conf);
2609         if (defined($size)) {
2610             $d->{disk} = 0; # no info available
2611             $d->{maxdisk} = $size;
2612         } else {
2613             $d->{disk} = 0;
2614             $d->{maxdisk} = 0;
2615         }
2616
2617         $d->{cpus} = ($conf->{sockets} || 1) * ($conf->{cores} || 1);
2618         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2619         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2620
2621         $d->{name} = $conf->{name} || "VM $vmid";
2622         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024) : 0;
2623
2624         if ($conf->{balloon}) {
2625             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2626             $d->{shares} = defined($conf->{shares}) ? $conf->{shares} : 1000;
2627         }
2628
2629         $d->{uptime} = 0;
2630         $d->{cpu} = 0;
2631         $d->{mem} = 0;
2632
2633         $d->{netout} = 0;
2634         $d->{netin} = 0;
2635
2636         $d->{diskread} = 0;
2637         $d->{diskwrite} = 0;
2638
2639         $d->{template} = PVE::QemuConfig->is_template($conf);
2640
2641         $res->{$vmid} = $d;
2642     }
2643
2644     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2645     foreach my $dev (keys %$netdev) {
2646         next if $dev !~ m/^tap([1-9]\d*)i/;
2647         my $vmid = $1;
2648         my $d = $res->{$vmid};
2649         next if !$d;
2650
2651         $d->{netout} += $netdev->{$dev}->{receive};
2652         $d->{netin} += $netdev->{$dev}->{transmit};
2653
2654         if ($full) {
2655             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2656             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2657         }
2658
2659     }
2660
2661     my $ctime = gettimeofday;
2662
2663     foreach my $vmid (keys %$list) {
2664
2665         my $d = $res->{$vmid};
2666         my $pid = $d->{pid};
2667         next if !$pid;
2668
2669         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2670         next if !$pstat; # not running
2671
2672         my $used = $pstat->{utime} + $pstat->{stime};
2673
2674         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2675
2676         if ($pstat->{vsize}) {
2677             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2678         }
2679
2680         my $old = $last_proc_pid_stat->{$pid};
2681         if (!$old) {
2682             $last_proc_pid_stat->{$pid} = {
2683                 time => $ctime,
2684                 used => $used,
2685                 cpu => 0,
2686             };
2687             next;
2688         }
2689
2690         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2691
2692         if ($dtime > 1000) {
2693             my $dutime = $used -  $old->{used};
2694
2695             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2696             $last_proc_pid_stat->{$pid} = {
2697                 time => $ctime,
2698                 used => $used,
2699                 cpu => $d->{cpu},
2700             };
2701         } else {
2702             $d->{cpu} = $old->{cpu};
2703         }
2704     }
2705
2706     return $res if !$full;
2707
2708     my $qmpclient = PVE::QMPClient->new();
2709
2710     my $ballooncb = sub {
2711         my ($vmid, $resp) = @_;
2712
2713         my $info = $resp->{'return'};
2714         return if !$info->{max_mem};
2715
2716         my $d = $res->{$vmid};
2717
2718         # use memory assigned to VM
2719         $d->{maxmem} = $info->{max_mem};
2720         $d->{balloon} = $info->{actual};
2721
2722         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2723             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2724             $d->{freemem} = $info->{free_mem};
2725         }
2726
2727         $d->{ballooninfo} = $info;
2728     };
2729
2730     my $blockstatscb = sub {
2731         my ($vmid, $resp) = @_;
2732         my $data = $resp->{'return'} || [];
2733         my $totalrdbytes = 0;
2734         my $totalwrbytes = 0;
2735
2736         for my $blockstat (@$data) {
2737             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2738             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2739
2740             $blockstat->{device} =~ s/drive-//;
2741             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2742         }
2743         $res->{$vmid}->{diskread} = $totalrdbytes;
2744         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2745     };
2746
2747     my $statuscb = sub {
2748         my ($vmid, $resp) = @_;
2749
2750         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2751         # this fails if ballon driver is not loaded, so this must be
2752         # the last commnand (following command are aborted if this fails).
2753         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2754
2755         my $status = 'unknown';
2756         if (!defined($status = $resp->{'return'}->{status})) {
2757             warn "unable to get VM status\n";
2758             return;
2759         }
2760
2761         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2762     };
2763
2764     foreach my $vmid (keys %$list) {
2765         next if $opt_vmid && ($vmid ne $opt_vmid);
2766         next if !$res->{$vmid}->{pid}; # not running
2767         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2768     }
2769
2770     $qmpclient->queue_execute(undef, 2);
2771
2772     foreach my $vmid (keys %$list) {
2773         next if $opt_vmid && ($vmid ne $opt_vmid);
2774         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2775     }
2776
2777     return $res;
2778 }
2779
2780 sub foreach_drive {
2781     my ($conf, $func, @param) = @_;
2782
2783     foreach my $ds (valid_drive_names()) {
2784         next if !defined($conf->{$ds});
2785
2786         my $drive = parse_drive($ds, $conf->{$ds});
2787         next if !$drive;
2788
2789         &$func($ds, $drive, @param);
2790     }
2791 }
2792
2793 sub foreach_volid {
2794     my ($conf, $func, @param) = @_;
2795
2796     my $volhash = {};
2797
2798     my $test_volid = sub {
2799         my ($volid, $is_cdrom, $replicate, $snapname) = @_;
2800
2801         return if !$volid;
2802
2803         $volhash->{$volid}->{cdrom} //= 1;
2804         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
2805
2806         $volhash->{$volid}->{replicate} //= 0;
2807         $volhash->{$volid}->{replicate} = 1 if $replicate;
2808
2809         $volhash->{$volid}->{referenced_in_config} //= 0;
2810         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
2811
2812         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
2813             if defined($snapname);
2814     };
2815
2816     foreach_drive($conf, sub {
2817         my ($ds, $drive) = @_;
2818         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, undef);
2819     });
2820
2821     foreach my $snapname (keys %{$conf->{snapshots}}) {
2822         my $snap = $conf->{snapshots}->{$snapname};
2823         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
2824         foreach_drive($snap, sub {
2825             my ($ds, $drive) = @_;
2826             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $snapname);
2827         });
2828     }
2829
2830     foreach my $volid (keys %$volhash) {
2831         &$func($volid, $volhash->{$volid}, @param);
2832     }
2833 }
2834
2835 sub vga_conf_has_spice {
2836     my ($vga) = @_;
2837
2838     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
2839
2840     return $1 || 1;
2841 }
2842
2843 sub config_to_command {
2844     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2845
2846     my $cmd = [];
2847     my $globalFlags = [];
2848     my $machineFlags = [];
2849     my $rtcFlags = [];
2850     my $cpuFlags = [];
2851     my $devices = [];
2852     my $pciaddr = '';
2853     my $bridges = {};
2854     my $kvmver = kvm_user_version();
2855     my $vernum = 0; # unknown
2856     my $ostype = $conf->{ostype};
2857     my $winversion = windows_version($ostype);
2858     my $kvm = $conf->{kvm} // 1;
2859
2860     die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n" if (!$cpuinfo->{hvm} && $kvm);
2861
2862     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
2863         $vernum = $1*1000000+$2*1000;
2864     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
2865         $vernum = $1*1000000+$2*1000+$3;
2866     }
2867
2868     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
2869
2870     my $have_ovz = -f '/proc/vz/vestat';
2871
2872     my $q35 = machine_type_is_q35($conf);
2873     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2874     my $machine_type = $forcemachine || $conf->{machine};
2875     my $use_old_bios_files = undef;
2876     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
2877
2878     my $cpuunits = defined($conf->{cpuunits}) ?
2879             $conf->{cpuunits} : $defaults->{cpuunits};
2880
2881     push @$cmd, '/usr/bin/kvm';
2882
2883     push @$cmd, '-id', $vmid;
2884
2885     my $use_virtio = 0;
2886
2887     my $qmpsocket = qmp_socket($vmid);
2888     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2889     push @$cmd, '-mon', "chardev=qmp,mode=control";
2890
2891
2892     push @$cmd, '-pidfile' , pidfile_name($vmid);
2893
2894     push @$cmd, '-daemonize';
2895
2896     if ($conf->{smbios1}) {
2897         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
2898     }
2899
2900     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2901         die "uefi base image not found\n" if ! -f $OVMF_CODE;
2902
2903         my $path;
2904         if (my $efidisk = $conf->{efidisk0}) {
2905             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
2906             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
2907             if ($storeid) {
2908                 $path = PVE::Storage::path($storecfg, $d->{file});
2909             } else {
2910                 $path = $d->{file};
2911             }
2912         } else {
2913             warn "no efidisk configured! Using temporary efivars disk.\n";
2914             $path = "/tmp/$vmid-ovmf.fd";
2915             PVE::Tools::file_copy($OVMF_VARS, $path, -s $OVMF_VARS);
2916         }
2917
2918         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$OVMF_CODE";
2919         push @$cmd, '-drive', "if=pflash,unit=1,id=drive-efidisk0,file=$path";
2920     }
2921
2922
2923     # add usb controllers
2924     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $q35, $usbdesc->{format}, $MAX_USB_DEVICES);
2925     push @$devices, @usbcontrollers if @usbcontrollers;
2926     my $vga = $conf->{vga};
2927
2928     my $qxlnum = vga_conf_has_spice($vga);
2929     $vga = 'qxl' if $qxlnum;
2930
2931     if (!$vga) {
2932         if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 9)) {
2933             $vga = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
2934         } else {
2935             $vga = ($winversion >= 6) ? 'std' : 'cirrus';
2936         }
2937     }
2938
2939     # enable absolute mouse coordinates (needed by vnc)
2940     my $tablet;
2941     if (defined($conf->{tablet})) {
2942         $tablet = $conf->{tablet};
2943     } else {
2944         $tablet = $defaults->{tablet};
2945         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
2946         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
2947     }
2948
2949     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
2950
2951     my $kvm_off = 0;
2952     my $gpu_passthrough;
2953
2954     # host pci devices
2955     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
2956         my $d = parse_hostpci($conf->{"hostpci$i"});
2957         next if !$d;
2958
2959         my $pcie = $d->{pcie};
2960         if($pcie){
2961             die "q35 machine model is not enabled" if !$q35;
2962             $pciaddr = print_pcie_addr("hostpci$i");
2963         }else{
2964             $pciaddr = print_pci_addr("hostpci$i", $bridges);
2965         }
2966
2967         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
2968         my $romfile = $d->{romfile};
2969
2970         my $xvga = '';
2971         if ($d->{'x-vga'}) {
2972             $xvga = ',x-vga=on';
2973             $kvm_off = 1;
2974             $vga = 'none';
2975             $gpu_passthrough = 1;
2976
2977             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2978                 $xvga = "";
2979             }
2980         }
2981         my $pcidevices = $d->{pciid};
2982         my $multifunction = 1 if @$pcidevices > 1;
2983
2984         my $j=0;
2985         foreach my $pcidevice (@$pcidevices) {
2986
2987             my $id = "hostpci$i";
2988             $id .= ".$j" if $multifunction;
2989             my $addr = $pciaddr;
2990             $addr .= ".$j" if $multifunction;
2991             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
2992
2993             if($j == 0){
2994                 $devicestr .= "$rombar$xvga";
2995                 $devicestr .= ",multifunction=on" if $multifunction;
2996                 $devicestr .= ",romfile=/usr/share/kvm/$romfile" if $romfile;
2997             }
2998
2999             push @$devices, '-device', $devicestr;
3000             $j++;
3001         }
3002     }
3003
3004     # usb devices
3005     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES);
3006     push @$devices, @usbdevices if @usbdevices;
3007     # serial devices
3008     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3009         if (my $path = $conf->{"serial$i"}) {
3010             if ($path eq 'socket') {
3011                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3012                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3013                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3014             } else {
3015                 die "no such serial device\n" if ! -c $path;
3016                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3017                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3018             }
3019         }
3020     }
3021
3022     # parallel devices
3023     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3024         if (my $path = $conf->{"parallel$i"}) {
3025             die "no such parallel device\n" if ! -c $path;
3026             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3027             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3028             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3029         }
3030     }
3031
3032     my $vmname = $conf->{name} || "vm$vmid";
3033
3034     push @$cmd, '-name', $vmname;
3035
3036     my $sockets = 1;
3037     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3038     $sockets = $conf->{sockets} if  $conf->{sockets};
3039
3040     my $cores = $conf->{cores} || 1;
3041
3042     my $maxcpus = $sockets * $cores;
3043
3044     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3045
3046     my $allowed_vcpus = $cpuinfo->{cpus};
3047
3048     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3049         if ($allowed_vcpus < $maxcpus);
3050
3051     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
3052
3053         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3054         for (my $i = 2; $i <= $vcpus; $i++)  {
3055             my $cpustr = print_cpu_device($conf,$i);
3056             push @$cmd, '-device', $cpustr;
3057         }
3058
3059     } else {
3060
3061         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3062     }
3063     push @$cmd, '-nodefaults';
3064
3065     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3066
3067     my $bootindex_hash = {};
3068     my $i = 1;
3069     foreach my $o (split(//, $bootorder)) {
3070         $bootindex_hash->{$o} = $i*100;
3071         $i++;
3072     }
3073
3074     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3075
3076     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3077
3078     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3079
3080     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
3081
3082     if ($vga && $vga !~ m/^serial\d+$/ && $vga ne 'none'){
3083         my $socket = vnc_socket($vmid);
3084         push @$cmd,  '-vnc', "unix:$socket,x509,password";
3085     } else {
3086         push @$cmd, '-nographic';
3087     }
3088
3089     # time drift fix
3090     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3091
3092     my $useLocaltime = $conf->{localtime};
3093
3094     if ($winversion >= 5) { # windows
3095         $useLocaltime = 1 if !defined($conf->{localtime});
3096
3097         # use time drift fix when acpi is enabled
3098         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3099             $tdf = 1 if !defined($conf->{tdf});
3100         }
3101     }
3102
3103     if ($winversion >= 6) {
3104         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3105         push @$cmd, '-no-hpet';
3106     }
3107
3108     push @$rtcFlags, 'driftfix=slew' if $tdf;
3109
3110     if (!$kvm) {
3111         push @$machineFlags, 'accel=tcg';
3112     }
3113
3114     if ($machine_type) {
3115         push @$machineFlags, "type=${machine_type}";
3116     }
3117
3118     if ($conf->{startdate}) {
3119         push @$rtcFlags, "base=$conf->{startdate}";
3120     } elsif ($useLocaltime) {
3121         push @$rtcFlags, 'base=localtime';
3122     }
3123
3124     my $cpu = $kvm ? "kvm64" : "qemu64";
3125     if (my $cputype = $conf->{cpu}) {
3126         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3127             or die "Cannot parse cpu description: $cputype\n";
3128         $cpu = $cpuconf->{cputype};
3129         $kvm_off = 1 if $cpuconf->{hidden};
3130     }
3131
3132     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
3133
3134     push @$cpuFlags , '-x2apic'
3135         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3136
3137     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3138
3139     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3140
3141     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3142
3143         push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3144         push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3145     }
3146
3147     add_hyperv_enlightenments($cpuFlags, $winversion, $machine_type, $kvmver, $conf->{bios}, $gpu_passthrough) if $kvm;
3148
3149     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm;
3150
3151     push @$cpuFlags, 'kvm=off' if $kvm_off;
3152
3153     my $cpu_vendor = $cpu_vendor_list->{$cpu} ||
3154         die "internal error"; # should not happen
3155
3156     push @$cpuFlags, "vendor=${cpu_vendor}"
3157         if $cpu_vendor ne 'default';
3158
3159     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3160
3161     push @$cmd, '-cpu', $cpu;
3162
3163     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3164
3165     push @$cmd, '-S' if $conf->{freeze};
3166
3167     # set keyboard layout
3168     my $kb = $conf->{keyboard} || $defaults->{keyboard};
3169     push @$cmd, '-k', $kb if $kb;
3170
3171     # enable sound
3172     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3173     #push @$cmd, '-soundhw', 'es1370';
3174     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3175
3176     if($conf->{agent}) {
3177         my $qgasocket = qmp_socket($vmid, 1);
3178         my $pciaddr = print_pci_addr("qga0", $bridges);
3179         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3180         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3181         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3182     }
3183
3184     my $spice_port;
3185
3186     if ($qxlnum) {
3187         if ($qxlnum > 1) {
3188             if ($winversion){
3189                 for(my $i = 1; $i < $qxlnum; $i++){
3190                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3191                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3192                 }
3193             } else {
3194                 # assume other OS works like Linux
3195                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3196                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3197             }
3198         }
3199
3200         my $pciaddr = print_pci_addr("spice", $bridges);
3201
3202         my $nodename = PVE::INotify::nodename();
3203         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3204         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3205         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3206         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3207         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3208
3209         push @$devices, '-spice', "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3210
3211         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3212         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3213         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3214     }
3215
3216     # enable balloon by default, unless explicitly disabled
3217     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3218         $pciaddr = print_pci_addr("balloon0", $bridges);
3219         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3220     }
3221
3222     if ($conf->{watchdog}) {
3223         my $wdopts = parse_watchdog($conf->{watchdog});
3224         $pciaddr = print_pci_addr("watchdog", $bridges);
3225         my $watchdog = $wdopts->{model} || 'i6300esb';
3226         push @$devices, '-device', "$watchdog$pciaddr";
3227         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3228     }
3229
3230     my $vollist = [];
3231     my $scsicontroller = {};
3232     my $ahcicontroller = {};
3233     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3234
3235     # Add iscsi initiator name if available
3236     if (my $initiator = get_initiator_name()) {
3237         push @$devices, '-iscsi', "initiator-name=$initiator";
3238     }
3239
3240     foreach_drive($conf, sub {
3241         my ($ds, $drive) = @_;
3242
3243         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3244             push @$vollist, $drive->{file};
3245         }
3246
3247         # ignore efidisk here, already added in bios/fw handling code above
3248         return if $drive->{interface} eq 'efidisk';
3249
3250         $use_virtio = 1 if $ds =~ m/^virtio/;
3251
3252         if (drive_is_cdrom ($drive)) {
3253             if ($bootindex_hash->{d}) {
3254                 $drive->{bootindex} = $bootindex_hash->{d};
3255                 $bootindex_hash->{d} += 1;
3256             }
3257         } else {
3258             if ($bootindex_hash->{c}) {
3259                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3260                 $bootindex_hash->{c} += 1;
3261             }
3262         }
3263
3264         if($drive->{interface} eq 'virtio'){
3265            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3266         }
3267
3268         if ($drive->{interface} eq 'scsi') {
3269
3270             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3271
3272             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3273             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3274
3275             my $iothread = '';
3276             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3277                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3278                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3279             } elsif ($drive->{iothread}) {
3280                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3281             }
3282
3283             my $queues = '';
3284             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3285                 $queues = ",num_queues=$drive->{queues}";
3286             }
3287
3288             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3289             $scsicontroller->{$controller}=1;
3290         }
3291
3292         if ($drive->{interface} eq 'sata') {
3293            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3294            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3295            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3296            $ahcicontroller->{$controller}=1;
3297         }
3298
3299         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3300         push @$devices, '-drive',$drive_cmd;
3301         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3302     });
3303
3304     for (my $i = 0; $i < $MAX_NETS; $i++) {
3305          next if !$conf->{"net$i"};
3306          my $d = parse_net($conf->{"net$i"});
3307          next if !$d;
3308
3309          $use_virtio = 1 if $d->{model} eq 'virtio';
3310
3311          if ($bootindex_hash->{n}) {
3312             $d->{bootindex} = $bootindex_hash->{n};
3313             $bootindex_hash->{n} += 1;
3314          }
3315
3316          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3317          push @$devices, '-netdev', $netdevfull;
3318
3319          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3320          push @$devices, '-device', $netdevicefull;
3321     }
3322
3323     if (!$q35) {
3324         # add pci bridges
3325         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3326            $bridges->{1} = 1;
3327            $bridges->{2} = 1;
3328         }
3329
3330         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3331
3332         while (my ($k, $v) = each %$bridges) {
3333             $pciaddr = print_pci_addr("pci.$k");
3334             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3335         }
3336     }
3337
3338     # add custom args
3339     if ($conf->{args}) {
3340         my $aa = PVE::Tools::split_args($conf->{args});
3341         push @$cmd, @$aa;
3342     }
3343
3344     push @$cmd, @$devices;
3345     push @$cmd, '-rtc', join(',', @$rtcFlags)
3346         if scalar(@$rtcFlags);
3347     push @$cmd, '-machine', join(',', @$machineFlags)
3348         if scalar(@$machineFlags);
3349     push @$cmd, '-global', join(',', @$globalFlags)
3350         if scalar(@$globalFlags);
3351
3352     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3353 }
3354
3355 sub vnc_socket {
3356     my ($vmid) = @_;
3357     return "${var_run_tmpdir}/$vmid.vnc";
3358 }
3359
3360 sub spice_port {
3361     my ($vmid) = @_;
3362
3363     my $res = vm_mon_cmd($vmid, 'query-spice');
3364
3365     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3366 }
3367
3368 sub qmp_socket {
3369     my ($vmid, $qga) = @_;
3370     my $sockettype = $qga ? 'qga' : 'qmp';
3371     return "${var_run_tmpdir}/$vmid.$sockettype";
3372 }
3373
3374 sub pidfile_name {
3375     my ($vmid) = @_;
3376     return "${var_run_tmpdir}/$vmid.pid";
3377 }
3378
3379 sub vm_devices_list {
3380     my ($vmid) = @_;
3381
3382     my $res = vm_mon_cmd($vmid, 'query-pci');
3383     my $devices = {};
3384     foreach my $pcibus (@$res) {
3385         foreach my $device (@{$pcibus->{devices}}) {
3386             next if !$device->{'qdev_id'};
3387             if ($device->{'pci_bridge'}) {
3388                 $devices->{$device->{'qdev_id'}} = 1;
3389                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3390                     next if !$bridge_device->{'qdev_id'};
3391                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3392                     $devices->{$device->{'qdev_id'}}++;
3393                 }
3394             } else {
3395                 $devices->{$device->{'qdev_id'}} = 1;
3396             }
3397         }
3398     }
3399
3400     my $resblock = vm_mon_cmd($vmid, 'query-block');
3401     foreach my $block (@$resblock) {
3402         if($block->{device} =~ m/^drive-(\S+)/){
3403                 $devices->{$1} = 1;
3404         }
3405     }
3406
3407     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3408     foreach my $mice (@$resmice) {
3409         if ($mice->{name} eq 'QEMU HID Tablet') {
3410             $devices->{tablet} = 1;
3411             last;
3412         }
3413     }
3414
3415     # for usb devices there is no query-usb
3416     # but we can iterate over the entries in
3417     # qom-list path=/machine/peripheral
3418     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3419     foreach my $per (@$resperipheral) {
3420         if ($per->{name} =~ m/^usb\d+$/) {
3421             $devices->{$per->{name}} = 1;
3422         }
3423     }
3424
3425     return $devices;
3426 }
3427
3428 sub vm_deviceplug {
3429     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3430
3431     my $q35 = machine_type_is_q35($conf);
3432
3433     my $devices_list = vm_devices_list($vmid);
3434     return 1 if defined($devices_list->{$deviceid});
3435
3436     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3437
3438     if ($deviceid eq 'tablet') {
3439
3440         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3441
3442     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3443
3444         die "usb hotplug currently not reliable\n";
3445         # since we can't reliably hot unplug all added usb devices
3446         # and usb passthrough disables live migration
3447         # we disable usb hotplugging for now
3448         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3449
3450     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3451
3452         qemu_iothread_add($vmid, $deviceid, $device);
3453
3454         qemu_driveadd($storecfg, $vmid, $device);
3455         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3456
3457         qemu_deviceadd($vmid, $devicefull);
3458         eval { qemu_deviceaddverify($vmid, $deviceid); };
3459         if (my $err = $@) {
3460             eval { qemu_drivedel($vmid, $deviceid); };
3461             warn $@ if $@;
3462             die $err;
3463         }
3464
3465     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3466
3467
3468         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3469         my $pciaddr = print_pci_addr($deviceid);
3470         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3471
3472         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3473
3474         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3475             qemu_iothread_add($vmid, $deviceid, $device);
3476             $devicefull .= ",iothread=iothread-$deviceid";
3477         }
3478
3479         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3480             $devicefull .= ",num_queues=$device->{queues}";
3481         }
3482
3483         qemu_deviceadd($vmid, $devicefull);
3484         qemu_deviceaddverify($vmid, $deviceid);
3485
3486     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3487
3488         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3489         qemu_driveadd($storecfg, $vmid, $device);
3490
3491         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3492         eval { qemu_deviceadd($vmid, $devicefull); };
3493         if (my $err = $@) {
3494             eval { qemu_drivedel($vmid, $deviceid); };
3495             warn $@ if $@;
3496             die $err;
3497         }
3498
3499     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3500
3501         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3502
3503         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
3504         my $use_old_bios_files = undef;
3505         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3506
3507         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3508         qemu_deviceadd($vmid, $netdevicefull);
3509         eval { qemu_deviceaddverify($vmid, $deviceid); };
3510         if (my $err = $@) {
3511             eval { qemu_netdevdel($vmid, $deviceid); };
3512             warn $@ if $@;
3513             die $err;
3514         }
3515
3516     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3517
3518         my $bridgeid = $2;
3519         my $pciaddr = print_pci_addr($deviceid);
3520         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3521
3522         qemu_deviceadd($vmid, $devicefull);
3523         qemu_deviceaddverify($vmid, $deviceid);
3524
3525     } else {
3526         die "can't hotplug device '$deviceid'\n";
3527     }
3528
3529     return 1;
3530 }
3531
3532 # fixme: this should raise exceptions on error!
3533 sub vm_deviceunplug {
3534     my ($vmid, $conf, $deviceid) = @_;
3535
3536     my $devices_list = vm_devices_list($vmid);
3537     return 1 if !defined($devices_list->{$deviceid});
3538
3539     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3540
3541     if ($deviceid eq 'tablet') {
3542
3543         qemu_devicedel($vmid, $deviceid);
3544
3545     } elsif ($deviceid =~ m/^usb\d+$/) {
3546
3547         die "usb hotplug currently not reliable\n";
3548         # when unplugging usb devices this way,
3549         # there may be remaining usb controllers/hubs
3550         # so we disable it for now
3551         qemu_devicedel($vmid, $deviceid);
3552         qemu_devicedelverify($vmid, $deviceid);
3553
3554     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3555
3556         qemu_devicedel($vmid, $deviceid);
3557         qemu_devicedelverify($vmid, $deviceid);
3558         qemu_drivedel($vmid, $deviceid);
3559         qemu_iothread_del($conf, $vmid, $deviceid);
3560
3561     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3562
3563         qemu_devicedel($vmid, $deviceid);
3564         qemu_devicedelverify($vmid, $deviceid);
3565         qemu_iothread_del($conf, $vmid, $deviceid);
3566
3567     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3568
3569         #qemu 2.3 segfault on drive_del with virtioscsi + iothread
3570         my $device = parse_drive($deviceid, $conf->{$deviceid});
3571         die "virtioscsi with iothread is not hot-unplugglable currently" if $device->{iothread};
3572
3573         qemu_devicedel($vmid, $deviceid);
3574         qemu_drivedel($vmid, $deviceid);
3575         qemu_deletescsihw($conf, $vmid, $deviceid);
3576
3577     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3578
3579         qemu_devicedel($vmid, $deviceid);
3580         qemu_devicedelverify($vmid, $deviceid);
3581         qemu_netdevdel($vmid, $deviceid);
3582
3583     } else {
3584         die "can't unplug device '$deviceid'\n";
3585     }
3586
3587     return 1;
3588 }
3589
3590 sub qemu_deviceadd {
3591     my ($vmid, $devicefull) = @_;
3592
3593     $devicefull = "driver=".$devicefull;
3594     my %options =  split(/[=,]/, $devicefull);
3595
3596     vm_mon_cmd($vmid, "device_add" , %options);
3597 }
3598
3599 sub qemu_devicedel {
3600     my ($vmid, $deviceid) = @_;
3601
3602     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3603 }
3604
3605 sub qemu_iothread_add {
3606     my($vmid, $deviceid, $device) = @_;
3607
3608     if ($device->{iothread}) {
3609         my $iothreads = vm_iothreads_list($vmid);
3610         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3611     }
3612 }
3613
3614 sub qemu_iothread_del {
3615     my($conf, $vmid, $deviceid) = @_;
3616
3617     my $device = parse_drive($deviceid, $conf->{$deviceid});
3618     if ($device->{iothread}) {
3619         my $iothreads = vm_iothreads_list($vmid);
3620         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3621     }
3622 }
3623
3624 sub qemu_objectadd {
3625     my($vmid, $objectid, $qomtype) = @_;
3626
3627     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3628
3629     return 1;
3630 }
3631
3632 sub qemu_objectdel {
3633     my($vmid, $objectid) = @_;
3634
3635     vm_mon_cmd($vmid, "object-del", id => $objectid);
3636
3637     return 1;
3638 }
3639
3640 sub qemu_driveadd {
3641     my ($storecfg, $vmid, $device) = @_;
3642
3643     my $drive = print_drive_full($storecfg, $vmid, $device);
3644     $drive =~ s/\\/\\\\/g;
3645     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3646
3647     # If the command succeeds qemu prints: "OK"
3648     return 1 if $ret =~ m/OK/s;
3649
3650     die "adding drive failed: $ret\n";
3651 }
3652
3653 sub qemu_drivedel {
3654     my($vmid, $deviceid) = @_;
3655
3656     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3657     $ret =~ s/^\s+//;
3658
3659     return 1 if $ret eq "";
3660
3661     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3662     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3663
3664     die "deleting drive $deviceid failed : $ret\n";
3665 }
3666
3667 sub qemu_deviceaddverify {
3668     my ($vmid, $deviceid) = @_;
3669
3670     for (my $i = 0; $i <= 5; $i++) {
3671          my $devices_list = vm_devices_list($vmid);
3672          return 1 if defined($devices_list->{$deviceid});
3673          sleep 1;
3674     }
3675
3676     die "error on hotplug device '$deviceid'\n";
3677 }
3678
3679
3680 sub qemu_devicedelverify {
3681     my ($vmid, $deviceid) = @_;
3682
3683     # need to verify that the device is correctly removed as device_del
3684     # is async and empty return is not reliable
3685
3686     for (my $i = 0; $i <= 5; $i++) {
3687          my $devices_list = vm_devices_list($vmid);
3688          return 1 if !defined($devices_list->{$deviceid});
3689          sleep 1;
3690     }
3691
3692     die "error on hot-unplugging device '$deviceid'\n";
3693 }
3694
3695 sub qemu_findorcreatescsihw {
3696     my ($storecfg, $conf, $vmid, $device) = @_;
3697
3698     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3699
3700     my $scsihwid="$controller_prefix$controller";
3701     my $devices_list = vm_devices_list($vmid);
3702
3703     if(!defined($devices_list->{$scsihwid})) {
3704         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3705     }
3706
3707     return 1;
3708 }
3709
3710 sub qemu_deletescsihw {
3711     my ($conf, $vmid, $opt) = @_;
3712
3713     my $device = parse_drive($opt, $conf->{$opt});
3714
3715     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3716         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3717         return 1;
3718     }
3719
3720     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3721
3722     my $devices_list = vm_devices_list($vmid);
3723     foreach my $opt (keys %{$devices_list}) {
3724         if (PVE::QemuServer::is_valid_drivename($opt)) {
3725             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3726             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3727                 return 1;
3728             }
3729         }
3730     }
3731
3732     my $scsihwid="scsihw$controller";
3733
3734     vm_deviceunplug($vmid, $conf, $scsihwid);
3735
3736     return 1;
3737 }
3738
3739 sub qemu_add_pci_bridge {
3740     my ($storecfg, $conf, $vmid, $device) = @_;
3741
3742     my $bridges = {};
3743
3744     my $bridgeid;
3745
3746     print_pci_addr($device, $bridges);
3747
3748     while (my ($k, $v) = each %$bridges) {
3749         $bridgeid = $k;
3750     }
3751     return 1 if !defined($bridgeid) || $bridgeid < 1;
3752
3753     my $bridge = "pci.$bridgeid";
3754     my $devices_list = vm_devices_list($vmid);
3755
3756     if (!defined($devices_list->{$bridge})) {
3757         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3758     }
3759
3760     return 1;
3761 }
3762
3763 sub qemu_set_link_status {
3764     my ($vmid, $device, $up) = @_;
3765
3766     vm_mon_cmd($vmid, "set_link", name => $device,
3767                up => $up ? JSON::true : JSON::false);
3768 }
3769
3770 sub qemu_netdevadd {
3771     my ($vmid, $conf, $device, $deviceid) = @_;
3772
3773     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
3774     my %options =  split(/[=,]/, $netdev);
3775
3776     vm_mon_cmd($vmid, "netdev_add",  %options);
3777     return 1;
3778 }
3779
3780 sub qemu_netdevdel {
3781     my ($vmid, $deviceid) = @_;
3782
3783     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3784 }
3785
3786 sub qemu_usb_hotplug {
3787     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3788
3789     return if !$device;
3790
3791     # remove the old one first
3792     vm_deviceunplug($vmid, $conf, $deviceid);
3793
3794     # check if xhci controller is necessary and available
3795     if ($device->{usb3}) {
3796
3797         my $devicelist = vm_devices_list($vmid);
3798
3799         if (!$devicelist->{xhci}) {
3800             my $pciaddr = print_pci_addr("xhci");
3801             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
3802         }
3803     }
3804     my $d = parse_usb_device($device->{host});
3805     $d->{usb3} = $device->{usb3};
3806
3807     # add the new one
3808     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d);
3809 }
3810
3811 sub qemu_cpu_hotplug {
3812     my ($vmid, $conf, $vcpus) = @_;
3813
3814     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
3815
3816     my $sockets = 1;
3817     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3818     $sockets = $conf->{sockets} if  $conf->{sockets};
3819     my $cores = $conf->{cores} || 1;
3820     my $maxcpus = $sockets * $cores;
3821
3822     $vcpus = $maxcpus if !$vcpus;
3823
3824     die "you can't add more vcpus than maxcpus\n"
3825         if $vcpus > $maxcpus;
3826
3827     my $currentvcpus = $conf->{vcpus} || $maxcpus;
3828
3829     if ($vcpus < $currentvcpus) {
3830
3831         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
3832
3833             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
3834                 qemu_devicedel($vmid, "cpu$i");
3835                 my $retry = 0;
3836                 my $currentrunningvcpus = undef;
3837                 while (1) {
3838                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3839                     last if scalar(@{$currentrunningvcpus}) == $i-1;
3840                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
3841                     $retry++;
3842                     sleep 1;
3843                 }
3844                 #update conf after each succesfull cpu unplug
3845                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
3846                 PVE::QemuConfig->write_config($vmid, $conf);
3847             }
3848         } else {
3849             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
3850         }
3851
3852         return;
3853     }
3854
3855     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3856     die "vcpus in running vm does not match its configuration\n"
3857         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
3858
3859     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
3860
3861         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
3862             my $cpustr = print_cpu_device($conf, $i);
3863             qemu_deviceadd($vmid, $cpustr);
3864
3865             my $retry = 0;
3866             my $currentrunningvcpus = undef;
3867             while (1) {
3868                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3869                 last if scalar(@{$currentrunningvcpus}) == $i;
3870                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
3871                 sleep 1;
3872                 $retry++;
3873             }
3874             #update conf after each succesfull cpu hotplug
3875             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
3876             PVE::QemuConfig->write_config($vmid, $conf);
3877         }
3878     } else {
3879
3880         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
3881             vm_mon_cmd($vmid, "cpu-add", id => int($i));
3882         }
3883     }
3884 }
3885
3886 sub qemu_block_set_io_throttle {
3887     my ($vmid, $deviceid,
3888         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
3889         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
3890         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
3891         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
3892
3893     return if !check_running($vmid) ;
3894
3895     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
3896         bps => int($bps),
3897         bps_rd => int($bps_rd),
3898         bps_wr => int($bps_wr),
3899         iops => int($iops),
3900         iops_rd => int($iops_rd),
3901         iops_wr => int($iops_wr),
3902         bps_max => int($bps_max),
3903         bps_rd_max => int($bps_rd_max),
3904         bps_wr_max => int($bps_wr_max),
3905         iops_max => int($iops_max),
3906         iops_rd_max => int($iops_rd_max),
3907         iops_wr_max => int($iops_wr_max),
3908         bps_max_length => int($bps_max_length),
3909         bps_rd_max_length => int($bps_rd_max_length),
3910         bps_wr_max_length => int($bps_wr_max_length),
3911         iops_max_length => int($iops_max_length),
3912         iops_rd_max_length => int($iops_rd_max_length),
3913         iops_wr_max_length => int($iops_wr_max_length),
3914     );
3915
3916 }
3917
3918 # old code, only used to shutdown old VM after update
3919 sub __read_avail {
3920     my ($fh, $timeout) = @_;
3921
3922     my $sel = new IO::Select;
3923     $sel->add($fh);
3924
3925     my $res = '';
3926     my $buf;
3927
3928     my @ready;
3929     while (scalar (@ready = $sel->can_read($timeout))) {
3930         my $count;
3931         if ($count = $fh->sysread($buf, 8192)) {
3932             if ($buf =~ /^(.*)\(qemu\) $/s) {
3933                 $res .= $1;
3934                 last;
3935             } else {
3936                 $res .= $buf;
3937             }
3938         } else {
3939             if (!defined($count)) {
3940                 die "$!\n";
3941             }
3942             last;
3943         }
3944     }
3945
3946     die "monitor read timeout\n" if !scalar(@ready);
3947
3948     return $res;
3949 }
3950
3951 # old code, only used to shutdown old VM after update
3952 sub vm_monitor_command {
3953     my ($vmid, $cmdstr, $nocheck) = @_;
3954
3955     my $res;
3956
3957     eval {
3958         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
3959
3960         my $sname = "${var_run_tmpdir}/$vmid.mon";
3961
3962         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
3963             die "unable to connect to VM $vmid socket - $!\n";
3964
3965         my $timeout = 3;
3966
3967         # hack: migrate sometime blocks the monitor (when migrate_downtime
3968         # is set)
3969         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3970             $timeout = 60*60; # 1 hour
3971         }
3972
3973         # read banner;
3974         my $data = __read_avail($sock, $timeout);
3975
3976         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
3977             die "got unexpected qemu monitor banner\n";
3978         }
3979
3980         my $sel = new IO::Select;
3981         $sel->add($sock);
3982
3983         if (!scalar(my @ready = $sel->can_write($timeout))) {
3984             die "monitor write error - timeout";
3985         }
3986
3987         my $fullcmd = "$cmdstr\r";
3988
3989         # syslog('info', "VM $vmid monitor command: $cmdstr");
3990
3991         my $b;
3992         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
3993             die "monitor write error - $!";
3994         }
3995
3996         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
3997
3998         $timeout = 20;
3999
4000         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
4001             $timeout = 60*60; # 1 hour
4002         } elsif ($cmdstr =~ m/^(eject|change)/) {
4003             $timeout = 60; # note: cdrom mount command is slow
4004         }
4005         if ($res = __read_avail($sock, $timeout)) {
4006
4007             my @lines = split("\r?\n", $res);
4008
4009             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
4010
4011             $res = join("\n", @lines);
4012             $res .= "\n";
4013         }
4014     };
4015
4016     my $err = $@;
4017
4018     if ($err) {
4019         syslog("err", "VM $vmid monitor command failed - $err");
4020         die $err;
4021     }
4022
4023     return $res;
4024 }
4025
4026 sub qemu_block_resize {
4027     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4028
4029     my $running = check_running($vmid);
4030
4031     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4032
4033     return if !$running;
4034
4035     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4036
4037 }
4038
4039 sub qemu_volume_snapshot {
4040     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4041
4042     my $running = check_running($vmid);
4043
4044     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4045         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
4046     } else {
4047         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4048     }
4049 }
4050
4051 sub qemu_volume_snapshot_delete {
4052     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4053
4054     my $running = check_running($vmid);
4055
4056     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4057         vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
4058     } else {
4059         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4060     }
4061 }
4062
4063 sub set_migration_caps {
4064     my ($vmid) = @_;
4065
4066     my $cap_ref = [];
4067
4068     my $enabled_cap = {
4069         "auto-converge" => 1,
4070         "xbzrle" => 1,
4071         "x-rdma-pin-all" => 0,
4072         "zero-blocks" => 0,
4073         "compress" => 0
4074     };
4075
4076     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4077
4078     for my $supported_capability (@$supported_capabilities) {
4079         push @$cap_ref, {
4080             capability => $supported_capability->{capability},
4081             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4082         };
4083     }
4084
4085     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4086 }
4087
4088 my $fast_plug_option = {
4089     'lock' => 1,
4090     'name' => 1,
4091     'onboot' => 1,
4092     'shares' => 1,
4093     'startup' => 1,
4094     'description' => 1,
4095     'protection' => 1,
4096     'vmstatestorage' => 1,
4097 };
4098
4099 # hotplug changes in [PENDING]
4100 # $selection hash can be used to only apply specified options, for
4101 # example: { cores => 1 } (only apply changed 'cores')
4102 # $errors ref is used to return error messages
4103 sub vmconfig_hotplug_pending {
4104     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4105
4106     my $defaults = load_defaults();
4107
4108     # commit values which do not have any impact on running VM first
4109     # Note: those option cannot raise errors, we we do not care about
4110     # $selection and always apply them.
4111
4112     my $add_error = sub {
4113         my ($opt, $msg) = @_;
4114         $errors->{$opt} = "hotplug problem - $msg";
4115     };
4116
4117     my $changes = 0;
4118     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4119         if ($fast_plug_option->{$opt}) {
4120             $conf->{$opt} = $conf->{pending}->{$opt};
4121             delete $conf->{pending}->{$opt};
4122             $changes = 1;
4123         }
4124     }
4125
4126     if ($changes) {
4127         PVE::QemuConfig->write_config($vmid, $conf);
4128         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4129     }
4130
4131     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4132
4133     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4134     while (my ($opt, $force) = each %$pending_delete_hash) {
4135         next if $selection && !$selection->{$opt};
4136         eval {
4137             if ($opt eq 'hotplug') {
4138                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4139             } elsif ($opt eq 'tablet') {
4140                 die "skip\n" if !$hotplug_features->{usb};
4141                 if ($defaults->{tablet}) {
4142                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4143                 } else {
4144                     vm_deviceunplug($vmid, $conf, $opt);
4145                 }
4146             } elsif ($opt =~ m/^usb\d+/) {
4147                 die "skip\n";
4148                 # since we cannot reliably hot unplug usb devices
4149                 # we are disabling it
4150                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4151                 vm_deviceunplug($vmid, $conf, $opt);
4152             } elsif ($opt eq 'vcpus') {
4153                 die "skip\n" if !$hotplug_features->{cpu};
4154                 qemu_cpu_hotplug($vmid, $conf, undef);
4155             } elsif ($opt eq 'balloon') {
4156                 # enable balloon device is not hotpluggable
4157                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
4158             } elsif ($fast_plug_option->{$opt}) {
4159                 # do nothing
4160             } elsif ($opt =~ m/^net(\d+)$/) {
4161                 die "skip\n" if !$hotplug_features->{network};
4162                 vm_deviceunplug($vmid, $conf, $opt);
4163             } elsif (is_valid_drivename($opt)) {
4164                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4165                 vm_deviceunplug($vmid, $conf, $opt);
4166                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4167             } elsif ($opt =~ m/^memory$/) {
4168                 die "skip\n" if !$hotplug_features->{memory};
4169                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4170             } elsif ($opt eq 'cpuunits') {
4171                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4172             } elsif ($opt eq 'cpulimit') {
4173                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4174             } else {
4175                 die "skip\n";
4176             }
4177         };
4178         if (my $err = $@) {
4179             &$add_error($opt, $err) if $err ne "skip\n";
4180         } else {
4181             # save new config if hotplug was successful
4182             delete $conf->{$opt};
4183             vmconfig_undelete_pending_option($conf, $opt);
4184             PVE::QemuConfig->write_config($vmid, $conf);
4185             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4186         }
4187     }
4188
4189     foreach my $opt (keys %{$conf->{pending}}) {
4190         next if $selection && !$selection->{$opt};
4191         my $value = $conf->{pending}->{$opt};
4192         eval {
4193             if ($opt eq 'hotplug') {
4194                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4195             } elsif ($opt eq 'tablet') {
4196                 die "skip\n" if !$hotplug_features->{usb};
4197                 if ($value == 1) {
4198                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4199                 } elsif ($value == 0) {
4200                     vm_deviceunplug($vmid, $conf, $opt);
4201                 }
4202             } elsif ($opt =~ m/^usb\d+$/) {
4203                 die "skip\n";
4204                 # since we cannot reliably hot unplug usb devices
4205                 # we are disabling it
4206                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4207                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4208                 die "skip\n" if !$d;
4209                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d);
4210             } elsif ($opt eq 'vcpus') {
4211                 die "skip\n" if !$hotplug_features->{cpu};
4212                 qemu_cpu_hotplug($vmid, $conf, $value);
4213             } elsif ($opt eq 'balloon') {
4214                 # enable/disable balloning device is not hotpluggable
4215                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4216                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4217                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4218
4219                 # allow manual ballooning if shares is set to zero
4220                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4221                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4222                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4223                 }
4224             } elsif ($opt =~ m/^net(\d+)$/) {
4225                 # some changes can be done without hotplug
4226                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4227                                     $vmid, $opt, $value);
4228             } elsif (is_valid_drivename($opt)) {
4229                 # some changes can be done without hotplug
4230                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4231                                      $vmid, $opt, $value, 1);
4232             } elsif ($opt =~ m/^memory$/) { #dimms
4233                 die "skip\n" if !$hotplug_features->{memory};
4234                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4235             } elsif ($opt eq 'cpuunits') {
4236                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4237             } elsif ($opt eq 'cpulimit') {
4238                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4239                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4240             } else {
4241                 die "skip\n";  # skip non-hot-pluggable options
4242             }
4243         };
4244         if (my $err = $@) {
4245             &$add_error($opt, $err) if $err ne "skip\n";
4246         } else {
4247             # save new config if hotplug was successful
4248             $conf->{$opt} = $value;
4249             delete $conf->{pending}->{$opt};
4250             PVE::QemuConfig->write_config($vmid, $conf);
4251             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4252         }
4253     }
4254 }
4255
4256 sub try_deallocate_drive {
4257     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4258
4259     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4260         my $volid = $drive->{file};
4261         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4262             my $sid = PVE::Storage::parse_volume_id($volid);
4263             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4264
4265             # check if the disk is really unused
4266             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4267                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4268             PVE::Storage::vdisk_free($storecfg, $volid);
4269             return 1;
4270         } else {
4271             # If vm is not owner of this disk remove from config
4272             return 1;
4273         }
4274     }
4275
4276     return undef;
4277 }
4278
4279 sub vmconfig_delete_or_detach_drive {
4280     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4281
4282     my $drive = parse_drive($opt, $conf->{$opt});
4283
4284     my $rpcenv = PVE::RPCEnvironment::get();
4285     my $authuser = $rpcenv->get_user();
4286
4287     if ($force) {
4288         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4289         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4290     } else {
4291         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4292     }
4293 }
4294
4295 sub vmconfig_apply_pending {
4296     my ($vmid, $conf, $storecfg) = @_;
4297
4298     # cold plug
4299
4300     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4301     while (my ($opt, $force) = each %$pending_delete_hash) {
4302         die "internal error" if $opt =~ m/^unused/;
4303         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4304         if (!defined($conf->{$opt})) {
4305             vmconfig_undelete_pending_option($conf, $opt);
4306             PVE::QemuConfig->write_config($vmid, $conf);
4307         } elsif (is_valid_drivename($opt)) {
4308             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4309             vmconfig_undelete_pending_option($conf, $opt);
4310             delete $conf->{$opt};
4311             PVE::QemuConfig->write_config($vmid, $conf);
4312         } else {
4313             vmconfig_undelete_pending_option($conf, $opt);
4314             delete $conf->{$opt};
4315             PVE::QemuConfig->write_config($vmid, $conf);
4316         }
4317     }
4318
4319     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4320
4321     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4322         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4323
4324         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4325             # skip if nothing changed
4326         } elsif (is_valid_drivename($opt)) {
4327             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4328                 if defined($conf->{$opt});
4329             $conf->{$opt} = $conf->{pending}->{$opt};
4330         } else {
4331             $conf->{$opt} = $conf->{pending}->{$opt};
4332         }
4333
4334         delete $conf->{pending}->{$opt};
4335         PVE::QemuConfig->write_config($vmid, $conf);
4336     }
4337 }
4338
4339 my $safe_num_ne = sub {
4340     my ($a, $b) = @_;
4341
4342     return 0 if !defined($a) && !defined($b);
4343     return 1 if !defined($a);
4344     return 1 if !defined($b);
4345
4346     return $a != $b;
4347 };
4348
4349 my $safe_string_ne = sub {
4350     my ($a, $b) = @_;
4351
4352     return 0 if !defined($a) && !defined($b);
4353     return 1 if !defined($a);
4354     return 1 if !defined($b);
4355
4356     return $a ne $b;
4357 };
4358
4359 sub vmconfig_update_net {
4360     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4361
4362     my $newnet = parse_net($value);
4363
4364     if ($conf->{$opt}) {
4365         my $oldnet = parse_net($conf->{$opt});
4366
4367         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4368             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4369             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4370             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4371
4372             # for non online change, we try to hot-unplug
4373             die "skip\n" if !$hotplug;
4374             vm_deviceunplug($vmid, $conf, $opt);
4375         } else {
4376
4377             die "internal error" if $opt !~ m/net(\d+)/;
4378             my $iface = "tap${vmid}i$1";
4379
4380             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4381                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4382                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4383                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4384                 PVE::Network::tap_unplug($iface);
4385                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4386             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4387                 # Rate can be applied on its own but any change above needs to
4388                 # include the rate in tap_plug since OVS resets everything.
4389                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4390             }
4391
4392             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4393                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4394             }
4395
4396             return 1;
4397         }
4398     }
4399
4400     if ($hotplug) {
4401         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4402     } else {
4403         die "skip\n";
4404     }
4405 }
4406
4407 sub vmconfig_update_disk {
4408     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4409
4410     # fixme: do we need force?
4411
4412     my $drive = parse_drive($opt, $value);
4413
4414     if ($conf->{$opt}) {
4415
4416         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4417
4418             my $media = $drive->{media} || 'disk';
4419             my $oldmedia = $old_drive->{media} || 'disk';
4420             die "unable to change media type\n" if $media ne $oldmedia;
4421
4422             if (!drive_is_cdrom($old_drive)) {
4423
4424                 if ($drive->{file} ne $old_drive->{file}) {
4425
4426                     die "skip\n" if !$hotplug;
4427
4428                     # unplug and register as unused
4429                     vm_deviceunplug($vmid, $conf, $opt);
4430                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4431
4432                 } else {
4433                     # update existing disk
4434
4435                     # skip non hotpluggable value
4436                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4437                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4438                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4439                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4440                         die "skip\n";
4441                     }
4442
4443                     # apply throttle
4444                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4445                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4446                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4447                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4448                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4449                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4450                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4451                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4452                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4453                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4454                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4455                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4456                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4457                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4458                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4459                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4460                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4461                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4462
4463                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4464                                                    ($drive->{mbps} || 0)*1024*1024,
4465                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4466                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4467                                                    $drive->{iops} || 0,
4468                                                    $drive->{iops_rd} || 0,
4469                                                    $drive->{iops_wr} || 0,
4470                                                    ($drive->{mbps_max} || 0)*1024*1024,
4471                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4472                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4473                                                    $drive->{iops_max} || 0,
4474                                                    $drive->{iops_rd_max} || 0,
4475                                                    $drive->{iops_wr_max} || 0,
4476                                                    $drive->{bps_max_length} || 1,
4477                                                    $drive->{bps_rd_max_length} || 1,
4478                                                    $drive->{bps_wr_max_length} || 1,
4479                                                    $drive->{iops_max_length} || 1,
4480                                                    $drive->{iops_rd_max_length} || 1,
4481                                                    $drive->{iops_wr_max_length} || 1);
4482
4483                     }
4484
4485                     return 1;
4486                 }
4487
4488             } else { # cdrom
4489
4490                 if ($drive->{file} eq 'none') {
4491                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4492                 } else {
4493                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4494                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4495                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4496                 }
4497
4498                 return 1;
4499             }
4500         }
4501     }
4502
4503     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4504     # hotplug new disks
4505     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4506     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4507 }
4508
4509 sub vm_start {
4510     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4511         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
4512
4513     PVE::QemuConfig->lock_config($vmid, sub {
4514         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4515
4516         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
4517
4518         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4519
4520         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4521
4522         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4523             vmconfig_apply_pending($vmid, $conf, $storecfg);
4524             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4525         }
4526
4527         my $defaults = load_defaults();
4528
4529         # set environment variable useful inside network script
4530         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4531
4532         my $local_volumes = {};
4533
4534         if ($targetstorage) {
4535             foreach_drive($conf, sub {
4536                 my ($ds, $drive) = @_;
4537
4538                 return if drive_is_cdrom($drive);
4539
4540                 my $volid = $drive->{file};
4541
4542                 return if !$volid;
4543
4544                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4545
4546                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4547                 return if $scfg->{shared};
4548                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
4549             });
4550
4551             my $format = undef;
4552
4553             foreach my $opt (sort keys %$local_volumes) {
4554
4555                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
4556                 my $drive = parse_drive($opt, $conf->{$opt});
4557
4558                 #if remote storage is specified, use default format
4559                 if ($targetstorage && $targetstorage ne "1") {
4560                     $storeid = $targetstorage;
4561                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
4562                     $format = $defFormat;
4563                 } else {
4564                     #else we use same format than original
4565                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4566                     $format = qemu_img_format($scfg, $volid);
4567                 }
4568
4569                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
4570                 my $newdrive = $drive;
4571                 $newdrive->{format} = $format;
4572                 $newdrive->{file} = $newvolid;
4573                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
4574                 $local_volumes->{$opt} = $drivestr;
4575                 #pass drive to conf for command line
4576                 $conf->{$opt} = $drivestr;
4577             }
4578         }
4579
4580         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4581
4582         my $migrate_port = 0;
4583         my $migrate_uri;
4584         if ($statefile) {
4585             if ($statefile eq 'tcp') {
4586                 my $localip = "localhost";
4587                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4588                 my $nodename = PVE::INotify::nodename();
4589
4590                 if (!defined($migration_type)) {
4591                     if (defined($datacenterconf->{migration}->{type})) {
4592                         $migration_type = $datacenterconf->{migration}->{type};
4593                     } else {
4594                         $migration_type = 'secure';
4595                     }
4596                 }
4597
4598                 if ($migration_type eq 'insecure') {
4599                     my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4600                     if ($migrate_network_addr) {
4601                         $localip = $migrate_network_addr;
4602                     } else {
4603                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4604                     }
4605
4606                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4607                 }
4608
4609                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4610                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4611                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4612                 push @$cmd, '-incoming', $migrate_uri;
4613                 push @$cmd, '-S';
4614
4615             } elsif ($statefile eq 'unix') {
4616                 # should be default for secure migrations as a ssh TCP forward
4617                 # tunnel is not deterministic reliable ready and fails regurarly
4618                 # to set up in time, so use UNIX socket forwards
4619                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
4620                 unlink $socket_addr;
4621
4622                 $migrate_uri = "unix:$socket_addr";
4623
4624                 push @$cmd, '-incoming', $migrate_uri;
4625                 push @$cmd, '-S';
4626
4627             } else {
4628                 push @$cmd, '-loadstate', $statefile;
4629             }
4630         } elsif ($paused) {
4631             push @$cmd, '-S';
4632         }
4633
4634         # host pci devices
4635         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4636           my $d = parse_hostpci($conf->{"hostpci$i"});
4637           next if !$d;
4638           my $pcidevices = $d->{pciid};
4639           foreach my $pcidevice (@$pcidevices) {
4640                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4641
4642                 my $info = pci_device_info("0000:$pciid");
4643                 die "IOMMU not present\n" if !check_iommu_support();
4644                 die "no pci device info for device '$pciid'\n" if !$info;
4645                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4646                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4647           }
4648         }
4649
4650         PVE::Storage::activate_volumes($storecfg, $vollist);
4651
4652         if (!check_running($vmid, 1) && -d "/sys/fs/cgroup/systemd/qemu.slice/$vmid.scope") {
4653             my $cmd = [];
4654             push @$cmd, '/bin/systemctl', 'stop', "$vmid.scope";
4655             eval  { run_command($cmd); };
4656         }
4657
4658         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
4659                                                   : $defaults->{cpuunits};
4660
4661         my %run_params = (timeout => $statefile ? undef : 30, umask => 0077);
4662
4663         my %properties = (
4664             Slice => 'qemu.slice',
4665             KillMode => 'none',
4666             CPUShares => $cpuunits
4667         );
4668
4669         if (my $cpulimit = $conf->{cpulimit}) {
4670             $properties{CPUQuota} = int($cpulimit * 100);
4671         }
4672         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
4673
4674         if ($conf->{hugepages}) {
4675
4676             my $code = sub {
4677                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
4678                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
4679
4680                 PVE::QemuServer::Memory::hugepages_mount();
4681                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
4682
4683                 eval  {
4684                     PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4685                     run_command($cmd, %run_params);
4686                 };
4687
4688                 if (my $err = $@) {
4689                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
4690                     die $err;
4691                 }
4692
4693                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
4694             };
4695             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
4696
4697         } else {
4698             eval  {
4699                 PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4700                 run_command($cmd, %run_params);
4701             };
4702         }
4703
4704         if (my $err = $@) {
4705             # deactivate volumes if start fails
4706             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
4707             die "start failed: $err";
4708         }
4709
4710         print "migration listens on $migrate_uri\n" if $migrate_uri;
4711
4712         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
4713             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4714             warn $@ if $@;
4715         }
4716
4717         #start nbd server for storage migration
4718         if ($targetstorage) {
4719             my $nodename = PVE::INotify::nodename();
4720             my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4721             my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
4722             my $pfamily = PVE::Tools::get_host_address_family($nodename);
4723             $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4724
4725             vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${migrate_port}" } } );
4726
4727             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4728
4729             foreach my $opt (sort keys %$local_volumes) {
4730                 my $volid = $local_volumes->{$opt};
4731                 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
4732                 my $migrate_storage_uri = "nbd:${localip}:${migrate_port}:exportname=drive-$opt";
4733                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
4734             }
4735         }
4736
4737         if ($migratedfrom) {
4738             eval {
4739                 set_migration_caps($vmid);
4740             };
4741             warn $@ if $@;
4742
4743             if ($spice_port) {
4744                 print "spice listens on port $spice_port\n";
4745                 if ($spice_ticket) {
4746                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4747                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4748                 }
4749             }
4750
4751         } else {
4752             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4753                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4754                     if $conf->{balloon};
4755             }
4756
4757             foreach my $opt (keys %$conf) {
4758                 next if $opt !~  m/^net\d+$/;
4759                 my $nicconf = parse_net($conf->{$opt});
4760                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4761             }
4762         }
4763
4764         vm_mon_cmd_nocheck($vmid, 'qom-set',
4765                     path => "machine/peripheral/balloon0",
4766                     property => "guest-stats-polling-interval",
4767                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4768
4769     });
4770 }
4771
4772 sub vm_mon_cmd {
4773     my ($vmid, $execute, %params) = @_;
4774
4775     my $cmd = { execute => $execute, arguments => \%params };
4776     vm_qmp_command($vmid, $cmd);
4777 }
4778
4779 sub vm_mon_cmd_nocheck {
4780     my ($vmid, $execute, %params) = @_;
4781
4782     my $cmd = { execute => $execute, arguments => \%params };
4783     vm_qmp_command($vmid, $cmd, 1);
4784 }
4785
4786 sub vm_qmp_command {
4787     my ($vmid, $cmd, $nocheck) = @_;
4788
4789     my $res;
4790
4791     my $timeout;
4792     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4793         $timeout = $cmd->{arguments}->{timeout};
4794         delete $cmd->{arguments}->{timeout};
4795     }
4796
4797     eval {
4798         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4799         my $sname = qmp_socket($vmid);
4800         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4801             my $qmpclient = PVE::QMPClient->new();
4802
4803             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4804         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
4805             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
4806                 if scalar(%{$cmd->{arguments}});
4807             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
4808         } else {
4809             die "unable to open monitor socket\n";
4810         }
4811     };
4812     if (my $err = $@) {
4813         syslog("err", "VM $vmid qmp command failed - $err");
4814         die $err;
4815     }
4816
4817     return $res;
4818 }
4819
4820 sub vm_human_monitor_command {
4821     my ($vmid, $cmdline) = @_;
4822
4823     my $res;
4824
4825     my $cmd = {
4826         execute => 'human-monitor-command',
4827         arguments => { 'command-line' => $cmdline},
4828     };
4829
4830     return vm_qmp_command($vmid, $cmd);
4831 }
4832
4833 sub vm_commandline {
4834     my ($storecfg, $vmid) = @_;
4835
4836     my $conf = PVE::QemuConfig->load_config($vmid);
4837
4838     my $defaults = load_defaults();
4839
4840     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
4841
4842     return PVE::Tools::cmd2string($cmd);
4843 }
4844
4845 sub vm_reset {
4846     my ($vmid, $skiplock) = @_;
4847
4848     PVE::QemuConfig->lock_config($vmid, sub {
4849
4850         my $conf = PVE::QemuConfig->load_config($vmid);
4851
4852         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4853
4854         vm_mon_cmd($vmid, "system_reset");
4855     });
4856 }
4857
4858 sub get_vm_volumes {
4859     my ($conf) = @_;
4860
4861     my $vollist = [];
4862     foreach_volid($conf, sub {
4863         my ($volid, $attr) = @_;
4864
4865         return if $volid =~ m|^/|;
4866
4867         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4868         return if !$sid;
4869
4870         push @$vollist, $volid;
4871     });
4872
4873     return $vollist;
4874 }
4875
4876 sub vm_stop_cleanup {
4877     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
4878
4879     eval {
4880
4881         if (!$keepActive) {
4882             my $vollist = get_vm_volumes($conf);
4883             PVE::Storage::deactivate_volumes($storecfg, $vollist);
4884         }
4885
4886         foreach my $ext (qw(mon qmp pid vnc qga)) {
4887             unlink "/var/run/qemu-server/${vmid}.$ext";
4888         }
4889
4890         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
4891     };
4892     warn $@ if $@; # avoid errors - just warn
4893 }
4894
4895 # Note: use $nockeck to skip tests if VM configuration file exists.
4896 # We need that when migration VMs to other nodes (files already moved)
4897 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
4898 sub vm_stop {
4899     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
4900
4901     $force = 1 if !defined($force) && !$shutdown;
4902
4903     if ($migratedfrom){
4904         my $pid = check_running($vmid, $nocheck, $migratedfrom);
4905         kill 15, $pid if $pid;
4906         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4907         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
4908         return;
4909     }
4910
4911     PVE::QemuConfig->lock_config($vmid, sub {
4912
4913         my $pid = check_running($vmid, $nocheck);
4914         return if !$pid;
4915
4916         my $conf;
4917         if (!$nocheck) {
4918             $conf = PVE::QemuConfig->load_config($vmid);
4919             PVE::QemuConfig->check_lock($conf) if !$skiplock;
4920             if (!defined($timeout) && $shutdown && $conf->{startup}) {
4921                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
4922                 $timeout = $opts->{down} if $opts->{down};
4923             }
4924         }
4925
4926         $timeout = 60 if !defined($timeout);
4927
4928         eval {
4929             if ($shutdown) {
4930                 if (defined($conf) && $conf->{agent}) {
4931                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
4932                 } else {
4933                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
4934                 }
4935             } else {
4936                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
4937             }
4938         };
4939         my $err = $@;
4940
4941         if (!$err) {
4942             my $count = 0;
4943             while (($count < $timeout) && check_running($vmid, $nocheck)) {
4944                 $count++;
4945                 sleep 1;
4946             }
4947
4948             if ($count >= $timeout) {
4949                 if ($force) {
4950                     warn "VM still running - terminating now with SIGTERM\n";
4951                     kill 15, $pid;
4952                 } else {
4953                     die "VM quit/powerdown failed - got timeout\n";
4954                 }
4955             } else {
4956                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4957                 return;
4958             }
4959         } else {
4960             if ($force) {
4961                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
4962                 kill 15, $pid;
4963             } else {
4964                 die "VM quit/powerdown failed\n";
4965             }
4966         }
4967
4968         # wait again
4969         $timeout = 10;
4970
4971         my $count = 0;
4972         while (($count < $timeout) && check_running($vmid, $nocheck)) {
4973             $count++;
4974             sleep 1;
4975         }
4976
4977         if ($count >= $timeout) {
4978             warn "VM still running - terminating now with SIGKILL\n";
4979             kill 9, $pid;
4980             sleep 1;
4981         }
4982
4983         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4984    });
4985 }
4986
4987 sub vm_suspend {
4988     my ($vmid, $skiplock) = @_;
4989
4990     PVE::QemuConfig->lock_config($vmid, sub {
4991
4992         my $conf = PVE::QemuConfig->load_config($vmid);
4993
4994         PVE::QemuConfig->check_lock($conf)
4995             if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
4996
4997         vm_mon_cmd($vmid, "stop");
4998     });
4999 }
5000
5001 sub vm_resume {
5002     my ($vmid, $skiplock, $nocheck) = @_;
5003
5004     PVE::QemuConfig->lock_config($vmid, sub {
5005
5006         if (!$nocheck) {
5007
5008             my $conf = PVE::QemuConfig->load_config($vmid);
5009
5010             PVE::QemuConfig->check_lock($conf)
5011                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5012
5013             vm_mon_cmd($vmid, "cont");
5014
5015         } else {
5016             vm_mon_cmd_nocheck($vmid, "cont");
5017         }
5018     });
5019 }
5020
5021 sub vm_sendkey {
5022     my ($vmid, $skiplock, $key) = @_;
5023
5024     PVE::QemuConfig->lock_config($vmid, sub {
5025
5026         my $conf = PVE::QemuConfig->load_config($vmid);
5027
5028         # there is no qmp command, so we use the human monitor command
5029         vm_human_monitor_command($vmid, "sendkey $key");
5030     });
5031 }
5032
5033 sub vm_destroy {
5034     my ($storecfg, $vmid, $skiplock) = @_;
5035
5036     PVE::QemuConfig->lock_config($vmid, sub {
5037
5038         my $conf = PVE::QemuConfig->load_config($vmid);
5039
5040         if (!check_running($vmid)) {
5041             destroy_vm($storecfg, $vmid, undef, $skiplock);
5042         } else {
5043             die "VM $vmid is running - destroy failed\n";
5044         }
5045     });
5046 }
5047
5048 # pci helpers
5049
5050 sub file_write {
5051     my ($filename, $buf) = @_;
5052
5053     my $fh = IO::File->new($filename, "w");
5054     return undef if !$fh;
5055
5056     my $res = print $fh $buf;
5057
5058     $fh->close();
5059
5060     return $res;
5061 }
5062
5063 sub pci_device_info {
5064     my ($name) = @_;
5065
5066     my $res;
5067
5068     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
5069     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
5070
5071     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
5072     return undef if !defined($irq) || $irq !~ m/^\d+$/;
5073
5074     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
5075     return undef if !defined($vendor) || $vendor !~ s/^0x//;
5076
5077     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
5078     return undef if !defined($product) || $product !~ s/^0x//;
5079
5080     $res = {
5081         name => $name,
5082         vendor => $vendor,
5083         product => $product,
5084         domain => $domain,
5085         bus => $bus,
5086         slot => $slot,
5087         func => $func,
5088         irq => $irq,
5089         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
5090     };
5091
5092     return $res;
5093 }
5094
5095 sub pci_dev_reset {
5096     my ($dev) = @_;
5097
5098     my $name = $dev->{name};
5099
5100     my $fn = "$pcisysfs/devices/$name/reset";
5101
5102     return file_write($fn, "1");
5103 }
5104
5105 sub pci_dev_bind_to_vfio {
5106     my ($dev) = @_;
5107
5108     my $name = $dev->{name};
5109
5110     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5111
5112     if (!-d $vfio_basedir) {
5113         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5114     }
5115     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5116
5117     my $testdir = "$vfio_basedir/$name";
5118     return 1 if -d $testdir;
5119
5120     my $data = "$dev->{vendor} $dev->{product}";
5121     return undef if !file_write("$vfio_basedir/new_id", $data);
5122
5123     my $fn = "$pcisysfs/devices/$name/driver/unbind";
5124     if (!file_write($fn, $name)) {
5125         return undef if -f $fn;
5126     }
5127
5128     $fn = "$vfio_basedir/bind";
5129     if (! -d $testdir) {
5130         return undef if !file_write($fn, $name);
5131     }
5132
5133     return -d $testdir;
5134 }
5135
5136 sub pci_dev_group_bind_to_vfio {
5137     my ($pciid) = @_;
5138
5139     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5140
5141     if (!-d $vfio_basedir) {
5142         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5143     }
5144     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5145
5146     # get IOMMU group devices
5147     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
5148       my @devs = grep /^0000:/, readdir($D);
5149     closedir($D);
5150
5151     foreach my $pciid (@devs) {
5152         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
5153
5154         # pci bridges, switches or root ports are not supported
5155         # they have a pci_bus subdirectory so skip them
5156         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
5157
5158         my $info = pci_device_info($1);
5159         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
5160     }
5161
5162     return 1;
5163 }
5164
5165 # vzdump restore implementaion
5166
5167 sub tar_archive_read_firstfile {
5168     my $archive = shift;
5169
5170     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5171
5172     # try to detect archive type first
5173     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5174         die "unable to open file '$archive'\n";
5175     my $firstfile = <$fh>;
5176     kill 15, $pid;
5177     close $fh;
5178
5179     die "ERROR: archive contaions no data\n" if !$firstfile;
5180     chomp $firstfile;
5181
5182     return $firstfile;
5183 }
5184
5185 sub tar_restore_cleanup {
5186     my ($storecfg, $statfile) = @_;
5187
5188     print STDERR "starting cleanup\n";
5189
5190     if (my $fd = IO::File->new($statfile, "r")) {
5191         while (defined(my $line = <$fd>)) {
5192             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5193                 my $volid = $2;
5194                 eval {
5195                     if ($volid =~ m|^/|) {
5196                         unlink $volid || die 'unlink failed\n';
5197                     } else {
5198                         PVE::Storage::vdisk_free($storecfg, $volid);
5199                     }
5200                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5201                 };
5202                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5203             } else {
5204                 print STDERR "unable to parse line in statfile - $line";
5205             }
5206         }
5207         $fd->close();
5208     }
5209 }
5210
5211 sub restore_archive {
5212     my ($archive, $vmid, $user, $opts) = @_;
5213
5214     my $format = $opts->{format};
5215     my $comp;
5216
5217     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5218         $format = 'tar' if !$format;
5219         $comp = 'gzip';
5220     } elsif ($archive =~ m/\.tar$/) {
5221         $format = 'tar' if !$format;
5222     } elsif ($archive =~ m/.tar.lzo$/) {
5223         $format = 'tar' if !$format;
5224         $comp = 'lzop';
5225     } elsif ($archive =~ m/\.vma$/) {
5226         $format = 'vma' if !$format;
5227     } elsif ($archive =~ m/\.vma\.gz$/) {
5228         $format = 'vma' if !$format;
5229         $comp = 'gzip';
5230     } elsif ($archive =~ m/\.vma\.lzo$/) {
5231         $format = 'vma' if !$format;
5232         $comp = 'lzop';
5233     } else {
5234         $format = 'vma' if !$format; # default
5235     }
5236
5237     # try to detect archive format
5238     if ($format eq 'tar') {
5239         return restore_tar_archive($archive, $vmid, $user, $opts);
5240     } else {
5241         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5242     }
5243 }
5244
5245 sub restore_update_config_line {
5246     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5247
5248     return if $line =~ m/^\#qmdump\#/;
5249     return if $line =~ m/^\#vzdump\#/;
5250     return if $line =~ m/^lock:/;
5251     return if $line =~ m/^unused\d+:/;
5252     return if $line =~ m/^parent:/;
5253     return if $line =~ m/^template:/; # restored VM is never a template
5254
5255     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5256     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5257         # try to convert old 1.X settings
5258         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5259         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5260             my ($model, $macaddr) = split(/\=/, $devconfig);
5261             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5262             my $net = {
5263                 model => $model,
5264                 bridge => "vmbr$ind",
5265                 macaddr => $macaddr,
5266             };
5267             my $netstr = print_net($net);
5268
5269             print $outfd "net$cookie->{netcount}: $netstr\n";
5270             $cookie->{netcount}++;
5271         }
5272     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5273         my ($id, $netstr) = ($1, $2);
5274         my $net = parse_net($netstr);
5275         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5276         $netstr = print_net($net);
5277         print $outfd "$id: $netstr\n";
5278     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5279         my $virtdev = $1;
5280         my $value = $3;
5281         my $di = parse_drive($virtdev, $value);
5282         if (defined($di->{backup}) && !$di->{backup}) {
5283             print $outfd "#$line";
5284         } elsif ($map->{$virtdev}) {
5285             delete $di->{format}; # format can change on restore
5286             $di->{file} = $map->{$virtdev};
5287             $value = print_drive($vmid, $di);
5288             print $outfd "$virtdev: $value\n";
5289         } else {
5290             print $outfd $line;
5291         }
5292     } else {
5293         print $outfd $line;
5294     }
5295 }
5296
5297 sub scan_volids {
5298     my ($cfg, $vmid) = @_;
5299
5300     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5301
5302     my $volid_hash = {};
5303     foreach my $storeid (keys %$info) {
5304         foreach my $item (@{$info->{$storeid}}) {
5305             next if !($item->{volid} && $item->{size});
5306             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5307             $volid_hash->{$item->{volid}} = $item;
5308         }
5309     }
5310
5311     return $volid_hash;
5312 }
5313
5314 sub is_volume_in_use {
5315     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5316
5317     my $path = PVE::Storage::path($storecfg, $volid);
5318
5319     my $scan_config = sub {
5320         my ($cref, $snapname) = @_;
5321
5322         foreach my $key (keys %$cref) {
5323             my $value = $cref->{$key};
5324             if (is_valid_drivename($key)) {
5325                 next if $skip_drive && $key eq $skip_drive;
5326                 my $drive = parse_drive($key, $value);
5327                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5328                 return 1 if $volid eq $drive->{file};
5329                 if ($drive->{file} =~ m!^/!) {
5330                     return 1 if $drive->{file} eq $path;
5331                 } else {
5332                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5333                     next if !$storeid;
5334                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5335                     next if !$scfg;
5336                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5337                 }
5338             }
5339         }
5340
5341         return 0;
5342     };
5343
5344     return 1 if &$scan_config($conf);
5345
5346     undef $skip_drive;
5347
5348     foreach my $snapname (keys %{$conf->{snapshots}}) {
5349         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5350     }
5351
5352     return 0;
5353 }
5354
5355 sub update_disksize {
5356     my ($vmid, $conf, $volid_hash) = @_;
5357
5358     my $changes;
5359
5360     my $used = {};
5361
5362     # Note: it is allowed to define multiple storages with same path (alias), so
5363     # we need to check both 'volid' and real 'path' (two different volid can point
5364     # to the same path).
5365
5366     my $usedpath = {};
5367
5368     # update size info
5369     foreach my $opt (keys %$conf) {
5370         if (is_valid_drivename($opt)) {
5371             my $drive = parse_drive($opt, $conf->{$opt});
5372             my $volid = $drive->{file};
5373             next if !$volid;
5374
5375             $used->{$volid} = 1;
5376             if ($volid_hash->{$volid} &&
5377                 (my $path = $volid_hash->{$volid}->{path})) {
5378                 $usedpath->{$path} = 1;
5379             }
5380
5381             next if drive_is_cdrom($drive);
5382             next if !$volid_hash->{$volid};
5383
5384             $drive->{size} = $volid_hash->{$volid}->{size};
5385             my $new = print_drive($vmid, $drive);
5386             if ($new ne $conf->{$opt}) {
5387                 $changes = 1;
5388                 $conf->{$opt} = $new;
5389             }
5390         }
5391     }
5392
5393     # remove 'unusedX' entry if volume is used
5394     foreach my $opt (keys %$conf) {
5395         next if $opt !~ m/^unused\d+$/;
5396         my $volid = $conf->{$opt};
5397         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5398         if ($used->{$volid} || ($path && $usedpath->{$path})) {
5399             $changes = 1;
5400             delete $conf->{$opt};
5401         }
5402     }
5403
5404     foreach my $volid (sort keys %$volid_hash) {
5405         next if $volid =~ m/vm-$vmid-state-/;
5406         next if $used->{$volid};
5407         my $path = $volid_hash->{$volid}->{path};
5408         next if !$path; # just to be sure
5409         next if $usedpath->{$path};
5410         $changes = 1;
5411         PVE::QemuConfig->add_unused_volume($conf, $volid);
5412         $usedpath->{$path} = 1; # avoid to add more than once (aliases)
5413     }
5414
5415     return $changes;
5416 }
5417
5418 sub rescan {
5419     my ($vmid, $nolock) = @_;
5420
5421     my $cfg = PVE::Storage::config();
5422
5423     my $volid_hash = scan_volids($cfg, $vmid);
5424
5425     my $updatefn =  sub {
5426         my ($vmid) = @_;
5427
5428         my $conf = PVE::QemuConfig->load_config($vmid);
5429
5430         PVE::QemuConfig->check_lock($conf);
5431
5432         my $vm_volids = {};
5433         foreach my $volid (keys %$volid_hash) {
5434             my $info = $volid_hash->{$volid};
5435             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5436         }
5437
5438         my $changes = update_disksize($vmid, $conf, $vm_volids);
5439
5440         PVE::QemuConfig->write_config($vmid, $conf) if $changes;
5441     };
5442
5443     if (defined($vmid)) {
5444         if ($nolock) {
5445             &$updatefn($vmid);
5446         } else {
5447             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5448         }
5449     } else {
5450         my $vmlist = config_list();
5451         foreach my $vmid (keys %$vmlist) {
5452             if ($nolock) {
5453                 &$updatefn($vmid);
5454             } else {
5455                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5456             }
5457         }
5458     }
5459 }
5460
5461 sub restore_vma_archive {
5462     my ($archive, $vmid, $user, $opts, $comp) = @_;
5463
5464     my $input = $archive eq '-' ? "<&STDIN" : undef;
5465     my $readfrom = $archive;
5466
5467     my $uncomp = '';
5468     if ($comp) {
5469         $readfrom = '-';
5470         my $qarchive = PVE::Tools::shellquote($archive);
5471         if ($comp eq 'gzip') {
5472             $uncomp = "zcat $qarchive|";
5473         } elsif ($comp eq 'lzop') {
5474             $uncomp = "lzop -d -c $qarchive|";
5475         } else {
5476             die "unknown compression method '$comp'\n";
5477         }
5478
5479     }
5480
5481     my $tmpdir = "/var/tmp/vzdumptmp$$";
5482     rmtree $tmpdir;
5483
5484     # disable interrupts (always do cleanups)
5485     local $SIG{INT} =
5486         local $SIG{TERM} =
5487         local $SIG{QUIT} =
5488         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
5489
5490     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5491     POSIX::mkfifo($mapfifo, 0600);
5492     my $fifofh;
5493
5494     my $openfifo = sub {
5495         open($fifofh, '>', $mapfifo) || die $!;
5496     };
5497
5498     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5499
5500     my $oldtimeout;
5501     my $timeout = 5;
5502
5503     my $devinfo = {};
5504
5505     my $rpcenv = PVE::RPCEnvironment::get();
5506
5507     my $conffile = PVE::QemuConfig->config_file($vmid);
5508     my $tmpfn = "$conffile.$$.tmp";
5509
5510     # Note: $oldconf is undef if VM does not exists
5511     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
5512     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
5513
5514     my $print_devmap = sub {
5515         my $virtdev_hash = {};
5516
5517         my $cfgfn = "$tmpdir/qemu-server.conf";
5518
5519         # we can read the config - that is already extracted
5520         my $fh = IO::File->new($cfgfn, "r") ||
5521             "unable to read qemu-server.conf - $!\n";
5522
5523         my $fwcfgfn = "$tmpdir/qemu-server.fw";
5524         if (-f $fwcfgfn) {
5525             my $pve_firewall_dir = '/etc/pve/firewall';
5526             mkdir $pve_firewall_dir; # make sure the dir exists
5527             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
5528         }
5529
5530         while (defined(my $line = <$fh>)) {
5531             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5532                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5533                 die "archive does not contain data for drive '$virtdev'\n"
5534                     if !$devinfo->{$devname};
5535                 if (defined($opts->{storage})) {
5536                     $storeid = $opts->{storage} || 'local';
5537                 } elsif (!$storeid) {
5538                     $storeid = 'local';
5539                 }
5540                 $format = 'raw' if !$format;
5541                 $devinfo->{$devname}->{devname} = $devname;
5542                 $devinfo->{$devname}->{virtdev} = $virtdev;
5543                 $devinfo->{$devname}->{format} = $format;
5544                 $devinfo->{$devname}->{storeid} = $storeid;
5545
5546                 # check permission on storage
5547                 my $pool = $opts->{pool}; # todo: do we need that?
5548                 if ($user ne 'root@pam') {
5549                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5550                 }
5551
5552                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5553             }
5554         }
5555
5556         foreach my $devname (keys %$devinfo) {
5557             die "found no device mapping information for device '$devname'\n"
5558                 if !$devinfo->{$devname}->{virtdev};
5559         }
5560
5561         my $cfg = PVE::Storage::config();
5562
5563         # create empty/temp config
5564         if ($oldconf) {
5565             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5566             foreach_drive($oldconf, sub {
5567                 my ($ds, $drive) = @_;
5568
5569                 return if drive_is_cdrom($drive);
5570
5571                 my $volid = $drive->{file};
5572
5573                 return if !$volid || $volid =~ m|^/|;
5574
5575                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5576                 return if !$path || !$owner || ($owner != $vmid);
5577
5578                 # Note: only delete disk we want to restore
5579                 # other volumes will become unused
5580                 if ($virtdev_hash->{$ds}) {
5581                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
5582                     if (my $err = $@) {
5583                         warn $err;
5584                     }
5585                 }
5586             });
5587
5588             # delete vmstate files
5589             # since after the restore we have no snapshots anymore
5590             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5591                 my $snap = $oldconf->{snapshots}->{$snapname};
5592                 if ($snap->{vmstate}) {
5593                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
5594                     if (my $err = $@) {
5595                         warn $err;
5596                     }
5597                 }
5598             }
5599         }
5600
5601         my $map = {};
5602         foreach my $virtdev (sort keys %$virtdev_hash) {
5603             my $d = $virtdev_hash->{$virtdev};
5604             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5605             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5606
5607             # test if requested format is supported
5608             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5609             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5610             $d->{format} = $defFormat if !$supported;
5611
5612             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5613                                                   $d->{format}, undef, $alloc_size);
5614             print STDERR "new volume ID is '$volid'\n";
5615             $d->{volid} = $volid;
5616             my $path = PVE::Storage::path($cfg, $volid);
5617
5618             PVE::Storage::activate_volumes($cfg,[$volid]);
5619
5620             my $write_zeros = 1;
5621             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
5622                 $write_zeros = 0;
5623             }
5624
5625             print $fifofh "format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
5626
5627             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5628             $map->{$virtdev} = $volid;
5629         }
5630
5631         $fh->seek(0, 0) || die "seek failed - $!\n";
5632
5633         my $outfd = new IO::File ($tmpfn, "w") ||
5634             die "unable to write config for VM $vmid\n";
5635
5636         my $cookie = { netcount => 0 };
5637         while (defined(my $line = <$fh>)) {
5638             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5639         }
5640
5641         $fh->close();
5642         $outfd->close();
5643     };
5644
5645     eval {
5646         # enable interrupts
5647         local $SIG{INT} =
5648             local $SIG{TERM} =
5649             local $SIG{QUIT} =
5650             local $SIG{HUP} =
5651             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
5652         local $SIG{ALRM} = sub { die "got timeout\n"; };
5653
5654         $oldtimeout = alarm($timeout);
5655
5656         my $parser = sub {
5657             my $line = shift;
5658
5659             print "$line\n";
5660
5661             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5662                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5663                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5664             } elsif ($line =~ m/^CTIME: /) {
5665                 # we correctly received the vma config, so we can disable
5666                 # the timeout now for disk allocation (set to 10 minutes, so
5667                 # that we always timeout if something goes wrong)
5668                 alarm(600);
5669                 &$print_devmap();
5670                 print $fifofh "done\n";
5671                 my $tmp = $oldtimeout || 0;
5672                 $oldtimeout = undef;
5673                 alarm($tmp);
5674                 close($fifofh);
5675             }
5676         };
5677
5678         print "restore vma archive: $cmd\n";
5679         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5680     };
5681     my $err = $@;
5682
5683     alarm($oldtimeout) if $oldtimeout;
5684
5685     my $vollist = [];
5686     foreach my $devname (keys %$devinfo) {
5687         my $volid = $devinfo->{$devname}->{volid};
5688         push @$vollist, $volid if $volid;
5689     }
5690
5691     my $cfg = PVE::Storage::config();
5692     PVE::Storage::deactivate_volumes($cfg, $vollist);
5693
5694     unlink $mapfifo;
5695
5696     if ($err) {
5697         rmtree $tmpdir;
5698         unlink $tmpfn;
5699
5700         foreach my $devname (keys %$devinfo) {
5701             my $volid = $devinfo->{$devname}->{volid};
5702             next if !$volid;
5703             eval {
5704                 if ($volid =~ m|^/|) {
5705                     unlink $volid || die 'unlink failed\n';
5706                 } else {
5707                     PVE::Storage::vdisk_free($cfg, $volid);
5708                 }
5709                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5710             };
5711             print STDERR "unable to cleanup '$volid' - $@" if $@;
5712         }
5713         die $err;
5714     }
5715
5716     rmtree $tmpdir;
5717
5718     rename($tmpfn, $conffile) ||
5719         die "unable to commit configuration file '$conffile'\n";
5720
5721     PVE::Cluster::cfs_update(); # make sure we read new file
5722
5723     eval { rescan($vmid, 1); };
5724     warn $@ if $@;
5725 }
5726
5727 sub restore_tar_archive {
5728     my ($archive, $vmid, $user, $opts) = @_;
5729
5730     if ($archive ne '-') {
5731         my $firstfile = tar_archive_read_firstfile($archive);
5732         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5733             if $firstfile ne 'qemu-server.conf';
5734     }
5735
5736     my $storecfg = PVE::Storage::config();
5737
5738     # destroy existing data - keep empty config
5739     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
5740     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5741
5742     my $tocmd = "/usr/lib/qemu-server/qmextract";
5743
5744     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5745     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5746     $tocmd .= ' --prealloc' if $opts->{prealloc};
5747     $tocmd .= ' --info' if $opts->{info};
5748
5749     # tar option "xf" does not autodetect compression when read from STDIN,
5750     # so we pipe to zcat
5751     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5752         PVE::Tools::shellquote("--to-command=$tocmd");
5753
5754     my $tmpdir = "/var/tmp/vzdumptmp$$";
5755     mkpath $tmpdir;
5756
5757     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5758     local $ENV{VZDUMP_VMID} = $vmid;
5759     local $ENV{VZDUMP_USER} = $user;
5760
5761     my $conffile = PVE::QemuConfig->config_file($vmid);
5762     my $tmpfn = "$conffile.$$.tmp";
5763
5764     # disable interrupts (always do cleanups)
5765     local $SIG{INT} =
5766         local $SIG{TERM} =
5767         local $SIG{QUIT} =
5768         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
5769
5770     eval {
5771         # enable interrupts
5772         local $SIG{INT} =
5773             local $SIG{TERM} =
5774             local $SIG{QUIT} =
5775             local $SIG{HUP} =
5776             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
5777
5778         if ($archive eq '-') {
5779             print "extracting archive from STDIN\n";
5780             run_command($cmd, input => "<&STDIN");
5781         } else {
5782             print "extracting archive '$archive'\n";
5783             run_command($cmd);
5784         }
5785
5786         return if $opts->{info};
5787
5788         # read new mapping
5789         my $map = {};
5790         my $statfile = "$tmpdir/qmrestore.stat";
5791         if (my $fd = IO::File->new($statfile, "r")) {
5792             while (defined (my $line = <$fd>)) {
5793                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5794                     $map->{$1} = $2 if $1;
5795                 } else {
5796                     print STDERR "unable to parse line in statfile - $line\n";
5797                 }
5798             }
5799             $fd->close();
5800         }
5801
5802         my $confsrc = "$tmpdir/qemu-server.conf";
5803
5804         my $srcfd = new IO::File($confsrc, "r") ||
5805             die "unable to open file '$confsrc'\n";
5806
5807         my $outfd = new IO::File ($tmpfn, "w") ||
5808             die "unable to write config for VM $vmid\n";
5809
5810         my $cookie = { netcount => 0 };
5811         while (defined (my $line = <$srcfd>)) {
5812             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5813         }
5814
5815         $srcfd->close();
5816         $outfd->close();
5817     };
5818     my $err = $@;
5819
5820     if ($err) {
5821
5822         unlink $tmpfn;
5823
5824         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
5825
5826         die $err;
5827     }
5828
5829     rmtree $tmpdir;
5830
5831     rename $tmpfn, $conffile ||
5832         die "unable to commit configuration file '$conffile'\n";
5833
5834     PVE::Cluster::cfs_update(); # make sure we read new file
5835
5836     eval { rescan($vmid, 1); };
5837     warn $@ if $@;
5838 };
5839
5840 sub foreach_storage_used_by_vm {
5841     my ($conf, $func) = @_;
5842
5843     my $sidhash = {};
5844
5845     foreach_drive($conf, sub {
5846         my ($ds, $drive) = @_;
5847         return if drive_is_cdrom($drive);
5848
5849         my $volid = $drive->{file};
5850
5851         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5852         $sidhash->{$sid} = $sid if $sid;
5853     });
5854
5855     foreach my $sid (sort keys %$sidhash) {
5856         &$func($sid);
5857     }
5858 }
5859
5860 sub do_snapshots_with_qemu {
5861     my ($storecfg, $volid) = @_;
5862
5863     my $storage_name = PVE::Storage::parse_volume_id($volid);
5864
5865     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
5866         && !$storecfg->{ids}->{$storage_name}->{krbd}){
5867         return 1;
5868     }
5869
5870     if ($volid =~ m/\.(qcow2|qed)$/){
5871         return 1;
5872     }
5873
5874     return undef;
5875 }
5876
5877 sub qga_check_running {
5878     my ($vmid) = @_;
5879
5880     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
5881     if ($@) {
5882         warn "Qemu Guest Agent are not running - $@";
5883         return 0;
5884     }
5885     return 1;
5886 }
5887
5888 sub template_create {
5889     my ($vmid, $conf, $disk) = @_;
5890
5891     my $storecfg = PVE::Storage::config();
5892
5893     foreach_drive($conf, sub {
5894         my ($ds, $drive) = @_;
5895
5896         return if drive_is_cdrom($drive);
5897         return if $disk && $ds ne $disk;
5898
5899         my $volid = $drive->{file};
5900         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
5901
5902         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
5903         $drive->{file} = $voliddst;
5904         $conf->{$ds} = print_drive($vmid, $drive);
5905         PVE::QemuConfig->write_config($vmid, $conf);
5906     });
5907 }
5908
5909 sub qemu_img_convert {
5910     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
5911
5912     my $storecfg = PVE::Storage::config();
5913     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
5914     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
5915
5916     if ($src_storeid && $dst_storeid) {
5917
5918         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
5919
5920         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
5921         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5922
5923         my $src_format = qemu_img_format($src_scfg, $src_volname);
5924         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
5925
5926         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
5927         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5928
5929         my $cmd = [];
5930         push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
5931         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
5932         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path;
5933         if ($is_zero_initialized) {
5934             push @$cmd, "zeroinit:$dst_path";
5935         } else {
5936             push @$cmd, $dst_path;
5937         }
5938
5939         my $parser = sub {
5940             my $line = shift;
5941             if($line =~ m/\((\S+)\/100\%\)/){
5942                 my $percent = $1;
5943                 my $transferred = int($size * $percent / 100);
5944                 my $remaining = $size - $transferred;
5945
5946                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
5947             }
5948
5949         };
5950
5951         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
5952         my $err = $@;
5953         die "copy failed: $err" if $err;
5954     }
5955 }
5956
5957 sub qemu_img_format {
5958     my ($scfg, $volname) = @_;
5959
5960     if ($scfg->{path} && $volname =~ m/\.(raw|cow|qcow|qcow2|qed|vmdk|cloop)$/) {
5961         return $1;
5962     } else {
5963         return "raw";
5964     }
5965 }
5966
5967 sub qemu_drive_mirror {
5968     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga) = @_;
5969
5970     $jobs = {} if !$jobs;
5971
5972     my $qemu_target;
5973     my $format;
5974     $jobs->{"drive-$drive"} = {};
5975
5976     if ($dst_volid =~ /^nbd:(localhost|[\d\.]+|\[[\d\.:a-fA-F]+\]):(\d+):exportname=(\S+)/) {
5977         my $server = $1;
5978         my $port = $2;
5979         my $exportname = $3;
5980
5981         $format = "nbd";
5982         my $unixsocket = "/run/qemu-server/$vmid.mirror-drive-$drive";
5983         $qemu_target = "nbd+unix:///$exportname?socket=$unixsocket";
5984         my $cmd = ['socat', '-T30', "UNIX-LISTEN:$unixsocket,fork", "TCP:$server:$2,connect-timeout=5"];
5985
5986         my $pid = fork();
5987         if (!defined($pid)) {
5988             die "forking socat tunnel failed\n";
5989         } elsif ($pid == 0) {
5990             exec(@$cmd);
5991             warn "exec failed: $!\n";
5992             POSIX::_exit(-1);
5993         }
5994         $jobs->{"drive-$drive"}->{pid} = $pid;
5995
5996         my $timeout = 0;
5997         while (!-S $unixsocket) {
5998             die "nbd connection helper timed out\n"
5999                 if $timeout++ > 5;
6000             sleep 1;
6001         }
6002     } else {
6003         my $storecfg = PVE::Storage::config();
6004         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6005
6006         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6007
6008         $format = qemu_img_format($dst_scfg, $dst_volname);
6009
6010         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6011
6012         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6013     }
6014
6015     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6016     $opts->{format} = $format if $format;
6017
6018     print "drive mirror is starting for drive-$drive\n";
6019
6020     eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); }; #if a job already run for this device,it's throw an error
6021
6022     if (my $err = $@) {
6023         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6024         die "mirroring error: $err";
6025     }
6026
6027     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6028 }
6029
6030 sub qemu_drive_mirror_monitor {
6031     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6032
6033     eval {
6034         my $err_complete = 0;
6035
6036         while (1) {
6037             die "storage migration timed out\n" if $err_complete > 300;
6038
6039             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6040
6041             my $running_mirror_jobs = {};
6042             foreach my $stat (@$stats) {
6043                 next if $stat->{type} ne 'mirror';
6044                 $running_mirror_jobs->{$stat->{device}} = $stat;
6045             }
6046
6047             my $readycounter = 0;
6048
6049             foreach my $job (keys %$jobs) {
6050
6051                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6052                     print "$job : finished\n";
6053                     delete $jobs->{$job};
6054                     next;
6055                 }
6056
6057                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6058
6059                 my $busy = $running_mirror_jobs->{$job}->{busy};
6060                 my $ready = $running_mirror_jobs->{$job}->{ready};
6061                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6062                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6063                     my $remaining = $total - $transferred;
6064                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6065
6066                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6067                 }
6068
6069                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6070             }
6071
6072             last if scalar(keys %$jobs) == 0;
6073
6074             if ($readycounter == scalar(keys %$jobs)) {
6075                 print "all mirroring jobs are ready \n";
6076                 last if $skipcomplete; #do the complete later
6077
6078                 if ($vmiddst && $vmiddst != $vmid) {
6079                     if ($qga) {
6080                         print "freeze filesystem\n";
6081                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6082                     } else {
6083                         print "suspend vm\n";
6084                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6085                     }
6086
6087                     # if we clone a disk for a new target vm, we don't switch the disk
6088                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6089
6090                     if ($qga) {
6091                         print "unfreeze filesystem\n";
6092                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6093                     } else {
6094                         print "resume vm\n";
6095                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6096                     }
6097
6098                     last;
6099                 } else {
6100
6101                     foreach my $job (keys %$jobs) {
6102                         # try to switch the disk if source and destination are on the same guest
6103                         print "$job: Completing block job...\n";
6104
6105                         eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
6106                         if ($@ =~ m/cannot be completed/) {
6107                             print "$job: Block job cannot be completed, try again.\n";
6108                             $err_complete++;
6109                         }else {
6110                             print "$job: Completed successfully.\n";
6111                             $jobs->{$job}->{complete} = 1;
6112                             eval { qemu_blockjobs_finish_tunnel($vmid, $job, $jobs->{$job}->{pid}) } ;
6113                         }
6114                     }
6115                 }
6116             }
6117             sleep 1;
6118         }
6119     };
6120     my $err = $@;
6121
6122     if ($err) {
6123         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6124         die "mirroring error: $err";
6125     }
6126
6127 }
6128
6129 sub qemu_blockjobs_cancel {
6130     my ($vmid, $jobs) = @_;
6131
6132     foreach my $job (keys %$jobs) {
6133         print "$job: Cancelling block job\n";
6134         eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
6135         $jobs->{$job}->{cancel} = 1;
6136     }
6137
6138     while (1) {
6139         my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6140
6141         my $running_jobs = {};
6142         foreach my $stat (@$stats) {
6143             $running_jobs->{$stat->{device}} = $stat;
6144         }
6145
6146         foreach my $job (keys %$jobs) {
6147
6148             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6149                 print "$job: Done.\n";
6150                 eval { qemu_blockjobs_finish_tunnel($vmid, $job, $jobs->{$job}->{pid}) } ;
6151                 delete $jobs->{$job};
6152             }
6153         }
6154
6155         last if scalar(keys %$jobs) == 0;
6156
6157         sleep 1;
6158     }
6159 }
6160
6161 sub qemu_blockjobs_finish_tunnel {
6162    my ($vmid, $job, $cpid) = @_;
6163
6164    return if !$cpid;
6165
6166    for (my $i = 1; $i < 20; $i++) {
6167         my $waitpid = waitpid($cpid, WNOHANG);
6168         last if (defined($waitpid) && ($waitpid == $cpid));
6169
6170         if ($i == 10) {
6171             kill(15, $cpid);
6172         } elsif ($i >= 15) {
6173             kill(9, $cpid);
6174         }
6175         sleep (1);
6176     }
6177     unlink "/run/qemu-server/$vmid.mirror-$job";
6178 }
6179
6180 sub clone_disk {
6181     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6182         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga) = @_;
6183
6184     my $newvolid;
6185
6186     if (!$full) {
6187         print "create linked clone of drive $drivename ($drive->{file})\n";
6188         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6189         push @$newvollist, $newvolid;
6190     } else {
6191
6192         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6193         $storeid = $storage if $storage;
6194
6195         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
6196         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6197
6198         print "create full clone of drive $drivename ($drive->{file})\n";
6199         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, undef, ($size/1024));
6200         push @$newvollist, $newvolid;
6201
6202         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
6203
6204         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6205         if (!$running || $snapname) {
6206             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6207         } else {
6208
6209             my $kvmver = get_running_qemu_version ($vmid);
6210             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
6211                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6212                     if $drive->{iothread};
6213             }
6214
6215             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga);
6216         }
6217     }
6218
6219     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6220
6221     my $disk = $drive;
6222     $disk->{format} = undef;
6223     $disk->{file} = $newvolid;
6224     $disk->{size} = $size;
6225
6226     return $disk;
6227 }
6228
6229 # this only works if VM is running
6230 sub get_current_qemu_machine {
6231     my ($vmid) = @_;
6232
6233     my $cmd = { execute => 'query-machines', arguments => {} };
6234     my $res = vm_qmp_command($vmid, $cmd);
6235
6236     my ($current, $default);
6237     foreach my $e (@$res) {
6238         $default = $e->{name} if $e->{'is-default'};
6239         $current = $e->{name} if $e->{'is-current'};
6240     }
6241
6242     # fallback to the default machine if current is not supported by qemu
6243     return $current || $default || 'pc';
6244 }
6245
6246 sub get_running_qemu_version {
6247     my ($vmid) = @_;
6248     my $cmd = { execute => 'query-version', arguments => {} };
6249     my $res = vm_qmp_command($vmid, $cmd);
6250     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
6251 }
6252
6253 sub qemu_machine_feature_enabled {
6254     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6255
6256     my $current_major;
6257     my $current_minor;
6258
6259     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6260
6261         $current_major = $3;
6262         $current_minor = $4;
6263
6264     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6265
6266         $current_major = $1;
6267         $current_minor = $2;
6268     }
6269
6270     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
6271
6272
6273 }
6274
6275 sub qemu_machine_pxe {
6276     my ($vmid, $conf, $machine) = @_;
6277
6278     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6279
6280     foreach my $opt (keys %$conf) {
6281         next if $opt !~ m/^net(\d+)$/;
6282         my $net = PVE::QemuServer::parse_net($conf->{$opt});
6283         next if !$net;
6284         my $romfile = PVE::QemuServer::vm_mon_cmd_nocheck($vmid, 'qom-get', path => $opt, property => 'romfile');
6285         return $machine.".pxe" if $romfile =~ m/pxe/;
6286         last;
6287     }
6288
6289     return $machine;
6290 }
6291
6292 sub qemu_use_old_bios_files {
6293     my ($machine_type) = @_;
6294
6295     return if !$machine_type;
6296
6297     my $use_old_bios_files = undef;
6298
6299     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6300         $machine_type = $1;
6301         $use_old_bios_files = 1;
6302     } else {
6303         my $kvmver = kvm_user_version();
6304         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6305         # load new efi bios files on migration. So this hack is required to allow
6306         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6307         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6308         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
6309     }
6310
6311     return ($use_old_bios_files, $machine_type);
6312 }
6313
6314 sub create_efidisk {
6315     my ($storecfg, $storeid, $vmid, $fmt) = @_;
6316
6317     die "EFI vars default image not found\n" if ! -f $OVMF_VARS;
6318
6319     my $vars_size = PVE::Tools::convert_size(-s $OVMF_VARS, 'b' => 'kb');
6320     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
6321     PVE::Storage::activate_volumes($storecfg, [$volid]);
6322
6323     my $path = PVE::Storage::path($storecfg, $volid);
6324     eval {
6325         run_command(['/usr/bin/qemu-img', 'convert', '-n', '-f', 'raw', '-O', $fmt, $OVMF_VARS, $path]);
6326     };
6327     die "Copying EFI vars image failed: $@" if $@;
6328
6329     return ($volid, $vars_size);
6330 }
6331
6332 sub lspci {
6333
6334     my $devices = {};
6335
6336     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6337             my (undef, $id, $function) = @_;
6338             my $res = { id => $id, function => $function};
6339             push @{$devices->{$id}}, $res;
6340     });
6341
6342     # Entries should be sorted by functions.
6343     foreach my $id (keys %$devices) {
6344         my $dev = $devices->{$id};
6345         $devices->{$id} = [ sort { $a->{function} <=> $b->{function} } @$dev ];
6346     }
6347
6348     return $devices;
6349 }
6350
6351 sub vm_iothreads_list {
6352     my ($vmid) = @_;
6353
6354     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6355
6356     my $iothreads = {};
6357     foreach my $iothread (@$res) {
6358         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6359     }
6360
6361     return $iothreads;
6362 }
6363
6364 sub scsihw_infos {
6365     my ($conf, $drive) = @_;
6366
6367     my $maxdev = 0;
6368
6369     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
6370         $maxdev = 7;
6371     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6372         $maxdev = 1;
6373     } else {
6374         $maxdev = 256;
6375     }
6376
6377     my $controller = int($drive->{index} / $maxdev);
6378     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6379
6380     return ($maxdev, $controller, $controller_prefix);
6381 }
6382
6383 sub add_hyperv_enlightenments {
6384     my ($cpuFlags, $winversion, $machine_type, $kvmver, $bios, $gpu_passthrough) = @_;
6385
6386     return if $winversion < 6;
6387     return if $bios && $bios eq 'ovmf' && $winversion < 8;
6388
6389     push @$cpuFlags , 'hv_vendor_id=proxmox' if $gpu_passthrough;
6390
6391     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
6392         push @$cpuFlags , 'hv_spinlocks=0x1fff';
6393         push @$cpuFlags , 'hv_vapic';
6394         push @$cpuFlags , 'hv_time';
6395     } else {
6396         push @$cpuFlags , 'hv_spinlocks=0xffff';
6397     }
6398
6399     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
6400         push @$cpuFlags , 'hv_reset';
6401         push @$cpuFlags , 'hv_vpindex';
6402         push @$cpuFlags , 'hv_runtime';
6403     }
6404
6405     if ($winversion >= 7) {
6406         push @$cpuFlags , 'hv_relaxed';
6407     }
6408 }
6409
6410 sub windows_version {
6411     my ($ostype) = @_;
6412
6413     return 0 if !$ostype;
6414
6415     my $winversion = 0;
6416
6417     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
6418         $winversion = 5;
6419     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
6420         $winversion = 6;
6421     } elsif ($ostype =~ m/^win(\d+)$/) {
6422         $winversion = $1;
6423     }
6424
6425     return $winversion;
6426 }
6427
6428 sub resolve_dst_disk_format {
6429         my ($storecfg, $storeid, $src_volname, $format) = @_;
6430         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6431
6432         if (!$format) {
6433             # if no target format is specified, use the source disk format as hint
6434             if ($src_volname) {
6435                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6436                 $format = qemu_img_format($scfg, $src_volname);
6437             } else {
6438                 return $defFormat;
6439             }
6440         }
6441
6442         # test if requested format is supported - else use default
6443         my $supported = grep { $_ eq $format } @$validFormats;
6444         $format = $defFormat if !$supported;
6445         return $format;
6446 }
6447
6448 sub resolve_first_disk {
6449     my $conf = shift;
6450     my @disks = PVE::QemuServer::valid_drive_names();
6451     my $firstdisk;
6452     foreach my $ds (reverse @disks) {
6453         next if !$conf->{$ds};
6454         my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
6455         next if PVE::QemuServer::drive_is_cdrom($disk);
6456         $firstdisk = $ds;
6457     }
6458     return $firstdisk;
6459 }
6460
6461 sub generate_smbios1_uuid {
6462     my ($uuid, $uuid_str);
6463     UUID::generate($uuid);
6464     UUID::unparse($uuid, $uuid_str);
6465     return "uuid=$uuid_str";
6466 }
6467
6468 # bash completion helper
6469
6470 sub complete_backup_archives {
6471     my ($cmdname, $pname, $cvalue) = @_;
6472
6473     my $cfg = PVE::Storage::config();
6474
6475     my $storeid;
6476
6477     if ($cvalue =~ m/^([^:]+):/) {
6478         $storeid = $1;
6479     }
6480
6481     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6482
6483     my $res = [];
6484     foreach my $id (keys %$data) {
6485         foreach my $item (@{$data->{$id}}) {
6486             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6487             push @$res, $item->{volid} if defined($item->{volid});
6488         }
6489     }
6490
6491     return $res;
6492 }
6493
6494 my $complete_vmid_full = sub {
6495     my ($running) = @_;
6496
6497     my $idlist = vmstatus();
6498
6499     my $res = [];
6500
6501     foreach my $id (keys %$idlist) {
6502         my $d = $idlist->{$id};
6503         if (defined($running)) {
6504             next if $d->{template};
6505             next if $running && $d->{status} ne 'running';
6506             next if !$running && $d->{status} eq 'running';
6507         }
6508         push @$res, $id;
6509
6510     }
6511     return $res;
6512 };
6513
6514 sub complete_vmid {
6515     return &$complete_vmid_full();
6516 }
6517
6518 sub complete_vmid_stopped {
6519     return &$complete_vmid_full(0);
6520 }
6521
6522 sub complete_vmid_running {
6523     return &$complete_vmid_full(1);
6524 }
6525
6526 sub complete_storage {
6527
6528     my $cfg = PVE::Storage::config();
6529     my $ids = $cfg->{ids};
6530
6531     my $res = [];
6532     foreach my $sid (keys %$ids) {
6533         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6534         next if !$ids->{$sid}->{content}->{images};
6535         push @$res, $sid;
6536     }
6537
6538     return $res;
6539 }
6540
6541 sub nbd_stop {
6542     my ($vmid) = @_;
6543
6544     vm_mon_cmd($vmid, 'nbd-server-stop');
6545 }
6546
6547 1;