PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QemuConfig;
  31 use PVE::QMPClient;
  32 use PVE::RPCEnvironment;
  33 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
  34 use PVE::QemuServer::Memory;
  35 use PVE::QemuServer::USB qw(parse_usb_device);
  36 use Time::HiRes qw(gettimeofday);
  37 use File::Copy qw(copy);
  38 use URI::Escape;
  39
  40 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  41
  42 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  43
  44 # Note about locking: we use flock on the config file protect
  45 # against concurent actions.
  46 # Aditionaly, we have a 'lock' setting in the config file. This
  47 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  48 # allowed when such lock is set. But you can ignore this kind of
  49 # lock with the --skiplock flag.
  50
  51 cfs_register_file('/qemu-server/',
  52                   \&parse_vm_config,
  53                   \&write_vm_config);
  54
  55 PVE::JSONSchema::register_standard_option('skiplock', {
  56     description => "Ignore locks - only root is allowed to use this option.",
  57     type => 'boolean',
  58     optional => 1,
  59 });
  60
  61 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  62     description => "Some command save/restore state from this location.",
  63     type => 'string',
  64     maxLength => 128,
  65     optional => 1,
  66 });
  67
  68 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  69     description => "The name of the snapshot.",
  70     type => 'string', format => 'pve-configid',
  71     maxLength => 40,
  72 });
  73
  74 #no warnings 'redefine';
  75
  76 sub cgroups_write {
  77    my ($controller, $vmid, $option, $value) = @_;
  78
  79    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  80    PVE::ProcFSTools::write_proc_entry($path, $value);
  81
  82 }
  83
  84 my $nodename = PVE::INotify::nodename();
  85
  86 mkdir "/etc/pve/nodes/$nodename";
  87 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  88 mkdir $confdir;
  89
  90 my $var_run_tmpdir = "/var/run/qemu-server";
  91 mkdir $var_run_tmpdir;
  92
  93 my $lock_dir = "/var/lock/qemu-server";
  94 mkdir $lock_dir;
  95
  96 my $pcisysfs = "/sys/bus/pci";
  97
  98 my $cpu_vendor_list = {
  99     # Intel CPUs
 100     486 => 'GenuineIntel',
 101     pentium => 'GenuineIntel',
 102     pentium2  => 'GenuineIntel',
 103     pentium3  => 'GenuineIntel',
 104     coreduo => 'GenuineIntel',
 105     core2duo => 'GenuineIntel',
 106     Conroe  => 'GenuineIntel',
 107     Penryn  => 'GenuineIntel',
 108     Nehalem  => 'GenuineIntel',
 109     Westmere => 'GenuineIntel',
 110     SandyBridge => 'GenuineIntel',
 111     IvyBridge => 'GenuineIntel',
 112     Haswell => 'GenuineIntel',
 113     'Haswell-noTSX' => 'GenuineIntel',
 114     Broadwell => 'GenuineIntel',
 115     'Broadwell-noTSX' => 'GenuineIntel',
 116
 117     # AMD CPUs
 118     athlon => 'AuthenticAMD',
 119     phenom  => 'AuthenticAMD',
 120     Opteron_G1  => 'AuthenticAMD',
 121     Opteron_G2  => 'AuthenticAMD',
 122     Opteron_G3  => 'AuthenticAMD',
 123     Opteron_G4  => 'AuthenticAMD',
 124     Opteron_G5  => 'AuthenticAMD',
 125
 126     # generic types, use vendor from host node
 127     host => 'default',
 128     kvm32 => 'default',
 129     kvm64 => 'default',
 130     qemu32 => 'default',
 131     qemu64 => 'default',
 132 };
 133
 134 my $cpu_fmt = {
 135     cputype => {
 136         description => "Emulated CPU type.",
 137         type => 'string',
 138         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 139         format_description => 'cputype',
 140         default => 'kvm64',
 141         default_key => 1,
 142     },
 143     hidden => {
 144         description => "Do not identify as a KVM virtual machine.",
 145         type => 'boolean',
 146         optional => 1,
 147         default => 0
 148     },
 149 };
 150
 151 my $watchdog_fmt = {
 152     model => {
 153         default_key => 1,
 154         type => 'string',
 155         enum => [qw(i6300esb ib700)],
 156         description => "Watchdog type to emulate.",
 157         default => 'i6300esb',
 158         optional => 1,
 159     },
 160     action => {
 161         type => 'string',
 162         enum => [qw(reset shutdown poweroff pause debug none)],
 163         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 164         optional => 1,
 165     },
 166 };
 167 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 168
 169 my $confdesc = {
 170     onboot => {
 171         optional => 1,
 172         type => 'boolean',
 173         description => "Specifies whether a VM will be started during system bootup.",
 174         default => 0,
 175     },
 176     autostart => {
 177         optional => 1,
 178         type => 'boolean',
 179         description => "Automatic restart after crash (currently ignored).",
 180         default => 0,
 181     },
 182     hotplug => {
 183         optional => 1,
 184         type => 'string', format => 'pve-hotplug-features',
 185         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 186         default => 'network,disk,usb',
 187     },
 188     reboot => {
 189         optional => 1,
 190         type => 'boolean',
 191         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 192         default => 1,
 193     },
 194     lock => {
 195         optional => 1,
 196         type => 'string',
 197         description => "Lock/unlock the VM.",
 198         enum => [qw(migrate backup snapshot rollback)],
 199     },
 200     cpulimit => {
 201         optional => 1,
 202         type => 'number',
 203         description => "Limit of CPU usage.",
 204         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 205         minimum => 0,
 206         maximum => 128,
 207         default => 0,
 208     },
 209     cpuunits => {
 210         optional => 1,
 211         type => 'integer',
 212         description => "CPU weight for a VM.",
 213         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 214         minimum => 0,
 215         maximum => 500000,
 216         default => 1000,
 217     },
 218     memory => {
 219         optional => 1,
 220         type => 'integer',
 221         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 222         minimum => 16,
 223         default => 512,
 224     },
 225     balloon => {
 226         optional => 1,
 227         type => 'integer',
 228         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 229         minimum => 0,
 230     },
 231     shares => {
 232         optional => 1,
 233         type => 'integer',
 234         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 235         minimum => 0,
 236         maximum => 50000,
 237         default => 1000,
 238     },
 239     keyboard => {
 240         optional => 1,
 241         type => 'string',
 242         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.conf' configuration file.",
 243         enum => PVE::Tools::kvmkeymaplist(),
 244         default => 'en-us',
 245     },
 246     name => {
 247         optional => 1,
 248         type => 'string', format => 'dns-name',
 249         description => "Set a name for the VM. Only used on the configuration web interface.",
 250     },
 251     scsihw => {
 252         optional => 1,
 253         type => 'string',
 254         description => "SCSI controller model",
 255         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 256         default => 'lsi',
 257     },
 258     description => {
 259         optional => 1,
 260         type => 'string',
 261         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 262     },
 263     ostype => {
 264         optional => 1,
 265         type => 'string',
 266         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 l24 l26 solaris)],
 267         description => "Specify guest operating system.",
 268         verbose_description => <<EODESC,
 269 Specify guest operating system. This is used to enable special
 270 optimization/features for specific operating systems:
 271
 272 [horizontal]
 273 other;; unspecified OS
 274 wxp;; Microsoft Windows XP
 275 w2k;; Microsoft Windows 2000
 276 w2k3;; Microsoft Windows 2003
 277 w2k8;; Microsoft Windows 2008
 278 wvista;; Microsoft Windows Vista
 279 win7;; Microsoft Windows 7
 280 win8;; Microsoft Windows 8/2012
 281 l24;; Linux 2.4 Kernel
 282 l26;; Linux 2.6/3.X Kernel
 283 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 284 EODESC
 285     },
 286     boot => {
 287         optional => 1,
 288         type => 'string',
 289         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 290         pattern => '[acdn]{1,4}',
 291         default => 'cdn',
 292     },
 293     bootdisk => {
 294         optional => 1,
 295         type => 'string', format => 'pve-qm-bootdisk',
 296         description => "Enable booting from specified disk.",
 297         pattern => '(ide|sata|scsi|virtio)\d+',
 298     },
 299     smp => {
 300         optional => 1,
 301         type => 'integer',
 302         description => "The number of CPUs. Please use option -sockets instead.",
 303         minimum => 1,
 304         default => 1,
 305     },
 306     sockets => {
 307         optional => 1,
 308         type => 'integer',
 309         description => "The number of CPU sockets.",
 310         minimum => 1,
 311         default => 1,
 312     },
 313     cores => {
 314         optional => 1,
 315         type => 'integer',
 316         description => "The number of cores per socket.",
 317         minimum => 1,
 318         default => 1,
 319     },
 320     numa => {
 321         optional => 1,
 322         type => 'boolean',
 323         description => "Enable/disable NUMA.",
 324         default => 0,
 325     },
 326     hugepages => {
 327         optional => 1,
 328         type => 'string',
 329         description => "Enable/disable hugepages memory.",
 330         enum => [qw(any 2 1024)],
 331     },
 332     vcpus => {
 333         optional => 1,
 334         type => 'integer',
 335         description => "Number of hotplugged vcpus.",
 336         minimum => 1,
 337         default => 0,
 338     },
 339     acpi => {
 340         optional => 1,
 341         type => 'boolean',
 342         description => "Enable/disable ACPI.",
 343         default => 1,
 344     },
 345     agent => {
 346         optional => 1,
 347         type => 'boolean',
 348         description => "Enable/disable Qemu GuestAgent.",
 349         default => 0,
 350     },
 351     kvm => {
 352         optional => 1,
 353         type => 'boolean',
 354         description => "Enable/disable KVM hardware virtualization.",
 355         default => 1,
 356     },
 357     tdf => {
 358         optional => 1,
 359         type => 'boolean',
 360         description => "Enable/disable time drift fix.",
 361         default => 0,
 362     },
 363     localtime => {
 364         optional => 1,
 365         type => 'boolean',
 366         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 367     },
 368     freeze => {
 369         optional => 1,
 370         type => 'boolean',
 371         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 372     },
 373     vga => {
 374         optional => 1,
 375         type => 'string',
 376         description => "Select the VGA type.",
 377         verbose_description => "Select the VGA type. If you want to use high resolution" .
 378             " modes (>= 1280x1024x16) then you should use the options " .
 379             "'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and " .
 380             "'cirrus' for other OS types. The 'qxl' option enables the SPICE " .
 381             "display sever. For win* OS you can select how many independent " .
 382             "displays you want, Linux guests can add displays them self. " .
 383             "You can also run without any graphic card, using a serial device" .
 384             " as terminal.",
 385         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 386     },
 387     watchdog => {
 388         optional => 1,
 389         type => 'string', format => 'pve-qm-watchdog',
 390         description => "Create a virtual hardware watchdog device.",
 391         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 392             " (by a guest action), the watchdog must be periodically polled " .
 393             "by an agent inside the guest or else the watchdog will reset " .
 394             "the guest (or execute the respective action specified)",
 395     },
 396     startdate => {
 397         optional => 1,
 398         type => 'string',
 399         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 400         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 401         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 402         default => 'now',
 403     },
 404     startup =>  get_standard_option('pve-startup-order'),
 405     template => {
 406         optional => 1,
 407         type => 'boolean',
 408         description => "Enable/disable Template.",
 409         default => 0,
 410     },
 411     args => {
 412         optional => 1,
 413         type => 'string',
 414         description => "Arbitrary arguments passed to kvm.",
 415         verbose_description => <<EODESCR,
 416 Arbitrary arguments passed to kvm, for example:
 417
 418 args: -no-reboot -no-hpet
 419
 420 NOTE: this option is for experts only.
 421 EODESCR
 422     },
 423     tablet => {
 424         optional => 1,
 425         type => 'boolean',
 426         default => 1,
 427         description => "Enable/disable the USB tablet device.",
 428         verbose_description => "Enable/disable the USB tablet device. This device is " .
 429             "usually needed to allow absolute mouse positioning with VNC. " .
 430             "Else the mouse runs out of sync with normal VNC clients. " .
 431             "If you're running lots of console-only guests on one host, " .
 432             "you may consider disabling this to save some context switches. " .
 433             "This is turned off by default if you use spice (-vga=qxl).",
 434     },
 435     migrate_speed => {
 436         optional => 1,
 437         type => 'integer',
 438         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 439         minimum => 0,
 440         default => 0,
 441     },
 442     migrate_downtime => {
 443         optional => 1,
 444         type => 'number',
 445         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 446         minimum => 0,
 447         default => 0.1,
 448     },
 449     cdrom => {
 450         optional => 1,
 451         type => 'string', format => 'pve-qm-drive',
 452         typetext => 'volume',
 453         description => "This is an alias for option -ide2",
 454     },
 455     cpu => {
 456         optional => 1,
 457         description => "Emulated CPU type.",
 458         type => 'string',
 459         format => $cpu_fmt,
 460     },
 461     parent => get_standard_option('pve-snapshot-name', {
 462         optional => 1,
 463         description => "Parent snapshot name. This is used internally, and should not be modified.",
 464     }),
 465     snaptime => {
 466         optional => 1,
 467         description => "Timestamp for snapshots.",
 468         type => 'integer',
 469         minimum => 0,
 470     },
 471     vmstate => {
 472         optional => 1,
 473         type => 'string', format => 'pve-volume-id',
 474         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 475     },
 476     machine => {
 477         description => "Specific the Qemu machine type.",
 478         type => 'string',
 479         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
 480         maxLength => 40,
 481         optional => 1,
 482     },
 483     smbios1 => {
 484         description => "Specify SMBIOS type 1 fields.",
 485         type => 'string', format => 'pve-qm-smbios1',
 486         maxLength => 256,
 487         optional => 1,
 488     },
 489     protection => {
 490         optional => 1,
 491         type => 'boolean',
 492         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 493         default => 0,
 494     },
 495     bios => {
 496         optional => 1,
 497         type => 'string',
 498         enum => [ qw(seabios ovmf) ],
 499         description => "Select BIOS implementation.",
 500         default => 'seabios',
 501     },
 502 };
 503
 504 # what about other qemu settings ?
 505 #cpu => 'string',
 506 #machine => 'string',
 507 #fda => 'file',
 508 #fdb => 'file',
 509 #mtdblock => 'file',
 510 #sd => 'file',
 511 #pflash => 'file',
 512 #snapshot => 'bool',
 513 #bootp => 'file',
 514 ##tftp => 'dir',
 515 ##smb => 'dir',
 516 #kernel => 'file',
 517 #append => 'string',
 518 #initrd => 'file',
 519 ##soundhw => 'string',
 520
 521 while (my ($k, $v) = each %$confdesc) {
 522     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 523 }
 524
 525 my $MAX_IDE_DISKS = 4;
 526 my $MAX_SCSI_DISKS = 14;
 527 my $MAX_VIRTIO_DISKS = 16;
 528 my $MAX_SATA_DISKS = 6;
 529 my $MAX_USB_DEVICES = 5;
 530 my $MAX_NETS = 32;
 531 my $MAX_UNUSED_DISKS = 8;
 532 my $MAX_HOSTPCI_DEVICES = 4;
 533 my $MAX_SERIAL_PORTS = 4;
 534 my $MAX_PARALLEL_PORTS = 3;
 535 my $MAX_NUMA = 8;
 536
 537 my $numa_fmt = {
 538     cpus => {
 539         type => "string",
 540         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 541         description => "CPUs accessing this NUMA node.",
 542         format_description => "id[-id];...",
 543     },
 544     memory => {
 545         type => "number",
 546         description => "Amount of memory this NUMA node provides.",
 547         optional => 1,
 548     },
 549     hostnodes => {
 550         type => "string",
 551         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 552         description => "Host NUMA nodes to use.",
 553         format_description => "id[-id];...",
 554         optional => 1,
 555     },
 556     policy => {
 557         type => 'string',
 558         enum => [qw(preferred bind interleave)],
 559         description => "NUMA allocation policy.",
 560         optional => 1,
 561     },
 562 };
 563 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 564 my $numadesc = {
 565     optional => 1,
 566     type => 'string', format => $numa_fmt,
 567     description => "NUMA topology.",
 568 };
 569 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 570
 571 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 572     $confdesc->{"numa$i"} = $numadesc;
 573 }
 574
 575 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 576                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 577                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 578 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 579
 580 my $net_fmt_bridge_descr = <<__EOD__;
 581 Bridge to attach the network device to. The Proxmox VE standard bridge
 582 is called 'vmbr0'.
 583
 584 If you do not specify a bridge, we create a kvm user (NATed) network
 585 device, which provides DHCP and DNS services. The following addresses
 586 are used:
 587
 588  10.0.2.2   Gateway
 589  10.0.2.3   DNS Server
 590  10.0.2.4   SMB Server
 591
 592 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 593 __EOD__
 594
 595 my $net_fmt = {
 596     macaddr => {
 597         type => 'string',
 598         pattern => qr/[0-9a-f]{2}(?::[0-9a-f]{2}){5}/i,
 599         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 600         format_description => "XX:XX:XX:XX:XX:XX",
 601         optional => 1,
 602     },
 603     model => {
 604         type => 'string',
 605         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 606         format_description => 'model',
 607         enum => $nic_model_list,
 608         default_key => 1,
 609     },
 610     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 611     bridge => {
 612         type => 'string',
 613         description => $net_fmt_bridge_descr,
 614         format_description => 'bridge',
 615         optional => 1,
 616     },
 617     queues => {
 618         type => 'integer',
 619         minimum => 0, maximum => 16,
 620         description => 'Number of packet queues to be used on the device.',
 621         optional => 1,
 622     },
 623     rate => {
 624         type => 'number',
 625         minimum => 0,
 626         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 627         optional => 1,
 628     },
 629     tag => {
 630         type => 'integer',
 631         minimum => 1, maximum => 4094,
 632         description => 'VLAN tag to apply to packets on this interface.',
 633         optional => 1,
 634     },
 635     trunks => {
 636         type => 'string',
 637         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 638         description => 'VLAN trunks to pass through this interface.',
 639         format_description => 'vlanid[;vlanid...]',
 640         optional => 1,
 641     },
 642     firewall => {
 643         type => 'boolean',
 644         description => 'Whether this interface should be protected by the firewall.',
 645         optional => 1,
 646     },
 647     link_down => {
 648         type => 'boolean',
 649         description => 'Whether this interface should be disconnected (like pulling the plug).',
 650         optional => 1,
 651     },
 652 };
 653
 654 my $netdesc = {
 655     optional => 1,
 656     type => 'string', format => $net_fmt,
 657     description => "Specify network devices.",
 658 };
 659
 660 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 661
 662 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 663     $confdesc->{"net$i"} = $netdesc;
 664 }
 665
 666 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 667 sub verify_volume_id_or_qm_path {
 668     my ($volid, $noerr) = @_;
 669
 670     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 671         return $volid;
 672     }
 673
 674     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 675     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 676     if ($@) {
 677         return undef if $noerr;
 678         die $@;
 679     }
 680     return $volid;
 681 }
 682
 683 my $drivename_hash;
 684
 685 my %drivedesc_base = (
 686     volume => { alias => 'file' },
 687     file => {
 688         type => 'string',
 689         format => 'pve-volume-id-or-qm-path',
 690         default_key => 1,
 691         format_description => 'volume',
 692         description => "The drive's backing volume.",
 693     },
 694     media => {
 695         type => 'string',
 696         enum => [qw(cdrom disk)],
 697         description => "The drive's media type.",
 698         default => 'disk',
 699         optional => 1
 700     },
 701     cyls => {
 702         type => 'integer',
 703         description => "Force the drive's physical geometry to have a specific cylinder count.",
 704         optional => 1
 705     },
 706     heads => {
 707         type => 'integer',
 708         description => "Force the drive's physical geometry to have a specific head count.",
 709         optional => 1
 710     },
 711     secs => {
 712         type => 'integer',
 713         description => "Force the drive's physical geometry to have a specific sector count.",
 714         optional => 1
 715     },
 716     trans => {
 717         type => 'string',
 718         enum => [qw(none lba auto)],
 719         description => "Force disk geometry bios translation mode.",
 720         optional => 1,
 721     },
 722     snapshot => {
 723         type => 'boolean',
 724         description => "Whether the drive should be included when making snapshots.",
 725         optional => 1,
 726     },
 727     cache => {
 728         type => 'string',
 729         enum => [qw(none writethrough writeback unsafe directsync)],
 730         description => "The drive's cache mode",
 731         optional => 1,
 732     },
 733     format => {
 734         type => 'string',
 735         format_description => 'image format',
 736         enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
 737         description => "The drive's backing file's data format.",
 738         optional => 1,
 739     },
 740     size => {
 741         type => 'string',
 742         format => 'disk-size',
 743         format_description => 'DiskSize',
 744         description => "Disk size. This is purely informational and has no effect.",
 745         optional => 1,
 746     },
 747     backup => {
 748         type => 'boolean',
 749         description => "Whether the drive should be included when making backups.",
 750         optional => 1,
 751     },
 752     werror => {
 753         type => 'string',
 754         enum => [qw(enospc ignore report stop)],
 755         description => 'Write error action.',
 756         optional => 1,
 757     },
 758     aio => {
 759         type => 'string',
 760         enum => [qw(native threads)],
 761         description => 'AIO type to use.',
 762         optional => 1,
 763     },
 764     discard => {
 765         type => 'string',
 766         enum => [qw(ignore on)],
 767         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 768         optional => 1,
 769     },
 770     detect_zeroes => {
 771         type => 'boolean',
 772         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 773         optional => 1,
 774     },
 775     serial => {
 776         type => 'string',
 777         format => 'urlencoded',
 778         format_description => 'serial',
 779         maxLength => 20*3, # *3 since it's %xx url enoded
 780         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
 781         optional => 1,
 782     }
 783 );
 784
 785 my %rerror_fmt = (
 786     rerror => {
 787         type => 'string',
 788         enum => [qw(ignore report stop)],
 789         description => 'Read error action.',
 790         optional => 1,
 791     },
 792 );
 793
 794 my %iothread_fmt = ( iothread => {
 795         type => 'boolean',
 796         description => "Whether to use iothreads for this drive",
 797         optional => 1,
 798 });
 799
 800 my %model_fmt = (
 801     model => {
 802         type => 'string',
 803         format => 'urlencoded',
 804         format_description => 'model',
 805         maxLength => 40*3, # *3 since it's %xx url enoded
 806         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
 807         optional => 1,
 808     },
 809 );
 810
 811 my %queues_fmt = (
 812     queues => {
 813         type => 'integer',
 814         description => "Number of queues.",
 815         minimum => 2,
 816         optional => 1
 817     }
 818 );
 819
 820 my $add_throttle_desc = sub {
 821     my ($key, $type, $what, $unit, $longunit) = @_;
 822     $drivedesc_base{$key} = {
 823         type => $type,
 824         format_description => $unit,
 825         description => "Maximum $what speed in $longunit per second.",
 826         optional => 1,
 827     };
 828 };
 829 # throughput: (leaky bucket)
 830 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes');
 831 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes');
 832 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes');
 833 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes');
 834 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes');
 835 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes');
 836 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations');
 837 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations');
 838 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations');
 839
 840 # pools: (pool of IO before throttling starts taking effect)
 841 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes');
 842 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes');
 843 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes');
 844 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations');
 845 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations');
 846 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations');
 847
 848 my $ide_fmt = {
 849     %drivedesc_base,
 850     %rerror_fmt,
 851     %model_fmt,
 852 };
 853
 854 my $idedesc = {
 855     optional => 1,
 856     type => 'string', format => $ide_fmt,
 857     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
 858 };
 859 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
 860
 861 my $scsi_fmt = {
 862     %drivedesc_base,
 863     %iothread_fmt,
 864     %queues_fmt,
 865 };
 866 my $scsidesc = {
 867     optional => 1,
 868     type => 'string', format => $scsi_fmt,
 869     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
 870 };
 871 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 872
 873 my $sata_fmt = {
 874     %drivedesc_base,
 875     %rerror_fmt,
 876 };
 877 my $satadesc = {
 878     optional => 1,
 879     type => 'string', format => $sata_fmt,
 880     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
 881 };
 882 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
 883
 884 my $virtio_fmt = {
 885     %drivedesc_base,
 886     %iothread_fmt,
 887     %rerror_fmt,
 888 };
 889 my $virtiodesc = {
 890     optional => 1,
 891     type => 'string', format => $virtio_fmt,
 892     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
 893 };
 894 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
 895
 896 my $alldrive_fmt = {
 897     %drivedesc_base,
 898     %rerror_fmt,
 899     %iothread_fmt,
 900     %model_fmt,
 901     %queues_fmt,
 902 };
 903
 904 my $usb_fmt = {
 905     host => {
 906         default_key => 1,
 907         type => 'string', format => 'pve-qm-usb-device',
 908         format_description => 'HOSTUSBDEVICE|spice',
 909         description => <<EODESCR,
 910 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
 911
 912  'bus-port(.port)*' (decimal numbers) or
 913  'vendor_id:product_id' (hexadeciaml numbers) or
 914  'spice'
 915
 916 You can use the 'lsusb -t' command to list existing usb devices.
 917
 918 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 919
 920 The value 'spice' can be used to add a usb redirection devices for spice.
 921 EODESCR
 922     },
 923     usb3 => {
 924         optional => 1,
 925         type => 'boolean',
 926         description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
 927         default => 0,
 928     },
 929 };
 930
 931 my $usbdesc = {
 932     optional => 1,
 933     type => 'string', format => $usb_fmt,
 934     description => "Configure an USB device (n is 0 to 4).",
 935 };
 936 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
 937
 938 # NOTE: the match-groups of this regex are used in parse_hostpci
 939 my $PCIRE = qr/([a-f0-9]{2}:[a-f0-9]{2})(?:\.([a-f0-9]))?/;
 940 my $hostpci_fmt = {
 941     host => {
 942         default_key => 1,
 943         type => 'string',
 944         pattern => qr/$PCIRE(;$PCIRE)*/,
 945         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
 946         description => <<EODESCR,
 947 Host PCI device pass through. The PCI ID of a host's PCI device or a list
 948 of PCI virtual functions of the host. HOSTPCIID syntax is:
 949
 950 'bus:dev.func' (hexadecimal numbers)
 951
 952 You can us the 'lspci' command to list existing PCI devices.
 953 EODESCR
 954     },
 955     rombar => {
 956         type => 'boolean',
 957         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
 958         optional => 1,
 959         default => 1,
 960     },
 961     pcie => {
 962         type => 'boolean',
 963         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
 964         optional => 1,
 965         default => 0,
 966     },
 967     'x-vga' => {
 968         type => 'boolean',
 969         description =>  "Enable vfio-vga device support.",
 970         optional => 1,
 971         default => 0,
 972     },
 973 };
 974 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
 975
 976 my $hostpcidesc = {
 977         optional => 1,
 978         type => 'string', format => 'pve-qm-hostpci',
 979         description => "Map host PCI devices into guest.",
 980         verbose_description =>  <<EODESCR,
 981 Map host PCI devices into guest.
 982
 983 NOTE: This option allows direct access to host hardware. So it is no longer
 984 possible to migrate such machines - use with special care.
 985
 986 CAUTION: Experimental! User reported problems with this option.
 987 EODESCR
 988 };
 989 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
 990
 991 my $serialdesc = {
 992         optional => 1,
 993         type => 'string',
 994         pattern => '(/dev/.+|socket)',
 995         description =>  "Create a serial device inside the VM (n is 0 to 3)",
 996         verbose_description =>  <<EODESCR,
 997 Create a serial device inside the VM (n is 0 to 3), and pass through a
 998 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
 999 host side (use 'qm terminal' to open a terminal connection).
1000
1001 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1002
1003 CAUTION: Experimental! User reported problems with this option.
1004 EODESCR
1005 };
1006
1007 my $paralleldesc= {
1008         optional => 1,
1009         type => 'string',
1010         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1011         description =>  "Map host parallel devices (n is 0 to 2).",
1012         verbose_description =>  <<EODESCR,
1013 Map host parallel devices (n is 0 to 2).
1014
1015 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1016
1017 CAUTION: Experimental! User reported problems with this option.
1018 EODESCR
1019 };
1020
1021 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1022     $confdesc->{"parallel$i"} = $paralleldesc;
1023 }
1024
1025 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1026     $confdesc->{"serial$i"} = $serialdesc;
1027 }
1028
1029 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1030     $confdesc->{"hostpci$i"} = $hostpcidesc;
1031 }
1032
1033 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1034     $drivename_hash->{"ide$i"} = 1;
1035     $confdesc->{"ide$i"} = $idedesc;
1036 }
1037
1038 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1039     $drivename_hash->{"sata$i"} = 1;
1040     $confdesc->{"sata$i"} = $satadesc;
1041 }
1042
1043 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1044     $drivename_hash->{"scsi$i"} = 1;
1045     $confdesc->{"scsi$i"} = $scsidesc ;
1046 }
1047
1048 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1049     $drivename_hash->{"virtio$i"} = 1;
1050     $confdesc->{"virtio$i"} = $virtiodesc;
1051 }
1052
1053 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1054     $confdesc->{"usb$i"} = $usbdesc;
1055 }
1056
1057 my $unuseddesc = {
1058     optional => 1,
1059     type => 'string', format => 'pve-volume-id',
1060     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1061 };
1062
1063 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1064     $confdesc->{"unused$i"} = $unuseddesc;
1065 }
1066
1067 my $kvm_api_version = 0;
1068
1069 sub kvm_version {
1070
1071     return $kvm_api_version if $kvm_api_version;
1072
1073     my $fh = IO::File->new("</dev/kvm") ||
1074         return 0;
1075
1076     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
1077         $kvm_api_version = $v;
1078     }
1079
1080     $fh->close();
1081
1082     return  $kvm_api_version;
1083 }
1084
1085 my $kvm_user_version;
1086
1087 sub kvm_user_version {
1088
1089     return $kvm_user_version if $kvm_user_version;
1090
1091     $kvm_user_version = 'unknown';
1092
1093     my $code = sub {
1094         my $line = shift;
1095         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1096             $kvm_user_version = $2;
1097         }
1098     };
1099
1100     eval { run_command("kvm -version", outfunc => $code); };
1101     warn $@ if $@;
1102
1103     return $kvm_user_version;
1104
1105 }
1106
1107 my $kernel_has_vhost_net = -c '/dev/vhost-net';
1108
1109 sub valid_drive_names {
1110     # order is important - used to autoselect boot disk
1111     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1112             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1113             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1114             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))));
1115 }
1116
1117 sub is_valid_drivename {
1118     my $dev = shift;
1119
1120     return defined($drivename_hash->{$dev});
1121 }
1122
1123 sub option_exists {
1124     my $key = shift;
1125     return defined($confdesc->{$key});
1126 }
1127
1128 sub nic_models {
1129     return $nic_model_list;
1130 }
1131
1132 sub os_list_description {
1133
1134     return {
1135         other => 'Other',
1136         wxp => 'Windows XP',
1137         w2k => 'Windows 2000',
1138         w2k3 =>, 'Windows 2003',
1139         w2k8 => 'Windows 2008',
1140         wvista => 'Windows Vista',
1141         win7 => 'Windows 7',
1142         win8 => 'Windows 8/2012',
1143         l24 => 'Linux 2.4',
1144         l26 => 'Linux 2.6',
1145     };
1146 }
1147
1148 my $cdrom_path;
1149
1150 sub get_cdrom_path {
1151
1152     return  $cdrom_path if $cdrom_path;
1153
1154     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1155     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1156     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1157 }
1158
1159 sub get_iso_path {
1160     my ($storecfg, $vmid, $cdrom) = @_;
1161
1162     if ($cdrom eq 'cdrom') {
1163         return get_cdrom_path();
1164     } elsif ($cdrom eq 'none') {
1165         return '';
1166     } elsif ($cdrom =~ m|^/|) {
1167         return $cdrom;
1168     } else {
1169         return PVE::Storage::path($storecfg, $cdrom);
1170     }
1171 }
1172
1173 # try to convert old style file names to volume IDs
1174 sub filename_to_volume_id {
1175     my ($vmid, $file, $media) = @_;
1176
1177     if (!($file eq 'none' || $file eq 'cdrom' ||
1178           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1179
1180         return undef if $file =~ m|/|;
1181
1182         if ($media && $media eq 'cdrom') {
1183             $file = "local:iso/$file";
1184         } else {
1185             $file = "local:$vmid/$file";
1186         }
1187     }
1188
1189     return $file;
1190 }
1191
1192 sub verify_media_type {
1193     my ($opt, $vtype, $media) = @_;
1194
1195     return if !$media;
1196
1197     my $etype;
1198     if ($media eq 'disk') {
1199         $etype = 'images';
1200     } elsif ($media eq 'cdrom') {
1201         $etype = 'iso';
1202     } else {
1203         die "internal error";
1204     }
1205
1206     return if ($vtype eq $etype);
1207
1208     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1209 }
1210
1211 sub cleanup_drive_path {
1212     my ($opt, $storecfg, $drive) = @_;
1213
1214     # try to convert filesystem paths to volume IDs
1215
1216     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1217         ($drive->{file} !~ m|^/dev/.+|) &&
1218         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1219         ($drive->{file} !~ m/^\d+$/)) {
1220         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1221         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1222         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1223         verify_media_type($opt, $vtype, $drive->{media});
1224         $drive->{file} = $volid;
1225     }
1226
1227     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1228 }
1229
1230 sub parse_hotplug_features {
1231     my ($data) = @_;
1232
1233     my $res = {};
1234
1235     return $res if $data eq '0';
1236
1237     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1238
1239     foreach my $feature (PVE::Tools::split_list($data)) {
1240         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1241             $res->{$1} = 1;
1242         } else {
1243             die "invalid hotplug feature '$feature'\n";
1244         }
1245     }
1246     return $res;
1247 }
1248
1249 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1250 sub pve_verify_hotplug_features {
1251     my ($value, $noerr) = @_;
1252
1253     return $value if parse_hotplug_features($value);
1254
1255     return undef if $noerr;
1256
1257     die "unable to parse hotplug option\n";
1258 }
1259
1260 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1261 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1262 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1263 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1264 #        [,iothread=on][,serial=serial][,model=model]
1265
1266 sub parse_drive {
1267     my ($key, $data) = @_;
1268
1269     my ($interface, $index);
1270
1271     if ($key =~ m/^([^\d]+)(\d+)$/) {
1272         $interface = $1;
1273         $index = $2;
1274     } else {
1275         return undef;
1276     }
1277
1278     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1279                                      : $confdesc->{$key}->{format};
1280     if (!$desc) {
1281         warn "invalid drive key: $key\n";
1282         return undef;
1283     }
1284     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1285     return undef if !$res;
1286     $res->{interface} = $interface;
1287     $res->{index} = $index;
1288
1289     my $error = 0;
1290     foreach my $opt (qw(bps bps_rd bps_wr)) {
1291         if (my $bps = defined(delete $res->{$opt})) {
1292             if (defined($res->{"m$opt"})) {
1293                 warn "both $opt and m$opt specified\n";
1294                 ++$error;
1295                 next;
1296             }
1297             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1298         }
1299     }
1300     return undef if $error;
1301
1302     return undef if $res->{mbps_rd} && $res->{mbps};
1303     return undef if $res->{mbps_wr} && $res->{mbps};
1304     return undef if $res->{iops_rd} && $res->{iops};
1305     return undef if $res->{iops_wr} && $res->{iops};
1306
1307     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1308         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1309         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1310         return undef if $res->{interface} eq 'virtio';
1311     }
1312
1313     if (my $size = $res->{size}) {
1314         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1315     }
1316
1317     return $res;
1318 }
1319
1320 sub print_drive {
1321     my ($vmid, $drive) = @_;
1322     my $data = { %$drive };
1323     delete $data->{$_} for qw(index interface);
1324     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1325 }
1326
1327 sub scsi_inquiry {
1328     my($fh, $noerr) = @_;
1329
1330     my $SG_IO = 0x2285;
1331     my $SG_GET_VERSION_NUM = 0x2282;
1332
1333     my $versionbuf = "\x00" x 8;
1334     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1335     if (!$ret) {
1336         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1337         return undef;
1338     }
1339     my $version = unpack("I", $versionbuf);
1340     if ($version < 30000) {
1341         die "scsi generic interface too old\n"  if !$noerr;
1342         return undef;
1343     }
1344
1345     my $buf = "\x00" x 36;
1346     my $sensebuf = "\x00" x 8;
1347     my $cmd = pack("C x3 C x1", 0x12, 36);
1348
1349     # see /usr/include/scsi/sg.h
1350     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1351
1352     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1353                       length($sensebuf), 0, length($buf), $buf,
1354                       $cmd, $sensebuf, 6000);
1355
1356     $ret = ioctl($fh, $SG_IO, $packet);
1357     if (!$ret) {
1358         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1359         return undef;
1360     }
1361
1362     my @res = unpack($sg_io_hdr_t, $packet);
1363     if ($res[17] || $res[18]) {
1364         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1365         return undef;
1366     }
1367
1368     my $res = {};
1369     (my $byte0, my $byte1, $res->{vendor},
1370      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1371
1372     $res->{removable} = $byte1 & 128 ? 1 : 0;
1373     $res->{type} = $byte0 & 31;
1374
1375     return $res;
1376 }
1377
1378 sub path_is_scsi {
1379     my ($path) = @_;
1380
1381     my $fh = IO::File->new("+<$path") || return undef;
1382     my $res = scsi_inquiry($fh, 1);
1383     close($fh);
1384
1385     return $res;
1386 }
1387
1388 sub machine_type_is_q35 {
1389     my ($conf) = @_;
1390
1391     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1392 }
1393
1394 sub print_tabletdevice_full {
1395     my ($conf) = @_;
1396
1397     my $q35 = machine_type_is_q35($conf);
1398
1399     # we use uhci for old VMs because tablet driver was buggy in older qemu
1400     my $usbbus = $q35 ? "ehci" : "uhci";
1401
1402     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1403 }
1404
1405 sub print_drivedevice_full {
1406     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1407
1408     my $device = '';
1409     my $maxdev = 0;
1410
1411     if ($drive->{interface} eq 'virtio') {
1412         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1413         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1414         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1415     } elsif ($drive->{interface} eq 'scsi') {
1416
1417         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1418         my $unit = $drive->{index} % $maxdev;
1419         my $devicetype = 'hd';
1420         my $path = '';
1421         if (drive_is_cdrom($drive)) {
1422             $devicetype = 'cd';
1423         } else {
1424             if ($drive->{file} =~ m|^/|) {
1425                 $path = $drive->{file};
1426                 if (my $info = path_is_scsi($path)) {
1427                     if ($info->{type} == 0) {
1428                         $devicetype = 'block';
1429                     } elsif ($info->{type} == 1) { # tape
1430                         $devicetype = 'generic';
1431                     }
1432                 }
1433             } else {
1434                  $path = PVE::Storage::path($storecfg, $drive->{file});
1435             }
1436
1437             if($path =~ m/^iscsi\:\/\//){
1438                 $devicetype = 'generic';
1439             }
1440         }
1441
1442         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1443             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1444         } else {
1445             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1446         }
1447
1448     } elsif ($drive->{interface} eq 'ide'){
1449         $maxdev = 2;
1450         my $controller = int($drive->{index} / $maxdev);
1451         my $unit = $drive->{index} % $maxdev;
1452         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1453
1454         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1455         if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
1456             $model = URI::Escape::uri_unescape($model);
1457             $device .= ",model=$model";
1458         }
1459     } elsif ($drive->{interface} eq 'sata'){
1460         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1461         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1462         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1463     } elsif ($drive->{interface} eq 'usb') {
1464         die "implement me";
1465         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1466     } else {
1467         die "unsupported interface type";
1468     }
1469
1470     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1471
1472     return $device;
1473 }
1474
1475 sub get_initiator_name {
1476     my $initiator;
1477
1478     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1479     while (defined(my $line = <$fh>)) {
1480         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1481         $initiator = $1;
1482         last;
1483     }
1484     $fh->close();
1485
1486     return $initiator;
1487 }
1488
1489 sub print_drive_full {
1490     my ($storecfg, $vmid, $drive) = @_;
1491
1492     my $path;
1493     my $volid = $drive->{file};
1494     my $format;
1495
1496     if (drive_is_cdrom($drive)) {
1497         $path = get_iso_path($storecfg, $vmid, $volid);
1498     } else {
1499         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1500         if ($storeid) {
1501             $path = PVE::Storage::path($storecfg, $volid);
1502             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1503             $format = qemu_img_format($scfg, $volname);
1504         } else {
1505             $path = $volid;
1506             $format = "raw";
1507         }
1508    }
1509
1510     my $opts = '';
1511     my @qemu_drive_options = qw(heads secs cyls trans media format cache snapshot rerror werror aio discard iops iops_rd iops_wr iops_max iops_rd_max iops_wr_max);
1512     foreach my $o (@qemu_drive_options) {
1513         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1514     }
1515     if (my $serial = $drive->{serial}) {
1516         $serial = URI::Escape::uri_unescape($serial);
1517         $opts .= ",serial=$serial";
1518     }
1519
1520     $opts .= ",format=$format" if $format && !$drive->{format};
1521
1522     foreach my $o (qw(bps bps_rd bps_wr)) {
1523         my $v = $drive->{"m$o"};
1524         $opts .= ",$o=" . int($v*1024*1024) if $v;
1525     }
1526
1527     my $cache_direct = 0;
1528
1529     if (my $cache = $drive->{cache}) {
1530         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1531     } elsif (!drive_is_cdrom($drive)) {
1532         $opts .= ",cache=none";
1533         $cache_direct = 1;
1534     }
1535
1536     # aio native works only with O_DIRECT
1537     if (!$drive->{aio}) {
1538         if($cache_direct) {
1539             $opts .= ",aio=native";
1540         } else {
1541             $opts .= ",aio=threads";
1542         }
1543     }
1544
1545     if (!drive_is_cdrom($drive)) {
1546         my $detectzeroes;
1547         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1548             $detectzeroes = 'off';
1549         } elsif ($drive->{discard}) {
1550             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1551         } else {
1552             # This used to be our default with discard not being specified:
1553             $detectzeroes = 'on';
1554         }
1555         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1556     }
1557
1558     my $pathinfo = $path ? "file=$path," : '';
1559
1560     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1561 }
1562
1563 sub print_netdevice_full {
1564     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1565
1566     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1567
1568     my $device = $net->{model};
1569     if ($net->{model} eq 'virtio') {
1570          $device = 'virtio-net-pci';
1571      };
1572
1573     my $pciaddr = print_pci_addr("$netid", $bridges);
1574     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1575     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1576         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1577         my $vectors = $net->{queues} * 2 + 2;
1578         $tmpstr .= ",vectors=$vectors,mq=on";
1579     }
1580     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1581
1582     if ($use_old_bios_files) {
1583         my $romfile;
1584         if ($device eq 'virtio-net-pci') {
1585             $romfile = 'pxe-virtio.rom';
1586         } elsif ($device eq 'e1000') {
1587             $romfile = 'pxe-e1000.rom';
1588         } elsif ($device eq 'ne2k') {
1589             $romfile = 'pxe-ne2k_pci.rom';
1590         } elsif ($device eq 'pcnet') {
1591             $romfile = 'pxe-pcnet.rom';
1592         } elsif ($device eq 'rtl8139') {
1593             $romfile = 'pxe-rtl8139.rom';
1594         }
1595         $tmpstr .= ",romfile=$romfile" if $romfile;
1596     }
1597
1598     return $tmpstr;
1599 }
1600
1601 sub print_netdev_full {
1602     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1603
1604     my $i = '';
1605     if ($netid =~ m/^net(\d+)$/) {
1606         $i = int($1);
1607     }
1608
1609     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1610
1611     my $ifname = "tap${vmid}i$i";
1612
1613     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1614     die "interface name '$ifname' is too long (max 15 character)\n"
1615         if length($ifname) >= 16;
1616
1617     my $vhostparam = '';
1618     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1619
1620     my $vmname = $conf->{name} || "vm$vmid";
1621
1622     my $netdev = "";
1623     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1624
1625     if ($net->{bridge}) {
1626         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1627     } else {
1628         $netdev = "type=user,id=$netid,hostname=$vmname";
1629     }
1630
1631     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1632
1633     return $netdev;
1634 }
1635
1636 sub drive_is_cdrom {
1637     my ($drive) = @_;
1638
1639     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1640
1641 }
1642
1643 sub parse_number_sets {
1644     my ($set) = @_;
1645     my $res = [];
1646     foreach my $part (split(/;/, $set)) {
1647         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1648             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1649             push @$res, [ $1, $2 ];
1650         } else {
1651             die "invalid range: $part\n";
1652         }
1653     }
1654     return $res;
1655 }
1656
1657 sub parse_numa {
1658     my ($data) = @_;
1659
1660     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
1661     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1662     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1663     return $res;
1664 }
1665
1666 sub parse_hostpci {
1667     my ($value) = @_;
1668
1669     return undef if !$value;
1670
1671     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
1672
1673     my @idlist = split(/;/, $res->{host});
1674     delete $res->{host};
1675     foreach my $id (@idlist) {
1676         if ($id =~ /^$PCIRE$/) {
1677             push @{$res->{pciid}}, { id => $1, function => ($2//'0') };
1678         } else {
1679             # should have been caught by parse_property_string already
1680             die "failed to parse PCI id: $id\n";
1681         }
1682     }
1683     return $res;
1684 }
1685
1686 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1687 sub parse_net {
1688     my ($data) = @_;
1689
1690     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
1691     if ($@) {
1692         warn $@;
1693         return undef;
1694     }
1695     $res->{macaddr} = PVE::Tools::random_ether_addr() if !defined($res->{macaddr});
1696     return $res;
1697 }
1698
1699 sub print_net {
1700     my $net = shift;
1701
1702     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1703 }
1704
1705 sub add_random_macs {
1706     my ($settings) = @_;
1707
1708     foreach my $opt (keys %$settings) {
1709         next if $opt !~ m/^net(\d+)$/;
1710         my $net = parse_net($settings->{$opt});
1711         next if !$net;
1712         $settings->{$opt} = print_net($net);
1713     }
1714 }
1715
1716 sub vm_is_volid_owner {
1717     my ($storecfg, $vmid, $volid) = @_;
1718
1719     if ($volid !~  m|^/|) {
1720         my ($path, $owner);
1721         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1722         if ($owner && ($owner == $vmid)) {
1723             return 1;
1724         }
1725     }
1726
1727     return undef;
1728 }
1729
1730 sub split_flagged_list {
1731     my $text = shift || '';
1732     $text =~ s/[,;]/ /g;
1733     $text =~ s/^\s+//;
1734     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
1735 }
1736
1737 sub join_flagged_list {
1738     my ($how, $lst) = @_;
1739     join $how, map { $lst->{$_} . $_ } keys %$lst;
1740 }
1741
1742 sub vmconfig_delete_pending_option {
1743     my ($conf, $key, $force) = @_;
1744
1745     delete $conf->{pending}->{$key};
1746     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1747     $pending_delete_hash->{$key} = $force ? '!' : '';
1748     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1749 }
1750
1751 sub vmconfig_undelete_pending_option {
1752     my ($conf, $key) = @_;
1753
1754     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1755     delete $pending_delete_hash->{$key};
1756
1757     if (%$pending_delete_hash) {
1758         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1759     } else {
1760         delete $conf->{pending}->{delete};
1761     }
1762 }
1763
1764 sub vmconfig_register_unused_drive {
1765     my ($storecfg, $vmid, $conf, $drive) = @_;
1766
1767     if (!drive_is_cdrom($drive)) {
1768         my $volid = $drive->{file};
1769         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1770             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1771         }
1772     }
1773 }
1774
1775 sub vmconfig_cleanup_pending {
1776     my ($conf) = @_;
1777
1778     # remove pending changes when nothing changed
1779     my $changes;
1780     foreach my $opt (keys %{$conf->{pending}}) {
1781         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
1782             $changes = 1;
1783             delete $conf->{pending}->{$opt};
1784         }
1785     }
1786
1787     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
1788     my $pending_delete_hash = {};
1789     while (my ($opt, $force) = each %$current_delete_hash) {
1790         if (defined($conf->{$opt})) {
1791             $pending_delete_hash->{$opt} = $force;
1792         } else {
1793             $changes = 1;
1794         }
1795     }
1796
1797     if (%$pending_delete_hash) {
1798         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1799     } else {
1800         delete $conf->{pending}->{delete};
1801     }
1802
1803     return $changes;
1804 }
1805
1806 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
1807 my $smbios1_fmt = {
1808     uuid => {
1809         type => 'string',
1810         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1811         format_description => 'UUID',
1812         description => "Set SMBIOS1 UUID.",
1813         optional => 1,
1814     },
1815     version => {
1816         type => 'string',
1817         pattern => '\S+',
1818         format_description => 'string',
1819         description => "Set SMBIOS1 version.",
1820         optional => 1,
1821     },
1822     serial => {
1823         type => 'string',
1824         pattern => '\S+',
1825         format_description => 'string',
1826         description => "Set SMBIOS1 serial number.",
1827         optional => 1,
1828     },
1829     manufacturer => {
1830         type => 'string',
1831         pattern => '\S+',
1832         format_description => 'string',
1833         description => "Set SMBIOS1 manufacturer.",
1834         optional => 1,
1835     },
1836     product => {
1837         type => 'string',
1838         pattern => '\S+',
1839         format_description => 'string',
1840         description => "Set SMBIOS1 product ID.",
1841         optional => 1,
1842     },
1843     sku => {
1844         type => 'string',
1845         pattern => '\S+',
1846         format_description => 'string',
1847         description => "Set SMBIOS1 SKU string.",
1848         optional => 1,
1849     },
1850     family => {
1851         type => 'string',
1852         pattern => '\S+',
1853         format_description => 'string',
1854         description => "Set SMBIOS1 family string.",
1855         optional => 1,
1856     },
1857 };
1858
1859 sub parse_smbios1 {
1860     my ($data) = @_;
1861
1862     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
1863     warn $@ if $@;
1864     return $res;
1865 }
1866
1867 sub print_smbios1 {
1868     my ($smbios1) = @_;
1869     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
1870 }
1871
1872 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
1873
1874 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
1875 sub verify_bootdisk {
1876     my ($value, $noerr) = @_;
1877
1878     return $value if is_valid_drivename($value);
1879
1880     return undef if $noerr;
1881
1882     die "invalid boot disk '$value'\n";
1883 }
1884
1885 sub parse_watchdog {
1886     my ($value) = @_;
1887
1888     return undef if !$value;
1889
1890     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
1891     warn $@ if $@;
1892     return $res;
1893 }
1894
1895 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
1896 sub verify_usb_device {
1897     my ($value, $noerr) = @_;
1898
1899     return $value if parse_usb_device($value);
1900
1901     return undef if $noerr;
1902
1903     die "unable to parse usb device\n";
1904 }
1905
1906 # add JSON properties for create and set function
1907 sub json_config_properties {
1908     my $prop = shift;
1909
1910     foreach my $opt (keys %$confdesc) {
1911         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
1912         $prop->{$opt} = $confdesc->{$opt};
1913     }
1914
1915     return $prop;
1916 }
1917
1918 sub check_type {
1919     my ($key, $value) = @_;
1920
1921     die "unknown setting '$key'\n" if !$confdesc->{$key};
1922
1923     my $type = $confdesc->{$key}->{type};
1924
1925     if (!defined($value)) {
1926         die "got undefined value\n";
1927     }
1928
1929     if ($value =~ m/[\n\r]/) {
1930         die "property contains a line feed\n";
1931     }
1932
1933     if ($type eq 'boolean') {
1934         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
1935         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
1936         die "type check ('boolean') failed - got '$value'\n";
1937     } elsif ($type eq 'integer') {
1938         return int($1) if $value =~ m/^(\d+)$/;
1939         die "type check ('integer') failed - got '$value'\n";
1940     } elsif ($type eq 'number') {
1941         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
1942         die "type check ('number') failed - got '$value'\n";
1943     } elsif ($type eq 'string') {
1944         if (my $fmt = $confdesc->{$key}->{format}) {
1945             if ($fmt eq 'pve-qm-drive') {
1946                 # special case - we need to pass $key to parse_drive()
1947                 my $drive = parse_drive($key, $value);
1948                 return $value if $drive;
1949                 die "unable to parse drive options\n";
1950             }
1951             PVE::JSONSchema::check_format($fmt, $value);
1952             return $value;
1953         }
1954         $value =~ s/^\"(.*)\"$/$1/;
1955         return $value;
1956     } else {
1957         die "internal error"
1958     }
1959 }
1960
1961 sub check_iommu_support{
1962     #fixme : need to check IOMMU support
1963     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
1964
1965     my $iommu=1;
1966     return $iommu;
1967
1968 }
1969
1970 sub touch_config {
1971     my ($vmid) = @_;
1972
1973     my $conf = PVE::QemuConfig->config_file($vmid);
1974     utime undef, undef, $conf;
1975 }
1976
1977 sub destroy_vm {
1978     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
1979
1980     my $conffile = PVE::QemuConfig->config_file($vmid);
1981
1982     my $conf = PVE::QemuConfig->load_config($vmid);
1983
1984     PVE::QemuConfig->check_lock($conf) if !$skiplock;
1985
1986     # only remove disks owned by this VM
1987     foreach_drive($conf, sub {
1988         my ($ds, $drive) = @_;
1989
1990         return if drive_is_cdrom($drive);
1991
1992         my $volid = $drive->{file};
1993
1994         return if !$volid || $volid =~ m|^/|;
1995
1996         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
1997         return if !$path || !$owner || ($owner != $vmid);
1998
1999         PVE::Storage::vdisk_free($storecfg, $volid);
2000     });
2001
2002     if ($keep_empty_config) {
2003         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2004     } else {
2005         unlink $conffile;
2006     }
2007
2008     # also remove unused disk
2009     eval {
2010         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2011
2012         eval {
2013             PVE::Storage::foreach_volid($dl, sub {
2014                 my ($volid, $sid, $volname, $d) = @_;
2015                 PVE::Storage::vdisk_free($storecfg, $volid);
2016             });
2017         };
2018         warn $@ if $@;
2019
2020     };
2021     warn $@ if $@;
2022 }
2023
2024 sub parse_vm_config {
2025     my ($filename, $raw) = @_;
2026
2027     return undef if !defined($raw);
2028
2029     my $res = {
2030         digest => Digest::SHA::sha1_hex($raw),
2031         snapshots => {},
2032         pending => {},
2033     };
2034
2035     $filename =~ m|/qemu-server/(\d+)\.conf$|
2036         || die "got strange filename '$filename'";
2037
2038     my $vmid = $1;
2039
2040     my $conf = $res;
2041     my $descr;
2042     my $section = '';
2043
2044     my @lines = split(/\n/, $raw);
2045     foreach my $line (@lines) {
2046         next if $line =~ m/^\s*$/;
2047
2048         if ($line =~ m/^\[PENDING\]\s*$/i) {
2049             $section = 'pending';
2050             if (defined($descr)) {
2051                 $descr =~ s/\s+$//;
2052                 $conf->{description} = $descr;
2053             }
2054             $descr = undef;
2055             $conf = $res->{$section} = {};
2056             next;
2057
2058         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2059             $section = $1;
2060             if (defined($descr)) {
2061                 $descr =~ s/\s+$//;
2062                 $conf->{description} = $descr;
2063             }
2064             $descr = undef;
2065             $conf = $res->{snapshots}->{$section} = {};
2066             next;
2067         }
2068
2069         if ($line =~ m/^\#(.*)\s*$/) {
2070             $descr = '' if !defined($descr);
2071             $descr .= PVE::Tools::decode_text($1) . "\n";
2072             next;
2073         }
2074
2075         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2076             $descr = '' if !defined($descr);
2077             $descr .= PVE::Tools::decode_text($2);
2078         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2079             $conf->{snapstate} = $1;
2080         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2081             my $key = $1;
2082             my $value = $2;
2083             $conf->{$key} = $value;
2084         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2085             my $value = $1;
2086             if ($section eq 'pending') {
2087                 $conf->{delete} = $value; # we parse this later
2088             } else {
2089                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2090             }
2091         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
2092             my $key = $1;
2093             my $value = $2;
2094             eval { $value = check_type($key, $value); };
2095             if ($@) {
2096                 warn "vm $vmid - unable to parse value of '$key' - $@";
2097             } else {
2098                 my $fmt = $confdesc->{$key}->{format};
2099                 if ($fmt && $fmt eq 'pve-qm-drive') {
2100                     my $v = parse_drive($key, $value);
2101                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2102                         $v->{file} = $volid;
2103                         $value = print_drive($vmid, $v);
2104                     } else {
2105                         warn "vm $vmid - unable to parse value of '$key'\n";
2106                         next;
2107                     }
2108                 }
2109
2110                 if ($key eq 'cdrom') {
2111                     $conf->{ide2} = $value;
2112                 } else {
2113                     $conf->{$key} = $value;
2114                 }
2115             }
2116         }
2117     }
2118
2119     if (defined($descr)) {
2120         $descr =~ s/\s+$//;
2121         $conf->{description} = $descr;
2122     }
2123     delete $res->{snapstate}; # just to be sure
2124
2125     return $res;
2126 }
2127
2128 sub write_vm_config {
2129     my ($filename, $conf) = @_;
2130
2131     delete $conf->{snapstate}; # just to be sure
2132
2133     if ($conf->{cdrom}) {
2134         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2135         $conf->{ide2} = $conf->{cdrom};
2136         delete $conf->{cdrom};
2137     }
2138
2139     # we do not use 'smp' any longer
2140     if ($conf->{sockets}) {
2141         delete $conf->{smp};
2142     } elsif ($conf->{smp}) {
2143         $conf->{sockets} = $conf->{smp};
2144         delete $conf->{cores};
2145         delete $conf->{smp};
2146     }
2147
2148     my $used_volids = {};
2149
2150     my $cleanup_config = sub {
2151         my ($cref, $pending, $snapname) = @_;
2152
2153         foreach my $key (keys %$cref) {
2154             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2155                 $key eq 'snapstate' || $key eq 'pending';
2156             my $value = $cref->{$key};
2157             if ($key eq 'delete') {
2158                 die "propertry 'delete' is only allowed in [PENDING]\n"
2159                     if !$pending;
2160                 # fixme: check syntax?
2161                 next;
2162             }
2163             eval { $value = check_type($key, $value); };
2164             die "unable to parse value of '$key' - $@" if $@;
2165
2166             $cref->{$key} = $value;
2167
2168             if (!$snapname && is_valid_drivename($key)) {
2169                 my $drive = parse_drive($key, $value);
2170                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2171             }
2172         }
2173     };
2174
2175     &$cleanup_config($conf);
2176
2177     &$cleanup_config($conf->{pending}, 1);
2178
2179     foreach my $snapname (keys %{$conf->{snapshots}}) {
2180         die "internal error" if $snapname eq 'pending';
2181         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2182     }
2183
2184     # remove 'unusedX' settings if we re-add a volume
2185     foreach my $key (keys %$conf) {
2186         my $value = $conf->{$key};
2187         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2188             delete $conf->{$key};
2189         }
2190     }
2191
2192     my $generate_raw_config = sub {
2193         my ($conf, $pending) = @_;
2194
2195         my $raw = '';
2196
2197         # add description as comment to top of file
2198         if (defined(my $descr = $conf->{description})) {
2199             if ($descr) {
2200                 foreach my $cl (split(/\n/, $descr)) {
2201                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2202                 }
2203             } else {
2204                 $raw .= "#\n" if $pending;
2205             }
2206         }
2207
2208         foreach my $key (sort keys %$conf) {
2209             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2210             $raw .= "$key: $conf->{$key}\n";
2211         }
2212         return $raw;
2213     };
2214
2215     my $raw = &$generate_raw_config($conf);
2216
2217     if (scalar(keys %{$conf->{pending}})){
2218         $raw .= "\n[PENDING]\n";
2219         $raw .= &$generate_raw_config($conf->{pending}, 1);
2220     }
2221
2222     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2223         $raw .= "\n[$snapname]\n";
2224         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2225     }
2226
2227     return $raw;
2228 }
2229
2230 sub load_defaults {
2231
2232     my $res = {};
2233
2234     # we use static defaults from our JSON schema configuration
2235     foreach my $key (keys %$confdesc) {
2236         if (defined(my $default = $confdesc->{$key}->{default})) {
2237             $res->{$key} = $default;
2238         }
2239     }
2240
2241     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2242     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2243
2244     return $res;
2245 }
2246
2247 sub config_list {
2248     my $vmlist = PVE::Cluster::get_vmlist();
2249     my $res = {};
2250     return $res if !$vmlist || !$vmlist->{ids};
2251     my $ids = $vmlist->{ids};
2252
2253     foreach my $vmid (keys %$ids) {
2254         my $d = $ids->{$vmid};
2255         next if !$d->{node} || $d->{node} ne $nodename;
2256         next if !$d->{type} || $d->{type} ne 'qemu';
2257         $res->{$vmid}->{exists} = 1;
2258     }
2259     return $res;
2260 }
2261
2262 # test if VM uses local resources (to prevent migration)
2263 sub check_local_resources {
2264     my ($conf, $noerr) = @_;
2265
2266     my $loc_res = 0;
2267
2268     $loc_res = 1 if $conf->{hostusb}; # old syntax
2269     $loc_res = 1 if $conf->{hostpci}; # old syntax
2270
2271     foreach my $k (keys %$conf) {
2272         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2273         # sockets are safe: they will recreated be on the target side post-migrate
2274         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2275         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2276     }
2277
2278     die "VM uses local resources\n" if $loc_res && !$noerr;
2279
2280     return $loc_res;
2281 }
2282
2283 # check if used storages are available on all nodes (use by migrate)
2284 sub check_storage_availability {
2285     my ($storecfg, $conf, $node) = @_;
2286
2287     foreach_drive($conf, sub {
2288         my ($ds, $drive) = @_;
2289
2290         my $volid = $drive->{file};
2291         return if !$volid;
2292
2293         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2294         return if !$sid;
2295
2296         # check if storage is available on both nodes
2297         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2298         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2299    });
2300 }
2301
2302 # list nodes where all VM images are available (used by has_feature API)
2303 sub shared_nodes {
2304     my ($conf, $storecfg) = @_;
2305
2306     my $nodelist = PVE::Cluster::get_nodelist();
2307     my $nodehash = { map { $_ => 1 } @$nodelist };
2308     my $nodename = PVE::INotify::nodename();
2309
2310     foreach_drive($conf, sub {
2311         my ($ds, $drive) = @_;
2312
2313         my $volid = $drive->{file};
2314         return if !$volid;
2315
2316         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2317         if ($storeid) {
2318             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2319             if ($scfg->{disable}) {
2320                 $nodehash = {};
2321             } elsif (my $avail = $scfg->{nodes}) {
2322                 foreach my $node (keys %$nodehash) {
2323                     delete $nodehash->{$node} if !$avail->{$node};
2324                 }
2325             } elsif (!$scfg->{shared}) {
2326                 foreach my $node (keys %$nodehash) {
2327                     delete $nodehash->{$node} if $node ne $nodename
2328                 }
2329             }
2330         }
2331     });
2332
2333     return $nodehash
2334 }
2335
2336 sub check_cmdline {
2337     my ($pidfile, $pid) = @_;
2338
2339     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2340     if (defined($fh)) {
2341         my $line = <$fh>;
2342         $fh->close;
2343         return undef if !$line;
2344         my @param = split(/\0/, $line);
2345
2346         my $cmd = $param[0];
2347         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2348
2349         for (my $i = 0; $i < scalar (@param); $i++) {
2350             my $p = $param[$i];
2351             next if !$p;
2352             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2353                 my $p = $param[$i+1];
2354                 return 1 if $p && ($p eq $pidfile);
2355                 return undef;
2356             }
2357         }
2358     }
2359     return undef;
2360 }
2361
2362 sub check_running {
2363     my ($vmid, $nocheck, $node) = @_;
2364
2365     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2366
2367     die "unable to find configuration file for VM $vmid - no such machine\n"
2368         if !$nocheck && ! -f $filename;
2369
2370     my $pidfile = pidfile_name($vmid);
2371
2372     if (my $fd = IO::File->new("<$pidfile")) {
2373         my $st = stat($fd);
2374         my $line = <$fd>;
2375         close($fd);
2376
2377         my $mtime = $st->mtime;
2378         if ($mtime > time()) {
2379             warn "file '$filename' modified in future\n";
2380         }
2381
2382         if ($line =~ m/^(\d+)$/) {
2383             my $pid = $1;
2384             if (check_cmdline($pidfile, $pid)) {
2385                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2386                     return $pid;
2387                 }
2388             }
2389         }
2390     }
2391
2392     return undef;
2393 }
2394
2395 sub vzlist {
2396
2397     my $vzlist = config_list();
2398
2399     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2400
2401     while (defined(my $de = $fd->read)) {
2402         next if $de !~ m/^(\d+)\.pid$/;
2403         my $vmid = $1;
2404         next if !defined($vzlist->{$vmid});
2405         if (my $pid = check_running($vmid)) {
2406             $vzlist->{$vmid}->{pid} = $pid;
2407         }
2408     }
2409
2410     return $vzlist;
2411 }
2412
2413 sub disksize {
2414     my ($storecfg, $conf) = @_;
2415
2416     my $bootdisk = $conf->{bootdisk};
2417     return undef if !$bootdisk;
2418     return undef if !is_valid_drivename($bootdisk);
2419
2420     return undef if !$conf->{$bootdisk};
2421
2422     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2423     return undef if !defined($drive);
2424
2425     return undef if drive_is_cdrom($drive);
2426
2427     my $volid = $drive->{file};
2428     return undef if !$volid;
2429
2430     return $drive->{size};
2431 }
2432
2433 my $last_proc_pid_stat;
2434
2435 # get VM status information
2436 # This must be fast and should not block ($full == false)
2437 # We only query KVM using QMP if $full == true (this can be slow)
2438 sub vmstatus {
2439     my ($opt_vmid, $full) = @_;
2440
2441     my $res = {};
2442
2443     my $storecfg = PVE::Storage::config();
2444
2445     my $list = vzlist();
2446     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2447
2448     my $cpucount = $cpuinfo->{cpus} || 1;
2449
2450     foreach my $vmid (keys %$list) {
2451         next if $opt_vmid && ($vmid ne $opt_vmid);
2452
2453         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
2454         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2455
2456         my $d = {};
2457         $d->{pid} = $list->{$vmid}->{pid};
2458
2459         # fixme: better status?
2460         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2461
2462         my $size = disksize($storecfg, $conf);
2463         if (defined($size)) {
2464             $d->{disk} = 0; # no info available
2465             $d->{maxdisk} = $size;
2466         } else {
2467             $d->{disk} = 0;
2468             $d->{maxdisk} = 0;
2469         }
2470
2471         $d->{cpus} = ($conf->{sockets} || 1) * ($conf->{cores} || 1);
2472         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2473         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2474
2475         $d->{name} = $conf->{name} || "VM $vmid";
2476         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024) : 0;
2477
2478         if ($conf->{balloon}) {
2479             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2480             $d->{shares} = defined($conf->{shares}) ? $conf->{shares} : 1000;
2481         }
2482
2483         $d->{uptime} = 0;
2484         $d->{cpu} = 0;
2485         $d->{mem} = 0;
2486
2487         $d->{netout} = 0;
2488         $d->{netin} = 0;
2489
2490         $d->{diskread} = 0;
2491         $d->{diskwrite} = 0;
2492
2493         $d->{template} = PVE::QemuConfig->is_template($conf);
2494
2495         $res->{$vmid} = $d;
2496     }
2497
2498     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2499     foreach my $dev (keys %$netdev) {
2500         next if $dev !~ m/^tap([1-9]\d*)i/;
2501         my $vmid = $1;
2502         my $d = $res->{$vmid};
2503         next if !$d;
2504
2505         $d->{netout} += $netdev->{$dev}->{receive};
2506         $d->{netin} += $netdev->{$dev}->{transmit};
2507
2508         if ($full) {
2509             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2510             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2511         }
2512
2513     }
2514
2515     my $ctime = gettimeofday;
2516
2517     foreach my $vmid (keys %$list) {
2518
2519         my $d = $res->{$vmid};
2520         my $pid = $d->{pid};
2521         next if !$pid;
2522
2523         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2524         next if !$pstat; # not running
2525
2526         my $used = $pstat->{utime} + $pstat->{stime};
2527
2528         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2529
2530         if ($pstat->{vsize}) {
2531             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2532         }
2533
2534         my $old = $last_proc_pid_stat->{$pid};
2535         if (!$old) {
2536             $last_proc_pid_stat->{$pid} = {
2537                 time => $ctime,
2538                 used => $used,
2539                 cpu => 0,
2540             };
2541             next;
2542         }
2543
2544         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2545
2546         if ($dtime > 1000) {
2547             my $dutime = $used -  $old->{used};
2548
2549             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2550             $last_proc_pid_stat->{$pid} = {
2551                 time => $ctime,
2552                 used => $used,
2553                 cpu => $d->{cpu},
2554             };
2555         } else {
2556             $d->{cpu} = $old->{cpu};
2557         }
2558     }
2559
2560     return $res if !$full;
2561
2562     my $qmpclient = PVE::QMPClient->new();
2563
2564     my $ballooncb = sub {
2565         my ($vmid, $resp) = @_;
2566
2567         my $info = $resp->{'return'};
2568         return if !$info->{max_mem};
2569
2570         my $d = $res->{$vmid};
2571
2572         # use memory assigned to VM
2573         $d->{maxmem} = $info->{max_mem};
2574         $d->{balloon} = $info->{actual};
2575
2576         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2577             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2578             $d->{freemem} = $info->{free_mem};
2579         }
2580
2581         $d->{ballooninfo} = $info;
2582     };
2583
2584     my $blockstatscb = sub {
2585         my ($vmid, $resp) = @_;
2586         my $data = $resp->{'return'} || [];
2587         my $totalrdbytes = 0;
2588         my $totalwrbytes = 0;
2589
2590         for my $blockstat (@$data) {
2591             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2592             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2593
2594             $blockstat->{device} =~ s/drive-//;
2595             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2596         }
2597         $res->{$vmid}->{diskread} = $totalrdbytes;
2598         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2599     };
2600
2601     my $statuscb = sub {
2602         my ($vmid, $resp) = @_;
2603
2604         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2605         # this fails if ballon driver is not loaded, so this must be
2606         # the last commnand (following command are aborted if this fails).
2607         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2608
2609         my $status = 'unknown';
2610         if (!defined($status = $resp->{'return'}->{status})) {
2611             warn "unable to get VM status\n";
2612             return;
2613         }
2614
2615         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2616     };
2617
2618     foreach my $vmid (keys %$list) {
2619         next if $opt_vmid && ($vmid ne $opt_vmid);
2620         next if !$res->{$vmid}->{pid}; # not running
2621         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2622     }
2623
2624     $qmpclient->queue_execute(undef, 1);
2625
2626     foreach my $vmid (keys %$list) {
2627         next if $opt_vmid && ($vmid ne $opt_vmid);
2628         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2629     }
2630
2631     return $res;
2632 }
2633
2634 sub foreach_drive {
2635     my ($conf, $func, @param) = @_;
2636
2637     foreach my $ds (valid_drive_names()) {
2638         next if !defined($conf->{$ds});
2639
2640         my $drive = parse_drive($ds, $conf->{$ds});
2641         next if !$drive;
2642
2643         &$func($ds, $drive, @param);
2644     }
2645 }
2646
2647 sub foreach_volid {
2648     my ($conf, $func, @param) = @_;
2649
2650     my $volhash = {};
2651
2652     my $test_volid = sub {
2653         my ($volid, $is_cdrom) = @_;
2654
2655         return if !$volid;
2656
2657         $volhash->{$volid} = $is_cdrom || 0;
2658     };
2659
2660     foreach_drive($conf, sub {
2661         my ($ds, $drive) = @_;
2662         &$test_volid($drive->{file}, drive_is_cdrom($drive));
2663     });
2664
2665     foreach my $snapname (keys %{$conf->{snapshots}}) {
2666         my $snap = $conf->{snapshots}->{$snapname};
2667         &$test_volid($snap->{vmstate}, 0);
2668         foreach_drive($snap, sub {
2669             my ($ds, $drive) = @_;
2670             &$test_volid($drive->{file}, drive_is_cdrom($drive));
2671         });
2672     }
2673
2674     foreach my $volid (keys %$volhash) {
2675         &$func($volid, $volhash->{$volid}, @param);
2676     }
2677 }
2678
2679 sub vga_conf_has_spice {
2680     my ($vga) = @_;
2681
2682     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
2683
2684     return $1 || 1;
2685 }
2686
2687 sub config_to_command {
2688     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2689
2690     my $cmd = [];
2691     my $globalFlags = [];
2692     my $machineFlags = [];
2693     my $rtcFlags = [];
2694     my $cpuFlags = [];
2695     my $devices = [];
2696     my $pciaddr = '';
2697     my $bridges = {};
2698     my $kvmver = kvm_user_version();
2699     my $vernum = 0; # unknown
2700     my $ostype = $conf->{ostype};
2701     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
2702         $vernum = $1*1000000+$2*1000;
2703     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
2704         $vernum = $1*1000000+$2*1000+$3;
2705     }
2706
2707     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
2708
2709     my $have_ovz = -f '/proc/vz/vestat';
2710
2711     my $q35 = machine_type_is_q35($conf);
2712     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2713     my $machine_type = $forcemachine || $conf->{machine};
2714     my $use_old_bios_files = undef;
2715     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
2716
2717     my $cpuunits = defined($conf->{cpuunits}) ?
2718             $conf->{cpuunits} : $defaults->{cpuunits};
2719
2720     push @$cmd, '/usr/bin/kvm';
2721
2722     push @$cmd, '-id', $vmid;
2723
2724     my $use_virtio = 0;
2725
2726     my $qmpsocket = qmp_socket($vmid);
2727     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2728     push @$cmd, '-mon', "chardev=qmp,mode=control";
2729
2730
2731     push @$cmd, '-pidfile' , pidfile_name($vmid);
2732
2733     push @$cmd, '-daemonize';
2734
2735     if ($conf->{smbios1}) {
2736         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
2737     }
2738
2739     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2740         my $ovmfvar = "OVMF_VARS-pure-efi.fd";
2741         my $ovmfvar_src = "/usr/share/kvm/$ovmfvar";
2742         my $ovmfvar_dst = "/tmp/$vmid-$ovmfvar";
2743         PVE::Tools::file_copy($ovmfvar_src, $ovmfvar_dst, 256*1024);
2744         push @$cmd, '-drive', "if=pflash,format=raw,readonly,file=/usr/share/kvm/OVMF-pure-efi.fd";
2745         push @$cmd, '-drive', "if=pflash,format=raw,file=$ovmfvar_dst";
2746     }
2747
2748
2749     # add usb controllers
2750     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $q35, $usbdesc->{format}, $MAX_USB_DEVICES);
2751     push @$devices, @usbcontrollers if @usbcontrollers;
2752     my $vga = $conf->{vga};
2753
2754     my $qxlnum = vga_conf_has_spice($vga);
2755     $vga = 'qxl' if $qxlnum;
2756
2757     if (!$vga) {
2758         if ($conf->{ostype} && ($conf->{ostype} eq 'win8' ||
2759                                 $conf->{ostype} eq 'win7' ||
2760                                 $conf->{ostype} eq 'w2k8')) {
2761             $vga = 'std';
2762         } else {
2763             $vga = 'cirrus';
2764         }
2765     }
2766
2767     # enable absolute mouse coordinates (needed by vnc)
2768     my $tablet;
2769     if (defined($conf->{tablet})) {
2770         $tablet = $conf->{tablet};
2771     } else {
2772         $tablet = $defaults->{tablet};
2773         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
2774         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
2775     }
2776
2777     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
2778
2779     my $kvm_off = 0;
2780     # host pci devices
2781     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
2782         my $d = parse_hostpci($conf->{"hostpci$i"});
2783         next if !$d;
2784
2785         my $pcie = $d->{pcie};
2786         if($pcie){
2787             die "q35 machine model is not enabled" if !$q35;
2788             $pciaddr = print_pcie_addr("hostpci$i");
2789         }else{
2790             $pciaddr = print_pci_addr("hostpci$i", $bridges);
2791         }
2792
2793         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
2794         my $xvga = '';
2795         if ($d->{'x-vga'}) {
2796             $xvga = ',x-vga=on';
2797             $kvm_off = 1;
2798             $vga = 'none';
2799             if ($ostype eq 'win7' || $ostype eq 'win8' || $ostype eq 'w2k8') {
2800                 push @$cpuFlags , 'hv_vendor_id=proxmox';
2801             }
2802             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2803                 $xvga = "";
2804             }
2805         }
2806         my $pcidevices = $d->{pciid};
2807         my $multifunction = 1 if @$pcidevices > 1;
2808
2809         my $j=0;
2810         foreach my $pcidevice (@$pcidevices) {
2811
2812             my $id = "hostpci$i";
2813             $id .= ".$j" if $multifunction;
2814             my $addr = $pciaddr;
2815             $addr .= ".$j" if $multifunction;
2816             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
2817
2818             if($j == 0){
2819                 $devicestr .= "$rombar$xvga";
2820                 $devicestr .= ",multifunction=on" if $multifunction;
2821             }
2822
2823             push @$devices, '-device', $devicestr;
2824             $j++;
2825         }
2826     }
2827
2828     # usb devices
2829     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES);
2830     push @$devices, @usbdevices if @usbdevices;
2831     # serial devices
2832     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2833         if (my $path = $conf->{"serial$i"}) {
2834             if ($path eq 'socket') {
2835                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
2836                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
2837                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2838             } else {
2839                 die "no such serial device\n" if ! -c $path;
2840                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
2841                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2842             }
2843         }
2844     }
2845
2846     # parallel devices
2847     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
2848         if (my $path = $conf->{"parallel$i"}) {
2849             die "no such parallel device\n" if ! -c $path;
2850             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
2851             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
2852             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
2853         }
2854     }
2855
2856     my $vmname = $conf->{name} || "vm$vmid";
2857
2858     push @$cmd, '-name', $vmname;
2859
2860     my $sockets = 1;
2861     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
2862     $sockets = $conf->{sockets} if  $conf->{sockets};
2863
2864     my $cores = $conf->{cores} || 1;
2865
2866     my $maxcpus = $sockets * $cores;
2867
2868     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
2869
2870     my $allowed_vcpus = $cpuinfo->{cpus};
2871
2872     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
2873         if ($allowed_vcpus < $maxcpus);
2874
2875     push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
2876
2877     push @$cmd, '-nodefaults';
2878
2879     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2880
2881     my $bootindex_hash = {};
2882     my $i = 1;
2883     foreach my $o (split(//, $bootorder)) {
2884         $bootindex_hash->{$o} = $i*100;
2885         $i++;
2886     }
2887
2888     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000";
2889
2890     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
2891
2892     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
2893
2894     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
2895
2896     if ($vga && $vga !~ m/^serial\d+$/ && $vga ne 'none'){
2897         my $socket = vnc_socket($vmid);
2898         push @$cmd,  '-vnc', "unix:$socket,x509,password";
2899     } else {
2900         push @$cmd, '-nographic';
2901     }
2902
2903     # time drift fix
2904     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
2905
2906     my $nokvm = defined($conf->{kvm}) && $conf->{kvm} == 0 ? 1 : 0;
2907     my $useLocaltime = $conf->{localtime};
2908
2909     if ($ostype) {
2910         # other, wxp, w2k, w2k3, w2k8, wvista, win7, win8, l24, l26, solaris
2911
2912         if ($ostype =~ m/^w/) { # windows
2913             $useLocaltime = 1 if !defined($conf->{localtime});
2914
2915             # use time drift fix when acpi is enabled
2916             if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
2917                 $tdf = 1 if !defined($conf->{tdf});
2918             }
2919         }
2920
2921         if ($ostype eq 'win7' || $ostype eq 'win8' || $ostype eq 'w2k8' ||
2922             $ostype eq 'wvista') {
2923             push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
2924             push @$cmd, '-no-hpet';
2925             if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
2926                 push @$cpuFlags , 'hv_spinlocks=0x1fff' if !$nokvm;
2927                 push @$cpuFlags , 'hv_vapic' if !$nokvm;
2928                 push @$cpuFlags , 'hv_time' if !$nokvm;
2929
2930                 if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
2931                     push @$cpuFlags , 'hv_reset' if !$nokvm;
2932                     push @$cpuFlags , 'hv_vpindex' if !$nokvm;
2933                     push @$cpuFlags , 'hv_runtime' if !$nokvm;
2934                 }
2935
2936             } else {
2937                 push @$cpuFlags , 'hv_spinlocks=0xffff' if !$nokvm;
2938             }
2939         }
2940
2941         if ($ostype eq 'win7' || $ostype eq 'win8') {
2942             push @$cpuFlags , 'hv_relaxed' if !$nokvm;
2943         }
2944     }
2945
2946     push @$rtcFlags, 'driftfix=slew' if $tdf;
2947
2948     if ($nokvm) {
2949         push @$machineFlags, 'accel=tcg';
2950     } else {
2951         die "No accelerator found!\n" if !$cpuinfo->{hvm};
2952     }
2953
2954     if ($machine_type) {
2955         push @$machineFlags, "type=${machine_type}";
2956     }
2957
2958     if ($conf->{startdate}) {
2959         push @$rtcFlags, "base=$conf->{startdate}";
2960     } elsif ($useLocaltime) {
2961         push @$rtcFlags, 'base=localtime';
2962     }
2963
2964     my $cpu = $nokvm ? "qemu64" : "kvm64";
2965     if (my $cputype = $conf->{cpu}) {
2966         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
2967             or die "Cannot parse cpu description: $cputype\n";
2968         $cpu = $cpuconf->{cputype};
2969         $kvm_off = 1 if $cpuconf->{hidden};
2970     }
2971
2972     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
2973
2974     push @$cpuFlags , '-x2apic'
2975         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
2976
2977     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
2978
2979     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
2980
2981     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
2982
2983         push @$cpuFlags , '+kvm_pv_unhalt' if !$nokvm;
2984         push @$cpuFlags , '+kvm_pv_eoi' if !$nokvm;
2985     }
2986
2987     push @$cpuFlags, 'enforce' if $cpu ne 'host' && !$nokvm;
2988
2989     push @$cpuFlags, 'kvm=off' if $kvm_off;
2990
2991     my $cpu_vendor = $cpu_vendor_list->{$cpu} ||
2992         die "internal error"; # should not happen
2993
2994     push @$cpuFlags, "vendor=${cpu_vendor}"
2995         if $cpu_vendor ne 'default';
2996
2997     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
2998
2999     push @$cmd, '-cpu', $cpu;
3000
3001     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3002
3003     push @$cmd, '-S' if $conf->{freeze};
3004
3005     # set keyboard layout
3006     my $kb = $conf->{keyboard} || $defaults->{keyboard};
3007     push @$cmd, '-k', $kb if $kb;
3008
3009     # enable sound
3010     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3011     #push @$cmd, '-soundhw', 'es1370';
3012     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3013
3014     if($conf->{agent}) {
3015         my $qgasocket = qmp_socket($vmid, 1);
3016         my $pciaddr = print_pci_addr("qga0", $bridges);
3017         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3018         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3019         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3020     }
3021
3022     my $spice_port;
3023
3024     if ($qxlnum) {
3025         if ($qxlnum > 1) {
3026             if ($conf->{ostype} && $conf->{ostype} =~ m/^w/){
3027                 for(my $i = 1; $i < $qxlnum; $i++){
3028                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3029                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3030                 }
3031             } else {
3032                 # assume other OS works like Linux
3033                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3034                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3035             }
3036         }
3037
3038         my $pciaddr = print_pci_addr("spice", $bridges);
3039
3040         my $nodename = PVE::INotify::nodename();
3041         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3042         $spice_port = PVE::Tools::next_spice_port($pfamily);
3043
3044         push @$devices, '-spice', "tls-port=${spice_port},addr=localhost,tls-ciphers=DES-CBC3-SHA,seamless-migration=on";
3045
3046         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3047         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3048         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3049     }
3050
3051     # enable balloon by default, unless explicitly disabled
3052     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3053         $pciaddr = print_pci_addr("balloon0", $bridges);
3054         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3055     }
3056
3057     if ($conf->{watchdog}) {
3058         my $wdopts = parse_watchdog($conf->{watchdog});
3059         $pciaddr = print_pci_addr("watchdog", $bridges);
3060         my $watchdog = $wdopts->{model} || 'i6300esb';
3061         push @$devices, '-device', "$watchdog$pciaddr";
3062         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3063     }
3064
3065     my $vollist = [];
3066     my $scsicontroller = {};
3067     my $ahcicontroller = {};
3068     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3069
3070     # Add iscsi initiator name if available
3071     if (my $initiator = get_initiator_name()) {
3072         push @$devices, '-iscsi', "initiator-name=$initiator";
3073     }
3074
3075     foreach_drive($conf, sub {
3076         my ($ds, $drive) = @_;
3077
3078         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3079             push @$vollist, $drive->{file};
3080         }
3081
3082         $use_virtio = 1 if $ds =~ m/^virtio/;
3083
3084         if (drive_is_cdrom ($drive)) {
3085             if ($bootindex_hash->{d}) {
3086                 $drive->{bootindex} = $bootindex_hash->{d};
3087                 $bootindex_hash->{d} += 1;
3088             }
3089         } else {
3090             if ($bootindex_hash->{c}) {
3091                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3092                 $bootindex_hash->{c} += 1;
3093             }
3094         }
3095
3096         if($drive->{interface} eq 'virtio'){
3097            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3098         }
3099
3100         if ($drive->{interface} eq 'scsi') {
3101
3102             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3103
3104             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3105             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3106
3107             my $iothread = '';
3108             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3109                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3110                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3111             } elsif ($drive->{iothread}) {
3112                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3113             }
3114
3115             my $queues = '';
3116             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3117                 $queues = ",num_queues=$drive->{queues}";
3118             }
3119
3120             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3121             $scsicontroller->{$controller}=1;
3122         }
3123
3124         if ($drive->{interface} eq 'sata') {
3125            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3126            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3127            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3128            $ahcicontroller->{$controller}=1;
3129         }
3130
3131         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3132         push @$devices, '-drive',$drive_cmd;
3133         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3134     });
3135
3136     for (my $i = 0; $i < $MAX_NETS; $i++) {
3137          next if !$conf->{"net$i"};
3138          my $d = parse_net($conf->{"net$i"});
3139          next if !$d;
3140
3141          $use_virtio = 1 if $d->{model} eq 'virtio';
3142
3143          if ($bootindex_hash->{n}) {
3144             $d->{bootindex} = $bootindex_hash->{n};
3145             $bootindex_hash->{n} += 1;
3146          }
3147
3148          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3149          push @$devices, '-netdev', $netdevfull;
3150
3151          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3152          push @$devices, '-device', $netdevicefull;
3153     }
3154
3155     if (!$q35) {
3156         # add pci bridges
3157         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3158            $bridges->{1} = 1;
3159            $bridges->{2} = 1;
3160         }
3161
3162         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3163
3164         while (my ($k, $v) = each %$bridges) {
3165             $pciaddr = print_pci_addr("pci.$k");
3166             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3167         }
3168     }
3169
3170     # add custom args
3171     if ($conf->{args}) {
3172         my $aa = PVE::Tools::split_args($conf->{args});
3173         push @$cmd, @$aa;
3174     }
3175
3176     push @$cmd, @$devices;
3177     push @$cmd, '-rtc', join(',', @$rtcFlags)
3178         if scalar(@$rtcFlags);
3179     push @$cmd, '-machine', join(',', @$machineFlags)
3180         if scalar(@$machineFlags);
3181     push @$cmd, '-global', join(',', @$globalFlags)
3182         if scalar(@$globalFlags);
3183
3184     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3185 }
3186
3187 sub vnc_socket {
3188     my ($vmid) = @_;
3189     return "${var_run_tmpdir}/$vmid.vnc";
3190 }
3191
3192 sub spice_port {
3193     my ($vmid) = @_;
3194
3195     my $res = vm_mon_cmd($vmid, 'query-spice');
3196
3197     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3198 }
3199
3200 sub qmp_socket {
3201     my ($vmid, $qga) = @_;
3202     my $sockettype = $qga ? 'qga' : 'qmp';
3203     return "${var_run_tmpdir}/$vmid.$sockettype";
3204 }
3205
3206 sub pidfile_name {
3207     my ($vmid) = @_;
3208     return "${var_run_tmpdir}/$vmid.pid";
3209 }
3210
3211 sub vm_devices_list {
3212     my ($vmid) = @_;
3213
3214     my $res = vm_mon_cmd($vmid, 'query-pci');
3215     my $devices = {};
3216     foreach my $pcibus (@$res) {
3217         foreach my $device (@{$pcibus->{devices}}) {
3218             next if !$device->{'qdev_id'};
3219             if ($device->{'pci_bridge'}) {
3220                 $devices->{$device->{'qdev_id'}} = 1;
3221                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3222                     next if !$bridge_device->{'qdev_id'};
3223                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3224                     $devices->{$device->{'qdev_id'}}++;
3225                 }
3226             } else {
3227                 $devices->{$device->{'qdev_id'}} = 1;
3228             }
3229         }
3230     }
3231
3232     my $resblock = vm_mon_cmd($vmid, 'query-block');
3233     foreach my $block (@$resblock) {
3234         if($block->{device} =~ m/^drive-(\S+)/){
3235                 $devices->{$1} = 1;
3236         }
3237     }
3238
3239     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3240     foreach my $mice (@$resmice) {
3241         if ($mice->{name} eq 'QEMU HID Tablet') {
3242             $devices->{tablet} = 1;
3243             last;
3244         }
3245     }
3246
3247     # for usb devices there is no query-usb
3248     # but we can iterate over the entries in
3249     # qom-list path=/machine/peripheral
3250     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3251     foreach my $per (@$resperipheral) {
3252         if ($per->{name} =~ m/^usb\d+$/) {
3253             $devices->{$per->{name}} = 1;
3254         }
3255     }
3256
3257     return $devices;
3258 }
3259
3260 sub vm_deviceplug {
3261     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3262
3263     my $q35 = machine_type_is_q35($conf);
3264
3265     my $devices_list = vm_devices_list($vmid);
3266     return 1 if defined($devices_list->{$deviceid});
3267
3268     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3269
3270     if ($deviceid eq 'tablet') {
3271
3272         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3273
3274     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3275
3276         qemu_iothread_add($vmid, $deviceid, $device);
3277
3278         qemu_driveadd($storecfg, $vmid, $device);
3279         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3280
3281         qemu_deviceadd($vmid, $devicefull);
3282         eval { qemu_deviceaddverify($vmid, $deviceid); };
3283         if (my $err = $@) {
3284             eval { qemu_drivedel($vmid, $deviceid); };
3285             warn $@ if $@;
3286             die $err;
3287         }
3288
3289     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3290
3291
3292         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3293         my $pciaddr = print_pci_addr($deviceid);
3294         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3295
3296         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3297
3298         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3299             qemu_iothread_add($vmid, $deviceid, $device);
3300             $devicefull .= ",iothread=iothread-$deviceid";
3301         }
3302
3303         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3304             $devicefull .= ",num_queues=$device->{queues}";
3305         }
3306
3307         qemu_deviceadd($vmid, $devicefull);
3308         qemu_deviceaddverify($vmid, $deviceid);
3309
3310     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3311
3312         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3313         qemu_driveadd($storecfg, $vmid, $device);
3314
3315         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3316         eval { qemu_deviceadd($vmid, $devicefull); };
3317         if (my $err = $@) {
3318             eval { qemu_drivedel($vmid, $deviceid); };
3319             warn $@ if $@;
3320             die $err;
3321         }
3322
3323     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3324
3325         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3326
3327         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
3328         my $use_old_bios_files = undef;
3329         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3330
3331         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3332         qemu_deviceadd($vmid, $netdevicefull);
3333         eval { qemu_deviceaddverify($vmid, $deviceid); };
3334         if (my $err = $@) {
3335             eval { qemu_netdevdel($vmid, $deviceid); };
3336             warn $@ if $@;
3337             die $err;
3338         }
3339
3340     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3341
3342         my $bridgeid = $2;
3343         my $pciaddr = print_pci_addr($deviceid);
3344         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3345
3346         qemu_deviceadd($vmid, $devicefull);
3347         qemu_deviceaddverify($vmid, $deviceid);
3348
3349     } else {
3350         die "can't hotplug device '$deviceid'\n";
3351     }
3352
3353     return 1;
3354 }
3355
3356 # fixme: this should raise exceptions on error!
3357 sub vm_deviceunplug {
3358     my ($vmid, $conf, $deviceid) = @_;
3359
3360     my $devices_list = vm_devices_list($vmid);
3361     return 1 if !defined($devices_list->{$deviceid});
3362
3363     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3364
3365     if ($deviceid eq 'tablet') {
3366
3367         qemu_devicedel($vmid, $deviceid);
3368
3369     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3370
3371         qemu_devicedel($vmid, $deviceid);
3372         qemu_devicedelverify($vmid, $deviceid);
3373         qemu_drivedel($vmid, $deviceid);
3374         qemu_iothread_del($conf, $vmid, $deviceid);
3375
3376     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3377
3378         qemu_devicedel($vmid, $deviceid);
3379         qemu_devicedelverify($vmid, $deviceid);
3380         qemu_iothread_del($conf, $vmid, $deviceid);
3381
3382     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3383
3384         #qemu 2.3 segfault on drive_del with virtioscsi + iothread
3385         my $device = parse_drive($deviceid, $conf->{$deviceid});
3386         die "virtioscsi with iothread is not hot-unplugglable currently" if $device->{iothread};
3387
3388         qemu_devicedel($vmid, $deviceid);
3389         qemu_drivedel($vmid, $deviceid);
3390         qemu_deletescsihw($conf, $vmid, $deviceid);
3391
3392     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3393
3394         qemu_devicedel($vmid, $deviceid);
3395         qemu_devicedelverify($vmid, $deviceid);
3396         qemu_netdevdel($vmid, $deviceid);
3397
3398     } else {
3399         die "can't unplug device '$deviceid'\n";
3400     }
3401
3402     return 1;
3403 }
3404
3405 sub qemu_deviceadd {
3406     my ($vmid, $devicefull) = @_;
3407
3408     $devicefull = "driver=".$devicefull;
3409     my %options =  split(/[=,]/, $devicefull);
3410
3411     vm_mon_cmd($vmid, "device_add" , %options);
3412 }
3413
3414 sub qemu_devicedel {
3415     my ($vmid, $deviceid) = @_;
3416
3417     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3418 }
3419
3420 sub qemu_iothread_add {
3421     my($vmid, $deviceid, $device) = @_;
3422
3423     if ($device->{iothread}) {
3424         my $iothreads = vm_iothreads_list($vmid);
3425         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3426     }
3427 }
3428
3429 sub qemu_iothread_del {
3430     my($conf, $vmid, $deviceid) = @_;
3431
3432     my $device = parse_drive($deviceid, $conf->{$deviceid});
3433     if ($device->{iothread}) {
3434         my $iothreads = vm_iothreads_list($vmid);
3435         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3436     }
3437 }
3438
3439 sub qemu_objectadd {
3440     my($vmid, $objectid, $qomtype) = @_;
3441
3442     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3443
3444     return 1;
3445 }
3446
3447 sub qemu_objectdel {
3448     my($vmid, $objectid) = @_;
3449
3450     vm_mon_cmd($vmid, "object-del", id => $objectid);
3451
3452     return 1;
3453 }
3454
3455 sub qemu_driveadd {
3456     my ($storecfg, $vmid, $device) = @_;
3457
3458     my $drive = print_drive_full($storecfg, $vmid, $device);
3459     $drive =~ s/\\/\\\\/g;
3460     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3461
3462     # If the command succeeds qemu prints: "OK"
3463     return 1 if $ret =~ m/OK/s;
3464
3465     die "adding drive failed: $ret\n";
3466 }
3467
3468 sub qemu_drivedel {
3469     my($vmid, $deviceid) = @_;
3470
3471     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3472     $ret =~ s/^\s+//;
3473
3474     return 1 if $ret eq "";
3475
3476     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3477     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3478
3479     die "deleting drive $deviceid failed : $ret\n";
3480 }
3481
3482 sub qemu_deviceaddverify {
3483     my ($vmid, $deviceid) = @_;
3484
3485     for (my $i = 0; $i <= 5; $i++) {
3486          my $devices_list = vm_devices_list($vmid);
3487          return 1 if defined($devices_list->{$deviceid});
3488          sleep 1;
3489     }
3490
3491     die "error on hotplug device '$deviceid'\n";
3492 }
3493
3494
3495 sub qemu_devicedelverify {
3496     my ($vmid, $deviceid) = @_;
3497
3498     # need to verify that the device is correctly removed as device_del
3499     # is async and empty return is not reliable
3500
3501     for (my $i = 0; $i <= 5; $i++) {
3502          my $devices_list = vm_devices_list($vmid);
3503          return 1 if !defined($devices_list->{$deviceid});
3504          sleep 1;
3505     }
3506
3507     die "error on hot-unplugging device '$deviceid'\n";
3508 }
3509
3510 sub qemu_findorcreatescsihw {
3511     my ($storecfg, $conf, $vmid, $device) = @_;
3512
3513     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3514
3515     my $scsihwid="$controller_prefix$controller";
3516     my $devices_list = vm_devices_list($vmid);
3517
3518     if(!defined($devices_list->{$scsihwid})) {
3519         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3520     }
3521
3522     return 1;
3523 }
3524
3525 sub qemu_deletescsihw {
3526     my ($conf, $vmid, $opt) = @_;
3527
3528     my $device = parse_drive($opt, $conf->{$opt});
3529
3530     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3531         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3532         return 1;
3533     }
3534
3535     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3536
3537     my $devices_list = vm_devices_list($vmid);
3538     foreach my $opt (keys %{$devices_list}) {
3539         if (PVE::QemuServer::is_valid_drivename($opt)) {
3540             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3541             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3542                 return 1;
3543             }
3544         }
3545     }
3546
3547     my $scsihwid="scsihw$controller";
3548
3549     vm_deviceunplug($vmid, $conf, $scsihwid);
3550
3551     return 1;
3552 }
3553
3554 sub qemu_add_pci_bridge {
3555     my ($storecfg, $conf, $vmid, $device) = @_;
3556
3557     my $bridges = {};
3558
3559     my $bridgeid;
3560
3561     print_pci_addr($device, $bridges);
3562
3563     while (my ($k, $v) = each %$bridges) {
3564         $bridgeid = $k;
3565     }
3566     return 1 if !defined($bridgeid) || $bridgeid < 1;
3567
3568     my $bridge = "pci.$bridgeid";
3569     my $devices_list = vm_devices_list($vmid);
3570
3571     if (!defined($devices_list->{$bridge})) {
3572         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3573     }
3574
3575     return 1;
3576 }
3577
3578 sub qemu_set_link_status {
3579     my ($vmid, $device, $up) = @_;
3580
3581     vm_mon_cmd($vmid, "set_link", name => $device,
3582                up => $up ? JSON::true : JSON::false);
3583 }
3584
3585 sub qemu_netdevadd {
3586     my ($vmid, $conf, $device, $deviceid) = @_;
3587
3588     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
3589     my %options =  split(/[=,]/, $netdev);
3590
3591     vm_mon_cmd($vmid, "netdev_add",  %options);
3592     return 1;
3593 }
3594
3595 sub qemu_netdevdel {
3596     my ($vmid, $deviceid) = @_;
3597
3598     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3599 }
3600
3601 sub qemu_usb_hotplug {
3602     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3603
3604     return if !$device;
3605
3606     # remove the old one first
3607     vm_deviceunplug($vmid, $conf, $deviceid);
3608
3609     # check if xhci controller is necessary and available
3610     if ($device->{usb3}) {
3611
3612         my $devicelist = vm_devices_list($vmid);
3613
3614         if (!$devicelist->{xhci}) {
3615             my $pciaddr = print_pci_addr("xhci");
3616             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
3617         }
3618     }
3619     my $d = parse_usb_device($device->{host});
3620     $d->{usb3} = $device->{usb3};
3621
3622     # add the new one
3623     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d);
3624 }
3625
3626 sub qemu_cpu_hotplug {
3627     my ($vmid, $conf, $vcpus) = @_;
3628
3629     my $sockets = 1;
3630     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3631     $sockets = $conf->{sockets} if  $conf->{sockets};
3632     my $cores = $conf->{cores} || 1;
3633     my $maxcpus = $sockets * $cores;
3634
3635     $vcpus = $maxcpus if !$vcpus;
3636
3637     die "you can't add more vcpus than maxcpus\n"
3638         if $vcpus > $maxcpus;
3639
3640     my $currentvcpus = $conf->{vcpus} || $maxcpus;
3641     die "online cpu unplug is not yet possible\n"
3642         if $vcpus < $currentvcpus;
3643
3644     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3645     die "vcpus in running vm is different than configuration\n"
3646         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
3647
3648     for (my $i = $currentvcpus; $i < $vcpus; $i++) {
3649         vm_mon_cmd($vmid, "cpu-add", id => int($i));
3650     }
3651 }
3652
3653 sub qemu_block_set_io_throttle {
3654     my ($vmid, $deviceid,
3655         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
3656         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max) = @_;
3657
3658     return if !check_running($vmid) ;
3659
3660     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
3661         bps => int($bps),
3662         bps_rd => int($bps_rd),
3663         bps_wr => int($bps_wr),
3664         iops => int($iops),
3665         iops_rd => int($iops_rd),
3666         iops_wr => int($iops_wr),
3667         bps_max => int($bps_max),
3668         bps_rd_max => int($bps_rd_max),
3669         bps_wr_max => int($bps_wr_max),
3670         iops_max => int($iops_max),
3671         iops_rd_max => int($iops_rd_max),
3672         iops_wr_max => int($iops_wr_max)
3673     );
3674
3675 }
3676
3677 # old code, only used to shutdown old VM after update
3678 sub __read_avail {
3679     my ($fh, $timeout) = @_;
3680
3681     my $sel = new IO::Select;
3682     $sel->add($fh);
3683
3684     my $res = '';
3685     my $buf;
3686
3687     my @ready;
3688     while (scalar (@ready = $sel->can_read($timeout))) {
3689         my $count;
3690         if ($count = $fh->sysread($buf, 8192)) {
3691             if ($buf =~ /^(.*)\(qemu\) $/s) {
3692                 $res .= $1;
3693                 last;
3694             } else {
3695                 $res .= $buf;
3696             }
3697         } else {
3698             if (!defined($count)) {
3699                 die "$!\n";
3700             }
3701             last;
3702         }
3703     }
3704
3705     die "monitor read timeout\n" if !scalar(@ready);
3706
3707     return $res;
3708 }
3709
3710 # old code, only used to shutdown old VM after update
3711 sub vm_monitor_command {
3712     my ($vmid, $cmdstr, $nocheck) = @_;
3713
3714     my $res;
3715
3716     eval {
3717         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
3718
3719         my $sname = "${var_run_tmpdir}/$vmid.mon";
3720
3721         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
3722             die "unable to connect to VM $vmid socket - $!\n";
3723
3724         my $timeout = 3;
3725
3726         # hack: migrate sometime blocks the monitor (when migrate_downtime
3727         # is set)
3728         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3729             $timeout = 60*60; # 1 hour
3730         }
3731
3732         # read banner;
3733         my $data = __read_avail($sock, $timeout);
3734
3735         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
3736             die "got unexpected qemu monitor banner\n";
3737         }
3738
3739         my $sel = new IO::Select;
3740         $sel->add($sock);
3741
3742         if (!scalar(my @ready = $sel->can_write($timeout))) {
3743             die "monitor write error - timeout";
3744         }
3745
3746         my $fullcmd = "$cmdstr\r";
3747
3748         # syslog('info', "VM $vmid monitor command: $cmdstr");
3749
3750         my $b;
3751         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
3752             die "monitor write error - $!";
3753         }
3754
3755         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
3756
3757         $timeout = 20;
3758
3759         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3760             $timeout = 60*60; # 1 hour
3761         } elsif ($cmdstr =~ m/^(eject|change)/) {
3762             $timeout = 60; # note: cdrom mount command is slow
3763         }
3764         if ($res = __read_avail($sock, $timeout)) {
3765
3766             my @lines = split("\r?\n", $res);
3767
3768             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
3769
3770             $res = join("\n", @lines);
3771             $res .= "\n";
3772         }
3773     };
3774
3775     my $err = $@;
3776
3777     if ($err) {
3778         syslog("err", "VM $vmid monitor command failed - $err");
3779         die $err;
3780     }
3781
3782     return $res;
3783 }
3784
3785 sub qemu_block_resize {
3786     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
3787
3788     my $running = check_running($vmid);
3789
3790     return if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
3791
3792     return if !$running;
3793
3794     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
3795
3796 }
3797
3798 sub qemu_volume_snapshot {
3799     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3800
3801     my $running = check_running($vmid);
3802
3803     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
3804         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
3805     } else {
3806         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
3807     }
3808 }
3809
3810 sub qemu_volume_snapshot_delete {
3811     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3812
3813     my $running = check_running($vmid);
3814
3815     return if !PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
3816
3817     return if !$running;
3818
3819     vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
3820 }
3821
3822 sub set_migration_caps {
3823     my ($vmid) = @_;
3824
3825     my $cap_ref = [];
3826
3827     my $enabled_cap = {
3828         "auto-converge" => 1,
3829         "xbzrle" => 1,
3830         "x-rdma-pin-all" => 0,
3831         "zero-blocks" => 0,
3832         "compress" => 0
3833     };
3834
3835     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
3836
3837     for my $supported_capability (@$supported_capabilities) {
3838         push @$cap_ref, {
3839             capability => $supported_capability->{capability},
3840             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
3841         };
3842     }
3843
3844     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
3845 }
3846
3847 my $fast_plug_option = {
3848     'lock' => 1,
3849     'name' => 1,
3850     'onboot' => 1,
3851     'shares' => 1,
3852     'startup' => 1,
3853     'description' => 1,
3854 };
3855
3856 # hotplug changes in [PENDING]
3857 # $selection hash can be used to only apply specified options, for
3858 # example: { cores => 1 } (only apply changed 'cores')
3859 # $errors ref is used to return error messages
3860 sub vmconfig_hotplug_pending {
3861     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
3862
3863     my $defaults = load_defaults();
3864
3865     # commit values which do not have any impact on running VM first
3866     # Note: those option cannot raise errors, we we do not care about
3867     # $selection and always apply them.
3868
3869     my $add_error = sub {
3870         my ($opt, $msg) = @_;
3871         $errors->{$opt} = "hotplug problem - $msg";
3872     };
3873
3874     my $changes = 0;
3875     foreach my $opt (keys %{$conf->{pending}}) { # add/change
3876         if ($fast_plug_option->{$opt}) {
3877             $conf->{$opt} = $conf->{pending}->{$opt};
3878             delete $conf->{pending}->{$opt};
3879             $changes = 1;
3880         }
3881     }
3882
3883     if ($changes) {
3884         PVE::QemuConfig->write_config($vmid, $conf);
3885         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
3886     }
3887
3888     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3889
3890     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
3891     while (my ($opt, $force) = each %$pending_delete_hash) {
3892         next if $selection && !$selection->{$opt};
3893         eval {
3894             if ($opt eq 'hotplug') {
3895                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
3896             } elsif ($opt eq 'tablet') {
3897                 die "skip\n" if !$hotplug_features->{usb};
3898                 if ($defaults->{tablet}) {
3899                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
3900                 } else {
3901                     vm_deviceunplug($vmid, $conf, $opt);
3902                 }
3903             } elsif ($opt eq 'vcpus') {
3904                 die "skip\n" if !$hotplug_features->{cpu};
3905                 qemu_cpu_hotplug($vmid, $conf, undef);
3906             } elsif ($opt eq 'balloon') {
3907                 # enable balloon device is not hotpluggable
3908                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
3909             } elsif ($fast_plug_option->{$opt}) {
3910                 # do nothing
3911             } elsif ($opt =~ m/^net(\d+)$/) {
3912                 die "skip\n" if !$hotplug_features->{network};
3913                 vm_deviceunplug($vmid, $conf, $opt);
3914             } elsif (is_valid_drivename($opt)) {
3915                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
3916                 vm_deviceunplug($vmid, $conf, $opt);
3917                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
3918             } elsif ($opt =~ m/^memory$/) {
3919                 die "skip\n" if !$hotplug_features->{memory};
3920                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
3921             } elsif ($opt eq 'cpuunits') {
3922                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
3923             } elsif ($opt eq 'cpulimit') {
3924                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
3925             } else {
3926                 die "skip\n";
3927             }
3928         };
3929         if (my $err = $@) {
3930             &$add_error($opt, $err) if $err ne "skip\n";
3931         } else {
3932             # save new config if hotplug was successful
3933             delete $conf->{$opt};
3934             vmconfig_undelete_pending_option($conf, $opt);
3935             PVE::QemuConfig->write_config($vmid, $conf);
3936             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
3937         }
3938     }
3939
3940     foreach my $opt (keys %{$conf->{pending}}) {
3941         next if $selection && !$selection->{$opt};
3942         my $value = $conf->{pending}->{$opt};
3943         eval {
3944             if ($opt eq 'hotplug') {
3945                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
3946             } elsif ($opt eq 'tablet') {
3947                 die "skip\n" if !$hotplug_features->{usb};
3948                 if ($value == 1) {
3949                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
3950                 } elsif ($value == 0) {
3951                     vm_deviceunplug($vmid, $conf, $opt);
3952                 }
3953             } elsif ($opt eq 'vcpus') {
3954                 die "skip\n" if !$hotplug_features->{cpu};
3955                 qemu_cpu_hotplug($vmid, $conf, $value);
3956             } elsif ($opt eq 'balloon') {
3957                 # enable/disable balloning device is not hotpluggable
3958                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
3959                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
3960                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
3961
3962                 # allow manual ballooning if shares is set to zero
3963                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
3964                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
3965                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
3966                 }
3967             } elsif ($opt =~ m/^net(\d+)$/) {
3968                 # some changes can be done without hotplug
3969                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
3970                                     $vmid, $opt, $value);
3971             } elsif (is_valid_drivename($opt)) {
3972                 # some changes can be done without hotplug
3973                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
3974                                      $vmid, $opt, $value, 1);
3975             } elsif ($opt =~ m/^memory$/) { #dimms
3976                 die "skip\n" if !$hotplug_features->{memory};
3977                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
3978             } elsif ($opt eq 'cpuunits') {
3979                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
3980             } elsif ($opt eq 'cpulimit') {
3981                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
3982                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
3983             } else {
3984                 die "skip\n";  # skip non-hot-pluggable options
3985             }
3986         };
3987         if (my $err = $@) {
3988             &$add_error($opt, $err) if $err ne "skip\n";
3989         } else {
3990             # save new config if hotplug was successful
3991             $conf->{$opt} = $value;
3992             delete $conf->{pending}->{$opt};
3993             PVE::QemuConfig->write_config($vmid, $conf);
3994             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
3995         }
3996     }
3997 }
3998
3999 sub try_deallocate_drive {
4000     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4001
4002     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4003         my $volid = $drive->{file};
4004         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4005             my $sid = PVE::Storage::parse_volume_id($volid);
4006             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4007
4008             # check if the disk is really unused
4009             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4010                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4011             PVE::Storage::vdisk_free($storecfg, $volid);
4012             return 1;
4013         } else {
4014             # If vm is not owner of this disk remove from config
4015             return 1;
4016         }
4017     }
4018
4019     return undef;
4020 }
4021
4022 sub vmconfig_delete_or_detach_drive {
4023     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4024
4025     my $drive = parse_drive($opt, $conf->{$opt});
4026
4027     my $rpcenv = PVE::RPCEnvironment::get();
4028     my $authuser = $rpcenv->get_user();
4029
4030     if ($force) {
4031         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4032         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4033     } else {
4034         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4035     }
4036 }
4037
4038 sub vmconfig_apply_pending {
4039     my ($vmid, $conf, $storecfg) = @_;
4040
4041     # cold plug
4042
4043     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4044     while (my ($opt, $force) = each %$pending_delete_hash) {
4045         die "internal error" if $opt =~ m/^unused/;
4046         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4047         if (!defined($conf->{$opt})) {
4048             vmconfig_undelete_pending_option($conf, $opt);
4049             PVE::QemuConfig->write_config($vmid, $conf);
4050         } elsif (is_valid_drivename($opt)) {
4051             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4052             vmconfig_undelete_pending_option($conf, $opt);
4053             delete $conf->{$opt};
4054             PVE::QemuConfig->write_config($vmid, $conf);
4055         } else {
4056             vmconfig_undelete_pending_option($conf, $opt);
4057             delete $conf->{$opt};
4058             PVE::QemuConfig->write_config($vmid, $conf);
4059         }
4060     }
4061
4062     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4063
4064     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4065         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4066
4067         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4068             # skip if nothing changed
4069         } elsif (is_valid_drivename($opt)) {
4070             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4071                 if defined($conf->{$opt});
4072             $conf->{$opt} = $conf->{pending}->{$opt};
4073         } else {
4074             $conf->{$opt} = $conf->{pending}->{$opt};
4075         }
4076
4077         delete $conf->{pending}->{$opt};
4078         PVE::QemuConfig->write_config($vmid, $conf);
4079     }
4080 }
4081
4082 my $safe_num_ne = sub {
4083     my ($a, $b) = @_;
4084
4085     return 0 if !defined($a) && !defined($b);
4086     return 1 if !defined($a);
4087     return 1 if !defined($b);
4088
4089     return $a != $b;
4090 };
4091
4092 my $safe_string_ne = sub {
4093     my ($a, $b) = @_;
4094
4095     return 0 if !defined($a) && !defined($b);
4096     return 1 if !defined($a);
4097     return 1 if !defined($b);
4098
4099     return $a ne $b;
4100 };
4101
4102 sub vmconfig_update_net {
4103     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4104
4105     my $newnet = parse_net($value);
4106
4107     if ($conf->{$opt}) {
4108         my $oldnet = parse_net($conf->{$opt});
4109
4110         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4111             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4112             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4113             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4114
4115             # for non online change, we try to hot-unplug
4116             die "skip\n" if !$hotplug;
4117             vm_deviceunplug($vmid, $conf, $opt);
4118         } else {
4119
4120             die "internal error" if $opt !~ m/net(\d+)/;
4121             my $iface = "tap${vmid}i$1";
4122
4123             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4124                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4125                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4126                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4127                 PVE::Network::tap_unplug($iface);
4128                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4129             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4130                 # Rate can be applied on its own but any change above needs to
4131                 # include the rate in tap_plug since OVS resets everything.
4132                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4133             }
4134
4135             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4136                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4137             }
4138
4139             return 1;
4140         }
4141     }
4142
4143     if ($hotplug) {
4144         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4145     } else {
4146         die "skip\n";
4147     }
4148 }
4149
4150 sub vmconfig_update_disk {
4151     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4152
4153     # fixme: do we need force?
4154
4155     my $drive = parse_drive($opt, $value);
4156
4157     if ($conf->{$opt}) {
4158
4159         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4160
4161             my $media = $drive->{media} || 'disk';
4162             my $oldmedia = $old_drive->{media} || 'disk';
4163             die "unable to change media type\n" if $media ne $oldmedia;
4164
4165             if (!drive_is_cdrom($old_drive)) {
4166
4167                 if ($drive->{file} ne $old_drive->{file}) {
4168
4169                     die "skip\n" if !$hotplug;
4170
4171                     # unplug and register as unused
4172                     vm_deviceunplug($vmid, $conf, $opt);
4173                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4174
4175                 } else {
4176                     # update existing disk
4177
4178                     # skip non hotpluggable value
4179                     if (&$safe_num_ne($drive->{discard}, $old_drive->{discard}) ||
4180                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4181                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4182                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4183                         die "skip\n";
4184                     }
4185
4186                     # apply throttle
4187                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4188                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4189                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4190                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4191                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4192                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4193                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4194                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4195                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4196                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4197                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4198                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max})) {
4199
4200                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4201                                                    ($drive->{mbps} || 0)*1024*1024,
4202                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4203                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4204                                                    $drive->{iops} || 0,
4205                                                    $drive->{iops_rd} || 0,
4206                                                    $drive->{iops_wr} || 0,
4207                                                    ($drive->{mbps_max} || 0)*1024*1024,
4208                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4209                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4210                                                    $drive->{iops_max} || 0,
4211                                                    $drive->{iops_rd_max} || 0,
4212                                                    $drive->{iops_wr_max} || 0);
4213
4214                     }
4215
4216                     return 1;
4217                 }
4218
4219             } else { # cdrom
4220
4221                 if ($drive->{file} eq 'none') {
4222                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4223                 } else {
4224                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4225                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4226                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4227                 }
4228
4229                 return 1;
4230             }
4231         }
4232     }
4233
4234     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4235     # hotplug new disks
4236     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4237     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4238 }
4239
4240 sub vm_start {
4241     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4242         $forcemachine, $spice_ticket) = @_;
4243
4244     PVE::QemuConfig->lock_config($vmid, sub {
4245         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4246
4247         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
4248
4249         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4250
4251         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4252
4253         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4254             vmconfig_apply_pending($vmid, $conf, $storecfg);
4255             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4256         }
4257
4258         my $defaults = load_defaults();
4259
4260         # set environment variable useful inside network script
4261         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4262
4263         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4264
4265         my $migrate_port = 0;
4266         my $migrate_uri;
4267         if ($statefile) {
4268             if ($statefile eq 'tcp') {
4269                 my $localip = "localhost";
4270                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4271                 my $nodename = PVE::INotify::nodename();
4272                 if ($datacenterconf->{migration_unsecure}) {
4273                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4274                         $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4275                 }
4276                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4277                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4278                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4279                 push @$cmd, '-incoming', $migrate_uri;
4280                 push @$cmd, '-S';
4281
4282             } elsif ($statefile eq 'unix') {
4283                 # should be default for secure migrations as a ssh TCP forward
4284                 # tunnel is not deterministic reliable ready and fails regurarly
4285                 # to set up in time, so use UNIX socket forwards
4286                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
4287                 unlink $socket_addr;
4288
4289                 $migrate_uri = "unix:$socket_addr";
4290
4291                 push @$cmd, '-incoming', $migrate_uri;
4292                 push @$cmd, '-S';
4293
4294             } else {
4295                 push @$cmd, '-loadstate', $statefile;
4296             }
4297         } elsif ($paused) {
4298             push @$cmd, '-S';
4299         }
4300
4301         # host pci devices
4302         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4303           my $d = parse_hostpci($conf->{"hostpci$i"});
4304           next if !$d;
4305           my $pcidevices = $d->{pciid};
4306           foreach my $pcidevice (@$pcidevices) {
4307                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4308
4309                 my $info = pci_device_info("0000:$pciid");
4310                 die "IOMMU not present\n" if !check_iommu_support();
4311                 die "no pci device info for device '$pciid'\n" if !$info;
4312                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4313                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4314           }
4315         }
4316
4317         PVE::Storage::activate_volumes($storecfg, $vollist);
4318
4319         if (!check_running($vmid, 1) && -d "/sys/fs/cgroup/systemd/qemu.slice/$vmid.scope") {
4320             my $cmd = [];
4321             push @$cmd, '/bin/systemctl', 'stop', "$vmid.scope";
4322             eval  { run_command($cmd); };
4323         }
4324
4325         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
4326                                                   : $defaults->{cpuunits};
4327
4328         my %run_params = (timeout => $statefile ? undef : 30, umask => 0077);
4329
4330         my %properties = (
4331             Slice => 'qemu.slice',
4332             KillMode => 'none',
4333             CPUShares => $cpuunits
4334         );
4335
4336         if (my $cpulimit = $conf->{cpulimit}) {
4337             $properties{CPUQuota} = int($cpulimit * 100);
4338         }
4339         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
4340
4341         if ($conf->{hugepages}) {
4342
4343             my $code = sub {
4344                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
4345                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
4346
4347                 PVE::QemuServer::Memory::hugepages_mount();
4348                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
4349
4350                 eval  {
4351                     PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4352                     run_command($cmd, %run_params);
4353                 };
4354
4355                 if (my $err = $@) {
4356                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
4357                     die $err;
4358                 }
4359
4360                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
4361             };
4362             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
4363
4364         } else {
4365             eval  {
4366                 PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4367                 run_command($cmd, %run_params);
4368             };
4369         }
4370
4371         if (my $err = $@) {
4372             # deactivate volumes if start fails
4373             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
4374             die "start failed: $err";
4375         }
4376
4377         print "migration listens on $migrate_uri\n" if $migrate_uri;
4378
4379         if ($statefile && $statefile ne 'tcp')  {
4380             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4381             warn $@ if $@;
4382         }
4383
4384         if ($migratedfrom) {
4385
4386             eval {
4387                 set_migration_caps($vmid);
4388             };
4389             warn $@ if $@;
4390
4391             if ($spice_port) {
4392                 print "spice listens on port $spice_port\n";
4393                 if ($spice_ticket) {
4394                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4395                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4396                 }
4397             }
4398
4399         } else {
4400
4401             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4402                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4403                     if $conf->{balloon};
4404             }
4405
4406             foreach my $opt (keys %$conf) {
4407                 next if $opt !~  m/^net\d+$/;
4408                 my $nicconf = parse_net($conf->{$opt});
4409                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4410             }
4411         }
4412
4413         vm_mon_cmd_nocheck($vmid, 'qom-set',
4414                     path => "machine/peripheral/balloon0",
4415                     property => "guest-stats-polling-interval",
4416                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4417
4418     });
4419 }
4420
4421 sub vm_mon_cmd {
4422     my ($vmid, $execute, %params) = @_;
4423
4424     my $cmd = { execute => $execute, arguments => \%params };
4425     vm_qmp_command($vmid, $cmd);
4426 }
4427
4428 sub vm_mon_cmd_nocheck {
4429     my ($vmid, $execute, %params) = @_;
4430
4431     my $cmd = { execute => $execute, arguments => \%params };
4432     vm_qmp_command($vmid, $cmd, 1);
4433 }
4434
4435 sub vm_qmp_command {
4436     my ($vmid, $cmd, $nocheck) = @_;
4437
4438     my $res;
4439
4440     my $timeout;
4441     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4442         $timeout = $cmd->{arguments}->{timeout};
4443         delete $cmd->{arguments}->{timeout};
4444     }
4445
4446     eval {
4447         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4448         my $sname = qmp_socket($vmid);
4449         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4450             my $qmpclient = PVE::QMPClient->new();
4451
4452             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4453         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
4454             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
4455                 if scalar(%{$cmd->{arguments}});
4456             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
4457         } else {
4458             die "unable to open monitor socket\n";
4459         }
4460     };
4461     if (my $err = $@) {
4462         syslog("err", "VM $vmid qmp command failed - $err");
4463         die $err;
4464     }
4465
4466     return $res;
4467 }
4468
4469 sub vm_human_monitor_command {
4470     my ($vmid, $cmdline) = @_;
4471
4472     my $res;
4473
4474     my $cmd = {
4475         execute => 'human-monitor-command',
4476         arguments => { 'command-line' => $cmdline},
4477     };
4478
4479     return vm_qmp_command($vmid, $cmd);
4480 }
4481
4482 sub vm_commandline {
4483     my ($storecfg, $vmid) = @_;
4484
4485     my $conf = PVE::QemuConfig->load_config($vmid);
4486
4487     my $defaults = load_defaults();
4488
4489     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
4490
4491     return join(' ', @$cmd);
4492 }
4493
4494 sub vm_reset {
4495     my ($vmid, $skiplock) = @_;
4496
4497     PVE::QemuConfig->lock_config($vmid, sub {
4498
4499         my $conf = PVE::QemuConfig->load_config($vmid);
4500
4501         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4502
4503         vm_mon_cmd($vmid, "system_reset");
4504     });
4505 }
4506
4507 sub get_vm_volumes {
4508     my ($conf) = @_;
4509
4510     my $vollist = [];
4511     foreach_volid($conf, sub {
4512         my ($volid, $is_cdrom) = @_;
4513
4514         return if $volid =~ m|^/|;
4515
4516         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4517         return if !$sid;
4518
4519         push @$vollist, $volid;
4520     });
4521
4522     return $vollist;
4523 }
4524
4525 sub vm_stop_cleanup {
4526     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
4527
4528     eval {
4529
4530         if (!$keepActive) {
4531             my $vollist = get_vm_volumes($conf);
4532             PVE::Storage::deactivate_volumes($storecfg, $vollist);
4533         }
4534
4535         foreach my $ext (qw(mon qmp pid vnc qga)) {
4536             unlink "/var/run/qemu-server/${vmid}.$ext";
4537         }
4538
4539         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
4540     };
4541     warn $@ if $@; # avoid errors - just warn
4542 }
4543
4544 # Note: use $nockeck to skip tests if VM configuration file exists.
4545 # We need that when migration VMs to other nodes (files already moved)
4546 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
4547 sub vm_stop {
4548     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
4549
4550     $force = 1 if !defined($force) && !$shutdown;
4551
4552     if ($migratedfrom){
4553         my $pid = check_running($vmid, $nocheck, $migratedfrom);
4554         kill 15, $pid if $pid;
4555         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4556         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
4557         return;
4558     }
4559
4560     PVE::QemuConfig->lock_config($vmid, sub {
4561
4562         my $pid = check_running($vmid, $nocheck);
4563         return if !$pid;
4564
4565         my $conf;
4566         if (!$nocheck) {
4567             $conf = PVE::QemuConfig->load_config($vmid);
4568             PVE::QemuConfig->check_lock($conf) if !$skiplock;
4569             if (!defined($timeout) && $shutdown && $conf->{startup}) {
4570                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
4571                 $timeout = $opts->{down} if $opts->{down};
4572             }
4573         }
4574
4575         $timeout = 60 if !defined($timeout);
4576
4577         eval {
4578             if ($shutdown) {
4579                 if (defined($conf) && $conf->{agent}) {
4580                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
4581                 } else {
4582                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
4583                 }
4584             } else {
4585                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
4586             }
4587         };
4588         my $err = $@;
4589
4590         if (!$err) {
4591             my $count = 0;
4592             while (($count < $timeout) && check_running($vmid, $nocheck)) {
4593                 $count++;
4594                 sleep 1;
4595             }
4596
4597             if ($count >= $timeout) {
4598                 if ($force) {
4599                     warn "VM still running - terminating now with SIGTERM\n";
4600                     kill 15, $pid;
4601                 } else {
4602                     die "VM quit/powerdown failed - got timeout\n";
4603                 }
4604             } else {
4605                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4606                 return;
4607             }
4608         } else {
4609             if ($force) {
4610                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
4611                 kill 15, $pid;
4612             } else {
4613                 die "VM quit/powerdown failed\n";
4614             }
4615         }
4616
4617         # wait again
4618         $timeout = 10;
4619
4620         my $count = 0;
4621         while (($count < $timeout) && check_running($vmid, $nocheck)) {
4622             $count++;
4623             sleep 1;
4624         }
4625
4626         if ($count >= $timeout) {
4627             warn "VM still running - terminating now with SIGKILL\n";
4628             kill 9, $pid;
4629             sleep 1;
4630         }
4631
4632         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4633    });
4634 }
4635
4636 sub vm_suspend {
4637     my ($vmid, $skiplock) = @_;
4638
4639     PVE::QemuConfig->lock_config($vmid, sub {
4640
4641         my $conf = PVE::QemuConfig->load_config($vmid);
4642
4643         PVE::QemuConfig->check_lock($conf)
4644             if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
4645
4646         vm_mon_cmd($vmid, "stop");
4647     });
4648 }
4649
4650 sub vm_resume {
4651     my ($vmid, $skiplock, $nocheck) = @_;
4652
4653     PVE::QemuConfig->lock_config($vmid, sub {
4654
4655         if (!$nocheck) {
4656
4657             my $conf = PVE::QemuConfig->load_config($vmid);
4658
4659             PVE::QemuConfig->check_lock($conf)
4660                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
4661
4662             vm_mon_cmd($vmid, "cont");
4663
4664         } else {
4665             vm_mon_cmd_nocheck($vmid, "cont");
4666         }
4667     });
4668 }
4669
4670 sub vm_sendkey {
4671     my ($vmid, $skiplock, $key) = @_;
4672
4673     PVE::QemuConfig->lock_config($vmid, sub {
4674
4675         my $conf = PVE::QemuConfig->load_config($vmid);
4676
4677         # there is no qmp command, so we use the human monitor command
4678         vm_human_monitor_command($vmid, "sendkey $key");
4679     });
4680 }
4681
4682 sub vm_destroy {
4683     my ($storecfg, $vmid, $skiplock) = @_;
4684
4685     PVE::QemuConfig->lock_config($vmid, sub {
4686
4687         my $conf = PVE::QemuConfig->load_config($vmid);
4688
4689         if (!check_running($vmid)) {
4690             destroy_vm($storecfg, $vmid, undef, $skiplock);
4691         } else {
4692             die "VM $vmid is running - destroy failed\n";
4693         }
4694     });
4695 }
4696
4697 # pci helpers
4698
4699 sub file_write {
4700     my ($filename, $buf) = @_;
4701
4702     my $fh = IO::File->new($filename, "w");
4703     return undef if !$fh;
4704
4705     my $res = print $fh $buf;
4706
4707     $fh->close();
4708
4709     return $res;
4710 }
4711
4712 sub pci_device_info {
4713     my ($name) = @_;
4714
4715     my $res;
4716
4717     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
4718     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
4719
4720     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
4721     return undef if !defined($irq) || $irq !~ m/^\d+$/;
4722
4723     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
4724     return undef if !defined($vendor) || $vendor !~ s/^0x//;
4725
4726     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
4727     return undef if !defined($product) || $product !~ s/^0x//;
4728
4729     $res = {
4730         name => $name,
4731         vendor => $vendor,
4732         product => $product,
4733         domain => $domain,
4734         bus => $bus,
4735         slot => $slot,
4736         func => $func,
4737         irq => $irq,
4738         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
4739     };
4740
4741     return $res;
4742 }
4743
4744 sub pci_dev_reset {
4745     my ($dev) = @_;
4746
4747     my $name = $dev->{name};
4748
4749     my $fn = "$pcisysfs/devices/$name/reset";
4750
4751     return file_write($fn, "1");
4752 }
4753
4754 sub pci_dev_bind_to_vfio {
4755     my ($dev) = @_;
4756
4757     my $name = $dev->{name};
4758
4759     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4760
4761     if (!-d $vfio_basedir) {
4762         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4763     }
4764     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4765
4766     my $testdir = "$vfio_basedir/$name";
4767     return 1 if -d $testdir;
4768
4769     my $data = "$dev->{vendor} $dev->{product}";
4770     return undef if !file_write("$vfio_basedir/new_id", $data);
4771
4772     my $fn = "$pcisysfs/devices/$name/driver/unbind";
4773     if (!file_write($fn, $name)) {
4774         return undef if -f $fn;
4775     }
4776
4777     $fn = "$vfio_basedir/bind";
4778     if (! -d $testdir) {
4779         return undef if !file_write($fn, $name);
4780     }
4781
4782     return -d $testdir;
4783 }
4784
4785 sub pci_dev_group_bind_to_vfio {
4786     my ($pciid) = @_;
4787
4788     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4789
4790     if (!-d $vfio_basedir) {
4791         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4792     }
4793     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4794
4795     # get IOMMU group devices
4796     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
4797       my @devs = grep /^0000:/, readdir($D);
4798     closedir($D);
4799
4800     foreach my $pciid (@devs) {
4801         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
4802
4803         # pci bridges, switches or root ports are not supported
4804         # they have a pci_bus subdirectory so skip them
4805         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
4806
4807         my $info = pci_device_info($1);
4808         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
4809     }
4810
4811     return 1;
4812 }
4813
4814 # vzdump restore implementaion
4815
4816 sub tar_archive_read_firstfile {
4817     my $archive = shift;
4818
4819     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
4820
4821     # try to detect archive type first
4822     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
4823         die "unable to open file '$archive'\n";
4824     my $firstfile = <$fh>;
4825     kill 15, $pid;
4826     close $fh;
4827
4828     die "ERROR: archive contaions no data\n" if !$firstfile;
4829     chomp $firstfile;
4830
4831     return $firstfile;
4832 }
4833
4834 sub tar_restore_cleanup {
4835     my ($storecfg, $statfile) = @_;
4836
4837     print STDERR "starting cleanup\n";
4838
4839     if (my $fd = IO::File->new($statfile, "r")) {
4840         while (defined(my $line = <$fd>)) {
4841             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
4842                 my $volid = $2;
4843                 eval {
4844                     if ($volid =~ m|^/|) {
4845                         unlink $volid || die 'unlink failed\n';
4846                     } else {
4847                         PVE::Storage::vdisk_free($storecfg, $volid);
4848                     }
4849                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
4850                 };
4851                 print STDERR "unable to cleanup '$volid' - $@" if $@;
4852             } else {
4853                 print STDERR "unable to parse line in statfile - $line";
4854             }
4855         }
4856         $fd->close();
4857     }
4858 }
4859
4860 sub restore_archive {
4861     my ($archive, $vmid, $user, $opts) = @_;
4862
4863     my $format = $opts->{format};
4864     my $comp;
4865
4866     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
4867         $format = 'tar' if !$format;
4868         $comp = 'gzip';
4869     } elsif ($archive =~ m/\.tar$/) {
4870         $format = 'tar' if !$format;
4871     } elsif ($archive =~ m/.tar.lzo$/) {
4872         $format = 'tar' if !$format;
4873         $comp = 'lzop';
4874     } elsif ($archive =~ m/\.vma$/) {
4875         $format = 'vma' if !$format;
4876     } elsif ($archive =~ m/\.vma\.gz$/) {
4877         $format = 'vma' if !$format;
4878         $comp = 'gzip';
4879     } elsif ($archive =~ m/\.vma\.lzo$/) {
4880         $format = 'vma' if !$format;
4881         $comp = 'lzop';
4882     } else {
4883         $format = 'vma' if !$format; # default
4884     }
4885
4886     # try to detect archive format
4887     if ($format eq 'tar') {
4888         return restore_tar_archive($archive, $vmid, $user, $opts);
4889     } else {
4890         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
4891     }
4892 }
4893
4894 sub restore_update_config_line {
4895     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
4896
4897     return if $line =~ m/^\#qmdump\#/;
4898     return if $line =~ m/^\#vzdump\#/;
4899     return if $line =~ m/^lock:/;
4900     return if $line =~ m/^unused\d+:/;
4901     return if $line =~ m/^parent:/;
4902     return if $line =~ m/^template:/; # restored VM is never a template
4903
4904     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
4905         # try to convert old 1.X settings
4906         my ($id, $ind, $ethcfg) = ($1, $2, $3);
4907         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
4908             my ($model, $macaddr) = split(/\=/, $devconfig);
4909             $macaddr = PVE::Tools::random_ether_addr() if !$macaddr || $unique;
4910             my $net = {
4911                 model => $model,
4912                 bridge => "vmbr$ind",
4913                 macaddr => $macaddr,
4914             };
4915             my $netstr = print_net($net);
4916
4917             print $outfd "net$cookie->{netcount}: $netstr\n";
4918             $cookie->{netcount}++;
4919         }
4920     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
4921         my ($id, $netstr) = ($1, $2);
4922         my $net = parse_net($netstr);
4923         $net->{macaddr} = PVE::Tools::random_ether_addr() if $net->{macaddr};
4924         $netstr = print_net($net);
4925         print $outfd "$id: $netstr\n";
4926     } elsif ($line =~ m/^((ide|scsi|virtio|sata)\d+):\s*(\S+)\s*$/) {
4927         my $virtdev = $1;
4928         my $value = $3;
4929         my $di = parse_drive($virtdev, $value);
4930         if (defined($di->{backup}) && !$di->{backup}) {
4931             print $outfd "#$line";
4932         } elsif ($map->{$virtdev}) {
4933             delete $di->{format}; # format can change on restore
4934             $di->{file} = $map->{$virtdev};
4935             $value = print_drive($vmid, $di);
4936             print $outfd "$virtdev: $value\n";
4937         } else {
4938             print $outfd $line;
4939         }
4940     } else {
4941         print $outfd $line;
4942     }
4943 }
4944
4945 sub scan_volids {
4946     my ($cfg, $vmid) = @_;
4947
4948     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
4949
4950     my $volid_hash = {};
4951     foreach my $storeid (keys %$info) {
4952         foreach my $item (@{$info->{$storeid}}) {
4953             next if !($item->{volid} && $item->{size});
4954             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
4955             $volid_hash->{$item->{volid}} = $item;
4956         }
4957     }
4958
4959     return $volid_hash;
4960 }
4961
4962 sub is_volume_in_use {
4963     my ($storecfg, $conf, $skip_drive, $volid) = @_;
4964
4965     my $path = PVE::Storage::path($storecfg, $volid);
4966
4967     my $scan_config = sub {
4968         my ($cref, $snapname) = @_;
4969
4970         foreach my $key (keys %$cref) {
4971             my $value = $cref->{$key};
4972             if (is_valid_drivename($key)) {
4973                 next if $skip_drive && $key eq $skip_drive;
4974                 my $drive = parse_drive($key, $value);
4975                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
4976                 return 1 if $volid eq $drive->{file};
4977                 if ($drive->{file} =~ m!^/!) {
4978                     return 1 if $drive->{file} eq $path;
4979                 } else {
4980                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
4981                     next if !$storeid;
4982                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
4983                     next if !$scfg;
4984                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
4985                 }
4986             }
4987         }
4988
4989         return 0;
4990     };
4991
4992     return 1 if &$scan_config($conf);
4993
4994     undef $skip_drive;
4995
4996     foreach my $snapname (keys %{$conf->{snapshots}}) {
4997         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
4998     }
4999
5000     return 0;
5001 }
5002
5003 sub update_disksize {
5004     my ($vmid, $conf, $volid_hash) = @_;
5005
5006     my $changes;
5007
5008     my $used = {};
5009
5010     # Note: it is allowed to define multiple storages with same path (alias), so
5011     # we need to check both 'volid' and real 'path' (two different volid can point
5012     # to the same path).
5013
5014     my $usedpath = {};
5015
5016     # update size info
5017     foreach my $opt (keys %$conf) {
5018         if (is_valid_drivename($opt)) {
5019             my $drive = parse_drive($opt, $conf->{$opt});
5020             my $volid = $drive->{file};
5021             next if !$volid;
5022
5023             $used->{$volid} = 1;
5024             if ($volid_hash->{$volid} &&
5025                 (my $path = $volid_hash->{$volid}->{path})) {
5026                 $usedpath->{$path} = 1;
5027             }
5028
5029             next if drive_is_cdrom($drive);
5030             next if !$volid_hash->{$volid};
5031
5032             $drive->{size} = $volid_hash->{$volid}->{size};
5033             my $new = print_drive($vmid, $drive);
5034             if ($new ne $conf->{$opt}) {
5035                 $changes = 1;
5036                 $conf->{$opt} = $new;
5037             }
5038         }
5039     }
5040
5041     # remove 'unusedX' entry if volume is used
5042     foreach my $opt (keys %$conf) {
5043         next if $opt !~ m/^unused\d+$/;
5044         my $volid = $conf->{$opt};
5045         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5046         if ($used->{$volid} || ($path && $usedpath->{$path})) {
5047             $changes = 1;
5048             delete $conf->{$opt};
5049         }
5050     }
5051
5052     foreach my $volid (sort keys %$volid_hash) {
5053         next if $volid =~ m/vm-$vmid-state-/;
5054         next if $used->{$volid};
5055         my $path = $volid_hash->{$volid}->{path};
5056         next if !$path; # just to be sure
5057         next if $usedpath->{$path};
5058         $changes = 1;
5059         PVE::QemuConfig->add_unused_volume($conf, $volid);
5060         $usedpath->{$path} = 1; # avoid to add more than once (aliases)
5061     }
5062
5063     return $changes;
5064 }
5065
5066 sub rescan {
5067     my ($vmid, $nolock) = @_;
5068
5069     my $cfg = PVE::Storage::config();
5070
5071     my $volid_hash = scan_volids($cfg, $vmid);
5072
5073     my $updatefn =  sub {
5074         my ($vmid) = @_;
5075
5076         my $conf = PVE::QemuConfig->load_config($vmid);
5077
5078         PVE::QemuConfig->check_lock($conf);
5079
5080         my $vm_volids = {};
5081         foreach my $volid (keys %$volid_hash) {
5082             my $info = $volid_hash->{$volid};
5083             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5084         }
5085
5086         my $changes = update_disksize($vmid, $conf, $vm_volids);
5087
5088         PVE::QemuConfig->write_config($vmid, $conf) if $changes;
5089     };
5090
5091     if (defined($vmid)) {
5092         if ($nolock) {
5093             &$updatefn($vmid);
5094         } else {
5095             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5096         }
5097     } else {
5098         my $vmlist = config_list();
5099         foreach my $vmid (keys %$vmlist) {
5100             if ($nolock) {
5101                 &$updatefn($vmid);
5102             } else {
5103                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5104             }
5105         }
5106     }
5107 }
5108
5109 sub restore_vma_archive {
5110     my ($archive, $vmid, $user, $opts, $comp) = @_;
5111
5112     my $input = $archive eq '-' ? "<&STDIN" : undef;
5113     my $readfrom = $archive;
5114
5115     my $uncomp = '';
5116     if ($comp) {
5117         $readfrom = '-';
5118         my $qarchive = PVE::Tools::shellquote($archive);
5119         if ($comp eq 'gzip') {
5120             $uncomp = "zcat $qarchive|";
5121         } elsif ($comp eq 'lzop') {
5122             $uncomp = "lzop -d -c $qarchive|";
5123         } else {
5124             die "unknown compression method '$comp'\n";
5125         }
5126
5127     }
5128
5129     my $tmpdir = "/var/tmp/vzdumptmp$$";
5130     rmtree $tmpdir;
5131
5132     # disable interrupts (always do cleanups)
5133     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5134         warn "got interrupt - ignored\n";
5135     };
5136
5137     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5138     POSIX::mkfifo($mapfifo, 0600);
5139     my $fifofh;
5140
5141     my $openfifo = sub {
5142         open($fifofh, '>', $mapfifo) || die $!;
5143     };
5144
5145     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5146
5147     my $oldtimeout;
5148     my $timeout = 5;
5149
5150     my $devinfo = {};
5151
5152     my $rpcenv = PVE::RPCEnvironment::get();
5153
5154     my $conffile = PVE::QemuConfig->config_file($vmid);
5155     my $tmpfn = "$conffile.$$.tmp";
5156
5157     # Note: $oldconf is undef if VM does not exists
5158     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
5159     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
5160
5161     my $print_devmap = sub {
5162         my $virtdev_hash = {};
5163
5164         my $cfgfn = "$tmpdir/qemu-server.conf";
5165
5166         # we can read the config - that is already extracted
5167         my $fh = IO::File->new($cfgfn, "r") ||
5168             "unable to read qemu-server.conf - $!\n";
5169
5170         my $fwcfgfn = "$tmpdir/qemu-server.fw";
5171         if (-f $fwcfgfn) {
5172             my $pve_firewall_dir = '/etc/pve/firewall';
5173             mkdir $pve_firewall_dir; # make sure the dir exists
5174             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
5175         }
5176
5177         while (defined(my $line = <$fh>)) {
5178             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5179                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5180                 die "archive does not contain data for drive '$virtdev'\n"
5181                     if !$devinfo->{$devname};
5182                 if (defined($opts->{storage})) {
5183                     $storeid = $opts->{storage} || 'local';
5184                 } elsif (!$storeid) {
5185                     $storeid = 'local';
5186                 }
5187                 $format = 'raw' if !$format;
5188                 $devinfo->{$devname}->{devname} = $devname;
5189                 $devinfo->{$devname}->{virtdev} = $virtdev;
5190                 $devinfo->{$devname}->{format} = $format;
5191                 $devinfo->{$devname}->{storeid} = $storeid;
5192
5193                 # check permission on storage
5194                 my $pool = $opts->{pool}; # todo: do we need that?
5195                 if ($user ne 'root@pam') {
5196                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5197                 }
5198
5199                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5200             }
5201         }
5202
5203         foreach my $devname (keys %$devinfo) {
5204             die "found no device mapping information for device '$devname'\n"
5205                 if !$devinfo->{$devname}->{virtdev};
5206         }
5207
5208         my $cfg = PVE::Storage::config();
5209
5210         # create empty/temp config
5211         if ($oldconf) {
5212             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5213             foreach_drive($oldconf, sub {
5214                 my ($ds, $drive) = @_;
5215
5216                 return if drive_is_cdrom($drive);
5217
5218                 my $volid = $drive->{file};
5219
5220                 return if !$volid || $volid =~ m|^/|;
5221
5222                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5223                 return if !$path || !$owner || ($owner != $vmid);
5224
5225                 # Note: only delete disk we want to restore
5226                 # other volumes will become unused
5227                 if ($virtdev_hash->{$ds}) {
5228                     PVE::Storage::vdisk_free($cfg, $volid);
5229                 }
5230             });
5231
5232             # delete vmstate files
5233             # since after the restore we have no snapshots anymore
5234             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5235                 my $snap = $oldconf->{snapshots}->{$snapname};
5236                 if ($snap->{vmstate}) {
5237                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
5238                     if (my $err = $@) {
5239                         warn $err;
5240                     }
5241                 }
5242             }
5243         }
5244
5245         my $map = {};
5246         foreach my $virtdev (sort keys %$virtdev_hash) {
5247             my $d = $virtdev_hash->{$virtdev};
5248             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5249             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5250
5251             # test if requested format is supported
5252             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5253             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5254             $d->{format} = $defFormat if !$supported;
5255
5256             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5257                                                   $d->{format}, undef, $alloc_size);
5258             print STDERR "new volume ID is '$volid'\n";
5259             $d->{volid} = $volid;
5260             my $path = PVE::Storage::path($cfg, $volid);
5261
5262             PVE::Storage::activate_volumes($cfg,[$volid]);
5263
5264             my $write_zeros = 1;
5265             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
5266                 $write_zeros = 0;
5267             }
5268
5269             print $fifofh "format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
5270
5271             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5272             $map->{$virtdev} = $volid;
5273         }
5274
5275         $fh->seek(0, 0) || die "seek failed - $!\n";
5276
5277         my $outfd = new IO::File ($tmpfn, "w") ||
5278             die "unable to write config for VM $vmid\n";
5279
5280         my $cookie = { netcount => 0 };
5281         while (defined(my $line = <$fh>)) {
5282             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5283         }
5284
5285         $fh->close();
5286         $outfd->close();
5287     };
5288
5289     eval {
5290         # enable interrupts
5291         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5292             die "interrupted by signal\n";
5293         };
5294         local $SIG{ALRM} = sub { die "got timeout\n"; };
5295
5296         $oldtimeout = alarm($timeout);
5297
5298         my $parser = sub {
5299             my $line = shift;
5300
5301             print "$line\n";
5302
5303             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5304                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5305                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5306             } elsif ($line =~ m/^CTIME: /) {
5307                 # we correctly received the vma config, so we can disable
5308                 # the timeout now for disk allocation (set to 10 minutes, so
5309                 # that we always timeout if something goes wrong)
5310                 alarm(600);
5311                 &$print_devmap();
5312                 print $fifofh "done\n";
5313                 my $tmp = $oldtimeout || 0;
5314                 $oldtimeout = undef;
5315                 alarm($tmp);
5316                 close($fifofh);
5317             }
5318         };
5319
5320         print "restore vma archive: $cmd\n";
5321         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5322     };
5323     my $err = $@;
5324
5325     alarm($oldtimeout) if $oldtimeout;
5326
5327     my $vollist = [];
5328     foreach my $devname (keys %$devinfo) {
5329         my $volid = $devinfo->{$devname}->{volid};
5330         push @$vollist, $volid if $volid;
5331     }
5332
5333     my $cfg = PVE::Storage::config();
5334     PVE::Storage::deactivate_volumes($cfg, $vollist);
5335
5336     unlink $mapfifo;
5337
5338     if ($err) {
5339         rmtree $tmpdir;
5340         unlink $tmpfn;
5341
5342         foreach my $devname (keys %$devinfo) {
5343             my $volid = $devinfo->{$devname}->{volid};
5344             next if !$volid;
5345             eval {
5346                 if ($volid =~ m|^/|) {
5347                     unlink $volid || die 'unlink failed\n';
5348                 } else {
5349                     PVE::Storage::vdisk_free($cfg, $volid);
5350                 }
5351                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5352             };
5353             print STDERR "unable to cleanup '$volid' - $@" if $@;
5354         }
5355         die $err;
5356     }
5357
5358     rmtree $tmpdir;
5359
5360     rename($tmpfn, $conffile) ||
5361         die "unable to commit configuration file '$conffile'\n";
5362
5363     PVE::Cluster::cfs_update(); # make sure we read new file
5364
5365     eval { rescan($vmid, 1); };
5366     warn $@ if $@;
5367 }
5368
5369 sub restore_tar_archive {
5370     my ($archive, $vmid, $user, $opts) = @_;
5371
5372     if ($archive ne '-') {
5373         my $firstfile = tar_archive_read_firstfile($archive);
5374         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5375             if $firstfile ne 'qemu-server.conf';
5376     }
5377
5378     my $storecfg = PVE::Storage::config();
5379
5380     # destroy existing data - keep empty config
5381     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
5382     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5383
5384     my $tocmd = "/usr/lib/qemu-server/qmextract";
5385
5386     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5387     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5388     $tocmd .= ' --prealloc' if $opts->{prealloc};
5389     $tocmd .= ' --info' if $opts->{info};
5390
5391     # tar option "xf" does not autodetect compression when read from STDIN,
5392     # so we pipe to zcat
5393     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5394         PVE::Tools::shellquote("--to-command=$tocmd");
5395
5396     my $tmpdir = "/var/tmp/vzdumptmp$$";
5397     mkpath $tmpdir;
5398
5399     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5400     local $ENV{VZDUMP_VMID} = $vmid;
5401     local $ENV{VZDUMP_USER} = $user;
5402
5403     my $conffile = PVE::QemuConfig->config_file($vmid);
5404     my $tmpfn = "$conffile.$$.tmp";
5405
5406     # disable interrupts (always do cleanups)
5407     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5408         print STDERR "got interrupt - ignored\n";
5409     };
5410
5411     eval {
5412         # enable interrupts
5413         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5414             die "interrupted by signal\n";
5415         };
5416
5417         if ($archive eq '-') {
5418             print "extracting archive from STDIN\n";
5419             run_command($cmd, input => "<&STDIN");
5420         } else {
5421             print "extracting archive '$archive'\n";
5422             run_command($cmd);
5423         }
5424
5425         return if $opts->{info};
5426
5427         # read new mapping
5428         my $map = {};
5429         my $statfile = "$tmpdir/qmrestore.stat";
5430         if (my $fd = IO::File->new($statfile, "r")) {
5431             while (defined (my $line = <$fd>)) {
5432                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5433                     $map->{$1} = $2 if $1;
5434                 } else {
5435                     print STDERR "unable to parse line in statfile - $line\n";
5436                 }
5437             }
5438             $fd->close();
5439         }
5440
5441         my $confsrc = "$tmpdir/qemu-server.conf";
5442
5443         my $srcfd = new IO::File($confsrc, "r") ||
5444             die "unable to open file '$confsrc'\n";
5445
5446         my $outfd = new IO::File ($tmpfn, "w") ||
5447             die "unable to write config for VM $vmid\n";
5448
5449         my $cookie = { netcount => 0 };
5450         while (defined (my $line = <$srcfd>)) {
5451             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5452         }
5453
5454         $srcfd->close();
5455         $outfd->close();
5456     };
5457     my $err = $@;
5458
5459     if ($err) {
5460
5461         unlink $tmpfn;
5462
5463         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
5464
5465         die $err;
5466     }
5467
5468     rmtree $tmpdir;
5469
5470     rename $tmpfn, $conffile ||
5471         die "unable to commit configuration file '$conffile'\n";
5472
5473     PVE::Cluster::cfs_update(); # make sure we read new file
5474
5475     eval { rescan($vmid, 1); };
5476     warn $@ if $@;
5477 };
5478
5479 sub foreach_writable_storage {
5480     my ($conf, $func) = @_;
5481
5482     my $sidhash = {};
5483
5484     foreach my $ds (keys %$conf) {
5485         next if !is_valid_drivename($ds);
5486
5487         my $drive = parse_drive($ds, $conf->{$ds});
5488         next if !$drive;
5489         next if drive_is_cdrom($drive);
5490
5491         my $volid = $drive->{file};
5492
5493         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5494         $sidhash->{$sid} = $sid if $sid;
5495     }
5496
5497     foreach my $sid (sort keys %$sidhash) {
5498         &$func($sid);
5499     }
5500 }
5501
5502 sub do_snapshots_with_qemu {
5503     my ($storecfg, $volid) = @_;
5504
5505     my $storage_name = PVE::Storage::parse_volume_id($volid);
5506
5507     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
5508         && !$storecfg->{ids}->{$storage_name}->{krbd}){
5509         return 1;
5510     }
5511
5512     if ($volid =~ m/\.(qcow2|qed)$/){
5513         return 1;
5514     }
5515
5516     return undef;
5517 }
5518
5519 sub qga_check_running {
5520     my ($vmid) = @_;
5521
5522     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
5523     if ($@) {
5524         warn "Qemu Guest Agent are not running - $@";
5525         return 0;
5526     }
5527     return 1;
5528 }
5529
5530 sub template_create {
5531     my ($vmid, $conf, $disk) = @_;
5532
5533     my $storecfg = PVE::Storage::config();
5534
5535     foreach_drive($conf, sub {
5536         my ($ds, $drive) = @_;
5537
5538         return if drive_is_cdrom($drive);
5539         return if $disk && $ds ne $disk;
5540
5541         my $volid = $drive->{file};
5542         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
5543
5544         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
5545         $drive->{file} = $voliddst;
5546         $conf->{$ds} = print_drive($vmid, $drive);
5547         PVE::QemuConfig->write_config($vmid, $conf);
5548     });
5549 }
5550
5551 sub qemu_img_convert {
5552     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
5553
5554     my $storecfg = PVE::Storage::config();
5555     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
5556     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
5557
5558     if ($src_storeid && $dst_storeid) {
5559
5560         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
5561
5562         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
5563         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5564
5565         my $src_format = qemu_img_format($src_scfg, $src_volname);
5566         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
5567
5568         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
5569         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5570
5571         my $cmd = [];
5572         push @$cmd, '/usr/bin/qemu-img', 'convert', '-t', 'writeback', '-p', '-n';
5573         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
5574         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path;
5575         if ($is_zero_initialized) {
5576             push @$cmd, "zeroinit:$dst_path";
5577         } else {
5578             push @$cmd, $dst_path;
5579         }
5580
5581         my $parser = sub {
5582             my $line = shift;
5583             if($line =~ m/\((\S+)\/100\%\)/){
5584                 my $percent = $1;
5585                 my $transferred = int($size * $percent / 100);
5586                 my $remaining = $size - $transferred;
5587
5588                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
5589             }
5590
5591         };
5592
5593         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
5594         my $err = $@;
5595         die "copy failed: $err" if $err;
5596     }
5597 }
5598
5599 sub qemu_img_format {
5600     my ($scfg, $volname) = @_;
5601
5602     if ($scfg->{path} && $volname =~ m/\.(raw|cow|qcow|qcow2|qed|vmdk|cloop)$/) {
5603         return $1;
5604     } else {
5605         return "raw";
5606     }
5607 }
5608
5609 sub qemu_drive_mirror {
5610     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized) = @_;
5611
5612     my $storecfg = PVE::Storage::config();
5613     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
5614
5615     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5616
5617     my $format = qemu_img_format($dst_scfg, $dst_volname);
5618
5619     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5620
5621     my $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
5622
5623     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
5624     $opts->{format} = $format if $format;
5625
5626     print "drive mirror is starting (scanning bitmap) : this step can take some minutes/hours, depend of disk size and storage speed\n";
5627
5628     my $finish_job = sub {
5629         while (1) {
5630             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
5631             my $stat = @$stats[0];
5632             last if !$stat;
5633             sleep 1;
5634         }
5635     };
5636
5637     eval {
5638     vm_mon_cmd($vmid, "drive-mirror", %$opts);
5639         while (1) {
5640             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
5641             my $stat = @$stats[0];
5642             die "mirroring job seem to have die. Maybe do you have bad sectors?" if !$stat;
5643             die "error job is not mirroring" if $stat->{type} ne "mirror";
5644
5645             my $busy = $stat->{busy};
5646             my $ready = $stat->{ready};
5647
5648             if (my $total = $stat->{len}) {
5649                 my $transferred = $stat->{offset} || 0;
5650                 my $remaining = $total - $transferred;
5651                 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
5652
5653                 print "transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
5654             }
5655
5656
5657             if ($stat->{ready} eq 'true') {
5658
5659                 last if $vmiddst != $vmid;
5660
5661                 # try to switch the disk if source and destination are on the same guest
5662                 eval { vm_mon_cmd($vmid, "block-job-complete", device => "drive-$drive") };
5663                 if (!$@) {
5664                     &$finish_job();
5665                     last;
5666                 }
5667                 die $@ if $@ !~ m/cannot be completed/;
5668             }
5669             sleep 1;
5670         }
5671
5672
5673     };
5674     my $err = $@;
5675
5676     my $cancel_job = sub {
5677         vm_mon_cmd($vmid, "block-job-cancel", device => "drive-$drive");
5678         &$finish_job();
5679     };
5680
5681     if ($err) {
5682         eval { &$cancel_job(); };
5683         die "mirroring error: $err";
5684     }
5685
5686     if ($vmiddst != $vmid) {
5687         # if we clone a disk for a new target vm, we don't switch the disk
5688         &$cancel_job(); # so we call block-job-cancel
5689     }
5690 }
5691
5692 sub clone_disk {
5693     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
5694         $newvmid, $storage, $format, $full, $newvollist) = @_;
5695
5696     my $newvolid;
5697
5698     if (!$full) {
5699         print "create linked clone of drive $drivename ($drive->{file})\n";
5700         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
5701         push @$newvollist, $newvolid;
5702     } else {
5703         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
5704         $storeid = $storage if $storage;
5705
5706         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5707         if (!$format) {
5708             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5709             $format = qemu_img_format($scfg, $volname);
5710         }
5711
5712         # test if requested format is supported - else use default
5713         my $supported = grep { $_ eq $format } @$validFormats;
5714         $format = $defFormat if !$supported;
5715
5716         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
5717
5718         print "create full clone of drive $drivename ($drive->{file})\n";
5719         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $format, undef, ($size/1024));
5720         push @$newvollist, $newvolid;
5721
5722         PVE::Storage::activate_volumes($storecfg, $newvollist);
5723
5724         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
5725         if (!$running || $snapname) {
5726             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
5727         } else {
5728             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit);
5729         }
5730     }
5731
5732     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
5733
5734     my $disk = $drive;
5735     $disk->{format} = undef;
5736     $disk->{file} = $newvolid;
5737     $disk->{size} = $size;
5738
5739     return $disk;
5740 }
5741
5742 # this only works if VM is running
5743 sub get_current_qemu_machine {
5744     my ($vmid) = @_;
5745
5746     my $cmd = { execute => 'query-machines', arguments => {} };
5747     my $res = vm_qmp_command($vmid, $cmd);
5748
5749     my ($current, $default);
5750     foreach my $e (@$res) {
5751         $default = $e->{name} if $e->{'is-default'};
5752         $current = $e->{name} if $e->{'is-current'};
5753     }
5754
5755     # fallback to the default machine if current is not supported by qemu
5756     return $current || $default || 'pc';
5757 }
5758
5759 sub qemu_machine_feature_enabled {
5760     my ($machine, $kvmver, $version_major, $version_minor) = @_;
5761
5762     my $current_major;
5763     my $current_minor;
5764
5765     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
5766
5767         $current_major = $3;
5768         $current_minor = $4;
5769
5770     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
5771
5772         $current_major = $1;
5773         $current_minor = $2;
5774     }
5775
5776     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
5777
5778
5779 }
5780
5781 sub qemu_machine_pxe {
5782     my ($vmid, $conf, $machine) = @_;
5783
5784     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
5785
5786     foreach my $opt (keys %$conf) {
5787         next if $opt !~ m/^net(\d+)$/;
5788         my $net = PVE::QemuServer::parse_net($conf->{$opt});
5789         next if !$net;
5790         my $romfile = PVE::QemuServer::vm_mon_cmd_nocheck($vmid, 'qom-get', path => $opt, property => 'romfile');
5791         return $machine.".pxe" if $romfile =~ m/pxe/;
5792         last;
5793     }
5794
5795     return $machine;
5796 }
5797
5798 sub qemu_use_old_bios_files {
5799     my ($machine_type) = @_;
5800
5801     return if !$machine_type;
5802
5803     my $use_old_bios_files = undef;
5804
5805     if ($machine_type =~ m/^(\S+)\.pxe$/) {
5806         $machine_type = $1;
5807         $use_old_bios_files = 1;
5808     } else {
5809         my $kvmver = kvm_user_version();
5810         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
5811         # load new efi bios files on migration. So this hack is required to allow
5812         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
5813         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
5814         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
5815     }
5816
5817     return ($use_old_bios_files, $machine_type);
5818 }
5819
5820 sub lspci {
5821
5822     my $devices = {};
5823
5824     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
5825             my (undef, $id, $function) = @_;
5826             my $res = { id => $id, function => $function};
5827             push @{$devices->{$id}}, $res;
5828     });
5829
5830     return $devices;
5831 }
5832
5833 sub vm_iothreads_list {
5834     my ($vmid) = @_;
5835
5836     my $res = vm_mon_cmd($vmid, 'query-iothreads');
5837
5838     my $iothreads = {};
5839     foreach my $iothread (@$res) {
5840         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
5841     }
5842
5843     return $iothreads;
5844 }
5845
5846 sub scsihw_infos {
5847     my ($conf, $drive) = @_;
5848
5849     my $maxdev = 0;
5850
5851     if ($conf->{scsihw} && ($conf->{scsihw} =~ m/^lsi/)) {
5852         $maxdev = 7;
5853     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
5854         $maxdev = 1;
5855     } else {
5856         $maxdev = 256;
5857     }
5858
5859     my $controller = int($drive->{index} / $maxdev);
5860     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
5861
5862     return ($maxdev, $controller, $controller_prefix);
5863 }
5864
5865 # bash completion helper
5866
5867 sub complete_backup_archives {
5868     my ($cmdname, $pname, $cvalue) = @_;
5869
5870     my $cfg = PVE::Storage::config();
5871
5872     my $storeid;
5873
5874     if ($cvalue =~ m/^([^:]+):/) {
5875         $storeid = $1;
5876     }
5877
5878     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
5879
5880     my $res = [];
5881     foreach my $id (keys %$data) {
5882         foreach my $item (@{$data->{$id}}) {
5883             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
5884             push @$res, $item->{volid} if defined($item->{volid});
5885         }
5886     }
5887
5888     return $res;
5889 }
5890
5891 my $complete_vmid_full = sub {
5892     my ($running) = @_;
5893
5894     my $idlist = vmstatus();
5895
5896     my $res = [];
5897
5898     foreach my $id (keys %$idlist) {
5899         my $d = $idlist->{$id};
5900         if (defined($running)) {
5901             next if $d->{template};
5902             next if $running && $d->{status} ne 'running';
5903             next if !$running && $d->{status} eq 'running';
5904         }
5905         push @$res, $id;
5906
5907     }
5908     return $res;
5909 };
5910
5911 sub complete_vmid {
5912     return &$complete_vmid_full();
5913 }
5914
5915 sub complete_vmid_stopped {
5916     return &$complete_vmid_full(0);
5917 }
5918
5919 sub complete_vmid_running {
5920     return &$complete_vmid_full(1);
5921 }
5922
5923 sub complete_storage {
5924
5925     my $cfg = PVE::Storage::config();
5926     my $ids = $cfg->{ids};
5927
5928     my $res = [];
5929     foreach my $sid (keys %$ids) {
5930         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
5931         next if !$ids->{$sid}->{content}->{images};
5932         push @$res, $sid;
5933     }
5934
5935     return $res;
5936 }
5937
5938 1;