PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QemuConfig;
  31 use PVE::QMPClient;
  32 use PVE::RPCEnvironment;
  33 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
  34 use PVE::QemuServer::Memory;
  35 use PVE::QemuServer::USB qw(parse_usb_device);
  36 use Time::HiRes qw(gettimeofday);
  37 use File::Copy qw(copy);
  38 use URI::Escape;
  39
  40 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  41
  42 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  43
  44 # Note about locking: we use flock on the config file protect
  45 # against concurent actions.
  46 # Aditionaly, we have a 'lock' setting in the config file. This
  47 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  48 # allowed when such lock is set. But you can ignore this kind of
  49 # lock with the --skiplock flag.
  50
  51 cfs_register_file('/qemu-server/',
  52                   \&parse_vm_config,
  53                   \&write_vm_config);
  54
  55 PVE::JSONSchema::register_standard_option('skiplock', {
  56     description => "Ignore locks - only root is allowed to use this option.",
  57     type => 'boolean',
  58     optional => 1,
  59 });
  60
  61 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  62     description => "Some command save/restore state from this location.",
  63     type => 'string',
  64     maxLength => 128,
  65     optional => 1,
  66 });
  67
  68 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  69     description => "The name of the snapshot.",
  70     type => 'string', format => 'pve-configid',
  71     maxLength => 40,
  72 });
  73
  74 #no warnings 'redefine';
  75
  76 sub cgroups_write {
  77    my ($controller, $vmid, $option, $value) = @_;
  78
  79    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  80    PVE::ProcFSTools::write_proc_entry($path, $value);
  81
  82 }
  83
  84 my $nodename = PVE::INotify::nodename();
  85
  86 mkdir "/etc/pve/nodes/$nodename";
  87 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  88 mkdir $confdir;
  89
  90 my $var_run_tmpdir = "/var/run/qemu-server";
  91 mkdir $var_run_tmpdir;
  92
  93 my $lock_dir = "/var/lock/qemu-server";
  94 mkdir $lock_dir;
  95
  96 my $pcisysfs = "/sys/bus/pci";
  97
  98 my $cpu_vendor_list = {
  99     # Intel CPUs
 100     486 => 'GenuineIntel',
 101     pentium => 'GenuineIntel',
 102     pentium2  => 'GenuineIntel',
 103     pentium3  => 'GenuineIntel',
 104     coreduo => 'GenuineIntel',
 105     core2duo => 'GenuineIntel',
 106     Conroe  => 'GenuineIntel',
 107     Penryn  => 'GenuineIntel',
 108     Nehalem  => 'GenuineIntel',
 109     Westmere => 'GenuineIntel',
 110     SandyBridge => 'GenuineIntel',
 111     IvyBridge => 'GenuineIntel',
 112     Haswell => 'GenuineIntel',
 113     'Haswell-noTSX' => 'GenuineIntel',
 114     Broadwell => 'GenuineIntel',
 115     'Broadwell-noTSX' => 'GenuineIntel',
 116
 117     # AMD CPUs
 118     athlon => 'AuthenticAMD',
 119     phenom  => 'AuthenticAMD',
 120     Opteron_G1  => 'AuthenticAMD',
 121     Opteron_G2  => 'AuthenticAMD',
 122     Opteron_G3  => 'AuthenticAMD',
 123     Opteron_G4  => 'AuthenticAMD',
 124     Opteron_G5  => 'AuthenticAMD',
 125
 126     # generic types, use vendor from host node
 127     host => 'default',
 128     kvm32 => 'default',
 129     kvm64 => 'default',
 130     qemu32 => 'default',
 131     qemu64 => 'default',
 132 };
 133
 134 my $cpu_fmt = {
 135     cputype => {
 136         description => "Emulated CPU type.",
 137         type => 'string',
 138         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 139         format_description => 'cputype',
 140         default => 'kvm64',
 141         default_key => 1,
 142     },
 143     hidden => {
 144         description => "Do not identify as a KVM virtual machine.",
 145         type => 'boolean',
 146         optional => 1,
 147         default => 0
 148     },
 149 };
 150
 151 my $watchdog_fmt = {
 152     model => {
 153         default_key => 1,
 154         type => 'string',
 155         enum => [qw(i6300esb ib700)],
 156         description => "Watchdog type to emulate.",
 157         default => 'i6300esb',
 158         optional => 1,
 159     },
 160     action => {
 161         type => 'string',
 162         enum => [qw(reset shutdown poweroff pause debug none)],
 163         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 164         optional => 1,
 165     },
 166 };
 167 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 168
 169 my $confdesc = {
 170     onboot => {
 171         optional => 1,
 172         type => 'boolean',
 173         description => "Specifies whether a VM will be started during system bootup.",
 174         default => 0,
 175     },
 176     autostart => {
 177         optional => 1,
 178         type => 'boolean',
 179         description => "Automatic restart after crash (currently ignored).",
 180         default => 0,
 181     },
 182     hotplug => {
 183         optional => 1,
 184         type => 'string', format => 'pve-hotplug-features',
 185         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 186         default => 'network,disk,usb',
 187     },
 188     reboot => {
 189         optional => 1,
 190         type => 'boolean',
 191         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 192         default => 1,
 193     },
 194     lock => {
 195         optional => 1,
 196         type => 'string',
 197         description => "Lock/unlock the VM.",
 198         enum => [qw(migrate backup snapshot rollback)],
 199     },
 200     cpulimit => {
 201         optional => 1,
 202         type => 'number',
 203         description => "Limit of CPU usage.",
 204         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 205         minimum => 0,
 206         maximum => 128,
 207         default => 0,
 208     },
 209     cpuunits => {
 210         optional => 1,
 211         type => 'integer',
 212         description => "CPU weight for a VM.",
 213         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 214         minimum => 0,
 215         maximum => 500000,
 216         default => 1000,
 217     },
 218     memory => {
 219         optional => 1,
 220         type => 'integer',
 221         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 222         minimum => 16,
 223         default => 512,
 224     },
 225     balloon => {
 226         optional => 1,
 227         type => 'integer',
 228         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 229         minimum => 0,
 230     },
 231     shares => {
 232         optional => 1,
 233         type => 'integer',
 234         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 235         minimum => 0,
 236         maximum => 50000,
 237         default => 1000,
 238     },
 239     keyboard => {
 240         optional => 1,
 241         type => 'string',
 242         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.conf' configuration file.",
 243         enum => PVE::Tools::kvmkeymaplist(),
 244         default => 'en-us',
 245     },
 246     name => {
 247         optional => 1,
 248         type => 'string', format => 'dns-name',
 249         description => "Set a name for the VM. Only used on the configuration web interface.",
 250     },
 251     scsihw => {
 252         optional => 1,
 253         type => 'string',
 254         description => "SCSI controller model",
 255         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 256         default => 'lsi',
 257     },
 258     description => {
 259         optional => 1,
 260         type => 'string',
 261         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 262     },
 263     ostype => {
 264         optional => 1,
 265         type => 'string',
 266         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 l24 l26 solaris)],
 267         description => "Specify guest operating system.",
 268         verbose_description => <<EODESC,
 269 Specify guest operating system. This is used to enable special
 270 optimization/features for specific operating systems:
 271
 272 [horizontal]
 273 other;; unspecified OS
 274 wxp;; Microsoft Windows XP
 275 w2k;; Microsoft Windows 2000
 276 w2k3;; Microsoft Windows 2003
 277 w2k8;; Microsoft Windows 2008
 278 wvista;; Microsoft Windows Vista
 279 win7;; Microsoft Windows 7
 280 win8;; Microsoft Windows 8/2012
 281 l24;; Linux 2.4 Kernel
 282 l26;; Linux 2.6/3.X Kernel
 283 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 284 EODESC
 285     },
 286     boot => {
 287         optional => 1,
 288         type => 'string',
 289         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 290         pattern => '[acdn]{1,4}',
 291         default => 'cdn',
 292     },
 293     bootdisk => {
 294         optional => 1,
 295         type => 'string', format => 'pve-qm-bootdisk',
 296         description => "Enable booting from specified disk.",
 297         pattern => '(ide|sata|scsi|virtio)\d+',
 298     },
 299     smp => {
 300         optional => 1,
 301         type => 'integer',
 302         description => "The number of CPUs. Please use option -sockets instead.",
 303         minimum => 1,
 304         default => 1,
 305     },
 306     sockets => {
 307         optional => 1,
 308         type => 'integer',
 309         description => "The number of CPU sockets.",
 310         minimum => 1,
 311         default => 1,
 312     },
 313     cores => {
 314         optional => 1,
 315         type => 'integer',
 316         description => "The number of cores per socket.",
 317         minimum => 1,
 318         default => 1,
 319     },
 320     numa => {
 321         optional => 1,
 322         type => 'boolean',
 323         description => "Enable/disable NUMA.",
 324         default => 0,
 325     },
 326     hugepages => {
 327         optional => 1,
 328         type => 'string',
 329         description => "Enable/disable hugepages memory.",
 330         enum => [qw(any 2 1024)],
 331     },
 332     vcpus => {
 333         optional => 1,
 334         type => 'integer',
 335         description => "Number of hotplugged vcpus.",
 336         minimum => 1,
 337         default => 0,
 338     },
 339     acpi => {
 340         optional => 1,
 341         type => 'boolean',
 342         description => "Enable/disable ACPI.",
 343         default => 1,
 344     },
 345     agent => {
 346         optional => 1,
 347         type => 'boolean',
 348         description => "Enable/disable Qemu GuestAgent.",
 349         default => 0,
 350     },
 351     kvm => {
 352         optional => 1,
 353         type => 'boolean',
 354         description => "Enable/disable KVM hardware virtualization.",
 355         default => 1,
 356     },
 357     tdf => {
 358         optional => 1,
 359         type => 'boolean',
 360         description => "Enable/disable time drift fix.",
 361         default => 0,
 362     },
 363     localtime => {
 364         optional => 1,
 365         type => 'boolean',
 366         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 367     },
 368     freeze => {
 369         optional => 1,
 370         type => 'boolean',
 371         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 372     },
 373     vga => {
 374         optional => 1,
 375         type => 'string',
 376         description => "Select the VGA type.",
 377         verbose_description => "Select the VGA type. If you want to use high resolution" .
 378             " modes (>= 1280x1024x16) then you should use the options " .
 379             "'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and " .
 380             "'cirrus' for other OS types. The 'qxl' option enables the SPICE " .
 381             "display sever. For win* OS you can select how many independent " .
 382             "displays you want, Linux guests can add displays them self. " .
 383             "You can also run without any graphic card, using a serial device" .
 384             " as terminal.",
 385         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 386     },
 387     watchdog => {
 388         optional => 1,
 389         type => 'string', format => 'pve-qm-watchdog',
 390         description => "Create a virtual hardware watchdog device.",
 391         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 392             " (by a guest action), the watchdog must be periodically polled " .
 393             "by an agent inside the guest or else the watchdog will reset " .
 394             "the guest (or execute the respective action specified)",
 395     },
 396     startdate => {
 397         optional => 1,
 398         type => 'string',
 399         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 400         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 401         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 402         default => 'now',
 403     },
 404     startup =>  get_standard_option('pve-startup-order'),
 405     template => {
 406         optional => 1,
 407         type => 'boolean',
 408         description => "Enable/disable Template.",
 409         default => 0,
 410     },
 411     args => {
 412         optional => 1,
 413         type => 'string',
 414         description => "Arbitrary arguments passed to kvm.",
 415         verbose_description => <<EODESCR,
 416 Arbitrary arguments passed to kvm, for example:
 417
 418 args: -no-reboot -no-hpet
 419
 420 NOTE: this option is for experts only.
 421 EODESCR
 422     },
 423     tablet => {
 424         optional => 1,
 425         type => 'boolean',
 426         default => 1,
 427         description => "Enable/disable the USB tablet device.",
 428         verbose_description => "Enable/disable the USB tablet device. This device is " .
 429             "usually needed to allow absolute mouse positioning with VNC. " .
 430             "Else the mouse runs out of sync with normal VNC clients. " .
 431             "If you're running lots of console-only guests on one host, " .
 432             "you may consider disabling this to save some context switches. " .
 433             "This is turned off by default if you use spice (-vga=qxl).",
 434     },
 435     migrate_speed => {
 436         optional => 1,
 437         type => 'integer',
 438         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 439         minimum => 0,
 440         default => 0,
 441     },
 442     migrate_downtime => {
 443         optional => 1,
 444         type => 'number',
 445         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 446         minimum => 0,
 447         default => 0.1,
 448     },
 449     cdrom => {
 450         optional => 1,
 451         type => 'string', format => 'pve-qm-drive',
 452         typetext => 'volume',
 453         description => "This is an alias for option -ide2",
 454     },
 455     cpu => {
 456         optional => 1,
 457         description => "Emulated CPU type.",
 458         type => 'string',
 459         format => $cpu_fmt,
 460     },
 461     parent => get_standard_option('pve-snapshot-name', {
 462         optional => 1,
 463         description => "Parent snapshot name. This is used internally, and should not be modified.",
 464     }),
 465     snaptime => {
 466         optional => 1,
 467         description => "Timestamp for snapshots.",
 468         type => 'integer',
 469         minimum => 0,
 470     },
 471     vmstate => {
 472         optional => 1,
 473         type => 'string', format => 'pve-volume-id',
 474         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 475     },
 476     machine => {
 477         description => "Specific the Qemu machine type.",
 478         type => 'string',
 479         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
 480         maxLength => 40,
 481         optional => 1,
 482     },
 483     smbios1 => {
 484         description => "Specify SMBIOS type 1 fields.",
 485         type => 'string', format => 'pve-qm-smbios1',
 486         maxLength => 256,
 487         optional => 1,
 488     },
 489     protection => {
 490         optional => 1,
 491         type => 'boolean',
 492         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 493         default => 0,
 494     },
 495     bios => {
 496         optional => 1,
 497         type => 'string',
 498         enum => [ qw(seabios ovmf) ],
 499         description => "Select BIOS implementation.",
 500         default => 'seabios',
 501     },
 502 };
 503
 504 # what about other qemu settings ?
 505 #cpu => 'string',
 506 #machine => 'string',
 507 #fda => 'file',
 508 #fdb => 'file',
 509 #mtdblock => 'file',
 510 #sd => 'file',
 511 #pflash => 'file',
 512 #snapshot => 'bool',
 513 #bootp => 'file',
 514 ##tftp => 'dir',
 515 ##smb => 'dir',
 516 #kernel => 'file',
 517 #append => 'string',
 518 #initrd => 'file',
 519 ##soundhw => 'string',
 520
 521 while (my ($k, $v) = each %$confdesc) {
 522     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 523 }
 524
 525 my $MAX_IDE_DISKS = 4;
 526 my $MAX_SCSI_DISKS = 14;
 527 my $MAX_VIRTIO_DISKS = 16;
 528 my $MAX_SATA_DISKS = 6;
 529 my $MAX_USB_DEVICES = 5;
 530 my $MAX_NETS = 32;
 531 my $MAX_UNUSED_DISKS = 8;
 532 my $MAX_HOSTPCI_DEVICES = 4;
 533 my $MAX_SERIAL_PORTS = 4;
 534 my $MAX_PARALLEL_PORTS = 3;
 535 my $MAX_NUMA = 8;
 536
 537 my $numa_fmt = {
 538     cpus => {
 539         type => "string",
 540         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 541         description => "CPUs accessing this NUMA node.",
 542         format_description => "id[-id];...",
 543     },
 544     memory => {
 545         type => "number",
 546         description => "Amount of memory this NUMA node provides.",
 547         optional => 1,
 548     },
 549     hostnodes => {
 550         type => "string",
 551         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 552         description => "Host NUMA nodes to use.",
 553         format_description => "id[-id];...",
 554         optional => 1,
 555     },
 556     policy => {
 557         type => 'string',
 558         enum => [qw(preferred bind interleave)],
 559         description => "NUMA allocation policy.",
 560         optional => 1,
 561     },
 562 };
 563 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 564 my $numadesc = {
 565     optional => 1,
 566     type => 'string', format => $numa_fmt,
 567     description => "NUMA topology.",
 568 };
 569 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 570
 571 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 572     $confdesc->{"numa$i"} = $numadesc;
 573 }
 574
 575 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 576                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 577                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 578 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 579
 580 my $net_fmt_bridge_descr = <<__EOD__;
 581 Bridge to attach the network device to. The Proxmox VE standard bridge
 582 is called 'vmbr0'.
 583
 584 If you do not specify a bridge, we create a kvm user (NATed) network
 585 device, which provides DHCP and DNS services. The following addresses
 586 are used:
 587
 588  10.0.2.2   Gateway
 589  10.0.2.3   DNS Server
 590  10.0.2.4   SMB Server
 591
 592 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 593 __EOD__
 594
 595 my $net_fmt = {
 596     macaddr => {
 597         type => 'string',
 598         pattern => qr/[0-9a-f]{2}(?::[0-9a-f]{2}){5}/i,
 599         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 600         format_description => "XX:XX:XX:XX:XX:XX",
 601         optional => 1,
 602     },
 603     model => {
 604         type => 'string',
 605         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 606         format_description => 'model',
 607         enum => $nic_model_list,
 608         default_key => 1,
 609     },
 610     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 611     bridge => {
 612         type => 'string',
 613         description => $net_fmt_bridge_descr,
 614         format_description => 'bridge',
 615         optional => 1,
 616     },
 617     queues => {
 618         type => 'integer',
 619         minimum => 0, maximum => 16,
 620         description => 'Number of packet queues to be used on the device.',
 621         optional => 1,
 622     },
 623     rate => {
 624         type => 'number',
 625         minimum => 0,
 626         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 627         optional => 1,
 628     },
 629     tag => {
 630         type => 'integer',
 631         minimum => 1, maximum => 4094,
 632         description => 'VLAN tag to apply to packets on this interface.',
 633         optional => 1,
 634     },
 635     trunks => {
 636         type => 'string',
 637         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 638         description => 'VLAN trunks to pass through this interface.',
 639         format_description => 'vlanid[;vlanid...]',
 640         optional => 1,
 641     },
 642     firewall => {
 643         type => 'boolean',
 644         description => 'Whether this interface should be protected by the firewall.',
 645         optional => 1,
 646     },
 647     link_down => {
 648         type => 'boolean',
 649         description => 'Whether this interface should be disconnected (like pulling the plug).',
 650         optional => 1,
 651     },
 652 };
 653
 654 my $netdesc = {
 655     optional => 1,
 656     type => 'string', format => $net_fmt,
 657     description => "Specify network devices.",
 658 };
 659
 660 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 661
 662 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 663     $confdesc->{"net$i"} = $netdesc;
 664 }
 665
 666 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 667 sub verify_volume_id_or_qm_path {
 668     my ($volid, $noerr) = @_;
 669
 670     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 671         return $volid;
 672     }
 673
 674     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 675     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 676     if ($@) {
 677         return undef if $noerr;
 678         die $@;
 679     }
 680     return $volid;
 681 }
 682
 683 my $drivename_hash;
 684
 685 my %drivedesc_base = (
 686     volume => { alias => 'file' },
 687     file => {
 688         type => 'string',
 689         format => 'pve-volume-id-or-qm-path',
 690         default_key => 1,
 691         format_description => 'volume',
 692         description => "The drive's backing volume.",
 693     },
 694     media => {
 695         type => 'string',
 696         enum => [qw(cdrom disk)],
 697         description => "The drive's media type.",
 698         default => 'disk',
 699         optional => 1
 700     },
 701     cyls => {
 702         type => 'integer',
 703         description => "Force the drive's physical geometry to have a specific cylinder count.",
 704         optional => 1
 705     },
 706     heads => {
 707         type => 'integer',
 708         description => "Force the drive's physical geometry to have a specific head count.",
 709         optional => 1
 710     },
 711     secs => {
 712         type => 'integer',
 713         description => "Force the drive's physical geometry to have a specific sector count.",
 714         optional => 1
 715     },
 716     trans => {
 717         type => 'string',
 718         enum => [qw(none lba auto)],
 719         description => "Force disk geometry bios translation mode.",
 720         optional => 1,
 721     },
 722     snapshot => {
 723         type => 'boolean',
 724         description => "Whether the drive should be included when making snapshots.",
 725         optional => 1,
 726     },
 727     cache => {
 728         type => 'string',
 729         enum => [qw(none writethrough writeback unsafe directsync)],
 730         description => "The drive's cache mode",
 731         optional => 1,
 732     },
 733     format => {
 734         type => 'string',
 735         format_description => 'image format',
 736         enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
 737         description => "The drive's backing file's data format.",
 738         optional => 1,
 739     },
 740     size => {
 741         type => 'string',
 742         format => 'disk-size',
 743         format_description => 'DiskSize',
 744         description => "Disk size. This is purely informational and has no effect.",
 745         optional => 1,
 746     },
 747     backup => {
 748         type => 'boolean',
 749         description => "Whether the drive should be included when making backups.",
 750         optional => 1,
 751     },
 752     werror => {
 753         type => 'string',
 754         enum => [qw(enospc ignore report stop)],
 755         description => 'Write error action.',
 756         optional => 1,
 757     },
 758     aio => {
 759         type => 'string',
 760         enum => [qw(native threads)],
 761         description => 'AIO type to use.',
 762         optional => 1,
 763     },
 764     discard => {
 765         type => 'string',
 766         enum => [qw(ignore on)],
 767         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 768         optional => 1,
 769     },
 770     detect_zeroes => {
 771         type => 'boolean',
 772         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 773         optional => 1,
 774     },
 775     serial => {
 776         type => 'string',
 777         format => 'urlencoded',
 778         format_description => 'serial',
 779         maxLength => 20*3, # *3 since it's %xx url enoded
 780         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
 781         optional => 1,
 782     }
 783 );
 784
 785 my %rerror_fmt = (
 786     rerror => {
 787         type => 'string',
 788         enum => [qw(ignore report stop)],
 789         description => 'Read error action.',
 790         optional => 1,
 791     },
 792 );
 793
 794 my %iothread_fmt = ( iothread => {
 795         type => 'boolean',
 796         description => "Whether to use iothreads for this drive",
 797         optional => 1,
 798 });
 799
 800 my %model_fmt = (
 801     model => {
 802         type => 'string',
 803         format => 'urlencoded',
 804         format_description => 'model',
 805         maxLength => 40*3, # *3 since it's %xx url enoded
 806         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
 807         optional => 1,
 808     },
 809 );
 810
 811 my %queues_fmt = (
 812     queues => {
 813         type => 'integer',
 814         description => "Number of queues.",
 815         minimum => 2,
 816         optional => 1
 817     }
 818 );
 819
 820 my $add_throttle_desc = sub {
 821     my ($key, $type, $what, $unit, $longunit) = @_;
 822     $drivedesc_base{$key} = {
 823         type => $type,
 824         format_description => $unit,
 825         description => "Maximum $what speed in $longunit per second.",
 826         optional => 1,
 827     };
 828 };
 829 # throughput: (leaky bucket)
 830 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes');
 831 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes');
 832 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes');
 833 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes');
 834 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes');
 835 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes');
 836 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations');
 837 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations');
 838 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations');
 839
 840 # pools: (pool of IO before throttling starts taking effect)
 841 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes');
 842 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes');
 843 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes');
 844 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations');
 845 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations');
 846 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations');
 847
 848 my $ide_fmt = {
 849     %drivedesc_base,
 850     %rerror_fmt,
 851     %model_fmt,
 852 };
 853
 854 my $idedesc = {
 855     optional => 1,
 856     type => 'string', format => $ide_fmt,
 857     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
 858 };
 859 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
 860
 861 my $scsi_fmt = {
 862     %drivedesc_base,
 863     %iothread_fmt,
 864     %queues_fmt,
 865 };
 866 my $scsidesc = {
 867     optional => 1,
 868     type => 'string', format => $scsi_fmt,
 869     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
 870 };
 871 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 872
 873 my $sata_fmt = {
 874     %drivedesc_base,
 875     %rerror_fmt,
 876 };
 877 my $satadesc = {
 878     optional => 1,
 879     type => 'string', format => $sata_fmt,
 880     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
 881 };
 882 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
 883
 884 my $virtio_fmt = {
 885     %drivedesc_base,
 886     %iothread_fmt,
 887     %rerror_fmt,
 888 };
 889 my $virtiodesc = {
 890     optional => 1,
 891     type => 'string', format => $virtio_fmt,
 892     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
 893 };
 894 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
 895
 896 my $alldrive_fmt = {
 897     %drivedesc_base,
 898     %rerror_fmt,
 899     %iothread_fmt,
 900     %model_fmt,
 901     %queues_fmt,
 902 };
 903
 904 my $usb_fmt = {
 905     host => {
 906         default_key => 1,
 907         type => 'string', format => 'pve-qm-usb-device',
 908         format_description => 'HOSTUSBDEVICE|spice',
 909         description => <<EODESCR,
 910 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
 911
 912  'bus-port(.port)*' (decimal numbers) or
 913  'vendor_id:product_id' (hexadeciaml numbers) or
 914  'spice'
 915
 916 You can use the 'lsusb -t' command to list existing usb devices.
 917
 918 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 919
 920 The value 'spice' can be used to add a usb redirection devices for spice.
 921 EODESCR
 922     },
 923     usb3 => {
 924         optional => 1,
 925         type => 'boolean',
 926         description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
 927         default => 0,
 928     },
 929 };
 930
 931 my $usbdesc = {
 932     optional => 1,
 933     type => 'string', format => $usb_fmt,
 934     description => "Configure an USB device (n is 0 to 4).",
 935 };
 936 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
 937
 938 # NOTE: the match-groups of this regex are used in parse_hostpci
 939 my $PCIRE = qr/([a-f0-9]{2}:[a-f0-9]{2})(?:\.([a-f0-9]))?/;
 940 my $hostpci_fmt = {
 941     host => {
 942         default_key => 1,
 943         type => 'string',
 944         pattern => qr/$PCIRE(;$PCIRE)*/,
 945         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
 946         description => <<EODESCR,
 947 Host PCI device pass through. The PCI ID of a host's PCI device or a list
 948 of PCI virtual functions of the host. HOSTPCIID syntax is:
 949
 950 'bus:dev.func' (hexadecimal numbers)
 951
 952 You can us the 'lspci' command to list existing PCI devices.
 953 EODESCR
 954     },
 955     rombar => {
 956         type => 'boolean',
 957         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
 958         optional => 1,
 959         default => 1,
 960     },
 961     pcie => {
 962         type => 'boolean',
 963         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
 964         optional => 1,
 965         default => 0,
 966     },
 967     'x-vga' => {
 968         type => 'boolean',
 969         description =>  "Enable vfio-vga device support.",
 970         optional => 1,
 971         default => 0,
 972     },
 973 };
 974 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
 975
 976 my $hostpcidesc = {
 977         optional => 1,
 978         type => 'string', format => 'pve-qm-hostpci',
 979         description => "Map host PCI devices into guest.",
 980         verbose_description =>  <<EODESCR,
 981 Map host PCI devices into guest.
 982
 983 NOTE: This option allows direct access to host hardware. So it is no longer
 984 possible to migrate such machines - use with special care.
 985
 986 CAUTION: Experimental! User reported problems with this option.
 987 EODESCR
 988 };
 989 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
 990
 991 my $serialdesc = {
 992         optional => 1,
 993         type => 'string',
 994         pattern => '(/dev/.+|socket)',
 995         description =>  "Create a serial device inside the VM (n is 0 to 3)",
 996         verbose_description =>  <<EODESCR,
 997 Create a serial device inside the VM (n is 0 to 3), and pass through a
 998 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
 999 host side (use 'qm terminal' to open a terminal connection).
1000
1001 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1002
1003 CAUTION: Experimental! User reported problems with this option.
1004 EODESCR
1005 };
1006
1007 my $paralleldesc= {
1008         optional => 1,
1009         type => 'string',
1010         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1011         description =>  "Map host parallel devices (n is 0 to 2).",
1012         verbose_description =>  <<EODESCR,
1013 Map host parallel devices (n is 0 to 2).
1014
1015 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1016
1017 CAUTION: Experimental! User reported problems with this option.
1018 EODESCR
1019 };
1020
1021 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1022     $confdesc->{"parallel$i"} = $paralleldesc;
1023 }
1024
1025 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1026     $confdesc->{"serial$i"} = $serialdesc;
1027 }
1028
1029 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1030     $confdesc->{"hostpci$i"} = $hostpcidesc;
1031 }
1032
1033 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1034     $drivename_hash->{"ide$i"} = 1;
1035     $confdesc->{"ide$i"} = $idedesc;
1036 }
1037
1038 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1039     $drivename_hash->{"sata$i"} = 1;
1040     $confdesc->{"sata$i"} = $satadesc;
1041 }
1042
1043 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1044     $drivename_hash->{"scsi$i"} = 1;
1045     $confdesc->{"scsi$i"} = $scsidesc ;
1046 }
1047
1048 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1049     $drivename_hash->{"virtio$i"} = 1;
1050     $confdesc->{"virtio$i"} = $virtiodesc;
1051 }
1052
1053 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1054     $confdesc->{"usb$i"} = $usbdesc;
1055 }
1056
1057 my $unuseddesc = {
1058     optional => 1,
1059     type => 'string', format => 'pve-volume-id',
1060     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1061 };
1062
1063 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1064     $confdesc->{"unused$i"} = $unuseddesc;
1065 }
1066
1067 my $kvm_api_version = 0;
1068
1069 sub kvm_version {
1070
1071     return $kvm_api_version if $kvm_api_version;
1072
1073     my $fh = IO::File->new("</dev/kvm") ||
1074         return 0;
1075
1076     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
1077         $kvm_api_version = $v;
1078     }
1079
1080     $fh->close();
1081
1082     return  $kvm_api_version;
1083 }
1084
1085 my $kvm_user_version;
1086
1087 sub kvm_user_version {
1088
1089     return $kvm_user_version if $kvm_user_version;
1090
1091     $kvm_user_version = 'unknown';
1092
1093     my $code = sub {
1094         my $line = shift;
1095         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1096             $kvm_user_version = $2;
1097         }
1098     };
1099
1100     eval { run_command("kvm -version", outfunc => $code); };
1101     warn $@ if $@;
1102
1103     return $kvm_user_version;
1104
1105 }
1106
1107 my $kernel_has_vhost_net = -c '/dev/vhost-net';
1108
1109 sub valid_drive_names {
1110     # order is important - used to autoselect boot disk
1111     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1112             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1113             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1114             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))));
1115 }
1116
1117 sub is_valid_drivename {
1118     my $dev = shift;
1119
1120     return defined($drivename_hash->{$dev});
1121 }
1122
1123 sub option_exists {
1124     my $key = shift;
1125     return defined($confdesc->{$key});
1126 }
1127
1128 sub nic_models {
1129     return $nic_model_list;
1130 }
1131
1132 sub os_list_description {
1133
1134     return {
1135         other => 'Other',
1136         wxp => 'Windows XP',
1137         w2k => 'Windows 2000',
1138         w2k3 =>, 'Windows 2003',
1139         w2k8 => 'Windows 2008',
1140         wvista => 'Windows Vista',
1141         win7 => 'Windows 7',
1142         win8 => 'Windows 8/2012',
1143         l24 => 'Linux 2.4',
1144         l26 => 'Linux 2.6',
1145     };
1146 }
1147
1148 my $cdrom_path;
1149
1150 sub get_cdrom_path {
1151
1152     return  $cdrom_path if $cdrom_path;
1153
1154     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1155     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1156     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1157 }
1158
1159 sub get_iso_path {
1160     my ($storecfg, $vmid, $cdrom) = @_;
1161
1162     if ($cdrom eq 'cdrom') {
1163         return get_cdrom_path();
1164     } elsif ($cdrom eq 'none') {
1165         return '';
1166     } elsif ($cdrom =~ m|^/|) {
1167         return $cdrom;
1168     } else {
1169         return PVE::Storage::path($storecfg, $cdrom);
1170     }
1171 }
1172
1173 # try to convert old style file names to volume IDs
1174 sub filename_to_volume_id {
1175     my ($vmid, $file, $media) = @_;
1176
1177     if (!($file eq 'none' || $file eq 'cdrom' ||
1178           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1179
1180         return undef if $file =~ m|/|;
1181
1182         if ($media && $media eq 'cdrom') {
1183             $file = "local:iso/$file";
1184         } else {
1185             $file = "local:$vmid/$file";
1186         }
1187     }
1188
1189     return $file;
1190 }
1191
1192 sub verify_media_type {
1193     my ($opt, $vtype, $media) = @_;
1194
1195     return if !$media;
1196
1197     my $etype;
1198     if ($media eq 'disk') {
1199         $etype = 'images';
1200     } elsif ($media eq 'cdrom') {
1201         $etype = 'iso';
1202     } else {
1203         die "internal error";
1204     }
1205
1206     return if ($vtype eq $etype);
1207
1208     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1209 }
1210
1211 sub cleanup_drive_path {
1212     my ($opt, $storecfg, $drive) = @_;
1213
1214     # try to convert filesystem paths to volume IDs
1215
1216     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1217         ($drive->{file} !~ m|^/dev/.+|) &&
1218         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1219         ($drive->{file} !~ m/^\d+$/)) {
1220         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1221         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1222         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1223         verify_media_type($opt, $vtype, $drive->{media});
1224         $drive->{file} = $volid;
1225     }
1226
1227     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1228 }
1229
1230 sub parse_hotplug_features {
1231     my ($data) = @_;
1232
1233     my $res = {};
1234
1235     return $res if $data eq '0';
1236
1237     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1238
1239     foreach my $feature (PVE::Tools::split_list($data)) {
1240         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1241             $res->{$1} = 1;
1242         } else {
1243             die "invalid hotplug feature '$feature'\n";
1244         }
1245     }
1246     return $res;
1247 }
1248
1249 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1250 sub pve_verify_hotplug_features {
1251     my ($value, $noerr) = @_;
1252
1253     return $value if parse_hotplug_features($value);
1254
1255     return undef if $noerr;
1256
1257     die "unable to parse hotplug option\n";
1258 }
1259
1260 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1261 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1262 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1263 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1264 #        [,iothread=on][,serial=serial][,model=model]
1265
1266 sub parse_drive {
1267     my ($key, $data) = @_;
1268
1269     my ($interface, $index);
1270
1271     if ($key =~ m/^([^\d]+)(\d+)$/) {
1272         $interface = $1;
1273         $index = $2;
1274     } else {
1275         return undef;
1276     }
1277
1278     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1279                                      : $confdesc->{$key}->{format};
1280     if (!$desc) {
1281         warn "invalid drive key: $key\n";
1282         return undef;
1283     }
1284     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1285     return undef if !$res;
1286     $res->{interface} = $interface;
1287     $res->{index} = $index;
1288
1289     my $error = 0;
1290     foreach my $opt (qw(bps bps_rd bps_wr)) {
1291         if (my $bps = defined(delete $res->{$opt})) {
1292             if (defined($res->{"m$opt"})) {
1293                 warn "both $opt and m$opt specified\n";
1294                 ++$error;
1295                 next;
1296             }
1297             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1298         }
1299     }
1300     return undef if $error;
1301
1302     return undef if $res->{mbps_rd} && $res->{mbps};
1303     return undef if $res->{mbps_wr} && $res->{mbps};
1304     return undef if $res->{iops_rd} && $res->{iops};
1305     return undef if $res->{iops_wr} && $res->{iops};
1306
1307     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1308         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1309         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1310         return undef if $res->{interface} eq 'virtio';
1311     }
1312
1313     if (my $size = $res->{size}) {
1314         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1315     }
1316
1317     return $res;
1318 }
1319
1320 sub print_drive {
1321     my ($vmid, $drive) = @_;
1322     my $data = { %$drive };
1323     delete $data->{$_} for qw(index interface);
1324     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1325 }
1326
1327 sub scsi_inquiry {
1328     my($fh, $noerr) = @_;
1329
1330     my $SG_IO = 0x2285;
1331     my $SG_GET_VERSION_NUM = 0x2282;
1332
1333     my $versionbuf = "\x00" x 8;
1334     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1335     if (!$ret) {
1336         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1337         return undef;
1338     }
1339     my $version = unpack("I", $versionbuf);
1340     if ($version < 30000) {
1341         die "scsi generic interface too old\n"  if !$noerr;
1342         return undef;
1343     }
1344
1345     my $buf = "\x00" x 36;
1346     my $sensebuf = "\x00" x 8;
1347     my $cmd = pack("C x3 C x1", 0x12, 36);
1348
1349     # see /usr/include/scsi/sg.h
1350     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1351
1352     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1353                       length($sensebuf), 0, length($buf), $buf,
1354                       $cmd, $sensebuf, 6000);
1355
1356     $ret = ioctl($fh, $SG_IO, $packet);
1357     if (!$ret) {
1358         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1359         return undef;
1360     }
1361
1362     my @res = unpack($sg_io_hdr_t, $packet);
1363     if ($res[17] || $res[18]) {
1364         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1365         return undef;
1366     }
1367
1368     my $res = {};
1369     (my $byte0, my $byte1, $res->{vendor},
1370      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1371
1372     $res->{removable} = $byte1 & 128 ? 1 : 0;
1373     $res->{type} = $byte0 & 31;
1374
1375     return $res;
1376 }
1377
1378 sub path_is_scsi {
1379     my ($path) = @_;
1380
1381     my $fh = IO::File->new("+<$path") || return undef;
1382     my $res = scsi_inquiry($fh, 1);
1383     close($fh);
1384
1385     return $res;
1386 }
1387
1388 sub machine_type_is_q35 {
1389     my ($conf) = @_;
1390
1391     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1392 }
1393
1394 sub print_tabletdevice_full {
1395     my ($conf) = @_;
1396
1397     my $q35 = machine_type_is_q35($conf);
1398
1399     # we use uhci for old VMs because tablet driver was buggy in older qemu
1400     my $usbbus = $q35 ? "ehci" : "uhci";
1401
1402     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1403 }
1404
1405 sub print_drivedevice_full {
1406     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1407
1408     my $device = '';
1409     my $maxdev = 0;
1410
1411     if ($drive->{interface} eq 'virtio') {
1412         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1413         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1414         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1415     } elsif ($drive->{interface} eq 'scsi') {
1416
1417         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1418         my $unit = $drive->{index} % $maxdev;
1419         my $devicetype = 'hd';
1420         my $path = '';
1421         if (drive_is_cdrom($drive)) {
1422             $devicetype = 'cd';
1423         } else {
1424             if ($drive->{file} =~ m|^/|) {
1425                 $path = $drive->{file};
1426                 if (my $info = path_is_scsi($path)) {
1427                     if ($info->{type} == 0) {
1428                         $devicetype = 'block';
1429                     } elsif ($info->{type} == 1) { # tape
1430                         $devicetype = 'generic';
1431                     }
1432                 }
1433             } else {
1434                  $path = PVE::Storage::path($storecfg, $drive->{file});
1435             }
1436
1437             if($path =~ m/^iscsi\:\/\//){
1438                 $devicetype = 'generic';
1439             }
1440         }
1441
1442         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1443             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1444         } else {
1445             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1446         }
1447
1448     } elsif ($drive->{interface} eq 'ide'){
1449         $maxdev = 2;
1450         my $controller = int($drive->{index} / $maxdev);
1451         my $unit = $drive->{index} % $maxdev;
1452         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1453
1454         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1455         if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
1456             $model = URI::Escape::uri_unescape($model);
1457             $device .= ",model=$model";
1458         }
1459     } elsif ($drive->{interface} eq 'sata'){
1460         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1461         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1462         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1463     } elsif ($drive->{interface} eq 'usb') {
1464         die "implement me";
1465         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1466     } else {
1467         die "unsupported interface type";
1468     }
1469
1470     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1471
1472     return $device;
1473 }
1474
1475 sub get_initiator_name {
1476     my $initiator;
1477
1478     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1479     while (defined(my $line = <$fh>)) {
1480         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1481         $initiator = $1;
1482         last;
1483     }
1484     $fh->close();
1485
1486     return $initiator;
1487 }
1488
1489 sub print_drive_full {
1490     my ($storecfg, $vmid, $drive) = @_;
1491
1492     my $path;
1493     my $volid = $drive->{file};
1494     my $format;
1495
1496     if (drive_is_cdrom($drive)) {
1497         $path = get_iso_path($storecfg, $vmid, $volid);
1498     } else {
1499         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1500         if ($storeid) {
1501             $path = PVE::Storage::path($storecfg, $volid);
1502             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1503             $format = qemu_img_format($scfg, $volname);
1504         } else {
1505             $path = $volid;
1506             $format = "raw";
1507         }
1508    }
1509
1510     my $opts = '';
1511     my @qemu_drive_options = qw(heads secs cyls trans media format cache snapshot rerror werror aio discard iops iops_rd iops_wr iops_max iops_rd_max iops_wr_max);
1512     foreach my $o (@qemu_drive_options) {
1513         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1514     }
1515     if (my $serial = $drive->{serial}) {
1516         $serial = URI::Escape::uri_unescape($serial);
1517         $opts .= ",serial=$serial";
1518     }
1519
1520     $opts .= ",format=$format" if $format && !$drive->{format};
1521
1522     foreach my $o (qw(bps bps_rd bps_wr)) {
1523         my $v = $drive->{"m$o"};
1524         $opts .= ",$o=" . int($v*1024*1024) if $v;
1525     }
1526
1527     my $cache_direct = 0;
1528
1529     if (my $cache = $drive->{cache}) {
1530         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1531     } elsif (!drive_is_cdrom($drive)) {
1532         $opts .= ",cache=none";
1533         $cache_direct = 1;
1534     }
1535
1536     # aio native works only with O_DIRECT
1537     if (!$drive->{aio}) {
1538         if($cache_direct) {
1539             $opts .= ",aio=native";
1540         } else {
1541             $opts .= ",aio=threads";
1542         }
1543     }
1544
1545     if (!drive_is_cdrom($drive)) {
1546         my $detectzeroes;
1547         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1548             $detectzeroes = 'off';
1549         } elsif ($drive->{discard}) {
1550             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1551         } else {
1552             # This used to be our default with discard not being specified:
1553             $detectzeroes = 'on';
1554         }
1555         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1556     }
1557
1558     my $pathinfo = $path ? "file=$path," : '';
1559
1560     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1561 }
1562
1563 sub print_netdevice_full {
1564     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1565
1566     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1567
1568     my $device = $net->{model};
1569     if ($net->{model} eq 'virtio') {
1570          $device = 'virtio-net-pci';
1571      };
1572
1573     my $pciaddr = print_pci_addr("$netid", $bridges);
1574     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1575     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1576         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1577         my $vectors = $net->{queues} * 2 + 2;
1578         $tmpstr .= ",vectors=$vectors,mq=on";
1579     }
1580     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1581
1582     if ($use_old_bios_files) {
1583         my $romfile;
1584         if ($device eq 'virtio-net-pci') {
1585             $romfile = 'pxe-virtio.rom';
1586         } elsif ($device eq 'e1000') {
1587             $romfile = 'pxe-e1000.rom';
1588         } elsif ($device eq 'ne2k') {
1589             $romfile = 'pxe-ne2k_pci.rom';
1590         } elsif ($device eq 'pcnet') {
1591             $romfile = 'pxe-pcnet.rom';
1592         } elsif ($device eq 'rtl8139') {
1593             $romfile = 'pxe-rtl8139.rom';
1594         }
1595         $tmpstr .= ",romfile=$romfile" if $romfile;
1596     }
1597
1598     return $tmpstr;
1599 }
1600
1601 sub print_netdev_full {
1602     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1603
1604     my $i = '';
1605     if ($netid =~ m/^net(\d+)$/) {
1606         $i = int($1);
1607     }
1608
1609     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1610
1611     my $ifname = "tap${vmid}i$i";
1612
1613     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1614     die "interface name '$ifname' is too long (max 15 character)\n"
1615         if length($ifname) >= 16;
1616
1617     my $vhostparam = '';
1618     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1619
1620     my $vmname = $conf->{name} || "vm$vmid";
1621
1622     my $netdev = "";
1623     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1624
1625     if ($net->{bridge}) {
1626         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1627     } else {
1628         $netdev = "type=user,id=$netid,hostname=$vmname";
1629     }
1630
1631     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1632
1633     return $netdev;
1634 }
1635
1636 sub drive_is_cdrom {
1637     my ($drive) = @_;
1638
1639     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1640
1641 }
1642
1643 sub parse_number_sets {
1644     my ($set) = @_;
1645     my $res = [];
1646     foreach my $part (split(/;/, $set)) {
1647         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1648             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1649             push @$res, [ $1, $2 ];
1650         } else {
1651             die "invalid range: $part\n";
1652         }
1653     }
1654     return $res;
1655 }
1656
1657 sub parse_numa {
1658     my ($data) = @_;
1659
1660     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
1661     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1662     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1663     return $res;
1664 }
1665
1666 sub parse_hostpci {
1667     my ($value) = @_;
1668
1669     return undef if !$value;
1670
1671     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
1672
1673     my @idlist = split(/;/, $res->{host});
1674     delete $res->{host};
1675     foreach my $id (@idlist) {
1676         if ($id =~ /^$PCIRE$/) {
1677             push @{$res->{pciid}}, { id => $1, function => ($2//'0') };
1678         } else {
1679             # should have been caught by parse_property_string already
1680             die "failed to parse PCI id: $id\n";
1681         }
1682     }
1683     return $res;
1684 }
1685
1686 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1687 sub parse_net {
1688     my ($data) = @_;
1689
1690     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
1691     if ($@) {
1692         warn $@;
1693         return undef;
1694     }
1695     $res->{macaddr} = PVE::Tools::random_ether_addr() if !defined($res->{macaddr});
1696     return $res;
1697 }
1698
1699 sub print_net {
1700     my $net = shift;
1701
1702     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1703 }
1704
1705 sub add_random_macs {
1706     my ($settings) = @_;
1707
1708     foreach my $opt (keys %$settings) {
1709         next if $opt !~ m/^net(\d+)$/;
1710         my $net = parse_net($settings->{$opt});
1711         next if !$net;
1712         $settings->{$opt} = print_net($net);
1713     }
1714 }
1715
1716 sub vm_is_volid_owner {
1717     my ($storecfg, $vmid, $volid) = @_;
1718
1719     if ($volid !~  m|^/|) {
1720         my ($path, $owner);
1721         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1722         if ($owner && ($owner == $vmid)) {
1723             return 1;
1724         }
1725     }
1726
1727     return undef;
1728 }
1729
1730 sub split_flagged_list {
1731     my $text = shift || '';
1732     $text =~ s/[,;]/ /g;
1733     $text =~ s/^\s+//;
1734     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
1735 }
1736
1737 sub join_flagged_list {
1738     my ($how, $lst) = @_;
1739     join $how, map { $lst->{$_} . $_ } keys %$lst;
1740 }
1741
1742 sub vmconfig_delete_pending_option {
1743     my ($conf, $key, $force) = @_;
1744
1745     delete $conf->{pending}->{$key};
1746     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1747     $pending_delete_hash->{$key} = $force ? '!' : '';
1748     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1749 }
1750
1751 sub vmconfig_undelete_pending_option {
1752     my ($conf, $key) = @_;
1753
1754     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1755     delete $pending_delete_hash->{$key};
1756
1757     if (%$pending_delete_hash) {
1758         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1759     } else {
1760         delete $conf->{pending}->{delete};
1761     }
1762 }
1763
1764 sub vmconfig_register_unused_drive {
1765     my ($storecfg, $vmid, $conf, $drive) = @_;
1766
1767     if (!drive_is_cdrom($drive)) {
1768         my $volid = $drive->{file};
1769         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1770             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1771         }
1772     }
1773 }
1774
1775 sub vmconfig_cleanup_pending {
1776     my ($conf) = @_;
1777
1778     # remove pending changes when nothing changed
1779     my $changes;
1780     foreach my $opt (keys %{$conf->{pending}}) {
1781         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
1782             $changes = 1;
1783             delete $conf->{pending}->{$opt};
1784         }
1785     }
1786
1787     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
1788     my $pending_delete_hash = {};
1789     while (my ($opt, $force) = each %$current_delete_hash) {
1790         if (defined($conf->{$opt})) {
1791             $pending_delete_hash->{$opt} = $force;
1792         } else {
1793             $changes = 1;
1794         }
1795     }
1796
1797     if (%$pending_delete_hash) {
1798         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1799     } else {
1800         delete $conf->{pending}->{delete};
1801     }
1802
1803     return $changes;
1804 }
1805
1806 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
1807 my $smbios1_fmt = {
1808     uuid => {
1809         type => 'string',
1810         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1811         format_description => 'UUID',
1812         description => "Set SMBIOS1 UUID.",
1813         optional => 1,
1814     },
1815     version => {
1816         type => 'string',
1817         pattern => '\S+',
1818         format_description => 'string',
1819         description => "Set SMBIOS1 version.",
1820         optional => 1,
1821     },
1822     serial => {
1823         type => 'string',
1824         pattern => '\S+',
1825         format_description => 'string',
1826         description => "Set SMBIOS1 serial number.",
1827         optional => 1,
1828     },
1829     manufacturer => {
1830         type => 'string',
1831         pattern => '\S+',
1832         format_description => 'string',
1833         description => "Set SMBIOS1 manufacturer.",
1834         optional => 1,
1835     },
1836     product => {
1837         type => 'string',
1838         pattern => '\S+',
1839         format_description => 'string',
1840         description => "Set SMBIOS1 product ID.",
1841         optional => 1,
1842     },
1843     sku => {
1844         type => 'string',
1845         pattern => '\S+',
1846         format_description => 'string',
1847         description => "Set SMBIOS1 SKU string.",
1848         optional => 1,
1849     },
1850     family => {
1851         type => 'string',
1852         pattern => '\S+',
1853         format_description => 'string',
1854         description => "Set SMBIOS1 family string.",
1855         optional => 1,
1856     },
1857 };
1858
1859 sub parse_smbios1 {
1860     my ($data) = @_;
1861
1862     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
1863     warn $@ if $@;
1864     return $res;
1865 }
1866
1867 sub print_smbios1 {
1868     my ($smbios1) = @_;
1869     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
1870 }
1871
1872 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
1873
1874 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
1875 sub verify_bootdisk {
1876     my ($value, $noerr) = @_;
1877
1878     return $value if is_valid_drivename($value);
1879
1880     return undef if $noerr;
1881
1882     die "invalid boot disk '$value'\n";
1883 }
1884
1885 sub parse_watchdog {
1886     my ($value) = @_;
1887
1888     return undef if !$value;
1889
1890     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
1891     warn $@ if $@;
1892     return $res;
1893 }
1894
1895 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
1896 sub verify_usb_device {
1897     my ($value, $noerr) = @_;
1898
1899     return $value if parse_usb_device($value);
1900
1901     return undef if $noerr;
1902
1903     die "unable to parse usb device\n";
1904 }
1905
1906 # add JSON properties for create and set function
1907 sub json_config_properties {
1908     my $prop = shift;
1909
1910     foreach my $opt (keys %$confdesc) {
1911         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
1912         $prop->{$opt} = $confdesc->{$opt};
1913     }
1914
1915     return $prop;
1916 }
1917
1918 sub check_type {
1919     my ($key, $value) = @_;
1920
1921     die "unknown setting '$key'\n" if !$confdesc->{$key};
1922
1923     my $type = $confdesc->{$key}->{type};
1924
1925     if (!defined($value)) {
1926         die "got undefined value\n";
1927     }
1928
1929     if ($value =~ m/[\n\r]/) {
1930         die "property contains a line feed\n";
1931     }
1932
1933     if ($type eq 'boolean') {
1934         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
1935         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
1936         die "type check ('boolean') failed - got '$value'\n";
1937     } elsif ($type eq 'integer') {
1938         return int($1) if $value =~ m/^(\d+)$/;
1939         die "type check ('integer') failed - got '$value'\n";
1940     } elsif ($type eq 'number') {
1941         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
1942         die "type check ('number') failed - got '$value'\n";
1943     } elsif ($type eq 'string') {
1944         if (my $fmt = $confdesc->{$key}->{format}) {
1945             if ($fmt eq 'pve-qm-drive') {
1946                 # special case - we need to pass $key to parse_drive()
1947                 my $drive = parse_drive($key, $value);
1948                 return $value if $drive;
1949                 die "unable to parse drive options\n";
1950             }
1951             PVE::JSONSchema::check_format($fmt, $value);
1952             return $value;
1953         }
1954         $value =~ s/^\"(.*)\"$/$1/;
1955         return $value;
1956     } else {
1957         die "internal error"
1958     }
1959 }
1960
1961 sub check_iommu_support{
1962     #fixme : need to check IOMMU support
1963     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
1964
1965     my $iommu=1;
1966     return $iommu;
1967
1968 }
1969
1970 sub touch_config {
1971     my ($vmid) = @_;
1972
1973     my $conf = PVE::QemuConfig->config_file($vmid);
1974     utime undef, undef, $conf;
1975 }
1976
1977 sub destroy_vm {
1978     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
1979
1980     my $conffile = PVE::QemuConfig->config_file($vmid);
1981
1982     my $conf = PVE::QemuConfig->load_config($vmid);
1983
1984     PVE::QemuConfig->check_lock($conf) if !$skiplock;
1985
1986     # only remove disks owned by this VM
1987     foreach_drive($conf, sub {
1988         my ($ds, $drive) = @_;
1989
1990         return if drive_is_cdrom($drive);
1991
1992         my $volid = $drive->{file};
1993
1994         return if !$volid || $volid =~ m|^/|;
1995
1996         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
1997         return if !$path || !$owner || ($owner != $vmid);
1998
1999         PVE::Storage::vdisk_free($storecfg, $volid);
2000     });
2001
2002     if ($keep_empty_config) {
2003         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2004     } else {
2005         unlink $conffile;
2006     }
2007
2008     # also remove unused disk
2009     eval {
2010         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2011
2012         eval {
2013             PVE::Storage::foreach_volid($dl, sub {
2014                 my ($volid, $sid, $volname, $d) = @_;
2015                 PVE::Storage::vdisk_free($storecfg, $volid);
2016             });
2017         };
2018         warn $@ if $@;
2019
2020     };
2021     warn $@ if $@;
2022 }
2023
2024 sub parse_vm_config {
2025     my ($filename, $raw) = @_;
2026
2027     return undef if !defined($raw);
2028
2029     my $res = {
2030         digest => Digest::SHA::sha1_hex($raw),
2031         snapshots => {},
2032         pending => {},
2033     };
2034
2035     $filename =~ m|/qemu-server/(\d+)\.conf$|
2036         || die "got strange filename '$filename'";
2037
2038     my $vmid = $1;
2039
2040     my $conf = $res;
2041     my $descr;
2042     my $section = '';
2043
2044     my @lines = split(/\n/, $raw);
2045     foreach my $line (@lines) {
2046         next if $line =~ m/^\s*$/;
2047
2048         if ($line =~ m/^\[PENDING\]\s*$/i) {
2049             $section = 'pending';
2050             if (defined($descr)) {
2051                 $descr =~ s/\s+$//;
2052                 $conf->{description} = $descr;
2053             }
2054             $descr = undef;
2055             $conf = $res->{$section} = {};
2056             next;
2057
2058         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2059             $section = $1;
2060             if (defined($descr)) {
2061                 $descr =~ s/\s+$//;
2062                 $conf->{description} = $descr;
2063             }
2064             $descr = undef;
2065             $conf = $res->{snapshots}->{$section} = {};
2066             next;
2067         }
2068
2069         if ($line =~ m/^\#(.*)\s*$/) {
2070             $descr = '' if !defined($descr);
2071             $descr .= PVE::Tools::decode_text($1) . "\n";
2072             next;
2073         }
2074
2075         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2076             $descr = '' if !defined($descr);
2077             $descr .= PVE::Tools::decode_text($2);
2078         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2079             $conf->{snapstate} = $1;
2080         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2081             my $key = $1;
2082             my $value = $2;
2083             $conf->{$key} = $value;
2084         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2085             my $value = $1;
2086             if ($section eq 'pending') {
2087                 $conf->{delete} = $value; # we parse this later
2088             } else {
2089                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2090             }
2091         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
2092             my $key = $1;
2093             my $value = $2;
2094             eval { $value = check_type($key, $value); };
2095             if ($@) {
2096                 warn "vm $vmid - unable to parse value of '$key' - $@";
2097             } else {
2098                 my $fmt = $confdesc->{$key}->{format};
2099                 if ($fmt && $fmt eq 'pve-qm-drive') {
2100                     my $v = parse_drive($key, $value);
2101                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2102                         $v->{file} = $volid;
2103                         $value = print_drive($vmid, $v);
2104                     } else {
2105                         warn "vm $vmid - unable to parse value of '$key'\n";
2106                         next;
2107                     }
2108                 }
2109
2110                 if ($key eq 'cdrom') {
2111                     $conf->{ide2} = $value;
2112                 } else {
2113                     $conf->{$key} = $value;
2114                 }
2115             }
2116         }
2117     }
2118
2119     if (defined($descr)) {
2120         $descr =~ s/\s+$//;
2121         $conf->{description} = $descr;
2122     }
2123     delete $res->{snapstate}; # just to be sure
2124
2125     return $res;
2126 }
2127
2128 sub write_vm_config {
2129     my ($filename, $conf) = @_;
2130
2131     delete $conf->{snapstate}; # just to be sure
2132
2133     if ($conf->{cdrom}) {
2134         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2135         $conf->{ide2} = $conf->{cdrom};
2136         delete $conf->{cdrom};
2137     }
2138
2139     # we do not use 'smp' any longer
2140     if ($conf->{sockets}) {
2141         delete $conf->{smp};
2142     } elsif ($conf->{smp}) {
2143         $conf->{sockets} = $conf->{smp};
2144         delete $conf->{cores};
2145         delete $conf->{smp};
2146     }
2147
2148     my $used_volids = {};
2149
2150     my $cleanup_config = sub {
2151         my ($cref, $pending, $snapname) = @_;
2152
2153         foreach my $key (keys %$cref) {
2154             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2155                 $key eq 'snapstate' || $key eq 'pending';
2156             my $value = $cref->{$key};
2157             if ($key eq 'delete') {
2158                 die "propertry 'delete' is only allowed in [PENDING]\n"
2159                     if !$pending;
2160                 # fixme: check syntax?
2161                 next;
2162             }
2163             eval { $value = check_type($key, $value); };
2164             die "unable to parse value of '$key' - $@" if $@;
2165
2166             $cref->{$key} = $value;
2167
2168             if (!$snapname && is_valid_drivename($key)) {
2169                 my $drive = parse_drive($key, $value);
2170                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2171             }
2172         }
2173     };
2174
2175     &$cleanup_config($conf);
2176
2177     &$cleanup_config($conf->{pending}, 1);
2178
2179     foreach my $snapname (keys %{$conf->{snapshots}}) {
2180         die "internal error" if $snapname eq 'pending';
2181         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2182     }
2183
2184     # remove 'unusedX' settings if we re-add a volume
2185     foreach my $key (keys %$conf) {
2186         my $value = $conf->{$key};
2187         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2188             delete $conf->{$key};
2189         }
2190     }
2191
2192     my $generate_raw_config = sub {
2193         my ($conf, $pending) = @_;
2194
2195         my $raw = '';
2196
2197         # add description as comment to top of file
2198         if (defined(my $descr = $conf->{description})) {
2199             if ($descr) {
2200                 foreach my $cl (split(/\n/, $descr)) {
2201                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2202                 }
2203             } else {
2204                 $raw .= "#\n" if $pending;
2205             }
2206         }
2207
2208         foreach my $key (sort keys %$conf) {
2209             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2210             $raw .= "$key: $conf->{$key}\n";
2211         }
2212         return $raw;
2213     };
2214
2215     my $raw = &$generate_raw_config($conf);
2216
2217     if (scalar(keys %{$conf->{pending}})){
2218         $raw .= "\n[PENDING]\n";
2219         $raw .= &$generate_raw_config($conf->{pending}, 1);
2220     }
2221
2222     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2223         $raw .= "\n[$snapname]\n";
2224         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2225     }
2226
2227     return $raw;
2228 }
2229
2230 sub load_defaults {
2231
2232     my $res = {};
2233
2234     # we use static defaults from our JSON schema configuration
2235     foreach my $key (keys %$confdesc) {
2236         if (defined(my $default = $confdesc->{$key}->{default})) {
2237             $res->{$key} = $default;
2238         }
2239     }
2240
2241     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2242     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2243
2244     return $res;
2245 }
2246
2247 sub config_list {
2248     my $vmlist = PVE::Cluster::get_vmlist();
2249     my $res = {};
2250     return $res if !$vmlist || !$vmlist->{ids};
2251     my $ids = $vmlist->{ids};
2252
2253     foreach my $vmid (keys %$ids) {
2254         my $d = $ids->{$vmid};
2255         next if !$d->{node} || $d->{node} ne $nodename;
2256         next if !$d->{type} || $d->{type} ne 'qemu';
2257         $res->{$vmid}->{exists} = 1;
2258     }
2259     return $res;
2260 }
2261
2262 # test if VM uses local resources (to prevent migration)
2263 sub check_local_resources {
2264     my ($conf, $noerr) = @_;
2265
2266     my $loc_res = 0;
2267
2268     $loc_res = 1 if $conf->{hostusb}; # old syntax
2269     $loc_res = 1 if $conf->{hostpci}; # old syntax
2270
2271     foreach my $k (keys %$conf) {
2272         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2273         # sockets are safe: they will recreated be on the target side post-migrate
2274         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2275         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2276     }
2277
2278     die "VM uses local resources\n" if $loc_res && !$noerr;
2279
2280     return $loc_res;
2281 }
2282
2283 # check if used storages are available on all nodes (use by migrate)
2284 sub check_storage_availability {
2285     my ($storecfg, $conf, $node) = @_;
2286
2287     foreach_drive($conf, sub {
2288         my ($ds, $drive) = @_;
2289
2290         my $volid = $drive->{file};
2291         return if !$volid;
2292
2293         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2294         return if !$sid;
2295
2296         # check if storage is available on both nodes
2297         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2298         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2299    });
2300 }
2301
2302 # list nodes where all VM images are available (used by has_feature API)
2303 sub shared_nodes {
2304     my ($conf, $storecfg) = @_;
2305
2306     my $nodelist = PVE::Cluster::get_nodelist();
2307     my $nodehash = { map { $_ => 1 } @$nodelist };
2308     my $nodename = PVE::INotify::nodename();
2309
2310     foreach_drive($conf, sub {
2311         my ($ds, $drive) = @_;
2312
2313         my $volid = $drive->{file};
2314         return if !$volid;
2315
2316         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2317         if ($storeid) {
2318             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2319             if ($scfg->{disable}) {
2320                 $nodehash = {};
2321             } elsif (my $avail = $scfg->{nodes}) {
2322                 foreach my $node (keys %$nodehash) {
2323                     delete $nodehash->{$node} if !$avail->{$node};
2324                 }
2325             } elsif (!$scfg->{shared}) {
2326                 foreach my $node (keys %$nodehash) {
2327                     delete $nodehash->{$node} if $node ne $nodename
2328                 }
2329             }
2330         }
2331     });
2332
2333     return $nodehash
2334 }
2335
2336 sub check_cmdline {
2337     my ($pidfile, $pid) = @_;
2338
2339     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2340     if (defined($fh)) {
2341         my $line = <$fh>;
2342         $fh->close;
2343         return undef if !$line;
2344         my @param = split(/\0/, $line);
2345
2346         my $cmd = $param[0];
2347         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2348
2349         for (my $i = 0; $i < scalar (@param); $i++) {
2350             my $p = $param[$i];
2351             next if !$p;
2352             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2353                 my $p = $param[$i+1];
2354                 return 1 if $p && ($p eq $pidfile);
2355                 return undef;
2356             }
2357         }
2358     }
2359     return undef;
2360 }
2361
2362 sub check_running {
2363     my ($vmid, $nocheck, $node) = @_;
2364
2365     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2366
2367     die "unable to find configuration file for VM $vmid - no such machine\n"
2368         if !$nocheck && ! -f $filename;
2369
2370     my $pidfile = pidfile_name($vmid);
2371
2372     if (my $fd = IO::File->new("<$pidfile")) {
2373         my $st = stat($fd);
2374         my $line = <$fd>;
2375         close($fd);
2376
2377         my $mtime = $st->mtime;
2378         if ($mtime > time()) {
2379             warn "file '$filename' modified in future\n";
2380         }
2381
2382         if ($line =~ m/^(\d+)$/) {
2383             my $pid = $1;
2384             if (check_cmdline($pidfile, $pid)) {
2385                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2386                     return $pid;
2387                 }
2388             }
2389         }
2390     }
2391
2392     return undef;
2393 }
2394
2395 sub vzlist {
2396
2397     my $vzlist = config_list();
2398
2399     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2400
2401     while (defined(my $de = $fd->read)) {
2402         next if $de !~ m/^(\d+)\.pid$/;
2403         my $vmid = $1;
2404         next if !defined($vzlist->{$vmid});
2405         if (my $pid = check_running($vmid)) {
2406             $vzlist->{$vmid}->{pid} = $pid;
2407         }
2408     }
2409
2410     return $vzlist;
2411 }
2412
2413 sub disksize {
2414     my ($storecfg, $conf) = @_;
2415
2416     my $bootdisk = $conf->{bootdisk};
2417     return undef if !$bootdisk;
2418     return undef if !is_valid_drivename($bootdisk);
2419
2420     return undef if !$conf->{$bootdisk};
2421
2422     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2423     return undef if !defined($drive);
2424
2425     return undef if drive_is_cdrom($drive);
2426
2427     my $volid = $drive->{file};
2428     return undef if !$volid;
2429
2430     return $drive->{size};
2431 }
2432
2433 my $last_proc_pid_stat;
2434
2435 # get VM status information
2436 # This must be fast and should not block ($full == false)
2437 # We only query KVM using QMP if $full == true (this can be slow)
2438 sub vmstatus {
2439     my ($opt_vmid, $full) = @_;
2440
2441     my $res = {};
2442
2443     my $storecfg = PVE::Storage::config();
2444
2445     my $list = vzlist();
2446     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2447
2448     my $cpucount = $cpuinfo->{cpus} || 1;
2449
2450     foreach my $vmid (keys %$list) {
2451         next if $opt_vmid && ($vmid ne $opt_vmid);
2452
2453         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
2454         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2455
2456         my $d = {};
2457         $d->{pid} = $list->{$vmid}->{pid};
2458
2459         # fixme: better status?
2460         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2461
2462         my $size = disksize($storecfg, $conf);
2463         if (defined($size)) {
2464             $d->{disk} = 0; # no info available
2465             $d->{maxdisk} = $size;
2466         } else {
2467             $d->{disk} = 0;
2468             $d->{maxdisk} = 0;
2469         }
2470
2471         $d->{cpus} = ($conf->{sockets} || 1) * ($conf->{cores} || 1);
2472         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2473         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2474
2475         $d->{name} = $conf->{name} || "VM $vmid";
2476         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024) : 0;
2477
2478         if ($conf->{balloon}) {
2479             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2480             $d->{shares} = defined($conf->{shares}) ? $conf->{shares} : 1000;
2481         }
2482
2483         $d->{uptime} = 0;
2484         $d->{cpu} = 0;
2485         $d->{mem} = 0;
2486
2487         $d->{netout} = 0;
2488         $d->{netin} = 0;
2489
2490         $d->{diskread} = 0;
2491         $d->{diskwrite} = 0;
2492
2493         $d->{template} = PVE::QemuConfig->is_template($conf);
2494
2495         $res->{$vmid} = $d;
2496     }
2497
2498     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2499     foreach my $dev (keys %$netdev) {
2500         next if $dev !~ m/^tap([1-9]\d*)i/;
2501         my $vmid = $1;
2502         my $d = $res->{$vmid};
2503         next if !$d;
2504
2505         $d->{netout} += $netdev->{$dev}->{receive};
2506         $d->{netin} += $netdev->{$dev}->{transmit};
2507
2508         if ($full) {
2509             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2510             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2511         }
2512
2513     }
2514
2515     my $ctime = gettimeofday;
2516
2517     foreach my $vmid (keys %$list) {
2518
2519         my $d = $res->{$vmid};
2520         my $pid = $d->{pid};
2521         next if !$pid;
2522
2523         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2524         next if !$pstat; # not running
2525
2526         my $used = $pstat->{utime} + $pstat->{stime};
2527
2528         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2529
2530         if ($pstat->{vsize}) {
2531             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2532         }
2533
2534         my $old = $last_proc_pid_stat->{$pid};
2535         if (!$old) {
2536             $last_proc_pid_stat->{$pid} = {
2537                 time => $ctime,
2538                 used => $used,
2539                 cpu => 0,
2540             };
2541             next;
2542         }
2543
2544         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2545
2546         if ($dtime > 1000) {
2547             my $dutime = $used -  $old->{used};
2548
2549             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2550             $last_proc_pid_stat->{$pid} = {
2551                 time => $ctime,
2552                 used => $used,
2553                 cpu => $d->{cpu},
2554             };
2555         } else {
2556             $d->{cpu} = $old->{cpu};
2557         }
2558     }
2559
2560     return $res if !$full;
2561
2562     my $qmpclient = PVE::QMPClient->new();
2563
2564     my $ballooncb = sub {
2565         my ($vmid, $resp) = @_;
2566
2567         my $info = $resp->{'return'};
2568         return if !$info->{max_mem};
2569
2570         my $d = $res->{$vmid};
2571
2572         # use memory assigned to VM
2573         $d->{maxmem} = $info->{max_mem};
2574         $d->{balloon} = $info->{actual};
2575
2576         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2577             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2578             $d->{freemem} = $info->{free_mem};
2579         }
2580
2581         $d->{ballooninfo} = $info;
2582     };
2583
2584     my $blockstatscb = sub {
2585         my ($vmid, $resp) = @_;
2586         my $data = $resp->{'return'} || [];
2587         my $totalrdbytes = 0;
2588         my $totalwrbytes = 0;
2589
2590         for my $blockstat (@$data) {
2591             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2592             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2593
2594             $blockstat->{device} =~ s/drive-//;
2595             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2596         }
2597         $res->{$vmid}->{diskread} = $totalrdbytes;
2598         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2599     };
2600
2601     my $statuscb = sub {
2602         my ($vmid, $resp) = @_;
2603
2604         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2605         # this fails if ballon driver is not loaded, so this must be
2606         # the last commnand (following command are aborted if this fails).
2607         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2608
2609         my $status = 'unknown';
2610         if (!defined($status = $resp->{'return'}->{status})) {
2611             warn "unable to get VM status\n";
2612             return;
2613         }
2614
2615         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2616     };
2617
2618     foreach my $vmid (keys %$list) {
2619         next if $opt_vmid && ($vmid ne $opt_vmid);
2620         next if !$res->{$vmid}->{pid}; # not running
2621         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2622     }
2623
2624     $qmpclient->queue_execute(undef, 1);
2625
2626     foreach my $vmid (keys %$list) {
2627         next if $opt_vmid && ($vmid ne $opt_vmid);
2628         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2629     }
2630
2631     return $res;
2632 }
2633
2634 sub foreach_drive {
2635     my ($conf, $func, @param) = @_;
2636
2637     foreach my $ds (valid_drive_names()) {
2638         next if !defined($conf->{$ds});
2639
2640         my $drive = parse_drive($ds, $conf->{$ds});
2641         next if !$drive;
2642
2643         &$func($ds, $drive, @param);
2644     }
2645 }
2646
2647 sub foreach_volid {
2648     my ($conf, $func, @param) = @_;
2649
2650     my $volhash = {};
2651
2652     my $test_volid = sub {
2653         my ($volid, $is_cdrom) = @_;
2654
2655         return if !$volid;
2656
2657         $volhash->{$volid} = $is_cdrom || 0;
2658     };
2659
2660     foreach_drive($conf, sub {
2661         my ($ds, $drive) = @_;
2662         &$test_volid($drive->{file}, drive_is_cdrom($drive));
2663     });
2664
2665     foreach my $snapname (keys %{$conf->{snapshots}}) {
2666         my $snap = $conf->{snapshots}->{$snapname};
2667         &$test_volid($snap->{vmstate}, 0);
2668         foreach_drive($snap, sub {
2669             my ($ds, $drive) = @_;
2670             &$test_volid($drive->{file}, drive_is_cdrom($drive));
2671         });
2672     }
2673
2674     foreach my $volid (keys %$volhash) {
2675         &$func($volid, $volhash->{$volid}, @param);
2676     }
2677 }
2678
2679 sub vga_conf_has_spice {
2680     my ($vga) = @_;
2681
2682     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
2683
2684     return $1 || 1;
2685 }
2686
2687 sub config_to_command {
2688     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2689
2690     my $cmd = [];
2691     my $globalFlags = [];
2692     my $machineFlags = [];
2693     my $rtcFlags = [];
2694     my $cpuFlags = [];
2695     my $devices = [];
2696     my $pciaddr = '';
2697     my $bridges = {};
2698     my $kvmver = kvm_user_version();
2699     my $vernum = 0; # unknown
2700     my $ostype = $conf->{ostype};
2701     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
2702         $vernum = $1*1000000+$2*1000;
2703     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
2704         $vernum = $1*1000000+$2*1000+$3;
2705     }
2706
2707     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
2708
2709     my $have_ovz = -f '/proc/vz/vestat';
2710
2711     my $q35 = machine_type_is_q35($conf);
2712     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2713     my $machine_type = $forcemachine || $conf->{machine};
2714     my $use_old_bios_files = undef;
2715     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
2716
2717     my $cpuunits = defined($conf->{cpuunits}) ?
2718             $conf->{cpuunits} : $defaults->{cpuunits};
2719
2720     push @$cmd, '/usr/bin/kvm';
2721
2722     push @$cmd, '-id', $vmid;
2723
2724     my $use_virtio = 0;
2725
2726     my $qmpsocket = qmp_socket($vmid);
2727     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2728     push @$cmd, '-mon', "chardev=qmp,mode=control";
2729
2730
2731     push @$cmd, '-pidfile' , pidfile_name($vmid);
2732
2733     push @$cmd, '-daemonize';
2734
2735     if ($conf->{smbios1}) {
2736         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
2737     }
2738
2739     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2740         my $ovmfvar = "OVMF_VARS-pure-efi.fd";
2741         my $ovmfvar_src = "/usr/share/kvm/$ovmfvar";
2742         my $ovmfvar_dst = "/tmp/$vmid-$ovmfvar";
2743         PVE::Tools::file_copy($ovmfvar_src, $ovmfvar_dst, 256*1024);
2744         push @$cmd, '-drive', "if=pflash,format=raw,readonly,file=/usr/share/kvm/OVMF-pure-efi.fd";
2745         push @$cmd, '-drive', "if=pflash,format=raw,file=$ovmfvar_dst";
2746     }
2747
2748
2749     # add usb controllers
2750     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $q35, $usbdesc->{format}, $MAX_USB_DEVICES);
2751     push @$devices, @usbcontrollers if @usbcontrollers;
2752     my $vga = $conf->{vga};
2753
2754     my $qxlnum = vga_conf_has_spice($vga);
2755     $vga = 'qxl' if $qxlnum;
2756
2757     if (!$vga) {
2758         if ($conf->{ostype} && ($conf->{ostype} eq 'win8' ||
2759                                 $conf->{ostype} eq 'win7' ||
2760                                 $conf->{ostype} eq 'w2k8')) {
2761             $vga = 'std';
2762         } else {
2763             $vga = 'cirrus';
2764         }
2765     }
2766
2767     # enable absolute mouse coordinates (needed by vnc)
2768     my $tablet;
2769     if (defined($conf->{tablet})) {
2770         $tablet = $conf->{tablet};
2771     } else {
2772         $tablet = $defaults->{tablet};
2773         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
2774         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
2775     }
2776
2777     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
2778
2779     my $kvm_off = 0;
2780     # host pci devices
2781     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
2782         my $d = parse_hostpci($conf->{"hostpci$i"});
2783         next if !$d;
2784
2785         my $pcie = $d->{pcie};
2786         if($pcie){
2787             die "q35 machine model is not enabled" if !$q35;
2788             $pciaddr = print_pcie_addr("hostpci$i");
2789         }else{
2790             $pciaddr = print_pci_addr("hostpci$i", $bridges);
2791         }
2792
2793         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
2794         my $xvga = '';
2795         if ($d->{'x-vga'}) {
2796             $xvga = ',x-vga=on';
2797             $kvm_off = 1;
2798             $vga = 'none';
2799             if ($ostype eq 'win7' || $ostype eq 'win8' || $ostype eq 'w2k8') {
2800                 push @$cpuFlags , 'hv_vendor_id=proxmox';
2801             }
2802             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2803                 $xvga = "";
2804             }
2805         }
2806         my $pcidevices = $d->{pciid};
2807         my $multifunction = 1 if @$pcidevices > 1;
2808
2809         my $j=0;
2810         foreach my $pcidevice (@$pcidevices) {
2811
2812             my $id = "hostpci$i";
2813             $id .= ".$j" if $multifunction;
2814             my $addr = $pciaddr;
2815             $addr .= ".$j" if $multifunction;
2816             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
2817
2818             if($j == 0){
2819                 $devicestr .= "$rombar$xvga";
2820                 $devicestr .= ",multifunction=on" if $multifunction;
2821             }
2822
2823             push @$devices, '-device', $devicestr;
2824             $j++;
2825         }
2826     }
2827
2828     # usb devices
2829     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES);
2830     push @$devices, @usbdevices if @usbdevices;
2831     # serial devices
2832     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2833         if (my $path = $conf->{"serial$i"}) {
2834             if ($path eq 'socket') {
2835                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
2836                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
2837                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2838             } else {
2839                 die "no such serial device\n" if ! -c $path;
2840                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
2841                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2842             }
2843         }
2844     }
2845
2846     # parallel devices
2847     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
2848         if (my $path = $conf->{"parallel$i"}) {
2849             die "no such parallel device\n" if ! -c $path;
2850             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
2851             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
2852             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
2853         }
2854     }
2855
2856     my $vmname = $conf->{name} || "vm$vmid";
2857
2858     push @$cmd, '-name', $vmname;
2859
2860     my $sockets = 1;
2861     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
2862     $sockets = $conf->{sockets} if  $conf->{sockets};
2863
2864     my $cores = $conf->{cores} || 1;
2865
2866     my $maxcpus = $sockets * $cores;
2867
2868     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
2869
2870     my $allowed_vcpus = $cpuinfo->{cpus};
2871
2872     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
2873         if ($allowed_vcpus < $maxcpus);
2874
2875     push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
2876
2877     push @$cmd, '-nodefaults';
2878
2879     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2880
2881     my $bootindex_hash = {};
2882     my $i = 1;
2883     foreach my $o (split(//, $bootorder)) {
2884         $bootindex_hash->{$o} = $i*100;
2885         $i++;
2886     }
2887
2888     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000";
2889
2890     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
2891
2892     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
2893
2894     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
2895
2896     if ($vga && $vga !~ m/^serial\d+$/ && $vga ne 'none'){
2897         my $socket = vnc_socket($vmid);
2898         push @$cmd,  '-vnc', "unix:$socket,x509,password";
2899     } else {
2900         push @$cmd, '-nographic';
2901     }
2902
2903     # time drift fix
2904     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
2905
2906     my $nokvm = defined($conf->{kvm}) && $conf->{kvm} == 0 ? 1 : 0;
2907     my $useLocaltime = $conf->{localtime};
2908
2909     if ($ostype) {
2910         # other, wxp, w2k, w2k3, w2k8, wvista, win7, win8, l24, l26, solaris
2911
2912         if ($ostype =~ m/^w/) { # windows
2913             $useLocaltime = 1 if !defined($conf->{localtime});
2914
2915             # use time drift fix when acpi is enabled
2916             if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
2917                 $tdf = 1 if !defined($conf->{tdf});
2918             }
2919         }
2920
2921         if ($ostype eq 'win7' || $ostype eq 'win8' || $ostype eq 'w2k8' ||
2922             $ostype eq 'wvista') {
2923             push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
2924             push @$cmd, '-no-hpet';
2925             if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
2926                 push @$cpuFlags , 'hv_spinlocks=0x1fff' if !$nokvm;
2927                 push @$cpuFlags , 'hv_vapic' if !$nokvm;
2928                 push @$cpuFlags , 'hv_time' if !$nokvm;
2929
2930                 if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
2931                     push @$cpuFlags , 'hv_reset' if !$nokvm;
2932                     push @$cpuFlags , 'hv_vpindex' if !$nokvm;
2933                     push @$cpuFlags , 'hv_runtime' if !$nokvm;
2934                 }
2935
2936             } else {
2937                 push @$cpuFlags , 'hv_spinlocks=0xffff' if !$nokvm;
2938             }
2939         }
2940
2941         if ($ostype eq 'win7' || $ostype eq 'win8') {
2942             push @$cpuFlags , 'hv_relaxed' if !$nokvm;
2943         }
2944     }
2945
2946     push @$rtcFlags, 'driftfix=slew' if $tdf;
2947
2948     if ($nokvm) {
2949         push @$machineFlags, 'accel=tcg';
2950     } else {
2951         die "No accelerator found!\n" if !$cpuinfo->{hvm};
2952     }
2953
2954     if ($machine_type) {
2955         push @$machineFlags, "type=${machine_type}";
2956     }
2957
2958     if ($conf->{startdate}) {
2959         push @$rtcFlags, "base=$conf->{startdate}";
2960     } elsif ($useLocaltime) {
2961         push @$rtcFlags, 'base=localtime';
2962     }
2963
2964     my $cpu = $nokvm ? "qemu64" : "kvm64";
2965     if (my $cputype = $conf->{cpu}) {
2966         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
2967             or die "Cannot parse cpu description: $cputype\n";
2968         $cpu = $cpuconf->{cputype};
2969         $kvm_off = 1 if $cpuconf->{hidden};
2970     }
2971
2972     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
2973
2974     push @$cpuFlags , '-x2apic'
2975         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
2976
2977     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
2978
2979     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
2980
2981     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
2982
2983         push @$cpuFlags , '+kvm_pv_unhalt' if !$nokvm;
2984         push @$cpuFlags , '+kvm_pv_eoi' if !$nokvm;
2985     }
2986
2987     push @$cpuFlags, 'enforce' if $cpu ne 'host' && !$nokvm;
2988
2989     push @$cpuFlags, 'kvm=off' if $kvm_off;
2990
2991     my $cpu_vendor = $cpu_vendor_list->{$cpu} ||
2992         die "internal error"; # should not happen
2993
2994     push @$cpuFlags, "vendor=${cpu_vendor}"
2995         if $cpu_vendor ne 'default';
2996
2997     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
2998
2999     push @$cmd, '-cpu', $cpu;
3000
3001     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3002
3003     push @$cmd, '-S' if $conf->{freeze};
3004
3005     # set keyboard layout
3006     my $kb = $conf->{keyboard} || $defaults->{keyboard};
3007     push @$cmd, '-k', $kb if $kb;
3008
3009     # enable sound
3010     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3011     #push @$cmd, '-soundhw', 'es1370';
3012     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3013
3014     if($conf->{agent}) {
3015         my $qgasocket = qmp_socket($vmid, 1);
3016         my $pciaddr = print_pci_addr("qga0", $bridges);
3017         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3018         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3019         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3020     }
3021
3022     my $spice_port;
3023
3024     if ($qxlnum) {
3025         if ($qxlnum > 1) {
3026             if ($conf->{ostype} && $conf->{ostype} =~ m/^w/){
3027                 for(my $i = 1; $i < $qxlnum; $i++){
3028                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3029                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3030                 }
3031             } else {
3032                 # assume other OS works like Linux
3033                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3034                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3035             }
3036         }
3037
3038         my $pciaddr = print_pci_addr("spice", $bridges);
3039
3040         my $nodename = PVE::INotify::nodename();
3041         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3042         $spice_port = PVE::Tools::next_spice_port($pfamily);
3043
3044         push @$devices, '-spice', "tls-port=${spice_port},addr=localhost,tls-ciphers=DES-CBC3-SHA,seamless-migration=on";
3045
3046         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3047         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3048         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3049     }
3050
3051     # enable balloon by default, unless explicitly disabled
3052     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3053         $pciaddr = print_pci_addr("balloon0", $bridges);
3054         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3055     }
3056
3057     if ($conf->{watchdog}) {
3058         my $wdopts = parse_watchdog($conf->{watchdog});
3059         $pciaddr = print_pci_addr("watchdog", $bridges);
3060         my $watchdog = $wdopts->{model} || 'i6300esb';
3061         push @$devices, '-device', "$watchdog$pciaddr";
3062         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3063     }
3064
3065     my $vollist = [];
3066     my $scsicontroller = {};
3067     my $ahcicontroller = {};
3068     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3069
3070     # Add iscsi initiator name if available
3071     if (my $initiator = get_initiator_name()) {
3072         push @$devices, '-iscsi', "initiator-name=$initiator";
3073     }
3074
3075     foreach_drive($conf, sub {
3076         my ($ds, $drive) = @_;
3077
3078         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3079             push @$vollist, $drive->{file};
3080         }
3081
3082         $use_virtio = 1 if $ds =~ m/^virtio/;
3083
3084         if (drive_is_cdrom ($drive)) {
3085             if ($bootindex_hash->{d}) {
3086                 $drive->{bootindex} = $bootindex_hash->{d};
3087                 $bootindex_hash->{d} += 1;
3088             }
3089         } else {
3090             if ($bootindex_hash->{c}) {
3091                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3092                 $bootindex_hash->{c} += 1;
3093             }
3094         }
3095
3096         if($drive->{interface} eq 'virtio'){
3097            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3098         }
3099
3100         if ($drive->{interface} eq 'scsi') {
3101
3102             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3103
3104             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3105             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3106
3107             my $iothread = '';
3108             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3109                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3110                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3111             } elsif ($drive->{iothread}) {
3112                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3113             }
3114
3115             my $queues = '';
3116             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3117                 $queues = ",num_queues=$drive->{queues}";
3118             }
3119
3120             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3121             $scsicontroller->{$controller}=1;
3122         }
3123
3124         if ($drive->{interface} eq 'sata') {
3125            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3126            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3127            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3128            $ahcicontroller->{$controller}=1;
3129         }
3130
3131         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3132         push @$devices, '-drive',$drive_cmd;
3133         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3134     });
3135
3136     for (my $i = 0; $i < $MAX_NETS; $i++) {
3137          next if !$conf->{"net$i"};
3138          my $d = parse_net($conf->{"net$i"});
3139          next if !$d;
3140
3141          $use_virtio = 1 if $d->{model} eq 'virtio';
3142
3143          if ($bootindex_hash->{n}) {
3144             $d->{bootindex} = $bootindex_hash->{n};
3145             $bootindex_hash->{n} += 1;
3146          }
3147
3148          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3149          push @$devices, '-netdev', $netdevfull;
3150
3151          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3152          push @$devices, '-device', $netdevicefull;
3153     }
3154
3155     if (!$q35) {
3156         # add pci bridges
3157         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3158            $bridges->{1} = 1;
3159            $bridges->{2} = 1;
3160         }
3161
3162         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3163
3164         while (my ($k, $v) = each %$bridges) {
3165             $pciaddr = print_pci_addr("pci.$k");
3166             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3167         }
3168     }
3169
3170     # add custom args
3171     if ($conf->{args}) {
3172         my $aa = PVE::Tools::split_args($conf->{args});
3173         push @$cmd, @$aa;
3174     }
3175
3176     push @$cmd, @$devices;
3177     push @$cmd, '-rtc', join(',', @$rtcFlags)
3178         if scalar(@$rtcFlags);
3179     push @$cmd, '-machine', join(',', @$machineFlags)
3180         if scalar(@$machineFlags);
3181     push @$cmd, '-global', join(',', @$globalFlags)
3182         if scalar(@$globalFlags);
3183
3184     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3185 }
3186
3187 sub vnc_socket {
3188     my ($vmid) = @_;
3189     return "${var_run_tmpdir}/$vmid.vnc";
3190 }
3191
3192 sub spice_port {
3193     my ($vmid) = @_;
3194
3195     my $res = vm_mon_cmd($vmid, 'query-spice');
3196
3197     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3198 }
3199
3200 sub qmp_socket {
3201     my ($vmid, $qga) = @_;
3202     my $sockettype = $qga ? 'qga' : 'qmp';
3203     return "${var_run_tmpdir}/$vmid.$sockettype";
3204 }
3205
3206 sub pidfile_name {
3207     my ($vmid) = @_;
3208     return "${var_run_tmpdir}/$vmid.pid";
3209 }
3210
3211 sub vm_devices_list {
3212     my ($vmid) = @_;
3213
3214     my $res = vm_mon_cmd($vmid, 'query-pci');
3215     my $devices = {};
3216     foreach my $pcibus (@$res) {
3217         foreach my $device (@{$pcibus->{devices}}) {
3218             next if !$device->{'qdev_id'};
3219             if ($device->{'pci_bridge'}) {
3220                 $devices->{$device->{'qdev_id'}} = 1;
3221                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3222                     next if !$bridge_device->{'qdev_id'};
3223                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3224                     $devices->{$device->{'qdev_id'}}++;
3225                 }
3226             } else {
3227                 $devices->{$device->{'qdev_id'}} = 1;
3228             }
3229         }
3230     }
3231
3232     my $resblock = vm_mon_cmd($vmid, 'query-block');
3233     foreach my $block (@$resblock) {
3234         if($block->{device} =~ m/^drive-(\S+)/){
3235                 $devices->{$1} = 1;
3236         }
3237     }
3238
3239     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3240     foreach my $mice (@$resmice) {
3241         if ($mice->{name} eq 'QEMU HID Tablet') {
3242             $devices->{tablet} = 1;
3243             last;
3244         }
3245     }
3246
3247     return $devices;
3248 }
3249
3250 sub vm_deviceplug {
3251     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3252
3253     my $q35 = machine_type_is_q35($conf);
3254
3255     my $devices_list = vm_devices_list($vmid);
3256     return 1 if defined($devices_list->{$deviceid});
3257
3258     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3259
3260     if ($deviceid eq 'tablet') {
3261
3262         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3263
3264     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3265
3266         qemu_iothread_add($vmid, $deviceid, $device);
3267
3268         qemu_driveadd($storecfg, $vmid, $device);
3269         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3270
3271         qemu_deviceadd($vmid, $devicefull);
3272         eval { qemu_deviceaddverify($vmid, $deviceid); };
3273         if (my $err = $@) {
3274             eval { qemu_drivedel($vmid, $deviceid); };
3275             warn $@ if $@;
3276             die $err;
3277         }
3278
3279     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3280
3281
3282         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3283         my $pciaddr = print_pci_addr($deviceid);
3284         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3285
3286         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3287
3288         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3289             qemu_iothread_add($vmid, $deviceid, $device);
3290             $devicefull .= ",iothread=iothread-$deviceid";
3291         }
3292
3293         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3294             $devicefull .= ",num_queues=$device->{queues}";
3295         }
3296
3297         qemu_deviceadd($vmid, $devicefull);
3298         qemu_deviceaddverify($vmid, $deviceid);
3299
3300     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3301
3302         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3303         qemu_driveadd($storecfg, $vmid, $device);
3304
3305         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3306         eval { qemu_deviceadd($vmid, $devicefull); };
3307         if (my $err = $@) {
3308             eval { qemu_drivedel($vmid, $deviceid); };
3309             warn $@ if $@;
3310             die $err;
3311         }
3312
3313     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3314
3315         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3316
3317         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
3318         my $use_old_bios_files = undef;
3319         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3320
3321         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3322         qemu_deviceadd($vmid, $netdevicefull);
3323         eval { qemu_deviceaddverify($vmid, $deviceid); };
3324         if (my $err = $@) {
3325             eval { qemu_netdevdel($vmid, $deviceid); };
3326             warn $@ if $@;
3327             die $err;
3328         }
3329
3330     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3331
3332         my $bridgeid = $2;
3333         my $pciaddr = print_pci_addr($deviceid);
3334         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3335
3336         qemu_deviceadd($vmid, $devicefull);
3337         qemu_deviceaddverify($vmid, $deviceid);
3338
3339     } else {
3340         die "can't hotplug device '$deviceid'\n";
3341     }
3342
3343     return 1;
3344 }
3345
3346 # fixme: this should raise exceptions on error!
3347 sub vm_deviceunplug {
3348     my ($vmid, $conf, $deviceid) = @_;
3349
3350     my $devices_list = vm_devices_list($vmid);
3351     return 1 if !defined($devices_list->{$deviceid});
3352
3353     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3354
3355     if ($deviceid eq 'tablet') {
3356
3357         qemu_devicedel($vmid, $deviceid);
3358
3359     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3360
3361         qemu_devicedel($vmid, $deviceid);
3362         qemu_devicedelverify($vmid, $deviceid);
3363         qemu_drivedel($vmid, $deviceid);
3364         qemu_iothread_del($conf, $vmid, $deviceid);
3365
3366     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3367
3368         qemu_devicedel($vmid, $deviceid);
3369         qemu_devicedelverify($vmid, $deviceid);
3370         qemu_iothread_del($conf, $vmid, $deviceid);
3371
3372     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3373
3374         #qemu 2.3 segfault on drive_del with virtioscsi + iothread
3375         my $device = parse_drive($deviceid, $conf->{$deviceid});
3376         die "virtioscsi with iothread is not hot-unplugglable currently" if $device->{iothread};
3377
3378         qemu_devicedel($vmid, $deviceid);
3379         qemu_drivedel($vmid, $deviceid);
3380         qemu_deletescsihw($conf, $vmid, $deviceid);
3381
3382     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3383
3384         qemu_devicedel($vmid, $deviceid);
3385         qemu_devicedelverify($vmid, $deviceid);
3386         qemu_netdevdel($vmid, $deviceid);
3387
3388     } else {
3389         die "can't unplug device '$deviceid'\n";
3390     }
3391
3392     return 1;
3393 }
3394
3395 sub qemu_deviceadd {
3396     my ($vmid, $devicefull) = @_;
3397
3398     $devicefull = "driver=".$devicefull;
3399     my %options =  split(/[=,]/, $devicefull);
3400
3401     vm_mon_cmd($vmid, "device_add" , %options);
3402 }
3403
3404 sub qemu_devicedel {
3405     my ($vmid, $deviceid) = @_;
3406
3407     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3408 }
3409
3410 sub qemu_iothread_add {
3411     my($vmid, $deviceid, $device) = @_;
3412
3413     if ($device->{iothread}) {
3414         my $iothreads = vm_iothreads_list($vmid);
3415         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3416     }
3417 }
3418
3419 sub qemu_iothread_del {
3420     my($conf, $vmid, $deviceid) = @_;
3421
3422     my $device = parse_drive($deviceid, $conf->{$deviceid});
3423     if ($device->{iothread}) {
3424         my $iothreads = vm_iothreads_list($vmid);
3425         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3426     }
3427 }
3428
3429 sub qemu_objectadd {
3430     my($vmid, $objectid, $qomtype) = @_;
3431
3432     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3433
3434     return 1;
3435 }
3436
3437 sub qemu_objectdel {
3438     my($vmid, $objectid) = @_;
3439
3440     vm_mon_cmd($vmid, "object-del", id => $objectid);
3441
3442     return 1;
3443 }
3444
3445 sub qemu_driveadd {
3446     my ($storecfg, $vmid, $device) = @_;
3447
3448     my $drive = print_drive_full($storecfg, $vmid, $device);
3449     $drive =~ s/\\/\\\\/g;
3450     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3451
3452     # If the command succeeds qemu prints: "OK"
3453     return 1 if $ret =~ m/OK/s;
3454
3455     die "adding drive failed: $ret\n";
3456 }
3457
3458 sub qemu_drivedel {
3459     my($vmid, $deviceid) = @_;
3460
3461     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3462     $ret =~ s/^\s+//;
3463
3464     return 1 if $ret eq "";
3465
3466     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3467     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3468
3469     die "deleting drive $deviceid failed : $ret\n";
3470 }
3471
3472 sub qemu_deviceaddverify {
3473     my ($vmid, $deviceid) = @_;
3474
3475     for (my $i = 0; $i <= 5; $i++) {
3476          my $devices_list = vm_devices_list($vmid);
3477          return 1 if defined($devices_list->{$deviceid});
3478          sleep 1;
3479     }
3480
3481     die "error on hotplug device '$deviceid'\n";
3482 }
3483
3484
3485 sub qemu_devicedelverify {
3486     my ($vmid, $deviceid) = @_;
3487
3488     # need to verify that the device is correctly removed as device_del
3489     # is async and empty return is not reliable
3490
3491     for (my $i = 0; $i <= 5; $i++) {
3492          my $devices_list = vm_devices_list($vmid);
3493          return 1 if !defined($devices_list->{$deviceid});
3494          sleep 1;
3495     }
3496
3497     die "error on hot-unplugging device '$deviceid'\n";
3498 }
3499
3500 sub qemu_findorcreatescsihw {
3501     my ($storecfg, $conf, $vmid, $device) = @_;
3502
3503     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3504
3505     my $scsihwid="$controller_prefix$controller";
3506     my $devices_list = vm_devices_list($vmid);
3507
3508     if(!defined($devices_list->{$scsihwid})) {
3509         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3510     }
3511
3512     return 1;
3513 }
3514
3515 sub qemu_deletescsihw {
3516     my ($conf, $vmid, $opt) = @_;
3517
3518     my $device = parse_drive($opt, $conf->{$opt});
3519
3520     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3521         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3522         return 1;
3523     }
3524
3525     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3526
3527     my $devices_list = vm_devices_list($vmid);
3528     foreach my $opt (keys %{$devices_list}) {
3529         if (PVE::QemuServer::is_valid_drivename($opt)) {
3530             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3531             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3532                 return 1;
3533             }
3534         }
3535     }
3536
3537     my $scsihwid="scsihw$controller";
3538
3539     vm_deviceunplug($vmid, $conf, $scsihwid);
3540
3541     return 1;
3542 }
3543
3544 sub qemu_add_pci_bridge {
3545     my ($storecfg, $conf, $vmid, $device) = @_;
3546
3547     my $bridges = {};
3548
3549     my $bridgeid;
3550
3551     print_pci_addr($device, $bridges);
3552
3553     while (my ($k, $v) = each %$bridges) {
3554         $bridgeid = $k;
3555     }
3556     return 1 if !defined($bridgeid) || $bridgeid < 1;
3557
3558     my $bridge = "pci.$bridgeid";
3559     my $devices_list = vm_devices_list($vmid);
3560
3561     if (!defined($devices_list->{$bridge})) {
3562         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3563     }
3564
3565     return 1;
3566 }
3567
3568 sub qemu_set_link_status {
3569     my ($vmid, $device, $up) = @_;
3570
3571     vm_mon_cmd($vmid, "set_link", name => $device,
3572                up => $up ? JSON::true : JSON::false);
3573 }
3574
3575 sub qemu_netdevadd {
3576     my ($vmid, $conf, $device, $deviceid) = @_;
3577
3578     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
3579     my %options =  split(/[=,]/, $netdev);
3580
3581     vm_mon_cmd($vmid, "netdev_add",  %options);
3582     return 1;
3583 }
3584
3585 sub qemu_netdevdel {
3586     my ($vmid, $deviceid) = @_;
3587
3588     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3589 }
3590
3591 sub qemu_cpu_hotplug {
3592     my ($vmid, $conf, $vcpus) = @_;
3593
3594     my $sockets = 1;
3595     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3596     $sockets = $conf->{sockets} if  $conf->{sockets};
3597     my $cores = $conf->{cores} || 1;
3598     my $maxcpus = $sockets * $cores;
3599
3600     $vcpus = $maxcpus if !$vcpus;
3601
3602     die "you can't add more vcpus than maxcpus\n"
3603         if $vcpus > $maxcpus;
3604
3605     my $currentvcpus = $conf->{vcpus} || $maxcpus;
3606     die "online cpu unplug is not yet possible\n"
3607         if $vcpus < $currentvcpus;
3608
3609     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3610     die "vcpus in running vm is different than configuration\n"
3611         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
3612
3613     for (my $i = $currentvcpus; $i < $vcpus; $i++) {
3614         vm_mon_cmd($vmid, "cpu-add", id => int($i));
3615     }
3616 }
3617
3618 sub qemu_block_set_io_throttle {
3619     my ($vmid, $deviceid,
3620         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
3621         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max) = @_;
3622
3623     return if !check_running($vmid) ;
3624
3625     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
3626         bps => int($bps),
3627         bps_rd => int($bps_rd),
3628         bps_wr => int($bps_wr),
3629         iops => int($iops),
3630         iops_rd => int($iops_rd),
3631         iops_wr => int($iops_wr),
3632         bps_max => int($bps_max),
3633         bps_rd_max => int($bps_rd_max),
3634         bps_wr_max => int($bps_wr_max),
3635         iops_max => int($iops_max),
3636         iops_rd_max => int($iops_rd_max),
3637         iops_wr_max => int($iops_wr_max)
3638     );
3639
3640 }
3641
3642 # old code, only used to shutdown old VM after update
3643 sub __read_avail {
3644     my ($fh, $timeout) = @_;
3645
3646     my $sel = new IO::Select;
3647     $sel->add($fh);
3648
3649     my $res = '';
3650     my $buf;
3651
3652     my @ready;
3653     while (scalar (@ready = $sel->can_read($timeout))) {
3654         my $count;
3655         if ($count = $fh->sysread($buf, 8192)) {
3656             if ($buf =~ /^(.*)\(qemu\) $/s) {
3657                 $res .= $1;
3658                 last;
3659             } else {
3660                 $res .= $buf;
3661             }
3662         } else {
3663             if (!defined($count)) {
3664                 die "$!\n";
3665             }
3666             last;
3667         }
3668     }
3669
3670     die "monitor read timeout\n" if !scalar(@ready);
3671
3672     return $res;
3673 }
3674
3675 # old code, only used to shutdown old VM after update
3676 sub vm_monitor_command {
3677     my ($vmid, $cmdstr, $nocheck) = @_;
3678
3679     my $res;
3680
3681     eval {
3682         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
3683
3684         my $sname = "${var_run_tmpdir}/$vmid.mon";
3685
3686         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
3687             die "unable to connect to VM $vmid socket - $!\n";
3688
3689         my $timeout = 3;
3690
3691         # hack: migrate sometime blocks the monitor (when migrate_downtime
3692         # is set)
3693         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3694             $timeout = 60*60; # 1 hour
3695         }
3696
3697         # read banner;
3698         my $data = __read_avail($sock, $timeout);
3699
3700         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
3701             die "got unexpected qemu monitor banner\n";
3702         }
3703
3704         my $sel = new IO::Select;
3705         $sel->add($sock);
3706
3707         if (!scalar(my @ready = $sel->can_write($timeout))) {
3708             die "monitor write error - timeout";
3709         }
3710
3711         my $fullcmd = "$cmdstr\r";
3712
3713         # syslog('info', "VM $vmid monitor command: $cmdstr");
3714
3715         my $b;
3716         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
3717             die "monitor write error - $!";
3718         }
3719
3720         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
3721
3722         $timeout = 20;
3723
3724         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3725             $timeout = 60*60; # 1 hour
3726         } elsif ($cmdstr =~ m/^(eject|change)/) {
3727             $timeout = 60; # note: cdrom mount command is slow
3728         }
3729         if ($res = __read_avail($sock, $timeout)) {
3730
3731             my @lines = split("\r?\n", $res);
3732
3733             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
3734
3735             $res = join("\n", @lines);
3736             $res .= "\n";
3737         }
3738     };
3739
3740     my $err = $@;
3741
3742     if ($err) {
3743         syslog("err", "VM $vmid monitor command failed - $err");
3744         die $err;
3745     }
3746
3747     return $res;
3748 }
3749
3750 sub qemu_block_resize {
3751     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
3752
3753     my $running = check_running($vmid);
3754
3755     return if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
3756
3757     return if !$running;
3758
3759     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
3760
3761 }
3762
3763 sub qemu_volume_snapshot {
3764     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3765
3766     my $running = check_running($vmid);
3767
3768     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
3769         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
3770     } else {
3771         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
3772     }
3773 }
3774
3775 sub qemu_volume_snapshot_delete {
3776     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3777
3778     my $running = check_running($vmid);
3779
3780     return if !PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
3781
3782     return if !$running;
3783
3784     vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
3785 }
3786
3787 sub set_migration_caps {
3788     my ($vmid) = @_;
3789
3790     my $cap_ref = [];
3791
3792     my $enabled_cap = {
3793         "auto-converge" => 1,
3794         "xbzrle" => 1,
3795         "x-rdma-pin-all" => 0,
3796         "zero-blocks" => 0,
3797         "compress" => 0
3798     };
3799
3800     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
3801
3802     for my $supported_capability (@$supported_capabilities) {
3803         push @$cap_ref, {
3804             capability => $supported_capability->{capability},
3805             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
3806         };
3807     }
3808
3809     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
3810 }
3811
3812 my $fast_plug_option = {
3813     'lock' => 1,
3814     'name' => 1,
3815     'onboot' => 1,
3816     'shares' => 1,
3817     'startup' => 1,
3818     'description' => 1,
3819 };
3820
3821 # hotplug changes in [PENDING]
3822 # $selection hash can be used to only apply specified options, for
3823 # example: { cores => 1 } (only apply changed 'cores')
3824 # $errors ref is used to return error messages
3825 sub vmconfig_hotplug_pending {
3826     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
3827
3828     my $defaults = load_defaults();
3829
3830     # commit values which do not have any impact on running VM first
3831     # Note: those option cannot raise errors, we we do not care about
3832     # $selection and always apply them.
3833
3834     my $add_error = sub {
3835         my ($opt, $msg) = @_;
3836         $errors->{$opt} = "hotplug problem - $msg";
3837     };
3838
3839     my $changes = 0;
3840     foreach my $opt (keys %{$conf->{pending}}) { # add/change
3841         if ($fast_plug_option->{$opt}) {
3842             $conf->{$opt} = $conf->{pending}->{$opt};
3843             delete $conf->{pending}->{$opt};
3844             $changes = 1;
3845         }
3846     }
3847
3848     if ($changes) {
3849         PVE::QemuConfig->write_config($vmid, $conf);
3850         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
3851     }
3852
3853     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3854
3855     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
3856     while (my ($opt, $force) = each %$pending_delete_hash) {
3857         next if $selection && !$selection->{$opt};
3858         eval {
3859             if ($opt eq 'hotplug') {
3860                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
3861             } elsif ($opt eq 'tablet') {
3862                 die "skip\n" if !$hotplug_features->{usb};
3863                 if ($defaults->{tablet}) {
3864                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
3865                 } else {
3866                     vm_deviceunplug($vmid, $conf, $opt);
3867                 }
3868             } elsif ($opt eq 'vcpus') {
3869                 die "skip\n" if !$hotplug_features->{cpu};
3870                 qemu_cpu_hotplug($vmid, $conf, undef);
3871             } elsif ($opt eq 'balloon') {
3872                 # enable balloon device is not hotpluggable
3873                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
3874             } elsif ($fast_plug_option->{$opt}) {
3875                 # do nothing
3876             } elsif ($opt =~ m/^net(\d+)$/) {
3877                 die "skip\n" if !$hotplug_features->{network};
3878                 vm_deviceunplug($vmid, $conf, $opt);
3879             } elsif (is_valid_drivename($opt)) {
3880                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
3881                 vm_deviceunplug($vmid, $conf, $opt);
3882                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
3883             } elsif ($opt =~ m/^memory$/) {
3884                 die "skip\n" if !$hotplug_features->{memory};
3885                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
3886             } elsif ($opt eq 'cpuunits') {
3887                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
3888             } elsif ($opt eq 'cpulimit') {
3889                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
3890             } else {
3891                 die "skip\n";
3892             }
3893         };
3894         if (my $err = $@) {
3895             &$add_error($opt, $err) if $err ne "skip\n";
3896         } else {
3897             # save new config if hotplug was successful
3898             delete $conf->{$opt};
3899             vmconfig_undelete_pending_option($conf, $opt);
3900             PVE::QemuConfig->write_config($vmid, $conf);
3901             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
3902         }
3903     }
3904
3905     foreach my $opt (keys %{$conf->{pending}}) {
3906         next if $selection && !$selection->{$opt};
3907         my $value = $conf->{pending}->{$opt};
3908         eval {
3909             if ($opt eq 'hotplug') {
3910                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
3911             } elsif ($opt eq 'tablet') {
3912                 die "skip\n" if !$hotplug_features->{usb};
3913                 if ($value == 1) {
3914                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
3915                 } elsif ($value == 0) {
3916                     vm_deviceunplug($vmid, $conf, $opt);
3917                 }
3918             } elsif ($opt eq 'vcpus') {
3919                 die "skip\n" if !$hotplug_features->{cpu};
3920                 qemu_cpu_hotplug($vmid, $conf, $value);
3921             } elsif ($opt eq 'balloon') {
3922                 # enable/disable balloning device is not hotpluggable
3923                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
3924                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
3925                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
3926
3927                 # allow manual ballooning if shares is set to zero
3928                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
3929                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
3930                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
3931                 }
3932             } elsif ($opt =~ m/^net(\d+)$/) {
3933                 # some changes can be done without hotplug
3934                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
3935                                     $vmid, $opt, $value);
3936             } elsif (is_valid_drivename($opt)) {
3937                 # some changes can be done without hotplug
3938                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
3939                                      $vmid, $opt, $value, 1);
3940             } elsif ($opt =~ m/^memory$/) { #dimms
3941                 die "skip\n" if !$hotplug_features->{memory};
3942                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
3943             } elsif ($opt eq 'cpuunits') {
3944                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
3945             } elsif ($opt eq 'cpulimit') {
3946                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
3947                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
3948             } else {
3949                 die "skip\n";  # skip non-hot-pluggable options
3950             }
3951         };
3952         if (my $err = $@) {
3953             &$add_error($opt, $err) if $err ne "skip\n";
3954         } else {
3955             # save new config if hotplug was successful
3956             $conf->{$opt} = $value;
3957             delete $conf->{pending}->{$opt};
3958             PVE::QemuConfig->write_config($vmid, $conf);
3959             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
3960         }
3961     }
3962 }
3963
3964 sub try_deallocate_drive {
3965     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
3966
3967     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
3968         my $volid = $drive->{file};
3969         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
3970             my $sid = PVE::Storage::parse_volume_id($volid);
3971             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
3972
3973             # check if the disk is really unused
3974             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
3975                 if is_volume_in_use($storecfg, $conf, $key, $volid);
3976             PVE::Storage::vdisk_free($storecfg, $volid);
3977             return 1;
3978         } else {
3979             # If vm is not owner of this disk remove from config
3980             return 1;
3981         }
3982     }
3983
3984     return undef;
3985 }
3986
3987 sub vmconfig_delete_or_detach_drive {
3988     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
3989
3990     my $drive = parse_drive($opt, $conf->{$opt});
3991
3992     my $rpcenv = PVE::RPCEnvironment::get();
3993     my $authuser = $rpcenv->get_user();
3994
3995     if ($force) {
3996         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
3997         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
3998     } else {
3999         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4000     }
4001 }
4002
4003 sub vmconfig_apply_pending {
4004     my ($vmid, $conf, $storecfg) = @_;
4005
4006     # cold plug
4007
4008     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4009     while (my ($opt, $force) = each %$pending_delete_hash) {
4010         die "internal error" if $opt =~ m/^unused/;
4011         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4012         if (!defined($conf->{$opt})) {
4013             vmconfig_undelete_pending_option($conf, $opt);
4014             PVE::QemuConfig->write_config($vmid, $conf);
4015         } elsif (is_valid_drivename($opt)) {
4016             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4017             vmconfig_undelete_pending_option($conf, $opt);
4018             delete $conf->{$opt};
4019             PVE::QemuConfig->write_config($vmid, $conf);
4020         } else {
4021             vmconfig_undelete_pending_option($conf, $opt);
4022             delete $conf->{$opt};
4023             PVE::QemuConfig->write_config($vmid, $conf);
4024         }
4025     }
4026
4027     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4028
4029     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4030         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4031
4032         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4033             # skip if nothing changed
4034         } elsif (is_valid_drivename($opt)) {
4035             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4036                 if defined($conf->{$opt});
4037             $conf->{$opt} = $conf->{pending}->{$opt};
4038         } else {
4039             $conf->{$opt} = $conf->{pending}->{$opt};
4040         }
4041
4042         delete $conf->{pending}->{$opt};
4043         PVE::QemuConfig->write_config($vmid, $conf);
4044     }
4045 }
4046
4047 my $safe_num_ne = sub {
4048     my ($a, $b) = @_;
4049
4050     return 0 if !defined($a) && !defined($b);
4051     return 1 if !defined($a);
4052     return 1 if !defined($b);
4053
4054     return $a != $b;
4055 };
4056
4057 my $safe_string_ne = sub {
4058     my ($a, $b) = @_;
4059
4060     return 0 if !defined($a) && !defined($b);
4061     return 1 if !defined($a);
4062     return 1 if !defined($b);
4063
4064     return $a ne $b;
4065 };
4066
4067 sub vmconfig_update_net {
4068     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4069
4070     my $newnet = parse_net($value);
4071
4072     if ($conf->{$opt}) {
4073         my $oldnet = parse_net($conf->{$opt});
4074
4075         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4076             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4077             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4078             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4079
4080             # for non online change, we try to hot-unplug
4081             die "skip\n" if !$hotplug;
4082             vm_deviceunplug($vmid, $conf, $opt);
4083         } else {
4084
4085             die "internal error" if $opt !~ m/net(\d+)/;
4086             my $iface = "tap${vmid}i$1";
4087
4088             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4089                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4090                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4091                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4092                 PVE::Network::tap_unplug($iface);
4093                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4094             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4095                 # Rate can be applied on its own but any change above needs to
4096                 # include the rate in tap_plug since OVS resets everything.
4097                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4098             }
4099
4100             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4101                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4102             }
4103
4104             return 1;
4105         }
4106     }
4107
4108     if ($hotplug) {
4109         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4110     } else {
4111         die "skip\n";
4112     }
4113 }
4114
4115 sub vmconfig_update_disk {
4116     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4117
4118     # fixme: do we need force?
4119
4120     my $drive = parse_drive($opt, $value);
4121
4122     if ($conf->{$opt}) {
4123
4124         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4125
4126             my $media = $drive->{media} || 'disk';
4127             my $oldmedia = $old_drive->{media} || 'disk';
4128             die "unable to change media type\n" if $media ne $oldmedia;
4129
4130             if (!drive_is_cdrom($old_drive)) {
4131
4132                 if ($drive->{file} ne $old_drive->{file}) {
4133
4134                     die "skip\n" if !$hotplug;
4135
4136                     # unplug and register as unused
4137                     vm_deviceunplug($vmid, $conf, $opt);
4138                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4139
4140                 } else {
4141                     # update existing disk
4142
4143                     # skip non hotpluggable value
4144                     if (&$safe_num_ne($drive->{discard}, $old_drive->{discard}) ||
4145                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4146                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4147                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4148                         die "skip\n";
4149                     }
4150
4151                     # apply throttle
4152                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4153                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4154                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4155                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4156                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4157                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4158                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4159                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4160                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4161                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4162                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4163                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max})) {
4164
4165                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4166                                                    ($drive->{mbps} || 0)*1024*1024,
4167                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4168                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4169                                                    $drive->{iops} || 0,
4170                                                    $drive->{iops_rd} || 0,
4171                                                    $drive->{iops_wr} || 0,
4172                                                    ($drive->{mbps_max} || 0)*1024*1024,
4173                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4174                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4175                                                    $drive->{iops_max} || 0,
4176                                                    $drive->{iops_rd_max} || 0,
4177                                                    $drive->{iops_wr_max} || 0);
4178
4179                     }
4180
4181                     return 1;
4182                 }
4183
4184             } else { # cdrom
4185
4186                 if ($drive->{file} eq 'none') {
4187                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4188                 } else {
4189                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4190                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4191                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4192                 }
4193
4194                 return 1;
4195             }
4196         }
4197     }
4198
4199     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4200     # hotplug new disks
4201     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4202     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4203 }
4204
4205 sub vm_start {
4206     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4207         $forcemachine, $spice_ticket) = @_;
4208
4209     PVE::QemuConfig->lock_config($vmid, sub {
4210         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4211
4212         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
4213
4214         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4215
4216         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4217
4218         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4219             vmconfig_apply_pending($vmid, $conf, $storecfg);
4220             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4221         }
4222
4223         my $defaults = load_defaults();
4224
4225         # set environment variable useful inside network script
4226         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4227
4228         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4229
4230         my $migrate_port = 0;
4231         my $migrate_uri;
4232         if ($statefile) {
4233             if ($statefile eq 'tcp') {
4234                 my $localip = "localhost";
4235                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4236                 my $nodename = PVE::INotify::nodename();
4237                 if ($datacenterconf->{migration_unsecure}) {
4238                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4239                         $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4240                 }
4241                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4242                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4243                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4244                 push @$cmd, '-incoming', $migrate_uri;
4245                 push @$cmd, '-S';
4246
4247             } elsif ($statefile eq 'unix') {
4248                 # should be default for secure migrations as a ssh TCP forward
4249                 # tunnel is not deterministic reliable ready and fails regurarly
4250                 # to set up in time, so use UNIX socket forwards
4251                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
4252                 unlink $socket_addr;
4253
4254                 $migrate_uri = "unix:$socket_addr";
4255
4256                 push @$cmd, '-incoming', $migrate_uri;
4257                 push @$cmd, '-S';
4258
4259             } else {
4260                 push @$cmd, '-loadstate', $statefile;
4261             }
4262         } elsif ($paused) {
4263             push @$cmd, '-S';
4264         }
4265
4266         # host pci devices
4267         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4268           my $d = parse_hostpci($conf->{"hostpci$i"});
4269           next if !$d;
4270           my $pcidevices = $d->{pciid};
4271           foreach my $pcidevice (@$pcidevices) {
4272                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4273
4274                 my $info = pci_device_info("0000:$pciid");
4275                 die "IOMMU not present\n" if !check_iommu_support();
4276                 die "no pci device info for device '$pciid'\n" if !$info;
4277                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4278                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4279           }
4280         }
4281
4282         PVE::Storage::activate_volumes($storecfg, $vollist);
4283
4284         if (!check_running($vmid, 1) && -d "/sys/fs/cgroup/systemd/qemu.slice/$vmid.scope") {
4285             my $cmd = [];
4286             push @$cmd, '/bin/systemctl', 'stop', "$vmid.scope";
4287             eval  { run_command($cmd); };
4288         }
4289
4290         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
4291                                                   : $defaults->{cpuunits};
4292
4293         my %run_params = (timeout => $statefile ? undef : 30, umask => 0077);
4294
4295         my %properties = (
4296             Slice => 'qemu.slice',
4297             KillMode => 'none',
4298             CPUShares => $cpuunits
4299         );
4300
4301         if (my $cpulimit = $conf->{cpulimit}) {
4302             $properties{CPUQuota} = int($cpulimit * 100);
4303         }
4304         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
4305
4306         if ($conf->{hugepages}) {
4307
4308             my $code = sub {
4309                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
4310                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
4311
4312                 PVE::QemuServer::Memory::hugepages_mount();
4313                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
4314
4315                 eval  {
4316                     PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4317                     run_command($cmd, %run_params);
4318                 };
4319
4320                 if (my $err = $@) {
4321                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
4322                     die $err;
4323                 }
4324
4325                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
4326             };
4327             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
4328
4329         } else {
4330             eval  {
4331                 PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4332                 run_command($cmd, %run_params);
4333             };
4334         }
4335
4336         if (my $err = $@) {
4337             # deactivate volumes if start fails
4338             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
4339             die "start failed: $err";
4340         }
4341
4342         print "migration listens on $migrate_uri\n" if $migrate_uri;
4343
4344         if ($statefile && $statefile ne 'tcp')  {
4345             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4346             warn $@ if $@;
4347         }
4348
4349         if ($migratedfrom) {
4350
4351             eval {
4352                 set_migration_caps($vmid);
4353             };
4354             warn $@ if $@;
4355
4356             if ($spice_port) {
4357                 print "spice listens on port $spice_port\n";
4358                 if ($spice_ticket) {
4359                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4360                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4361                 }
4362             }
4363
4364         } else {
4365
4366             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4367                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4368                     if $conf->{balloon};
4369             }
4370
4371             foreach my $opt (keys %$conf) {
4372                 next if $opt !~  m/^net\d+$/;
4373                 my $nicconf = parse_net($conf->{$opt});
4374                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4375             }
4376         }
4377
4378         vm_mon_cmd_nocheck($vmid, 'qom-set',
4379                     path => "machine/peripheral/balloon0",
4380                     property => "guest-stats-polling-interval",
4381                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4382
4383     });
4384 }
4385
4386 sub vm_mon_cmd {
4387     my ($vmid, $execute, %params) = @_;
4388
4389     my $cmd = { execute => $execute, arguments => \%params };
4390     vm_qmp_command($vmid, $cmd);
4391 }
4392
4393 sub vm_mon_cmd_nocheck {
4394     my ($vmid, $execute, %params) = @_;
4395
4396     my $cmd = { execute => $execute, arguments => \%params };
4397     vm_qmp_command($vmid, $cmd, 1);
4398 }
4399
4400 sub vm_qmp_command {
4401     my ($vmid, $cmd, $nocheck) = @_;
4402
4403     my $res;
4404
4405     my $timeout;
4406     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4407         $timeout = $cmd->{arguments}->{timeout};
4408         delete $cmd->{arguments}->{timeout};
4409     }
4410
4411     eval {
4412         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4413         my $sname = qmp_socket($vmid);
4414         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4415             my $qmpclient = PVE::QMPClient->new();
4416
4417             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4418         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
4419             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
4420                 if scalar(%{$cmd->{arguments}});
4421             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
4422         } else {
4423             die "unable to open monitor socket\n";
4424         }
4425     };
4426     if (my $err = $@) {
4427         syslog("err", "VM $vmid qmp command failed - $err");
4428         die $err;
4429     }
4430
4431     return $res;
4432 }
4433
4434 sub vm_human_monitor_command {
4435     my ($vmid, $cmdline) = @_;
4436
4437     my $res;
4438
4439     my $cmd = {
4440         execute => 'human-monitor-command',
4441         arguments => { 'command-line' => $cmdline},
4442     };
4443
4444     return vm_qmp_command($vmid, $cmd);
4445 }
4446
4447 sub vm_commandline {
4448     my ($storecfg, $vmid) = @_;
4449
4450     my $conf = PVE::QemuConfig->load_config($vmid);
4451
4452     my $defaults = load_defaults();
4453
4454     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
4455
4456     return join(' ', @$cmd);
4457 }
4458
4459 sub vm_reset {
4460     my ($vmid, $skiplock) = @_;
4461
4462     PVE::QemuConfig->lock_config($vmid, sub {
4463
4464         my $conf = PVE::QemuConfig->load_config($vmid);
4465
4466         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4467
4468         vm_mon_cmd($vmid, "system_reset");
4469     });
4470 }
4471
4472 sub get_vm_volumes {
4473     my ($conf) = @_;
4474
4475     my $vollist = [];
4476     foreach_volid($conf, sub {
4477         my ($volid, $is_cdrom) = @_;
4478
4479         return if $volid =~ m|^/|;
4480
4481         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4482         return if !$sid;
4483
4484         push @$vollist, $volid;
4485     });
4486
4487     return $vollist;
4488 }
4489
4490 sub vm_stop_cleanup {
4491     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
4492
4493     eval {
4494
4495         if (!$keepActive) {
4496             my $vollist = get_vm_volumes($conf);
4497             PVE::Storage::deactivate_volumes($storecfg, $vollist);
4498         }
4499
4500         foreach my $ext (qw(mon qmp pid vnc qga)) {
4501             unlink "/var/run/qemu-server/${vmid}.$ext";
4502         }
4503
4504         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
4505     };
4506     warn $@ if $@; # avoid errors - just warn
4507 }
4508
4509 # Note: use $nockeck to skip tests if VM configuration file exists.
4510 # We need that when migration VMs to other nodes (files already moved)
4511 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
4512 sub vm_stop {
4513     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
4514
4515     $force = 1 if !defined($force) && !$shutdown;
4516
4517     if ($migratedfrom){
4518         my $pid = check_running($vmid, $nocheck, $migratedfrom);
4519         kill 15, $pid if $pid;
4520         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4521         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
4522         return;
4523     }
4524
4525     PVE::QemuConfig->lock_config($vmid, sub {
4526
4527         my $pid = check_running($vmid, $nocheck);
4528         return if !$pid;
4529
4530         my $conf;
4531         if (!$nocheck) {
4532             $conf = PVE::QemuConfig->load_config($vmid);
4533             PVE::QemuConfig->check_lock($conf) if !$skiplock;
4534             if (!defined($timeout) && $shutdown && $conf->{startup}) {
4535                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
4536                 $timeout = $opts->{down} if $opts->{down};
4537             }
4538         }
4539
4540         $timeout = 60 if !defined($timeout);
4541
4542         eval {
4543             if ($shutdown) {
4544                 if (defined($conf) && $conf->{agent}) {
4545                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
4546                 } else {
4547                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
4548                 }
4549             } else {
4550                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
4551             }
4552         };
4553         my $err = $@;
4554
4555         if (!$err) {
4556             my $count = 0;
4557             while (($count < $timeout) && check_running($vmid, $nocheck)) {
4558                 $count++;
4559                 sleep 1;
4560             }
4561
4562             if ($count >= $timeout) {
4563                 if ($force) {
4564                     warn "VM still running - terminating now with SIGTERM\n";
4565                     kill 15, $pid;
4566                 } else {
4567                     die "VM quit/powerdown failed - got timeout\n";
4568                 }
4569             } else {
4570                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4571                 return;
4572             }
4573         } else {
4574             if ($force) {
4575                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
4576                 kill 15, $pid;
4577             } else {
4578                 die "VM quit/powerdown failed\n";
4579             }
4580         }
4581
4582         # wait again
4583         $timeout = 10;
4584
4585         my $count = 0;
4586         while (($count < $timeout) && check_running($vmid, $nocheck)) {
4587             $count++;
4588             sleep 1;
4589         }
4590
4591         if ($count >= $timeout) {
4592             warn "VM still running - terminating now with SIGKILL\n";
4593             kill 9, $pid;
4594             sleep 1;
4595         }
4596
4597         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4598    });
4599 }
4600
4601 sub vm_suspend {
4602     my ($vmid, $skiplock) = @_;
4603
4604     PVE::QemuConfig->lock_config($vmid, sub {
4605
4606         my $conf = PVE::QemuConfig->load_config($vmid);
4607
4608         PVE::QemuConfig->check_lock($conf)
4609             if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
4610
4611         vm_mon_cmd($vmid, "stop");
4612     });
4613 }
4614
4615 sub vm_resume {
4616     my ($vmid, $skiplock, $nocheck) = @_;
4617
4618     PVE::QemuConfig->lock_config($vmid, sub {
4619
4620         if (!$nocheck) {
4621
4622             my $conf = PVE::QemuConfig->load_config($vmid);
4623
4624             PVE::QemuConfig->check_lock($conf)
4625                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
4626
4627             vm_mon_cmd($vmid, "cont");
4628
4629         } else {
4630             vm_mon_cmd_nocheck($vmid, "cont");
4631         }
4632     });
4633 }
4634
4635 sub vm_sendkey {
4636     my ($vmid, $skiplock, $key) = @_;
4637
4638     PVE::QemuConfig->lock_config($vmid, sub {
4639
4640         my $conf = PVE::QemuConfig->load_config($vmid);
4641
4642         # there is no qmp command, so we use the human monitor command
4643         vm_human_monitor_command($vmid, "sendkey $key");
4644     });
4645 }
4646
4647 sub vm_destroy {
4648     my ($storecfg, $vmid, $skiplock) = @_;
4649
4650     PVE::QemuConfig->lock_config($vmid, sub {
4651
4652         my $conf = PVE::QemuConfig->load_config($vmid);
4653
4654         if (!check_running($vmid)) {
4655             destroy_vm($storecfg, $vmid, undef, $skiplock);
4656         } else {
4657             die "VM $vmid is running - destroy failed\n";
4658         }
4659     });
4660 }
4661
4662 # pci helpers
4663
4664 sub file_write {
4665     my ($filename, $buf) = @_;
4666
4667     my $fh = IO::File->new($filename, "w");
4668     return undef if !$fh;
4669
4670     my $res = print $fh $buf;
4671
4672     $fh->close();
4673
4674     return $res;
4675 }
4676
4677 sub pci_device_info {
4678     my ($name) = @_;
4679
4680     my $res;
4681
4682     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
4683     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
4684
4685     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
4686     return undef if !defined($irq) || $irq !~ m/^\d+$/;
4687
4688     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
4689     return undef if !defined($vendor) || $vendor !~ s/^0x//;
4690
4691     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
4692     return undef if !defined($product) || $product !~ s/^0x//;
4693
4694     $res = {
4695         name => $name,
4696         vendor => $vendor,
4697         product => $product,
4698         domain => $domain,
4699         bus => $bus,
4700         slot => $slot,
4701         func => $func,
4702         irq => $irq,
4703         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
4704     };
4705
4706     return $res;
4707 }
4708
4709 sub pci_dev_reset {
4710     my ($dev) = @_;
4711
4712     my $name = $dev->{name};
4713
4714     my $fn = "$pcisysfs/devices/$name/reset";
4715
4716     return file_write($fn, "1");
4717 }
4718
4719 sub pci_dev_bind_to_vfio {
4720     my ($dev) = @_;
4721
4722     my $name = $dev->{name};
4723
4724     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4725
4726     if (!-d $vfio_basedir) {
4727         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4728     }
4729     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4730
4731     my $testdir = "$vfio_basedir/$name";
4732     return 1 if -d $testdir;
4733
4734     my $data = "$dev->{vendor} $dev->{product}";
4735     return undef if !file_write("$vfio_basedir/new_id", $data);
4736
4737     my $fn = "$pcisysfs/devices/$name/driver/unbind";
4738     if (!file_write($fn, $name)) {
4739         return undef if -f $fn;
4740     }
4741
4742     $fn = "$vfio_basedir/bind";
4743     if (! -d $testdir) {
4744         return undef if !file_write($fn, $name);
4745     }
4746
4747     return -d $testdir;
4748 }
4749
4750 sub pci_dev_group_bind_to_vfio {
4751     my ($pciid) = @_;
4752
4753     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4754
4755     if (!-d $vfio_basedir) {
4756         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4757     }
4758     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4759
4760     # get IOMMU group devices
4761     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
4762       my @devs = grep /^0000:/, readdir($D);
4763     closedir($D);
4764
4765     foreach my $pciid (@devs) {
4766         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
4767
4768         # pci bridges, switches or root ports are not supported
4769         # they have a pci_bus subdirectory so skip them
4770         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
4771
4772         my $info = pci_device_info($1);
4773         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
4774     }
4775
4776     return 1;
4777 }
4778
4779 # vzdump restore implementaion
4780
4781 sub tar_archive_read_firstfile {
4782     my $archive = shift;
4783
4784     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
4785
4786     # try to detect archive type first
4787     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
4788         die "unable to open file '$archive'\n";
4789     my $firstfile = <$fh>;
4790     kill 15, $pid;
4791     close $fh;
4792
4793     die "ERROR: archive contaions no data\n" if !$firstfile;
4794     chomp $firstfile;
4795
4796     return $firstfile;
4797 }
4798
4799 sub tar_restore_cleanup {
4800     my ($storecfg, $statfile) = @_;
4801
4802     print STDERR "starting cleanup\n";
4803
4804     if (my $fd = IO::File->new($statfile, "r")) {
4805         while (defined(my $line = <$fd>)) {
4806             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
4807                 my $volid = $2;
4808                 eval {
4809                     if ($volid =~ m|^/|) {
4810                         unlink $volid || die 'unlink failed\n';
4811                     } else {
4812                         PVE::Storage::vdisk_free($storecfg, $volid);
4813                     }
4814                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
4815                 };
4816                 print STDERR "unable to cleanup '$volid' - $@" if $@;
4817             } else {
4818                 print STDERR "unable to parse line in statfile - $line";
4819             }
4820         }
4821         $fd->close();
4822     }
4823 }
4824
4825 sub restore_archive {
4826     my ($archive, $vmid, $user, $opts) = @_;
4827
4828     my $format = $opts->{format};
4829     my $comp;
4830
4831     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
4832         $format = 'tar' if !$format;
4833         $comp = 'gzip';
4834     } elsif ($archive =~ m/\.tar$/) {
4835         $format = 'tar' if !$format;
4836     } elsif ($archive =~ m/.tar.lzo$/) {
4837         $format = 'tar' if !$format;
4838         $comp = 'lzop';
4839     } elsif ($archive =~ m/\.vma$/) {
4840         $format = 'vma' if !$format;
4841     } elsif ($archive =~ m/\.vma\.gz$/) {
4842         $format = 'vma' if !$format;
4843         $comp = 'gzip';
4844     } elsif ($archive =~ m/\.vma\.lzo$/) {
4845         $format = 'vma' if !$format;
4846         $comp = 'lzop';
4847     } else {
4848         $format = 'vma' if !$format; # default
4849     }
4850
4851     # try to detect archive format
4852     if ($format eq 'tar') {
4853         return restore_tar_archive($archive, $vmid, $user, $opts);
4854     } else {
4855         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
4856     }
4857 }
4858
4859 sub restore_update_config_line {
4860     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
4861
4862     return if $line =~ m/^\#qmdump\#/;
4863     return if $line =~ m/^\#vzdump\#/;
4864     return if $line =~ m/^lock:/;
4865     return if $line =~ m/^unused\d+:/;
4866     return if $line =~ m/^parent:/;
4867     return if $line =~ m/^template:/; # restored VM is never a template
4868
4869     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
4870         # try to convert old 1.X settings
4871         my ($id, $ind, $ethcfg) = ($1, $2, $3);
4872         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
4873             my ($model, $macaddr) = split(/\=/, $devconfig);
4874             $macaddr = PVE::Tools::random_ether_addr() if !$macaddr || $unique;
4875             my $net = {
4876                 model => $model,
4877                 bridge => "vmbr$ind",
4878                 macaddr => $macaddr,
4879             };
4880             my $netstr = print_net($net);
4881
4882             print $outfd "net$cookie->{netcount}: $netstr\n";
4883             $cookie->{netcount}++;
4884         }
4885     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
4886         my ($id, $netstr) = ($1, $2);
4887         my $net = parse_net($netstr);
4888         $net->{macaddr} = PVE::Tools::random_ether_addr() if $net->{macaddr};
4889         $netstr = print_net($net);
4890         print $outfd "$id: $netstr\n";
4891     } elsif ($line =~ m/^((ide|scsi|virtio|sata)\d+):\s*(\S+)\s*$/) {
4892         my $virtdev = $1;
4893         my $value = $3;
4894         my $di = parse_drive($virtdev, $value);
4895         if (defined($di->{backup}) && !$di->{backup}) {
4896             print $outfd "#$line";
4897         } elsif ($map->{$virtdev}) {
4898             delete $di->{format}; # format can change on restore
4899             $di->{file} = $map->{$virtdev};
4900             $value = print_drive($vmid, $di);
4901             print $outfd "$virtdev: $value\n";
4902         } else {
4903             print $outfd $line;
4904         }
4905     } else {
4906         print $outfd $line;
4907     }
4908 }
4909
4910 sub scan_volids {
4911     my ($cfg, $vmid) = @_;
4912
4913     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
4914
4915     my $volid_hash = {};
4916     foreach my $storeid (keys %$info) {
4917         foreach my $item (@{$info->{$storeid}}) {
4918             next if !($item->{volid} && $item->{size});
4919             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
4920             $volid_hash->{$item->{volid}} = $item;
4921         }
4922     }
4923
4924     return $volid_hash;
4925 }
4926
4927 sub is_volume_in_use {
4928     my ($storecfg, $conf, $skip_drive, $volid) = @_;
4929
4930     my $path = PVE::Storage::path($storecfg, $volid);
4931
4932     my $scan_config = sub {
4933         my ($cref, $snapname) = @_;
4934
4935         foreach my $key (keys %$cref) {
4936             my $value = $cref->{$key};
4937             if (is_valid_drivename($key)) {
4938                 next if $skip_drive && $key eq $skip_drive;
4939                 my $drive = parse_drive($key, $value);
4940                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
4941                 return 1 if $volid eq $drive->{file};
4942                 if ($drive->{file} =~ m!^/!) {
4943                     return 1 if $drive->{file} eq $path;
4944                 } else {
4945                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
4946                     next if !$storeid;
4947                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
4948                     next if !$scfg;
4949                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
4950                 }
4951             }
4952         }
4953
4954         return 0;
4955     };
4956
4957     return 1 if &$scan_config($conf);
4958
4959     undef $skip_drive;
4960
4961     foreach my $snapname (keys %{$conf->{snapshots}}) {
4962         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
4963     }
4964
4965     return 0;
4966 }
4967
4968 sub update_disksize {
4969     my ($vmid, $conf, $volid_hash) = @_;
4970
4971     my $changes;
4972
4973     my $used = {};
4974
4975     # Note: it is allowed to define multiple storages with same path (alias), so
4976     # we need to check both 'volid' and real 'path' (two different volid can point
4977     # to the same path).
4978
4979     my $usedpath = {};
4980
4981     # update size info
4982     foreach my $opt (keys %$conf) {
4983         if (is_valid_drivename($opt)) {
4984             my $drive = parse_drive($opt, $conf->{$opt});
4985             my $volid = $drive->{file};
4986             next if !$volid;
4987
4988             $used->{$volid} = 1;
4989             if ($volid_hash->{$volid} &&
4990                 (my $path = $volid_hash->{$volid}->{path})) {
4991                 $usedpath->{$path} = 1;
4992             }
4993
4994             next if drive_is_cdrom($drive);
4995             next if !$volid_hash->{$volid};
4996
4997             $drive->{size} = $volid_hash->{$volid}->{size};
4998             my $new = print_drive($vmid, $drive);
4999             if ($new ne $conf->{$opt}) {
5000                 $changes = 1;
5001                 $conf->{$opt} = $new;
5002             }
5003         }
5004     }
5005
5006     # remove 'unusedX' entry if volume is used
5007     foreach my $opt (keys %$conf) {
5008         next if $opt !~ m/^unused\d+$/;
5009         my $volid = $conf->{$opt};
5010         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5011         if ($used->{$volid} || ($path && $usedpath->{$path})) {
5012             $changes = 1;
5013             delete $conf->{$opt};
5014         }
5015     }
5016
5017     foreach my $volid (sort keys %$volid_hash) {
5018         next if $volid =~ m/vm-$vmid-state-/;
5019         next if $used->{$volid};
5020         my $path = $volid_hash->{$volid}->{path};
5021         next if !$path; # just to be sure
5022         next if $usedpath->{$path};
5023         $changes = 1;
5024         PVE::QemuConfig->add_unused_volume($conf, $volid);
5025         $usedpath->{$path} = 1; # avoid to add more than once (aliases)
5026     }
5027
5028     return $changes;
5029 }
5030
5031 sub rescan {
5032     my ($vmid, $nolock) = @_;
5033
5034     my $cfg = PVE::Storage::config();
5035
5036     my $volid_hash = scan_volids($cfg, $vmid);
5037
5038     my $updatefn =  sub {
5039         my ($vmid) = @_;
5040
5041         my $conf = PVE::QemuConfig->load_config($vmid);
5042
5043         PVE::QemuConfig->check_lock($conf);
5044
5045         my $vm_volids = {};
5046         foreach my $volid (keys %$volid_hash) {
5047             my $info = $volid_hash->{$volid};
5048             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5049         }
5050
5051         my $changes = update_disksize($vmid, $conf, $vm_volids);
5052
5053         PVE::QemuConfig->write_config($vmid, $conf) if $changes;
5054     };
5055
5056     if (defined($vmid)) {
5057         if ($nolock) {
5058             &$updatefn($vmid);
5059         } else {
5060             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5061         }
5062     } else {
5063         my $vmlist = config_list();
5064         foreach my $vmid (keys %$vmlist) {
5065             if ($nolock) {
5066                 &$updatefn($vmid);
5067             } else {
5068                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5069             }
5070         }
5071     }
5072 }
5073
5074 sub restore_vma_archive {
5075     my ($archive, $vmid, $user, $opts, $comp) = @_;
5076
5077     my $input = $archive eq '-' ? "<&STDIN" : undef;
5078     my $readfrom = $archive;
5079
5080     my $uncomp = '';
5081     if ($comp) {
5082         $readfrom = '-';
5083         my $qarchive = PVE::Tools::shellquote($archive);
5084         if ($comp eq 'gzip') {
5085             $uncomp = "zcat $qarchive|";
5086         } elsif ($comp eq 'lzop') {
5087             $uncomp = "lzop -d -c $qarchive|";
5088         } else {
5089             die "unknown compression method '$comp'\n";
5090         }
5091
5092     }
5093
5094     my $tmpdir = "/var/tmp/vzdumptmp$$";
5095     rmtree $tmpdir;
5096
5097     # disable interrupts (always do cleanups)
5098     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5099         warn "got interrupt - ignored\n";
5100     };
5101
5102     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5103     POSIX::mkfifo($mapfifo, 0600);
5104     my $fifofh;
5105
5106     my $openfifo = sub {
5107         open($fifofh, '>', $mapfifo) || die $!;
5108     };
5109
5110     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5111
5112     my $oldtimeout;
5113     my $timeout = 5;
5114
5115     my $devinfo = {};
5116
5117     my $rpcenv = PVE::RPCEnvironment::get();
5118
5119     my $conffile = PVE::QemuConfig->config_file($vmid);
5120     my $tmpfn = "$conffile.$$.tmp";
5121
5122     # Note: $oldconf is undef if VM does not exists
5123     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
5124     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
5125
5126     my $print_devmap = sub {
5127         my $virtdev_hash = {};
5128
5129         my $cfgfn = "$tmpdir/qemu-server.conf";
5130
5131         # we can read the config - that is already extracted
5132         my $fh = IO::File->new($cfgfn, "r") ||
5133             "unable to read qemu-server.conf - $!\n";
5134
5135         my $fwcfgfn = "$tmpdir/qemu-server.fw";
5136         if (-f $fwcfgfn) {
5137             my $pve_firewall_dir = '/etc/pve/firewall';
5138             mkdir $pve_firewall_dir; # make sure the dir exists
5139             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
5140         }
5141
5142         while (defined(my $line = <$fh>)) {
5143             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5144                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5145                 die "archive does not contain data for drive '$virtdev'\n"
5146                     if !$devinfo->{$devname};
5147                 if (defined($opts->{storage})) {
5148                     $storeid = $opts->{storage} || 'local';
5149                 } elsif (!$storeid) {
5150                     $storeid = 'local';
5151                 }
5152                 $format = 'raw' if !$format;
5153                 $devinfo->{$devname}->{devname} = $devname;
5154                 $devinfo->{$devname}->{virtdev} = $virtdev;
5155                 $devinfo->{$devname}->{format} = $format;
5156                 $devinfo->{$devname}->{storeid} = $storeid;
5157
5158                 # check permission on storage
5159                 my $pool = $opts->{pool}; # todo: do we need that?
5160                 if ($user ne 'root@pam') {
5161                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5162                 }
5163
5164                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5165             }
5166         }
5167
5168         foreach my $devname (keys %$devinfo) {
5169             die "found no device mapping information for device '$devname'\n"
5170                 if !$devinfo->{$devname}->{virtdev};
5171         }
5172
5173         my $cfg = PVE::Storage::config();
5174
5175         # create empty/temp config
5176         if ($oldconf) {
5177             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5178             foreach_drive($oldconf, sub {
5179                 my ($ds, $drive) = @_;
5180
5181                 return if drive_is_cdrom($drive);
5182
5183                 my $volid = $drive->{file};
5184
5185                 return if !$volid || $volid =~ m|^/|;
5186
5187                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5188                 return if !$path || !$owner || ($owner != $vmid);
5189
5190                 # Note: only delete disk we want to restore
5191                 # other volumes will become unused
5192                 if ($virtdev_hash->{$ds}) {
5193                     PVE::Storage::vdisk_free($cfg, $volid);
5194                 }
5195             });
5196
5197             # delete vmstate files
5198             # since after the restore we have no snapshots anymore
5199             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5200                 my $snap = $oldconf->{snapshots}->{$snapname};
5201                 if ($snap->{vmstate}) {
5202                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
5203                     if (my $err = $@) {
5204                         warn $err;
5205                     }
5206                 }
5207             }
5208         }
5209
5210         my $map = {};
5211         foreach my $virtdev (sort keys %$virtdev_hash) {
5212             my $d = $virtdev_hash->{$virtdev};
5213             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5214             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5215
5216             # test if requested format is supported
5217             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5218             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5219             $d->{format} = $defFormat if !$supported;
5220
5221             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5222                                                   $d->{format}, undef, $alloc_size);
5223             print STDERR "new volume ID is '$volid'\n";
5224             $d->{volid} = $volid;
5225             my $path = PVE::Storage::path($cfg, $volid);
5226
5227             PVE::Storage::activate_volumes($cfg,[$volid]);
5228
5229             my $write_zeros = 1;
5230             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
5231                 $write_zeros = 0;
5232             }
5233
5234             print $fifofh "format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
5235
5236             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5237             $map->{$virtdev} = $volid;
5238         }
5239
5240         $fh->seek(0, 0) || die "seek failed - $!\n";
5241
5242         my $outfd = new IO::File ($tmpfn, "w") ||
5243             die "unable to write config for VM $vmid\n";
5244
5245         my $cookie = { netcount => 0 };
5246         while (defined(my $line = <$fh>)) {
5247             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5248         }
5249
5250         $fh->close();
5251         $outfd->close();
5252     };
5253
5254     eval {
5255         # enable interrupts
5256         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5257             die "interrupted by signal\n";
5258         };
5259         local $SIG{ALRM} = sub { die "got timeout\n"; };
5260
5261         $oldtimeout = alarm($timeout);
5262
5263         my $parser = sub {
5264             my $line = shift;
5265
5266             print "$line\n";
5267
5268             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5269                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5270                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5271             } elsif ($line =~ m/^CTIME: /) {
5272                 # we correctly received the vma config, so we can disable
5273                 # the timeout now for disk allocation (set to 10 minutes, so
5274                 # that we always timeout if something goes wrong)
5275                 alarm(600);
5276                 &$print_devmap();
5277                 print $fifofh "done\n";
5278                 my $tmp = $oldtimeout || 0;
5279                 $oldtimeout = undef;
5280                 alarm($tmp);
5281                 close($fifofh);
5282             }
5283         };
5284
5285         print "restore vma archive: $cmd\n";
5286         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5287     };
5288     my $err = $@;
5289
5290     alarm($oldtimeout) if $oldtimeout;
5291
5292     my $vollist = [];
5293     foreach my $devname (keys %$devinfo) {
5294         my $volid = $devinfo->{$devname}->{volid};
5295         push @$vollist, $volid if $volid;
5296     }
5297
5298     my $cfg = PVE::Storage::config();
5299     PVE::Storage::deactivate_volumes($cfg, $vollist);
5300
5301     unlink $mapfifo;
5302
5303     if ($err) {
5304         rmtree $tmpdir;
5305         unlink $tmpfn;
5306
5307         foreach my $devname (keys %$devinfo) {
5308             my $volid = $devinfo->{$devname}->{volid};
5309             next if !$volid;
5310             eval {
5311                 if ($volid =~ m|^/|) {
5312                     unlink $volid || die 'unlink failed\n';
5313                 } else {
5314                     PVE::Storage::vdisk_free($cfg, $volid);
5315                 }
5316                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5317             };
5318             print STDERR "unable to cleanup '$volid' - $@" if $@;
5319         }
5320         die $err;
5321     }
5322
5323     rmtree $tmpdir;
5324
5325     rename($tmpfn, $conffile) ||
5326         die "unable to commit configuration file '$conffile'\n";
5327
5328     PVE::Cluster::cfs_update(); # make sure we read new file
5329
5330     eval { rescan($vmid, 1); };
5331     warn $@ if $@;
5332 }
5333
5334 sub restore_tar_archive {
5335     my ($archive, $vmid, $user, $opts) = @_;
5336
5337     if ($archive ne '-') {
5338         my $firstfile = tar_archive_read_firstfile($archive);
5339         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5340             if $firstfile ne 'qemu-server.conf';
5341     }
5342
5343     my $storecfg = PVE::Storage::config();
5344
5345     # destroy existing data - keep empty config
5346     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
5347     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5348
5349     my $tocmd = "/usr/lib/qemu-server/qmextract";
5350
5351     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5352     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5353     $tocmd .= ' --prealloc' if $opts->{prealloc};
5354     $tocmd .= ' --info' if $opts->{info};
5355
5356     # tar option "xf" does not autodetect compression when read from STDIN,
5357     # so we pipe to zcat
5358     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5359         PVE::Tools::shellquote("--to-command=$tocmd");
5360
5361     my $tmpdir = "/var/tmp/vzdumptmp$$";
5362     mkpath $tmpdir;
5363
5364     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5365     local $ENV{VZDUMP_VMID} = $vmid;
5366     local $ENV{VZDUMP_USER} = $user;
5367
5368     my $conffile = PVE::QemuConfig->config_file($vmid);
5369     my $tmpfn = "$conffile.$$.tmp";
5370
5371     # disable interrupts (always do cleanups)
5372     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5373         print STDERR "got interrupt - ignored\n";
5374     };
5375
5376     eval {
5377         # enable interrupts
5378         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5379             die "interrupted by signal\n";
5380         };
5381
5382         if ($archive eq '-') {
5383             print "extracting archive from STDIN\n";
5384             run_command($cmd, input => "<&STDIN");
5385         } else {
5386             print "extracting archive '$archive'\n";
5387             run_command($cmd);
5388         }
5389
5390         return if $opts->{info};
5391
5392         # read new mapping
5393         my $map = {};
5394         my $statfile = "$tmpdir/qmrestore.stat";
5395         if (my $fd = IO::File->new($statfile, "r")) {
5396             while (defined (my $line = <$fd>)) {
5397                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5398                     $map->{$1} = $2 if $1;
5399                 } else {
5400                     print STDERR "unable to parse line in statfile - $line\n";
5401                 }
5402             }
5403             $fd->close();
5404         }
5405
5406         my $confsrc = "$tmpdir/qemu-server.conf";
5407
5408         my $srcfd = new IO::File($confsrc, "r") ||
5409             die "unable to open file '$confsrc'\n";
5410
5411         my $outfd = new IO::File ($tmpfn, "w") ||
5412             die "unable to write config for VM $vmid\n";
5413
5414         my $cookie = { netcount => 0 };
5415         while (defined (my $line = <$srcfd>)) {
5416             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5417         }
5418
5419         $srcfd->close();
5420         $outfd->close();
5421     };
5422     my $err = $@;
5423
5424     if ($err) {
5425
5426         unlink $tmpfn;
5427
5428         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
5429
5430         die $err;
5431     }
5432
5433     rmtree $tmpdir;
5434
5435     rename $tmpfn, $conffile ||
5436         die "unable to commit configuration file '$conffile'\n";
5437
5438     PVE::Cluster::cfs_update(); # make sure we read new file
5439
5440     eval { rescan($vmid, 1); };
5441     warn $@ if $@;
5442 };
5443
5444 sub foreach_writable_storage {
5445     my ($conf, $func) = @_;
5446
5447     my $sidhash = {};
5448
5449     foreach my $ds (keys %$conf) {
5450         next if !is_valid_drivename($ds);
5451
5452         my $drive = parse_drive($ds, $conf->{$ds});
5453         next if !$drive;
5454         next if drive_is_cdrom($drive);
5455
5456         my $volid = $drive->{file};
5457
5458         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5459         $sidhash->{$sid} = $sid if $sid;
5460     }
5461
5462     foreach my $sid (sort keys %$sidhash) {
5463         &$func($sid);
5464     }
5465 }
5466
5467 sub do_snapshots_with_qemu {
5468     my ($storecfg, $volid) = @_;
5469
5470     my $storage_name = PVE::Storage::parse_volume_id($volid);
5471
5472     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
5473         && !$storecfg->{ids}->{$storage_name}->{krbd}){
5474         return 1;
5475     }
5476
5477     if ($volid =~ m/\.(qcow2|qed)$/){
5478         return 1;
5479     }
5480
5481     return undef;
5482 }
5483
5484 sub qga_check_running {
5485     my ($vmid) = @_;
5486
5487     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
5488     if ($@) {
5489         warn "Qemu Guest Agent are not running - $@";
5490         return 0;
5491     }
5492     return 1;
5493 }
5494
5495 sub template_create {
5496     my ($vmid, $conf, $disk) = @_;
5497
5498     my $storecfg = PVE::Storage::config();
5499
5500     foreach_drive($conf, sub {
5501         my ($ds, $drive) = @_;
5502
5503         return if drive_is_cdrom($drive);
5504         return if $disk && $ds ne $disk;
5505
5506         my $volid = $drive->{file};
5507         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
5508
5509         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
5510         $drive->{file} = $voliddst;
5511         $conf->{$ds} = print_drive($vmid, $drive);
5512         PVE::QemuConfig->write_config($vmid, $conf);
5513     });
5514 }
5515
5516 sub qemu_img_convert {
5517     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
5518
5519     my $storecfg = PVE::Storage::config();
5520     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
5521     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
5522
5523     if ($src_storeid && $dst_storeid) {
5524
5525         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
5526
5527         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
5528         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5529
5530         my $src_format = qemu_img_format($src_scfg, $src_volname);
5531         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
5532
5533         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
5534         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5535
5536         my $cmd = [];
5537         push @$cmd, '/usr/bin/qemu-img', 'convert', '-t', 'writeback', '-p', '-n';
5538         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
5539         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path;
5540         if ($is_zero_initialized) {
5541             push @$cmd, "zeroinit:$dst_path";
5542         } else {
5543             push @$cmd, $dst_path;
5544         }
5545
5546         my $parser = sub {
5547             my $line = shift;
5548             if($line =~ m/\((\S+)\/100\%\)/){
5549                 my $percent = $1;
5550                 my $transferred = int($size * $percent / 100);
5551                 my $remaining = $size - $transferred;
5552
5553                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
5554             }
5555
5556         };
5557
5558         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
5559         my $err = $@;
5560         die "copy failed: $err" if $err;
5561     }
5562 }
5563
5564 sub qemu_img_format {
5565     my ($scfg, $volname) = @_;
5566
5567     if ($scfg->{path} && $volname =~ m/\.(raw|cow|qcow|qcow2|qed|vmdk|cloop)$/) {
5568         return $1;
5569     } else {
5570         return "raw";
5571     }
5572 }
5573
5574 sub qemu_drive_mirror {
5575     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized) = @_;
5576
5577     my $storecfg = PVE::Storage::config();
5578     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
5579
5580     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5581
5582     my $format = qemu_img_format($dst_scfg, $dst_volname);
5583
5584     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5585
5586     my $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
5587
5588     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
5589     $opts->{format} = $format if $format;
5590
5591     print "drive mirror is starting (scanning bitmap) : this step can take some minutes/hours, depend of disk size and storage speed\n";
5592
5593     my $finish_job = sub {
5594         while (1) {
5595             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
5596             my $stat = @$stats[0];
5597             last if !$stat;
5598             sleep 1;
5599         }
5600     };
5601
5602     eval {
5603     vm_mon_cmd($vmid, "drive-mirror", %$opts);
5604         while (1) {
5605             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
5606             my $stat = @$stats[0];
5607             die "mirroring job seem to have die. Maybe do you have bad sectors?" if !$stat;
5608             die "error job is not mirroring" if $stat->{type} ne "mirror";
5609
5610             my $busy = $stat->{busy};
5611             my $ready = $stat->{ready};
5612
5613             if (my $total = $stat->{len}) {
5614                 my $transferred = $stat->{offset} || 0;
5615                 my $remaining = $total - $transferred;
5616                 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
5617
5618                 print "transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
5619             }
5620
5621
5622             if ($stat->{ready} eq 'true') {
5623
5624                 last if $vmiddst != $vmid;
5625
5626                 # try to switch the disk if source and destination are on the same guest
5627                 eval { vm_mon_cmd($vmid, "block-job-complete", device => "drive-$drive") };
5628                 if (!$@) {
5629                     &$finish_job();
5630                     last;
5631                 }
5632                 die $@ if $@ !~ m/cannot be completed/;
5633             }
5634             sleep 1;
5635         }
5636
5637
5638     };
5639     my $err = $@;
5640
5641     my $cancel_job = sub {
5642         vm_mon_cmd($vmid, "block-job-cancel", device => "drive-$drive");
5643         &$finish_job();
5644     };
5645
5646     if ($err) {
5647         eval { &$cancel_job(); };
5648         die "mirroring error: $err";
5649     }
5650
5651     if ($vmiddst != $vmid) {
5652         # if we clone a disk for a new target vm, we don't switch the disk
5653         &$cancel_job(); # so we call block-job-cancel
5654     }
5655 }
5656
5657 sub clone_disk {
5658     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
5659         $newvmid, $storage, $format, $full, $newvollist) = @_;
5660
5661     my $newvolid;
5662
5663     if (!$full) {
5664         print "create linked clone of drive $drivename ($drive->{file})\n";
5665         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
5666         push @$newvollist, $newvolid;
5667     } else {
5668         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
5669         $storeid = $storage if $storage;
5670
5671         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5672         if (!$format) {
5673             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5674             $format = qemu_img_format($scfg, $volname);
5675         }
5676
5677         # test if requested format is supported - else use default
5678         my $supported = grep { $_ eq $format } @$validFormats;
5679         $format = $defFormat if !$supported;
5680
5681         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
5682
5683         print "create full clone of drive $drivename ($drive->{file})\n";
5684         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $format, undef, ($size/1024));
5685         push @$newvollist, $newvolid;
5686
5687         PVE::Storage::activate_volumes($storecfg, $newvollist);
5688
5689         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
5690         if (!$running || $snapname) {
5691             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
5692         } else {
5693             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit);
5694         }
5695     }
5696
5697     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
5698
5699     my $disk = $drive;
5700     $disk->{format} = undef;
5701     $disk->{file} = $newvolid;
5702     $disk->{size} = $size;
5703
5704     return $disk;
5705 }
5706
5707 # this only works if VM is running
5708 sub get_current_qemu_machine {
5709     my ($vmid) = @_;
5710
5711     my $cmd = { execute => 'query-machines', arguments => {} };
5712     my $res = vm_qmp_command($vmid, $cmd);
5713
5714     my ($current, $default);
5715     foreach my $e (@$res) {
5716         $default = $e->{name} if $e->{'is-default'};
5717         $current = $e->{name} if $e->{'is-current'};
5718     }
5719
5720     # fallback to the default machine if current is not supported by qemu
5721     return $current || $default || 'pc';
5722 }
5723
5724 sub qemu_machine_feature_enabled {
5725     my ($machine, $kvmver, $version_major, $version_minor) = @_;
5726
5727     my $current_major;
5728     my $current_minor;
5729
5730     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
5731
5732         $current_major = $3;
5733         $current_minor = $4;
5734
5735     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
5736
5737         $current_major = $1;
5738         $current_minor = $2;
5739     }
5740
5741     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
5742
5743
5744 }
5745
5746 sub qemu_machine_pxe {
5747     my ($vmid, $conf, $machine) = @_;
5748
5749     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
5750
5751     foreach my $opt (keys %$conf) {
5752         next if $opt !~ m/^net(\d+)$/;
5753         my $net = PVE::QemuServer::parse_net($conf->{$opt});
5754         next if !$net;
5755         my $romfile = PVE::QemuServer::vm_mon_cmd_nocheck($vmid, 'qom-get', path => $opt, property => 'romfile');
5756         return $machine.".pxe" if $romfile =~ m/pxe/;
5757         last;
5758     }
5759
5760     return $machine;
5761 }
5762
5763 sub qemu_use_old_bios_files {
5764     my ($machine_type) = @_;
5765
5766     return if !$machine_type;
5767
5768     my $use_old_bios_files = undef;
5769
5770     if ($machine_type =~ m/^(\S+)\.pxe$/) {
5771         $machine_type = $1;
5772         $use_old_bios_files = 1;
5773     } else {
5774         my $kvmver = kvm_user_version();
5775         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
5776         # load new efi bios files on migration. So this hack is required to allow
5777         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
5778         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
5779         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
5780     }
5781
5782     return ($use_old_bios_files, $machine_type);
5783 }
5784
5785 sub lspci {
5786
5787     my $devices = {};
5788
5789     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
5790             my (undef, $id, $function) = @_;
5791             my $res = { id => $id, function => $function};
5792             push @{$devices->{$id}}, $res;
5793     });
5794
5795     return $devices;
5796 }
5797
5798 sub vm_iothreads_list {
5799     my ($vmid) = @_;
5800
5801     my $res = vm_mon_cmd($vmid, 'query-iothreads');
5802
5803     my $iothreads = {};
5804     foreach my $iothread (@$res) {
5805         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
5806     }
5807
5808     return $iothreads;
5809 }
5810
5811 sub scsihw_infos {
5812     my ($conf, $drive) = @_;
5813
5814     my $maxdev = 0;
5815
5816     if ($conf->{scsihw} && ($conf->{scsihw} =~ m/^lsi/)) {
5817         $maxdev = 7;
5818     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
5819         $maxdev = 1;
5820     } else {
5821         $maxdev = 256;
5822     }
5823
5824     my $controller = int($drive->{index} / $maxdev);
5825     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
5826
5827     return ($maxdev, $controller, $controller_prefix);
5828 }
5829
5830 # bash completion helper
5831
5832 sub complete_backup_archives {
5833     my ($cmdname, $pname, $cvalue) = @_;
5834
5835     my $cfg = PVE::Storage::config();
5836
5837     my $storeid;
5838
5839     if ($cvalue =~ m/^([^:]+):/) {
5840         $storeid = $1;
5841     }
5842
5843     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
5844
5845     my $res = [];
5846     foreach my $id (keys %$data) {
5847         foreach my $item (@{$data->{$id}}) {
5848             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
5849             push @$res, $item->{volid} if defined($item->{volid});
5850         }
5851     }
5852
5853     return $res;
5854 }
5855
5856 my $complete_vmid_full = sub {
5857     my ($running) = @_;
5858
5859     my $idlist = vmstatus();
5860
5861     my $res = [];
5862
5863     foreach my $id (keys %$idlist) {
5864         my $d = $idlist->{$id};
5865         if (defined($running)) {
5866             next if $d->{template};
5867             next if $running && $d->{status} ne 'running';
5868             next if !$running && $d->{status} eq 'running';
5869         }
5870         push @$res, $id;
5871
5872     }
5873     return $res;
5874 };
5875
5876 sub complete_vmid {
5877     return &$complete_vmid_full();
5878 }
5879
5880 sub complete_vmid_stopped {
5881     return &$complete_vmid_full(0);
5882 }
5883
5884 sub complete_vmid_running {
5885     return &$complete_vmid_full(1);
5886 }
5887
5888 sub complete_storage {
5889
5890     my $cfg = PVE::Storage::config();
5891     my $ids = $cfg->{ids};
5892
5893     my $res = [];
5894     foreach my $sid (keys %$ids) {
5895         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
5896         next if !$ids->{$sid}->{content}->{images};
5897         push @$res, $sid;
5898     }
5899
5900     return $res;
5901 }
5902
5903 1;