PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QemuConfig;
  31 use PVE::QMPClient;
  32 use PVE::RPCEnvironment;
  33 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
  34 use PVE::QemuServer::Memory;
  35 use Time::HiRes qw(gettimeofday);
  36 use File::Copy qw(copy);
  37 use URI::Escape;
  38
  39 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  40
  41 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  42
  43 # Note about locking: we use flock on the config file protect
  44 # against concurent actions.
  45 # Aditionaly, we have a 'lock' setting in the config file. This
  46 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  47 # allowed when such lock is set. But you can ignore this kind of
  48 # lock with the --skiplock flag.
  49
  50 cfs_register_file('/qemu-server/',
  51                   \&parse_vm_config,
  52                   \&write_vm_config);
  53
  54 PVE::JSONSchema::register_standard_option('skiplock', {
  55     description => "Ignore locks - only root is allowed to use this option.",
  56     type => 'boolean',
  57     optional => 1,
  58 });
  59
  60 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  61     description => "Some command save/restore state from this location.",
  62     type => 'string',
  63     maxLength => 128,
  64     optional => 1,
  65 });
  66
  67 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  68     description => "The name of the snapshot.",
  69     type => 'string', format => 'pve-configid',
  70     maxLength => 40,
  71 });
  72
  73 #no warnings 'redefine';
  74
  75 sub cgroups_write {
  76    my ($controller, $vmid, $option, $value) = @_;
  77
  78    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  79    PVE::ProcFSTools::write_proc_entry($path, $value);
  80
  81 }
  82
  83 my $nodename = PVE::INotify::nodename();
  84
  85 mkdir "/etc/pve/nodes/$nodename";
  86 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  87 mkdir $confdir;
  88
  89 my $var_run_tmpdir = "/var/run/qemu-server";
  90 mkdir $var_run_tmpdir;
  91
  92 my $lock_dir = "/var/lock/qemu-server";
  93 mkdir $lock_dir;
  94
  95 my $pcisysfs = "/sys/bus/pci";
  96
  97 my $cpu_vendor_list = {
  98     # Intel CPUs
  99     486 => 'GenuineIntel',
 100     pentium => 'GenuineIntel',
 101     pentium2  => 'GenuineIntel',
 102     pentium3  => 'GenuineIntel',
 103     coreduo => 'GenuineIntel',
 104     core2duo => 'GenuineIntel',
 105     Conroe  => 'GenuineIntel',
 106     Penryn  => 'GenuineIntel',
 107     Nehalem  => 'GenuineIntel',
 108     Westmere => 'GenuineIntel',
 109     SandyBridge => 'GenuineIntel',
 110     IvyBridge => 'GenuineIntel',
 111     Haswell => 'GenuineIntel',
 112     'Haswell-noTSX' => 'GenuineIntel',
 113     Broadwell => 'GenuineIntel',
 114     'Broadwell-noTSX' => 'GenuineIntel',
 115
 116     # AMD CPUs
 117     athlon => 'AuthenticAMD',
 118     phenom  => 'AuthenticAMD',
 119     Opteron_G1  => 'AuthenticAMD',
 120     Opteron_G2  => 'AuthenticAMD',
 121     Opteron_G3  => 'AuthenticAMD',
 122     Opteron_G4  => 'AuthenticAMD',
 123     Opteron_G5  => 'AuthenticAMD',
 124
 125     # generic types, use vendor from host node
 126     host => 'default',
 127     kvm32 => 'default',
 128     kvm64 => 'default',
 129     qemu32 => 'default',
 130     qemu64 => 'default',
 131 };
 132
 133 my $cpu_fmt = {
 134     cputype => {
 135         description => "Emulated CPU type.",
 136         type => 'string',
 137         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 138         format_description => 'cputype',
 139         default => 'kvm64',
 140         default_key => 1,
 141     },
 142     hidden => {
 143         description => "Do not identify as a KVM virtual machine.",
 144         type => 'boolean',
 145         optional => 1,
 146         default => 0
 147     },
 148 };
 149
 150 my $watchdog_fmt = {
 151     model => {
 152         default_key => 1,
 153         type => 'string',
 154         enum => [qw(i6300esb ib700)],
 155         description => "Watchdog type to emulate.",
 156         default => 'i6300esb',
 157         optional => 1,
 158     },
 159     action => {
 160         type => 'string',
 161         enum => [qw(reset shutdown poweroff pause debug none)],
 162         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 163         optional => 1,
 164     },
 165 };
 166 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 167
 168 my $confdesc = {
 169     onboot => {
 170         optional => 1,
 171         type => 'boolean',
 172         description => "Specifies whether a VM will be started during system bootup.",
 173         default => 0,
 174     },
 175     autostart => {
 176         optional => 1,
 177         type => 'boolean',
 178         description => "Automatic restart after crash (currently ignored).",
 179         default => 0,
 180     },
 181     hotplug => {
 182         optional => 1,
 183         type => 'string', format => 'pve-hotplug-features',
 184         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 185         default => 'network,disk,usb',
 186     },
 187     reboot => {
 188         optional => 1,
 189         type => 'boolean',
 190         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 191         default => 1,
 192     },
 193     lock => {
 194         optional => 1,
 195         type => 'string',
 196         description => "Lock/unlock the VM.",
 197         enum => [qw(migrate backup snapshot rollback)],
 198     },
 199     cpulimit => {
 200         optional => 1,
 201         type => 'number',
 202         description => "Limit of CPU usage.",
 203         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 204         minimum => 0,
 205         maximum => 128,
 206         default => 0,
 207     },
 208     cpuunits => {
 209         optional => 1,
 210         type => 'integer',
 211         description => "CPU weight for a VM.",
 212         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 213         minimum => 0,
 214         maximum => 500000,
 215         default => 1000,
 216     },
 217     memory => {
 218         optional => 1,
 219         type => 'integer',
 220         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 221         minimum => 16,
 222         default => 512,
 223     },
 224     balloon => {
 225         optional => 1,
 226         type => 'integer',
 227         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 228         minimum => 0,
 229     },
 230     shares => {
 231         optional => 1,
 232         type => 'integer',
 233         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 234         minimum => 0,
 235         maximum => 50000,
 236         default => 1000,
 237     },
 238     keyboard => {
 239         optional => 1,
 240         type => 'string',
 241         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.conf' configuration file.",
 242         enum => PVE::Tools::kvmkeymaplist(),
 243         default => 'en-us',
 244     },
 245     name => {
 246         optional => 1,
 247         type => 'string', format => 'dns-name',
 248         description => "Set a name for the VM. Only used on the configuration web interface.",
 249     },
 250     scsihw => {
 251         optional => 1,
 252         type => 'string',
 253         description => "SCSI controller model",
 254         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 255         default => 'lsi',
 256     },
 257     description => {
 258         optional => 1,
 259         type => 'string',
 260         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 261     },
 262     ostype => {
 263         optional => 1,
 264         type => 'string',
 265         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 l24 l26 solaris)],
 266         description => "Specify guest operating system.",
 267         verbose_description => <<EODESC,
 268 Specify guest operating system. This is used to enable special
 269 optimization/features for specific operating systems:
 270
 271 [horizontal]
 272 other;; unspecified OS
 273 wxp;; Microsoft Windows XP
 274 w2k;; Microsoft Windows 2000
 275 w2k3;; Microsoft Windows 2003
 276 w2k8;; Microsoft Windows 2008
 277 wvista;; Microsoft Windows Vista
 278 win7;; Microsoft Windows 7
 279 win8;; Microsoft Windows 8/2012
 280 l24;; Linux 2.4 Kernel
 281 l26;; Linux 2.6/3.X Kernel
 282 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 283 EODESC
 284     },
 285     boot => {
 286         optional => 1,
 287         type => 'string',
 288         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 289         pattern => '[acdn]{1,4}',
 290         default => 'cdn',
 291     },
 292     bootdisk => {
 293         optional => 1,
 294         type => 'string', format => 'pve-qm-bootdisk',
 295         description => "Enable booting from specified disk.",
 296         pattern => '(ide|sata|scsi|virtio)\d+',
 297     },
 298     smp => {
 299         optional => 1,
 300         type => 'integer',
 301         description => "The number of CPUs. Please use option -sockets instead.",
 302         minimum => 1,
 303         default => 1,
 304     },
 305     sockets => {
 306         optional => 1,
 307         type => 'integer',
 308         description => "The number of CPU sockets.",
 309         minimum => 1,
 310         default => 1,
 311     },
 312     cores => {
 313         optional => 1,
 314         type => 'integer',
 315         description => "The number of cores per socket.",
 316         minimum => 1,
 317         default => 1,
 318     },
 319     numa => {
 320         optional => 1,
 321         type => 'boolean',
 322         description => "Enable/disable NUMA.",
 323         default => 0,
 324     },
 325     hugepages => {
 326         optional => 1,
 327         type => 'string',
 328         description => "Enable/disable hugepages memory.",
 329         enum => [qw(any 2 1024)],
 330     },
 331     vcpus => {
 332         optional => 1,
 333         type => 'integer',
 334         description => "Number of hotplugged vcpus.",
 335         minimum => 1,
 336         default => 0,
 337     },
 338     acpi => {
 339         optional => 1,
 340         type => 'boolean',
 341         description => "Enable/disable ACPI.",
 342         default => 1,
 343     },
 344     agent => {
 345         optional => 1,
 346         type => 'boolean',
 347         description => "Enable/disable Qemu GuestAgent.",
 348         default => 0,
 349     },
 350     kvm => {
 351         optional => 1,
 352         type => 'boolean',
 353         description => "Enable/disable KVM hardware virtualization.",
 354         default => 1,
 355     },
 356     tdf => {
 357         optional => 1,
 358         type => 'boolean',
 359         description => "Enable/disable time drift fix.",
 360         default => 0,
 361     },
 362     localtime => {
 363         optional => 1,
 364         type => 'boolean',
 365         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 366     },
 367     freeze => {
 368         optional => 1,
 369         type => 'boolean',
 370         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 371     },
 372     vga => {
 373         optional => 1,
 374         type => 'string',
 375         description => "Select the VGA type.",
 376         verbose_description => "Select the VGA type. If you want to use high resolution" .
 377             " modes (>= 1280x1024x16) then you should use the options " .
 378             "'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and " .
 379             "'cirrus' for other OS types. The 'qxl' option enables the SPICE " .
 380             "display sever. For win* OS you can select how many independent " .
 381             "displays you want, Linux guests can add displays them self. " .
 382             "You can also run without any graphic card, using a serial device" .
 383             " as terminal.",
 384         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 385     },
 386     watchdog => {
 387         optional => 1,
 388         type => 'string', format => 'pve-qm-watchdog',
 389         description => "Create a virtual hardware watchdog device.",
 390         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 391             " (by a guest action), the watchdog must be periodically polled " .
 392             "by an agent inside the guest or else the watchdog will reset " .
 393             "the guest (or execute the respective action specified)",
 394     },
 395     startdate => {
 396         optional => 1,
 397         type => 'string',
 398         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 399         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 400         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 401         default => 'now',
 402     },
 403     startup =>  get_standard_option('pve-startup-order'),
 404     template => {
 405         optional => 1,
 406         type => 'boolean',
 407         description => "Enable/disable Template.",
 408         default => 0,
 409     },
 410     args => {
 411         optional => 1,
 412         type => 'string',
 413         description => "Arbitrary arguments passed to kvm.",
 414         verbose_description => <<EODESCR,
 415 Arbitrary arguments passed to kvm, for example:
 416
 417 args: -no-reboot -no-hpet
 418
 419 NOTE: this option is for experts only.
 420 EODESCR
 421     },
 422     tablet => {
 423         optional => 1,
 424         type => 'boolean',
 425         default => 1,
 426         description => "Enable/disable the USB tablet device.",
 427         verbose_description => "Enable/disable the USB tablet device. This device is " .
 428             "usually needed to allow absolute mouse positioning with VNC. " .
 429             "Else the mouse runs out of sync with normal VNC clients. " .
 430             "If you're running lots of console-only guests on one host, " .
 431             "you may consider disabling this to save some context switches. " .
 432             "This is turned off by default if you use spice (-vga=qxl).",
 433     },
 434     migrate_speed => {
 435         optional => 1,
 436         type => 'integer',
 437         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 438         minimum => 0,
 439         default => 0,
 440     },
 441     migrate_downtime => {
 442         optional => 1,
 443         type => 'number',
 444         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 445         minimum => 0,
 446         default => 0.1,
 447     },
 448     cdrom => {
 449         optional => 1,
 450         type => 'string', format => 'pve-qm-drive',
 451         typetext => 'volume',
 452         description => "This is an alias for option -ide2",
 453     },
 454     cpu => {
 455         optional => 1,
 456         description => "Emulated CPU type.",
 457         type => 'string',
 458         format => $cpu_fmt,
 459     },
 460     parent => get_standard_option('pve-snapshot-name', {
 461         optional => 1,
 462         description => "Parent snapshot name. This is used internally, and should not be modified.",
 463     }),
 464     snaptime => {
 465         optional => 1,
 466         description => "Timestamp for snapshots.",
 467         type => 'integer',
 468         minimum => 0,
 469     },
 470     vmstate => {
 471         optional => 1,
 472         type => 'string', format => 'pve-volume-id',
 473         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 474     },
 475     machine => {
 476         description => "Specific the Qemu machine type.",
 477         type => 'string',
 478         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
 479         maxLength => 40,
 480         optional => 1,
 481     },
 482     smbios1 => {
 483         description => "Specify SMBIOS type 1 fields.",
 484         type => 'string', format => 'pve-qm-smbios1',
 485         maxLength => 256,
 486         optional => 1,
 487     },
 488     protection => {
 489         optional => 1,
 490         type => 'boolean',
 491         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 492         default => 0,
 493     },
 494     bios => {
 495         optional => 1,
 496         type => 'string',
 497         enum => [ qw(seabios ovmf) ],
 498         description => "Select BIOS implementation.",
 499         default => 'seabios',
 500     },
 501 };
 502
 503 # what about other qemu settings ?
 504 #cpu => 'string',
 505 #machine => 'string',
 506 #fda => 'file',
 507 #fdb => 'file',
 508 #mtdblock => 'file',
 509 #sd => 'file',
 510 #pflash => 'file',
 511 #snapshot => 'bool',
 512 #bootp => 'file',
 513 ##tftp => 'dir',
 514 ##smb => 'dir',
 515 #kernel => 'file',
 516 #append => 'string',
 517 #initrd => 'file',
 518 ##soundhw => 'string',
 519
 520 while (my ($k, $v) = each %$confdesc) {
 521     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 522 }
 523
 524 my $MAX_IDE_DISKS = 4;
 525 my $MAX_SCSI_DISKS = 14;
 526 my $MAX_VIRTIO_DISKS = 16;
 527 my $MAX_SATA_DISKS = 6;
 528 my $MAX_USB_DEVICES = 5;
 529 my $MAX_NETS = 32;
 530 my $MAX_UNUSED_DISKS = 8;
 531 my $MAX_HOSTPCI_DEVICES = 4;
 532 my $MAX_SERIAL_PORTS = 4;
 533 my $MAX_PARALLEL_PORTS = 3;
 534 my $MAX_NUMA = 8;
 535
 536 my $numa_fmt = {
 537     cpus => {
 538         type => "string",
 539         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 540         description => "CPUs accessing this NUMA node.",
 541         format_description => "id[-id];...",
 542     },
 543     memory => {
 544         type => "number",
 545         description => "Amount of memory this NUMA node provides.",
 546         optional => 1,
 547     },
 548     hostnodes => {
 549         type => "string",
 550         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 551         description => "Host NUMA nodes to use.",
 552         format_description => "id[-id];...",
 553         optional => 1,
 554     },
 555     policy => {
 556         type => 'string',
 557         enum => [qw(preferred bind interleave)],
 558         description => "NUMA allocation policy.",
 559         optional => 1,
 560     },
 561 };
 562 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 563 my $numadesc = {
 564     optional => 1,
 565     type => 'string', format => $numa_fmt,
 566     description => "NUMA topology.",
 567 };
 568 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 569
 570 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 571     $confdesc->{"numa$i"} = $numadesc;
 572 }
 573
 574 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 575                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 576                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 577 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 578
 579 my $net_fmt_bridge_descr = <<__EOD__;
 580 Bridge to attach the network device to. The Proxmox VE standard bridge
 581 is called 'vmbr0'.
 582
 583 If you do not specify a bridge, we create a kvm user (NATed) network
 584 device, which provides DHCP and DNS services. The following addresses
 585 are used:
 586
 587  10.0.2.2   Gateway
 588  10.0.2.3   DNS Server
 589  10.0.2.4   SMB Server
 590
 591 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 592 __EOD__
 593
 594 my $net_fmt = {
 595     macaddr => {
 596         type => 'string',
 597         pattern => qr/[0-9a-f]{2}(?::[0-9a-f]{2}){5}/i,
 598         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 599         format_description => "XX:XX:XX:XX:XX:XX",
 600         optional => 1,
 601     },
 602     model => {
 603         type => 'string',
 604         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 605         format_description => 'model',
 606         enum => $nic_model_list,
 607         default_key => 1,
 608     },
 609     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 610     bridge => {
 611         type => 'string',
 612         description => $net_fmt_bridge_descr,
 613         format_description => 'bridge',
 614         optional => 1,
 615     },
 616     queues => {
 617         type => 'integer',
 618         minimum => 0, maximum => 16,
 619         description => 'Number of packet queues to be used on the device.',
 620         optional => 1,
 621     },
 622     rate => {
 623         type => 'number',
 624         minimum => 0,
 625         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 626         optional => 1,
 627     },
 628     tag => {
 629         type => 'integer',
 630         minimum => 1, maximum => 4094,
 631         description => 'VLAN tag to apply to packets on this interface.',
 632         optional => 1,
 633     },
 634     trunks => {
 635         type => 'string',
 636         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 637         description => 'VLAN trunks to pass through this interface.',
 638         format_description => 'vlanid[;vlanid...]',
 639         optional => 1,
 640     },
 641     firewall => {
 642         type => 'boolean',
 643         description => 'Whether this interface should be protected by the firewall.',
 644         optional => 1,
 645     },
 646     link_down => {
 647         type => 'boolean',
 648         description => 'Whether this interface should be disconnected (like pulling the plug).',
 649         optional => 1,
 650     },
 651 };
 652
 653 my $netdesc = {
 654     optional => 1,
 655     type => 'string', format => $net_fmt,
 656     description => "Specify network devices.",
 657 };
 658
 659 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 660
 661 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 662     $confdesc->{"net$i"} = $netdesc;
 663 }
 664
 665 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 666 sub verify_volume_id_or_qm_path {
 667     my ($volid, $noerr) = @_;
 668
 669     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 670         return $volid;
 671     }
 672
 673     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 674     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 675     if ($@) {
 676         return undef if $noerr;
 677         die $@;
 678     }
 679     return $volid;
 680 }
 681
 682 my $drivename_hash;
 683
 684 my %drivedesc_base = (
 685     volume => { alias => 'file' },
 686     file => {
 687         type => 'string',
 688         format => 'pve-volume-id-or-qm-path',
 689         default_key => 1,
 690         format_description => 'volume',
 691         description => "The drive's backing volume.",
 692     },
 693     media => {
 694         type => 'string',
 695         enum => [qw(cdrom disk)],
 696         description => "The drive's media type.",
 697         default => 'disk',
 698         optional => 1
 699     },
 700     cyls => {
 701         type => 'integer',
 702         description => "Force the drive's physical geometry to have a specific cylinder count.",
 703         optional => 1
 704     },
 705     heads => {
 706         type => 'integer',
 707         description => "Force the drive's physical geometry to have a specific head count.",
 708         optional => 1
 709     },
 710     secs => {
 711         type => 'integer',
 712         description => "Force the drive's physical geometry to have a specific sector count.",
 713         optional => 1
 714     },
 715     trans => {
 716         type => 'string',
 717         enum => [qw(none lba auto)],
 718         description => "Force disk geometry bios translation mode.",
 719         optional => 1,
 720     },
 721     snapshot => {
 722         type => 'boolean',
 723         description => "Whether the drive should be included when making snapshots.",
 724         optional => 1,
 725     },
 726     cache => {
 727         type => 'string',
 728         enum => [qw(none writethrough writeback unsafe directsync)],
 729         description => "The drive's cache mode",
 730         optional => 1,
 731     },
 732     format => {
 733         type => 'string',
 734         format_description => 'image format',
 735         enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
 736         description => "The drive's backing file's data format.",
 737         optional => 1,
 738     },
 739     size => {
 740         type => 'string',
 741         format => 'disk-size',
 742         format_description => 'DiskSize',
 743         description => "Disk size. This is purely informational and has no effect.",
 744         optional => 1,
 745     },
 746     backup => {
 747         type => 'boolean',
 748         description => "Whether the drive should be included when making backups.",
 749         optional => 1,
 750     },
 751     werror => {
 752         type => 'string',
 753         enum => [qw(enospc ignore report stop)],
 754         description => 'Write error action.',
 755         optional => 1,
 756     },
 757     aio => {
 758         type => 'string',
 759         enum => [qw(native threads)],
 760         description => 'AIO type to use.',
 761         optional => 1,
 762     },
 763     discard => {
 764         type => 'string',
 765         enum => [qw(ignore on)],
 766         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 767         optional => 1,
 768     },
 769     detect_zeroes => {
 770         type => 'boolean',
 771         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 772         optional => 1,
 773     },
 774     serial => {
 775         type => 'string',
 776         format => 'urlencoded',
 777         format_description => 'serial',
 778         maxLength => 20*3, # *3 since it's %xx url enoded
 779         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
 780         optional => 1,
 781     }
 782 );
 783
 784 my %rerror_fmt = (
 785     rerror => {
 786         type => 'string',
 787         enum => [qw(ignore report stop)],
 788         description => 'Read error action.',
 789         optional => 1,
 790     },
 791 );
 792
 793 my %iothread_fmt = ( iothread => {
 794         type => 'boolean',
 795         description => "Whether to use iothreads for this drive",
 796         optional => 1,
 797 });
 798
 799 my %model_fmt = (
 800     model => {
 801         type => 'string',
 802         format => 'urlencoded',
 803         format_description => 'model',
 804         maxLength => 40*3, # *3 since it's %xx url enoded
 805         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
 806         optional => 1,
 807     },
 808 );
 809
 810 my %queues_fmt = (
 811     queues => {
 812         type => 'integer',
 813         description => "Number of queues.",
 814         minimum => 2,
 815         optional => 1
 816     }
 817 );
 818
 819 my $add_throttle_desc = sub {
 820     my ($key, $type, $what, $unit, $longunit) = @_;
 821     $drivedesc_base{$key} = {
 822         type => $type,
 823         format_description => $unit,
 824         description => "Maximum $what speed in $longunit per second.",
 825         optional => 1,
 826     };
 827 };
 828 # throughput: (leaky bucket)
 829 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes');
 830 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes');
 831 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes');
 832 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes');
 833 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes');
 834 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes');
 835 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations');
 836 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations');
 837 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations');
 838
 839 # pools: (pool of IO before throttling starts taking effect)
 840 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes');
 841 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes');
 842 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes');
 843 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations');
 844 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations');
 845 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations');
 846
 847 my $ide_fmt = {
 848     %drivedesc_base,
 849     %rerror_fmt,
 850     %model_fmt,
 851 };
 852
 853 my $idedesc = {
 854     optional => 1,
 855     type => 'string', format => $ide_fmt,
 856     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
 857 };
 858 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
 859
 860 my $scsi_fmt = {
 861     %drivedesc_base,
 862     %iothread_fmt,
 863     %queues_fmt,
 864 };
 865 my $scsidesc = {
 866     optional => 1,
 867     type => 'string', format => $scsi_fmt,
 868     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
 869 };
 870 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 871
 872 my $sata_fmt = {
 873     %drivedesc_base,
 874     %rerror_fmt,
 875 };
 876 my $satadesc = {
 877     optional => 1,
 878     type => 'string', format => $sata_fmt,
 879     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
 880 };
 881 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
 882
 883 my $virtio_fmt = {
 884     %drivedesc_base,
 885     %iothread_fmt,
 886     %rerror_fmt,
 887 };
 888 my $virtiodesc = {
 889     optional => 1,
 890     type => 'string', format => $virtio_fmt,
 891     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
 892 };
 893 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
 894
 895 my $alldrive_fmt = {
 896     %drivedesc_base,
 897     %rerror_fmt,
 898     %iothread_fmt,
 899     %model_fmt,
 900     %queues_fmt,
 901 };
 902
 903 my $usb_fmt = {
 904     host => {
 905         default_key => 1,
 906         type => 'string', format => 'pve-qm-usb-device',
 907         format_description => 'HOSTUSBDEVICE|spice',
 908         description => <<EODESCR,
 909 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
 910
 911  'bus-port(.port)*' (decimal numbers) or
 912  'vendor_id:product_id' (hexadeciaml numbers) or
 913  'spice'
 914
 915 You can use the 'lsusb -t' command to list existing usb devices.
 916
 917 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 918
 919 The value 'spice' can be used to add a usb redirection devices for spice.
 920 EODESCR
 921     },
 922     usb3 => {
 923         optional => 1,
 924         type => 'boolean',
 925         description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
 926         default => 0,
 927     },
 928 };
 929
 930 my $usbdesc = {
 931     optional => 1,
 932     type => 'string', format => $usb_fmt,
 933     description => "Configure an USB device (n is 0 to 4).",
 934 };
 935 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
 936
 937 # NOTE: the match-groups of this regex are used in parse_hostpci
 938 my $PCIRE = qr/([a-f0-9]{2}:[a-f0-9]{2})(?:\.([a-f0-9]))?/;
 939 my $hostpci_fmt = {
 940     host => {
 941         default_key => 1,
 942         type => 'string',
 943         pattern => qr/$PCIRE(;$PCIRE)*/,
 944         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
 945         description => <<EODESCR,
 946 Host PCI device pass through. The PCI ID of a host's PCI device or a list
 947 of PCI virtual functions of the host. HOSTPCIID syntax is:
 948
 949 'bus:dev.func' (hexadecimal numbers)
 950
 951 You can us the 'lspci' command to list existing PCI devices.
 952 EODESCR
 953     },
 954     rombar => {
 955         type => 'boolean',
 956         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
 957         optional => 1,
 958         default => 1,
 959     },
 960     pcie => {
 961         type => 'boolean',
 962         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
 963         optional => 1,
 964         default => 0,
 965     },
 966     'x-vga' => {
 967         type => 'boolean',
 968         description =>  "Enable vfio-vga device support.",
 969         optional => 1,
 970         default => 0,
 971     },
 972 };
 973 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
 974
 975 my $hostpcidesc = {
 976         optional => 1,
 977         type => 'string', format => 'pve-qm-hostpci',
 978         description => "Map host PCI devices into guest.",
 979         verbose_description =>  <<EODESCR,
 980 Map host PCI devices into guest.
 981
 982 NOTE: This option allows direct access to host hardware. So it is no longer
 983 possible to migrate such machines - use with special care.
 984
 985 CAUTION: Experimental! User reported problems with this option.
 986 EODESCR
 987 };
 988 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
 989
 990 my $serialdesc = {
 991         optional => 1,
 992         type => 'string',
 993         pattern => '(/dev/.+|socket)',
 994         description =>  "Create a serial device inside the VM (n is 0 to 3)",
 995         verbose_description =>  <<EODESCR,
 996 Create a serial device inside the VM (n is 0 to 3), and pass through a
 997 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
 998 host side (use 'qm terminal' to open a terminal connection).
 999
1000 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1001
1002 CAUTION: Experimental! User reported problems with this option.
1003 EODESCR
1004 };
1005
1006 my $paralleldesc= {
1007         optional => 1,
1008         type => 'string',
1009         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1010         description =>  "Map host parallel devices (n is 0 to 2).",
1011         verbose_description =>  <<EODESCR,
1012 Map host parallel devices (n is 0 to 2).
1013
1014 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1015
1016 CAUTION: Experimental! User reported problems with this option.
1017 EODESCR
1018 };
1019
1020 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1021     $confdesc->{"parallel$i"} = $paralleldesc;
1022 }
1023
1024 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1025     $confdesc->{"serial$i"} = $serialdesc;
1026 }
1027
1028 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1029     $confdesc->{"hostpci$i"} = $hostpcidesc;
1030 }
1031
1032 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1033     $drivename_hash->{"ide$i"} = 1;
1034     $confdesc->{"ide$i"} = $idedesc;
1035 }
1036
1037 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1038     $drivename_hash->{"sata$i"} = 1;
1039     $confdesc->{"sata$i"} = $satadesc;
1040 }
1041
1042 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1043     $drivename_hash->{"scsi$i"} = 1;
1044     $confdesc->{"scsi$i"} = $scsidesc ;
1045 }
1046
1047 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1048     $drivename_hash->{"virtio$i"} = 1;
1049     $confdesc->{"virtio$i"} = $virtiodesc;
1050 }
1051
1052 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1053     $confdesc->{"usb$i"} = $usbdesc;
1054 }
1055
1056 my $unuseddesc = {
1057     optional => 1,
1058     type => 'string', format => 'pve-volume-id',
1059     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1060 };
1061
1062 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1063     $confdesc->{"unused$i"} = $unuseddesc;
1064 }
1065
1066 my $kvm_api_version = 0;
1067
1068 sub kvm_version {
1069
1070     return $kvm_api_version if $kvm_api_version;
1071
1072     my $fh = IO::File->new("</dev/kvm") ||
1073         return 0;
1074
1075     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
1076         $kvm_api_version = $v;
1077     }
1078
1079     $fh->close();
1080
1081     return  $kvm_api_version;
1082 }
1083
1084 my $kvm_user_version;
1085
1086 sub kvm_user_version {
1087
1088     return $kvm_user_version if $kvm_user_version;
1089
1090     $kvm_user_version = 'unknown';
1091
1092     my $code = sub {
1093         my $line = shift;
1094         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1095             $kvm_user_version = $2;
1096         }
1097     };
1098
1099     eval { run_command("kvm -version", outfunc => $code); };
1100     warn $@ if $@;
1101
1102     return $kvm_user_version;
1103
1104 }
1105
1106 my $kernel_has_vhost_net = -c '/dev/vhost-net';
1107
1108 sub valid_drive_names {
1109     # order is important - used to autoselect boot disk
1110     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1111             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1112             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1113             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))));
1114 }
1115
1116 sub is_valid_drivename {
1117     my $dev = shift;
1118
1119     return defined($drivename_hash->{$dev});
1120 }
1121
1122 sub option_exists {
1123     my $key = shift;
1124     return defined($confdesc->{$key});
1125 }
1126
1127 sub nic_models {
1128     return $nic_model_list;
1129 }
1130
1131 sub os_list_description {
1132
1133     return {
1134         other => 'Other',
1135         wxp => 'Windows XP',
1136         w2k => 'Windows 2000',
1137         w2k3 =>, 'Windows 2003',
1138         w2k8 => 'Windows 2008',
1139         wvista => 'Windows Vista',
1140         win7 => 'Windows 7',
1141         win8 => 'Windows 8/2012',
1142         l24 => 'Linux 2.4',
1143         l26 => 'Linux 2.6',
1144     };
1145 }
1146
1147 my $cdrom_path;
1148
1149 sub get_cdrom_path {
1150
1151     return  $cdrom_path if $cdrom_path;
1152
1153     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1154     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1155     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1156 }
1157
1158 sub get_iso_path {
1159     my ($storecfg, $vmid, $cdrom) = @_;
1160
1161     if ($cdrom eq 'cdrom') {
1162         return get_cdrom_path();
1163     } elsif ($cdrom eq 'none') {
1164         return '';
1165     } elsif ($cdrom =~ m|^/|) {
1166         return $cdrom;
1167     } else {
1168         return PVE::Storage::path($storecfg, $cdrom);
1169     }
1170 }
1171
1172 # try to convert old style file names to volume IDs
1173 sub filename_to_volume_id {
1174     my ($vmid, $file, $media) = @_;
1175
1176     if (!($file eq 'none' || $file eq 'cdrom' ||
1177           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1178
1179         return undef if $file =~ m|/|;
1180
1181         if ($media && $media eq 'cdrom') {
1182             $file = "local:iso/$file";
1183         } else {
1184             $file = "local:$vmid/$file";
1185         }
1186     }
1187
1188     return $file;
1189 }
1190
1191 sub verify_media_type {
1192     my ($opt, $vtype, $media) = @_;
1193
1194     return if !$media;
1195
1196     my $etype;
1197     if ($media eq 'disk') {
1198         $etype = 'images';
1199     } elsif ($media eq 'cdrom') {
1200         $etype = 'iso';
1201     } else {
1202         die "internal error";
1203     }
1204
1205     return if ($vtype eq $etype);
1206
1207     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1208 }
1209
1210 sub cleanup_drive_path {
1211     my ($opt, $storecfg, $drive) = @_;
1212
1213     # try to convert filesystem paths to volume IDs
1214
1215     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1216         ($drive->{file} !~ m|^/dev/.+|) &&
1217         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1218         ($drive->{file} !~ m/^\d+$/)) {
1219         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1220         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1221         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1222         verify_media_type($opt, $vtype, $drive->{media});
1223         $drive->{file} = $volid;
1224     }
1225
1226     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1227 }
1228
1229 sub parse_hotplug_features {
1230     my ($data) = @_;
1231
1232     my $res = {};
1233
1234     return $res if $data eq '0';
1235
1236     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1237
1238     foreach my $feature (PVE::Tools::split_list($data)) {
1239         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1240             $res->{$1} = 1;
1241         } else {
1242             die "invalid hotplug feature '$feature'\n";
1243         }
1244     }
1245     return $res;
1246 }
1247
1248 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1249 sub pve_verify_hotplug_features {
1250     my ($value, $noerr) = @_;
1251
1252     return $value if parse_hotplug_features($value);
1253
1254     return undef if $noerr;
1255
1256     die "unable to parse hotplug option\n";
1257 }
1258
1259 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1260 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1261 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1262 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1263 #        [,iothread=on][,serial=serial][,model=model]
1264
1265 sub parse_drive {
1266     my ($key, $data) = @_;
1267
1268     my ($interface, $index);
1269
1270     if ($key =~ m/^([^\d]+)(\d+)$/) {
1271         $interface = $1;
1272         $index = $2;
1273     } else {
1274         return undef;
1275     }
1276
1277     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1278                                      : $confdesc->{$key}->{format};
1279     if (!$desc) {
1280         warn "invalid drive key: $key\n";
1281         return undef;
1282     }
1283     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1284     return undef if !$res;
1285     $res->{interface} = $interface;
1286     $res->{index} = $index;
1287
1288     my $error = 0;
1289     foreach my $opt (qw(bps bps_rd bps_wr)) {
1290         if (my $bps = defined(delete $res->{$opt})) {
1291             if (defined($res->{"m$opt"})) {
1292                 warn "both $opt and m$opt specified\n";
1293                 ++$error;
1294                 next;
1295             }
1296             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1297         }
1298     }
1299     return undef if $error;
1300
1301     return undef if $res->{mbps_rd} && $res->{mbps};
1302     return undef if $res->{mbps_wr} && $res->{mbps};
1303     return undef if $res->{iops_rd} && $res->{iops};
1304     return undef if $res->{iops_wr} && $res->{iops};
1305
1306     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1307         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1308         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1309         return undef if $res->{interface} eq 'virtio';
1310     }
1311
1312     if (my $size = $res->{size}) {
1313         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1314     }
1315
1316     return $res;
1317 }
1318
1319 sub print_drive {
1320     my ($vmid, $drive) = @_;
1321     my $data = { %$drive };
1322     delete $data->{$_} for qw(index interface);
1323     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1324 }
1325
1326 sub scsi_inquiry {
1327     my($fh, $noerr) = @_;
1328
1329     my $SG_IO = 0x2285;
1330     my $SG_GET_VERSION_NUM = 0x2282;
1331
1332     my $versionbuf = "\x00" x 8;
1333     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1334     if (!$ret) {
1335         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1336         return undef;
1337     }
1338     my $version = unpack("I", $versionbuf);
1339     if ($version < 30000) {
1340         die "scsi generic interface too old\n"  if !$noerr;
1341         return undef;
1342     }
1343
1344     my $buf = "\x00" x 36;
1345     my $sensebuf = "\x00" x 8;
1346     my $cmd = pack("C x3 C x1", 0x12, 36);
1347
1348     # see /usr/include/scsi/sg.h
1349     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1350
1351     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1352                       length($sensebuf), 0, length($buf), $buf,
1353                       $cmd, $sensebuf, 6000);
1354
1355     $ret = ioctl($fh, $SG_IO, $packet);
1356     if (!$ret) {
1357         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1358         return undef;
1359     }
1360
1361     my @res = unpack($sg_io_hdr_t, $packet);
1362     if ($res[17] || $res[18]) {
1363         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1364         return undef;
1365     }
1366
1367     my $res = {};
1368     (my $byte0, my $byte1, $res->{vendor},
1369      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1370
1371     $res->{removable} = $byte1 & 128 ? 1 : 0;
1372     $res->{type} = $byte0 & 31;
1373
1374     return $res;
1375 }
1376
1377 sub path_is_scsi {
1378     my ($path) = @_;
1379
1380     my $fh = IO::File->new("+<$path") || return undef;
1381     my $res = scsi_inquiry($fh, 1);
1382     close($fh);
1383
1384     return $res;
1385 }
1386
1387 sub machine_type_is_q35 {
1388     my ($conf) = @_;
1389
1390     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1391 }
1392
1393 sub print_tabletdevice_full {
1394     my ($conf) = @_;
1395
1396     my $q35 = machine_type_is_q35($conf);
1397
1398     # we use uhci for old VMs because tablet driver was buggy in older qemu
1399     my $usbbus = $q35 ? "ehci" : "uhci";
1400
1401     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1402 }
1403
1404 sub print_drivedevice_full {
1405     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1406
1407     my $device = '';
1408     my $maxdev = 0;
1409
1410     if ($drive->{interface} eq 'virtio') {
1411         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1412         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1413         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1414     } elsif ($drive->{interface} eq 'scsi') {
1415
1416         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1417         my $unit = $drive->{index} % $maxdev;
1418         my $devicetype = 'hd';
1419         my $path = '';
1420         if (drive_is_cdrom($drive)) {
1421             $devicetype = 'cd';
1422         } else {
1423             if ($drive->{file} =~ m|^/|) {
1424                 $path = $drive->{file};
1425                 if (my $info = path_is_scsi($path)) {
1426                     if ($info->{type} == 0) {
1427                         $devicetype = 'block';
1428                     } elsif ($info->{type} == 1) { # tape
1429                         $devicetype = 'generic';
1430                     }
1431                 }
1432             } else {
1433                  $path = PVE::Storage::path($storecfg, $drive->{file});
1434             }
1435
1436             if($path =~ m/^iscsi\:\/\//){
1437                 $devicetype = 'generic';
1438             }
1439         }
1440
1441         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1442             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1443         } else {
1444             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1445         }
1446
1447     } elsif ($drive->{interface} eq 'ide'){
1448         $maxdev = 2;
1449         my $controller = int($drive->{index} / $maxdev);
1450         my $unit = $drive->{index} % $maxdev;
1451         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1452
1453         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1454         if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
1455             $model = URI::Escape::uri_unescape($model);
1456             $device .= ",model=$model";
1457         }
1458     } elsif ($drive->{interface} eq 'sata'){
1459         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1460         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1461         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1462     } elsif ($drive->{interface} eq 'usb') {
1463         die "implement me";
1464         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1465     } else {
1466         die "unsupported interface type";
1467     }
1468
1469     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1470
1471     return $device;
1472 }
1473
1474 sub get_initiator_name {
1475     my $initiator;
1476
1477     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1478     while (defined(my $line = <$fh>)) {
1479         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1480         $initiator = $1;
1481         last;
1482     }
1483     $fh->close();
1484
1485     return $initiator;
1486 }
1487
1488 sub print_drive_full {
1489     my ($storecfg, $vmid, $drive) = @_;
1490
1491     my $path;
1492     my $volid = $drive->{file};
1493     my $format;
1494
1495     if (drive_is_cdrom($drive)) {
1496         $path = get_iso_path($storecfg, $vmid, $volid);
1497     } else {
1498         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1499         if ($storeid) {
1500             $path = PVE::Storage::path($storecfg, $volid);
1501             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1502             $format = qemu_img_format($scfg, $volname);
1503         } else {
1504             $path = $volid;
1505             $format = "raw";
1506         }
1507    }
1508
1509     my $opts = '';
1510     my @qemu_drive_options = qw(heads secs cyls trans media format cache snapshot rerror werror aio discard iops iops_rd iops_wr iops_max iops_rd_max iops_wr_max);
1511     foreach my $o (@qemu_drive_options) {
1512         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1513     }
1514     if (my $serial = $drive->{serial}) {
1515         $serial = URI::Escape::uri_unescape($serial);
1516         $opts .= ",serial=$serial";
1517     }
1518
1519     $opts .= ",format=$format" if $format && !$drive->{format};
1520
1521     foreach my $o (qw(bps bps_rd bps_wr)) {
1522         my $v = $drive->{"m$o"};
1523         $opts .= ",$o=" . int($v*1024*1024) if $v;
1524     }
1525
1526     my $cache_direct = 0;
1527
1528     if (my $cache = $drive->{cache}) {
1529         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1530     } elsif (!drive_is_cdrom($drive)) {
1531         $opts .= ",cache=none";
1532         $cache_direct = 1;
1533     }
1534
1535     # aio native works only with O_DIRECT
1536     if (!$drive->{aio}) {
1537         if($cache_direct) {
1538             $opts .= ",aio=native";
1539         } else {
1540             $opts .= ",aio=threads";
1541         }
1542     }
1543
1544     if (!drive_is_cdrom($drive)) {
1545         my $detectzeroes;
1546         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1547             $detectzeroes = 'off';
1548         } elsif ($drive->{discard}) {
1549             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1550         } else {
1551             # This used to be our default with discard not being specified:
1552             $detectzeroes = 'on';
1553         }
1554         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1555     }
1556
1557     my $pathinfo = $path ? "file=$path," : '';
1558
1559     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1560 }
1561
1562 sub print_netdevice_full {
1563     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1564
1565     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1566
1567     my $device = $net->{model};
1568     if ($net->{model} eq 'virtio') {
1569          $device = 'virtio-net-pci';
1570      };
1571
1572     my $pciaddr = print_pci_addr("$netid", $bridges);
1573     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1574     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1575         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1576         my $vectors = $net->{queues} * 2 + 2;
1577         $tmpstr .= ",vectors=$vectors,mq=on";
1578     }
1579     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1580
1581     if ($use_old_bios_files) {
1582         my $romfile;
1583         if ($device eq 'virtio-net-pci') {
1584             $romfile = 'pxe-virtio.rom';
1585         } elsif ($device eq 'e1000') {
1586             $romfile = 'pxe-e1000.rom';
1587         } elsif ($device eq 'ne2k') {
1588             $romfile = 'pxe-ne2k_pci.rom';
1589         } elsif ($device eq 'pcnet') {
1590             $romfile = 'pxe-pcnet.rom';
1591         } elsif ($device eq 'rtl8139') {
1592             $romfile = 'pxe-rtl8139.rom';
1593         }
1594         $tmpstr .= ",romfile=$romfile" if $romfile;
1595     }
1596
1597     return $tmpstr;
1598 }
1599
1600 sub print_netdev_full {
1601     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1602
1603     my $i = '';
1604     if ($netid =~ m/^net(\d+)$/) {
1605         $i = int($1);
1606     }
1607
1608     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1609
1610     my $ifname = "tap${vmid}i$i";
1611
1612     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1613     die "interface name '$ifname' is too long (max 15 character)\n"
1614         if length($ifname) >= 16;
1615
1616     my $vhostparam = '';
1617     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1618
1619     my $vmname = $conf->{name} || "vm$vmid";
1620
1621     my $netdev = "";
1622     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1623
1624     if ($net->{bridge}) {
1625         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1626     } else {
1627         $netdev = "type=user,id=$netid,hostname=$vmname";
1628     }
1629
1630     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1631
1632     return $netdev;
1633 }
1634
1635 sub drive_is_cdrom {
1636     my ($drive) = @_;
1637
1638     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1639
1640 }
1641
1642 sub parse_number_sets {
1643     my ($set) = @_;
1644     my $res = [];
1645     foreach my $part (split(/;/, $set)) {
1646         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1647             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1648             push @$res, [ $1, $2 ];
1649         } else {
1650             die "invalid range: $part\n";
1651         }
1652     }
1653     return $res;
1654 }
1655
1656 sub parse_numa {
1657     my ($data) = @_;
1658
1659     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
1660     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1661     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1662     return $res;
1663 }
1664
1665 sub parse_hostpci {
1666     my ($value) = @_;
1667
1668     return undef if !$value;
1669
1670     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
1671
1672     my @idlist = split(/;/, $res->{host});
1673     delete $res->{host};
1674     foreach my $id (@idlist) {
1675         if ($id =~ /^$PCIRE$/) {
1676             push @{$res->{pciid}}, { id => $1, function => ($2//'0') };
1677         } else {
1678             # should have been caught by parse_property_string already
1679             die "failed to parse PCI id: $id\n";
1680         }
1681     }
1682     return $res;
1683 }
1684
1685 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1686 sub parse_net {
1687     my ($data) = @_;
1688
1689     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
1690     if ($@) {
1691         warn $@;
1692         return undef;
1693     }
1694     $res->{macaddr} = PVE::Tools::random_ether_addr() if !defined($res->{macaddr});
1695     return $res;
1696 }
1697
1698 sub print_net {
1699     my $net = shift;
1700
1701     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1702 }
1703
1704 sub add_random_macs {
1705     my ($settings) = @_;
1706
1707     foreach my $opt (keys %$settings) {
1708         next if $opt !~ m/^net(\d+)$/;
1709         my $net = parse_net($settings->{$opt});
1710         next if !$net;
1711         $settings->{$opt} = print_net($net);
1712     }
1713 }
1714
1715 sub vm_is_volid_owner {
1716     my ($storecfg, $vmid, $volid) = @_;
1717
1718     if ($volid !~  m|^/|) {
1719         my ($path, $owner);
1720         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1721         if ($owner && ($owner == $vmid)) {
1722             return 1;
1723         }
1724     }
1725
1726     return undef;
1727 }
1728
1729 sub split_flagged_list {
1730     my $text = shift || '';
1731     $text =~ s/[,;]/ /g;
1732     $text =~ s/^\s+//;
1733     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
1734 }
1735
1736 sub join_flagged_list {
1737     my ($how, $lst) = @_;
1738     join $how, map { $lst->{$_} . $_ } keys %$lst;
1739 }
1740
1741 sub vmconfig_delete_pending_option {
1742     my ($conf, $key, $force) = @_;
1743
1744     delete $conf->{pending}->{$key};
1745     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1746     $pending_delete_hash->{$key} = $force ? '!' : '';
1747     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1748 }
1749
1750 sub vmconfig_undelete_pending_option {
1751     my ($conf, $key) = @_;
1752
1753     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1754     delete $pending_delete_hash->{$key};
1755
1756     if (%$pending_delete_hash) {
1757         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1758     } else {
1759         delete $conf->{pending}->{delete};
1760     }
1761 }
1762
1763 sub vmconfig_register_unused_drive {
1764     my ($storecfg, $vmid, $conf, $drive) = @_;
1765
1766     if (!drive_is_cdrom($drive)) {
1767         my $volid = $drive->{file};
1768         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1769             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1770         }
1771     }
1772 }
1773
1774 sub vmconfig_cleanup_pending {
1775     my ($conf) = @_;
1776
1777     # remove pending changes when nothing changed
1778     my $changes;
1779     foreach my $opt (keys %{$conf->{pending}}) {
1780         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
1781             $changes = 1;
1782             delete $conf->{pending}->{$opt};
1783         }
1784     }
1785
1786     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
1787     my $pending_delete_hash = {};
1788     while (my ($opt, $force) = each %$current_delete_hash) {
1789         if (defined($conf->{$opt})) {
1790             $pending_delete_hash->{$opt} = $force;
1791         } else {
1792             $changes = 1;
1793         }
1794     }
1795
1796     if (%$pending_delete_hash) {
1797         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1798     } else {
1799         delete $conf->{pending}->{delete};
1800     }
1801
1802     return $changes;
1803 }
1804
1805 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
1806 my $smbios1_fmt = {
1807     uuid => {
1808         type => 'string',
1809         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1810         format_description => 'UUID',
1811         description => "Set SMBIOS1 UUID.",
1812         optional => 1,
1813     },
1814     version => {
1815         type => 'string',
1816         pattern => '\S+',
1817         format_description => 'string',
1818         description => "Set SMBIOS1 version.",
1819         optional => 1,
1820     },
1821     serial => {
1822         type => 'string',
1823         pattern => '\S+',
1824         format_description => 'string',
1825         description => "Set SMBIOS1 serial number.",
1826         optional => 1,
1827     },
1828     manufacturer => {
1829         type => 'string',
1830         pattern => '\S+',
1831         format_description => 'string',
1832         description => "Set SMBIOS1 manufacturer.",
1833         optional => 1,
1834     },
1835     product => {
1836         type => 'string',
1837         pattern => '\S+',
1838         format_description => 'string',
1839         description => "Set SMBIOS1 product ID.",
1840         optional => 1,
1841     },
1842     sku => {
1843         type => 'string',
1844         pattern => '\S+',
1845         format_description => 'string',
1846         description => "Set SMBIOS1 SKU string.",
1847         optional => 1,
1848     },
1849     family => {
1850         type => 'string',
1851         pattern => '\S+',
1852         format_description => 'string',
1853         description => "Set SMBIOS1 family string.",
1854         optional => 1,
1855     },
1856 };
1857
1858 sub parse_smbios1 {
1859     my ($data) = @_;
1860
1861     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
1862     warn $@ if $@;
1863     return $res;
1864 }
1865
1866 sub print_smbios1 {
1867     my ($smbios1) = @_;
1868     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
1869 }
1870
1871 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
1872
1873 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
1874 sub verify_bootdisk {
1875     my ($value, $noerr) = @_;
1876
1877     return $value if is_valid_drivename($value);
1878
1879     return undef if $noerr;
1880
1881     die "invalid boot disk '$value'\n";
1882 }
1883
1884 sub parse_watchdog {
1885     my ($value) = @_;
1886
1887     return undef if !$value;
1888
1889     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
1890     warn $@ if $@;
1891     return $res;
1892 }
1893
1894 sub parse_usb_device {
1895     my ($value) = @_;
1896
1897     return undef if !$value;
1898
1899     my $res = {};
1900     if ($value =~ m/^(0x)?([0-9A-Fa-f]{4}):(0x)?([0-9A-Fa-f]{4})$/) {
1901         $res->{vendorid} = $2;
1902         $res->{productid} = $4;
1903     } elsif ($value =~ m/^(\d+)\-(\d+(\.\d+)*)$/) {
1904         $res->{hostbus} = $1;
1905         $res->{hostport} = $2;
1906     } elsif ($value =~ m/^spice$/i) {
1907         $res->{spice} = 1;
1908     } else {
1909         return undef;
1910     }
1911
1912     return $res;
1913 }
1914
1915 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
1916 sub verify_usb_device {
1917     my ($value, $noerr) = @_;
1918
1919     return $value if parse_usb_device($value);
1920
1921     return undef if $noerr;
1922
1923     die "unable to parse usb device\n";
1924 }
1925
1926 # add JSON properties for create and set function
1927 sub json_config_properties {
1928     my $prop = shift;
1929
1930     foreach my $opt (keys %$confdesc) {
1931         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
1932         $prop->{$opt} = $confdesc->{$opt};
1933     }
1934
1935     return $prop;
1936 }
1937
1938 sub check_type {
1939     my ($key, $value) = @_;
1940
1941     die "unknown setting '$key'\n" if !$confdesc->{$key};
1942
1943     my $type = $confdesc->{$key}->{type};
1944
1945     if (!defined($value)) {
1946         die "got undefined value\n";
1947     }
1948
1949     if ($value =~ m/[\n\r]/) {
1950         die "property contains a line feed\n";
1951     }
1952
1953     if ($type eq 'boolean') {
1954         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
1955         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
1956         die "type check ('boolean') failed - got '$value'\n";
1957     } elsif ($type eq 'integer') {
1958         return int($1) if $value =~ m/^(\d+)$/;
1959         die "type check ('integer') failed - got '$value'\n";
1960     } elsif ($type eq 'number') {
1961         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
1962         die "type check ('number') failed - got '$value'\n";
1963     } elsif ($type eq 'string') {
1964         if (my $fmt = $confdesc->{$key}->{format}) {
1965             if ($fmt eq 'pve-qm-drive') {
1966                 # special case - we need to pass $key to parse_drive()
1967                 my $drive = parse_drive($key, $value);
1968                 return $value if $drive;
1969                 die "unable to parse drive options\n";
1970             }
1971             PVE::JSONSchema::check_format($fmt, $value);
1972             return $value;
1973         }
1974         $value =~ s/^\"(.*)\"$/$1/;
1975         return $value;
1976     } else {
1977         die "internal error"
1978     }
1979 }
1980
1981 sub check_iommu_support{
1982     #fixme : need to check IOMMU support
1983     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
1984
1985     my $iommu=1;
1986     return $iommu;
1987
1988 }
1989
1990 sub touch_config {
1991     my ($vmid) = @_;
1992
1993     my $conf = PVE::QemuConfig->config_file($vmid);
1994     utime undef, undef, $conf;
1995 }
1996
1997 sub destroy_vm {
1998     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
1999
2000     my $conffile = PVE::QemuConfig->config_file($vmid);
2001
2002     my $conf = PVE::QemuConfig->load_config($vmid);
2003
2004     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2005
2006     # only remove disks owned by this VM
2007     foreach_drive($conf, sub {
2008         my ($ds, $drive) = @_;
2009
2010         return if drive_is_cdrom($drive);
2011
2012         my $volid = $drive->{file};
2013
2014         return if !$volid || $volid =~ m|^/|;
2015
2016         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2017         return if !$path || !$owner || ($owner != $vmid);
2018
2019         PVE::Storage::vdisk_free($storecfg, $volid);
2020     });
2021
2022     if ($keep_empty_config) {
2023         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2024     } else {
2025         unlink $conffile;
2026     }
2027
2028     # also remove unused disk
2029     eval {
2030         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2031
2032         eval {
2033             PVE::Storage::foreach_volid($dl, sub {
2034                 my ($volid, $sid, $volname, $d) = @_;
2035                 PVE::Storage::vdisk_free($storecfg, $volid);
2036             });
2037         };
2038         warn $@ if $@;
2039
2040     };
2041     warn $@ if $@;
2042 }
2043
2044 sub parse_vm_config {
2045     my ($filename, $raw) = @_;
2046
2047     return undef if !defined($raw);
2048
2049     my $res = {
2050         digest => Digest::SHA::sha1_hex($raw),
2051         snapshots => {},
2052         pending => {},
2053     };
2054
2055     $filename =~ m|/qemu-server/(\d+)\.conf$|
2056         || die "got strange filename '$filename'";
2057
2058     my $vmid = $1;
2059
2060     my $conf = $res;
2061     my $descr;
2062     my $section = '';
2063
2064     my @lines = split(/\n/, $raw);
2065     foreach my $line (@lines) {
2066         next if $line =~ m/^\s*$/;
2067
2068         if ($line =~ m/^\[PENDING\]\s*$/i) {
2069             $section = 'pending';
2070             if (defined($descr)) {
2071                 $descr =~ s/\s+$//;
2072                 $conf->{description} = $descr;
2073             }
2074             $descr = undef;
2075             $conf = $res->{$section} = {};
2076             next;
2077
2078         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2079             $section = $1;
2080             if (defined($descr)) {
2081                 $descr =~ s/\s+$//;
2082                 $conf->{description} = $descr;
2083             }
2084             $descr = undef;
2085             $conf = $res->{snapshots}->{$section} = {};
2086             next;
2087         }
2088
2089         if ($line =~ m/^\#(.*)\s*$/) {
2090             $descr = '' if !defined($descr);
2091             $descr .= PVE::Tools::decode_text($1) . "\n";
2092             next;
2093         }
2094
2095         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2096             $descr = '' if !defined($descr);
2097             $descr .= PVE::Tools::decode_text($2);
2098         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2099             $conf->{snapstate} = $1;
2100         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2101             my $key = $1;
2102             my $value = $2;
2103             $conf->{$key} = $value;
2104         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2105             my $value = $1;
2106             if ($section eq 'pending') {
2107                 $conf->{delete} = $value; # we parse this later
2108             } else {
2109                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2110             }
2111         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
2112             my $key = $1;
2113             my $value = $2;
2114             eval { $value = check_type($key, $value); };
2115             if ($@) {
2116                 warn "vm $vmid - unable to parse value of '$key' - $@";
2117             } else {
2118                 my $fmt = $confdesc->{$key}->{format};
2119                 if ($fmt && $fmt eq 'pve-qm-drive') {
2120                     my $v = parse_drive($key, $value);
2121                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2122                         $v->{file} = $volid;
2123                         $value = print_drive($vmid, $v);
2124                     } else {
2125                         warn "vm $vmid - unable to parse value of '$key'\n";
2126                         next;
2127                     }
2128                 }
2129
2130                 if ($key eq 'cdrom') {
2131                     $conf->{ide2} = $value;
2132                 } else {
2133                     $conf->{$key} = $value;
2134                 }
2135             }
2136         }
2137     }
2138
2139     if (defined($descr)) {
2140         $descr =~ s/\s+$//;
2141         $conf->{description} = $descr;
2142     }
2143     delete $res->{snapstate}; # just to be sure
2144
2145     return $res;
2146 }
2147
2148 sub write_vm_config {
2149     my ($filename, $conf) = @_;
2150
2151     delete $conf->{snapstate}; # just to be sure
2152
2153     if ($conf->{cdrom}) {
2154         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2155         $conf->{ide2} = $conf->{cdrom};
2156         delete $conf->{cdrom};
2157     }
2158
2159     # we do not use 'smp' any longer
2160     if ($conf->{sockets}) {
2161         delete $conf->{smp};
2162     } elsif ($conf->{smp}) {
2163         $conf->{sockets} = $conf->{smp};
2164         delete $conf->{cores};
2165         delete $conf->{smp};
2166     }
2167
2168     my $used_volids = {};
2169
2170     my $cleanup_config = sub {
2171         my ($cref, $pending, $snapname) = @_;
2172
2173         foreach my $key (keys %$cref) {
2174             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2175                 $key eq 'snapstate' || $key eq 'pending';
2176             my $value = $cref->{$key};
2177             if ($key eq 'delete') {
2178                 die "propertry 'delete' is only allowed in [PENDING]\n"
2179                     if !$pending;
2180                 # fixme: check syntax?
2181                 next;
2182             }
2183             eval { $value = check_type($key, $value); };
2184             die "unable to parse value of '$key' - $@" if $@;
2185
2186             $cref->{$key} = $value;
2187
2188             if (!$snapname && is_valid_drivename($key)) {
2189                 my $drive = parse_drive($key, $value);
2190                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2191             }
2192         }
2193     };
2194
2195     &$cleanup_config($conf);
2196
2197     &$cleanup_config($conf->{pending}, 1);
2198
2199     foreach my $snapname (keys %{$conf->{snapshots}}) {
2200         die "internal error" if $snapname eq 'pending';
2201         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2202     }
2203
2204     # remove 'unusedX' settings if we re-add a volume
2205     foreach my $key (keys %$conf) {
2206         my $value = $conf->{$key};
2207         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2208             delete $conf->{$key};
2209         }
2210     }
2211
2212     my $generate_raw_config = sub {
2213         my ($conf, $pending) = @_;
2214
2215         my $raw = '';
2216
2217         # add description as comment to top of file
2218         if (defined(my $descr = $conf->{description})) {
2219             if ($descr) {
2220                 foreach my $cl (split(/\n/, $descr)) {
2221                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2222                 }
2223             } else {
2224                 $raw .= "#\n" if $pending;
2225             }
2226         }
2227
2228         foreach my $key (sort keys %$conf) {
2229             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2230             $raw .= "$key: $conf->{$key}\n";
2231         }
2232         return $raw;
2233     };
2234
2235     my $raw = &$generate_raw_config($conf);
2236
2237     if (scalar(keys %{$conf->{pending}})){
2238         $raw .= "\n[PENDING]\n";
2239         $raw .= &$generate_raw_config($conf->{pending}, 1);
2240     }
2241
2242     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2243         $raw .= "\n[$snapname]\n";
2244         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2245     }
2246
2247     return $raw;
2248 }
2249
2250 sub load_defaults {
2251
2252     my $res = {};
2253
2254     # we use static defaults from our JSON schema configuration
2255     foreach my $key (keys %$confdesc) {
2256         if (defined(my $default = $confdesc->{$key}->{default})) {
2257             $res->{$key} = $default;
2258         }
2259     }
2260
2261     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2262     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2263
2264     return $res;
2265 }
2266
2267 sub config_list {
2268     my $vmlist = PVE::Cluster::get_vmlist();
2269     my $res = {};
2270     return $res if !$vmlist || !$vmlist->{ids};
2271     my $ids = $vmlist->{ids};
2272
2273     foreach my $vmid (keys %$ids) {
2274         my $d = $ids->{$vmid};
2275         next if !$d->{node} || $d->{node} ne $nodename;
2276         next if !$d->{type} || $d->{type} ne 'qemu';
2277         $res->{$vmid}->{exists} = 1;
2278     }
2279     return $res;
2280 }
2281
2282 # test if VM uses local resources (to prevent migration)
2283 sub check_local_resources {
2284     my ($conf, $noerr) = @_;
2285
2286     my $loc_res = 0;
2287
2288     $loc_res = 1 if $conf->{hostusb}; # old syntax
2289     $loc_res = 1 if $conf->{hostpci}; # old syntax
2290
2291     foreach my $k (keys %$conf) {
2292         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2293         # sockets are safe: they will recreated be on the target side post-migrate
2294         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2295         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2296     }
2297
2298     die "VM uses local resources\n" if $loc_res && !$noerr;
2299
2300     return $loc_res;
2301 }
2302
2303 # check if used storages are available on all nodes (use by migrate)
2304 sub check_storage_availability {
2305     my ($storecfg, $conf, $node) = @_;
2306
2307     foreach_drive($conf, sub {
2308         my ($ds, $drive) = @_;
2309
2310         my $volid = $drive->{file};
2311         return if !$volid;
2312
2313         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2314         return if !$sid;
2315
2316         # check if storage is available on both nodes
2317         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2318         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2319    });
2320 }
2321
2322 # list nodes where all VM images are available (used by has_feature API)
2323 sub shared_nodes {
2324     my ($conf, $storecfg) = @_;
2325
2326     my $nodelist = PVE::Cluster::get_nodelist();
2327     my $nodehash = { map { $_ => 1 } @$nodelist };
2328     my $nodename = PVE::INotify::nodename();
2329
2330     foreach_drive($conf, sub {
2331         my ($ds, $drive) = @_;
2332
2333         my $volid = $drive->{file};
2334         return if !$volid;
2335
2336         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2337         if ($storeid) {
2338             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2339             if ($scfg->{disable}) {
2340                 $nodehash = {};
2341             } elsif (my $avail = $scfg->{nodes}) {
2342                 foreach my $node (keys %$nodehash) {
2343                     delete $nodehash->{$node} if !$avail->{$node};
2344                 }
2345             } elsif (!$scfg->{shared}) {
2346                 foreach my $node (keys %$nodehash) {
2347                     delete $nodehash->{$node} if $node ne $nodename
2348                 }
2349             }
2350         }
2351     });
2352
2353     return $nodehash
2354 }
2355
2356 sub check_cmdline {
2357     my ($pidfile, $pid) = @_;
2358
2359     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2360     if (defined($fh)) {
2361         my $line = <$fh>;
2362         $fh->close;
2363         return undef if !$line;
2364         my @param = split(/\0/, $line);
2365
2366         my $cmd = $param[0];
2367         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2368
2369         for (my $i = 0; $i < scalar (@param); $i++) {
2370             my $p = $param[$i];
2371             next if !$p;
2372             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2373                 my $p = $param[$i+1];
2374                 return 1 if $p && ($p eq $pidfile);
2375                 return undef;
2376             }
2377         }
2378     }
2379     return undef;
2380 }
2381
2382 sub check_running {
2383     my ($vmid, $nocheck, $node) = @_;
2384
2385     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2386
2387     die "unable to find configuration file for VM $vmid - no such machine\n"
2388         if !$nocheck && ! -f $filename;
2389
2390     my $pidfile = pidfile_name($vmid);
2391
2392     if (my $fd = IO::File->new("<$pidfile")) {
2393         my $st = stat($fd);
2394         my $line = <$fd>;
2395         close($fd);
2396
2397         my $mtime = $st->mtime;
2398         if ($mtime > time()) {
2399             warn "file '$filename' modified in future\n";
2400         }
2401
2402         if ($line =~ m/^(\d+)$/) {
2403             my $pid = $1;
2404             if (check_cmdline($pidfile, $pid)) {
2405                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2406                     return $pid;
2407                 }
2408             }
2409         }
2410     }
2411
2412     return undef;
2413 }
2414
2415 sub vzlist {
2416
2417     my $vzlist = config_list();
2418
2419     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2420
2421     while (defined(my $de = $fd->read)) {
2422         next if $de !~ m/^(\d+)\.pid$/;
2423         my $vmid = $1;
2424         next if !defined($vzlist->{$vmid});
2425         if (my $pid = check_running($vmid)) {
2426             $vzlist->{$vmid}->{pid} = $pid;
2427         }
2428     }
2429
2430     return $vzlist;
2431 }
2432
2433 sub disksize {
2434     my ($storecfg, $conf) = @_;
2435
2436     my $bootdisk = $conf->{bootdisk};
2437     return undef if !$bootdisk;
2438     return undef if !is_valid_drivename($bootdisk);
2439
2440     return undef if !$conf->{$bootdisk};
2441
2442     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2443     return undef if !defined($drive);
2444
2445     return undef if drive_is_cdrom($drive);
2446
2447     my $volid = $drive->{file};
2448     return undef if !$volid;
2449
2450     return $drive->{size};
2451 }
2452
2453 my $last_proc_pid_stat;
2454
2455 # get VM status information
2456 # This must be fast and should not block ($full == false)
2457 # We only query KVM using QMP if $full == true (this can be slow)
2458 sub vmstatus {
2459     my ($opt_vmid, $full) = @_;
2460
2461     my $res = {};
2462
2463     my $storecfg = PVE::Storage::config();
2464
2465     my $list = vzlist();
2466     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2467
2468     my $cpucount = $cpuinfo->{cpus} || 1;
2469
2470     foreach my $vmid (keys %$list) {
2471         next if $opt_vmid && ($vmid ne $opt_vmid);
2472
2473         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
2474         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2475
2476         my $d = {};
2477         $d->{pid} = $list->{$vmid}->{pid};
2478
2479         # fixme: better status?
2480         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2481
2482         my $size = disksize($storecfg, $conf);
2483         if (defined($size)) {
2484             $d->{disk} = 0; # no info available
2485             $d->{maxdisk} = $size;
2486         } else {
2487             $d->{disk} = 0;
2488             $d->{maxdisk} = 0;
2489         }
2490
2491         $d->{cpus} = ($conf->{sockets} || 1) * ($conf->{cores} || 1);
2492         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2493         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2494
2495         $d->{name} = $conf->{name} || "VM $vmid";
2496         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024) : 0;
2497
2498         if ($conf->{balloon}) {
2499             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2500             $d->{shares} = defined($conf->{shares}) ? $conf->{shares} : 1000;
2501         }
2502
2503         $d->{uptime} = 0;
2504         $d->{cpu} = 0;
2505         $d->{mem} = 0;
2506
2507         $d->{netout} = 0;
2508         $d->{netin} = 0;
2509
2510         $d->{diskread} = 0;
2511         $d->{diskwrite} = 0;
2512
2513         $d->{template} = PVE::QemuConfig->is_template($conf);
2514
2515         $res->{$vmid} = $d;
2516     }
2517
2518     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2519     foreach my $dev (keys %$netdev) {
2520         next if $dev !~ m/^tap([1-9]\d*)i/;
2521         my $vmid = $1;
2522         my $d = $res->{$vmid};
2523         next if !$d;
2524
2525         $d->{netout} += $netdev->{$dev}->{receive};
2526         $d->{netin} += $netdev->{$dev}->{transmit};
2527
2528         if ($full) {
2529             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2530             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2531         }
2532
2533     }
2534
2535     my $ctime = gettimeofday;
2536
2537     foreach my $vmid (keys %$list) {
2538
2539         my $d = $res->{$vmid};
2540         my $pid = $d->{pid};
2541         next if !$pid;
2542
2543         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2544         next if !$pstat; # not running
2545
2546         my $used = $pstat->{utime} + $pstat->{stime};
2547
2548         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2549
2550         if ($pstat->{vsize}) {
2551             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2552         }
2553
2554         my $old = $last_proc_pid_stat->{$pid};
2555         if (!$old) {
2556             $last_proc_pid_stat->{$pid} = {
2557                 time => $ctime,
2558                 used => $used,
2559                 cpu => 0,
2560             };
2561             next;
2562         }
2563
2564         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2565
2566         if ($dtime > 1000) {
2567             my $dutime = $used -  $old->{used};
2568
2569             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2570             $last_proc_pid_stat->{$pid} = {
2571                 time => $ctime,
2572                 used => $used,
2573                 cpu => $d->{cpu},
2574             };
2575         } else {
2576             $d->{cpu} = $old->{cpu};
2577         }
2578     }
2579
2580     return $res if !$full;
2581
2582     my $qmpclient = PVE::QMPClient->new();
2583
2584     my $ballooncb = sub {
2585         my ($vmid, $resp) = @_;
2586
2587         my $info = $resp->{'return'};
2588         return if !$info->{max_mem};
2589
2590         my $d = $res->{$vmid};
2591
2592         # use memory assigned to VM
2593         $d->{maxmem} = $info->{max_mem};
2594         $d->{balloon} = $info->{actual};
2595
2596         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2597             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2598             $d->{freemem} = $info->{free_mem};
2599         }
2600
2601         $d->{ballooninfo} = $info;
2602     };
2603
2604     my $blockstatscb = sub {
2605         my ($vmid, $resp) = @_;
2606         my $data = $resp->{'return'} || [];
2607         my $totalrdbytes = 0;
2608         my $totalwrbytes = 0;
2609
2610         for my $blockstat (@$data) {
2611             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2612             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2613
2614             $blockstat->{device} =~ s/drive-//;
2615             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2616         }
2617         $res->{$vmid}->{diskread} = $totalrdbytes;
2618         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2619     };
2620
2621     my $statuscb = sub {
2622         my ($vmid, $resp) = @_;
2623
2624         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2625         # this fails if ballon driver is not loaded, so this must be
2626         # the last commnand (following command are aborted if this fails).
2627         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2628
2629         my $status = 'unknown';
2630         if (!defined($status = $resp->{'return'}->{status})) {
2631             warn "unable to get VM status\n";
2632             return;
2633         }
2634
2635         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2636     };
2637
2638     foreach my $vmid (keys %$list) {
2639         next if $opt_vmid && ($vmid ne $opt_vmid);
2640         next if !$res->{$vmid}->{pid}; # not running
2641         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2642     }
2643
2644     $qmpclient->queue_execute(undef, 1);
2645
2646     foreach my $vmid (keys %$list) {
2647         next if $opt_vmid && ($vmid ne $opt_vmid);
2648         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2649     }
2650
2651     return $res;
2652 }
2653
2654 sub foreach_drive {
2655     my ($conf, $func, @param) = @_;
2656
2657     foreach my $ds (valid_drive_names()) {
2658         next if !defined($conf->{$ds});
2659
2660         my $drive = parse_drive($ds, $conf->{$ds});
2661         next if !$drive;
2662
2663         &$func($ds, $drive, @param);
2664     }
2665 }
2666
2667 sub foreach_volid {
2668     my ($conf, $func, @param) = @_;
2669
2670     my $volhash = {};
2671
2672     my $test_volid = sub {
2673         my ($volid, $is_cdrom) = @_;
2674
2675         return if !$volid;
2676
2677         $volhash->{$volid} = $is_cdrom || 0;
2678     };
2679
2680     foreach_drive($conf, sub {
2681         my ($ds, $drive) = @_;
2682         &$test_volid($drive->{file}, drive_is_cdrom($drive));
2683     });
2684
2685     foreach my $snapname (keys %{$conf->{snapshots}}) {
2686         my $snap = $conf->{snapshots}->{$snapname};
2687         &$test_volid($snap->{vmstate}, 0);
2688         foreach_drive($snap, sub {
2689             my ($ds, $drive) = @_;
2690             &$test_volid($drive->{file}, drive_is_cdrom($drive));
2691         });
2692     }
2693
2694     foreach my $volid (keys %$volhash) {
2695         &$func($volid, $volhash->{$volid}, @param);
2696     }
2697 }
2698
2699 sub vga_conf_has_spice {
2700     my ($vga) = @_;
2701
2702     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
2703
2704     return $1 || 1;
2705 }
2706
2707 sub config_to_command {
2708     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2709
2710     my $cmd = [];
2711     my $globalFlags = [];
2712     my $machineFlags = [];
2713     my $rtcFlags = [];
2714     my $cpuFlags = [];
2715     my $devices = [];
2716     my $pciaddr = '';
2717     my $bridges = {};
2718     my $kvmver = kvm_user_version();
2719     my $vernum = 0; # unknown
2720     my $ostype = $conf->{ostype};
2721     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
2722         $vernum = $1*1000000+$2*1000;
2723     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
2724         $vernum = $1*1000000+$2*1000+$3;
2725     }
2726
2727     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
2728
2729     my $have_ovz = -f '/proc/vz/vestat';
2730
2731     my $q35 = machine_type_is_q35($conf);
2732     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2733     my $machine_type = $forcemachine || $conf->{machine};
2734     my $use_old_bios_files = undef;
2735     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
2736
2737     my $cpuunits = defined($conf->{cpuunits}) ?
2738             $conf->{cpuunits} : $defaults->{cpuunits};
2739
2740     push @$cmd, '/usr/bin/kvm';
2741
2742     push @$cmd, '-id', $vmid;
2743
2744     my $use_virtio = 0;
2745
2746     my $qmpsocket = qmp_socket($vmid);
2747     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2748     push @$cmd, '-mon', "chardev=qmp,mode=control";
2749
2750
2751     push @$cmd, '-pidfile' , pidfile_name($vmid);
2752
2753     push @$cmd, '-daemonize';
2754
2755     if ($conf->{smbios1}) {
2756         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
2757     }
2758
2759     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2760         my $ovmfvar = "OVMF_VARS-pure-efi.fd";
2761         my $ovmfvar_src = "/usr/share/kvm/$ovmfvar";
2762         my $ovmfvar_dst = "/tmp/$vmid-$ovmfvar";
2763         PVE::Tools::file_copy($ovmfvar_src, $ovmfvar_dst, 256*1024);
2764         push @$cmd, '-drive', "if=pflash,format=raw,readonly,file=/usr/share/kvm/OVMF-pure-efi.fd";
2765         push @$cmd, '-drive', "if=pflash,format=raw,file=$ovmfvar_dst";
2766     }
2767
2768     if ($q35) {
2769         # the q35 chipset support native usb2, so we enable usb controller
2770         # by default for this machine type
2771         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
2772     } else {
2773         $pciaddr = print_pci_addr("piix3", $bridges);
2774         push @$devices, '-device', "piix3-usb-uhci,id=uhci$pciaddr.0x2";
2775
2776         my $use_usb2 = 0;
2777         for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2778             next if !$conf->{"usb$i"};
2779             my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format},$conf->{"usb$i"}) };
2780             next if !$d || $d->{usb3}; # do not add usb2 controller if we have only usb3 devices
2781             $use_usb2 = 1;
2782         }
2783         # include usb device config
2784         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-usb.cfg' if $use_usb2;
2785     }
2786
2787     # add usb3 controller if needed
2788
2789     my $use_usb3 = 0;
2790     for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2791         next if !$conf->{"usb$i"};
2792         my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format},$conf->{"usb$i"}) };
2793         next if !$d || !$d->{usb3};
2794         $use_usb3 = 1;
2795     }
2796
2797     $pciaddr = print_pci_addr("xhci", $bridges);
2798     push @$devices, '-device', "nec-usb-xhci,id=xhci$pciaddr" if $use_usb3;
2799
2800     my $vga = $conf->{vga};
2801
2802     my $qxlnum = vga_conf_has_spice($vga);
2803     $vga = 'qxl' if $qxlnum;
2804
2805     if (!$vga) {
2806         if ($conf->{ostype} && ($conf->{ostype} eq 'win8' ||
2807                                 $conf->{ostype} eq 'win7' ||
2808                                 $conf->{ostype} eq 'w2k8')) {
2809             $vga = 'std';
2810         } else {
2811             $vga = 'cirrus';
2812         }
2813     }
2814
2815     # enable absolute mouse coordinates (needed by vnc)
2816     my $tablet;
2817     if (defined($conf->{tablet})) {
2818         $tablet = $conf->{tablet};
2819     } else {
2820         $tablet = $defaults->{tablet};
2821         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
2822         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
2823     }
2824
2825     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
2826
2827     my $kvm_off = 0;
2828     # host pci devices
2829     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
2830         my $d = parse_hostpci($conf->{"hostpci$i"});
2831         next if !$d;
2832
2833         my $pcie = $d->{pcie};
2834         if($pcie){
2835             die "q35 machine model is not enabled" if !$q35;
2836             $pciaddr = print_pcie_addr("hostpci$i");
2837         }else{
2838             $pciaddr = print_pci_addr("hostpci$i", $bridges);
2839         }
2840
2841         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
2842         my $xvga = '';
2843         if ($d->{'x-vga'}) {
2844             $xvga = ',x-vga=on';
2845             $kvm_off = 1;
2846             $vga = 'none';
2847             if ($ostype eq 'win7' || $ostype eq 'win8' || $ostype eq 'w2k8') {
2848                 push @$cpuFlags , 'hv_vendor_id=proxmox';
2849             }
2850             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2851                 $xvga = "";
2852             }
2853         }
2854         my $pcidevices = $d->{pciid};
2855         my $multifunction = 1 if @$pcidevices > 1;
2856
2857         my $j=0;
2858         foreach my $pcidevice (@$pcidevices) {
2859
2860             my $id = "hostpci$i";
2861             $id .= ".$j" if $multifunction;
2862             my $addr = $pciaddr;
2863             $addr .= ".$j" if $multifunction;
2864             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
2865
2866             if($j == 0){
2867                 $devicestr .= "$rombar$xvga";
2868                 $devicestr .= ",multifunction=on" if $multifunction;
2869             }
2870
2871             push @$devices, '-device', $devicestr;
2872             $j++;
2873         }
2874     }
2875
2876     # usb devices
2877     for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2878         next if !$conf->{"usb$i"};
2879         my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format},$conf->{"usb$i"}) };
2880         next if !$d;
2881
2882         # if it is a usb3 device, attach it to the xhci controller, else omit the bus option
2883         my $usbbus = '';
2884         if (defined($d->{usb3}) && $d->{usb3}) {
2885             $usbbus = ',bus=xhci.0';
2886         }
2887
2888         if (defined($d->{host})) {
2889             $d = parse_usb_device($d->{host});
2890             if (defined($d->{vendorid}) && defined($d->{productid})) {
2891                 push @$devices, '-device', "usb-host$usbbus,vendorid=0x$d->{vendorid},productid=0x$d->{productid}";
2892             } elsif (defined($d->{hostbus}) && defined($d->{hostport})) {
2893                 push @$devices, '-device', "usb-host$usbbus,hostbus=$d->{hostbus},hostport=$d->{hostport}";
2894             } elsif (defined($d->{spice}) && $d->{spice}) {
2895                 # usb redir support for spice, currently no usb3
2896                 push @$devices, '-chardev', "spicevmc,id=usbredirchardev$i,name=usbredir";
2897                 push @$devices, '-device', "usb-redir,chardev=usbredirchardev$i,id=usbredirdev$i,bus=ehci.0";
2898             }
2899         }
2900     }
2901
2902     # serial devices
2903     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2904         if (my $path = $conf->{"serial$i"}) {
2905             if ($path eq 'socket') {
2906                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
2907                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
2908                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2909             } else {
2910                 die "no such serial device\n" if ! -c $path;
2911                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
2912                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2913             }
2914         }
2915     }
2916
2917     # parallel devices
2918     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
2919         if (my $path = $conf->{"parallel$i"}) {
2920             die "no such parallel device\n" if ! -c $path;
2921             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
2922             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
2923             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
2924         }
2925     }
2926
2927     my $vmname = $conf->{name} || "vm$vmid";
2928
2929     push @$cmd, '-name', $vmname;
2930
2931     my $sockets = 1;
2932     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
2933     $sockets = $conf->{sockets} if  $conf->{sockets};
2934
2935     my $cores = $conf->{cores} || 1;
2936
2937     my $maxcpus = $sockets * $cores;
2938
2939     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
2940
2941     my $allowed_vcpus = $cpuinfo->{cpus};
2942
2943     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
2944         if ($allowed_vcpus < $maxcpus);
2945
2946     push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
2947
2948     push @$cmd, '-nodefaults';
2949
2950     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2951
2952     my $bootindex_hash = {};
2953     my $i = 1;
2954     foreach my $o (split(//, $bootorder)) {
2955         $bootindex_hash->{$o} = $i*100;
2956         $i++;
2957     }
2958
2959     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000";
2960
2961     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
2962
2963     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
2964
2965     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
2966
2967     if ($vga && $vga !~ m/^serial\d+$/ && $vga ne 'none'){
2968         my $socket = vnc_socket($vmid);
2969         push @$cmd,  '-vnc', "unix:$socket,x509,password";
2970     } else {
2971         push @$cmd, '-nographic';
2972     }
2973
2974     # time drift fix
2975     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
2976
2977     my $nokvm = defined($conf->{kvm}) && $conf->{kvm} == 0 ? 1 : 0;
2978     my $useLocaltime = $conf->{localtime};
2979
2980     if ($ostype) {
2981         # other, wxp, w2k, w2k3, w2k8, wvista, win7, win8, l24, l26, solaris
2982
2983         if ($ostype =~ m/^w/) { # windows
2984             $useLocaltime = 1 if !defined($conf->{localtime});
2985
2986             # use time drift fix when acpi is enabled
2987             if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
2988                 $tdf = 1 if !defined($conf->{tdf});
2989             }
2990         }
2991
2992         if ($ostype eq 'win7' || $ostype eq 'win8' || $ostype eq 'w2k8' ||
2993             $ostype eq 'wvista') {
2994             push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
2995             push @$cmd, '-no-hpet';
2996             if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
2997                 push @$cpuFlags , 'hv_spinlocks=0x1fff' if !$nokvm;
2998                 push @$cpuFlags , 'hv_vapic' if !$nokvm;
2999                 push @$cpuFlags , 'hv_time' if !$nokvm;
3000
3001                 if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
3002                     push @$cpuFlags , 'hv_reset' if !$nokvm;
3003                     push @$cpuFlags , 'hv_vpindex' if !$nokvm;
3004                     push @$cpuFlags , 'hv_runtime' if !$nokvm;
3005                 }
3006
3007             } else {
3008                 push @$cpuFlags , 'hv_spinlocks=0xffff' if !$nokvm;
3009             }
3010         }
3011
3012         if ($ostype eq 'win7' || $ostype eq 'win8') {
3013             push @$cpuFlags , 'hv_relaxed' if !$nokvm;
3014         }
3015     }
3016
3017     push @$rtcFlags, 'driftfix=slew' if $tdf;
3018
3019     if ($nokvm) {
3020         push @$machineFlags, 'accel=tcg';
3021     } else {
3022         die "No accelerator found!\n" if !$cpuinfo->{hvm};
3023     }
3024
3025     if ($machine_type) {
3026         push @$machineFlags, "type=${machine_type}";
3027     }
3028
3029     if ($conf->{startdate}) {
3030         push @$rtcFlags, "base=$conf->{startdate}";
3031     } elsif ($useLocaltime) {
3032         push @$rtcFlags, 'base=localtime';
3033     }
3034
3035     my $cpu = $nokvm ? "qemu64" : "kvm64";
3036     if (my $cputype = $conf->{cpu}) {
3037         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3038             or die "Cannot parse cpu description: $cputype\n";
3039         $cpu = $cpuconf->{cputype};
3040         $kvm_off = 1 if $cpuconf->{hidden};
3041     }
3042
3043     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
3044
3045     push @$cpuFlags , '-x2apic'
3046         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3047
3048     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3049
3050     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3051
3052     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3053
3054         push @$cpuFlags , '+kvm_pv_unhalt' if !$nokvm;
3055         push @$cpuFlags , '+kvm_pv_eoi' if !$nokvm;
3056     }
3057
3058     push @$cpuFlags, 'enforce' if $cpu ne 'host' && !$nokvm;
3059
3060     push @$cpuFlags, 'kvm=off' if $kvm_off;
3061
3062     my $cpu_vendor = $cpu_vendor_list->{$cpu} ||
3063         die "internal error"; # should not happen
3064
3065     push @$cpuFlags, "vendor=${cpu_vendor}"
3066         if $cpu_vendor ne 'default';
3067
3068     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3069
3070     push @$cmd, '-cpu', $cpu;
3071
3072     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3073
3074     push @$cmd, '-S' if $conf->{freeze};
3075
3076     # set keyboard layout
3077     my $kb = $conf->{keyboard} || $defaults->{keyboard};
3078     push @$cmd, '-k', $kb if $kb;
3079
3080     # enable sound
3081     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3082     #push @$cmd, '-soundhw', 'es1370';
3083     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3084
3085     if($conf->{agent}) {
3086         my $qgasocket = qmp_socket($vmid, 1);
3087         my $pciaddr = print_pci_addr("qga0", $bridges);
3088         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3089         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3090         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3091     }
3092
3093     my $spice_port;
3094
3095     if ($qxlnum) {
3096         if ($qxlnum > 1) {
3097             if ($conf->{ostype} && $conf->{ostype} =~ m/^w/){
3098                 for(my $i = 1; $i < $qxlnum; $i++){
3099                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3100                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3101                 }
3102             } else {
3103                 # assume other OS works like Linux
3104                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3105                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3106             }
3107         }
3108
3109         my $pciaddr = print_pci_addr("spice", $bridges);
3110
3111         my $nodename = PVE::INotify::nodename();
3112         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3113         $spice_port = PVE::Tools::next_spice_port($pfamily);
3114
3115         push @$devices, '-spice', "tls-port=${spice_port},addr=localhost,tls-ciphers=DES-CBC3-SHA,seamless-migration=on";
3116
3117         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3118         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3119         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3120     }
3121
3122     # enable balloon by default, unless explicitly disabled
3123     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3124         $pciaddr = print_pci_addr("balloon0", $bridges);
3125         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3126     }
3127
3128     if ($conf->{watchdog}) {
3129         my $wdopts = parse_watchdog($conf->{watchdog});
3130         $pciaddr = print_pci_addr("watchdog", $bridges);
3131         my $watchdog = $wdopts->{model} || 'i6300esb';
3132         push @$devices, '-device', "$watchdog$pciaddr";
3133         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3134     }
3135
3136     my $vollist = [];
3137     my $scsicontroller = {};
3138     my $ahcicontroller = {};
3139     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3140
3141     # Add iscsi initiator name if available
3142     if (my $initiator = get_initiator_name()) {
3143         push @$devices, '-iscsi', "initiator-name=$initiator";
3144     }
3145
3146     foreach_drive($conf, sub {
3147         my ($ds, $drive) = @_;
3148
3149         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3150             push @$vollist, $drive->{file};
3151         }
3152
3153         $use_virtio = 1 if $ds =~ m/^virtio/;
3154
3155         if (drive_is_cdrom ($drive)) {
3156             if ($bootindex_hash->{d}) {
3157                 $drive->{bootindex} = $bootindex_hash->{d};
3158                 $bootindex_hash->{d} += 1;
3159             }
3160         } else {
3161             if ($bootindex_hash->{c}) {
3162                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3163                 $bootindex_hash->{c} += 1;
3164             }
3165         }
3166
3167         if($drive->{interface} eq 'virtio'){
3168            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3169         }
3170
3171         if ($drive->{interface} eq 'scsi') {
3172
3173             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3174
3175             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3176             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3177
3178             my $iothread = '';
3179             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3180                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3181                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3182             } elsif ($drive->{iothread}) {
3183                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3184             }
3185
3186             my $queues = '';
3187             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3188                 $queues = ",num_queues=$drive->{queues}";
3189             }
3190
3191             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3192             $scsicontroller->{$controller}=1;
3193         }
3194
3195         if ($drive->{interface} eq 'sata') {
3196            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3197            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3198            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3199            $ahcicontroller->{$controller}=1;
3200         }
3201
3202         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3203         push @$devices, '-drive',$drive_cmd;
3204         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3205     });
3206
3207     for (my $i = 0; $i < $MAX_NETS; $i++) {
3208          next if !$conf->{"net$i"};
3209          my $d = parse_net($conf->{"net$i"});
3210          next if !$d;
3211
3212          $use_virtio = 1 if $d->{model} eq 'virtio';
3213
3214          if ($bootindex_hash->{n}) {
3215             $d->{bootindex} = $bootindex_hash->{n};
3216             $bootindex_hash->{n} += 1;
3217          }
3218
3219          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3220          push @$devices, '-netdev', $netdevfull;
3221
3222          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3223          push @$devices, '-device', $netdevicefull;
3224     }
3225
3226     if (!$q35) {
3227         # add pci bridges
3228         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3229            $bridges->{1} = 1;
3230            $bridges->{2} = 1;
3231         }
3232
3233         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3234
3235         while (my ($k, $v) = each %$bridges) {
3236             $pciaddr = print_pci_addr("pci.$k");
3237             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3238         }
3239     }
3240
3241     # add custom args
3242     if ($conf->{args}) {
3243         my $aa = PVE::Tools::split_args($conf->{args});
3244         push @$cmd, @$aa;
3245     }
3246
3247     push @$cmd, @$devices;
3248     push @$cmd, '-rtc', join(',', @$rtcFlags)
3249         if scalar(@$rtcFlags);
3250     push @$cmd, '-machine', join(',', @$machineFlags)
3251         if scalar(@$machineFlags);
3252     push @$cmd, '-global', join(',', @$globalFlags)
3253         if scalar(@$globalFlags);
3254
3255     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3256 }
3257
3258 sub vnc_socket {
3259     my ($vmid) = @_;
3260     return "${var_run_tmpdir}/$vmid.vnc";
3261 }
3262
3263 sub spice_port {
3264     my ($vmid) = @_;
3265
3266     my $res = vm_mon_cmd($vmid, 'query-spice');
3267
3268     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3269 }
3270
3271 sub qmp_socket {
3272     my ($vmid, $qga) = @_;
3273     my $sockettype = $qga ? 'qga' : 'qmp';
3274     return "${var_run_tmpdir}/$vmid.$sockettype";
3275 }
3276
3277 sub pidfile_name {
3278     my ($vmid) = @_;
3279     return "${var_run_tmpdir}/$vmid.pid";
3280 }
3281
3282 sub vm_devices_list {
3283     my ($vmid) = @_;
3284
3285     my $res = vm_mon_cmd($vmid, 'query-pci');
3286     my $devices = {};
3287     foreach my $pcibus (@$res) {
3288         foreach my $device (@{$pcibus->{devices}}) {
3289             next if !$device->{'qdev_id'};
3290             if ($device->{'pci_bridge'}) {
3291                 $devices->{$device->{'qdev_id'}} = 1;
3292                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3293                     next if !$bridge_device->{'qdev_id'};
3294                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3295                     $devices->{$device->{'qdev_id'}}++;
3296                 }
3297             } else {
3298                 $devices->{$device->{'qdev_id'}} = 1;
3299             }
3300         }
3301     }
3302
3303     my $resblock = vm_mon_cmd($vmid, 'query-block');
3304     foreach my $block (@$resblock) {
3305         if($block->{device} =~ m/^drive-(\S+)/){
3306                 $devices->{$1} = 1;
3307         }
3308     }
3309
3310     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3311     foreach my $mice (@$resmice) {
3312         if ($mice->{name} eq 'QEMU HID Tablet') {
3313             $devices->{tablet} = 1;
3314             last;
3315         }
3316     }
3317
3318     return $devices;
3319 }
3320
3321 sub vm_deviceplug {
3322     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3323
3324     my $q35 = machine_type_is_q35($conf);
3325
3326     my $devices_list = vm_devices_list($vmid);
3327     return 1 if defined($devices_list->{$deviceid});
3328
3329     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3330
3331     if ($deviceid eq 'tablet') {
3332
3333         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3334
3335     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3336
3337         qemu_iothread_add($vmid, $deviceid, $device);
3338
3339         qemu_driveadd($storecfg, $vmid, $device);
3340         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3341
3342         qemu_deviceadd($vmid, $devicefull);
3343         eval { qemu_deviceaddverify($vmid, $deviceid); };
3344         if (my $err = $@) {
3345             eval { qemu_drivedel($vmid, $deviceid); };
3346             warn $@ if $@;
3347             die $err;
3348         }
3349
3350     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3351
3352
3353         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3354         my $pciaddr = print_pci_addr($deviceid);
3355         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3356
3357         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3358
3359         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3360             qemu_iothread_add($vmid, $deviceid, $device);
3361             $devicefull .= ",iothread=iothread-$deviceid";
3362         }
3363
3364         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3365             $devicefull .= ",num_queues=$device->{queues}";
3366         }
3367
3368         qemu_deviceadd($vmid, $devicefull);
3369         qemu_deviceaddverify($vmid, $deviceid);
3370
3371     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3372
3373         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3374         qemu_driveadd($storecfg, $vmid, $device);
3375
3376         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3377         eval { qemu_deviceadd($vmid, $devicefull); };
3378         if (my $err = $@) {
3379             eval { qemu_drivedel($vmid, $deviceid); };
3380             warn $@ if $@;
3381             die $err;
3382         }
3383
3384     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3385
3386         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3387
3388         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
3389         my $use_old_bios_files = undef;
3390         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3391
3392         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3393         qemu_deviceadd($vmid, $netdevicefull);
3394         eval { qemu_deviceaddverify($vmid, $deviceid); };
3395         if (my $err = $@) {
3396             eval { qemu_netdevdel($vmid, $deviceid); };
3397             warn $@ if $@;
3398             die $err;
3399         }
3400
3401     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3402
3403         my $bridgeid = $2;
3404         my $pciaddr = print_pci_addr($deviceid);
3405         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3406
3407         qemu_deviceadd($vmid, $devicefull);
3408         qemu_deviceaddverify($vmid, $deviceid);
3409
3410     } else {
3411         die "can't hotplug device '$deviceid'\n";
3412     }
3413
3414     return 1;
3415 }
3416
3417 # fixme: this should raise exceptions on error!
3418 sub vm_deviceunplug {
3419     my ($vmid, $conf, $deviceid) = @_;
3420
3421     my $devices_list = vm_devices_list($vmid);
3422     return 1 if !defined($devices_list->{$deviceid});
3423
3424     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3425
3426     if ($deviceid eq 'tablet') {
3427
3428         qemu_devicedel($vmid, $deviceid);
3429
3430     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3431
3432         qemu_devicedel($vmid, $deviceid);
3433         qemu_devicedelverify($vmid, $deviceid);
3434         qemu_drivedel($vmid, $deviceid);
3435         qemu_iothread_del($conf, $vmid, $deviceid);
3436
3437     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3438
3439         qemu_devicedel($vmid, $deviceid);
3440         qemu_devicedelverify($vmid, $deviceid);
3441         qemu_iothread_del($conf, $vmid, $deviceid);
3442
3443     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3444
3445         #qemu 2.3 segfault on drive_del with virtioscsi + iothread
3446         my $device = parse_drive($deviceid, $conf->{$deviceid});
3447         die "virtioscsi with iothread is not hot-unplugglable currently" if $device->{iothread};
3448
3449         qemu_devicedel($vmid, $deviceid);
3450         qemu_drivedel($vmid, $deviceid);
3451         qemu_deletescsihw($conf, $vmid, $deviceid);
3452
3453     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3454
3455         qemu_devicedel($vmid, $deviceid);
3456         qemu_devicedelverify($vmid, $deviceid);
3457         qemu_netdevdel($vmid, $deviceid);
3458
3459     } else {
3460         die "can't unplug device '$deviceid'\n";
3461     }
3462
3463     return 1;
3464 }
3465
3466 sub qemu_deviceadd {
3467     my ($vmid, $devicefull) = @_;
3468
3469     $devicefull = "driver=".$devicefull;
3470     my %options =  split(/[=,]/, $devicefull);
3471
3472     vm_mon_cmd($vmid, "device_add" , %options);
3473 }
3474
3475 sub qemu_devicedel {
3476     my ($vmid, $deviceid) = @_;
3477
3478     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3479 }
3480
3481 sub qemu_iothread_add {
3482     my($vmid, $deviceid, $device) = @_;
3483
3484     if ($device->{iothread}) {
3485         my $iothreads = vm_iothreads_list($vmid);
3486         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3487     }
3488 }
3489
3490 sub qemu_iothread_del {
3491     my($conf, $vmid, $deviceid) = @_;
3492
3493     my $device = parse_drive($deviceid, $conf->{$deviceid});
3494     if ($device->{iothread}) {
3495         my $iothreads = vm_iothreads_list($vmid);
3496         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3497     }
3498 }
3499
3500 sub qemu_objectadd {
3501     my($vmid, $objectid, $qomtype) = @_;
3502
3503     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3504
3505     return 1;
3506 }
3507
3508 sub qemu_objectdel {
3509     my($vmid, $objectid) = @_;
3510
3511     vm_mon_cmd($vmid, "object-del", id => $objectid);
3512
3513     return 1;
3514 }
3515
3516 sub qemu_driveadd {
3517     my ($storecfg, $vmid, $device) = @_;
3518
3519     my $drive = print_drive_full($storecfg, $vmid, $device);
3520     $drive =~ s/\\/\\\\/g;
3521     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3522
3523     # If the command succeeds qemu prints: "OK"
3524     return 1 if $ret =~ m/OK/s;
3525
3526     die "adding drive failed: $ret\n";
3527 }
3528
3529 sub qemu_drivedel {
3530     my($vmid, $deviceid) = @_;
3531
3532     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3533     $ret =~ s/^\s+//;
3534
3535     return 1 if $ret eq "";
3536
3537     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3538     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3539
3540     die "deleting drive $deviceid failed : $ret\n";
3541 }
3542
3543 sub qemu_deviceaddverify {
3544     my ($vmid, $deviceid) = @_;
3545
3546     for (my $i = 0; $i <= 5; $i++) {
3547          my $devices_list = vm_devices_list($vmid);
3548          return 1 if defined($devices_list->{$deviceid});
3549          sleep 1;
3550     }
3551
3552     die "error on hotplug device '$deviceid'\n";
3553 }
3554
3555
3556 sub qemu_devicedelverify {
3557     my ($vmid, $deviceid) = @_;
3558
3559     # need to verify that the device is correctly removed as device_del
3560     # is async and empty return is not reliable
3561
3562     for (my $i = 0; $i <= 5; $i++) {
3563          my $devices_list = vm_devices_list($vmid);
3564          return 1 if !defined($devices_list->{$deviceid});
3565          sleep 1;
3566     }
3567
3568     die "error on hot-unplugging device '$deviceid'\n";
3569 }
3570
3571 sub qemu_findorcreatescsihw {
3572     my ($storecfg, $conf, $vmid, $device) = @_;
3573
3574     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3575
3576     my $scsihwid="$controller_prefix$controller";
3577     my $devices_list = vm_devices_list($vmid);
3578
3579     if(!defined($devices_list->{$scsihwid})) {
3580         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3581     }
3582
3583     return 1;
3584 }
3585
3586 sub qemu_deletescsihw {
3587     my ($conf, $vmid, $opt) = @_;
3588
3589     my $device = parse_drive($opt, $conf->{$opt});
3590
3591     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3592         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3593         return 1;
3594     }
3595
3596     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3597
3598     my $devices_list = vm_devices_list($vmid);
3599     foreach my $opt (keys %{$devices_list}) {
3600         if (PVE::QemuServer::is_valid_drivename($opt)) {
3601             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3602             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3603                 return 1;
3604             }
3605         }
3606     }
3607
3608     my $scsihwid="scsihw$controller";
3609
3610     vm_deviceunplug($vmid, $conf, $scsihwid);
3611
3612     return 1;
3613 }
3614
3615 sub qemu_add_pci_bridge {
3616     my ($storecfg, $conf, $vmid, $device) = @_;
3617
3618     my $bridges = {};
3619
3620     my $bridgeid;
3621
3622     print_pci_addr($device, $bridges);
3623
3624     while (my ($k, $v) = each %$bridges) {
3625         $bridgeid = $k;
3626     }
3627     return 1 if !defined($bridgeid) || $bridgeid < 1;
3628
3629     my $bridge = "pci.$bridgeid";
3630     my $devices_list = vm_devices_list($vmid);
3631
3632     if (!defined($devices_list->{$bridge})) {
3633         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3634     }
3635
3636     return 1;
3637 }
3638
3639 sub qemu_set_link_status {
3640     my ($vmid, $device, $up) = @_;
3641
3642     vm_mon_cmd($vmid, "set_link", name => $device,
3643                up => $up ? JSON::true : JSON::false);
3644 }
3645
3646 sub qemu_netdevadd {
3647     my ($vmid, $conf, $device, $deviceid) = @_;
3648
3649     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
3650     my %options =  split(/[=,]/, $netdev);
3651
3652     vm_mon_cmd($vmid, "netdev_add",  %options);
3653     return 1;
3654 }
3655
3656 sub qemu_netdevdel {
3657     my ($vmid, $deviceid) = @_;
3658
3659     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3660 }
3661
3662 sub qemu_cpu_hotplug {
3663     my ($vmid, $conf, $vcpus) = @_;
3664
3665     my $sockets = 1;
3666     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3667     $sockets = $conf->{sockets} if  $conf->{sockets};
3668     my $cores = $conf->{cores} || 1;
3669     my $maxcpus = $sockets * $cores;
3670
3671     $vcpus = $maxcpus if !$vcpus;
3672
3673     die "you can't add more vcpus than maxcpus\n"
3674         if $vcpus > $maxcpus;
3675
3676     my $currentvcpus = $conf->{vcpus} || $maxcpus;
3677     die "online cpu unplug is not yet possible\n"
3678         if $vcpus < $currentvcpus;
3679
3680     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3681     die "vcpus in running vm is different than configuration\n"
3682         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
3683
3684     for (my $i = $currentvcpus; $i < $vcpus; $i++) {
3685         vm_mon_cmd($vmid, "cpu-add", id => int($i));
3686     }
3687 }
3688
3689 sub qemu_block_set_io_throttle {
3690     my ($vmid, $deviceid,
3691         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
3692         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max) = @_;
3693
3694     return if !check_running($vmid) ;
3695
3696     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
3697         bps => int($bps),
3698         bps_rd => int($bps_rd),
3699         bps_wr => int($bps_wr),
3700         iops => int($iops),
3701         iops_rd => int($iops_rd),
3702         iops_wr => int($iops_wr),
3703         bps_max => int($bps_max),
3704         bps_rd_max => int($bps_rd_max),
3705         bps_wr_max => int($bps_wr_max),
3706         iops_max => int($iops_max),
3707         iops_rd_max => int($iops_rd_max),
3708         iops_wr_max => int($iops_wr_max)
3709     );
3710
3711 }
3712
3713 # old code, only used to shutdown old VM after update
3714 sub __read_avail {
3715     my ($fh, $timeout) = @_;
3716
3717     my $sel = new IO::Select;
3718     $sel->add($fh);
3719
3720     my $res = '';
3721     my $buf;
3722
3723     my @ready;
3724     while (scalar (@ready = $sel->can_read($timeout))) {
3725         my $count;
3726         if ($count = $fh->sysread($buf, 8192)) {
3727             if ($buf =~ /^(.*)\(qemu\) $/s) {
3728                 $res .= $1;
3729                 last;
3730             } else {
3731                 $res .= $buf;
3732             }
3733         } else {
3734             if (!defined($count)) {
3735                 die "$!\n";
3736             }
3737             last;
3738         }
3739     }
3740
3741     die "monitor read timeout\n" if !scalar(@ready);
3742
3743     return $res;
3744 }
3745
3746 # old code, only used to shutdown old VM after update
3747 sub vm_monitor_command {
3748     my ($vmid, $cmdstr, $nocheck) = @_;
3749
3750     my $res;
3751
3752     eval {
3753         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
3754
3755         my $sname = "${var_run_tmpdir}/$vmid.mon";
3756
3757         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
3758             die "unable to connect to VM $vmid socket - $!\n";
3759
3760         my $timeout = 3;
3761
3762         # hack: migrate sometime blocks the monitor (when migrate_downtime
3763         # is set)
3764         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3765             $timeout = 60*60; # 1 hour
3766         }
3767
3768         # read banner;
3769         my $data = __read_avail($sock, $timeout);
3770
3771         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
3772             die "got unexpected qemu monitor banner\n";
3773         }
3774
3775         my $sel = new IO::Select;
3776         $sel->add($sock);
3777
3778         if (!scalar(my @ready = $sel->can_write($timeout))) {
3779             die "monitor write error - timeout";
3780         }
3781
3782         my $fullcmd = "$cmdstr\r";
3783
3784         # syslog('info', "VM $vmid monitor command: $cmdstr");
3785
3786         my $b;
3787         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
3788             die "monitor write error - $!";
3789         }
3790
3791         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
3792
3793         $timeout = 20;
3794
3795         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3796             $timeout = 60*60; # 1 hour
3797         } elsif ($cmdstr =~ m/^(eject|change)/) {
3798             $timeout = 60; # note: cdrom mount command is slow
3799         }
3800         if ($res = __read_avail($sock, $timeout)) {
3801
3802             my @lines = split("\r?\n", $res);
3803
3804             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
3805
3806             $res = join("\n", @lines);
3807             $res .= "\n";
3808         }
3809     };
3810
3811     my $err = $@;
3812
3813     if ($err) {
3814         syslog("err", "VM $vmid monitor command failed - $err");
3815         die $err;
3816     }
3817
3818     return $res;
3819 }
3820
3821 sub qemu_block_resize {
3822     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
3823
3824     my $running = check_running($vmid);
3825
3826     return if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
3827
3828     return if !$running;
3829
3830     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
3831
3832 }
3833
3834 sub qemu_volume_snapshot {
3835     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3836
3837     my $running = check_running($vmid);
3838
3839     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
3840         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
3841     } else {
3842         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
3843     }
3844 }
3845
3846 sub qemu_volume_snapshot_delete {
3847     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3848
3849     my $running = check_running($vmid);
3850
3851     return if !PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
3852
3853     return if !$running;
3854
3855     vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
3856 }
3857
3858 sub set_migration_caps {
3859     my ($vmid) = @_;
3860
3861     my $cap_ref = [];
3862
3863     my $enabled_cap = {
3864         "auto-converge" => 1,
3865         "xbzrle" => 1,
3866         "x-rdma-pin-all" => 0,
3867         "zero-blocks" => 0,
3868         "compress" => 0
3869     };
3870
3871     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
3872
3873     for my $supported_capability (@$supported_capabilities) {
3874         push @$cap_ref, {
3875             capability => $supported_capability->{capability},
3876             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
3877         };
3878     }
3879
3880     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
3881 }
3882
3883 my $fast_plug_option = {
3884     'lock' => 1,
3885     'name' => 1,
3886     'onboot' => 1,
3887     'shares' => 1,
3888     'startup' => 1,
3889     'description' => 1,
3890 };
3891
3892 # hotplug changes in [PENDING]
3893 # $selection hash can be used to only apply specified options, for
3894 # example: { cores => 1 } (only apply changed 'cores')
3895 # $errors ref is used to return error messages
3896 sub vmconfig_hotplug_pending {
3897     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
3898
3899     my $defaults = load_defaults();
3900
3901     # commit values which do not have any impact on running VM first
3902     # Note: those option cannot raise errors, we we do not care about
3903     # $selection and always apply them.
3904
3905     my $add_error = sub {
3906         my ($opt, $msg) = @_;
3907         $errors->{$opt} = "hotplug problem - $msg";
3908     };
3909
3910     my $changes = 0;
3911     foreach my $opt (keys %{$conf->{pending}}) { # add/change
3912         if ($fast_plug_option->{$opt}) {
3913             $conf->{$opt} = $conf->{pending}->{$opt};
3914             delete $conf->{pending}->{$opt};
3915             $changes = 1;
3916         }
3917     }
3918
3919     if ($changes) {
3920         PVE::QemuConfig->write_config($vmid, $conf);
3921         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
3922     }
3923
3924     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3925
3926     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
3927     while (my ($opt, $force) = each %$pending_delete_hash) {
3928         next if $selection && !$selection->{$opt};
3929         eval {
3930             if ($opt eq 'hotplug') {
3931                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
3932             } elsif ($opt eq 'tablet') {
3933                 die "skip\n" if !$hotplug_features->{usb};
3934                 if ($defaults->{tablet}) {
3935                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
3936                 } else {
3937                     vm_deviceunplug($vmid, $conf, $opt);
3938                 }
3939             } elsif ($opt eq 'vcpus') {
3940                 die "skip\n" if !$hotplug_features->{cpu};
3941                 qemu_cpu_hotplug($vmid, $conf, undef);
3942             } elsif ($opt eq 'balloon') {
3943                 # enable balloon device is not hotpluggable
3944                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
3945             } elsif ($fast_plug_option->{$opt}) {
3946                 # do nothing
3947             } elsif ($opt =~ m/^net(\d+)$/) {
3948                 die "skip\n" if !$hotplug_features->{network};
3949                 vm_deviceunplug($vmid, $conf, $opt);
3950             } elsif (is_valid_drivename($opt)) {
3951                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
3952                 vm_deviceunplug($vmid, $conf, $opt);
3953                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
3954             } elsif ($opt =~ m/^memory$/) {
3955                 die "skip\n" if !$hotplug_features->{memory};
3956                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
3957             } elsif ($opt eq 'cpuunits') {
3958                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
3959             } elsif ($opt eq 'cpulimit') {
3960                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
3961             } else {
3962                 die "skip\n";
3963             }
3964         };
3965         if (my $err = $@) {
3966             &$add_error($opt, $err) if $err ne "skip\n";
3967         } else {
3968             # save new config if hotplug was successful
3969             delete $conf->{$opt};
3970             vmconfig_undelete_pending_option($conf, $opt);
3971             PVE::QemuConfig->write_config($vmid, $conf);
3972             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
3973         }
3974     }
3975
3976     foreach my $opt (keys %{$conf->{pending}}) {
3977         next if $selection && !$selection->{$opt};
3978         my $value = $conf->{pending}->{$opt};
3979         eval {
3980             if ($opt eq 'hotplug') {
3981                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
3982             } elsif ($opt eq 'tablet') {
3983                 die "skip\n" if !$hotplug_features->{usb};
3984                 if ($value == 1) {
3985                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
3986                 } elsif ($value == 0) {
3987                     vm_deviceunplug($vmid, $conf, $opt);
3988                 }
3989             } elsif ($opt eq 'vcpus') {
3990                 die "skip\n" if !$hotplug_features->{cpu};
3991                 qemu_cpu_hotplug($vmid, $conf, $value);
3992             } elsif ($opt eq 'balloon') {
3993                 # enable/disable balloning device is not hotpluggable
3994                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
3995                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
3996                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
3997
3998                 # allow manual ballooning if shares is set to zero
3999                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4000                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4001                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4002                 }
4003             } elsif ($opt =~ m/^net(\d+)$/) {
4004                 # some changes can be done without hotplug
4005                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4006                                     $vmid, $opt, $value);
4007             } elsif (is_valid_drivename($opt)) {
4008                 # some changes can be done without hotplug
4009                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4010                                      $vmid, $opt, $value, 1);
4011             } elsif ($opt =~ m/^memory$/) { #dimms
4012                 die "skip\n" if !$hotplug_features->{memory};
4013                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4014             } elsif ($opt eq 'cpuunits') {
4015                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4016             } elsif ($opt eq 'cpulimit') {
4017                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4018                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4019             } else {
4020                 die "skip\n";  # skip non-hot-pluggable options
4021             }
4022         };
4023         if (my $err = $@) {
4024             &$add_error($opt, $err) if $err ne "skip\n";
4025         } else {
4026             # save new config if hotplug was successful
4027             $conf->{$opt} = $value;
4028             delete $conf->{pending}->{$opt};
4029             PVE::QemuConfig->write_config($vmid, $conf);
4030             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4031         }
4032     }
4033 }
4034
4035 sub try_deallocate_drive {
4036     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4037
4038     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4039         my $volid = $drive->{file};
4040         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4041             my $sid = PVE::Storage::parse_volume_id($volid);
4042             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4043
4044             # check if the disk is really unused
4045             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4046                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4047             PVE::Storage::vdisk_free($storecfg, $volid);
4048             return 1;
4049         } else {
4050             # If vm is not owner of this disk remove from config
4051             return 1;
4052         }
4053     }
4054
4055     return undef;
4056 }
4057
4058 sub vmconfig_delete_or_detach_drive {
4059     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4060
4061     my $drive = parse_drive($opt, $conf->{$opt});
4062
4063     my $rpcenv = PVE::RPCEnvironment::get();
4064     my $authuser = $rpcenv->get_user();
4065
4066     if ($force) {
4067         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4068         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4069     } else {
4070         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4071     }
4072 }
4073
4074 sub vmconfig_apply_pending {
4075     my ($vmid, $conf, $storecfg) = @_;
4076
4077     # cold plug
4078
4079     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4080     while (my ($opt, $force) = each %$pending_delete_hash) {
4081         die "internal error" if $opt =~ m/^unused/;
4082         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4083         if (!defined($conf->{$opt})) {
4084             vmconfig_undelete_pending_option($conf, $opt);
4085             PVE::QemuConfig->write_config($vmid, $conf);
4086         } elsif (is_valid_drivename($opt)) {
4087             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4088             vmconfig_undelete_pending_option($conf, $opt);
4089             delete $conf->{$opt};
4090             PVE::QemuConfig->write_config($vmid, $conf);
4091         } else {
4092             vmconfig_undelete_pending_option($conf, $opt);
4093             delete $conf->{$opt};
4094             PVE::QemuConfig->write_config($vmid, $conf);
4095         }
4096     }
4097
4098     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4099
4100     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4101         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4102
4103         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4104             # skip if nothing changed
4105         } elsif (is_valid_drivename($opt)) {
4106             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4107                 if defined($conf->{$opt});
4108             $conf->{$opt} = $conf->{pending}->{$opt};
4109         } else {
4110             $conf->{$opt} = $conf->{pending}->{$opt};
4111         }
4112
4113         delete $conf->{pending}->{$opt};
4114         PVE::QemuConfig->write_config($vmid, $conf);
4115     }
4116 }
4117
4118 my $safe_num_ne = sub {
4119     my ($a, $b) = @_;
4120
4121     return 0 if !defined($a) && !defined($b);
4122     return 1 if !defined($a);
4123     return 1 if !defined($b);
4124
4125     return $a != $b;
4126 };
4127
4128 my $safe_string_ne = sub {
4129     my ($a, $b) = @_;
4130
4131     return 0 if !defined($a) && !defined($b);
4132     return 1 if !defined($a);
4133     return 1 if !defined($b);
4134
4135     return $a ne $b;
4136 };
4137
4138 sub vmconfig_update_net {
4139     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4140
4141     my $newnet = parse_net($value);
4142
4143     if ($conf->{$opt}) {
4144         my $oldnet = parse_net($conf->{$opt});
4145
4146         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4147             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4148             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4149             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4150
4151             # for non online change, we try to hot-unplug
4152             die "skip\n" if !$hotplug;
4153             vm_deviceunplug($vmid, $conf, $opt);
4154         } else {
4155
4156             die "internal error" if $opt !~ m/net(\d+)/;
4157             my $iface = "tap${vmid}i$1";
4158
4159             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4160                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4161                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4162                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4163                 PVE::Network::tap_unplug($iface);
4164                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4165             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4166                 # Rate can be applied on its own but any change above needs to
4167                 # include the rate in tap_plug since OVS resets everything.
4168                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4169             }
4170
4171             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4172                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4173             }
4174
4175             return 1;
4176         }
4177     }
4178
4179     if ($hotplug) {
4180         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4181     } else {
4182         die "skip\n";
4183     }
4184 }
4185
4186 sub vmconfig_update_disk {
4187     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4188
4189     # fixme: do we need force?
4190
4191     my $drive = parse_drive($opt, $value);
4192
4193     if ($conf->{$opt}) {
4194
4195         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4196
4197             my $media = $drive->{media} || 'disk';
4198             my $oldmedia = $old_drive->{media} || 'disk';
4199             die "unable to change media type\n" if $media ne $oldmedia;
4200
4201             if (!drive_is_cdrom($old_drive)) {
4202
4203                 if ($drive->{file} ne $old_drive->{file}) {
4204
4205                     die "skip\n" if !$hotplug;
4206
4207                     # unplug and register as unused
4208                     vm_deviceunplug($vmid, $conf, $opt);
4209                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4210
4211                 } else {
4212                     # update existing disk
4213
4214                     # skip non hotpluggable value
4215                     if (&$safe_num_ne($drive->{discard}, $old_drive->{discard}) ||
4216                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4217                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4218                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4219                         die "skip\n";
4220                     }
4221
4222                     # apply throttle
4223                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4224                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4225                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4226                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4227                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4228                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4229                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4230                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4231                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4232                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4233                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4234                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max})) {
4235
4236                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4237                                                    ($drive->{mbps} || 0)*1024*1024,
4238                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4239                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4240                                                    $drive->{iops} || 0,
4241                                                    $drive->{iops_rd} || 0,
4242                                                    $drive->{iops_wr} || 0,
4243                                                    ($drive->{mbps_max} || 0)*1024*1024,
4244                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4245                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4246                                                    $drive->{iops_max} || 0,
4247                                                    $drive->{iops_rd_max} || 0,
4248                                                    $drive->{iops_wr_max} || 0);
4249
4250                     }
4251
4252                     return 1;
4253                 }
4254
4255             } else { # cdrom
4256
4257                 if ($drive->{file} eq 'none') {
4258                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4259                 } else {
4260                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4261                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4262                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4263                 }
4264
4265                 return 1;
4266             }
4267         }
4268     }
4269
4270     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4271     # hotplug new disks
4272     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4273     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4274 }
4275
4276 sub vm_start {
4277     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4278         $forcemachine, $spice_ticket) = @_;
4279
4280     PVE::QemuConfig->lock_config($vmid, sub {
4281         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4282
4283         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
4284
4285         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4286
4287         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4288
4289         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4290             vmconfig_apply_pending($vmid, $conf, $storecfg);
4291             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4292         }
4293
4294         my $defaults = load_defaults();
4295
4296         # set environment variable useful inside network script
4297         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4298
4299         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4300
4301         my $migrate_port = 0;
4302         my $migrate_uri;
4303         if ($statefile) {
4304             if ($statefile eq 'tcp') {
4305                 my $localip = "localhost";
4306                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4307                 my $nodename = PVE::INotify::nodename();
4308                 if ($datacenterconf->{migration_unsecure}) {
4309                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4310                         $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4311                 }
4312                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4313                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4314                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4315                 push @$cmd, '-incoming', $migrate_uri;
4316                 push @$cmd, '-S';
4317
4318             } elsif ($statefile eq 'unix') {
4319                 # should be default for secure migrations as a ssh TCP forward
4320                 # tunnel is not deterministic reliable ready and fails regurarly
4321                 # to set up in time, so use UNIX socket forwards
4322                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
4323                 unlink $socket_addr;
4324
4325                 $migrate_uri = "unix:$socket_addr";
4326
4327                 push @$cmd, '-incoming', $migrate_uri;
4328                 push @$cmd, '-S';
4329
4330             } else {
4331                 push @$cmd, '-loadstate', $statefile;
4332             }
4333         } elsif ($paused) {
4334             push @$cmd, '-S';
4335         }
4336
4337         # host pci devices
4338         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4339           my $d = parse_hostpci($conf->{"hostpci$i"});
4340           next if !$d;
4341           my $pcidevices = $d->{pciid};
4342           foreach my $pcidevice (@$pcidevices) {
4343                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4344
4345                 my $info = pci_device_info("0000:$pciid");
4346                 die "IOMMU not present\n" if !check_iommu_support();
4347                 die "no pci device info for device '$pciid'\n" if !$info;
4348                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4349                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4350           }
4351         }
4352
4353         PVE::Storage::activate_volumes($storecfg, $vollist);
4354
4355         if (!check_running($vmid, 1) && -d "/sys/fs/cgroup/systemd/qemu.slice/$vmid.scope") {
4356             my $cmd = [];
4357             push @$cmd, '/bin/systemctl', 'stop', "$vmid.scope";
4358             eval  { run_command($cmd); };
4359         }
4360
4361         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
4362                                                   : $defaults->{cpuunits};
4363
4364         my %run_params = (timeout => $statefile ? undef : 30, umask => 0077);
4365
4366         my %properties = (
4367             Slice => 'qemu.slice',
4368             KillMode => 'none',
4369             CPUShares => $cpuunits
4370         );
4371
4372         if (my $cpulimit = $conf->{cpulimit}) {
4373             $properties{CPUQuota} = int($cpulimit * 100);
4374         }
4375         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
4376
4377         if ($conf->{hugepages}) {
4378
4379             my $code = sub {
4380                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
4381                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
4382
4383                 PVE::QemuServer::Memory::hugepages_mount();
4384                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
4385
4386                 eval  {
4387                     PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4388                     run_command($cmd, %run_params);
4389                 };
4390
4391                 if (my $err = $@) {
4392                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
4393                     die $err;
4394                 }
4395
4396                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
4397             };
4398             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
4399
4400         } else {
4401             eval  {
4402                 PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4403                 run_command($cmd, %run_params);
4404             };
4405         }
4406
4407         if (my $err = $@) {
4408             # deactivate volumes if start fails
4409             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
4410             die "start failed: $err";
4411         }
4412
4413         print "migration listens on $migrate_uri\n" if $migrate_uri;
4414
4415         if ($statefile && $statefile ne 'tcp')  {
4416             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4417             warn $@ if $@;
4418         }
4419
4420         if ($migratedfrom) {
4421
4422             eval {
4423                 set_migration_caps($vmid);
4424             };
4425             warn $@ if $@;
4426
4427             if ($spice_port) {
4428                 print "spice listens on port $spice_port\n";
4429                 if ($spice_ticket) {
4430                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4431                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4432                 }
4433             }
4434
4435         } else {
4436
4437             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4438                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4439                     if $conf->{balloon};
4440             }
4441
4442             foreach my $opt (keys %$conf) {
4443                 next if $opt !~  m/^net\d+$/;
4444                 my $nicconf = parse_net($conf->{$opt});
4445                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4446             }
4447         }
4448
4449         vm_mon_cmd_nocheck($vmid, 'qom-set',
4450                     path => "machine/peripheral/balloon0",
4451                     property => "guest-stats-polling-interval",
4452                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4453
4454     });
4455 }
4456
4457 sub vm_mon_cmd {
4458     my ($vmid, $execute, %params) = @_;
4459
4460     my $cmd = { execute => $execute, arguments => \%params };
4461     vm_qmp_command($vmid, $cmd);
4462 }
4463
4464 sub vm_mon_cmd_nocheck {
4465     my ($vmid, $execute, %params) = @_;
4466
4467     my $cmd = { execute => $execute, arguments => \%params };
4468     vm_qmp_command($vmid, $cmd, 1);
4469 }
4470
4471 sub vm_qmp_command {
4472     my ($vmid, $cmd, $nocheck) = @_;
4473
4474     my $res;
4475
4476     my $timeout;
4477     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4478         $timeout = $cmd->{arguments}->{timeout};
4479         delete $cmd->{arguments}->{timeout};
4480     }
4481
4482     eval {
4483         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4484         my $sname = qmp_socket($vmid);
4485         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4486             my $qmpclient = PVE::QMPClient->new();
4487
4488             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4489         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
4490             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
4491                 if scalar(%{$cmd->{arguments}});
4492             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
4493         } else {
4494             die "unable to open monitor socket\n";
4495         }
4496     };
4497     if (my $err = $@) {
4498         syslog("err", "VM $vmid qmp command failed - $err");
4499         die $err;
4500     }
4501
4502     return $res;
4503 }
4504
4505 sub vm_human_monitor_command {
4506     my ($vmid, $cmdline) = @_;
4507
4508     my $res;
4509
4510     my $cmd = {
4511         execute => 'human-monitor-command',
4512         arguments => { 'command-line' => $cmdline},
4513     };
4514
4515     return vm_qmp_command($vmid, $cmd);
4516 }
4517
4518 sub vm_commandline {
4519     my ($storecfg, $vmid) = @_;
4520
4521     my $conf = PVE::QemuConfig->load_config($vmid);
4522
4523     my $defaults = load_defaults();
4524
4525     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
4526
4527     return join(' ', @$cmd);
4528 }
4529
4530 sub vm_reset {
4531     my ($vmid, $skiplock) = @_;
4532
4533     PVE::QemuConfig->lock_config($vmid, sub {
4534
4535         my $conf = PVE::QemuConfig->load_config($vmid);
4536
4537         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4538
4539         vm_mon_cmd($vmid, "system_reset");
4540     });
4541 }
4542
4543 sub get_vm_volumes {
4544     my ($conf) = @_;
4545
4546     my $vollist = [];
4547     foreach_volid($conf, sub {
4548         my ($volid, $is_cdrom) = @_;
4549
4550         return if $volid =~ m|^/|;
4551
4552         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4553         return if !$sid;
4554
4555         push @$vollist, $volid;
4556     });
4557
4558     return $vollist;
4559 }
4560
4561 sub vm_stop_cleanup {
4562     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
4563
4564     eval {
4565
4566         if (!$keepActive) {
4567             my $vollist = get_vm_volumes($conf);
4568             PVE::Storage::deactivate_volumes($storecfg, $vollist);
4569         }
4570
4571         foreach my $ext (qw(mon qmp pid vnc qga)) {
4572             unlink "/var/run/qemu-server/${vmid}.$ext";
4573         }
4574
4575         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
4576     };
4577     warn $@ if $@; # avoid errors - just warn
4578 }
4579
4580 # Note: use $nockeck to skip tests if VM configuration file exists.
4581 # We need that when migration VMs to other nodes (files already moved)
4582 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
4583 sub vm_stop {
4584     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
4585
4586     $force = 1 if !defined($force) && !$shutdown;
4587
4588     if ($migratedfrom){
4589         my $pid = check_running($vmid, $nocheck, $migratedfrom);
4590         kill 15, $pid if $pid;
4591         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4592         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
4593         return;
4594     }
4595
4596     PVE::QemuConfig->lock_config($vmid, sub {
4597
4598         my $pid = check_running($vmid, $nocheck);
4599         return if !$pid;
4600
4601         my $conf;
4602         if (!$nocheck) {
4603             $conf = PVE::QemuConfig->load_config($vmid);
4604             PVE::QemuConfig->check_lock($conf) if !$skiplock;
4605             if (!defined($timeout) && $shutdown && $conf->{startup}) {
4606                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
4607                 $timeout = $opts->{down} if $opts->{down};
4608             }
4609         }
4610
4611         $timeout = 60 if !defined($timeout);
4612
4613         eval {
4614             if ($shutdown) {
4615                 if (defined($conf) && $conf->{agent}) {
4616                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
4617                 } else {
4618                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
4619                 }
4620             } else {
4621                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
4622             }
4623         };
4624         my $err = $@;
4625
4626         if (!$err) {
4627             my $count = 0;
4628             while (($count < $timeout) && check_running($vmid, $nocheck)) {
4629                 $count++;
4630                 sleep 1;
4631             }
4632
4633             if ($count >= $timeout) {
4634                 if ($force) {
4635                     warn "VM still running - terminating now with SIGTERM\n";
4636                     kill 15, $pid;
4637                 } else {
4638                     die "VM quit/powerdown failed - got timeout\n";
4639                 }
4640             } else {
4641                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4642                 return;
4643             }
4644         } else {
4645             if ($force) {
4646                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
4647                 kill 15, $pid;
4648             } else {
4649                 die "VM quit/powerdown failed\n";
4650             }
4651         }
4652
4653         # wait again
4654         $timeout = 10;
4655
4656         my $count = 0;
4657         while (($count < $timeout) && check_running($vmid, $nocheck)) {
4658             $count++;
4659             sleep 1;
4660         }
4661
4662         if ($count >= $timeout) {
4663             warn "VM still running - terminating now with SIGKILL\n";
4664             kill 9, $pid;
4665             sleep 1;
4666         }
4667
4668         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4669    });
4670 }
4671
4672 sub vm_suspend {
4673     my ($vmid, $skiplock) = @_;
4674
4675     PVE::QemuConfig->lock_config($vmid, sub {
4676
4677         my $conf = PVE::QemuConfig->load_config($vmid);
4678
4679         PVE::QemuConfig->check_lock($conf)
4680             if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
4681
4682         vm_mon_cmd($vmid, "stop");
4683     });
4684 }
4685
4686 sub vm_resume {
4687     my ($vmid, $skiplock, $nocheck) = @_;
4688
4689     PVE::QemuConfig->lock_config($vmid, sub {
4690
4691         if (!$nocheck) {
4692
4693             my $conf = PVE::QemuConfig->load_config($vmid);
4694
4695             PVE::QemuConfig->check_lock($conf)
4696                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
4697
4698             vm_mon_cmd($vmid, "cont");
4699
4700         } else {
4701             vm_mon_cmd_nocheck($vmid, "cont");
4702         }
4703     });
4704 }
4705
4706 sub vm_sendkey {
4707     my ($vmid, $skiplock, $key) = @_;
4708
4709     PVE::QemuConfig->lock_config($vmid, sub {
4710
4711         my $conf = PVE::QemuConfig->load_config($vmid);
4712
4713         # there is no qmp command, so we use the human monitor command
4714         vm_human_monitor_command($vmid, "sendkey $key");
4715     });
4716 }
4717
4718 sub vm_destroy {
4719     my ($storecfg, $vmid, $skiplock) = @_;
4720
4721     PVE::QemuConfig->lock_config($vmid, sub {
4722
4723         my $conf = PVE::QemuConfig->load_config($vmid);
4724
4725         if (!check_running($vmid)) {
4726             destroy_vm($storecfg, $vmid, undef, $skiplock);
4727         } else {
4728             die "VM $vmid is running - destroy failed\n";
4729         }
4730     });
4731 }
4732
4733 # pci helpers
4734
4735 sub file_write {
4736     my ($filename, $buf) = @_;
4737
4738     my $fh = IO::File->new($filename, "w");
4739     return undef if !$fh;
4740
4741     my $res = print $fh $buf;
4742
4743     $fh->close();
4744
4745     return $res;
4746 }
4747
4748 sub pci_device_info {
4749     my ($name) = @_;
4750
4751     my $res;
4752
4753     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
4754     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
4755
4756     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
4757     return undef if !defined($irq) || $irq !~ m/^\d+$/;
4758
4759     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
4760     return undef if !defined($vendor) || $vendor !~ s/^0x//;
4761
4762     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
4763     return undef if !defined($product) || $product !~ s/^0x//;
4764
4765     $res = {
4766         name => $name,
4767         vendor => $vendor,
4768         product => $product,
4769         domain => $domain,
4770         bus => $bus,
4771         slot => $slot,
4772         func => $func,
4773         irq => $irq,
4774         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
4775     };
4776
4777     return $res;
4778 }
4779
4780 sub pci_dev_reset {
4781     my ($dev) = @_;
4782
4783     my $name = $dev->{name};
4784
4785     my $fn = "$pcisysfs/devices/$name/reset";
4786
4787     return file_write($fn, "1");
4788 }
4789
4790 sub pci_dev_bind_to_vfio {
4791     my ($dev) = @_;
4792
4793     my $name = $dev->{name};
4794
4795     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4796
4797     if (!-d $vfio_basedir) {
4798         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4799     }
4800     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4801
4802     my $testdir = "$vfio_basedir/$name";
4803     return 1 if -d $testdir;
4804
4805     my $data = "$dev->{vendor} $dev->{product}";
4806     return undef if !file_write("$vfio_basedir/new_id", $data);
4807
4808     my $fn = "$pcisysfs/devices/$name/driver/unbind";
4809     if (!file_write($fn, $name)) {
4810         return undef if -f $fn;
4811     }
4812
4813     $fn = "$vfio_basedir/bind";
4814     if (! -d $testdir) {
4815         return undef if !file_write($fn, $name);
4816     }
4817
4818     return -d $testdir;
4819 }
4820
4821 sub pci_dev_group_bind_to_vfio {
4822     my ($pciid) = @_;
4823
4824     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4825
4826     if (!-d $vfio_basedir) {
4827         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4828     }
4829     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4830
4831     # get IOMMU group devices
4832     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
4833       my @devs = grep /^0000:/, readdir($D);
4834     closedir($D);
4835
4836     foreach my $pciid (@devs) {
4837         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
4838
4839         # pci bridges, switches or root ports are not supported
4840         # they have a pci_bus subdirectory so skip them
4841         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
4842
4843         my $info = pci_device_info($1);
4844         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
4845     }
4846
4847     return 1;
4848 }
4849
4850 # vzdump restore implementaion
4851
4852 sub tar_archive_read_firstfile {
4853     my $archive = shift;
4854
4855     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
4856
4857     # try to detect archive type first
4858     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
4859         die "unable to open file '$archive'\n";
4860     my $firstfile = <$fh>;
4861     kill 15, $pid;
4862     close $fh;
4863
4864     die "ERROR: archive contaions no data\n" if !$firstfile;
4865     chomp $firstfile;
4866
4867     return $firstfile;
4868 }
4869
4870 sub tar_restore_cleanup {
4871     my ($storecfg, $statfile) = @_;
4872
4873     print STDERR "starting cleanup\n";
4874
4875     if (my $fd = IO::File->new($statfile, "r")) {
4876         while (defined(my $line = <$fd>)) {
4877             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
4878                 my $volid = $2;
4879                 eval {
4880                     if ($volid =~ m|^/|) {
4881                         unlink $volid || die 'unlink failed\n';
4882                     } else {
4883                         PVE::Storage::vdisk_free($storecfg, $volid);
4884                     }
4885                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
4886                 };
4887                 print STDERR "unable to cleanup '$volid' - $@" if $@;
4888             } else {
4889                 print STDERR "unable to parse line in statfile - $line";
4890             }
4891         }
4892         $fd->close();
4893     }
4894 }
4895
4896 sub restore_archive {
4897     my ($archive, $vmid, $user, $opts) = @_;
4898
4899     my $format = $opts->{format};
4900     my $comp;
4901
4902     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
4903         $format = 'tar' if !$format;
4904         $comp = 'gzip';
4905     } elsif ($archive =~ m/\.tar$/) {
4906         $format = 'tar' if !$format;
4907     } elsif ($archive =~ m/.tar.lzo$/) {
4908         $format = 'tar' if !$format;
4909         $comp = 'lzop';
4910     } elsif ($archive =~ m/\.vma$/) {
4911         $format = 'vma' if !$format;
4912     } elsif ($archive =~ m/\.vma\.gz$/) {
4913         $format = 'vma' if !$format;
4914         $comp = 'gzip';
4915     } elsif ($archive =~ m/\.vma\.lzo$/) {
4916         $format = 'vma' if !$format;
4917         $comp = 'lzop';
4918     } else {
4919         $format = 'vma' if !$format; # default
4920     }
4921
4922     # try to detect archive format
4923     if ($format eq 'tar') {
4924         return restore_tar_archive($archive, $vmid, $user, $opts);
4925     } else {
4926         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
4927     }
4928 }
4929
4930 sub restore_update_config_line {
4931     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
4932
4933     return if $line =~ m/^\#qmdump\#/;
4934     return if $line =~ m/^\#vzdump\#/;
4935     return if $line =~ m/^lock:/;
4936     return if $line =~ m/^unused\d+:/;
4937     return if $line =~ m/^parent:/;
4938     return if $line =~ m/^template:/; # restored VM is never a template
4939
4940     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
4941         # try to convert old 1.X settings
4942         my ($id, $ind, $ethcfg) = ($1, $2, $3);
4943         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
4944             my ($model, $macaddr) = split(/\=/, $devconfig);
4945             $macaddr = PVE::Tools::random_ether_addr() if !$macaddr || $unique;
4946             my $net = {
4947                 model => $model,
4948                 bridge => "vmbr$ind",
4949                 macaddr => $macaddr,
4950             };
4951             my $netstr = print_net($net);
4952
4953             print $outfd "net$cookie->{netcount}: $netstr\n";
4954             $cookie->{netcount}++;
4955         }
4956     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
4957         my ($id, $netstr) = ($1, $2);
4958         my $net = parse_net($netstr);
4959         $net->{macaddr} = PVE::Tools::random_ether_addr() if $net->{macaddr};
4960         $netstr = print_net($net);
4961         print $outfd "$id: $netstr\n";
4962     } elsif ($line =~ m/^((ide|scsi|virtio|sata)\d+):\s*(\S+)\s*$/) {
4963         my $virtdev = $1;
4964         my $value = $3;
4965         my $di = parse_drive($virtdev, $value);
4966         if (defined($di->{backup}) && !$di->{backup}) {
4967             print $outfd "#$line";
4968         } elsif ($map->{$virtdev}) {
4969             delete $di->{format}; # format can change on restore
4970             $di->{file} = $map->{$virtdev};
4971             $value = print_drive($vmid, $di);
4972             print $outfd "$virtdev: $value\n";
4973         } else {
4974             print $outfd $line;
4975         }
4976     } else {
4977         print $outfd $line;
4978     }
4979 }
4980
4981 sub scan_volids {
4982     my ($cfg, $vmid) = @_;
4983
4984     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
4985
4986     my $volid_hash = {};
4987     foreach my $storeid (keys %$info) {
4988         foreach my $item (@{$info->{$storeid}}) {
4989             next if !($item->{volid} && $item->{size});
4990             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
4991             $volid_hash->{$item->{volid}} = $item;
4992         }
4993     }
4994
4995     return $volid_hash;
4996 }
4997
4998 sub is_volume_in_use {
4999     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5000
5001     my $path = PVE::Storage::path($storecfg, $volid);
5002
5003     my $scan_config = sub {
5004         my ($cref, $snapname) = @_;
5005
5006         foreach my $key (keys %$cref) {
5007             my $value = $cref->{$key};
5008             if (is_valid_drivename($key)) {
5009                 next if $skip_drive && $key eq $skip_drive;
5010                 my $drive = parse_drive($key, $value);
5011                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5012                 return 1 if $volid eq $drive->{file};
5013                 if ($drive->{file} =~ m!^/!) {
5014                     return 1 if $drive->{file} eq $path;
5015                 } else {
5016                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5017                     next if !$storeid;
5018                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5019                     next if !$scfg;
5020                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5021                 }
5022             }
5023         }
5024
5025         return 0;
5026     };
5027
5028     return 1 if &$scan_config($conf);
5029
5030     undef $skip_drive;
5031
5032     foreach my $snapname (keys %{$conf->{snapshots}}) {
5033         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5034     }
5035
5036     return 0;
5037 }
5038
5039 sub update_disksize {
5040     my ($vmid, $conf, $volid_hash) = @_;
5041
5042     my $changes;
5043
5044     my $used = {};
5045
5046     # Note: it is allowed to define multiple storages with same path (alias), so
5047     # we need to check both 'volid' and real 'path' (two different volid can point
5048     # to the same path).
5049
5050     my $usedpath = {};
5051
5052     # update size info
5053     foreach my $opt (keys %$conf) {
5054         if (is_valid_drivename($opt)) {
5055             my $drive = parse_drive($opt, $conf->{$opt});
5056             my $volid = $drive->{file};
5057             next if !$volid;
5058
5059             $used->{$volid} = 1;
5060             if ($volid_hash->{$volid} &&
5061                 (my $path = $volid_hash->{$volid}->{path})) {
5062                 $usedpath->{$path} = 1;
5063             }
5064
5065             next if drive_is_cdrom($drive);
5066             next if !$volid_hash->{$volid};
5067
5068             $drive->{size} = $volid_hash->{$volid}->{size};
5069             my $new = print_drive($vmid, $drive);
5070             if ($new ne $conf->{$opt}) {
5071                 $changes = 1;
5072                 $conf->{$opt} = $new;
5073             }
5074         }
5075     }
5076
5077     # remove 'unusedX' entry if volume is used
5078     foreach my $opt (keys %$conf) {
5079         next if $opt !~ m/^unused\d+$/;
5080         my $volid = $conf->{$opt};
5081         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5082         if ($used->{$volid} || ($path && $usedpath->{$path})) {
5083             $changes = 1;
5084             delete $conf->{$opt};
5085         }
5086     }
5087
5088     foreach my $volid (sort keys %$volid_hash) {
5089         next if $volid =~ m/vm-$vmid-state-/;
5090         next if $used->{$volid};
5091         my $path = $volid_hash->{$volid}->{path};
5092         next if !$path; # just to be sure
5093         next if $usedpath->{$path};
5094         $changes = 1;
5095         PVE::QemuConfig->add_unused_volume($conf, $volid);
5096         $usedpath->{$path} = 1; # avoid to add more than once (aliases)
5097     }
5098
5099     return $changes;
5100 }
5101
5102 sub rescan {
5103     my ($vmid, $nolock) = @_;
5104
5105     my $cfg = PVE::Storage::config();
5106
5107     my $volid_hash = scan_volids($cfg, $vmid);
5108
5109     my $updatefn =  sub {
5110         my ($vmid) = @_;
5111
5112         my $conf = PVE::QemuConfig->load_config($vmid);
5113
5114         PVE::QemuConfig->check_lock($conf);
5115
5116         my $vm_volids = {};
5117         foreach my $volid (keys %$volid_hash) {
5118             my $info = $volid_hash->{$volid};
5119             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5120         }
5121
5122         my $changes = update_disksize($vmid, $conf, $vm_volids);
5123
5124         PVE::QemuConfig->write_config($vmid, $conf) if $changes;
5125     };
5126
5127     if (defined($vmid)) {
5128         if ($nolock) {
5129             &$updatefn($vmid);
5130         } else {
5131             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5132         }
5133     } else {
5134         my $vmlist = config_list();
5135         foreach my $vmid (keys %$vmlist) {
5136             if ($nolock) {
5137                 &$updatefn($vmid);
5138             } else {
5139                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5140             }
5141         }
5142     }
5143 }
5144
5145 sub restore_vma_archive {
5146     my ($archive, $vmid, $user, $opts, $comp) = @_;
5147
5148     my $input = $archive eq '-' ? "<&STDIN" : undef;
5149     my $readfrom = $archive;
5150
5151     my $uncomp = '';
5152     if ($comp) {
5153         $readfrom = '-';
5154         my $qarchive = PVE::Tools::shellquote($archive);
5155         if ($comp eq 'gzip') {
5156             $uncomp = "zcat $qarchive|";
5157         } elsif ($comp eq 'lzop') {
5158             $uncomp = "lzop -d -c $qarchive|";
5159         } else {
5160             die "unknown compression method '$comp'\n";
5161         }
5162
5163     }
5164
5165     my $tmpdir = "/var/tmp/vzdumptmp$$";
5166     rmtree $tmpdir;
5167
5168     # disable interrupts (always do cleanups)
5169     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5170         warn "got interrupt - ignored\n";
5171     };
5172
5173     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5174     POSIX::mkfifo($mapfifo, 0600);
5175     my $fifofh;
5176
5177     my $openfifo = sub {
5178         open($fifofh, '>', $mapfifo) || die $!;
5179     };
5180
5181     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5182
5183     my $oldtimeout;
5184     my $timeout = 5;
5185
5186     my $devinfo = {};
5187
5188     my $rpcenv = PVE::RPCEnvironment::get();
5189
5190     my $conffile = PVE::QemuConfig->config_file($vmid);
5191     my $tmpfn = "$conffile.$$.tmp";
5192
5193     # Note: $oldconf is undef if VM does not exists
5194     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
5195     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
5196
5197     my $print_devmap = sub {
5198         my $virtdev_hash = {};
5199
5200         my $cfgfn = "$tmpdir/qemu-server.conf";
5201
5202         # we can read the config - that is already extracted
5203         my $fh = IO::File->new($cfgfn, "r") ||
5204             "unable to read qemu-server.conf - $!\n";
5205
5206         my $fwcfgfn = "$tmpdir/qemu-server.fw";
5207         if (-f $fwcfgfn) {
5208             my $pve_firewall_dir = '/etc/pve/firewall';
5209             mkdir $pve_firewall_dir; # make sure the dir exists
5210             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
5211         }
5212
5213         while (defined(my $line = <$fh>)) {
5214             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5215                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5216                 die "archive does not contain data for drive '$virtdev'\n"
5217                     if !$devinfo->{$devname};
5218                 if (defined($opts->{storage})) {
5219                     $storeid = $opts->{storage} || 'local';
5220                 } elsif (!$storeid) {
5221                     $storeid = 'local';
5222                 }
5223                 $format = 'raw' if !$format;
5224                 $devinfo->{$devname}->{devname} = $devname;
5225                 $devinfo->{$devname}->{virtdev} = $virtdev;
5226                 $devinfo->{$devname}->{format} = $format;
5227                 $devinfo->{$devname}->{storeid} = $storeid;
5228
5229                 # check permission on storage
5230                 my $pool = $opts->{pool}; # todo: do we need that?
5231                 if ($user ne 'root@pam') {
5232                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5233                 }
5234
5235                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5236             }
5237         }
5238
5239         foreach my $devname (keys %$devinfo) {
5240             die "found no device mapping information for device '$devname'\n"
5241                 if !$devinfo->{$devname}->{virtdev};
5242         }
5243
5244         my $cfg = PVE::Storage::config();
5245
5246         # create empty/temp config
5247         if ($oldconf) {
5248             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5249             foreach_drive($oldconf, sub {
5250                 my ($ds, $drive) = @_;
5251
5252                 return if drive_is_cdrom($drive);
5253
5254                 my $volid = $drive->{file};
5255
5256                 return if !$volid || $volid =~ m|^/|;
5257
5258                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5259                 return if !$path || !$owner || ($owner != $vmid);
5260
5261                 # Note: only delete disk we want to restore
5262                 # other volumes will become unused
5263                 if ($virtdev_hash->{$ds}) {
5264                     PVE::Storage::vdisk_free($cfg, $volid);
5265                 }
5266             });
5267
5268             # delete vmstate files
5269             # since after the restore we have no snapshots anymore
5270             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5271                 my $snap = $oldconf->{snapshots}->{$snapname};
5272                 if ($snap->{vmstate}) {
5273                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
5274                     if (my $err = $@) {
5275                         warn $err;
5276                     }
5277                 }
5278             }
5279         }
5280
5281         my $map = {};
5282         foreach my $virtdev (sort keys %$virtdev_hash) {
5283             my $d = $virtdev_hash->{$virtdev};
5284             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5285             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5286
5287             # test if requested format is supported
5288             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5289             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5290             $d->{format} = $defFormat if !$supported;
5291
5292             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5293                                                   $d->{format}, undef, $alloc_size);
5294             print STDERR "new volume ID is '$volid'\n";
5295             $d->{volid} = $volid;
5296             my $path = PVE::Storage::path($cfg, $volid);
5297
5298             PVE::Storage::activate_volumes($cfg,[$volid]);
5299
5300             my $write_zeros = 1;
5301             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
5302                 $write_zeros = 0;
5303             }
5304
5305             print $fifofh "format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
5306
5307             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5308             $map->{$virtdev} = $volid;
5309         }
5310
5311         $fh->seek(0, 0) || die "seek failed - $!\n";
5312
5313         my $outfd = new IO::File ($tmpfn, "w") ||
5314             die "unable to write config for VM $vmid\n";
5315
5316         my $cookie = { netcount => 0 };
5317         while (defined(my $line = <$fh>)) {
5318             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5319         }
5320
5321         $fh->close();
5322         $outfd->close();
5323     };
5324
5325     eval {
5326         # enable interrupts
5327         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5328             die "interrupted by signal\n";
5329         };
5330         local $SIG{ALRM} = sub { die "got timeout\n"; };
5331
5332         $oldtimeout = alarm($timeout);
5333
5334         my $parser = sub {
5335             my $line = shift;
5336
5337             print "$line\n";
5338
5339             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5340                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5341                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5342             } elsif ($line =~ m/^CTIME: /) {
5343                 # we correctly received the vma config, so we can disable
5344                 # the timeout now for disk allocation (set to 10 minutes, so
5345                 # that we always timeout if something goes wrong)
5346                 alarm(600);
5347                 &$print_devmap();
5348                 print $fifofh "done\n";
5349                 my $tmp = $oldtimeout || 0;
5350                 $oldtimeout = undef;
5351                 alarm($tmp);
5352                 close($fifofh);
5353             }
5354         };
5355
5356         print "restore vma archive: $cmd\n";
5357         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5358     };
5359     my $err = $@;
5360
5361     alarm($oldtimeout) if $oldtimeout;
5362
5363     my $vollist = [];
5364     foreach my $devname (keys %$devinfo) {
5365         my $volid = $devinfo->{$devname}->{volid};
5366         push @$vollist, $volid if $volid;
5367     }
5368
5369     my $cfg = PVE::Storage::config();
5370     PVE::Storage::deactivate_volumes($cfg, $vollist);
5371
5372     unlink $mapfifo;
5373
5374     if ($err) {
5375         rmtree $tmpdir;
5376         unlink $tmpfn;
5377
5378         foreach my $devname (keys %$devinfo) {
5379             my $volid = $devinfo->{$devname}->{volid};
5380             next if !$volid;
5381             eval {
5382                 if ($volid =~ m|^/|) {
5383                     unlink $volid || die 'unlink failed\n';
5384                 } else {
5385                     PVE::Storage::vdisk_free($cfg, $volid);
5386                 }
5387                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5388             };
5389             print STDERR "unable to cleanup '$volid' - $@" if $@;
5390         }
5391         die $err;
5392     }
5393
5394     rmtree $tmpdir;
5395
5396     rename($tmpfn, $conffile) ||
5397         die "unable to commit configuration file '$conffile'\n";
5398
5399     PVE::Cluster::cfs_update(); # make sure we read new file
5400
5401     eval { rescan($vmid, 1); };
5402     warn $@ if $@;
5403 }
5404
5405 sub restore_tar_archive {
5406     my ($archive, $vmid, $user, $opts) = @_;
5407
5408     if ($archive ne '-') {
5409         my $firstfile = tar_archive_read_firstfile($archive);
5410         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5411             if $firstfile ne 'qemu-server.conf';
5412     }
5413
5414     my $storecfg = PVE::Storage::config();
5415
5416     # destroy existing data - keep empty config
5417     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
5418     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5419
5420     my $tocmd = "/usr/lib/qemu-server/qmextract";
5421
5422     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5423     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5424     $tocmd .= ' --prealloc' if $opts->{prealloc};
5425     $tocmd .= ' --info' if $opts->{info};
5426
5427     # tar option "xf" does not autodetect compression when read from STDIN,
5428     # so we pipe to zcat
5429     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5430         PVE::Tools::shellquote("--to-command=$tocmd");
5431
5432     my $tmpdir = "/var/tmp/vzdumptmp$$";
5433     mkpath $tmpdir;
5434
5435     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5436     local $ENV{VZDUMP_VMID} = $vmid;
5437     local $ENV{VZDUMP_USER} = $user;
5438
5439     my $conffile = PVE::QemuConfig->config_file($vmid);
5440     my $tmpfn = "$conffile.$$.tmp";
5441
5442     # disable interrupts (always do cleanups)
5443     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5444         print STDERR "got interrupt - ignored\n";
5445     };
5446
5447     eval {
5448         # enable interrupts
5449         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5450             die "interrupted by signal\n";
5451         };
5452
5453         if ($archive eq '-') {
5454             print "extracting archive from STDIN\n";
5455             run_command($cmd, input => "<&STDIN");
5456         } else {
5457             print "extracting archive '$archive'\n";
5458             run_command($cmd);
5459         }
5460
5461         return if $opts->{info};
5462
5463         # read new mapping
5464         my $map = {};
5465         my $statfile = "$tmpdir/qmrestore.stat";
5466         if (my $fd = IO::File->new($statfile, "r")) {
5467             while (defined (my $line = <$fd>)) {
5468                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5469                     $map->{$1} = $2 if $1;
5470                 } else {
5471                     print STDERR "unable to parse line in statfile - $line\n";
5472                 }
5473             }
5474             $fd->close();
5475         }
5476
5477         my $confsrc = "$tmpdir/qemu-server.conf";
5478
5479         my $srcfd = new IO::File($confsrc, "r") ||
5480             die "unable to open file '$confsrc'\n";
5481
5482         my $outfd = new IO::File ($tmpfn, "w") ||
5483             die "unable to write config for VM $vmid\n";
5484
5485         my $cookie = { netcount => 0 };
5486         while (defined (my $line = <$srcfd>)) {
5487             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5488         }
5489
5490         $srcfd->close();
5491         $outfd->close();
5492     };
5493     my $err = $@;
5494
5495     if ($err) {
5496
5497         unlink $tmpfn;
5498
5499         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
5500
5501         die $err;
5502     }
5503
5504     rmtree $tmpdir;
5505
5506     rename $tmpfn, $conffile ||
5507         die "unable to commit configuration file '$conffile'\n";
5508
5509     PVE::Cluster::cfs_update(); # make sure we read new file
5510
5511     eval { rescan($vmid, 1); };
5512     warn $@ if $@;
5513 };
5514
5515 sub foreach_writable_storage {
5516     my ($conf, $func) = @_;
5517
5518     my $sidhash = {};
5519
5520     foreach my $ds (keys %$conf) {
5521         next if !is_valid_drivename($ds);
5522
5523         my $drive = parse_drive($ds, $conf->{$ds});
5524         next if !$drive;
5525         next if drive_is_cdrom($drive);
5526
5527         my $volid = $drive->{file};
5528
5529         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5530         $sidhash->{$sid} = $sid if $sid;
5531     }
5532
5533     foreach my $sid (sort keys %$sidhash) {
5534         &$func($sid);
5535     }
5536 }
5537
5538 sub do_snapshots_with_qemu {
5539     my ($storecfg, $volid) = @_;
5540
5541     my $storage_name = PVE::Storage::parse_volume_id($volid);
5542
5543     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
5544         && !$storecfg->{ids}->{$storage_name}->{krbd}){
5545         return 1;
5546     }
5547
5548     if ($volid =~ m/\.(qcow2|qed)$/){
5549         return 1;
5550     }
5551
5552     return undef;
5553 }
5554
5555 sub qga_check_running {
5556     my ($vmid) = @_;
5557
5558     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
5559     if ($@) {
5560         warn "Qemu Guest Agent are not running - $@";
5561         return 0;
5562     }
5563     return 1;
5564 }
5565
5566 sub template_create {
5567     my ($vmid, $conf, $disk) = @_;
5568
5569     my $storecfg = PVE::Storage::config();
5570
5571     foreach_drive($conf, sub {
5572         my ($ds, $drive) = @_;
5573
5574         return if drive_is_cdrom($drive);
5575         return if $disk && $ds ne $disk;
5576
5577         my $volid = $drive->{file};
5578         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
5579
5580         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
5581         $drive->{file} = $voliddst;
5582         $conf->{$ds} = print_drive($vmid, $drive);
5583         PVE::QemuConfig->write_config($vmid, $conf);
5584     });
5585 }
5586
5587 sub qemu_img_convert {
5588     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
5589
5590     my $storecfg = PVE::Storage::config();
5591     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
5592     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
5593
5594     if ($src_storeid && $dst_storeid) {
5595
5596         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
5597
5598         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
5599         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5600
5601         my $src_format = qemu_img_format($src_scfg, $src_volname);
5602         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
5603
5604         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
5605         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5606
5607         my $cmd = [];
5608         push @$cmd, '/usr/bin/qemu-img', 'convert', '-t', 'writeback', '-p', '-n';
5609         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
5610         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path;
5611         if ($is_zero_initialized) {
5612             push @$cmd, "zeroinit:$dst_path";
5613         } else {
5614             push @$cmd, $dst_path;
5615         }
5616
5617         my $parser = sub {
5618             my $line = shift;
5619             if($line =~ m/\((\S+)\/100\%\)/){
5620                 my $percent = $1;
5621                 my $transferred = int($size * $percent / 100);
5622                 my $remaining = $size - $transferred;
5623
5624                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
5625             }
5626
5627         };
5628
5629         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
5630         my $err = $@;
5631         die "copy failed: $err" if $err;
5632     }
5633 }
5634
5635 sub qemu_img_format {
5636     my ($scfg, $volname) = @_;
5637
5638     if ($scfg->{path} && $volname =~ m/\.(raw|cow|qcow|qcow2|qed|vmdk|cloop)$/) {
5639         return $1;
5640     } else {
5641         return "raw";
5642     }
5643 }
5644
5645 sub qemu_drive_mirror {
5646     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized) = @_;
5647
5648     my $storecfg = PVE::Storage::config();
5649     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
5650
5651     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5652
5653     my $format = qemu_img_format($dst_scfg, $dst_volname);
5654
5655     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5656
5657     my $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
5658
5659     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
5660     $opts->{format} = $format if $format;
5661
5662     print "drive mirror is starting (scanning bitmap) : this step can take some minutes/hours, depend of disk size and storage speed\n";
5663
5664     my $finish_job = sub {
5665         while (1) {
5666             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
5667             my $stat = @$stats[0];
5668             last if !$stat;
5669             sleep 1;
5670         }
5671     };
5672
5673     eval {
5674     vm_mon_cmd($vmid, "drive-mirror", %$opts);
5675         while (1) {
5676             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
5677             my $stat = @$stats[0];
5678             die "mirroring job seem to have die. Maybe do you have bad sectors?" if !$stat;
5679             die "error job is not mirroring" if $stat->{type} ne "mirror";
5680
5681             my $busy = $stat->{busy};
5682             my $ready = $stat->{ready};
5683
5684             if (my $total = $stat->{len}) {
5685                 my $transferred = $stat->{offset} || 0;
5686                 my $remaining = $total - $transferred;
5687                 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
5688
5689                 print "transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
5690             }
5691
5692
5693             if ($stat->{ready} eq 'true') {
5694
5695                 last if $vmiddst != $vmid;
5696
5697                 # try to switch the disk if source and destination are on the same guest
5698                 eval { vm_mon_cmd($vmid, "block-job-complete", device => "drive-$drive") };
5699                 if (!$@) {
5700                     &$finish_job();
5701                     last;
5702                 }
5703                 die $@ if $@ !~ m/cannot be completed/;
5704             }
5705             sleep 1;
5706         }
5707
5708
5709     };
5710     my $err = $@;
5711
5712     my $cancel_job = sub {
5713         vm_mon_cmd($vmid, "block-job-cancel", device => "drive-$drive");
5714         &$finish_job();
5715     };
5716
5717     if ($err) {
5718         eval { &$cancel_job(); };
5719         die "mirroring error: $err";
5720     }
5721
5722     if ($vmiddst != $vmid) {
5723         # if we clone a disk for a new target vm, we don't switch the disk
5724         &$cancel_job(); # so we call block-job-cancel
5725     }
5726 }
5727
5728 sub clone_disk {
5729     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
5730         $newvmid, $storage, $format, $full, $newvollist) = @_;
5731
5732     my $newvolid;
5733
5734     if (!$full) {
5735         print "create linked clone of drive $drivename ($drive->{file})\n";
5736         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
5737         push @$newvollist, $newvolid;
5738     } else {
5739         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
5740         $storeid = $storage if $storage;
5741
5742         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5743         if (!$format) {
5744             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5745             $format = qemu_img_format($scfg, $volname);
5746         }
5747
5748         # test if requested format is supported - else use default
5749         my $supported = grep { $_ eq $format } @$validFormats;
5750         $format = $defFormat if !$supported;
5751
5752         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
5753
5754         print "create full clone of drive $drivename ($drive->{file})\n";
5755         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $format, undef, ($size/1024));
5756         push @$newvollist, $newvolid;
5757
5758         PVE::Storage::activate_volumes($storecfg, $newvollist);
5759
5760         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
5761         if (!$running || $snapname) {
5762             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
5763         } else {
5764             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit);
5765         }
5766     }
5767
5768     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
5769
5770     my $disk = $drive;
5771     $disk->{format} = undef;
5772     $disk->{file} = $newvolid;
5773     $disk->{size} = $size;
5774
5775     return $disk;
5776 }
5777
5778 # this only works if VM is running
5779 sub get_current_qemu_machine {
5780     my ($vmid) = @_;
5781
5782     my $cmd = { execute => 'query-machines', arguments => {} };
5783     my $res = vm_qmp_command($vmid, $cmd);
5784
5785     my ($current, $default);
5786     foreach my $e (@$res) {
5787         $default = $e->{name} if $e->{'is-default'};
5788         $current = $e->{name} if $e->{'is-current'};
5789     }
5790
5791     # fallback to the default machine if current is not supported by qemu
5792     return $current || $default || 'pc';
5793 }
5794
5795 sub qemu_machine_feature_enabled {
5796     my ($machine, $kvmver, $version_major, $version_minor) = @_;
5797
5798     my $current_major;
5799     my $current_minor;
5800
5801     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
5802
5803         $current_major = $3;
5804         $current_minor = $4;
5805
5806     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
5807
5808         $current_major = $1;
5809         $current_minor = $2;
5810     }
5811
5812     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
5813
5814
5815 }
5816
5817 sub qemu_machine_pxe {
5818     my ($vmid, $conf, $machine) = @_;
5819
5820     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
5821
5822     foreach my $opt (keys %$conf) {
5823         next if $opt !~ m/^net(\d+)$/;
5824         my $net = PVE::QemuServer::parse_net($conf->{$opt});
5825         next if !$net;
5826         my $romfile = PVE::QemuServer::vm_mon_cmd_nocheck($vmid, 'qom-get', path => $opt, property => 'romfile');
5827         return $machine.".pxe" if $romfile =~ m/pxe/;
5828         last;
5829     }
5830
5831     return $machine;
5832 }
5833
5834 sub qemu_use_old_bios_files {
5835     my ($machine_type) = @_;
5836
5837     return if !$machine_type;
5838
5839     my $use_old_bios_files = undef;
5840
5841     if ($machine_type =~ m/^(\S+)\.pxe$/) {
5842         $machine_type = $1;
5843         $use_old_bios_files = 1;
5844     } else {
5845         my $kvmver = kvm_user_version();
5846         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
5847         # load new efi bios files on migration. So this hack is required to allow
5848         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
5849         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
5850         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
5851     }
5852
5853     return ($use_old_bios_files, $machine_type);
5854 }
5855
5856 sub lspci {
5857
5858     my $devices = {};
5859
5860     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
5861             my (undef, $id, $function) = @_;
5862             my $res = { id => $id, function => $function};
5863             push @{$devices->{$id}}, $res;
5864     });
5865
5866     return $devices;
5867 }
5868
5869 sub vm_iothreads_list {
5870     my ($vmid) = @_;
5871
5872     my $res = vm_mon_cmd($vmid, 'query-iothreads');
5873
5874     my $iothreads = {};
5875     foreach my $iothread (@$res) {
5876         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
5877     }
5878
5879     return $iothreads;
5880 }
5881
5882 sub scsihw_infos {
5883     my ($conf, $drive) = @_;
5884
5885     my $maxdev = 0;
5886
5887     if ($conf->{scsihw} && ($conf->{scsihw} =~ m/^lsi/)) {
5888         $maxdev = 7;
5889     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
5890         $maxdev = 1;
5891     } else {
5892         $maxdev = 256;
5893     }
5894
5895     my $controller = int($drive->{index} / $maxdev);
5896     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
5897
5898     return ($maxdev, $controller, $controller_prefix);
5899 }
5900
5901 # bash completion helper
5902
5903 sub complete_backup_archives {
5904     my ($cmdname, $pname, $cvalue) = @_;
5905
5906     my $cfg = PVE::Storage::config();
5907
5908     my $storeid;
5909
5910     if ($cvalue =~ m/^([^:]+):/) {
5911         $storeid = $1;
5912     }
5913
5914     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
5915
5916     my $res = [];
5917     foreach my $id (keys %$data) {
5918         foreach my $item (@{$data->{$id}}) {
5919             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
5920             push @$res, $item->{volid} if defined($item->{volid});
5921         }
5922     }
5923
5924     return $res;
5925 }
5926
5927 my $complete_vmid_full = sub {
5928     my ($running) = @_;
5929
5930     my $idlist = vmstatus();
5931
5932     my $res = [];
5933
5934     foreach my $id (keys %$idlist) {
5935         my $d = $idlist->{$id};
5936         if (defined($running)) {
5937             next if $d->{template};
5938             next if $running && $d->{status} ne 'running';
5939             next if !$running && $d->{status} eq 'running';
5940         }
5941         push @$res, $id;
5942
5943     }
5944     return $res;
5945 };
5946
5947 sub complete_vmid {
5948     return &$complete_vmid_full();
5949 }
5950
5951 sub complete_vmid_stopped {
5952     return &$complete_vmid_full(0);
5953 }
5954
5955 sub complete_vmid_running {
5956     return &$complete_vmid_full(1);
5957 }
5958
5959 sub complete_storage {
5960
5961     my $cfg = PVE::Storage::config();
5962     my $ids = $cfg->{ids};
5963
5964     my $res = [];
5965     foreach my $sid (keys %$ids) {
5966         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
5967         next if !$ids->{$sid}->{content}->{images};
5968         push @$res, $sid;
5969     }
5970
5971     return $res;
5972 }
5973
5974 1;