PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QemuConfig;
  31 use PVE::QMPClient;
  32 use PVE::RPCEnvironment;
  33 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
  34 use PVE::QemuServer::Memory;
  35 use PVE::QemuServer::USB qw(parse_usb_device);
  36 use Time::HiRes qw(gettimeofday);
  37 use File::Copy qw(copy);
  38 use URI::Escape;
  39
  40 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  41
  42 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  43
  44 # Note about locking: we use flock on the config file protect
  45 # against concurent actions.
  46 # Aditionaly, we have a 'lock' setting in the config file. This
  47 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  48 # allowed when such lock is set. But you can ignore this kind of
  49 # lock with the --skiplock flag.
  50
  51 cfs_register_file('/qemu-server/',
  52                   \&parse_vm_config,
  53                   \&write_vm_config);
  54
  55 PVE::JSONSchema::register_standard_option('skiplock', {
  56     description => "Ignore locks - only root is allowed to use this option.",
  57     type => 'boolean',
  58     optional => 1,
  59 });
  60
  61 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  62     description => "Some command save/restore state from this location.",
  63     type => 'string',
  64     maxLength => 128,
  65     optional => 1,
  66 });
  67
  68 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  69     description => "The name of the snapshot.",
  70     type => 'string', format => 'pve-configid',
  71     maxLength => 40,
  72 });
  73
  74 #no warnings 'redefine';
  75
  76 sub cgroups_write {
  77    my ($controller, $vmid, $option, $value) = @_;
  78
  79    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  80    PVE::ProcFSTools::write_proc_entry($path, $value);
  81
  82 }
  83
  84 my $nodename = PVE::INotify::nodename();
  85
  86 mkdir "/etc/pve/nodes/$nodename";
  87 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  88 mkdir $confdir;
  89
  90 my $var_run_tmpdir = "/var/run/qemu-server";
  91 mkdir $var_run_tmpdir;
  92
  93 my $lock_dir = "/var/lock/qemu-server";
  94 mkdir $lock_dir;
  95
  96 my $pcisysfs = "/sys/bus/pci";
  97
  98 my $cpu_vendor_list = {
  99     # Intel CPUs
 100     486 => 'GenuineIntel',
 101     pentium => 'GenuineIntel',
 102     pentium2  => 'GenuineIntel',
 103     pentium3  => 'GenuineIntel',
 104     coreduo => 'GenuineIntel',
 105     core2duo => 'GenuineIntel',
 106     Conroe  => 'GenuineIntel',
 107     Penryn  => 'GenuineIntel',
 108     Nehalem  => 'GenuineIntel',
 109     Westmere => 'GenuineIntel',
 110     SandyBridge => 'GenuineIntel',
 111     IvyBridge => 'GenuineIntel',
 112     Haswell => 'GenuineIntel',
 113     'Haswell-noTSX' => 'GenuineIntel',
 114     Broadwell => 'GenuineIntel',
 115     'Broadwell-noTSX' => 'GenuineIntel',
 116
 117     # AMD CPUs
 118     athlon => 'AuthenticAMD',
 119     phenom  => 'AuthenticAMD',
 120     Opteron_G1  => 'AuthenticAMD',
 121     Opteron_G2  => 'AuthenticAMD',
 122     Opteron_G3  => 'AuthenticAMD',
 123     Opteron_G4  => 'AuthenticAMD',
 124     Opteron_G5  => 'AuthenticAMD',
 125
 126     # generic types, use vendor from host node
 127     host => 'default',
 128     kvm32 => 'default',
 129     kvm64 => 'default',
 130     qemu32 => 'default',
 131     qemu64 => 'default',
 132 };
 133
 134 my $cpu_fmt = {
 135     cputype => {
 136         description => "Emulated CPU type.",
 137         type => 'string',
 138         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 139         format_description => 'cputype',
 140         default => 'kvm64',
 141         default_key => 1,
 142     },
 143     hidden => {
 144         description => "Do not identify as a KVM virtual machine.",
 145         type => 'boolean',
 146         optional => 1,
 147         default => 0
 148     },
 149 };
 150
 151 my $watchdog_fmt = {
 152     model => {
 153         default_key => 1,
 154         type => 'string',
 155         enum => [qw(i6300esb ib700)],
 156         description => "Watchdog type to emulate.",
 157         default => 'i6300esb',
 158         optional => 1,
 159     },
 160     action => {
 161         type => 'string',
 162         enum => [qw(reset shutdown poweroff pause debug none)],
 163         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 164         optional => 1,
 165     },
 166 };
 167 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 168
 169 my $confdesc = {
 170     onboot => {
 171         optional => 1,
 172         type => 'boolean',
 173         description => "Specifies whether a VM will be started during system bootup.",
 174         default => 0,
 175     },
 176     autostart => {
 177         optional => 1,
 178         type => 'boolean',
 179         description => "Automatic restart after crash (currently ignored).",
 180         default => 0,
 181     },
 182     hotplug => {
 183         optional => 1,
 184         type => 'string', format => 'pve-hotplug-features',
 185         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 186         default => 'network,disk,usb',
 187     },
 188     reboot => {
 189         optional => 1,
 190         type => 'boolean',
 191         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 192         default => 1,
 193     },
 194     lock => {
 195         optional => 1,
 196         type => 'string',
 197         description => "Lock/unlock the VM.",
 198         enum => [qw(migrate backup snapshot rollback)],
 199     },
 200     cpulimit => {
 201         optional => 1,
 202         type => 'number',
 203         description => "Limit of CPU usage.",
 204         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 205         minimum => 0,
 206         maximum => 128,
 207         default => 0,
 208     },
 209     cpuunits => {
 210         optional => 1,
 211         type => 'integer',
 212         description => "CPU weight for a VM.",
 213         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 214         minimum => 0,
 215         maximum => 500000,
 216         default => 1000,
 217     },
 218     memory => {
 219         optional => 1,
 220         type => 'integer',
 221         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 222         minimum => 16,
 223         default => 512,
 224     },
 225     balloon => {
 226         optional => 1,
 227         type => 'integer',
 228         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 229         minimum => 0,
 230     },
 231     shares => {
 232         optional => 1,
 233         type => 'integer',
 234         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 235         minimum => 0,
 236         maximum => 50000,
 237         default => 1000,
 238     },
 239     keyboard => {
 240         optional => 1,
 241         type => 'string',
 242         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.conf' configuration file.",
 243         enum => PVE::Tools::kvmkeymaplist(),
 244         default => 'en-us',
 245     },
 246     name => {
 247         optional => 1,
 248         type => 'string', format => 'dns-name',
 249         description => "Set a name for the VM. Only used on the configuration web interface.",
 250     },
 251     scsihw => {
 252         optional => 1,
 253         type => 'string',
 254         description => "SCSI controller model",
 255         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 256         default => 'lsi',
 257     },
 258     description => {
 259         optional => 1,
 260         type => 'string',
 261         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 262     },
 263     ostype => {
 264         optional => 1,
 265         type => 'string',
 266         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 l24 l26 solaris)],
 267         description => "Specify guest operating system.",
 268         verbose_description => <<EODESC,
 269 Specify guest operating system. This is used to enable special
 270 optimization/features for specific operating systems:
 271
 272 [horizontal]
 273 other;; unspecified OS
 274 wxp;; Microsoft Windows XP
 275 w2k;; Microsoft Windows 2000
 276 w2k3;; Microsoft Windows 2003
 277 w2k8;; Microsoft Windows 2008
 278 wvista;; Microsoft Windows Vista
 279 win7;; Microsoft Windows 7
 280 win8;; Microsoft Windows 8/2012
 281 l24;; Linux 2.4 Kernel
 282 l26;; Linux 2.6/3.X Kernel
 283 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 284 EODESC
 285     },
 286     boot => {
 287         optional => 1,
 288         type => 'string',
 289         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 290         pattern => '[acdn]{1,4}',
 291         default => 'cdn',
 292     },
 293     bootdisk => {
 294         optional => 1,
 295         type => 'string', format => 'pve-qm-bootdisk',
 296         description => "Enable booting from specified disk.",
 297         pattern => '(ide|sata|scsi|virtio)\d+',
 298     },
 299     smp => {
 300         optional => 1,
 301         type => 'integer',
 302         description => "The number of CPUs. Please use option -sockets instead.",
 303         minimum => 1,
 304         default => 1,
 305     },
 306     sockets => {
 307         optional => 1,
 308         type => 'integer',
 309         description => "The number of CPU sockets.",
 310         minimum => 1,
 311         default => 1,
 312     },
 313     cores => {
 314         optional => 1,
 315         type => 'integer',
 316         description => "The number of cores per socket.",
 317         minimum => 1,
 318         default => 1,
 319     },
 320     numa => {
 321         optional => 1,
 322         type => 'boolean',
 323         description => "Enable/disable NUMA.",
 324         default => 0,
 325     },
 326     hugepages => {
 327         optional => 1,
 328         type => 'string',
 329         description => "Enable/disable hugepages memory.",
 330         enum => [qw(any 2 1024)],
 331     },
 332     vcpus => {
 333         optional => 1,
 334         type => 'integer',
 335         description => "Number of hotplugged vcpus.",
 336         minimum => 1,
 337         default => 0,
 338     },
 339     acpi => {
 340         optional => 1,
 341         type => 'boolean',
 342         description => "Enable/disable ACPI.",
 343         default => 1,
 344     },
 345     agent => {
 346         optional => 1,
 347         type => 'boolean',
 348         description => "Enable/disable Qemu GuestAgent.",
 349         default => 0,
 350     },
 351     kvm => {
 352         optional => 1,
 353         type => 'boolean',
 354         description => "Enable/disable KVM hardware virtualization.",
 355         default => 1,
 356     },
 357     tdf => {
 358         optional => 1,
 359         type => 'boolean',
 360         description => "Enable/disable time drift fix.",
 361         default => 0,
 362     },
 363     localtime => {
 364         optional => 1,
 365         type => 'boolean',
 366         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 367     },
 368     freeze => {
 369         optional => 1,
 370         type => 'boolean',
 371         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 372     },
 373     vga => {
 374         optional => 1,
 375         type => 'string',
 376         description => "Select the VGA type.",
 377         verbose_description => "Select the VGA type. If you want to use high resolution" .
 378             " modes (>= 1280x1024x16) then you should use the options " .
 379             "'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and " .
 380             "'cirrus' for other OS types. The 'qxl' option enables the SPICE " .
 381             "display sever. For win* OS you can select how many independent " .
 382             "displays you want, Linux guests can add displays them self. " .
 383             "You can also run without any graphic card, using a serial device" .
 384             " as terminal.",
 385         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 386     },
 387     watchdog => {
 388         optional => 1,
 389         type => 'string', format => 'pve-qm-watchdog',
 390         description => "Create a virtual hardware watchdog device.",
 391         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 392             " (by a guest action), the watchdog must be periodically polled " .
 393             "by an agent inside the guest or else the watchdog will reset " .
 394             "the guest (or execute the respective action specified)",
 395     },
 396     startdate => {
 397         optional => 1,
 398         type => 'string',
 399         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 400         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 401         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 402         default => 'now',
 403     },
 404     startup =>  get_standard_option('pve-startup-order'),
 405     template => {
 406         optional => 1,
 407         type => 'boolean',
 408         description => "Enable/disable Template.",
 409         default => 0,
 410     },
 411     args => {
 412         optional => 1,
 413         type => 'string',
 414         description => "Arbitrary arguments passed to kvm.",
 415         verbose_description => <<EODESCR,
 416 Arbitrary arguments passed to kvm, for example:
 417
 418 args: -no-reboot -no-hpet
 419
 420 NOTE: this option is for experts only.
 421 EODESCR
 422     },
 423     tablet => {
 424         optional => 1,
 425         type => 'boolean',
 426         default => 1,
 427         description => "Enable/disable the USB tablet device.",
 428         verbose_description => "Enable/disable the USB tablet device. This device is " .
 429             "usually needed to allow absolute mouse positioning with VNC. " .
 430             "Else the mouse runs out of sync with normal VNC clients. " .
 431             "If you're running lots of console-only guests on one host, " .
 432             "you may consider disabling this to save some context switches. " .
 433             "This is turned off by default if you use spice (-vga=qxl).",
 434     },
 435     migrate_speed => {
 436         optional => 1,
 437         type => 'integer',
 438         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 439         minimum => 0,
 440         default => 0,
 441     },
 442     migrate_downtime => {
 443         optional => 1,
 444         type => 'number',
 445         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 446         minimum => 0,
 447         default => 0.1,
 448     },
 449     cdrom => {
 450         optional => 1,
 451         type => 'string', format => 'pve-qm-drive',
 452         typetext => 'volume',
 453         description => "This is an alias for option -ide2",
 454     },
 455     cpu => {
 456         optional => 1,
 457         description => "Emulated CPU type.",
 458         type => 'string',
 459         format => $cpu_fmt,
 460     },
 461     parent => get_standard_option('pve-snapshot-name', {
 462         optional => 1,
 463         description => "Parent snapshot name. This is used internally, and should not be modified.",
 464     }),
 465     snaptime => {
 466         optional => 1,
 467         description => "Timestamp for snapshots.",
 468         type => 'integer',
 469         minimum => 0,
 470     },
 471     vmstate => {
 472         optional => 1,
 473         type => 'string', format => 'pve-volume-id',
 474         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 475     },
 476     machine => {
 477         description => "Specific the Qemu machine type.",
 478         type => 'string',
 479         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
 480         maxLength => 40,
 481         optional => 1,
 482     },
 483     smbios1 => {
 484         description => "Specify SMBIOS type 1 fields.",
 485         type => 'string', format => 'pve-qm-smbios1',
 486         maxLength => 256,
 487         optional => 1,
 488     },
 489     protection => {
 490         optional => 1,
 491         type => 'boolean',
 492         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 493         default => 0,
 494     },
 495     bios => {
 496         optional => 1,
 497         type => 'string',
 498         enum => [ qw(seabios ovmf) ],
 499         description => "Select BIOS implementation.",
 500         default => 'seabios',
 501     },
 502 };
 503
 504 # what about other qemu settings ?
 505 #cpu => 'string',
 506 #machine => 'string',
 507 #fda => 'file',
 508 #fdb => 'file',
 509 #mtdblock => 'file',
 510 #sd => 'file',
 511 #pflash => 'file',
 512 #snapshot => 'bool',
 513 #bootp => 'file',
 514 ##tftp => 'dir',
 515 ##smb => 'dir',
 516 #kernel => 'file',
 517 #append => 'string',
 518 #initrd => 'file',
 519 ##soundhw => 'string',
 520
 521 while (my ($k, $v) = each %$confdesc) {
 522     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 523 }
 524
 525 my $MAX_IDE_DISKS = 4;
 526 my $MAX_SCSI_DISKS = 14;
 527 my $MAX_VIRTIO_DISKS = 16;
 528 my $MAX_SATA_DISKS = 6;
 529 my $MAX_USB_DEVICES = 5;
 530 my $MAX_NETS = 32;
 531 my $MAX_UNUSED_DISKS = 8;
 532 my $MAX_HOSTPCI_DEVICES = 4;
 533 my $MAX_SERIAL_PORTS = 4;
 534 my $MAX_PARALLEL_PORTS = 3;
 535 my $MAX_NUMA = 8;
 536
 537 my $numa_fmt = {
 538     cpus => {
 539         type => "string",
 540         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 541         description => "CPUs accessing this NUMA node.",
 542         format_description => "id[-id];...",
 543     },
 544     memory => {
 545         type => "number",
 546         description => "Amount of memory this NUMA node provides.",
 547         optional => 1,
 548     },
 549     hostnodes => {
 550         type => "string",
 551         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 552         description => "Host NUMA nodes to use.",
 553         format_description => "id[-id];...",
 554         optional => 1,
 555     },
 556     policy => {
 557         type => 'string',
 558         enum => [qw(preferred bind interleave)],
 559         description => "NUMA allocation policy.",
 560         optional => 1,
 561     },
 562 };
 563 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 564 my $numadesc = {
 565     optional => 1,
 566     type => 'string', format => $numa_fmt,
 567     description => "NUMA topology.",
 568 };
 569 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 570
 571 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 572     $confdesc->{"numa$i"} = $numadesc;
 573 }
 574
 575 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 576                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 577                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 578 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 579
 580 my $net_fmt_bridge_descr = <<__EOD__;
 581 Bridge to attach the network device to. The Proxmox VE standard bridge
 582 is called 'vmbr0'.
 583
 584 If you do not specify a bridge, we create a kvm user (NATed) network
 585 device, which provides DHCP and DNS services. The following addresses
 586 are used:
 587
 588  10.0.2.2   Gateway
 589  10.0.2.3   DNS Server
 590  10.0.2.4   SMB Server
 591
 592 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 593 __EOD__
 594
 595 my $net_fmt = {
 596     macaddr => {
 597         type => 'string',
 598         pattern => qr/[0-9a-f]{2}(?::[0-9a-f]{2}){5}/i,
 599         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 600         format_description => "XX:XX:XX:XX:XX:XX",
 601         optional => 1,
 602     },
 603     model => {
 604         type => 'string',
 605         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 606         format_description => 'model',
 607         enum => $nic_model_list,
 608         default_key => 1,
 609     },
 610     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 611     bridge => {
 612         type => 'string',
 613         description => $net_fmt_bridge_descr,
 614         format_description => 'bridge',
 615         optional => 1,
 616     },
 617     queues => {
 618         type => 'integer',
 619         minimum => 0, maximum => 16,
 620         description => 'Number of packet queues to be used on the device.',
 621         optional => 1,
 622     },
 623     rate => {
 624         type => 'number',
 625         minimum => 0,
 626         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 627         optional => 1,
 628     },
 629     tag => {
 630         type => 'integer',
 631         minimum => 1, maximum => 4094,
 632         description => 'VLAN tag to apply to packets on this interface.',
 633         optional => 1,
 634     },
 635     trunks => {
 636         type => 'string',
 637         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 638         description => 'VLAN trunks to pass through this interface.',
 639         format_description => 'vlanid[;vlanid...]',
 640         optional => 1,
 641     },
 642     firewall => {
 643         type => 'boolean',
 644         description => 'Whether this interface should be protected by the firewall.',
 645         optional => 1,
 646     },
 647     link_down => {
 648         type => 'boolean',
 649         description => 'Whether this interface should be disconnected (like pulling the plug).',
 650         optional => 1,
 651     },
 652 };
 653
 654 my $netdesc = {
 655     optional => 1,
 656     type => 'string', format => $net_fmt,
 657     description => "Specify network devices.",
 658 };
 659
 660 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 661
 662 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 663     $confdesc->{"net$i"} = $netdesc;
 664 }
 665
 666 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 667 sub verify_volume_id_or_qm_path {
 668     my ($volid, $noerr) = @_;
 669
 670     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 671         return $volid;
 672     }
 673
 674     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 675     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 676     if ($@) {
 677         return undef if $noerr;
 678         die $@;
 679     }
 680     return $volid;
 681 }
 682
 683 my $drivename_hash;
 684
 685 my %drivedesc_base = (
 686     volume => { alias => 'file' },
 687     file => {
 688         type => 'string',
 689         format => 'pve-volume-id-or-qm-path',
 690         default_key => 1,
 691         format_description => 'volume',
 692         description => "The drive's backing volume.",
 693     },
 694     media => {
 695         type => 'string',
 696         enum => [qw(cdrom disk)],
 697         description => "The drive's media type.",
 698         default => 'disk',
 699         optional => 1
 700     },
 701     cyls => {
 702         type => 'integer',
 703         description => "Force the drive's physical geometry to have a specific cylinder count.",
 704         optional => 1
 705     },
 706     heads => {
 707         type => 'integer',
 708         description => "Force the drive's physical geometry to have a specific head count.",
 709         optional => 1
 710     },
 711     secs => {
 712         type => 'integer',
 713         description => "Force the drive's physical geometry to have a specific sector count.",
 714         optional => 1
 715     },
 716     trans => {
 717         type => 'string',
 718         enum => [qw(none lba auto)],
 719         description => "Force disk geometry bios translation mode.",
 720         optional => 1,
 721     },
 722     snapshot => {
 723         type => 'boolean',
 724         description => "Whether the drive should be included when making snapshots.",
 725         optional => 1,
 726     },
 727     cache => {
 728         type => 'string',
 729         enum => [qw(none writethrough writeback unsafe directsync)],
 730         description => "The drive's cache mode",
 731         optional => 1,
 732     },
 733     format => {
 734         type => 'string',
 735         format_description => 'image format',
 736         enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
 737         description => "The drive's backing file's data format.",
 738         optional => 1,
 739     },
 740     size => {
 741         type => 'string',
 742         format => 'disk-size',
 743         format_description => 'DiskSize',
 744         description => "Disk size. This is purely informational and has no effect.",
 745         optional => 1,
 746     },
 747     backup => {
 748         type => 'boolean',
 749         description => "Whether the drive should be included when making backups.",
 750         optional => 1,
 751     },
 752     werror => {
 753         type => 'string',
 754         enum => [qw(enospc ignore report stop)],
 755         description => 'Write error action.',
 756         optional => 1,
 757     },
 758     aio => {
 759         type => 'string',
 760         enum => [qw(native threads)],
 761         description => 'AIO type to use.',
 762         optional => 1,
 763     },
 764     discard => {
 765         type => 'string',
 766         enum => [qw(ignore on)],
 767         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 768         optional => 1,
 769     },
 770     detect_zeroes => {
 771         type => 'boolean',
 772         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 773         optional => 1,
 774     },
 775     serial => {
 776         type => 'string',
 777         format => 'urlencoded',
 778         format_description => 'serial',
 779         maxLength => 20*3, # *3 since it's %xx url enoded
 780         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
 781         optional => 1,
 782     }
 783 );
 784
 785 my %rerror_fmt = (
 786     rerror => {
 787         type => 'string',
 788         enum => [qw(ignore report stop)],
 789         description => 'Read error action.',
 790         optional => 1,
 791     },
 792 );
 793
 794 my %iothread_fmt = ( iothread => {
 795         type => 'boolean',
 796         description => "Whether to use iothreads for this drive",
 797         optional => 1,
 798 });
 799
 800 my %model_fmt = (
 801     model => {
 802         type => 'string',
 803         format => 'urlencoded',
 804         format_description => 'model',
 805         maxLength => 40*3, # *3 since it's %xx url enoded
 806         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
 807         optional => 1,
 808     },
 809 );
 810
 811 my %queues_fmt = (
 812     queues => {
 813         type => 'integer',
 814         description => "Number of queues.",
 815         minimum => 2,
 816         optional => 1
 817     }
 818 );
 819
 820 my $add_throttle_desc = sub {
 821     my ($key, $type, $what, $unit, $longunit) = @_;
 822     $drivedesc_base{$key} = {
 823         type => $type,
 824         format_description => $unit,
 825         description => "Maximum $what speed in $longunit per second.",
 826         optional => 1,
 827     };
 828 };
 829 # throughput: (leaky bucket)
 830 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes');
 831 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes');
 832 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes');
 833 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes');
 834 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes');
 835 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes');
 836 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations');
 837 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations');
 838 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations');
 839
 840 # pools: (pool of IO before throttling starts taking effect)
 841 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes');
 842 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes');
 843 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes');
 844 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations');
 845 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations');
 846 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations');
 847
 848 my $ide_fmt = {
 849     %drivedesc_base,
 850     %rerror_fmt,
 851     %model_fmt,
 852 };
 853
 854 my $idedesc = {
 855     optional => 1,
 856     type => 'string', format => $ide_fmt,
 857     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
 858 };
 859 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
 860
 861 my $scsi_fmt = {
 862     %drivedesc_base,
 863     %iothread_fmt,
 864     %queues_fmt,
 865 };
 866 my $scsidesc = {
 867     optional => 1,
 868     type => 'string', format => $scsi_fmt,
 869     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
 870 };
 871 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 872
 873 my $sata_fmt = {
 874     %drivedesc_base,
 875     %rerror_fmt,
 876 };
 877 my $satadesc = {
 878     optional => 1,
 879     type => 'string', format => $sata_fmt,
 880     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
 881 };
 882 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
 883
 884 my $virtio_fmt = {
 885     %drivedesc_base,
 886     %iothread_fmt,
 887     %rerror_fmt,
 888 };
 889 my $virtiodesc = {
 890     optional => 1,
 891     type => 'string', format => $virtio_fmt,
 892     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
 893 };
 894 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
 895
 896 my $alldrive_fmt = {
 897     %drivedesc_base,
 898     %rerror_fmt,
 899     %iothread_fmt,
 900     %model_fmt,
 901     %queues_fmt,
 902 };
 903
 904 my $usb_fmt = {
 905     host => {
 906         default_key => 1,
 907         type => 'string', format => 'pve-qm-usb-device',
 908         format_description => 'HOSTUSBDEVICE|spice',
 909         description => <<EODESCR,
 910 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
 911
 912  'bus-port(.port)*' (decimal numbers) or
 913  'vendor_id:product_id' (hexadeciaml numbers) or
 914  'spice'
 915
 916 You can use the 'lsusb -t' command to list existing usb devices.
 917
 918 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 919
 920 The value 'spice' can be used to add a usb redirection devices for spice.
 921 EODESCR
 922     },
 923     usb3 => {
 924         optional => 1,
 925         type => 'boolean',
 926         description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
 927         default => 0,
 928     },
 929 };
 930
 931 my $usbdesc = {
 932     optional => 1,
 933     type => 'string', format => $usb_fmt,
 934     description => "Configure an USB device (n is 0 to 4).",
 935 };
 936 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
 937
 938 # NOTE: the match-groups of this regex are used in parse_hostpci
 939 my $PCIRE = qr/([a-f0-9]{2}:[a-f0-9]{2})(?:\.([a-f0-9]))?/;
 940 my $hostpci_fmt = {
 941     host => {
 942         default_key => 1,
 943         type => 'string',
 944         pattern => qr/$PCIRE(;$PCIRE)*/,
 945         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
 946         description => <<EODESCR,
 947 Host PCI device pass through. The PCI ID of a host's PCI device or a list
 948 of PCI virtual functions of the host. HOSTPCIID syntax is:
 949
 950 'bus:dev.func' (hexadecimal numbers)
 951
 952 You can us the 'lspci' command to list existing PCI devices.
 953 EODESCR
 954     },
 955     rombar => {
 956         type => 'boolean',
 957         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
 958         optional => 1,
 959         default => 1,
 960     },
 961     pcie => {
 962         type => 'boolean',
 963         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
 964         optional => 1,
 965         default => 0,
 966     },
 967     'x-vga' => {
 968         type => 'boolean',
 969         description =>  "Enable vfio-vga device support.",
 970         optional => 1,
 971         default => 0,
 972     },
 973 };
 974 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
 975
 976 my $hostpcidesc = {
 977         optional => 1,
 978         type => 'string', format => 'pve-qm-hostpci',
 979         description => "Map host PCI devices into guest.",
 980         verbose_description =>  <<EODESCR,
 981 Map host PCI devices into guest.
 982
 983 NOTE: This option allows direct access to host hardware. So it is no longer
 984 possible to migrate such machines - use with special care.
 985
 986 CAUTION: Experimental! User reported problems with this option.
 987 EODESCR
 988 };
 989 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
 990
 991 my $serialdesc = {
 992         optional => 1,
 993         type => 'string',
 994         pattern => '(/dev/.+|socket)',
 995         description =>  "Create a serial device inside the VM (n is 0 to 3)",
 996         verbose_description =>  <<EODESCR,
 997 Create a serial device inside the VM (n is 0 to 3), and pass through a
 998 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
 999 host side (use 'qm terminal' to open a terminal connection).
1000
1001 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1002
1003 CAUTION: Experimental! User reported problems with this option.
1004 EODESCR
1005 };
1006
1007 my $paralleldesc= {
1008         optional => 1,
1009         type => 'string',
1010         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1011         description =>  "Map host parallel devices (n is 0 to 2).",
1012         verbose_description =>  <<EODESCR,
1013 Map host parallel devices (n is 0 to 2).
1014
1015 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1016
1017 CAUTION: Experimental! User reported problems with this option.
1018 EODESCR
1019 };
1020
1021 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1022     $confdesc->{"parallel$i"} = $paralleldesc;
1023 }
1024
1025 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1026     $confdesc->{"serial$i"} = $serialdesc;
1027 }
1028
1029 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1030     $confdesc->{"hostpci$i"} = $hostpcidesc;
1031 }
1032
1033 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1034     $drivename_hash->{"ide$i"} = 1;
1035     $confdesc->{"ide$i"} = $idedesc;
1036 }
1037
1038 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1039     $drivename_hash->{"sata$i"} = 1;
1040     $confdesc->{"sata$i"} = $satadesc;
1041 }
1042
1043 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1044     $drivename_hash->{"scsi$i"} = 1;
1045     $confdesc->{"scsi$i"} = $scsidesc ;
1046 }
1047
1048 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1049     $drivename_hash->{"virtio$i"} = 1;
1050     $confdesc->{"virtio$i"} = $virtiodesc;
1051 }
1052
1053 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1054     $confdesc->{"usb$i"} = $usbdesc;
1055 }
1056
1057 my $unuseddesc = {
1058     optional => 1,
1059     type => 'string', format => 'pve-volume-id',
1060     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1061 };
1062
1063 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1064     $confdesc->{"unused$i"} = $unuseddesc;
1065 }
1066
1067 my $kvm_api_version = 0;
1068
1069 sub kvm_version {
1070
1071     return $kvm_api_version if $kvm_api_version;
1072
1073     my $fh = IO::File->new("</dev/kvm") ||
1074         return 0;
1075
1076     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
1077         $kvm_api_version = $v;
1078     }
1079
1080     $fh->close();
1081
1082     return  $kvm_api_version;
1083 }
1084
1085 my $kvm_user_version;
1086
1087 sub kvm_user_version {
1088
1089     return $kvm_user_version if $kvm_user_version;
1090
1091     $kvm_user_version = 'unknown';
1092
1093     my $code = sub {
1094         my $line = shift;
1095         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1096             $kvm_user_version = $2;
1097         }
1098     };
1099
1100     eval { run_command("kvm -version", outfunc => $code); };
1101     warn $@ if $@;
1102
1103     return $kvm_user_version;
1104
1105 }
1106
1107 my $kernel_has_vhost_net = -c '/dev/vhost-net';
1108
1109 sub valid_drive_names {
1110     # order is important - used to autoselect boot disk
1111     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1112             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1113             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1114             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))));
1115 }
1116
1117 sub is_valid_drivename {
1118     my $dev = shift;
1119
1120     return defined($drivename_hash->{$dev});
1121 }
1122
1123 sub option_exists {
1124     my $key = shift;
1125     return defined($confdesc->{$key});
1126 }
1127
1128 sub nic_models {
1129     return $nic_model_list;
1130 }
1131
1132 sub os_list_description {
1133
1134     return {
1135         other => 'Other',
1136         wxp => 'Windows XP',
1137         w2k => 'Windows 2000',
1138         w2k3 =>, 'Windows 2003',
1139         w2k8 => 'Windows 2008',
1140         wvista => 'Windows Vista',
1141         win7 => 'Windows 7',
1142         win8 => 'Windows 8/2012',
1143         l24 => 'Linux 2.4',
1144         l26 => 'Linux 2.6',
1145     };
1146 }
1147
1148 my $cdrom_path;
1149
1150 sub get_cdrom_path {
1151
1152     return  $cdrom_path if $cdrom_path;
1153
1154     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1155     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1156     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1157 }
1158
1159 sub get_iso_path {
1160     my ($storecfg, $vmid, $cdrom) = @_;
1161
1162     if ($cdrom eq 'cdrom') {
1163         return get_cdrom_path();
1164     } elsif ($cdrom eq 'none') {
1165         return '';
1166     } elsif ($cdrom =~ m|^/|) {
1167         return $cdrom;
1168     } else {
1169         return PVE::Storage::path($storecfg, $cdrom);
1170     }
1171 }
1172
1173 # try to convert old style file names to volume IDs
1174 sub filename_to_volume_id {
1175     my ($vmid, $file, $media) = @_;
1176
1177     if (!($file eq 'none' || $file eq 'cdrom' ||
1178           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1179
1180         return undef if $file =~ m|/|;
1181
1182         if ($media && $media eq 'cdrom') {
1183             $file = "local:iso/$file";
1184         } else {
1185             $file = "local:$vmid/$file";
1186         }
1187     }
1188
1189     return $file;
1190 }
1191
1192 sub verify_media_type {
1193     my ($opt, $vtype, $media) = @_;
1194
1195     return if !$media;
1196
1197     my $etype;
1198     if ($media eq 'disk') {
1199         $etype = 'images';
1200     } elsif ($media eq 'cdrom') {
1201         $etype = 'iso';
1202     } else {
1203         die "internal error";
1204     }
1205
1206     return if ($vtype eq $etype);
1207
1208     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1209 }
1210
1211 sub cleanup_drive_path {
1212     my ($opt, $storecfg, $drive) = @_;
1213
1214     # try to convert filesystem paths to volume IDs
1215
1216     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1217         ($drive->{file} !~ m|^/dev/.+|) &&
1218         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1219         ($drive->{file} !~ m/^\d+$/)) {
1220         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1221         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1222         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1223         verify_media_type($opt, $vtype, $drive->{media});
1224         $drive->{file} = $volid;
1225     }
1226
1227     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1228 }
1229
1230 sub parse_hotplug_features {
1231     my ($data) = @_;
1232
1233     my $res = {};
1234
1235     return $res if $data eq '0';
1236
1237     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1238
1239     foreach my $feature (PVE::Tools::split_list($data)) {
1240         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1241             $res->{$1} = 1;
1242         } else {
1243             die "invalid hotplug feature '$feature'\n";
1244         }
1245     }
1246     return $res;
1247 }
1248
1249 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1250 sub pve_verify_hotplug_features {
1251     my ($value, $noerr) = @_;
1252
1253     return $value if parse_hotplug_features($value);
1254
1255     return undef if $noerr;
1256
1257     die "unable to parse hotplug option\n";
1258 }
1259
1260 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1261 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1262 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1263 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1264 #        [,iothread=on][,serial=serial][,model=model]
1265
1266 sub parse_drive {
1267     my ($key, $data) = @_;
1268
1269     my ($interface, $index);
1270
1271     if ($key =~ m/^([^\d]+)(\d+)$/) {
1272         $interface = $1;
1273         $index = $2;
1274     } else {
1275         return undef;
1276     }
1277
1278     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1279                                      : $confdesc->{$key}->{format};
1280     if (!$desc) {
1281         warn "invalid drive key: $key\n";
1282         return undef;
1283     }
1284     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1285     return undef if !$res;
1286     $res->{interface} = $interface;
1287     $res->{index} = $index;
1288
1289     my $error = 0;
1290     foreach my $opt (qw(bps bps_rd bps_wr)) {
1291         if (my $bps = defined(delete $res->{$opt})) {
1292             if (defined($res->{"m$opt"})) {
1293                 warn "both $opt and m$opt specified\n";
1294                 ++$error;
1295                 next;
1296             }
1297             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1298         }
1299     }
1300     return undef if $error;
1301
1302     return undef if $res->{mbps_rd} && $res->{mbps};
1303     return undef if $res->{mbps_wr} && $res->{mbps};
1304     return undef if $res->{iops_rd} && $res->{iops};
1305     return undef if $res->{iops_wr} && $res->{iops};
1306
1307     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1308         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1309         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1310         return undef if $res->{interface} eq 'virtio';
1311     }
1312
1313     if (my $size = $res->{size}) {
1314         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1315     }
1316
1317     return $res;
1318 }
1319
1320 sub print_drive {
1321     my ($vmid, $drive) = @_;
1322     my $data = { %$drive };
1323     delete $data->{$_} for qw(index interface);
1324     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1325 }
1326
1327 sub scsi_inquiry {
1328     my($fh, $noerr) = @_;
1329
1330     my $SG_IO = 0x2285;
1331     my $SG_GET_VERSION_NUM = 0x2282;
1332
1333     my $versionbuf = "\x00" x 8;
1334     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1335     if (!$ret) {
1336         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1337         return undef;
1338     }
1339     my $version = unpack("I", $versionbuf);
1340     if ($version < 30000) {
1341         die "scsi generic interface too old\n"  if !$noerr;
1342         return undef;
1343     }
1344
1345     my $buf = "\x00" x 36;
1346     my $sensebuf = "\x00" x 8;
1347     my $cmd = pack("C x3 C x1", 0x12, 36);
1348
1349     # see /usr/include/scsi/sg.h
1350     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1351
1352     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1353                       length($sensebuf), 0, length($buf), $buf,
1354                       $cmd, $sensebuf, 6000);
1355
1356     $ret = ioctl($fh, $SG_IO, $packet);
1357     if (!$ret) {
1358         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1359         return undef;
1360     }
1361
1362     my @res = unpack($sg_io_hdr_t, $packet);
1363     if ($res[17] || $res[18]) {
1364         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1365         return undef;
1366     }
1367
1368     my $res = {};
1369     (my $byte0, my $byte1, $res->{vendor},
1370      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1371
1372     $res->{removable} = $byte1 & 128 ? 1 : 0;
1373     $res->{type} = $byte0 & 31;
1374
1375     return $res;
1376 }
1377
1378 sub path_is_scsi {
1379     my ($path) = @_;
1380
1381     my $fh = IO::File->new("+<$path") || return undef;
1382     my $res = scsi_inquiry($fh, 1);
1383     close($fh);
1384
1385     return $res;
1386 }
1387
1388 sub machine_type_is_q35 {
1389     my ($conf) = @_;
1390
1391     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1392 }
1393
1394 sub print_tabletdevice_full {
1395     my ($conf) = @_;
1396
1397     my $q35 = machine_type_is_q35($conf);
1398
1399     # we use uhci for old VMs because tablet driver was buggy in older qemu
1400     my $usbbus = $q35 ? "ehci" : "uhci";
1401
1402     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1403 }
1404
1405 sub print_drivedevice_full {
1406     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1407
1408     my $device = '';
1409     my $maxdev = 0;
1410
1411     if ($drive->{interface} eq 'virtio') {
1412         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1413         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1414         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1415     } elsif ($drive->{interface} eq 'scsi') {
1416
1417         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1418         my $unit = $drive->{index} % $maxdev;
1419         my $devicetype = 'hd';
1420         my $path = '';
1421         if (drive_is_cdrom($drive)) {
1422             $devicetype = 'cd';
1423         } else {
1424             if ($drive->{file} =~ m|^/|) {
1425                 $path = $drive->{file};
1426                 if (my $info = path_is_scsi($path)) {
1427                     if ($info->{type} == 0) {
1428                         $devicetype = 'block';
1429                     } elsif ($info->{type} == 1) { # tape
1430                         $devicetype = 'generic';
1431                     }
1432                 }
1433             } else {
1434                  $path = PVE::Storage::path($storecfg, $drive->{file});
1435             }
1436
1437             if($path =~ m/^iscsi\:\/\//){
1438                 $devicetype = 'generic';
1439             }
1440         }
1441
1442         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1443             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1444         } else {
1445             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1446         }
1447
1448     } elsif ($drive->{interface} eq 'ide'){
1449         $maxdev = 2;
1450         my $controller = int($drive->{index} / $maxdev);
1451         my $unit = $drive->{index} % $maxdev;
1452         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1453
1454         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1455         if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
1456             $model = URI::Escape::uri_unescape($model);
1457             $device .= ",model=$model";
1458         }
1459     } elsif ($drive->{interface} eq 'sata'){
1460         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1461         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1462         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1463     } elsif ($drive->{interface} eq 'usb') {
1464         die "implement me";
1465         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1466     } else {
1467         die "unsupported interface type";
1468     }
1469
1470     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1471
1472     return $device;
1473 }
1474
1475 sub get_initiator_name {
1476     my $initiator;
1477
1478     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1479     while (defined(my $line = <$fh>)) {
1480         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1481         $initiator = $1;
1482         last;
1483     }
1484     $fh->close();
1485
1486     return $initiator;
1487 }
1488
1489 sub print_drive_full {
1490     my ($storecfg, $vmid, $drive) = @_;
1491
1492     my $path;
1493     my $volid = $drive->{file};
1494     my $format;
1495
1496     if (drive_is_cdrom($drive)) {
1497         $path = get_iso_path($storecfg, $vmid, $volid);
1498     } else {
1499         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1500         if ($storeid) {
1501             $path = PVE::Storage::path($storecfg, $volid);
1502             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1503             $format = qemu_img_format($scfg, $volname);
1504         } else {
1505             $path = $volid;
1506             $format = "raw";
1507         }
1508    }
1509
1510     my $opts = '';
1511     my @qemu_drive_options = qw(heads secs cyls trans media format cache snapshot rerror werror aio discard iops iops_rd iops_wr iops_max iops_rd_max iops_wr_max);
1512     foreach my $o (@qemu_drive_options) {
1513         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1514     }
1515     if (my $serial = $drive->{serial}) {
1516         $serial = URI::Escape::uri_unescape($serial);
1517         $opts .= ",serial=$serial";
1518     }
1519
1520     $opts .= ",format=$format" if $format && !$drive->{format};
1521
1522     foreach my $o (qw(bps bps_rd bps_wr)) {
1523         my $v = $drive->{"m$o"};
1524         $opts .= ",$o=" . int($v*1024*1024) if $v;
1525     }
1526
1527     my $cache_direct = 0;
1528
1529     if (my $cache = $drive->{cache}) {
1530         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1531     } elsif (!drive_is_cdrom($drive)) {
1532         $opts .= ",cache=none";
1533         $cache_direct = 1;
1534     }
1535
1536     # aio native works only with O_DIRECT
1537     if (!$drive->{aio}) {
1538         if($cache_direct) {
1539             $opts .= ",aio=native";
1540         } else {
1541             $opts .= ",aio=threads";
1542         }
1543     }
1544
1545     if (!drive_is_cdrom($drive)) {
1546         my $detectzeroes;
1547         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1548             $detectzeroes = 'off';
1549         } elsif ($drive->{discard}) {
1550             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1551         } else {
1552             # This used to be our default with discard not being specified:
1553             $detectzeroes = 'on';
1554         }
1555         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1556     }
1557
1558     my $pathinfo = $path ? "file=$path," : '';
1559
1560     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1561 }
1562
1563 sub print_netdevice_full {
1564     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1565
1566     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1567
1568     my $device = $net->{model};
1569     if ($net->{model} eq 'virtio') {
1570          $device = 'virtio-net-pci';
1571      };
1572
1573     my $pciaddr = print_pci_addr("$netid", $bridges);
1574     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1575     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1576         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1577         my $vectors = $net->{queues} * 2 + 2;
1578         $tmpstr .= ",vectors=$vectors,mq=on";
1579     }
1580     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1581
1582     if ($use_old_bios_files) {
1583         my $romfile;
1584         if ($device eq 'virtio-net-pci') {
1585             $romfile = 'pxe-virtio.rom';
1586         } elsif ($device eq 'e1000') {
1587             $romfile = 'pxe-e1000.rom';
1588         } elsif ($device eq 'ne2k') {
1589             $romfile = 'pxe-ne2k_pci.rom';
1590         } elsif ($device eq 'pcnet') {
1591             $romfile = 'pxe-pcnet.rom';
1592         } elsif ($device eq 'rtl8139') {
1593             $romfile = 'pxe-rtl8139.rom';
1594         }
1595         $tmpstr .= ",romfile=$romfile" if $romfile;
1596     }
1597
1598     return $tmpstr;
1599 }
1600
1601 sub print_netdev_full {
1602     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1603
1604     my $i = '';
1605     if ($netid =~ m/^net(\d+)$/) {
1606         $i = int($1);
1607     }
1608
1609     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1610
1611     my $ifname = "tap${vmid}i$i";
1612
1613     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1614     die "interface name '$ifname' is too long (max 15 character)\n"
1615         if length($ifname) >= 16;
1616
1617     my $vhostparam = '';
1618     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1619
1620     my $vmname = $conf->{name} || "vm$vmid";
1621
1622     my $netdev = "";
1623     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1624
1625     if ($net->{bridge}) {
1626         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1627     } else {
1628         $netdev = "type=user,id=$netid,hostname=$vmname";
1629     }
1630
1631     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1632
1633     return $netdev;
1634 }
1635
1636 sub drive_is_cdrom {
1637     my ($drive) = @_;
1638
1639     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1640
1641 }
1642
1643 sub parse_number_sets {
1644     my ($set) = @_;
1645     my $res = [];
1646     foreach my $part (split(/;/, $set)) {
1647         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1648             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1649             push @$res, [ $1, $2 ];
1650         } else {
1651             die "invalid range: $part\n";
1652         }
1653     }
1654     return $res;
1655 }
1656
1657 sub parse_numa {
1658     my ($data) = @_;
1659
1660     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
1661     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1662     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1663     return $res;
1664 }
1665
1666 sub parse_hostpci {
1667     my ($value) = @_;
1668
1669     return undef if !$value;
1670
1671     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
1672
1673     my @idlist = split(/;/, $res->{host});
1674     delete $res->{host};
1675     foreach my $id (@idlist) {
1676         if ($id =~ /^$PCIRE$/) {
1677             push @{$res->{pciid}}, { id => $1, function => ($2//'0') };
1678         } else {
1679             # should have been caught by parse_property_string already
1680             die "failed to parse PCI id: $id\n";
1681         }
1682     }
1683     return $res;
1684 }
1685
1686 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1687 sub parse_net {
1688     my ($data) = @_;
1689
1690     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
1691     if ($@) {
1692         warn $@;
1693         return undef;
1694     }
1695     if (!defined($res->{macaddr})) {
1696         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1697         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1698     }
1699     return $res;
1700 }
1701
1702 sub print_net {
1703     my $net = shift;
1704
1705     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1706 }
1707
1708 sub add_random_macs {
1709     my ($settings) = @_;
1710
1711     foreach my $opt (keys %$settings) {
1712         next if $opt !~ m/^net(\d+)$/;
1713         my $net = parse_net($settings->{$opt});
1714         next if !$net;
1715         $settings->{$opt} = print_net($net);
1716     }
1717 }
1718
1719 sub vm_is_volid_owner {
1720     my ($storecfg, $vmid, $volid) = @_;
1721
1722     if ($volid !~  m|^/|) {
1723         my ($path, $owner);
1724         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1725         if ($owner && ($owner == $vmid)) {
1726             return 1;
1727         }
1728     }
1729
1730     return undef;
1731 }
1732
1733 sub split_flagged_list {
1734     my $text = shift || '';
1735     $text =~ s/[,;]/ /g;
1736     $text =~ s/^\s+//;
1737     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
1738 }
1739
1740 sub join_flagged_list {
1741     my ($how, $lst) = @_;
1742     join $how, map { $lst->{$_} . $_ } keys %$lst;
1743 }
1744
1745 sub vmconfig_delete_pending_option {
1746     my ($conf, $key, $force) = @_;
1747
1748     delete $conf->{pending}->{$key};
1749     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1750     $pending_delete_hash->{$key} = $force ? '!' : '';
1751     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1752 }
1753
1754 sub vmconfig_undelete_pending_option {
1755     my ($conf, $key) = @_;
1756
1757     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1758     delete $pending_delete_hash->{$key};
1759
1760     if (%$pending_delete_hash) {
1761         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1762     } else {
1763         delete $conf->{pending}->{delete};
1764     }
1765 }
1766
1767 sub vmconfig_register_unused_drive {
1768     my ($storecfg, $vmid, $conf, $drive) = @_;
1769
1770     if (!drive_is_cdrom($drive)) {
1771         my $volid = $drive->{file};
1772         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1773             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1774         }
1775     }
1776 }
1777
1778 sub vmconfig_cleanup_pending {
1779     my ($conf) = @_;
1780
1781     # remove pending changes when nothing changed
1782     my $changes;
1783     foreach my $opt (keys %{$conf->{pending}}) {
1784         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
1785             $changes = 1;
1786             delete $conf->{pending}->{$opt};
1787         }
1788     }
1789
1790     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
1791     my $pending_delete_hash = {};
1792     while (my ($opt, $force) = each %$current_delete_hash) {
1793         if (defined($conf->{$opt})) {
1794             $pending_delete_hash->{$opt} = $force;
1795         } else {
1796             $changes = 1;
1797         }
1798     }
1799
1800     if (%$pending_delete_hash) {
1801         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1802     } else {
1803         delete $conf->{pending}->{delete};
1804     }
1805
1806     return $changes;
1807 }
1808
1809 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
1810 my $smbios1_fmt = {
1811     uuid => {
1812         type => 'string',
1813         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1814         format_description => 'UUID',
1815         description => "Set SMBIOS1 UUID.",
1816         optional => 1,
1817     },
1818     version => {
1819         type => 'string',
1820         pattern => '\S+',
1821         format_description => 'string',
1822         description => "Set SMBIOS1 version.",
1823         optional => 1,
1824     },
1825     serial => {
1826         type => 'string',
1827         pattern => '\S+',
1828         format_description => 'string',
1829         description => "Set SMBIOS1 serial number.",
1830         optional => 1,
1831     },
1832     manufacturer => {
1833         type => 'string',
1834         pattern => '\S+',
1835         format_description => 'string',
1836         description => "Set SMBIOS1 manufacturer.",
1837         optional => 1,
1838     },
1839     product => {
1840         type => 'string',
1841         pattern => '\S+',
1842         format_description => 'string',
1843         description => "Set SMBIOS1 product ID.",
1844         optional => 1,
1845     },
1846     sku => {
1847         type => 'string',
1848         pattern => '\S+',
1849         format_description => 'string',
1850         description => "Set SMBIOS1 SKU string.",
1851         optional => 1,
1852     },
1853     family => {
1854         type => 'string',
1855         pattern => '\S+',
1856         format_description => 'string',
1857         description => "Set SMBIOS1 family string.",
1858         optional => 1,
1859     },
1860 };
1861
1862 sub parse_smbios1 {
1863     my ($data) = @_;
1864
1865     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
1866     warn $@ if $@;
1867     return $res;
1868 }
1869
1870 sub print_smbios1 {
1871     my ($smbios1) = @_;
1872     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
1873 }
1874
1875 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
1876
1877 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
1878 sub verify_bootdisk {
1879     my ($value, $noerr) = @_;
1880
1881     return $value if is_valid_drivename($value);
1882
1883     return undef if $noerr;
1884
1885     die "invalid boot disk '$value'\n";
1886 }
1887
1888 sub parse_watchdog {
1889     my ($value) = @_;
1890
1891     return undef if !$value;
1892
1893     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
1894     warn $@ if $@;
1895     return $res;
1896 }
1897
1898 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
1899 sub verify_usb_device {
1900     my ($value, $noerr) = @_;
1901
1902     return $value if parse_usb_device($value);
1903
1904     return undef if $noerr;
1905
1906     die "unable to parse usb device\n";
1907 }
1908
1909 # add JSON properties for create and set function
1910 sub json_config_properties {
1911     my $prop = shift;
1912
1913     foreach my $opt (keys %$confdesc) {
1914         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
1915         $prop->{$opt} = $confdesc->{$opt};
1916     }
1917
1918     return $prop;
1919 }
1920
1921 sub check_type {
1922     my ($key, $value) = @_;
1923
1924     die "unknown setting '$key'\n" if !$confdesc->{$key};
1925
1926     my $type = $confdesc->{$key}->{type};
1927
1928     if (!defined($value)) {
1929         die "got undefined value\n";
1930     }
1931
1932     if ($value =~ m/[\n\r]/) {
1933         die "property contains a line feed\n";
1934     }
1935
1936     if ($type eq 'boolean') {
1937         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
1938         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
1939         die "type check ('boolean') failed - got '$value'\n";
1940     } elsif ($type eq 'integer') {
1941         return int($1) if $value =~ m/^(\d+)$/;
1942         die "type check ('integer') failed - got '$value'\n";
1943     } elsif ($type eq 'number') {
1944         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
1945         die "type check ('number') failed - got '$value'\n";
1946     } elsif ($type eq 'string') {
1947         if (my $fmt = $confdesc->{$key}->{format}) {
1948             if ($fmt eq 'pve-qm-drive') {
1949                 # special case - we need to pass $key to parse_drive()
1950                 my $drive = parse_drive($key, $value);
1951                 return $value if $drive;
1952                 die "unable to parse drive options\n";
1953             }
1954             PVE::JSONSchema::check_format($fmt, $value);
1955             return $value;
1956         }
1957         $value =~ s/^\"(.*)\"$/$1/;
1958         return $value;
1959     } else {
1960         die "internal error"
1961     }
1962 }
1963
1964 sub check_iommu_support{
1965     #fixme : need to check IOMMU support
1966     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
1967
1968     my $iommu=1;
1969     return $iommu;
1970
1971 }
1972
1973 sub touch_config {
1974     my ($vmid) = @_;
1975
1976     my $conf = PVE::QemuConfig->config_file($vmid);
1977     utime undef, undef, $conf;
1978 }
1979
1980 sub destroy_vm {
1981     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
1982
1983     my $conffile = PVE::QemuConfig->config_file($vmid);
1984
1985     my $conf = PVE::QemuConfig->load_config($vmid);
1986
1987     PVE::QemuConfig->check_lock($conf) if !$skiplock;
1988
1989     # only remove disks owned by this VM
1990     foreach_drive($conf, sub {
1991         my ($ds, $drive) = @_;
1992
1993         return if drive_is_cdrom($drive);
1994
1995         my $volid = $drive->{file};
1996
1997         return if !$volid || $volid =~ m|^/|;
1998
1999         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2000         return if !$path || !$owner || ($owner != $vmid);
2001
2002         PVE::Storage::vdisk_free($storecfg, $volid);
2003     });
2004
2005     if ($keep_empty_config) {
2006         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2007     } else {
2008         unlink $conffile;
2009     }
2010
2011     # also remove unused disk
2012     eval {
2013         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2014
2015         eval {
2016             PVE::Storage::foreach_volid($dl, sub {
2017                 my ($volid, $sid, $volname, $d) = @_;
2018                 PVE::Storage::vdisk_free($storecfg, $volid);
2019             });
2020         };
2021         warn $@ if $@;
2022
2023     };
2024     warn $@ if $@;
2025 }
2026
2027 sub parse_vm_config {
2028     my ($filename, $raw) = @_;
2029
2030     return undef if !defined($raw);
2031
2032     my $res = {
2033         digest => Digest::SHA::sha1_hex($raw),
2034         snapshots => {},
2035         pending => {},
2036     };
2037
2038     $filename =~ m|/qemu-server/(\d+)\.conf$|
2039         || die "got strange filename '$filename'";
2040
2041     my $vmid = $1;
2042
2043     my $conf = $res;
2044     my $descr;
2045     my $section = '';
2046
2047     my @lines = split(/\n/, $raw);
2048     foreach my $line (@lines) {
2049         next if $line =~ m/^\s*$/;
2050
2051         if ($line =~ m/^\[PENDING\]\s*$/i) {
2052             $section = 'pending';
2053             if (defined($descr)) {
2054                 $descr =~ s/\s+$//;
2055                 $conf->{description} = $descr;
2056             }
2057             $descr = undef;
2058             $conf = $res->{$section} = {};
2059             next;
2060
2061         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2062             $section = $1;
2063             if (defined($descr)) {
2064                 $descr =~ s/\s+$//;
2065                 $conf->{description} = $descr;
2066             }
2067             $descr = undef;
2068             $conf = $res->{snapshots}->{$section} = {};
2069             next;
2070         }
2071
2072         if ($line =~ m/^\#(.*)\s*$/) {
2073             $descr = '' if !defined($descr);
2074             $descr .= PVE::Tools::decode_text($1) . "\n";
2075             next;
2076         }
2077
2078         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2079             $descr = '' if !defined($descr);
2080             $descr .= PVE::Tools::decode_text($2);
2081         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2082             $conf->{snapstate} = $1;
2083         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2084             my $key = $1;
2085             my $value = $2;
2086             $conf->{$key} = $value;
2087         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2088             my $value = $1;
2089             if ($section eq 'pending') {
2090                 $conf->{delete} = $value; # we parse this later
2091             } else {
2092                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2093             }
2094         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
2095             my $key = $1;
2096             my $value = $2;
2097             eval { $value = check_type($key, $value); };
2098             if ($@) {
2099                 warn "vm $vmid - unable to parse value of '$key' - $@";
2100             } else {
2101                 my $fmt = $confdesc->{$key}->{format};
2102                 if ($fmt && $fmt eq 'pve-qm-drive') {
2103                     my $v = parse_drive($key, $value);
2104                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2105                         $v->{file} = $volid;
2106                         $value = print_drive($vmid, $v);
2107                     } else {
2108                         warn "vm $vmid - unable to parse value of '$key'\n";
2109                         next;
2110                     }
2111                 }
2112
2113                 if ($key eq 'cdrom') {
2114                     $conf->{ide2} = $value;
2115                 } else {
2116                     $conf->{$key} = $value;
2117                 }
2118             }
2119         }
2120     }
2121
2122     if (defined($descr)) {
2123         $descr =~ s/\s+$//;
2124         $conf->{description} = $descr;
2125     }
2126     delete $res->{snapstate}; # just to be sure
2127
2128     return $res;
2129 }
2130
2131 sub write_vm_config {
2132     my ($filename, $conf) = @_;
2133
2134     delete $conf->{snapstate}; # just to be sure
2135
2136     if ($conf->{cdrom}) {
2137         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2138         $conf->{ide2} = $conf->{cdrom};
2139         delete $conf->{cdrom};
2140     }
2141
2142     # we do not use 'smp' any longer
2143     if ($conf->{sockets}) {
2144         delete $conf->{smp};
2145     } elsif ($conf->{smp}) {
2146         $conf->{sockets} = $conf->{smp};
2147         delete $conf->{cores};
2148         delete $conf->{smp};
2149     }
2150
2151     my $used_volids = {};
2152
2153     my $cleanup_config = sub {
2154         my ($cref, $pending, $snapname) = @_;
2155
2156         foreach my $key (keys %$cref) {
2157             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2158                 $key eq 'snapstate' || $key eq 'pending';
2159             my $value = $cref->{$key};
2160             if ($key eq 'delete') {
2161                 die "propertry 'delete' is only allowed in [PENDING]\n"
2162                     if !$pending;
2163                 # fixme: check syntax?
2164                 next;
2165             }
2166             eval { $value = check_type($key, $value); };
2167             die "unable to parse value of '$key' - $@" if $@;
2168
2169             $cref->{$key} = $value;
2170
2171             if (!$snapname && is_valid_drivename($key)) {
2172                 my $drive = parse_drive($key, $value);
2173                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2174             }
2175         }
2176     };
2177
2178     &$cleanup_config($conf);
2179
2180     &$cleanup_config($conf->{pending}, 1);
2181
2182     foreach my $snapname (keys %{$conf->{snapshots}}) {
2183         die "internal error" if $snapname eq 'pending';
2184         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2185     }
2186
2187     # remove 'unusedX' settings if we re-add a volume
2188     foreach my $key (keys %$conf) {
2189         my $value = $conf->{$key};
2190         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2191             delete $conf->{$key};
2192         }
2193     }
2194
2195     my $generate_raw_config = sub {
2196         my ($conf, $pending) = @_;
2197
2198         my $raw = '';
2199
2200         # add description as comment to top of file
2201         if (defined(my $descr = $conf->{description})) {
2202             if ($descr) {
2203                 foreach my $cl (split(/\n/, $descr)) {
2204                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2205                 }
2206             } else {
2207                 $raw .= "#\n" if $pending;
2208             }
2209         }
2210
2211         foreach my $key (sort keys %$conf) {
2212             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2213             $raw .= "$key: $conf->{$key}\n";
2214         }
2215         return $raw;
2216     };
2217
2218     my $raw = &$generate_raw_config($conf);
2219
2220     if (scalar(keys %{$conf->{pending}})){
2221         $raw .= "\n[PENDING]\n";
2222         $raw .= &$generate_raw_config($conf->{pending}, 1);
2223     }
2224
2225     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2226         $raw .= "\n[$snapname]\n";
2227         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2228     }
2229
2230     return $raw;
2231 }
2232
2233 sub load_defaults {
2234
2235     my $res = {};
2236
2237     # we use static defaults from our JSON schema configuration
2238     foreach my $key (keys %$confdesc) {
2239         if (defined(my $default = $confdesc->{$key}->{default})) {
2240             $res->{$key} = $default;
2241         }
2242     }
2243
2244     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2245     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2246
2247     return $res;
2248 }
2249
2250 sub config_list {
2251     my $vmlist = PVE::Cluster::get_vmlist();
2252     my $res = {};
2253     return $res if !$vmlist || !$vmlist->{ids};
2254     my $ids = $vmlist->{ids};
2255
2256     foreach my $vmid (keys %$ids) {
2257         my $d = $ids->{$vmid};
2258         next if !$d->{node} || $d->{node} ne $nodename;
2259         next if !$d->{type} || $d->{type} ne 'qemu';
2260         $res->{$vmid}->{exists} = 1;
2261     }
2262     return $res;
2263 }
2264
2265 # test if VM uses local resources (to prevent migration)
2266 sub check_local_resources {
2267     my ($conf, $noerr) = @_;
2268
2269     my $loc_res = 0;
2270
2271     $loc_res = 1 if $conf->{hostusb}; # old syntax
2272     $loc_res = 1 if $conf->{hostpci}; # old syntax
2273
2274     foreach my $k (keys %$conf) {
2275         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2276         # sockets are safe: they will recreated be on the target side post-migrate
2277         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2278         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2279     }
2280
2281     die "VM uses local resources\n" if $loc_res && !$noerr;
2282
2283     return $loc_res;
2284 }
2285
2286 # check if used storages are available on all nodes (use by migrate)
2287 sub check_storage_availability {
2288     my ($storecfg, $conf, $node) = @_;
2289
2290     foreach_drive($conf, sub {
2291         my ($ds, $drive) = @_;
2292
2293         my $volid = $drive->{file};
2294         return if !$volid;
2295
2296         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2297         return if !$sid;
2298
2299         # check if storage is available on both nodes
2300         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2301         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2302    });
2303 }
2304
2305 # list nodes where all VM images are available (used by has_feature API)
2306 sub shared_nodes {
2307     my ($conf, $storecfg) = @_;
2308
2309     my $nodelist = PVE::Cluster::get_nodelist();
2310     my $nodehash = { map { $_ => 1 } @$nodelist };
2311     my $nodename = PVE::INotify::nodename();
2312
2313     foreach_drive($conf, sub {
2314         my ($ds, $drive) = @_;
2315
2316         my $volid = $drive->{file};
2317         return if !$volid;
2318
2319         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2320         if ($storeid) {
2321             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2322             if ($scfg->{disable}) {
2323                 $nodehash = {};
2324             } elsif (my $avail = $scfg->{nodes}) {
2325                 foreach my $node (keys %$nodehash) {
2326                     delete $nodehash->{$node} if !$avail->{$node};
2327                 }
2328             } elsif (!$scfg->{shared}) {
2329                 foreach my $node (keys %$nodehash) {
2330                     delete $nodehash->{$node} if $node ne $nodename
2331                 }
2332             }
2333         }
2334     });
2335
2336     return $nodehash
2337 }
2338
2339 sub check_cmdline {
2340     my ($pidfile, $pid) = @_;
2341
2342     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2343     if (defined($fh)) {
2344         my $line = <$fh>;
2345         $fh->close;
2346         return undef if !$line;
2347         my @param = split(/\0/, $line);
2348
2349         my $cmd = $param[0];
2350         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2351
2352         for (my $i = 0; $i < scalar (@param); $i++) {
2353             my $p = $param[$i];
2354             next if !$p;
2355             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2356                 my $p = $param[$i+1];
2357                 return 1 if $p && ($p eq $pidfile);
2358                 return undef;
2359             }
2360         }
2361     }
2362     return undef;
2363 }
2364
2365 sub check_running {
2366     my ($vmid, $nocheck, $node) = @_;
2367
2368     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2369
2370     die "unable to find configuration file for VM $vmid - no such machine\n"
2371         if !$nocheck && ! -f $filename;
2372
2373     my $pidfile = pidfile_name($vmid);
2374
2375     if (my $fd = IO::File->new("<$pidfile")) {
2376         my $st = stat($fd);
2377         my $line = <$fd>;
2378         close($fd);
2379
2380         my $mtime = $st->mtime;
2381         if ($mtime > time()) {
2382             warn "file '$filename' modified in future\n";
2383         }
2384
2385         if ($line =~ m/^(\d+)$/) {
2386             my $pid = $1;
2387             if (check_cmdline($pidfile, $pid)) {
2388                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2389                     return $pid;
2390                 }
2391             }
2392         }
2393     }
2394
2395     return undef;
2396 }
2397
2398 sub vzlist {
2399
2400     my $vzlist = config_list();
2401
2402     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2403
2404     while (defined(my $de = $fd->read)) {
2405         next if $de !~ m/^(\d+)\.pid$/;
2406         my $vmid = $1;
2407         next if !defined($vzlist->{$vmid});
2408         if (my $pid = check_running($vmid)) {
2409             $vzlist->{$vmid}->{pid} = $pid;
2410         }
2411     }
2412
2413     return $vzlist;
2414 }
2415
2416 sub disksize {
2417     my ($storecfg, $conf) = @_;
2418
2419     my $bootdisk = $conf->{bootdisk};
2420     return undef if !$bootdisk;
2421     return undef if !is_valid_drivename($bootdisk);
2422
2423     return undef if !$conf->{$bootdisk};
2424
2425     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2426     return undef if !defined($drive);
2427
2428     return undef if drive_is_cdrom($drive);
2429
2430     my $volid = $drive->{file};
2431     return undef if !$volid;
2432
2433     return $drive->{size};
2434 }
2435
2436 my $last_proc_pid_stat;
2437
2438 # get VM status information
2439 # This must be fast and should not block ($full == false)
2440 # We only query KVM using QMP if $full == true (this can be slow)
2441 sub vmstatus {
2442     my ($opt_vmid, $full) = @_;
2443
2444     my $res = {};
2445
2446     my $storecfg = PVE::Storage::config();
2447
2448     my $list = vzlist();
2449     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2450
2451     my $cpucount = $cpuinfo->{cpus} || 1;
2452
2453     foreach my $vmid (keys %$list) {
2454         next if $opt_vmid && ($vmid ne $opt_vmid);
2455
2456         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
2457         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2458
2459         my $d = {};
2460         $d->{pid} = $list->{$vmid}->{pid};
2461
2462         # fixme: better status?
2463         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2464
2465         my $size = disksize($storecfg, $conf);
2466         if (defined($size)) {
2467             $d->{disk} = 0; # no info available
2468             $d->{maxdisk} = $size;
2469         } else {
2470             $d->{disk} = 0;
2471             $d->{maxdisk} = 0;
2472         }
2473
2474         $d->{cpus} = ($conf->{sockets} || 1) * ($conf->{cores} || 1);
2475         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2476         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2477
2478         $d->{name} = $conf->{name} || "VM $vmid";
2479         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024) : 0;
2480
2481         if ($conf->{balloon}) {
2482             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2483             $d->{shares} = defined($conf->{shares}) ? $conf->{shares} : 1000;
2484         }
2485
2486         $d->{uptime} = 0;
2487         $d->{cpu} = 0;
2488         $d->{mem} = 0;
2489
2490         $d->{netout} = 0;
2491         $d->{netin} = 0;
2492
2493         $d->{diskread} = 0;
2494         $d->{diskwrite} = 0;
2495
2496         $d->{template} = PVE::QemuConfig->is_template($conf);
2497
2498         $res->{$vmid} = $d;
2499     }
2500
2501     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2502     foreach my $dev (keys %$netdev) {
2503         next if $dev !~ m/^tap([1-9]\d*)i/;
2504         my $vmid = $1;
2505         my $d = $res->{$vmid};
2506         next if !$d;
2507
2508         $d->{netout} += $netdev->{$dev}->{receive};
2509         $d->{netin} += $netdev->{$dev}->{transmit};
2510
2511         if ($full) {
2512             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2513             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2514         }
2515
2516     }
2517
2518     my $ctime = gettimeofday;
2519
2520     foreach my $vmid (keys %$list) {
2521
2522         my $d = $res->{$vmid};
2523         my $pid = $d->{pid};
2524         next if !$pid;
2525
2526         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2527         next if !$pstat; # not running
2528
2529         my $used = $pstat->{utime} + $pstat->{stime};
2530
2531         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2532
2533         if ($pstat->{vsize}) {
2534             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2535         }
2536
2537         my $old = $last_proc_pid_stat->{$pid};
2538         if (!$old) {
2539             $last_proc_pid_stat->{$pid} = {
2540                 time => $ctime,
2541                 used => $used,
2542                 cpu => 0,
2543             };
2544             next;
2545         }
2546
2547         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2548
2549         if ($dtime > 1000) {
2550             my $dutime = $used -  $old->{used};
2551
2552             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2553             $last_proc_pid_stat->{$pid} = {
2554                 time => $ctime,
2555                 used => $used,
2556                 cpu => $d->{cpu},
2557             };
2558         } else {
2559             $d->{cpu} = $old->{cpu};
2560         }
2561     }
2562
2563     return $res if !$full;
2564
2565     my $qmpclient = PVE::QMPClient->new();
2566
2567     my $ballooncb = sub {
2568         my ($vmid, $resp) = @_;
2569
2570         my $info = $resp->{'return'};
2571         return if !$info->{max_mem};
2572
2573         my $d = $res->{$vmid};
2574
2575         # use memory assigned to VM
2576         $d->{maxmem} = $info->{max_mem};
2577         $d->{balloon} = $info->{actual};
2578
2579         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2580             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2581             $d->{freemem} = $info->{free_mem};
2582         }
2583
2584         $d->{ballooninfo} = $info;
2585     };
2586
2587     my $blockstatscb = sub {
2588         my ($vmid, $resp) = @_;
2589         my $data = $resp->{'return'} || [];
2590         my $totalrdbytes = 0;
2591         my $totalwrbytes = 0;
2592
2593         for my $blockstat (@$data) {
2594             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2595             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2596
2597             $blockstat->{device} =~ s/drive-//;
2598             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2599         }
2600         $res->{$vmid}->{diskread} = $totalrdbytes;
2601         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2602     };
2603
2604     my $statuscb = sub {
2605         my ($vmid, $resp) = @_;
2606
2607         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2608         # this fails if ballon driver is not loaded, so this must be
2609         # the last commnand (following command are aborted if this fails).
2610         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2611
2612         my $status = 'unknown';
2613         if (!defined($status = $resp->{'return'}->{status})) {
2614             warn "unable to get VM status\n";
2615             return;
2616         }
2617
2618         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2619     };
2620
2621     foreach my $vmid (keys %$list) {
2622         next if $opt_vmid && ($vmid ne $opt_vmid);
2623         next if !$res->{$vmid}->{pid}; # not running
2624         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2625     }
2626
2627     $qmpclient->queue_execute(undef, 1);
2628
2629     foreach my $vmid (keys %$list) {
2630         next if $opt_vmid && ($vmid ne $opt_vmid);
2631         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2632     }
2633
2634     return $res;
2635 }
2636
2637 sub foreach_drive {
2638     my ($conf, $func, @param) = @_;
2639
2640     foreach my $ds (valid_drive_names()) {
2641         next if !defined($conf->{$ds});
2642
2643         my $drive = parse_drive($ds, $conf->{$ds});
2644         next if !$drive;
2645
2646         &$func($ds, $drive, @param);
2647     }
2648 }
2649
2650 sub foreach_volid {
2651     my ($conf, $func, @param) = @_;
2652
2653     my $volhash = {};
2654
2655     my $test_volid = sub {
2656         my ($volid, $is_cdrom) = @_;
2657
2658         return if !$volid;
2659
2660         $volhash->{$volid} = $is_cdrom || 0;
2661     };
2662
2663     foreach_drive($conf, sub {
2664         my ($ds, $drive) = @_;
2665         &$test_volid($drive->{file}, drive_is_cdrom($drive));
2666     });
2667
2668     foreach my $snapname (keys %{$conf->{snapshots}}) {
2669         my $snap = $conf->{snapshots}->{$snapname};
2670         &$test_volid($snap->{vmstate}, 0);
2671         foreach_drive($snap, sub {
2672             my ($ds, $drive) = @_;
2673             &$test_volid($drive->{file}, drive_is_cdrom($drive));
2674         });
2675     }
2676
2677     foreach my $volid (keys %$volhash) {
2678         &$func($volid, $volhash->{$volid}, @param);
2679     }
2680 }
2681
2682 sub vga_conf_has_spice {
2683     my ($vga) = @_;
2684
2685     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
2686
2687     return $1 || 1;
2688 }
2689
2690 sub config_to_command {
2691     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2692
2693     my $cmd = [];
2694     my $globalFlags = [];
2695     my $machineFlags = [];
2696     my $rtcFlags = [];
2697     my $cpuFlags = [];
2698     my $devices = [];
2699     my $pciaddr = '';
2700     my $bridges = {};
2701     my $kvmver = kvm_user_version();
2702     my $vernum = 0; # unknown
2703     my $ostype = $conf->{ostype};
2704     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
2705         $vernum = $1*1000000+$2*1000;
2706     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
2707         $vernum = $1*1000000+$2*1000+$3;
2708     }
2709
2710     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
2711
2712     my $have_ovz = -f '/proc/vz/vestat';
2713
2714     my $q35 = machine_type_is_q35($conf);
2715     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2716     my $machine_type = $forcemachine || $conf->{machine};
2717     my $use_old_bios_files = undef;
2718     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
2719
2720     my $cpuunits = defined($conf->{cpuunits}) ?
2721             $conf->{cpuunits} : $defaults->{cpuunits};
2722
2723     push @$cmd, '/usr/bin/kvm';
2724
2725     push @$cmd, '-id', $vmid;
2726
2727     my $use_virtio = 0;
2728
2729     my $qmpsocket = qmp_socket($vmid);
2730     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2731     push @$cmd, '-mon', "chardev=qmp,mode=control";
2732
2733
2734     push @$cmd, '-pidfile' , pidfile_name($vmid);
2735
2736     push @$cmd, '-daemonize';
2737
2738     if ($conf->{smbios1}) {
2739         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
2740     }
2741
2742     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2743         my $ovmfvar = "OVMF_VARS-pure-efi.fd";
2744         my $ovmfvar_src = "/usr/share/kvm/$ovmfvar";
2745         my $ovmfvar_dst = "/tmp/$vmid-$ovmfvar";
2746         PVE::Tools::file_copy($ovmfvar_src, $ovmfvar_dst, 256*1024);
2747         push @$cmd, '-drive', "if=pflash,format=raw,readonly,file=/usr/share/kvm/OVMF-pure-efi.fd";
2748         push @$cmd, '-drive', "if=pflash,format=raw,file=$ovmfvar_dst";
2749     }
2750
2751
2752     # add usb controllers
2753     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $q35, $usbdesc->{format}, $MAX_USB_DEVICES);
2754     push @$devices, @usbcontrollers if @usbcontrollers;
2755     my $vga = $conf->{vga};
2756
2757     my $qxlnum = vga_conf_has_spice($vga);
2758     $vga = 'qxl' if $qxlnum;
2759
2760     if (!$vga) {
2761         if ($conf->{ostype} && ($conf->{ostype} eq 'win8' ||
2762                                 $conf->{ostype} eq 'win7' ||
2763                                 $conf->{ostype} eq 'w2k8')) {
2764             $vga = 'std';
2765         } else {
2766             $vga = 'cirrus';
2767         }
2768     }
2769
2770     # enable absolute mouse coordinates (needed by vnc)
2771     my $tablet;
2772     if (defined($conf->{tablet})) {
2773         $tablet = $conf->{tablet};
2774     } else {
2775         $tablet = $defaults->{tablet};
2776         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
2777         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
2778     }
2779
2780     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
2781
2782     my $kvm_off = 0;
2783     # host pci devices
2784     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
2785         my $d = parse_hostpci($conf->{"hostpci$i"});
2786         next if !$d;
2787
2788         my $pcie = $d->{pcie};
2789         if($pcie){
2790             die "q35 machine model is not enabled" if !$q35;
2791             $pciaddr = print_pcie_addr("hostpci$i");
2792         }else{
2793             $pciaddr = print_pci_addr("hostpci$i", $bridges);
2794         }
2795
2796         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
2797         my $xvga = '';
2798         if ($d->{'x-vga'}) {
2799             $xvga = ',x-vga=on';
2800             $kvm_off = 1;
2801             $vga = 'none';
2802             if ($ostype eq 'win7' || $ostype eq 'win8' || $ostype eq 'w2k8') {
2803                 push @$cpuFlags , 'hv_vendor_id=proxmox';
2804             }
2805             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2806                 $xvga = "";
2807             }
2808         }
2809         my $pcidevices = $d->{pciid};
2810         my $multifunction = 1 if @$pcidevices > 1;
2811
2812         my $j=0;
2813         foreach my $pcidevice (@$pcidevices) {
2814
2815             my $id = "hostpci$i";
2816             $id .= ".$j" if $multifunction;
2817             my $addr = $pciaddr;
2818             $addr .= ".$j" if $multifunction;
2819             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
2820
2821             if($j == 0){
2822                 $devicestr .= "$rombar$xvga";
2823                 $devicestr .= ",multifunction=on" if $multifunction;
2824             }
2825
2826             push @$devices, '-device', $devicestr;
2827             $j++;
2828         }
2829     }
2830
2831     # usb devices
2832     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES);
2833     push @$devices, @usbdevices if @usbdevices;
2834     # serial devices
2835     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2836         if (my $path = $conf->{"serial$i"}) {
2837             if ($path eq 'socket') {
2838                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
2839                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
2840                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2841             } else {
2842                 die "no such serial device\n" if ! -c $path;
2843                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
2844                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2845             }
2846         }
2847     }
2848
2849     # parallel devices
2850     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
2851         if (my $path = $conf->{"parallel$i"}) {
2852             die "no such parallel device\n" if ! -c $path;
2853             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
2854             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
2855             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
2856         }
2857     }
2858
2859     my $vmname = $conf->{name} || "vm$vmid";
2860
2861     push @$cmd, '-name', $vmname;
2862
2863     my $sockets = 1;
2864     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
2865     $sockets = $conf->{sockets} if  $conf->{sockets};
2866
2867     my $cores = $conf->{cores} || 1;
2868
2869     my $maxcpus = $sockets * $cores;
2870
2871     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
2872
2873     my $allowed_vcpus = $cpuinfo->{cpus};
2874
2875     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
2876         if ($allowed_vcpus < $maxcpus);
2877
2878     push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
2879
2880     push @$cmd, '-nodefaults';
2881
2882     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2883
2884     my $bootindex_hash = {};
2885     my $i = 1;
2886     foreach my $o (split(//, $bootorder)) {
2887         $bootindex_hash->{$o} = $i*100;
2888         $i++;
2889     }
2890
2891     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000";
2892
2893     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
2894
2895     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
2896
2897     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
2898
2899     if ($vga && $vga !~ m/^serial\d+$/ && $vga ne 'none'){
2900         my $socket = vnc_socket($vmid);
2901         push @$cmd,  '-vnc', "unix:$socket,x509,password";
2902     } else {
2903         push @$cmd, '-nographic';
2904     }
2905
2906     # time drift fix
2907     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
2908
2909     my $nokvm = defined($conf->{kvm}) && $conf->{kvm} == 0 ? 1 : 0;
2910     my $useLocaltime = $conf->{localtime};
2911
2912     if ($ostype) {
2913         # other, wxp, w2k, w2k3, w2k8, wvista, win7, win8, l24, l26, solaris
2914
2915         if ($ostype =~ m/^w/) { # windows
2916             $useLocaltime = 1 if !defined($conf->{localtime});
2917
2918             # use time drift fix when acpi is enabled
2919             if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
2920                 $tdf = 1 if !defined($conf->{tdf});
2921             }
2922         }
2923
2924         if ($ostype eq 'win7' || $ostype eq 'win8' || $ostype eq 'w2k8' ||
2925             $ostype eq 'wvista') {
2926             push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
2927             push @$cmd, '-no-hpet';
2928             if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
2929                 push @$cpuFlags , 'hv_spinlocks=0x1fff' if !$nokvm;
2930                 push @$cpuFlags , 'hv_vapic' if !$nokvm;
2931                 push @$cpuFlags , 'hv_time' if !$nokvm;
2932
2933                 if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
2934                     push @$cpuFlags , 'hv_reset' if !$nokvm;
2935                     push @$cpuFlags , 'hv_vpindex' if !$nokvm;
2936                     push @$cpuFlags , 'hv_runtime' if !$nokvm;
2937                 }
2938
2939             } else {
2940                 push @$cpuFlags , 'hv_spinlocks=0xffff' if !$nokvm;
2941             }
2942         }
2943
2944         if ($ostype eq 'win7' || $ostype eq 'win8') {
2945             push @$cpuFlags , 'hv_relaxed' if !$nokvm;
2946         }
2947     }
2948
2949     push @$rtcFlags, 'driftfix=slew' if $tdf;
2950
2951     if ($nokvm) {
2952         push @$machineFlags, 'accel=tcg';
2953     } else {
2954         die "No accelerator found!\n" if !$cpuinfo->{hvm};
2955     }
2956
2957     if ($machine_type) {
2958         push @$machineFlags, "type=${machine_type}";
2959     }
2960
2961     if ($conf->{startdate}) {
2962         push @$rtcFlags, "base=$conf->{startdate}";
2963     } elsif ($useLocaltime) {
2964         push @$rtcFlags, 'base=localtime';
2965     }
2966
2967     my $cpu = $nokvm ? "qemu64" : "kvm64";
2968     if (my $cputype = $conf->{cpu}) {
2969         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
2970             or die "Cannot parse cpu description: $cputype\n";
2971         $cpu = $cpuconf->{cputype};
2972         $kvm_off = 1 if $cpuconf->{hidden};
2973     }
2974
2975     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
2976
2977     push @$cpuFlags , '-x2apic'
2978         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
2979
2980     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
2981
2982     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
2983
2984     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
2985
2986         push @$cpuFlags , '+kvm_pv_unhalt' if !$nokvm;
2987         push @$cpuFlags , '+kvm_pv_eoi' if !$nokvm;
2988     }
2989
2990     push @$cpuFlags, 'enforce' if $cpu ne 'host' && !$nokvm;
2991
2992     push @$cpuFlags, 'kvm=off' if $kvm_off;
2993
2994     my $cpu_vendor = $cpu_vendor_list->{$cpu} ||
2995         die "internal error"; # should not happen
2996
2997     push @$cpuFlags, "vendor=${cpu_vendor}"
2998         if $cpu_vendor ne 'default';
2999
3000     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3001
3002     push @$cmd, '-cpu', $cpu;
3003
3004     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3005
3006     push @$cmd, '-S' if $conf->{freeze};
3007
3008     # set keyboard layout
3009     my $kb = $conf->{keyboard} || $defaults->{keyboard};
3010     push @$cmd, '-k', $kb if $kb;
3011
3012     # enable sound
3013     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3014     #push @$cmd, '-soundhw', 'es1370';
3015     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3016
3017     if($conf->{agent}) {
3018         my $qgasocket = qmp_socket($vmid, 1);
3019         my $pciaddr = print_pci_addr("qga0", $bridges);
3020         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3021         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3022         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3023     }
3024
3025     my $spice_port;
3026
3027     if ($qxlnum) {
3028         if ($qxlnum > 1) {
3029             if ($conf->{ostype} && $conf->{ostype} =~ m/^w/){
3030                 for(my $i = 1; $i < $qxlnum; $i++){
3031                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3032                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3033                 }
3034             } else {
3035                 # assume other OS works like Linux
3036                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3037                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3038             }
3039         }
3040
3041         my $pciaddr = print_pci_addr("spice", $bridges);
3042
3043         my $nodename = PVE::INotify::nodename();
3044         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3045         $spice_port = PVE::Tools::next_spice_port($pfamily);
3046
3047         push @$devices, '-spice', "tls-port=${spice_port},addr=localhost,tls-ciphers=DES-CBC3-SHA,seamless-migration=on";
3048
3049         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3050         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3051         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3052     }
3053
3054     # enable balloon by default, unless explicitly disabled
3055     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3056         $pciaddr = print_pci_addr("balloon0", $bridges);
3057         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3058     }
3059
3060     if ($conf->{watchdog}) {
3061         my $wdopts = parse_watchdog($conf->{watchdog});
3062         $pciaddr = print_pci_addr("watchdog", $bridges);
3063         my $watchdog = $wdopts->{model} || 'i6300esb';
3064         push @$devices, '-device', "$watchdog$pciaddr";
3065         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3066     }
3067
3068     my $vollist = [];
3069     my $scsicontroller = {};
3070     my $ahcicontroller = {};
3071     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3072
3073     # Add iscsi initiator name if available
3074     if (my $initiator = get_initiator_name()) {
3075         push @$devices, '-iscsi', "initiator-name=$initiator";
3076     }
3077
3078     foreach_drive($conf, sub {
3079         my ($ds, $drive) = @_;
3080
3081         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3082             push @$vollist, $drive->{file};
3083         }
3084
3085         $use_virtio = 1 if $ds =~ m/^virtio/;
3086
3087         if (drive_is_cdrom ($drive)) {
3088             if ($bootindex_hash->{d}) {
3089                 $drive->{bootindex} = $bootindex_hash->{d};
3090                 $bootindex_hash->{d} += 1;
3091             }
3092         } else {
3093             if ($bootindex_hash->{c}) {
3094                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3095                 $bootindex_hash->{c} += 1;
3096             }
3097         }
3098
3099         if($drive->{interface} eq 'virtio'){
3100            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3101         }
3102
3103         if ($drive->{interface} eq 'scsi') {
3104
3105             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3106
3107             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3108             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3109
3110             my $iothread = '';
3111             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3112                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3113                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3114             } elsif ($drive->{iothread}) {
3115                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3116             }
3117
3118             my $queues = '';
3119             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3120                 $queues = ",num_queues=$drive->{queues}";
3121             }
3122
3123             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3124             $scsicontroller->{$controller}=1;
3125         }
3126
3127         if ($drive->{interface} eq 'sata') {
3128            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3129            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3130            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3131            $ahcicontroller->{$controller}=1;
3132         }
3133
3134         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3135         push @$devices, '-drive',$drive_cmd;
3136         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3137     });
3138
3139     for (my $i = 0; $i < $MAX_NETS; $i++) {
3140          next if !$conf->{"net$i"};
3141          my $d = parse_net($conf->{"net$i"});
3142          next if !$d;
3143
3144          $use_virtio = 1 if $d->{model} eq 'virtio';
3145
3146          if ($bootindex_hash->{n}) {
3147             $d->{bootindex} = $bootindex_hash->{n};
3148             $bootindex_hash->{n} += 1;
3149          }
3150
3151          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3152          push @$devices, '-netdev', $netdevfull;
3153
3154          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3155          push @$devices, '-device', $netdevicefull;
3156     }
3157
3158     if (!$q35) {
3159         # add pci bridges
3160         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3161            $bridges->{1} = 1;
3162            $bridges->{2} = 1;
3163         }
3164
3165         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3166
3167         while (my ($k, $v) = each %$bridges) {
3168             $pciaddr = print_pci_addr("pci.$k");
3169             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3170         }
3171     }
3172
3173     # add custom args
3174     if ($conf->{args}) {
3175         my $aa = PVE::Tools::split_args($conf->{args});
3176         push @$cmd, @$aa;
3177     }
3178
3179     push @$cmd, @$devices;
3180     push @$cmd, '-rtc', join(',', @$rtcFlags)
3181         if scalar(@$rtcFlags);
3182     push @$cmd, '-machine', join(',', @$machineFlags)
3183         if scalar(@$machineFlags);
3184     push @$cmd, '-global', join(',', @$globalFlags)
3185         if scalar(@$globalFlags);
3186
3187     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3188 }
3189
3190 sub vnc_socket {
3191     my ($vmid) = @_;
3192     return "${var_run_tmpdir}/$vmid.vnc";
3193 }
3194
3195 sub spice_port {
3196     my ($vmid) = @_;
3197
3198     my $res = vm_mon_cmd($vmid, 'query-spice');
3199
3200     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3201 }
3202
3203 sub qmp_socket {
3204     my ($vmid, $qga) = @_;
3205     my $sockettype = $qga ? 'qga' : 'qmp';
3206     return "${var_run_tmpdir}/$vmid.$sockettype";
3207 }
3208
3209 sub pidfile_name {
3210     my ($vmid) = @_;
3211     return "${var_run_tmpdir}/$vmid.pid";
3212 }
3213
3214 sub vm_devices_list {
3215     my ($vmid) = @_;
3216
3217     my $res = vm_mon_cmd($vmid, 'query-pci');
3218     my $devices = {};
3219     foreach my $pcibus (@$res) {
3220         foreach my $device (@{$pcibus->{devices}}) {
3221             next if !$device->{'qdev_id'};
3222             if ($device->{'pci_bridge'}) {
3223                 $devices->{$device->{'qdev_id'}} = 1;
3224                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3225                     next if !$bridge_device->{'qdev_id'};
3226                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3227                     $devices->{$device->{'qdev_id'}}++;
3228                 }
3229             } else {
3230                 $devices->{$device->{'qdev_id'}} = 1;
3231             }
3232         }
3233     }
3234
3235     my $resblock = vm_mon_cmd($vmid, 'query-block');
3236     foreach my $block (@$resblock) {
3237         if($block->{device} =~ m/^drive-(\S+)/){
3238                 $devices->{$1} = 1;
3239         }
3240     }
3241
3242     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3243     foreach my $mice (@$resmice) {
3244         if ($mice->{name} eq 'QEMU HID Tablet') {
3245             $devices->{tablet} = 1;
3246             last;
3247         }
3248     }
3249
3250     # for usb devices there is no query-usb
3251     # but we can iterate over the entries in
3252     # qom-list path=/machine/peripheral
3253     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3254     foreach my $per (@$resperipheral) {
3255         if ($per->{name} =~ m/^usb\d+$/) {
3256             $devices->{$per->{name}} = 1;
3257         }
3258     }
3259
3260     return $devices;
3261 }
3262
3263 sub vm_deviceplug {
3264     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3265
3266     my $q35 = machine_type_is_q35($conf);
3267
3268     my $devices_list = vm_devices_list($vmid);
3269     return 1 if defined($devices_list->{$deviceid});
3270
3271     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3272
3273     if ($deviceid eq 'tablet') {
3274
3275         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3276
3277     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3278
3279         die "usb hotplug currently not reliable\n";
3280         # since we can't reliably hot unplug all added usb devices
3281         # and usb passthrough disables live migration
3282         # we disable usb hotplugging for now
3283         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3284
3285     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3286
3287         qemu_iothread_add($vmid, $deviceid, $device);
3288
3289         qemu_driveadd($storecfg, $vmid, $device);
3290         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3291
3292         qemu_deviceadd($vmid, $devicefull);
3293         eval { qemu_deviceaddverify($vmid, $deviceid); };
3294         if (my $err = $@) {
3295             eval { qemu_drivedel($vmid, $deviceid); };
3296             warn $@ if $@;
3297             die $err;
3298         }
3299
3300     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3301
3302
3303         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3304         my $pciaddr = print_pci_addr($deviceid);
3305         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3306
3307         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3308
3309         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3310             qemu_iothread_add($vmid, $deviceid, $device);
3311             $devicefull .= ",iothread=iothread-$deviceid";
3312         }
3313
3314         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3315             $devicefull .= ",num_queues=$device->{queues}";
3316         }
3317
3318         qemu_deviceadd($vmid, $devicefull);
3319         qemu_deviceaddverify($vmid, $deviceid);
3320
3321     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3322
3323         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3324         qemu_driveadd($storecfg, $vmid, $device);
3325
3326         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3327         eval { qemu_deviceadd($vmid, $devicefull); };
3328         if (my $err = $@) {
3329             eval { qemu_drivedel($vmid, $deviceid); };
3330             warn $@ if $@;
3331             die $err;
3332         }
3333
3334     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3335
3336         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3337
3338         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
3339         my $use_old_bios_files = undef;
3340         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3341
3342         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3343         qemu_deviceadd($vmid, $netdevicefull);
3344         eval { qemu_deviceaddverify($vmid, $deviceid); };
3345         if (my $err = $@) {
3346             eval { qemu_netdevdel($vmid, $deviceid); };
3347             warn $@ if $@;
3348             die $err;
3349         }
3350
3351     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3352
3353         my $bridgeid = $2;
3354         my $pciaddr = print_pci_addr($deviceid);
3355         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3356
3357         qemu_deviceadd($vmid, $devicefull);
3358         qemu_deviceaddverify($vmid, $deviceid);
3359
3360     } else {
3361         die "can't hotplug device '$deviceid'\n";
3362     }
3363
3364     return 1;
3365 }
3366
3367 # fixme: this should raise exceptions on error!
3368 sub vm_deviceunplug {
3369     my ($vmid, $conf, $deviceid) = @_;
3370
3371     my $devices_list = vm_devices_list($vmid);
3372     return 1 if !defined($devices_list->{$deviceid});
3373
3374     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3375
3376     if ($deviceid eq 'tablet') {
3377
3378         qemu_devicedel($vmid, $deviceid);
3379
3380     } elsif ($deviceid =~ m/^usb\d+$/) {
3381
3382         die "usb hotplug currently not reliable\n";
3383         # when unplugging usb devices this way,
3384         # there may be remaining usb controllers/hubs
3385         # so we disable it for now
3386         qemu_devicedel($vmid, $deviceid);
3387         qemu_devicedelverify($vmid, $deviceid);
3388
3389     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3390
3391         qemu_devicedel($vmid, $deviceid);
3392         qemu_devicedelverify($vmid, $deviceid);
3393         qemu_drivedel($vmid, $deviceid);
3394         qemu_iothread_del($conf, $vmid, $deviceid);
3395
3396     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3397
3398         qemu_devicedel($vmid, $deviceid);
3399         qemu_devicedelverify($vmid, $deviceid);
3400         qemu_iothread_del($conf, $vmid, $deviceid);
3401
3402     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3403
3404         #qemu 2.3 segfault on drive_del with virtioscsi + iothread
3405         my $device = parse_drive($deviceid, $conf->{$deviceid});
3406         die "virtioscsi with iothread is not hot-unplugglable currently" if $device->{iothread};
3407
3408         qemu_devicedel($vmid, $deviceid);
3409         qemu_drivedel($vmid, $deviceid);
3410         qemu_deletescsihw($conf, $vmid, $deviceid);
3411
3412     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3413
3414         qemu_devicedel($vmid, $deviceid);
3415         qemu_devicedelverify($vmid, $deviceid);
3416         qemu_netdevdel($vmid, $deviceid);
3417
3418     } else {
3419         die "can't unplug device '$deviceid'\n";
3420     }
3421
3422     return 1;
3423 }
3424
3425 sub qemu_deviceadd {
3426     my ($vmid, $devicefull) = @_;
3427
3428     $devicefull = "driver=".$devicefull;
3429     my %options =  split(/[=,]/, $devicefull);
3430
3431     vm_mon_cmd($vmid, "device_add" , %options);
3432 }
3433
3434 sub qemu_devicedel {
3435     my ($vmid, $deviceid) = @_;
3436
3437     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3438 }
3439
3440 sub qemu_iothread_add {
3441     my($vmid, $deviceid, $device) = @_;
3442
3443     if ($device->{iothread}) {
3444         my $iothreads = vm_iothreads_list($vmid);
3445         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3446     }
3447 }
3448
3449 sub qemu_iothread_del {
3450     my($conf, $vmid, $deviceid) = @_;
3451
3452     my $device = parse_drive($deviceid, $conf->{$deviceid});
3453     if ($device->{iothread}) {
3454         my $iothreads = vm_iothreads_list($vmid);
3455         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3456     }
3457 }
3458
3459 sub qemu_objectadd {
3460     my($vmid, $objectid, $qomtype) = @_;
3461
3462     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3463
3464     return 1;
3465 }
3466
3467 sub qemu_objectdel {
3468     my($vmid, $objectid) = @_;
3469
3470     vm_mon_cmd($vmid, "object-del", id => $objectid);
3471
3472     return 1;
3473 }
3474
3475 sub qemu_driveadd {
3476     my ($storecfg, $vmid, $device) = @_;
3477
3478     my $drive = print_drive_full($storecfg, $vmid, $device);
3479     $drive =~ s/\\/\\\\/g;
3480     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3481
3482     # If the command succeeds qemu prints: "OK"
3483     return 1 if $ret =~ m/OK/s;
3484
3485     die "adding drive failed: $ret\n";
3486 }
3487
3488 sub qemu_drivedel {
3489     my($vmid, $deviceid) = @_;
3490
3491     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3492     $ret =~ s/^\s+//;
3493
3494     return 1 if $ret eq "";
3495
3496     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3497     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3498
3499     die "deleting drive $deviceid failed : $ret\n";
3500 }
3501
3502 sub qemu_deviceaddverify {
3503     my ($vmid, $deviceid) = @_;
3504
3505     for (my $i = 0; $i <= 5; $i++) {
3506          my $devices_list = vm_devices_list($vmid);
3507          return 1 if defined($devices_list->{$deviceid});
3508          sleep 1;
3509     }
3510
3511     die "error on hotplug device '$deviceid'\n";
3512 }
3513
3514
3515 sub qemu_devicedelverify {
3516     my ($vmid, $deviceid) = @_;
3517
3518     # need to verify that the device is correctly removed as device_del
3519     # is async and empty return is not reliable
3520
3521     for (my $i = 0; $i <= 5; $i++) {
3522          my $devices_list = vm_devices_list($vmid);
3523          return 1 if !defined($devices_list->{$deviceid});
3524          sleep 1;
3525     }
3526
3527     die "error on hot-unplugging device '$deviceid'\n";
3528 }
3529
3530 sub qemu_findorcreatescsihw {
3531     my ($storecfg, $conf, $vmid, $device) = @_;
3532
3533     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3534
3535     my $scsihwid="$controller_prefix$controller";
3536     my $devices_list = vm_devices_list($vmid);
3537
3538     if(!defined($devices_list->{$scsihwid})) {
3539         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3540     }
3541
3542     return 1;
3543 }
3544
3545 sub qemu_deletescsihw {
3546     my ($conf, $vmid, $opt) = @_;
3547
3548     my $device = parse_drive($opt, $conf->{$opt});
3549
3550     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3551         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3552         return 1;
3553     }
3554
3555     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3556
3557     my $devices_list = vm_devices_list($vmid);
3558     foreach my $opt (keys %{$devices_list}) {
3559         if (PVE::QemuServer::is_valid_drivename($opt)) {
3560             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3561             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3562                 return 1;
3563             }
3564         }
3565     }
3566
3567     my $scsihwid="scsihw$controller";
3568
3569     vm_deviceunplug($vmid, $conf, $scsihwid);
3570
3571     return 1;
3572 }
3573
3574 sub qemu_add_pci_bridge {
3575     my ($storecfg, $conf, $vmid, $device) = @_;
3576
3577     my $bridges = {};
3578
3579     my $bridgeid;
3580
3581     print_pci_addr($device, $bridges);
3582
3583     while (my ($k, $v) = each %$bridges) {
3584         $bridgeid = $k;
3585     }
3586     return 1 if !defined($bridgeid) || $bridgeid < 1;
3587
3588     my $bridge = "pci.$bridgeid";
3589     my $devices_list = vm_devices_list($vmid);
3590
3591     if (!defined($devices_list->{$bridge})) {
3592         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3593     }
3594
3595     return 1;
3596 }
3597
3598 sub qemu_set_link_status {
3599     my ($vmid, $device, $up) = @_;
3600
3601     vm_mon_cmd($vmid, "set_link", name => $device,
3602                up => $up ? JSON::true : JSON::false);
3603 }
3604
3605 sub qemu_netdevadd {
3606     my ($vmid, $conf, $device, $deviceid) = @_;
3607
3608     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
3609     my %options =  split(/[=,]/, $netdev);
3610
3611     vm_mon_cmd($vmid, "netdev_add",  %options);
3612     return 1;
3613 }
3614
3615 sub qemu_netdevdel {
3616     my ($vmid, $deviceid) = @_;
3617
3618     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3619 }
3620
3621 sub qemu_usb_hotplug {
3622     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3623
3624     return if !$device;
3625
3626     # remove the old one first
3627     vm_deviceunplug($vmid, $conf, $deviceid);
3628
3629     # check if xhci controller is necessary and available
3630     if ($device->{usb3}) {
3631
3632         my $devicelist = vm_devices_list($vmid);
3633
3634         if (!$devicelist->{xhci}) {
3635             my $pciaddr = print_pci_addr("xhci");
3636             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
3637         }
3638     }
3639     my $d = parse_usb_device($device->{host});
3640     $d->{usb3} = $device->{usb3};
3641
3642     # add the new one
3643     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d);
3644 }
3645
3646 sub qemu_cpu_hotplug {
3647     my ($vmid, $conf, $vcpus) = @_;
3648
3649     my $sockets = 1;
3650     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3651     $sockets = $conf->{sockets} if  $conf->{sockets};
3652     my $cores = $conf->{cores} || 1;
3653     my $maxcpus = $sockets * $cores;
3654
3655     $vcpus = $maxcpus if !$vcpus;
3656
3657     die "you can't add more vcpus than maxcpus\n"
3658         if $vcpus > $maxcpus;
3659
3660     my $currentvcpus = $conf->{vcpus} || $maxcpus;
3661     die "online cpu unplug is not yet possible\n"
3662         if $vcpus < $currentvcpus;
3663
3664     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3665     die "vcpus in running vm is different than configuration\n"
3666         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
3667
3668     for (my $i = $currentvcpus; $i < $vcpus; $i++) {
3669         vm_mon_cmd($vmid, "cpu-add", id => int($i));
3670     }
3671 }
3672
3673 sub qemu_block_set_io_throttle {
3674     my ($vmid, $deviceid,
3675         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
3676         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max) = @_;
3677
3678     return if !check_running($vmid) ;
3679
3680     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
3681         bps => int($bps),
3682         bps_rd => int($bps_rd),
3683         bps_wr => int($bps_wr),
3684         iops => int($iops),
3685         iops_rd => int($iops_rd),
3686         iops_wr => int($iops_wr),
3687         bps_max => int($bps_max),
3688         bps_rd_max => int($bps_rd_max),
3689         bps_wr_max => int($bps_wr_max),
3690         iops_max => int($iops_max),
3691         iops_rd_max => int($iops_rd_max),
3692         iops_wr_max => int($iops_wr_max)
3693     );
3694
3695 }
3696
3697 # old code, only used to shutdown old VM after update
3698 sub __read_avail {
3699     my ($fh, $timeout) = @_;
3700
3701     my $sel = new IO::Select;
3702     $sel->add($fh);
3703
3704     my $res = '';
3705     my $buf;
3706
3707     my @ready;
3708     while (scalar (@ready = $sel->can_read($timeout))) {
3709         my $count;
3710         if ($count = $fh->sysread($buf, 8192)) {
3711             if ($buf =~ /^(.*)\(qemu\) $/s) {
3712                 $res .= $1;
3713                 last;
3714             } else {
3715                 $res .= $buf;
3716             }
3717         } else {
3718             if (!defined($count)) {
3719                 die "$!\n";
3720             }
3721             last;
3722         }
3723     }
3724
3725     die "monitor read timeout\n" if !scalar(@ready);
3726
3727     return $res;
3728 }
3729
3730 # old code, only used to shutdown old VM after update
3731 sub vm_monitor_command {
3732     my ($vmid, $cmdstr, $nocheck) = @_;
3733
3734     my $res;
3735
3736     eval {
3737         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
3738
3739         my $sname = "${var_run_tmpdir}/$vmid.mon";
3740
3741         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
3742             die "unable to connect to VM $vmid socket - $!\n";
3743
3744         my $timeout = 3;
3745
3746         # hack: migrate sometime blocks the monitor (when migrate_downtime
3747         # is set)
3748         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3749             $timeout = 60*60; # 1 hour
3750         }
3751
3752         # read banner;
3753         my $data = __read_avail($sock, $timeout);
3754
3755         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
3756             die "got unexpected qemu monitor banner\n";
3757         }
3758
3759         my $sel = new IO::Select;
3760         $sel->add($sock);
3761
3762         if (!scalar(my @ready = $sel->can_write($timeout))) {
3763             die "monitor write error - timeout";
3764         }
3765
3766         my $fullcmd = "$cmdstr\r";
3767
3768         # syslog('info', "VM $vmid monitor command: $cmdstr");
3769
3770         my $b;
3771         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
3772             die "monitor write error - $!";
3773         }
3774
3775         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
3776
3777         $timeout = 20;
3778
3779         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3780             $timeout = 60*60; # 1 hour
3781         } elsif ($cmdstr =~ m/^(eject|change)/) {
3782             $timeout = 60; # note: cdrom mount command is slow
3783         }
3784         if ($res = __read_avail($sock, $timeout)) {
3785
3786             my @lines = split("\r?\n", $res);
3787
3788             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
3789
3790             $res = join("\n", @lines);
3791             $res .= "\n";
3792         }
3793     };
3794
3795     my $err = $@;
3796
3797     if ($err) {
3798         syslog("err", "VM $vmid monitor command failed - $err");
3799         die $err;
3800     }
3801
3802     return $res;
3803 }
3804
3805 sub qemu_block_resize {
3806     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
3807
3808     my $running = check_running($vmid);
3809
3810     return if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
3811
3812     return if !$running;
3813
3814     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
3815
3816 }
3817
3818 sub qemu_volume_snapshot {
3819     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3820
3821     my $running = check_running($vmid);
3822
3823     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
3824         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
3825     } else {
3826         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
3827     }
3828 }
3829
3830 sub qemu_volume_snapshot_delete {
3831     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3832
3833     my $running = check_running($vmid);
3834
3835     return if !PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
3836
3837     return if !$running;
3838
3839     vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
3840 }
3841
3842 sub set_migration_caps {
3843     my ($vmid) = @_;
3844
3845     my $cap_ref = [];
3846
3847     my $enabled_cap = {
3848         "auto-converge" => 1,
3849         "xbzrle" => 1,
3850         "x-rdma-pin-all" => 0,
3851         "zero-blocks" => 0,
3852         "compress" => 0
3853     };
3854
3855     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
3856
3857     for my $supported_capability (@$supported_capabilities) {
3858         push @$cap_ref, {
3859             capability => $supported_capability->{capability},
3860             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
3861         };
3862     }
3863
3864     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
3865 }
3866
3867 my $fast_plug_option = {
3868     'lock' => 1,
3869     'name' => 1,
3870     'onboot' => 1,
3871     'shares' => 1,
3872     'startup' => 1,
3873     'description' => 1,
3874 };
3875
3876 # hotplug changes in [PENDING]
3877 # $selection hash can be used to only apply specified options, for
3878 # example: { cores => 1 } (only apply changed 'cores')
3879 # $errors ref is used to return error messages
3880 sub vmconfig_hotplug_pending {
3881     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
3882
3883     my $defaults = load_defaults();
3884
3885     # commit values which do not have any impact on running VM first
3886     # Note: those option cannot raise errors, we we do not care about
3887     # $selection and always apply them.
3888
3889     my $add_error = sub {
3890         my ($opt, $msg) = @_;
3891         $errors->{$opt} = "hotplug problem - $msg";
3892     };
3893
3894     my $changes = 0;
3895     foreach my $opt (keys %{$conf->{pending}}) { # add/change
3896         if ($fast_plug_option->{$opt}) {
3897             $conf->{$opt} = $conf->{pending}->{$opt};
3898             delete $conf->{pending}->{$opt};
3899             $changes = 1;
3900         }
3901     }
3902
3903     if ($changes) {
3904         PVE::QemuConfig->write_config($vmid, $conf);
3905         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
3906     }
3907
3908     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3909
3910     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
3911     while (my ($opt, $force) = each %$pending_delete_hash) {
3912         next if $selection && !$selection->{$opt};
3913         eval {
3914             if ($opt eq 'hotplug') {
3915                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
3916             } elsif ($opt eq 'tablet') {
3917                 die "skip\n" if !$hotplug_features->{usb};
3918                 if ($defaults->{tablet}) {
3919                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
3920                 } else {
3921                     vm_deviceunplug($vmid, $conf, $opt);
3922                 }
3923             } elsif ($opt =~ m/^usb\d+/) {
3924                 die "skip\n";
3925                 # since we cannot reliably hot unplug usb devices
3926                 # we are disabling it
3927                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
3928                 vm_deviceunplug($vmid, $conf, $opt);
3929             } elsif ($opt eq 'vcpus') {
3930                 die "skip\n" if !$hotplug_features->{cpu};
3931                 qemu_cpu_hotplug($vmid, $conf, undef);
3932             } elsif ($opt eq 'balloon') {
3933                 # enable balloon device is not hotpluggable
3934                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
3935             } elsif ($fast_plug_option->{$opt}) {
3936                 # do nothing
3937             } elsif ($opt =~ m/^net(\d+)$/) {
3938                 die "skip\n" if !$hotplug_features->{network};
3939                 vm_deviceunplug($vmid, $conf, $opt);
3940             } elsif (is_valid_drivename($opt)) {
3941                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
3942                 vm_deviceunplug($vmid, $conf, $opt);
3943                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
3944             } elsif ($opt =~ m/^memory$/) {
3945                 die "skip\n" if !$hotplug_features->{memory};
3946                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
3947             } elsif ($opt eq 'cpuunits') {
3948                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
3949             } elsif ($opt eq 'cpulimit') {
3950                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
3951             } else {
3952                 die "skip\n";
3953             }
3954         };
3955         if (my $err = $@) {
3956             &$add_error($opt, $err) if $err ne "skip\n";
3957         } else {
3958             # save new config if hotplug was successful
3959             delete $conf->{$opt};
3960             vmconfig_undelete_pending_option($conf, $opt);
3961             PVE::QemuConfig->write_config($vmid, $conf);
3962             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
3963         }
3964     }
3965
3966     foreach my $opt (keys %{$conf->{pending}}) {
3967         next if $selection && !$selection->{$opt};
3968         my $value = $conf->{pending}->{$opt};
3969         eval {
3970             if ($opt eq 'hotplug') {
3971                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
3972             } elsif ($opt eq 'tablet') {
3973                 die "skip\n" if !$hotplug_features->{usb};
3974                 if ($value == 1) {
3975                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
3976                 } elsif ($value == 0) {
3977                     vm_deviceunplug($vmid, $conf, $opt);
3978                 }
3979             } elsif ($opt =~ m/^usb\d+$/) {
3980                 die "skip\n";
3981                 # since we cannot reliably hot unplug usb devices
3982                 # we are disabling it
3983                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
3984                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
3985                 die "skip\n" if !$d;
3986                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d);
3987             } elsif ($opt eq 'vcpus') {
3988                 die "skip\n" if !$hotplug_features->{cpu};
3989                 qemu_cpu_hotplug($vmid, $conf, $value);
3990             } elsif ($opt eq 'balloon') {
3991                 # enable/disable balloning device is not hotpluggable
3992                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
3993                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
3994                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
3995
3996                 # allow manual ballooning if shares is set to zero
3997                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
3998                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
3999                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4000                 }
4001             } elsif ($opt =~ m/^net(\d+)$/) {
4002                 # some changes can be done without hotplug
4003                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4004                                     $vmid, $opt, $value);
4005             } elsif (is_valid_drivename($opt)) {
4006                 # some changes can be done without hotplug
4007                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4008                                      $vmid, $opt, $value, 1);
4009             } elsif ($opt =~ m/^memory$/) { #dimms
4010                 die "skip\n" if !$hotplug_features->{memory};
4011                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4012             } elsif ($opt eq 'cpuunits') {
4013                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4014             } elsif ($opt eq 'cpulimit') {
4015                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4016                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4017             } else {
4018                 die "skip\n";  # skip non-hot-pluggable options
4019             }
4020         };
4021         if (my $err = $@) {
4022             &$add_error($opt, $err) if $err ne "skip\n";
4023         } else {
4024             # save new config if hotplug was successful
4025             $conf->{$opt} = $value;
4026             delete $conf->{pending}->{$opt};
4027             PVE::QemuConfig->write_config($vmid, $conf);
4028             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4029         }
4030     }
4031 }
4032
4033 sub try_deallocate_drive {
4034     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4035
4036     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4037         my $volid = $drive->{file};
4038         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4039             my $sid = PVE::Storage::parse_volume_id($volid);
4040             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4041
4042             # check if the disk is really unused
4043             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4044                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4045             PVE::Storage::vdisk_free($storecfg, $volid);
4046             return 1;
4047         } else {
4048             # If vm is not owner of this disk remove from config
4049             return 1;
4050         }
4051     }
4052
4053     return undef;
4054 }
4055
4056 sub vmconfig_delete_or_detach_drive {
4057     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4058
4059     my $drive = parse_drive($opt, $conf->{$opt});
4060
4061     my $rpcenv = PVE::RPCEnvironment::get();
4062     my $authuser = $rpcenv->get_user();
4063
4064     if ($force) {
4065         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4066         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4067     } else {
4068         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4069     }
4070 }
4071
4072 sub vmconfig_apply_pending {
4073     my ($vmid, $conf, $storecfg) = @_;
4074
4075     # cold plug
4076
4077     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4078     while (my ($opt, $force) = each %$pending_delete_hash) {
4079         die "internal error" if $opt =~ m/^unused/;
4080         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4081         if (!defined($conf->{$opt})) {
4082             vmconfig_undelete_pending_option($conf, $opt);
4083             PVE::QemuConfig->write_config($vmid, $conf);
4084         } elsif (is_valid_drivename($opt)) {
4085             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4086             vmconfig_undelete_pending_option($conf, $opt);
4087             delete $conf->{$opt};
4088             PVE::QemuConfig->write_config($vmid, $conf);
4089         } else {
4090             vmconfig_undelete_pending_option($conf, $opt);
4091             delete $conf->{$opt};
4092             PVE::QemuConfig->write_config($vmid, $conf);
4093         }
4094     }
4095
4096     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4097
4098     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4099         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4100
4101         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4102             # skip if nothing changed
4103         } elsif (is_valid_drivename($opt)) {
4104             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4105                 if defined($conf->{$opt});
4106             $conf->{$opt} = $conf->{pending}->{$opt};
4107         } else {
4108             $conf->{$opt} = $conf->{pending}->{$opt};
4109         }
4110
4111         delete $conf->{pending}->{$opt};
4112         PVE::QemuConfig->write_config($vmid, $conf);
4113     }
4114 }
4115
4116 my $safe_num_ne = sub {
4117     my ($a, $b) = @_;
4118
4119     return 0 if !defined($a) && !defined($b);
4120     return 1 if !defined($a);
4121     return 1 if !defined($b);
4122
4123     return $a != $b;
4124 };
4125
4126 my $safe_string_ne = sub {
4127     my ($a, $b) = @_;
4128
4129     return 0 if !defined($a) && !defined($b);
4130     return 1 if !defined($a);
4131     return 1 if !defined($b);
4132
4133     return $a ne $b;
4134 };
4135
4136 sub vmconfig_update_net {
4137     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4138
4139     my $newnet = parse_net($value);
4140
4141     if ($conf->{$opt}) {
4142         my $oldnet = parse_net($conf->{$opt});
4143
4144         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4145             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4146             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4147             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4148
4149             # for non online change, we try to hot-unplug
4150             die "skip\n" if !$hotplug;
4151             vm_deviceunplug($vmid, $conf, $opt);
4152         } else {
4153
4154             die "internal error" if $opt !~ m/net(\d+)/;
4155             my $iface = "tap${vmid}i$1";
4156
4157             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4158                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4159                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4160                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4161                 PVE::Network::tap_unplug($iface);
4162                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4163             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4164                 # Rate can be applied on its own but any change above needs to
4165                 # include the rate in tap_plug since OVS resets everything.
4166                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4167             }
4168
4169             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4170                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4171             }
4172
4173             return 1;
4174         }
4175     }
4176
4177     if ($hotplug) {
4178         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4179     } else {
4180         die "skip\n";
4181     }
4182 }
4183
4184 sub vmconfig_update_disk {
4185     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4186
4187     # fixme: do we need force?
4188
4189     my $drive = parse_drive($opt, $value);
4190
4191     if ($conf->{$opt}) {
4192
4193         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4194
4195             my $media = $drive->{media} || 'disk';
4196             my $oldmedia = $old_drive->{media} || 'disk';
4197             die "unable to change media type\n" if $media ne $oldmedia;
4198
4199             if (!drive_is_cdrom($old_drive)) {
4200
4201                 if ($drive->{file} ne $old_drive->{file}) {
4202
4203                     die "skip\n" if !$hotplug;
4204
4205                     # unplug and register as unused
4206                     vm_deviceunplug($vmid, $conf, $opt);
4207                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4208
4209                 } else {
4210                     # update existing disk
4211
4212                     # skip non hotpluggable value
4213                     if (&$safe_num_ne($drive->{discard}, $old_drive->{discard}) ||
4214                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4215                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4216                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4217                         die "skip\n";
4218                     }
4219
4220                     # apply throttle
4221                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4222                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4223                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4224                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4225                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4226                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4227                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4228                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4229                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4230                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4231                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4232                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max})) {
4233
4234                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4235                                                    ($drive->{mbps} || 0)*1024*1024,
4236                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4237                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4238                                                    $drive->{iops} || 0,
4239                                                    $drive->{iops_rd} || 0,
4240                                                    $drive->{iops_wr} || 0,
4241                                                    ($drive->{mbps_max} || 0)*1024*1024,
4242                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4243                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4244                                                    $drive->{iops_max} || 0,
4245                                                    $drive->{iops_rd_max} || 0,
4246                                                    $drive->{iops_wr_max} || 0);
4247
4248                     }
4249
4250                     return 1;
4251                 }
4252
4253             } else { # cdrom
4254
4255                 if ($drive->{file} eq 'none') {
4256                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4257                 } else {
4258                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4259                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4260                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4261                 }
4262
4263                 return 1;
4264             }
4265         }
4266     }
4267
4268     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4269     # hotplug new disks
4270     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4271     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4272 }
4273
4274 sub vm_start {
4275     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4276         $forcemachine, $spice_ticket) = @_;
4277
4278     PVE::QemuConfig->lock_config($vmid, sub {
4279         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4280
4281         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
4282
4283         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4284
4285         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4286
4287         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4288             vmconfig_apply_pending($vmid, $conf, $storecfg);
4289             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4290         }
4291
4292         my $defaults = load_defaults();
4293
4294         # set environment variable useful inside network script
4295         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4296
4297         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4298
4299         my $migrate_port = 0;
4300         my $migrate_uri;
4301         if ($statefile) {
4302             if ($statefile eq 'tcp') {
4303                 my $localip = "localhost";
4304                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4305                 my $nodename = PVE::INotify::nodename();
4306                 if ($datacenterconf->{migration_unsecure}) {
4307                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4308                         $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4309                 }
4310                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4311                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4312                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4313                 push @$cmd, '-incoming', $migrate_uri;
4314                 push @$cmd, '-S';
4315
4316             } elsif ($statefile eq 'unix') {
4317                 # should be default for secure migrations as a ssh TCP forward
4318                 # tunnel is not deterministic reliable ready and fails regurarly
4319                 # to set up in time, so use UNIX socket forwards
4320                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
4321                 unlink $socket_addr;
4322
4323                 $migrate_uri = "unix:$socket_addr";
4324
4325                 push @$cmd, '-incoming', $migrate_uri;
4326                 push @$cmd, '-S';
4327
4328             } else {
4329                 push @$cmd, '-loadstate', $statefile;
4330             }
4331         } elsif ($paused) {
4332             push @$cmd, '-S';
4333         }
4334
4335         # host pci devices
4336         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4337           my $d = parse_hostpci($conf->{"hostpci$i"});
4338           next if !$d;
4339           my $pcidevices = $d->{pciid};
4340           foreach my $pcidevice (@$pcidevices) {
4341                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4342
4343                 my $info = pci_device_info("0000:$pciid");
4344                 die "IOMMU not present\n" if !check_iommu_support();
4345                 die "no pci device info for device '$pciid'\n" if !$info;
4346                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4347                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4348           }
4349         }
4350
4351         PVE::Storage::activate_volumes($storecfg, $vollist);
4352
4353         if (!check_running($vmid, 1) && -d "/sys/fs/cgroup/systemd/qemu.slice/$vmid.scope") {
4354             my $cmd = [];
4355             push @$cmd, '/bin/systemctl', 'stop', "$vmid.scope";
4356             eval  { run_command($cmd); };
4357         }
4358
4359         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
4360                                                   : $defaults->{cpuunits};
4361
4362         my %run_params = (timeout => $statefile ? undef : 30, umask => 0077);
4363
4364         my %properties = (
4365             Slice => 'qemu.slice',
4366             KillMode => 'none',
4367             CPUShares => $cpuunits
4368         );
4369
4370         if (my $cpulimit = $conf->{cpulimit}) {
4371             $properties{CPUQuota} = int($cpulimit * 100);
4372         }
4373         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
4374
4375         if ($conf->{hugepages}) {
4376
4377             my $code = sub {
4378                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
4379                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
4380
4381                 PVE::QemuServer::Memory::hugepages_mount();
4382                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
4383
4384                 eval  {
4385                     PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4386                     run_command($cmd, %run_params);
4387                 };
4388
4389                 if (my $err = $@) {
4390                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
4391                     die $err;
4392                 }
4393
4394                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
4395             };
4396             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
4397
4398         } else {
4399             eval  {
4400                 PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4401                 run_command($cmd, %run_params);
4402             };
4403         }
4404
4405         if (my $err = $@) {
4406             # deactivate volumes if start fails
4407             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
4408             die "start failed: $err";
4409         }
4410
4411         print "migration listens on $migrate_uri\n" if $migrate_uri;
4412
4413         if ($statefile && $statefile ne 'tcp')  {
4414             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4415             warn $@ if $@;
4416         }
4417
4418         if ($migratedfrom) {
4419
4420             eval {
4421                 set_migration_caps($vmid);
4422             };
4423             warn $@ if $@;
4424
4425             if ($spice_port) {
4426                 print "spice listens on port $spice_port\n";
4427                 if ($spice_ticket) {
4428                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4429                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4430                 }
4431             }
4432
4433         } else {
4434
4435             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4436                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4437                     if $conf->{balloon};
4438             }
4439
4440             foreach my $opt (keys %$conf) {
4441                 next if $opt !~  m/^net\d+$/;
4442                 my $nicconf = parse_net($conf->{$opt});
4443                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4444             }
4445         }
4446
4447         vm_mon_cmd_nocheck($vmid, 'qom-set',
4448                     path => "machine/peripheral/balloon0",
4449                     property => "guest-stats-polling-interval",
4450                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4451
4452     });
4453 }
4454
4455 sub vm_mon_cmd {
4456     my ($vmid, $execute, %params) = @_;
4457
4458     my $cmd = { execute => $execute, arguments => \%params };
4459     vm_qmp_command($vmid, $cmd);
4460 }
4461
4462 sub vm_mon_cmd_nocheck {
4463     my ($vmid, $execute, %params) = @_;
4464
4465     my $cmd = { execute => $execute, arguments => \%params };
4466     vm_qmp_command($vmid, $cmd, 1);
4467 }
4468
4469 sub vm_qmp_command {
4470     my ($vmid, $cmd, $nocheck) = @_;
4471
4472     my $res;
4473
4474     my $timeout;
4475     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4476         $timeout = $cmd->{arguments}->{timeout};
4477         delete $cmd->{arguments}->{timeout};
4478     }
4479
4480     eval {
4481         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4482         my $sname = qmp_socket($vmid);
4483         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4484             my $qmpclient = PVE::QMPClient->new();
4485
4486             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4487         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
4488             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
4489                 if scalar(%{$cmd->{arguments}});
4490             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
4491         } else {
4492             die "unable to open monitor socket\n";
4493         }
4494     };
4495     if (my $err = $@) {
4496         syslog("err", "VM $vmid qmp command failed - $err");
4497         die $err;
4498     }
4499
4500     return $res;
4501 }
4502
4503 sub vm_human_monitor_command {
4504     my ($vmid, $cmdline) = @_;
4505
4506     my $res;
4507
4508     my $cmd = {
4509         execute => 'human-monitor-command',
4510         arguments => { 'command-line' => $cmdline},
4511     };
4512
4513     return vm_qmp_command($vmid, $cmd);
4514 }
4515
4516 sub vm_commandline {
4517     my ($storecfg, $vmid) = @_;
4518
4519     my $conf = PVE::QemuConfig->load_config($vmid);
4520
4521     my $defaults = load_defaults();
4522
4523     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
4524
4525     return join(' ', @$cmd);
4526 }
4527
4528 sub vm_reset {
4529     my ($vmid, $skiplock) = @_;
4530
4531     PVE::QemuConfig->lock_config($vmid, sub {
4532
4533         my $conf = PVE::QemuConfig->load_config($vmid);
4534
4535         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4536
4537         vm_mon_cmd($vmid, "system_reset");
4538     });
4539 }
4540
4541 sub get_vm_volumes {
4542     my ($conf) = @_;
4543
4544     my $vollist = [];
4545     foreach_volid($conf, sub {
4546         my ($volid, $is_cdrom) = @_;
4547
4548         return if $volid =~ m|^/|;
4549
4550         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4551         return if !$sid;
4552
4553         push @$vollist, $volid;
4554     });
4555
4556     return $vollist;
4557 }
4558
4559 sub vm_stop_cleanup {
4560     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
4561
4562     eval {
4563
4564         if (!$keepActive) {
4565             my $vollist = get_vm_volumes($conf);
4566             PVE::Storage::deactivate_volumes($storecfg, $vollist);
4567         }
4568
4569         foreach my $ext (qw(mon qmp pid vnc qga)) {
4570             unlink "/var/run/qemu-server/${vmid}.$ext";
4571         }
4572
4573         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
4574     };
4575     warn $@ if $@; # avoid errors - just warn
4576 }
4577
4578 # Note: use $nockeck to skip tests if VM configuration file exists.
4579 # We need that when migration VMs to other nodes (files already moved)
4580 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
4581 sub vm_stop {
4582     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
4583
4584     $force = 1 if !defined($force) && !$shutdown;
4585
4586     if ($migratedfrom){
4587         my $pid = check_running($vmid, $nocheck, $migratedfrom);
4588         kill 15, $pid if $pid;
4589         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4590         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
4591         return;
4592     }
4593
4594     PVE::QemuConfig->lock_config($vmid, sub {
4595
4596         my $pid = check_running($vmid, $nocheck);
4597         return if !$pid;
4598
4599         my $conf;
4600         if (!$nocheck) {
4601             $conf = PVE::QemuConfig->load_config($vmid);
4602             PVE::QemuConfig->check_lock($conf) if !$skiplock;
4603             if (!defined($timeout) && $shutdown && $conf->{startup}) {
4604                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
4605                 $timeout = $opts->{down} if $opts->{down};
4606             }
4607         }
4608
4609         $timeout = 60 if !defined($timeout);
4610
4611         eval {
4612             if ($shutdown) {
4613                 if (defined($conf) && $conf->{agent}) {
4614                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
4615                 } else {
4616                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
4617                 }
4618             } else {
4619                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
4620             }
4621         };
4622         my $err = $@;
4623
4624         if (!$err) {
4625             my $count = 0;
4626             while (($count < $timeout) && check_running($vmid, $nocheck)) {
4627                 $count++;
4628                 sleep 1;
4629             }
4630
4631             if ($count >= $timeout) {
4632                 if ($force) {
4633                     warn "VM still running - terminating now with SIGTERM\n";
4634                     kill 15, $pid;
4635                 } else {
4636                     die "VM quit/powerdown failed - got timeout\n";
4637                 }
4638             } else {
4639                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4640                 return;
4641             }
4642         } else {
4643             if ($force) {
4644                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
4645                 kill 15, $pid;
4646             } else {
4647                 die "VM quit/powerdown failed\n";
4648             }
4649         }
4650
4651         # wait again
4652         $timeout = 10;
4653
4654         my $count = 0;
4655         while (($count < $timeout) && check_running($vmid, $nocheck)) {
4656             $count++;
4657             sleep 1;
4658         }
4659
4660         if ($count >= $timeout) {
4661             warn "VM still running - terminating now with SIGKILL\n";
4662             kill 9, $pid;
4663             sleep 1;
4664         }
4665
4666         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4667    });
4668 }
4669
4670 sub vm_suspend {
4671     my ($vmid, $skiplock) = @_;
4672
4673     PVE::QemuConfig->lock_config($vmid, sub {
4674
4675         my $conf = PVE::QemuConfig->load_config($vmid);
4676
4677         PVE::QemuConfig->check_lock($conf)
4678             if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
4679
4680         vm_mon_cmd($vmid, "stop");
4681     });
4682 }
4683
4684 sub vm_resume {
4685     my ($vmid, $skiplock, $nocheck) = @_;
4686
4687     PVE::QemuConfig->lock_config($vmid, sub {
4688
4689         if (!$nocheck) {
4690
4691             my $conf = PVE::QemuConfig->load_config($vmid);
4692
4693             PVE::QemuConfig->check_lock($conf)
4694                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
4695
4696             vm_mon_cmd($vmid, "cont");
4697
4698         } else {
4699             vm_mon_cmd_nocheck($vmid, "cont");
4700         }
4701     });
4702 }
4703
4704 sub vm_sendkey {
4705     my ($vmid, $skiplock, $key) = @_;
4706
4707     PVE::QemuConfig->lock_config($vmid, sub {
4708
4709         my $conf = PVE::QemuConfig->load_config($vmid);
4710
4711         # there is no qmp command, so we use the human monitor command
4712         vm_human_monitor_command($vmid, "sendkey $key");
4713     });
4714 }
4715
4716 sub vm_destroy {
4717     my ($storecfg, $vmid, $skiplock) = @_;
4718
4719     PVE::QemuConfig->lock_config($vmid, sub {
4720
4721         my $conf = PVE::QemuConfig->load_config($vmid);
4722
4723         if (!check_running($vmid)) {
4724             destroy_vm($storecfg, $vmid, undef, $skiplock);
4725         } else {
4726             die "VM $vmid is running - destroy failed\n";
4727         }
4728     });
4729 }
4730
4731 # pci helpers
4732
4733 sub file_write {
4734     my ($filename, $buf) = @_;
4735
4736     my $fh = IO::File->new($filename, "w");
4737     return undef if !$fh;
4738
4739     my $res = print $fh $buf;
4740
4741     $fh->close();
4742
4743     return $res;
4744 }
4745
4746 sub pci_device_info {
4747     my ($name) = @_;
4748
4749     my $res;
4750
4751     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
4752     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
4753
4754     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
4755     return undef if !defined($irq) || $irq !~ m/^\d+$/;
4756
4757     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
4758     return undef if !defined($vendor) || $vendor !~ s/^0x//;
4759
4760     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
4761     return undef if !defined($product) || $product !~ s/^0x//;
4762
4763     $res = {
4764         name => $name,
4765         vendor => $vendor,
4766         product => $product,
4767         domain => $domain,
4768         bus => $bus,
4769         slot => $slot,
4770         func => $func,
4771         irq => $irq,
4772         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
4773     };
4774
4775     return $res;
4776 }
4777
4778 sub pci_dev_reset {
4779     my ($dev) = @_;
4780
4781     my $name = $dev->{name};
4782
4783     my $fn = "$pcisysfs/devices/$name/reset";
4784
4785     return file_write($fn, "1");
4786 }
4787
4788 sub pci_dev_bind_to_vfio {
4789     my ($dev) = @_;
4790
4791     my $name = $dev->{name};
4792
4793     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4794
4795     if (!-d $vfio_basedir) {
4796         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4797     }
4798     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4799
4800     my $testdir = "$vfio_basedir/$name";
4801     return 1 if -d $testdir;
4802
4803     my $data = "$dev->{vendor} $dev->{product}";
4804     return undef if !file_write("$vfio_basedir/new_id", $data);
4805
4806     my $fn = "$pcisysfs/devices/$name/driver/unbind";
4807     if (!file_write($fn, $name)) {
4808         return undef if -f $fn;
4809     }
4810
4811     $fn = "$vfio_basedir/bind";
4812     if (! -d $testdir) {
4813         return undef if !file_write($fn, $name);
4814     }
4815
4816     return -d $testdir;
4817 }
4818
4819 sub pci_dev_group_bind_to_vfio {
4820     my ($pciid) = @_;
4821
4822     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4823
4824     if (!-d $vfio_basedir) {
4825         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4826     }
4827     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4828
4829     # get IOMMU group devices
4830     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
4831       my @devs = grep /^0000:/, readdir($D);
4832     closedir($D);
4833
4834     foreach my $pciid (@devs) {
4835         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
4836
4837         # pci bridges, switches or root ports are not supported
4838         # they have a pci_bus subdirectory so skip them
4839         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
4840
4841         my $info = pci_device_info($1);
4842         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
4843     }
4844
4845     return 1;
4846 }
4847
4848 # vzdump restore implementaion
4849
4850 sub tar_archive_read_firstfile {
4851     my $archive = shift;
4852
4853     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
4854
4855     # try to detect archive type first
4856     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
4857         die "unable to open file '$archive'\n";
4858     my $firstfile = <$fh>;
4859     kill 15, $pid;
4860     close $fh;
4861
4862     die "ERROR: archive contaions no data\n" if !$firstfile;
4863     chomp $firstfile;
4864
4865     return $firstfile;
4866 }
4867
4868 sub tar_restore_cleanup {
4869     my ($storecfg, $statfile) = @_;
4870
4871     print STDERR "starting cleanup\n";
4872
4873     if (my $fd = IO::File->new($statfile, "r")) {
4874         while (defined(my $line = <$fd>)) {
4875             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
4876                 my $volid = $2;
4877                 eval {
4878                     if ($volid =~ m|^/|) {
4879                         unlink $volid || die 'unlink failed\n';
4880                     } else {
4881                         PVE::Storage::vdisk_free($storecfg, $volid);
4882                     }
4883                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
4884                 };
4885                 print STDERR "unable to cleanup '$volid' - $@" if $@;
4886             } else {
4887                 print STDERR "unable to parse line in statfile - $line";
4888             }
4889         }
4890         $fd->close();
4891     }
4892 }
4893
4894 sub restore_archive {
4895     my ($archive, $vmid, $user, $opts) = @_;
4896
4897     my $format = $opts->{format};
4898     my $comp;
4899
4900     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
4901         $format = 'tar' if !$format;
4902         $comp = 'gzip';
4903     } elsif ($archive =~ m/\.tar$/) {
4904         $format = 'tar' if !$format;
4905     } elsif ($archive =~ m/.tar.lzo$/) {
4906         $format = 'tar' if !$format;
4907         $comp = 'lzop';
4908     } elsif ($archive =~ m/\.vma$/) {
4909         $format = 'vma' if !$format;
4910     } elsif ($archive =~ m/\.vma\.gz$/) {
4911         $format = 'vma' if !$format;
4912         $comp = 'gzip';
4913     } elsif ($archive =~ m/\.vma\.lzo$/) {
4914         $format = 'vma' if !$format;
4915         $comp = 'lzop';
4916     } else {
4917         $format = 'vma' if !$format; # default
4918     }
4919
4920     # try to detect archive format
4921     if ($format eq 'tar') {
4922         return restore_tar_archive($archive, $vmid, $user, $opts);
4923     } else {
4924         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
4925     }
4926 }
4927
4928 sub restore_update_config_line {
4929     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
4930
4931     return if $line =~ m/^\#qmdump\#/;
4932     return if $line =~ m/^\#vzdump\#/;
4933     return if $line =~ m/^lock:/;
4934     return if $line =~ m/^unused\d+:/;
4935     return if $line =~ m/^parent:/;
4936     return if $line =~ m/^template:/; # restored VM is never a template
4937
4938     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
4939     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
4940         # try to convert old 1.X settings
4941         my ($id, $ind, $ethcfg) = ($1, $2, $3);
4942         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
4943             my ($model, $macaddr) = split(/\=/, $devconfig);
4944             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
4945             my $net = {
4946                 model => $model,
4947                 bridge => "vmbr$ind",
4948                 macaddr => $macaddr,
4949             };
4950             my $netstr = print_net($net);
4951
4952             print $outfd "net$cookie->{netcount}: $netstr\n";
4953             $cookie->{netcount}++;
4954         }
4955     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
4956         my ($id, $netstr) = ($1, $2);
4957         my $net = parse_net($netstr);
4958         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
4959         $netstr = print_net($net);
4960         print $outfd "$id: $netstr\n";
4961     } elsif ($line =~ m/^((ide|scsi|virtio|sata)\d+):\s*(\S+)\s*$/) {
4962         my $virtdev = $1;
4963         my $value = $3;
4964         my $di = parse_drive($virtdev, $value);
4965         if (defined($di->{backup}) && !$di->{backup}) {
4966             print $outfd "#$line";
4967         } elsif ($map->{$virtdev}) {
4968             delete $di->{format}; # format can change on restore
4969             $di->{file} = $map->{$virtdev};
4970             $value = print_drive($vmid, $di);
4971             print $outfd "$virtdev: $value\n";
4972         } else {
4973             print $outfd $line;
4974         }
4975     } else {
4976         print $outfd $line;
4977     }
4978 }
4979
4980 sub scan_volids {
4981     my ($cfg, $vmid) = @_;
4982
4983     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
4984
4985     my $volid_hash = {};
4986     foreach my $storeid (keys %$info) {
4987         foreach my $item (@{$info->{$storeid}}) {
4988             next if !($item->{volid} && $item->{size});
4989             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
4990             $volid_hash->{$item->{volid}} = $item;
4991         }
4992     }
4993
4994     return $volid_hash;
4995 }
4996
4997 sub is_volume_in_use {
4998     my ($storecfg, $conf, $skip_drive, $volid) = @_;
4999
5000     my $path = PVE::Storage::path($storecfg, $volid);
5001
5002     my $scan_config = sub {
5003         my ($cref, $snapname) = @_;
5004
5005         foreach my $key (keys %$cref) {
5006             my $value = $cref->{$key};
5007             if (is_valid_drivename($key)) {
5008                 next if $skip_drive && $key eq $skip_drive;
5009                 my $drive = parse_drive($key, $value);
5010                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5011                 return 1 if $volid eq $drive->{file};
5012                 if ($drive->{file} =~ m!^/!) {
5013                     return 1 if $drive->{file} eq $path;
5014                 } else {
5015                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5016                     next if !$storeid;
5017                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5018                     next if !$scfg;
5019                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5020                 }
5021             }
5022         }
5023
5024         return 0;
5025     };
5026
5027     return 1 if &$scan_config($conf);
5028
5029     undef $skip_drive;
5030
5031     foreach my $snapname (keys %{$conf->{snapshots}}) {
5032         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5033     }
5034
5035     return 0;
5036 }
5037
5038 sub update_disksize {
5039     my ($vmid, $conf, $volid_hash) = @_;
5040
5041     my $changes;
5042
5043     my $used = {};
5044
5045     # Note: it is allowed to define multiple storages with same path (alias), so
5046     # we need to check both 'volid' and real 'path' (two different volid can point
5047     # to the same path).
5048
5049     my $usedpath = {};
5050
5051     # update size info
5052     foreach my $opt (keys %$conf) {
5053         if (is_valid_drivename($opt)) {
5054             my $drive = parse_drive($opt, $conf->{$opt});
5055             my $volid = $drive->{file};
5056             next if !$volid;
5057
5058             $used->{$volid} = 1;
5059             if ($volid_hash->{$volid} &&
5060                 (my $path = $volid_hash->{$volid}->{path})) {
5061                 $usedpath->{$path} = 1;
5062             }
5063
5064             next if drive_is_cdrom($drive);
5065             next if !$volid_hash->{$volid};
5066
5067             $drive->{size} = $volid_hash->{$volid}->{size};
5068             my $new = print_drive($vmid, $drive);
5069             if ($new ne $conf->{$opt}) {
5070                 $changes = 1;
5071                 $conf->{$opt} = $new;
5072             }
5073         }
5074     }
5075
5076     # remove 'unusedX' entry if volume is used
5077     foreach my $opt (keys %$conf) {
5078         next if $opt !~ m/^unused\d+$/;
5079         my $volid = $conf->{$opt};
5080         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5081         if ($used->{$volid} || ($path && $usedpath->{$path})) {
5082             $changes = 1;
5083             delete $conf->{$opt};
5084         }
5085     }
5086
5087     foreach my $volid (sort keys %$volid_hash) {
5088         next if $volid =~ m/vm-$vmid-state-/;
5089         next if $used->{$volid};
5090         my $path = $volid_hash->{$volid}->{path};
5091         next if !$path; # just to be sure
5092         next if $usedpath->{$path};
5093         $changes = 1;
5094         PVE::QemuConfig->add_unused_volume($conf, $volid);
5095         $usedpath->{$path} = 1; # avoid to add more than once (aliases)
5096     }
5097
5098     return $changes;
5099 }
5100
5101 sub rescan {
5102     my ($vmid, $nolock) = @_;
5103
5104     my $cfg = PVE::Storage::config();
5105
5106     my $volid_hash = scan_volids($cfg, $vmid);
5107
5108     my $updatefn =  sub {
5109         my ($vmid) = @_;
5110
5111         my $conf = PVE::QemuConfig->load_config($vmid);
5112
5113         PVE::QemuConfig->check_lock($conf);
5114
5115         my $vm_volids = {};
5116         foreach my $volid (keys %$volid_hash) {
5117             my $info = $volid_hash->{$volid};
5118             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5119         }
5120
5121         my $changes = update_disksize($vmid, $conf, $vm_volids);
5122
5123         PVE::QemuConfig->write_config($vmid, $conf) if $changes;
5124     };
5125
5126     if (defined($vmid)) {
5127         if ($nolock) {
5128             &$updatefn($vmid);
5129         } else {
5130             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5131         }
5132     } else {
5133         my $vmlist = config_list();
5134         foreach my $vmid (keys %$vmlist) {
5135             if ($nolock) {
5136                 &$updatefn($vmid);
5137             } else {
5138                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5139             }
5140         }
5141     }
5142 }
5143
5144 sub restore_vma_archive {
5145     my ($archive, $vmid, $user, $opts, $comp) = @_;
5146
5147     my $input = $archive eq '-' ? "<&STDIN" : undef;
5148     my $readfrom = $archive;
5149
5150     my $uncomp = '';
5151     if ($comp) {
5152         $readfrom = '-';
5153         my $qarchive = PVE::Tools::shellquote($archive);
5154         if ($comp eq 'gzip') {
5155             $uncomp = "zcat $qarchive|";
5156         } elsif ($comp eq 'lzop') {
5157             $uncomp = "lzop -d -c $qarchive|";
5158         } else {
5159             die "unknown compression method '$comp'\n";
5160         }
5161
5162     }
5163
5164     my $tmpdir = "/var/tmp/vzdumptmp$$";
5165     rmtree $tmpdir;
5166
5167     # disable interrupts (always do cleanups)
5168     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5169         warn "got interrupt - ignored\n";
5170     };
5171
5172     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5173     POSIX::mkfifo($mapfifo, 0600);
5174     my $fifofh;
5175
5176     my $openfifo = sub {
5177         open($fifofh, '>', $mapfifo) || die $!;
5178     };
5179
5180     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5181
5182     my $oldtimeout;
5183     my $timeout = 5;
5184
5185     my $devinfo = {};
5186
5187     my $rpcenv = PVE::RPCEnvironment::get();
5188
5189     my $conffile = PVE::QemuConfig->config_file($vmid);
5190     my $tmpfn = "$conffile.$$.tmp";
5191
5192     # Note: $oldconf is undef if VM does not exists
5193     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
5194     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
5195
5196     my $print_devmap = sub {
5197         my $virtdev_hash = {};
5198
5199         my $cfgfn = "$tmpdir/qemu-server.conf";
5200
5201         # we can read the config - that is already extracted
5202         my $fh = IO::File->new($cfgfn, "r") ||
5203             "unable to read qemu-server.conf - $!\n";
5204
5205         my $fwcfgfn = "$tmpdir/qemu-server.fw";
5206         if (-f $fwcfgfn) {
5207             my $pve_firewall_dir = '/etc/pve/firewall';
5208             mkdir $pve_firewall_dir; # make sure the dir exists
5209             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
5210         }
5211
5212         while (defined(my $line = <$fh>)) {
5213             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5214                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5215                 die "archive does not contain data for drive '$virtdev'\n"
5216                     if !$devinfo->{$devname};
5217                 if (defined($opts->{storage})) {
5218                     $storeid = $opts->{storage} || 'local';
5219                 } elsif (!$storeid) {
5220                     $storeid = 'local';
5221                 }
5222                 $format = 'raw' if !$format;
5223                 $devinfo->{$devname}->{devname} = $devname;
5224                 $devinfo->{$devname}->{virtdev} = $virtdev;
5225                 $devinfo->{$devname}->{format} = $format;
5226                 $devinfo->{$devname}->{storeid} = $storeid;
5227
5228                 # check permission on storage
5229                 my $pool = $opts->{pool}; # todo: do we need that?
5230                 if ($user ne 'root@pam') {
5231                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5232                 }
5233
5234                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5235             }
5236         }
5237
5238         foreach my $devname (keys %$devinfo) {
5239             die "found no device mapping information for device '$devname'\n"
5240                 if !$devinfo->{$devname}->{virtdev};
5241         }
5242
5243         my $cfg = PVE::Storage::config();
5244
5245         # create empty/temp config
5246         if ($oldconf) {
5247             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5248             foreach_drive($oldconf, sub {
5249                 my ($ds, $drive) = @_;
5250
5251                 return if drive_is_cdrom($drive);
5252
5253                 my $volid = $drive->{file};
5254
5255                 return if !$volid || $volid =~ m|^/|;
5256
5257                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5258                 return if !$path || !$owner || ($owner != $vmid);
5259
5260                 # Note: only delete disk we want to restore
5261                 # other volumes will become unused
5262                 if ($virtdev_hash->{$ds}) {
5263                     PVE::Storage::vdisk_free($cfg, $volid);
5264                 }
5265             });
5266
5267             # delete vmstate files
5268             # since after the restore we have no snapshots anymore
5269             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5270                 my $snap = $oldconf->{snapshots}->{$snapname};
5271                 if ($snap->{vmstate}) {
5272                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
5273                     if (my $err = $@) {
5274                         warn $err;
5275                     }
5276                 }
5277             }
5278         }
5279
5280         my $map = {};
5281         foreach my $virtdev (sort keys %$virtdev_hash) {
5282             my $d = $virtdev_hash->{$virtdev};
5283             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5284             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5285
5286             # test if requested format is supported
5287             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5288             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5289             $d->{format} = $defFormat if !$supported;
5290
5291             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5292                                                   $d->{format}, undef, $alloc_size);
5293             print STDERR "new volume ID is '$volid'\n";
5294             $d->{volid} = $volid;
5295             my $path = PVE::Storage::path($cfg, $volid);
5296
5297             PVE::Storage::activate_volumes($cfg,[$volid]);
5298
5299             my $write_zeros = 1;
5300             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
5301                 $write_zeros = 0;
5302             }
5303
5304             print $fifofh "format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
5305
5306             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5307             $map->{$virtdev} = $volid;
5308         }
5309
5310         $fh->seek(0, 0) || die "seek failed - $!\n";
5311
5312         my $outfd = new IO::File ($tmpfn, "w") ||
5313             die "unable to write config for VM $vmid\n";
5314
5315         my $cookie = { netcount => 0 };
5316         while (defined(my $line = <$fh>)) {
5317             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5318         }
5319
5320         $fh->close();
5321         $outfd->close();
5322     };
5323
5324     eval {
5325         # enable interrupts
5326         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5327             die "interrupted by signal\n";
5328         };
5329         local $SIG{ALRM} = sub { die "got timeout\n"; };
5330
5331         $oldtimeout = alarm($timeout);
5332
5333         my $parser = sub {
5334             my $line = shift;
5335
5336             print "$line\n";
5337
5338             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5339                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5340                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5341             } elsif ($line =~ m/^CTIME: /) {
5342                 # we correctly received the vma config, so we can disable
5343                 # the timeout now for disk allocation (set to 10 minutes, so
5344                 # that we always timeout if something goes wrong)
5345                 alarm(600);
5346                 &$print_devmap();
5347                 print $fifofh "done\n";
5348                 my $tmp = $oldtimeout || 0;
5349                 $oldtimeout = undef;
5350                 alarm($tmp);
5351                 close($fifofh);
5352             }
5353         };
5354
5355         print "restore vma archive: $cmd\n";
5356         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5357     };
5358     my $err = $@;
5359
5360     alarm($oldtimeout) if $oldtimeout;
5361
5362     my $vollist = [];
5363     foreach my $devname (keys %$devinfo) {
5364         my $volid = $devinfo->{$devname}->{volid};
5365         push @$vollist, $volid if $volid;
5366     }
5367
5368     my $cfg = PVE::Storage::config();
5369     PVE::Storage::deactivate_volumes($cfg, $vollist);
5370
5371     unlink $mapfifo;
5372
5373     if ($err) {
5374         rmtree $tmpdir;
5375         unlink $tmpfn;
5376
5377         foreach my $devname (keys %$devinfo) {
5378             my $volid = $devinfo->{$devname}->{volid};
5379             next if !$volid;
5380             eval {
5381                 if ($volid =~ m|^/|) {
5382                     unlink $volid || die 'unlink failed\n';
5383                 } else {
5384                     PVE::Storage::vdisk_free($cfg, $volid);
5385                 }
5386                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5387             };
5388             print STDERR "unable to cleanup '$volid' - $@" if $@;
5389         }
5390         die $err;
5391     }
5392
5393     rmtree $tmpdir;
5394
5395     rename($tmpfn, $conffile) ||
5396         die "unable to commit configuration file '$conffile'\n";
5397
5398     PVE::Cluster::cfs_update(); # make sure we read new file
5399
5400     eval { rescan($vmid, 1); };
5401     warn $@ if $@;
5402 }
5403
5404 sub restore_tar_archive {
5405     my ($archive, $vmid, $user, $opts) = @_;
5406
5407     if ($archive ne '-') {
5408         my $firstfile = tar_archive_read_firstfile($archive);
5409         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5410             if $firstfile ne 'qemu-server.conf';
5411     }
5412
5413     my $storecfg = PVE::Storage::config();
5414
5415     # destroy existing data - keep empty config
5416     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
5417     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5418
5419     my $tocmd = "/usr/lib/qemu-server/qmextract";
5420
5421     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5422     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5423     $tocmd .= ' --prealloc' if $opts->{prealloc};
5424     $tocmd .= ' --info' if $opts->{info};
5425
5426     # tar option "xf" does not autodetect compression when read from STDIN,
5427     # so we pipe to zcat
5428     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5429         PVE::Tools::shellquote("--to-command=$tocmd");
5430
5431     my $tmpdir = "/var/tmp/vzdumptmp$$";
5432     mkpath $tmpdir;
5433
5434     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5435     local $ENV{VZDUMP_VMID} = $vmid;
5436     local $ENV{VZDUMP_USER} = $user;
5437
5438     my $conffile = PVE::QemuConfig->config_file($vmid);
5439     my $tmpfn = "$conffile.$$.tmp";
5440
5441     # disable interrupts (always do cleanups)
5442     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5443         print STDERR "got interrupt - ignored\n";
5444     };
5445
5446     eval {
5447         # enable interrupts
5448         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5449             die "interrupted by signal\n";
5450         };
5451
5452         if ($archive eq '-') {
5453             print "extracting archive from STDIN\n";
5454             run_command($cmd, input => "<&STDIN");
5455         } else {
5456             print "extracting archive '$archive'\n";
5457             run_command($cmd);
5458         }
5459
5460         return if $opts->{info};
5461
5462         # read new mapping
5463         my $map = {};
5464         my $statfile = "$tmpdir/qmrestore.stat";
5465         if (my $fd = IO::File->new($statfile, "r")) {
5466             while (defined (my $line = <$fd>)) {
5467                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5468                     $map->{$1} = $2 if $1;
5469                 } else {
5470                     print STDERR "unable to parse line in statfile - $line\n";
5471                 }
5472             }
5473             $fd->close();
5474         }
5475
5476         my $confsrc = "$tmpdir/qemu-server.conf";
5477
5478         my $srcfd = new IO::File($confsrc, "r") ||
5479             die "unable to open file '$confsrc'\n";
5480
5481         my $outfd = new IO::File ($tmpfn, "w") ||
5482             die "unable to write config for VM $vmid\n";
5483
5484         my $cookie = { netcount => 0 };
5485         while (defined (my $line = <$srcfd>)) {
5486             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5487         }
5488
5489         $srcfd->close();
5490         $outfd->close();
5491     };
5492     my $err = $@;
5493
5494     if ($err) {
5495
5496         unlink $tmpfn;
5497
5498         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
5499
5500         die $err;
5501     }
5502
5503     rmtree $tmpdir;
5504
5505     rename $tmpfn, $conffile ||
5506         die "unable to commit configuration file '$conffile'\n";
5507
5508     PVE::Cluster::cfs_update(); # make sure we read new file
5509
5510     eval { rescan($vmid, 1); };
5511     warn $@ if $@;
5512 };
5513
5514 sub foreach_writable_storage {
5515     my ($conf, $func) = @_;
5516
5517     my $sidhash = {};
5518
5519     foreach my $ds (keys %$conf) {
5520         next if !is_valid_drivename($ds);
5521
5522         my $drive = parse_drive($ds, $conf->{$ds});
5523         next if !$drive;
5524         next if drive_is_cdrom($drive);
5525
5526         my $volid = $drive->{file};
5527
5528         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5529         $sidhash->{$sid} = $sid if $sid;
5530     }
5531
5532     foreach my $sid (sort keys %$sidhash) {
5533         &$func($sid);
5534     }
5535 }
5536
5537 sub do_snapshots_with_qemu {
5538     my ($storecfg, $volid) = @_;
5539
5540     my $storage_name = PVE::Storage::parse_volume_id($volid);
5541
5542     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
5543         && !$storecfg->{ids}->{$storage_name}->{krbd}){
5544         return 1;
5545     }
5546
5547     if ($volid =~ m/\.(qcow2|qed)$/){
5548         return 1;
5549     }
5550
5551     return undef;
5552 }
5553
5554 sub qga_check_running {
5555     my ($vmid) = @_;
5556
5557     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
5558     if ($@) {
5559         warn "Qemu Guest Agent are not running - $@";
5560         return 0;
5561     }
5562     return 1;
5563 }
5564
5565 sub template_create {
5566     my ($vmid, $conf, $disk) = @_;
5567
5568     my $storecfg = PVE::Storage::config();
5569
5570     foreach_drive($conf, sub {
5571         my ($ds, $drive) = @_;
5572
5573         return if drive_is_cdrom($drive);
5574         return if $disk && $ds ne $disk;
5575
5576         my $volid = $drive->{file};
5577         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
5578
5579         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
5580         $drive->{file} = $voliddst;
5581         $conf->{$ds} = print_drive($vmid, $drive);
5582         PVE::QemuConfig->write_config($vmid, $conf);
5583     });
5584 }
5585
5586 sub qemu_img_convert {
5587     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
5588
5589     my $storecfg = PVE::Storage::config();
5590     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
5591     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
5592
5593     if ($src_storeid && $dst_storeid) {
5594
5595         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
5596
5597         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
5598         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5599
5600         my $src_format = qemu_img_format($src_scfg, $src_volname);
5601         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
5602
5603         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
5604         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5605
5606         my $cmd = [];
5607         push @$cmd, '/usr/bin/qemu-img', 'convert', '-t', 'writeback', '-p', '-n';
5608         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
5609         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path;
5610         if ($is_zero_initialized) {
5611             push @$cmd, "zeroinit:$dst_path";
5612         } else {
5613             push @$cmd, $dst_path;
5614         }
5615
5616         my $parser = sub {
5617             my $line = shift;
5618             if($line =~ m/\((\S+)\/100\%\)/){
5619                 my $percent = $1;
5620                 my $transferred = int($size * $percent / 100);
5621                 my $remaining = $size - $transferred;
5622
5623                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
5624             }
5625
5626         };
5627
5628         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
5629         my $err = $@;
5630         die "copy failed: $err" if $err;
5631     }
5632 }
5633
5634 sub qemu_img_format {
5635     my ($scfg, $volname) = @_;
5636
5637     if ($scfg->{path} && $volname =~ m/\.(raw|cow|qcow|qcow2|qed|vmdk|cloop)$/) {
5638         return $1;
5639     } else {
5640         return "raw";
5641     }
5642 }
5643
5644 sub qemu_drive_mirror {
5645     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized) = @_;
5646
5647     my $storecfg = PVE::Storage::config();
5648     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
5649
5650     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5651
5652     my $format = qemu_img_format($dst_scfg, $dst_volname);
5653
5654     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5655
5656     my $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
5657
5658     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
5659     $opts->{format} = $format if $format;
5660
5661     print "drive mirror is starting (scanning bitmap) : this step can take some minutes/hours, depend of disk size and storage speed\n";
5662
5663     my $finish_job = sub {
5664         while (1) {
5665             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
5666             my $stat = @$stats[0];
5667             last if !$stat;
5668             sleep 1;
5669         }
5670     };
5671
5672     eval {
5673     vm_mon_cmd($vmid, "drive-mirror", %$opts);
5674         while (1) {
5675             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
5676             my $stat = @$stats[0];
5677             die "mirroring job seem to have die. Maybe do you have bad sectors?" if !$stat;
5678             die "error job is not mirroring" if $stat->{type} ne "mirror";
5679
5680             my $busy = $stat->{busy};
5681             my $ready = $stat->{ready};
5682
5683             if (my $total = $stat->{len}) {
5684                 my $transferred = $stat->{offset} || 0;
5685                 my $remaining = $total - $transferred;
5686                 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
5687
5688                 print "transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
5689             }
5690
5691
5692             if ($stat->{ready} eq 'true') {
5693
5694                 last if $vmiddst != $vmid;
5695
5696                 # try to switch the disk if source and destination are on the same guest
5697                 eval { vm_mon_cmd($vmid, "block-job-complete", device => "drive-$drive") };
5698                 if (!$@) {
5699                     &$finish_job();
5700                     last;
5701                 }
5702                 die $@ if $@ !~ m/cannot be completed/;
5703             }
5704             sleep 1;
5705         }
5706
5707
5708     };
5709     my $err = $@;
5710
5711     my $cancel_job = sub {
5712         vm_mon_cmd($vmid, "block-job-cancel", device => "drive-$drive");
5713         &$finish_job();
5714     };
5715
5716     if ($err) {
5717         eval { &$cancel_job(); };
5718         die "mirroring error: $err";
5719     }
5720
5721     if ($vmiddst != $vmid) {
5722         # if we clone a disk for a new target vm, we don't switch the disk
5723         &$cancel_job(); # so we call block-job-cancel
5724     }
5725 }
5726
5727 sub clone_disk {
5728     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
5729         $newvmid, $storage, $format, $full, $newvollist) = @_;
5730
5731     my $newvolid;
5732
5733     if (!$full) {
5734         print "create linked clone of drive $drivename ($drive->{file})\n";
5735         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
5736         push @$newvollist, $newvolid;
5737     } else {
5738         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
5739         $storeid = $storage if $storage;
5740
5741         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5742         if (!$format) {
5743             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5744             $format = qemu_img_format($scfg, $volname);
5745         }
5746
5747         # test if requested format is supported - else use default
5748         my $supported = grep { $_ eq $format } @$validFormats;
5749         $format = $defFormat if !$supported;
5750
5751         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
5752
5753         print "create full clone of drive $drivename ($drive->{file})\n";
5754         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $format, undef, ($size/1024));
5755         push @$newvollist, $newvolid;
5756
5757         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
5758
5759         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
5760         if (!$running || $snapname) {
5761             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
5762         } else {
5763             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit);
5764         }
5765     }
5766
5767     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
5768
5769     my $disk = $drive;
5770     $disk->{format} = undef;
5771     $disk->{file} = $newvolid;
5772     $disk->{size} = $size;
5773
5774     return $disk;
5775 }
5776
5777 # this only works if VM is running
5778 sub get_current_qemu_machine {
5779     my ($vmid) = @_;
5780
5781     my $cmd = { execute => 'query-machines', arguments => {} };
5782     my $res = vm_qmp_command($vmid, $cmd);
5783
5784     my ($current, $default);
5785     foreach my $e (@$res) {
5786         $default = $e->{name} if $e->{'is-default'};
5787         $current = $e->{name} if $e->{'is-current'};
5788     }
5789
5790     # fallback to the default machine if current is not supported by qemu
5791     return $current || $default || 'pc';
5792 }
5793
5794 sub qemu_machine_feature_enabled {
5795     my ($machine, $kvmver, $version_major, $version_minor) = @_;
5796
5797     my $current_major;
5798     my $current_minor;
5799
5800     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
5801
5802         $current_major = $3;
5803         $current_minor = $4;
5804
5805     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
5806
5807         $current_major = $1;
5808         $current_minor = $2;
5809     }
5810
5811     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
5812
5813
5814 }
5815
5816 sub qemu_machine_pxe {
5817     my ($vmid, $conf, $machine) = @_;
5818
5819     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
5820
5821     foreach my $opt (keys %$conf) {
5822         next if $opt !~ m/^net(\d+)$/;
5823         my $net = PVE::QemuServer::parse_net($conf->{$opt});
5824         next if !$net;
5825         my $romfile = PVE::QemuServer::vm_mon_cmd_nocheck($vmid, 'qom-get', path => $opt, property => 'romfile');
5826         return $machine.".pxe" if $romfile =~ m/pxe/;
5827         last;
5828     }
5829
5830     return $machine;
5831 }
5832
5833 sub qemu_use_old_bios_files {
5834     my ($machine_type) = @_;
5835
5836     return if !$machine_type;
5837
5838     my $use_old_bios_files = undef;
5839
5840     if ($machine_type =~ m/^(\S+)\.pxe$/) {
5841         $machine_type = $1;
5842         $use_old_bios_files = 1;
5843     } else {
5844         my $kvmver = kvm_user_version();
5845         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
5846         # load new efi bios files on migration. So this hack is required to allow
5847         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
5848         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
5849         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
5850     }
5851
5852     return ($use_old_bios_files, $machine_type);
5853 }
5854
5855 sub lspci {
5856
5857     my $devices = {};
5858
5859     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
5860             my (undef, $id, $function) = @_;
5861             my $res = { id => $id, function => $function};
5862             push @{$devices->{$id}}, $res;
5863     });
5864
5865     return $devices;
5866 }
5867
5868 sub vm_iothreads_list {
5869     my ($vmid) = @_;
5870
5871     my $res = vm_mon_cmd($vmid, 'query-iothreads');
5872
5873     my $iothreads = {};
5874     foreach my $iothread (@$res) {
5875         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
5876     }
5877
5878     return $iothreads;
5879 }
5880
5881 sub scsihw_infos {
5882     my ($conf, $drive) = @_;
5883
5884     my $maxdev = 0;
5885
5886     if ($conf->{scsihw} && ($conf->{scsihw} =~ m/^lsi/)) {
5887         $maxdev = 7;
5888     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
5889         $maxdev = 1;
5890     } else {
5891         $maxdev = 256;
5892     }
5893
5894     my $controller = int($drive->{index} / $maxdev);
5895     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
5896
5897     return ($maxdev, $controller, $controller_prefix);
5898 }
5899
5900 # bash completion helper
5901
5902 sub complete_backup_archives {
5903     my ($cmdname, $pname, $cvalue) = @_;
5904
5905     my $cfg = PVE::Storage::config();
5906
5907     my $storeid;
5908
5909     if ($cvalue =~ m/^([^:]+):/) {
5910         $storeid = $1;
5911     }
5912
5913     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
5914
5915     my $res = [];
5916     foreach my $id (keys %$data) {
5917         foreach my $item (@{$data->{$id}}) {
5918             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
5919             push @$res, $item->{volid} if defined($item->{volid});
5920         }
5921     }
5922
5923     return $res;
5924 }
5925
5926 my $complete_vmid_full = sub {
5927     my ($running) = @_;
5928
5929     my $idlist = vmstatus();
5930
5931     my $res = [];
5932
5933     foreach my $id (keys %$idlist) {
5934         my $d = $idlist->{$id};
5935         if (defined($running)) {
5936             next if $d->{template};
5937             next if $running && $d->{status} ne 'running';
5938             next if !$running && $d->{status} eq 'running';
5939         }
5940         push @$res, $id;
5941
5942     }
5943     return $res;
5944 };
5945
5946 sub complete_vmid {
5947     return &$complete_vmid_full();
5948 }
5949
5950 sub complete_vmid_stopped {
5951     return &$complete_vmid_full(0);
5952 }
5953
5954 sub complete_vmid_running {
5955     return &$complete_vmid_full(1);
5956 }
5957
5958 sub complete_storage {
5959
5960     my $cfg = PVE::Storage::config();
5961     my $ids = $cfg->{ids};
5962
5963     my $res = [];
5964     foreach my $sid (keys %$ids) {
5965         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
5966         next if !$ids->{$sid}->{content}->{images};
5967         push @$res, $sid;
5968     }
5969
5970     return $res;
5971 }
5972
5973 1;