PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QemuConfig;
  31 use PVE::QMPClient;
  32 use PVE::RPCEnvironment;
  33 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
  34 use PVE::QemuServer::Memory;
  35 use PVE::QemuServer::USB qw(parse_usb_device);
  36 use Time::HiRes qw(gettimeofday);
  37 use File::Copy qw(copy);
  38 use URI::Escape;
  39
  40 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  41
  42 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  43
  44 # Note about locking: we use flock on the config file protect
  45 # against concurent actions.
  46 # Aditionaly, we have a 'lock' setting in the config file. This
  47 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  48 # allowed when such lock is set. But you can ignore this kind of
  49 # lock with the --skiplock flag.
  50
  51 cfs_register_file('/qemu-server/',
  52                   \&parse_vm_config,
  53                   \&write_vm_config);
  54
  55 PVE::JSONSchema::register_standard_option('skiplock', {
  56     description => "Ignore locks - only root is allowed to use this option.",
  57     type => 'boolean',
  58     optional => 1,
  59 });
  60
  61 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  62     description => "Some command save/restore state from this location.",
  63     type => 'string',
  64     maxLength => 128,
  65     optional => 1,
  66 });
  67
  68 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  69     description => "The name of the snapshot.",
  70     type => 'string', format => 'pve-configid',
  71     maxLength => 40,
  72 });
  73
  74 #no warnings 'redefine';
  75
  76 sub cgroups_write {
  77    my ($controller, $vmid, $option, $value) = @_;
  78
  79    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  80    PVE::ProcFSTools::write_proc_entry($path, $value);
  81
  82 }
  83
  84 my $nodename = PVE::INotify::nodename();
  85
  86 mkdir "/etc/pve/nodes/$nodename";
  87 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  88 mkdir $confdir;
  89
  90 my $var_run_tmpdir = "/var/run/qemu-server";
  91 mkdir $var_run_tmpdir;
  92
  93 my $lock_dir = "/var/lock/qemu-server";
  94 mkdir $lock_dir;
  95
  96 my $pcisysfs = "/sys/bus/pci";
  97
  98 my $cpu_vendor_list = {
  99     # Intel CPUs
 100     486 => 'GenuineIntel',
 101     pentium => 'GenuineIntel',
 102     pentium2  => 'GenuineIntel',
 103     pentium3  => 'GenuineIntel',
 104     coreduo => 'GenuineIntel',
 105     core2duo => 'GenuineIntel',
 106     Conroe  => 'GenuineIntel',
 107     Penryn  => 'GenuineIntel',
 108     Nehalem  => 'GenuineIntel',
 109     Westmere => 'GenuineIntel',
 110     SandyBridge => 'GenuineIntel',
 111     IvyBridge => 'GenuineIntel',
 112     Haswell => 'GenuineIntel',
 113     'Haswell-noTSX' => 'GenuineIntel',
 114     Broadwell => 'GenuineIntel',
 115     'Broadwell-noTSX' => 'GenuineIntel',
 116
 117     # AMD CPUs
 118     athlon => 'AuthenticAMD',
 119     phenom  => 'AuthenticAMD',
 120     Opteron_G1  => 'AuthenticAMD',
 121     Opteron_G2  => 'AuthenticAMD',
 122     Opteron_G3  => 'AuthenticAMD',
 123     Opteron_G4  => 'AuthenticAMD',
 124     Opteron_G5  => 'AuthenticAMD',
 125
 126     # generic types, use vendor from host node
 127     host => 'default',
 128     kvm32 => 'default',
 129     kvm64 => 'default',
 130     qemu32 => 'default',
 131     qemu64 => 'default',
 132 };
 133
 134 my $cpu_fmt = {
 135     cputype => {
 136         description => "Emulated CPU type.",
 137         type => 'string',
 138         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 139         format_description => 'cputype',
 140         default => 'kvm64',
 141         default_key => 1,
 142     },
 143     hidden => {
 144         description => "Do not identify as a KVM virtual machine.",
 145         type => 'boolean',
 146         optional => 1,
 147         default => 0
 148     },
 149 };
 150
 151 my $watchdog_fmt = {
 152     model => {
 153         default_key => 1,
 154         type => 'string',
 155         enum => [qw(i6300esb ib700)],
 156         description => "Watchdog type to emulate.",
 157         default => 'i6300esb',
 158         optional => 1,
 159     },
 160     action => {
 161         type => 'string',
 162         enum => [qw(reset shutdown poweroff pause debug none)],
 163         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 164         optional => 1,
 165     },
 166 };
 167 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 168
 169 my $confdesc = {
 170     onboot => {
 171         optional => 1,
 172         type => 'boolean',
 173         description => "Specifies whether a VM will be started during system bootup.",
 174         default => 0,
 175     },
 176     autostart => {
 177         optional => 1,
 178         type => 'boolean',
 179         description => "Automatic restart after crash (currently ignored).",
 180         default => 0,
 181     },
 182     hotplug => {
 183         optional => 1,
 184         type => 'string', format => 'pve-hotplug-features',
 185         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 186         default => 'network,disk,usb',
 187     },
 188     reboot => {
 189         optional => 1,
 190         type => 'boolean',
 191         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 192         default => 1,
 193     },
 194     lock => {
 195         optional => 1,
 196         type => 'string',
 197         description => "Lock/unlock the VM.",
 198         enum => [qw(migrate backup snapshot rollback)],
 199     },
 200     cpulimit => {
 201         optional => 1,
 202         type => 'number',
 203         description => "Limit of CPU usage.",
 204         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 205         minimum => 0,
 206         maximum => 128,
 207         default => 0,
 208     },
 209     cpuunits => {
 210         optional => 1,
 211         type => 'integer',
 212         description => "CPU weight for a VM.",
 213         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 214         minimum => 0,
 215         maximum => 500000,
 216         default => 1000,
 217     },
 218     memory => {
 219         optional => 1,
 220         type => 'integer',
 221         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 222         minimum => 16,
 223         default => 512,
 224     },
 225     balloon => {
 226         optional => 1,
 227         type => 'integer',
 228         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 229         minimum => 0,
 230     },
 231     shares => {
 232         optional => 1,
 233         type => 'integer',
 234         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 235         minimum => 0,
 236         maximum => 50000,
 237         default => 1000,
 238     },
 239     keyboard => {
 240         optional => 1,
 241         type => 'string',
 242         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.conf' configuration file.",
 243         enum => PVE::Tools::kvmkeymaplist(),
 244         default => 'en-us',
 245     },
 246     name => {
 247         optional => 1,
 248         type => 'string', format => 'dns-name',
 249         description => "Set a name for the VM. Only used on the configuration web interface.",
 250     },
 251     scsihw => {
 252         optional => 1,
 253         type => 'string',
 254         description => "SCSI controller model",
 255         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 256         default => 'lsi',
 257     },
 258     description => {
 259         optional => 1,
 260         type => 'string',
 261         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 262     },
 263     ostype => {
 264         optional => 1,
 265         type => 'string',
 266         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 l24 l26 solaris)],
 267         description => "Specify guest operating system.",
 268         verbose_description => <<EODESC,
 269 Specify guest operating system. This is used to enable special
 270 optimization/features for specific operating systems:
 271
 272 [horizontal]
 273 other;; unspecified OS
 274 wxp;; Microsoft Windows XP
 275 w2k;; Microsoft Windows 2000
 276 w2k3;; Microsoft Windows 2003
 277 w2k8;; Microsoft Windows 2008
 278 wvista;; Microsoft Windows Vista
 279 win7;; Microsoft Windows 7
 280 win8;; Microsoft Windows 8/2012
 281 l24;; Linux 2.4 Kernel
 282 l26;; Linux 2.6/3.X Kernel
 283 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 284 EODESC
 285     },
 286     boot => {
 287         optional => 1,
 288         type => 'string',
 289         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 290         pattern => '[acdn]{1,4}',
 291         default => 'cdn',
 292     },
 293     bootdisk => {
 294         optional => 1,
 295         type => 'string', format => 'pve-qm-bootdisk',
 296         description => "Enable booting from specified disk.",
 297         pattern => '(ide|sata|scsi|virtio)\d+',
 298     },
 299     smp => {
 300         optional => 1,
 301         type => 'integer',
 302         description => "The number of CPUs. Please use option -sockets instead.",
 303         minimum => 1,
 304         default => 1,
 305     },
 306     sockets => {
 307         optional => 1,
 308         type => 'integer',
 309         description => "The number of CPU sockets.",
 310         minimum => 1,
 311         default => 1,
 312     },
 313     cores => {
 314         optional => 1,
 315         type => 'integer',
 316         description => "The number of cores per socket.",
 317         minimum => 1,
 318         default => 1,
 319     },
 320     numa => {
 321         optional => 1,
 322         type => 'boolean',
 323         description => "Enable/disable NUMA.",
 324         default => 0,
 325     },
 326     hugepages => {
 327         optional => 1,
 328         type => 'string',
 329         description => "Enable/disable hugepages memory.",
 330         enum => [qw(any 2 1024)],
 331     },
 332     vcpus => {
 333         optional => 1,
 334         type => 'integer',
 335         description => "Number of hotplugged vcpus.",
 336         minimum => 1,
 337         default => 0,
 338     },
 339     acpi => {
 340         optional => 1,
 341         type => 'boolean',
 342         description => "Enable/disable ACPI.",
 343         default => 1,
 344     },
 345     agent => {
 346         optional => 1,
 347         type => 'boolean',
 348         description => "Enable/disable Qemu GuestAgent.",
 349         default => 0,
 350     },
 351     kvm => {
 352         optional => 1,
 353         type => 'boolean',
 354         description => "Enable/disable KVM hardware virtualization.",
 355         default => 1,
 356     },
 357     tdf => {
 358         optional => 1,
 359         type => 'boolean',
 360         description => "Enable/disable time drift fix.",
 361         default => 0,
 362     },
 363     localtime => {
 364         optional => 1,
 365         type => 'boolean',
 366         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 367     },
 368     freeze => {
 369         optional => 1,
 370         type => 'boolean',
 371         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 372     },
 373     vga => {
 374         optional => 1,
 375         type => 'string',
 376         description => "Select the VGA type.",
 377         verbose_description => "Select the VGA type. If you want to use high resolution" .
 378             " modes (>= 1280x1024x16) then you should use the options " .
 379             "'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and " .
 380             "'cirrus' for other OS types. The 'qxl' option enables the SPICE " .
 381             "display sever. For win* OS you can select how many independent " .
 382             "displays you want, Linux guests can add displays them self. " .
 383             "You can also run without any graphic card, using a serial device" .
 384             " as terminal.",
 385         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 386     },
 387     watchdog => {
 388         optional => 1,
 389         type => 'string', format => 'pve-qm-watchdog',
 390         description => "Create a virtual hardware watchdog device.",
 391         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 392             " (by a guest action), the watchdog must be periodically polled " .
 393             "by an agent inside the guest or else the watchdog will reset " .
 394             "the guest (or execute the respective action specified)",
 395     },
 396     startdate => {
 397         optional => 1,
 398         type => 'string',
 399         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 400         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 401         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 402         default => 'now',
 403     },
 404     startup =>  get_standard_option('pve-startup-order'),
 405     template => {
 406         optional => 1,
 407         type => 'boolean',
 408         description => "Enable/disable Template.",
 409         default => 0,
 410     },
 411     args => {
 412         optional => 1,
 413         type => 'string',
 414         description => "Arbitrary arguments passed to kvm.",
 415         verbose_description => <<EODESCR,
 416 Arbitrary arguments passed to kvm, for example:
 417
 418 args: -no-reboot -no-hpet
 419
 420 NOTE: this option is for experts only.
 421 EODESCR
 422     },
 423     tablet => {
 424         optional => 1,
 425         type => 'boolean',
 426         default => 1,
 427         description => "Enable/disable the USB tablet device.",
 428         verbose_description => "Enable/disable the USB tablet device. This device is " .
 429             "usually needed to allow absolute mouse positioning with VNC. " .
 430             "Else the mouse runs out of sync with normal VNC clients. " .
 431             "If you're running lots of console-only guests on one host, " .
 432             "you may consider disabling this to save some context switches. " .
 433             "This is turned off by default if you use spice (-vga=qxl).",
 434     },
 435     migrate_speed => {
 436         optional => 1,
 437         type => 'integer',
 438         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 439         minimum => 0,
 440         default => 0,
 441     },
 442     migrate_downtime => {
 443         optional => 1,
 444         type => 'number',
 445         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 446         minimum => 0,
 447         default => 0.1,
 448     },
 449     cdrom => {
 450         optional => 1,
 451         type => 'string', format => 'pve-qm-drive',
 452         typetext => 'volume',
 453         description => "This is an alias for option -ide2",
 454     },
 455     cpu => {
 456         optional => 1,
 457         description => "Emulated CPU type.",
 458         type => 'string',
 459         format => $cpu_fmt,
 460     },
 461     parent => get_standard_option('pve-snapshot-name', {
 462         optional => 1,
 463         description => "Parent snapshot name. This is used internally, and should not be modified.",
 464     }),
 465     snaptime => {
 466         optional => 1,
 467         description => "Timestamp for snapshots.",
 468         type => 'integer',
 469         minimum => 0,
 470     },
 471     vmstate => {
 472         optional => 1,
 473         type => 'string', format => 'pve-volume-id',
 474         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 475     },
 476     machine => {
 477         description => "Specific the Qemu machine type.",
 478         type => 'string',
 479         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
 480         maxLength => 40,
 481         optional => 1,
 482     },
 483     smbios1 => {
 484         description => "Specify SMBIOS type 1 fields.",
 485         type => 'string', format => 'pve-qm-smbios1',
 486         maxLength => 256,
 487         optional => 1,
 488     },
 489     protection => {
 490         optional => 1,
 491         type => 'boolean',
 492         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 493         default => 0,
 494     },
 495     bios => {
 496         optional => 1,
 497         type => 'string',
 498         enum => [ qw(seabios ovmf) ],
 499         description => "Select BIOS implementation.",
 500         default => 'seabios',
 501     },
 502 };
 503
 504 # what about other qemu settings ?
 505 #cpu => 'string',
 506 #machine => 'string',
 507 #fda => 'file',
 508 #fdb => 'file',
 509 #mtdblock => 'file',
 510 #sd => 'file',
 511 #pflash => 'file',
 512 #snapshot => 'bool',
 513 #bootp => 'file',
 514 ##tftp => 'dir',
 515 ##smb => 'dir',
 516 #kernel => 'file',
 517 #append => 'string',
 518 #initrd => 'file',
 519 ##soundhw => 'string',
 520
 521 while (my ($k, $v) = each %$confdesc) {
 522     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 523 }
 524
 525 my $MAX_IDE_DISKS = 4;
 526 my $MAX_SCSI_DISKS = 14;
 527 my $MAX_VIRTIO_DISKS = 16;
 528 my $MAX_SATA_DISKS = 6;
 529 my $MAX_USB_DEVICES = 5;
 530 my $MAX_NETS = 32;
 531 my $MAX_UNUSED_DISKS = 8;
 532 my $MAX_HOSTPCI_DEVICES = 4;
 533 my $MAX_SERIAL_PORTS = 4;
 534 my $MAX_PARALLEL_PORTS = 3;
 535 my $MAX_NUMA = 8;
 536
 537 my $numa_fmt = {
 538     cpus => {
 539         type => "string",
 540         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 541         description => "CPUs accessing this NUMA node.",
 542         format_description => "id[-id];...",
 543     },
 544     memory => {
 545         type => "number",
 546         description => "Amount of memory this NUMA node provides.",
 547         optional => 1,
 548     },
 549     hostnodes => {
 550         type => "string",
 551         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 552         description => "Host NUMA nodes to use.",
 553         format_description => "id[-id];...",
 554         optional => 1,
 555     },
 556     policy => {
 557         type => 'string',
 558         enum => [qw(preferred bind interleave)],
 559         description => "NUMA allocation policy.",
 560         optional => 1,
 561     },
 562 };
 563 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 564 my $numadesc = {
 565     optional => 1,
 566     type => 'string', format => $numa_fmt,
 567     description => "NUMA topology.",
 568 };
 569 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 570
 571 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 572     $confdesc->{"numa$i"} = $numadesc;
 573 }
 574
 575 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 576                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 577                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 578 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 579
 580 my $net_fmt_bridge_descr = <<__EOD__;
 581 Bridge to attach the network device to. The Proxmox VE standard bridge
 582 is called 'vmbr0'.
 583
 584 If you do not specify a bridge, we create a kvm user (NATed) network
 585 device, which provides DHCP and DNS services. The following addresses
 586 are used:
 587
 588  10.0.2.2   Gateway
 589  10.0.2.3   DNS Server
 590  10.0.2.4   SMB Server
 591
 592 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 593 __EOD__
 594
 595 my $net_fmt = {
 596     macaddr => {
 597         type => 'string',
 598         pattern => qr/[0-9a-f]{2}(?::[0-9a-f]{2}){5}/i,
 599         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 600         format_description => "XX:XX:XX:XX:XX:XX",
 601         optional => 1,
 602     },
 603     model => {
 604         type => 'string',
 605         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 606         format_description => 'model',
 607         enum => $nic_model_list,
 608         default_key => 1,
 609     },
 610     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 611     bridge => {
 612         type => 'string',
 613         description => $net_fmt_bridge_descr,
 614         format_description => 'bridge',
 615         optional => 1,
 616     },
 617     queues => {
 618         type => 'integer',
 619         minimum => 0, maximum => 16,
 620         description => 'Number of packet queues to be used on the device.',
 621         optional => 1,
 622     },
 623     rate => {
 624         type => 'number',
 625         minimum => 0,
 626         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 627         optional => 1,
 628     },
 629     tag => {
 630         type => 'integer',
 631         minimum => 1, maximum => 4094,
 632         description => 'VLAN tag to apply to packets on this interface.',
 633         optional => 1,
 634     },
 635     trunks => {
 636         type => 'string',
 637         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 638         description => 'VLAN trunks to pass through this interface.',
 639         format_description => 'vlanid[;vlanid...]',
 640         optional => 1,
 641     },
 642     firewall => {
 643         type => 'boolean',
 644         description => 'Whether this interface should be protected by the firewall.',
 645         optional => 1,
 646     },
 647     link_down => {
 648         type => 'boolean',
 649         description => 'Whether this interface should be disconnected (like pulling the plug).',
 650         optional => 1,
 651     },
 652 };
 653
 654 my $netdesc = {
 655     optional => 1,
 656     type => 'string', format => $net_fmt,
 657     description => "Specify network devices.",
 658 };
 659
 660 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 661
 662 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 663     $confdesc->{"net$i"} = $netdesc;
 664 }
 665
 666 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 667 sub verify_volume_id_or_qm_path {
 668     my ($volid, $noerr) = @_;
 669
 670     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 671         return $volid;
 672     }
 673
 674     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 675     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 676     if ($@) {
 677         return undef if $noerr;
 678         die $@;
 679     }
 680     return $volid;
 681 }
 682
 683 my $drivename_hash;
 684
 685 my %drivedesc_base = (
 686     volume => { alias => 'file' },
 687     file => {
 688         type => 'string',
 689         format => 'pve-volume-id-or-qm-path',
 690         default_key => 1,
 691         format_description => 'volume',
 692         description => "The drive's backing volume.",
 693     },
 694     media => {
 695         type => 'string',
 696         enum => [qw(cdrom disk)],
 697         description => "The drive's media type.",
 698         default => 'disk',
 699         optional => 1
 700     },
 701     cyls => {
 702         type => 'integer',
 703         description => "Force the drive's physical geometry to have a specific cylinder count.",
 704         optional => 1
 705     },
 706     heads => {
 707         type => 'integer',
 708         description => "Force the drive's physical geometry to have a specific head count.",
 709         optional => 1
 710     },
 711     secs => {
 712         type => 'integer',
 713         description => "Force the drive's physical geometry to have a specific sector count.",
 714         optional => 1
 715     },
 716     trans => {
 717         type => 'string',
 718         enum => [qw(none lba auto)],
 719         description => "Force disk geometry bios translation mode.",
 720         optional => 1,
 721     },
 722     snapshot => {
 723         type => 'boolean',
 724         description => "Whether the drive should be included when making snapshots.",
 725         optional => 1,
 726     },
 727     cache => {
 728         type => 'string',
 729         enum => [qw(none writethrough writeback unsafe directsync)],
 730         description => "The drive's cache mode",
 731         optional => 1,
 732     },
 733     format => {
 734         type => 'string',
 735         format_description => 'image format',
 736         enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
 737         description => "The drive's backing file's data format.",
 738         optional => 1,
 739     },
 740     size => {
 741         type => 'string',
 742         format => 'disk-size',
 743         format_description => 'DiskSize',
 744         description => "Disk size. This is purely informational and has no effect.",
 745         optional => 1,
 746     },
 747     backup => {
 748         type => 'boolean',
 749         description => "Whether the drive should be included when making backups.",
 750         optional => 1,
 751     },
 752     werror => {
 753         type => 'string',
 754         enum => [qw(enospc ignore report stop)],
 755         description => 'Write error action.',
 756         optional => 1,
 757     },
 758     aio => {
 759         type => 'string',
 760         enum => [qw(native threads)],
 761         description => 'AIO type to use.',
 762         optional => 1,
 763     },
 764     discard => {
 765         type => 'string',
 766         enum => [qw(ignore on)],
 767         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 768         optional => 1,
 769     },
 770     detect_zeroes => {
 771         type => 'boolean',
 772         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 773         optional => 1,
 774     },
 775     serial => {
 776         type => 'string',
 777         format => 'urlencoded',
 778         format_description => 'serial',
 779         maxLength => 20*3, # *3 since it's %xx url enoded
 780         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
 781         optional => 1,
 782     }
 783 );
 784
 785 my %rerror_fmt = (
 786     rerror => {
 787         type => 'string',
 788         enum => [qw(ignore report stop)],
 789         description => 'Read error action.',
 790         optional => 1,
 791     },
 792 );
 793
 794 my %iothread_fmt = ( iothread => {
 795         type => 'boolean',
 796         description => "Whether to use iothreads for this drive",
 797         optional => 1,
 798 });
 799
 800 my %model_fmt = (
 801     model => {
 802         type => 'string',
 803         format => 'urlencoded',
 804         format_description => 'model',
 805         maxLength => 40*3, # *3 since it's %xx url enoded
 806         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
 807         optional => 1,
 808     },
 809 );
 810
 811 my %queues_fmt = (
 812     queues => {
 813         type => 'integer',
 814         description => "Number of queues.",
 815         minimum => 2,
 816         optional => 1
 817     }
 818 );
 819
 820 my $add_throttle_desc = sub {
 821     my ($key, $type, $what, $unit, $longunit) = @_;
 822     $drivedesc_base{$key} = {
 823         type => $type,
 824         format_description => $unit,
 825         description => "Maximum $what speed in $longunit per second.",
 826         optional => 1,
 827     };
 828 };
 829 # throughput: (leaky bucket)
 830 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes');
 831 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes');
 832 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes');
 833 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes');
 834 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes');
 835 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes');
 836 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations');
 837 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations');
 838 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations');
 839
 840 # pools: (pool of IO before throttling starts taking effect)
 841 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes');
 842 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes');
 843 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes');
 844 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations');
 845 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations');
 846 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations');
 847
 848 my $ide_fmt = {
 849     %drivedesc_base,
 850     %rerror_fmt,
 851     %model_fmt,
 852 };
 853
 854 my $idedesc = {
 855     optional => 1,
 856     type => 'string', format => $ide_fmt,
 857     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
 858 };
 859 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
 860
 861 my $scsi_fmt = {
 862     %drivedesc_base,
 863     %iothread_fmt,
 864     %queues_fmt,
 865 };
 866 my $scsidesc = {
 867     optional => 1,
 868     type => 'string', format => $scsi_fmt,
 869     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
 870 };
 871 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 872
 873 my $sata_fmt = {
 874     %drivedesc_base,
 875     %rerror_fmt,
 876 };
 877 my $satadesc = {
 878     optional => 1,
 879     type => 'string', format => $sata_fmt,
 880     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
 881 };
 882 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
 883
 884 my $virtio_fmt = {
 885     %drivedesc_base,
 886     %iothread_fmt,
 887     %rerror_fmt,
 888 };
 889 my $virtiodesc = {
 890     optional => 1,
 891     type => 'string', format => $virtio_fmt,
 892     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
 893 };
 894 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
 895
 896 my $alldrive_fmt = {
 897     %drivedesc_base,
 898     %rerror_fmt,
 899     %iothread_fmt,
 900     %model_fmt,
 901     %queues_fmt,
 902 };
 903
 904 my $usb_fmt = {
 905     host => {
 906         default_key => 1,
 907         type => 'string', format => 'pve-qm-usb-device',
 908         format_description => 'HOSTUSBDEVICE|spice',
 909         description => <<EODESCR,
 910 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
 911
 912  'bus-port(.port)*' (decimal numbers) or
 913  'vendor_id:product_id' (hexadeciaml numbers) or
 914  'spice'
 915
 916 You can use the 'lsusb -t' command to list existing usb devices.
 917
 918 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 919
 920 The value 'spice' can be used to add a usb redirection devices for spice.
 921 EODESCR
 922     },
 923     usb3 => {
 924         optional => 1,
 925         type => 'boolean',
 926         description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
 927         default => 0,
 928     },
 929 };
 930
 931 my $usbdesc = {
 932     optional => 1,
 933     type => 'string', format => $usb_fmt,
 934     description => "Configure an USB device (n is 0 to 4).",
 935 };
 936 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
 937
 938 # NOTE: the match-groups of this regex are used in parse_hostpci
 939 my $PCIRE = qr/([a-f0-9]{2}:[a-f0-9]{2})(?:\.([a-f0-9]))?/;
 940 my $hostpci_fmt = {
 941     host => {
 942         default_key => 1,
 943         type => 'string',
 944         pattern => qr/$PCIRE(;$PCIRE)*/,
 945         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
 946         description => <<EODESCR,
 947 Host PCI device pass through. The PCI ID of a host's PCI device or a list
 948 of PCI virtual functions of the host. HOSTPCIID syntax is:
 949
 950 'bus:dev.func' (hexadecimal numbers)
 951
 952 You can us the 'lspci' command to list existing PCI devices.
 953 EODESCR
 954     },
 955     rombar => {
 956         type => 'boolean',
 957         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
 958         optional => 1,
 959         default => 1,
 960     },
 961     pcie => {
 962         type => 'boolean',
 963         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
 964         optional => 1,
 965         default => 0,
 966     },
 967     'x-vga' => {
 968         type => 'boolean',
 969         description =>  "Enable vfio-vga device support.",
 970         optional => 1,
 971         default => 0,
 972     },
 973 };
 974 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
 975
 976 my $hostpcidesc = {
 977         optional => 1,
 978         type => 'string', format => 'pve-qm-hostpci',
 979         description => "Map host PCI devices into guest.",
 980         verbose_description =>  <<EODESCR,
 981 Map host PCI devices into guest.
 982
 983 NOTE: This option allows direct access to host hardware. So it is no longer
 984 possible to migrate such machines - use with special care.
 985
 986 CAUTION: Experimental! User reported problems with this option.
 987 EODESCR
 988 };
 989 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
 990
 991 my $serialdesc = {
 992         optional => 1,
 993         type => 'string',
 994         pattern => '(/dev/.+|socket)',
 995         description =>  "Create a serial device inside the VM (n is 0 to 3)",
 996         verbose_description =>  <<EODESCR,
 997 Create a serial device inside the VM (n is 0 to 3), and pass through a
 998 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
 999 host side (use 'qm terminal' to open a terminal connection).
1000
1001 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1002
1003 CAUTION: Experimental! User reported problems with this option.
1004 EODESCR
1005 };
1006
1007 my $paralleldesc= {
1008         optional => 1,
1009         type => 'string',
1010         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1011         description =>  "Map host parallel devices (n is 0 to 2).",
1012         verbose_description =>  <<EODESCR,
1013 Map host parallel devices (n is 0 to 2).
1014
1015 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1016
1017 CAUTION: Experimental! User reported problems with this option.
1018 EODESCR
1019 };
1020
1021 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1022     $confdesc->{"parallel$i"} = $paralleldesc;
1023 }
1024
1025 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1026     $confdesc->{"serial$i"} = $serialdesc;
1027 }
1028
1029 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1030     $confdesc->{"hostpci$i"} = $hostpcidesc;
1031 }
1032
1033 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1034     $drivename_hash->{"ide$i"} = 1;
1035     $confdesc->{"ide$i"} = $idedesc;
1036 }
1037
1038 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1039     $drivename_hash->{"sata$i"} = 1;
1040     $confdesc->{"sata$i"} = $satadesc;
1041 }
1042
1043 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1044     $drivename_hash->{"scsi$i"} = 1;
1045     $confdesc->{"scsi$i"} = $scsidesc ;
1046 }
1047
1048 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1049     $drivename_hash->{"virtio$i"} = 1;
1050     $confdesc->{"virtio$i"} = $virtiodesc;
1051 }
1052
1053 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1054     $confdesc->{"usb$i"} = $usbdesc;
1055 }
1056
1057 my $unuseddesc = {
1058     optional => 1,
1059     type => 'string', format => 'pve-volume-id',
1060     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1061 };
1062
1063 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1064     $confdesc->{"unused$i"} = $unuseddesc;
1065 }
1066
1067 my $kvm_api_version = 0;
1068
1069 sub kvm_version {
1070
1071     return $kvm_api_version if $kvm_api_version;
1072
1073     my $fh = IO::File->new("</dev/kvm") ||
1074         return 0;
1075
1076     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
1077         $kvm_api_version = $v;
1078     }
1079
1080     $fh->close();
1081
1082     return  $kvm_api_version;
1083 }
1084
1085 my $kvm_user_version;
1086
1087 sub kvm_user_version {
1088
1089     return $kvm_user_version if $kvm_user_version;
1090
1091     $kvm_user_version = 'unknown';
1092
1093     my $code = sub {
1094         my $line = shift;
1095         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1096             $kvm_user_version = $2;
1097         }
1098     };
1099
1100     eval { run_command("kvm -version", outfunc => $code); };
1101     warn $@ if $@;
1102
1103     return $kvm_user_version;
1104
1105 }
1106
1107 my $kernel_has_vhost_net = -c '/dev/vhost-net';
1108
1109 sub valid_drive_names {
1110     # order is important - used to autoselect boot disk
1111     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1112             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1113             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1114             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))));
1115 }
1116
1117 sub is_valid_drivename {
1118     my $dev = shift;
1119
1120     return defined($drivename_hash->{$dev});
1121 }
1122
1123 sub option_exists {
1124     my $key = shift;
1125     return defined($confdesc->{$key});
1126 }
1127
1128 sub nic_models {
1129     return $nic_model_list;
1130 }
1131
1132 sub os_list_description {
1133
1134     return {
1135         other => 'Other',
1136         wxp => 'Windows XP',
1137         w2k => 'Windows 2000',
1138         w2k3 =>, 'Windows 2003',
1139         w2k8 => 'Windows 2008',
1140         wvista => 'Windows Vista',
1141         win7 => 'Windows 7',
1142         win8 => 'Windows 8/2012',
1143         l24 => 'Linux 2.4',
1144         l26 => 'Linux 2.6',
1145     };
1146 }
1147
1148 my $cdrom_path;
1149
1150 sub get_cdrom_path {
1151
1152     return  $cdrom_path if $cdrom_path;
1153
1154     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1155     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1156     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1157 }
1158
1159 sub get_iso_path {
1160     my ($storecfg, $vmid, $cdrom) = @_;
1161
1162     if ($cdrom eq 'cdrom') {
1163         return get_cdrom_path();
1164     } elsif ($cdrom eq 'none') {
1165         return '';
1166     } elsif ($cdrom =~ m|^/|) {
1167         return $cdrom;
1168     } else {
1169         return PVE::Storage::path($storecfg, $cdrom);
1170     }
1171 }
1172
1173 # try to convert old style file names to volume IDs
1174 sub filename_to_volume_id {
1175     my ($vmid, $file, $media) = @_;
1176
1177     if (!($file eq 'none' || $file eq 'cdrom' ||
1178           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1179
1180         return undef if $file =~ m|/|;
1181
1182         if ($media && $media eq 'cdrom') {
1183             $file = "local:iso/$file";
1184         } else {
1185             $file = "local:$vmid/$file";
1186         }
1187     }
1188
1189     return $file;
1190 }
1191
1192 sub verify_media_type {
1193     my ($opt, $vtype, $media) = @_;
1194
1195     return if !$media;
1196
1197     my $etype;
1198     if ($media eq 'disk') {
1199         $etype = 'images';
1200     } elsif ($media eq 'cdrom') {
1201         $etype = 'iso';
1202     } else {
1203         die "internal error";
1204     }
1205
1206     return if ($vtype eq $etype);
1207
1208     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1209 }
1210
1211 sub cleanup_drive_path {
1212     my ($opt, $storecfg, $drive) = @_;
1213
1214     # try to convert filesystem paths to volume IDs
1215
1216     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1217         ($drive->{file} !~ m|^/dev/.+|) &&
1218         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1219         ($drive->{file} !~ m/^\d+$/)) {
1220         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1221         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1222         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1223         verify_media_type($opt, $vtype, $drive->{media});
1224         $drive->{file} = $volid;
1225     }
1226
1227     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1228 }
1229
1230 sub parse_hotplug_features {
1231     my ($data) = @_;
1232
1233     my $res = {};
1234
1235     return $res if $data eq '0';
1236
1237     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1238
1239     foreach my $feature (PVE::Tools::split_list($data)) {
1240         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1241             $res->{$1} = 1;
1242         } else {
1243             die "invalid hotplug feature '$feature'\n";
1244         }
1245     }
1246     return $res;
1247 }
1248
1249 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1250 sub pve_verify_hotplug_features {
1251     my ($value, $noerr) = @_;
1252
1253     return $value if parse_hotplug_features($value);
1254
1255     return undef if $noerr;
1256
1257     die "unable to parse hotplug option\n";
1258 }
1259
1260 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1261 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1262 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1263 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1264 #        [,iothread=on][,serial=serial][,model=model]
1265
1266 sub parse_drive {
1267     my ($key, $data) = @_;
1268
1269     my ($interface, $index);
1270
1271     if ($key =~ m/^([^\d]+)(\d+)$/) {
1272         $interface = $1;
1273         $index = $2;
1274     } else {
1275         return undef;
1276     }
1277
1278     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1279                                      : $confdesc->{$key}->{format};
1280     if (!$desc) {
1281         warn "invalid drive key: $key\n";
1282         return undef;
1283     }
1284     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1285     return undef if !$res;
1286     $res->{interface} = $interface;
1287     $res->{index} = $index;
1288
1289     my $error = 0;
1290     foreach my $opt (qw(bps bps_rd bps_wr)) {
1291         if (my $bps = defined(delete $res->{$opt})) {
1292             if (defined($res->{"m$opt"})) {
1293                 warn "both $opt and m$opt specified\n";
1294                 ++$error;
1295                 next;
1296             }
1297             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1298         }
1299     }
1300     return undef if $error;
1301
1302     return undef if $res->{mbps_rd} && $res->{mbps};
1303     return undef if $res->{mbps_wr} && $res->{mbps};
1304     return undef if $res->{iops_rd} && $res->{iops};
1305     return undef if $res->{iops_wr} && $res->{iops};
1306
1307     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1308         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1309         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1310         return undef if $res->{interface} eq 'virtio';
1311     }
1312
1313     if (my $size = $res->{size}) {
1314         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1315     }
1316
1317     return $res;
1318 }
1319
1320 sub print_drive {
1321     my ($vmid, $drive) = @_;
1322     my $data = { %$drive };
1323     delete $data->{$_} for qw(index interface);
1324     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1325 }
1326
1327 sub scsi_inquiry {
1328     my($fh, $noerr) = @_;
1329
1330     my $SG_IO = 0x2285;
1331     my $SG_GET_VERSION_NUM = 0x2282;
1332
1333     my $versionbuf = "\x00" x 8;
1334     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1335     if (!$ret) {
1336         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1337         return undef;
1338     }
1339     my $version = unpack("I", $versionbuf);
1340     if ($version < 30000) {
1341         die "scsi generic interface too old\n"  if !$noerr;
1342         return undef;
1343     }
1344
1345     my $buf = "\x00" x 36;
1346     my $sensebuf = "\x00" x 8;
1347     my $cmd = pack("C x3 C x1", 0x12, 36);
1348
1349     # see /usr/include/scsi/sg.h
1350     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1351
1352     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1353                       length($sensebuf), 0, length($buf), $buf,
1354                       $cmd, $sensebuf, 6000);
1355
1356     $ret = ioctl($fh, $SG_IO, $packet);
1357     if (!$ret) {
1358         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1359         return undef;
1360     }
1361
1362     my @res = unpack($sg_io_hdr_t, $packet);
1363     if ($res[17] || $res[18]) {
1364         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1365         return undef;
1366     }
1367
1368     my $res = {};
1369     (my $byte0, my $byte1, $res->{vendor},
1370      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1371
1372     $res->{removable} = $byte1 & 128 ? 1 : 0;
1373     $res->{type} = $byte0 & 31;
1374
1375     return $res;
1376 }
1377
1378 sub path_is_scsi {
1379     my ($path) = @_;
1380
1381     my $fh = IO::File->new("+<$path") || return undef;
1382     my $res = scsi_inquiry($fh, 1);
1383     close($fh);
1384
1385     return $res;
1386 }
1387
1388 sub machine_type_is_q35 {
1389     my ($conf) = @_;
1390
1391     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1392 }
1393
1394 sub print_tabletdevice_full {
1395     my ($conf) = @_;
1396
1397     my $q35 = machine_type_is_q35($conf);
1398
1399     # we use uhci for old VMs because tablet driver was buggy in older qemu
1400     my $usbbus = $q35 ? "ehci" : "uhci";
1401
1402     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1403 }
1404
1405 sub print_drivedevice_full {
1406     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1407
1408     my $device = '';
1409     my $maxdev = 0;
1410
1411     if ($drive->{interface} eq 'virtio') {
1412         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1413         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1414         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1415     } elsif ($drive->{interface} eq 'scsi') {
1416
1417         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1418         my $unit = $drive->{index} % $maxdev;
1419         my $devicetype = 'hd';
1420         my $path = '';
1421         if (drive_is_cdrom($drive)) {
1422             $devicetype = 'cd';
1423         } else {
1424             if ($drive->{file} =~ m|^/|) {
1425                 $path = $drive->{file};
1426                 if (my $info = path_is_scsi($path)) {
1427                     if ($info->{type} == 0) {
1428                         $devicetype = 'block';
1429                     } elsif ($info->{type} == 1) { # tape
1430                         $devicetype = 'generic';
1431                     }
1432                 }
1433             } else {
1434                  $path = PVE::Storage::path($storecfg, $drive->{file});
1435             }
1436
1437             if($path =~ m/^iscsi\:\/\//){
1438                 $devicetype = 'generic';
1439             }
1440         }
1441
1442         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1443             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1444         } else {
1445             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1446         }
1447
1448     } elsif ($drive->{interface} eq 'ide'){
1449         $maxdev = 2;
1450         my $controller = int($drive->{index} / $maxdev);
1451         my $unit = $drive->{index} % $maxdev;
1452         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1453
1454         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1455         if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
1456             $model = URI::Escape::uri_unescape($model);
1457             $device .= ",model=$model";
1458         }
1459     } elsif ($drive->{interface} eq 'sata'){
1460         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1461         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1462         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1463     } elsif ($drive->{interface} eq 'usb') {
1464         die "implement me";
1465         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1466     } else {
1467         die "unsupported interface type";
1468     }
1469
1470     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1471
1472     return $device;
1473 }
1474
1475 sub get_initiator_name {
1476     my $initiator;
1477
1478     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1479     while (defined(my $line = <$fh>)) {
1480         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1481         $initiator = $1;
1482         last;
1483     }
1484     $fh->close();
1485
1486     return $initiator;
1487 }
1488
1489 sub print_drive_full {
1490     my ($storecfg, $vmid, $drive) = @_;
1491
1492     my $path;
1493     my $volid = $drive->{file};
1494     my $format;
1495
1496     if (drive_is_cdrom($drive)) {
1497         $path = get_iso_path($storecfg, $vmid, $volid);
1498     } else {
1499         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1500         if ($storeid) {
1501             $path = PVE::Storage::path($storecfg, $volid);
1502             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1503             $format = qemu_img_format($scfg, $volname);
1504         } else {
1505             $path = $volid;
1506             $format = "raw";
1507         }
1508    }
1509
1510     my $opts = '';
1511     my @qemu_drive_options = qw(heads secs cyls trans media format cache snapshot rerror werror aio discard iops iops_rd iops_wr iops_max iops_rd_max iops_wr_max);
1512     foreach my $o (@qemu_drive_options) {
1513         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1514     }
1515     if (my $serial = $drive->{serial}) {
1516         $serial = URI::Escape::uri_unescape($serial);
1517         $opts .= ",serial=$serial";
1518     }
1519
1520     $opts .= ",format=$format" if $format && !$drive->{format};
1521
1522     foreach my $o (qw(bps bps_rd bps_wr)) {
1523         my $v = $drive->{"m$o"};
1524         $opts .= ",$o=" . int($v*1024*1024) if $v;
1525     }
1526
1527     my $cache_direct = 0;
1528
1529     if (my $cache = $drive->{cache}) {
1530         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1531     } elsif (!drive_is_cdrom($drive)) {
1532         $opts .= ",cache=none";
1533         $cache_direct = 1;
1534     }
1535
1536     # aio native works only with O_DIRECT
1537     if (!$drive->{aio}) {
1538         if($cache_direct) {
1539             $opts .= ",aio=native";
1540         } else {
1541             $opts .= ",aio=threads";
1542         }
1543     }
1544
1545     if (!drive_is_cdrom($drive)) {
1546         my $detectzeroes;
1547         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1548             $detectzeroes = 'off';
1549         } elsif ($drive->{discard}) {
1550             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1551         } else {
1552             # This used to be our default with discard not being specified:
1553             $detectzeroes = 'on';
1554         }
1555         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1556     }
1557
1558     my $pathinfo = $path ? "file=$path," : '';
1559
1560     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1561 }
1562
1563 sub print_netdevice_full {
1564     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1565
1566     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1567
1568     my $device = $net->{model};
1569     if ($net->{model} eq 'virtio') {
1570          $device = 'virtio-net-pci';
1571      };
1572
1573     my $pciaddr = print_pci_addr("$netid", $bridges);
1574     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1575     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1576         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1577         my $vectors = $net->{queues} * 2 + 2;
1578         $tmpstr .= ",vectors=$vectors,mq=on";
1579     }
1580     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1581
1582     if ($use_old_bios_files) {
1583         my $romfile;
1584         if ($device eq 'virtio-net-pci') {
1585             $romfile = 'pxe-virtio.rom';
1586         } elsif ($device eq 'e1000') {
1587             $romfile = 'pxe-e1000.rom';
1588         } elsif ($device eq 'ne2k') {
1589             $romfile = 'pxe-ne2k_pci.rom';
1590         } elsif ($device eq 'pcnet') {
1591             $romfile = 'pxe-pcnet.rom';
1592         } elsif ($device eq 'rtl8139') {
1593             $romfile = 'pxe-rtl8139.rom';
1594         }
1595         $tmpstr .= ",romfile=$romfile" if $romfile;
1596     }
1597
1598     return $tmpstr;
1599 }
1600
1601 sub print_netdev_full {
1602     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1603
1604     my $i = '';
1605     if ($netid =~ m/^net(\d+)$/) {
1606         $i = int($1);
1607     }
1608
1609     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1610
1611     my $ifname = "tap${vmid}i$i";
1612
1613     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1614     die "interface name '$ifname' is too long (max 15 character)\n"
1615         if length($ifname) >= 16;
1616
1617     my $vhostparam = '';
1618     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1619
1620     my $vmname = $conf->{name} || "vm$vmid";
1621
1622     my $netdev = "";
1623     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1624
1625     if ($net->{bridge}) {
1626         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1627     } else {
1628         $netdev = "type=user,id=$netid,hostname=$vmname";
1629     }
1630
1631     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1632
1633     return $netdev;
1634 }
1635
1636 sub drive_is_cdrom {
1637     my ($drive) = @_;
1638
1639     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1640
1641 }
1642
1643 sub parse_number_sets {
1644     my ($set) = @_;
1645     my $res = [];
1646     foreach my $part (split(/;/, $set)) {
1647         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1648             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1649             push @$res, [ $1, $2 ];
1650         } else {
1651             die "invalid range: $part\n";
1652         }
1653     }
1654     return $res;
1655 }
1656
1657 sub parse_numa {
1658     my ($data) = @_;
1659
1660     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
1661     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1662     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1663     return $res;
1664 }
1665
1666 sub parse_hostpci {
1667     my ($value) = @_;
1668
1669     return undef if !$value;
1670
1671     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
1672
1673     my @idlist = split(/;/, $res->{host});
1674     delete $res->{host};
1675     foreach my $id (@idlist) {
1676         if ($id =~ /^$PCIRE$/) {
1677             push @{$res->{pciid}}, { id => $1, function => ($2//'0') };
1678         } else {
1679             # should have been caught by parse_property_string already
1680             die "failed to parse PCI id: $id\n";
1681         }
1682     }
1683     return $res;
1684 }
1685
1686 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1687 sub parse_net {
1688     my ($data) = @_;
1689
1690     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
1691     if ($@) {
1692         warn $@;
1693         return undef;
1694     }
1695     $res->{macaddr} = PVE::Tools::random_ether_addr() if !defined($res->{macaddr});
1696     return $res;
1697 }
1698
1699 sub print_net {
1700     my $net = shift;
1701
1702     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1703 }
1704
1705 sub add_random_macs {
1706     my ($settings) = @_;
1707
1708     foreach my $opt (keys %$settings) {
1709         next if $opt !~ m/^net(\d+)$/;
1710         my $net = parse_net($settings->{$opt});
1711         next if !$net;
1712         $settings->{$opt} = print_net($net);
1713     }
1714 }
1715
1716 sub vm_is_volid_owner {
1717     my ($storecfg, $vmid, $volid) = @_;
1718
1719     if ($volid !~  m|^/|) {
1720         my ($path, $owner);
1721         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1722         if ($owner && ($owner == $vmid)) {
1723             return 1;
1724         }
1725     }
1726
1727     return undef;
1728 }
1729
1730 sub split_flagged_list {
1731     my $text = shift || '';
1732     $text =~ s/[,;]/ /g;
1733     $text =~ s/^\s+//;
1734     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
1735 }
1736
1737 sub join_flagged_list {
1738     my ($how, $lst) = @_;
1739     join $how, map { $lst->{$_} . $_ } keys %$lst;
1740 }
1741
1742 sub vmconfig_delete_pending_option {
1743     my ($conf, $key, $force) = @_;
1744
1745     delete $conf->{pending}->{$key};
1746     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1747     $pending_delete_hash->{$key} = $force ? '!' : '';
1748     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1749 }
1750
1751 sub vmconfig_undelete_pending_option {
1752     my ($conf, $key) = @_;
1753
1754     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1755     delete $pending_delete_hash->{$key};
1756
1757     if (%$pending_delete_hash) {
1758         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1759     } else {
1760         delete $conf->{pending}->{delete};
1761     }
1762 }
1763
1764 sub vmconfig_register_unused_drive {
1765     my ($storecfg, $vmid, $conf, $drive) = @_;
1766
1767     if (!drive_is_cdrom($drive)) {
1768         my $volid = $drive->{file};
1769         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1770             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1771         }
1772     }
1773 }
1774
1775 sub vmconfig_cleanup_pending {
1776     my ($conf) = @_;
1777
1778     # remove pending changes when nothing changed
1779     my $changes;
1780     foreach my $opt (keys %{$conf->{pending}}) {
1781         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
1782             $changes = 1;
1783             delete $conf->{pending}->{$opt};
1784         }
1785     }
1786
1787     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
1788     my $pending_delete_hash = {};
1789     while (my ($opt, $force) = each %$current_delete_hash) {
1790         if (defined($conf->{$opt})) {
1791             $pending_delete_hash->{$opt} = $force;
1792         } else {
1793             $changes = 1;
1794         }
1795     }
1796
1797     if (%$pending_delete_hash) {
1798         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1799     } else {
1800         delete $conf->{pending}->{delete};
1801     }
1802
1803     return $changes;
1804 }
1805
1806 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
1807 my $smbios1_fmt = {
1808     uuid => {
1809         type => 'string',
1810         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1811         format_description => 'UUID',
1812         description => "Set SMBIOS1 UUID.",
1813         optional => 1,
1814     },
1815     version => {
1816         type => 'string',
1817         pattern => '\S+',
1818         format_description => 'string',
1819         description => "Set SMBIOS1 version.",
1820         optional => 1,
1821     },
1822     serial => {
1823         type => 'string',
1824         pattern => '\S+',
1825         format_description => 'string',
1826         description => "Set SMBIOS1 serial number.",
1827         optional => 1,
1828     },
1829     manufacturer => {
1830         type => 'string',
1831         pattern => '\S+',
1832         format_description => 'string',
1833         description => "Set SMBIOS1 manufacturer.",
1834         optional => 1,
1835     },
1836     product => {
1837         type => 'string',
1838         pattern => '\S+',
1839         format_description => 'string',
1840         description => "Set SMBIOS1 product ID.",
1841         optional => 1,
1842     },
1843     sku => {
1844         type => 'string',
1845         pattern => '\S+',
1846         format_description => 'string',
1847         description => "Set SMBIOS1 SKU string.",
1848         optional => 1,
1849     },
1850     family => {
1851         type => 'string',
1852         pattern => '\S+',
1853         format_description => 'string',
1854         description => "Set SMBIOS1 family string.",
1855         optional => 1,
1856     },
1857 };
1858
1859 sub parse_smbios1 {
1860     my ($data) = @_;
1861
1862     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
1863     warn $@ if $@;
1864     return $res;
1865 }
1866
1867 sub print_smbios1 {
1868     my ($smbios1) = @_;
1869     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
1870 }
1871
1872 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
1873
1874 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
1875 sub verify_bootdisk {
1876     my ($value, $noerr) = @_;
1877
1878     return $value if is_valid_drivename($value);
1879
1880     return undef if $noerr;
1881
1882     die "invalid boot disk '$value'\n";
1883 }
1884
1885 sub parse_watchdog {
1886     my ($value) = @_;
1887
1888     return undef if !$value;
1889
1890     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
1891     warn $@ if $@;
1892     return $res;
1893 }
1894
1895 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
1896 sub verify_usb_device {
1897     my ($value, $noerr) = @_;
1898
1899     return $value if parse_usb_device($value);
1900
1901     return undef if $noerr;
1902
1903     die "unable to parse usb device\n";
1904 }
1905
1906 # add JSON properties for create and set function
1907 sub json_config_properties {
1908     my $prop = shift;
1909
1910     foreach my $opt (keys %$confdesc) {
1911         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
1912         $prop->{$opt} = $confdesc->{$opt};
1913     }
1914
1915     return $prop;
1916 }
1917
1918 sub check_type {
1919     my ($key, $value) = @_;
1920
1921     die "unknown setting '$key'\n" if !$confdesc->{$key};
1922
1923     my $type = $confdesc->{$key}->{type};
1924
1925     if (!defined($value)) {
1926         die "got undefined value\n";
1927     }
1928
1929     if ($value =~ m/[\n\r]/) {
1930         die "property contains a line feed\n";
1931     }
1932
1933     if ($type eq 'boolean') {
1934         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
1935         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
1936         die "type check ('boolean') failed - got '$value'\n";
1937     } elsif ($type eq 'integer') {
1938         return int($1) if $value =~ m/^(\d+)$/;
1939         die "type check ('integer') failed - got '$value'\n";
1940     } elsif ($type eq 'number') {
1941         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
1942         die "type check ('number') failed - got '$value'\n";
1943     } elsif ($type eq 'string') {
1944         if (my $fmt = $confdesc->{$key}->{format}) {
1945             if ($fmt eq 'pve-qm-drive') {
1946                 # special case - we need to pass $key to parse_drive()
1947                 my $drive = parse_drive($key, $value);
1948                 return $value if $drive;
1949                 die "unable to parse drive options\n";
1950             }
1951             PVE::JSONSchema::check_format($fmt, $value);
1952             return $value;
1953         }
1954         $value =~ s/^\"(.*)\"$/$1/;
1955         return $value;
1956     } else {
1957         die "internal error"
1958     }
1959 }
1960
1961 sub check_iommu_support{
1962     #fixme : need to check IOMMU support
1963     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
1964
1965     my $iommu=1;
1966     return $iommu;
1967
1968 }
1969
1970 sub touch_config {
1971     my ($vmid) = @_;
1972
1973     my $conf = PVE::QemuConfig->config_file($vmid);
1974     utime undef, undef, $conf;
1975 }
1976
1977 sub destroy_vm {
1978     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
1979
1980     my $conffile = PVE::QemuConfig->config_file($vmid);
1981
1982     my $conf = PVE::QemuConfig->load_config($vmid);
1983
1984     PVE::QemuConfig->check_lock($conf) if !$skiplock;
1985
1986     # only remove disks owned by this VM
1987     foreach_drive($conf, sub {
1988         my ($ds, $drive) = @_;
1989
1990         return if drive_is_cdrom($drive);
1991
1992         my $volid = $drive->{file};
1993
1994         return if !$volid || $volid =~ m|^/|;
1995
1996         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
1997         return if !$path || !$owner || ($owner != $vmid);
1998
1999         PVE::Storage::vdisk_free($storecfg, $volid);
2000     });
2001
2002     if ($keep_empty_config) {
2003         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2004     } else {
2005         unlink $conffile;
2006     }
2007
2008     # also remove unused disk
2009     eval {
2010         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2011
2012         eval {
2013             PVE::Storage::foreach_volid($dl, sub {
2014                 my ($volid, $sid, $volname, $d) = @_;
2015                 PVE::Storage::vdisk_free($storecfg, $volid);
2016             });
2017         };
2018         warn $@ if $@;
2019
2020     };
2021     warn $@ if $@;
2022 }
2023
2024 sub parse_vm_config {
2025     my ($filename, $raw) = @_;
2026
2027     return undef if !defined($raw);
2028
2029     my $res = {
2030         digest => Digest::SHA::sha1_hex($raw),
2031         snapshots => {},
2032         pending => {},
2033     };
2034
2035     $filename =~ m|/qemu-server/(\d+)\.conf$|
2036         || die "got strange filename '$filename'";
2037
2038     my $vmid = $1;
2039
2040     my $conf = $res;
2041     my $descr;
2042     my $section = '';
2043
2044     my @lines = split(/\n/, $raw);
2045     foreach my $line (@lines) {
2046         next if $line =~ m/^\s*$/;
2047
2048         if ($line =~ m/^\[PENDING\]\s*$/i) {
2049             $section = 'pending';
2050             if (defined($descr)) {
2051                 $descr =~ s/\s+$//;
2052                 $conf->{description} = $descr;
2053             }
2054             $descr = undef;
2055             $conf = $res->{$section} = {};
2056             next;
2057
2058         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2059             $section = $1;
2060             if (defined($descr)) {
2061                 $descr =~ s/\s+$//;
2062                 $conf->{description} = $descr;
2063             }
2064             $descr = undef;
2065             $conf = $res->{snapshots}->{$section} = {};
2066             next;
2067         }
2068
2069         if ($line =~ m/^\#(.*)\s*$/) {
2070             $descr = '' if !defined($descr);
2071             $descr .= PVE::Tools::decode_text($1) . "\n";
2072             next;
2073         }
2074
2075         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2076             $descr = '' if !defined($descr);
2077             $descr .= PVE::Tools::decode_text($2);
2078         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2079             $conf->{snapstate} = $1;
2080         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2081             my $key = $1;
2082             my $value = $2;
2083             $conf->{$key} = $value;
2084         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2085             my $value = $1;
2086             if ($section eq 'pending') {
2087                 $conf->{delete} = $value; # we parse this later
2088             } else {
2089                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2090             }
2091         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
2092             my $key = $1;
2093             my $value = $2;
2094             eval { $value = check_type($key, $value); };
2095             if ($@) {
2096                 warn "vm $vmid - unable to parse value of '$key' - $@";
2097             } else {
2098                 my $fmt = $confdesc->{$key}->{format};
2099                 if ($fmt && $fmt eq 'pve-qm-drive') {
2100                     my $v = parse_drive($key, $value);
2101                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2102                         $v->{file} = $volid;
2103                         $value = print_drive($vmid, $v);
2104                     } else {
2105                         warn "vm $vmid - unable to parse value of '$key'\n";
2106                         next;
2107                     }
2108                 }
2109
2110                 if ($key eq 'cdrom') {
2111                     $conf->{ide2} = $value;
2112                 } else {
2113                     $conf->{$key} = $value;
2114                 }
2115             }
2116         }
2117     }
2118
2119     if (defined($descr)) {
2120         $descr =~ s/\s+$//;
2121         $conf->{description} = $descr;
2122     }
2123     delete $res->{snapstate}; # just to be sure
2124
2125     return $res;
2126 }
2127
2128 sub write_vm_config {
2129     my ($filename, $conf) = @_;
2130
2131     delete $conf->{snapstate}; # just to be sure
2132
2133     if ($conf->{cdrom}) {
2134         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2135         $conf->{ide2} = $conf->{cdrom};
2136         delete $conf->{cdrom};
2137     }
2138
2139     # we do not use 'smp' any longer
2140     if ($conf->{sockets}) {
2141         delete $conf->{smp};
2142     } elsif ($conf->{smp}) {
2143         $conf->{sockets} = $conf->{smp};
2144         delete $conf->{cores};
2145         delete $conf->{smp};
2146     }
2147
2148     my $used_volids = {};
2149
2150     my $cleanup_config = sub {
2151         my ($cref, $pending, $snapname) = @_;
2152
2153         foreach my $key (keys %$cref) {
2154             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2155                 $key eq 'snapstate' || $key eq 'pending';
2156             my $value = $cref->{$key};
2157             if ($key eq 'delete') {
2158                 die "propertry 'delete' is only allowed in [PENDING]\n"
2159                     if !$pending;
2160                 # fixme: check syntax?
2161                 next;
2162             }
2163             eval { $value = check_type($key, $value); };
2164             die "unable to parse value of '$key' - $@" if $@;
2165
2166             $cref->{$key} = $value;
2167
2168             if (!$snapname && is_valid_drivename($key)) {
2169                 my $drive = parse_drive($key, $value);
2170                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2171             }
2172         }
2173     };
2174
2175     &$cleanup_config($conf);
2176
2177     &$cleanup_config($conf->{pending}, 1);
2178
2179     foreach my $snapname (keys %{$conf->{snapshots}}) {
2180         die "internal error" if $snapname eq 'pending';
2181         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2182     }
2183
2184     # remove 'unusedX' settings if we re-add a volume
2185     foreach my $key (keys %$conf) {
2186         my $value = $conf->{$key};
2187         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2188             delete $conf->{$key};
2189         }
2190     }
2191
2192     my $generate_raw_config = sub {
2193         my ($conf, $pending) = @_;
2194
2195         my $raw = '';
2196
2197         # add description as comment to top of file
2198         if (defined(my $descr = $conf->{description})) {
2199             if ($descr) {
2200                 foreach my $cl (split(/\n/, $descr)) {
2201                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2202                 }
2203             } else {
2204                 $raw .= "#\n" if $pending;
2205             }
2206         }
2207
2208         foreach my $key (sort keys %$conf) {
2209             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2210             $raw .= "$key: $conf->{$key}\n";
2211         }
2212         return $raw;
2213     };
2214
2215     my $raw = &$generate_raw_config($conf);
2216
2217     if (scalar(keys %{$conf->{pending}})){
2218         $raw .= "\n[PENDING]\n";
2219         $raw .= &$generate_raw_config($conf->{pending}, 1);
2220     }
2221
2222     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2223         $raw .= "\n[$snapname]\n";
2224         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2225     }
2226
2227     return $raw;
2228 }
2229
2230 sub load_defaults {
2231
2232     my $res = {};
2233
2234     # we use static defaults from our JSON schema configuration
2235     foreach my $key (keys %$confdesc) {
2236         if (defined(my $default = $confdesc->{$key}->{default})) {
2237             $res->{$key} = $default;
2238         }
2239     }
2240
2241     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2242     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2243
2244     return $res;
2245 }
2246
2247 sub config_list {
2248     my $vmlist = PVE::Cluster::get_vmlist();
2249     my $res = {};
2250     return $res if !$vmlist || !$vmlist->{ids};
2251     my $ids = $vmlist->{ids};
2252
2253     foreach my $vmid (keys %$ids) {
2254         my $d = $ids->{$vmid};
2255         next if !$d->{node} || $d->{node} ne $nodename;
2256         next if !$d->{type} || $d->{type} ne 'qemu';
2257         $res->{$vmid}->{exists} = 1;
2258     }
2259     return $res;
2260 }
2261
2262 # test if VM uses local resources (to prevent migration)
2263 sub check_local_resources {
2264     my ($conf, $noerr) = @_;
2265
2266     my $loc_res = 0;
2267
2268     $loc_res = 1 if $conf->{hostusb}; # old syntax
2269     $loc_res = 1 if $conf->{hostpci}; # old syntax
2270
2271     foreach my $k (keys %$conf) {
2272         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2273         # sockets are safe: they will recreated be on the target side post-migrate
2274         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2275         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2276     }
2277
2278     die "VM uses local resources\n" if $loc_res && !$noerr;
2279
2280     return $loc_res;
2281 }
2282
2283 # check if used storages are available on all nodes (use by migrate)
2284 sub check_storage_availability {
2285     my ($storecfg, $conf, $node) = @_;
2286
2287     foreach_drive($conf, sub {
2288         my ($ds, $drive) = @_;
2289
2290         my $volid = $drive->{file};
2291         return if !$volid;
2292
2293         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2294         return if !$sid;
2295
2296         # check if storage is available on both nodes
2297         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2298         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2299    });
2300 }
2301
2302 # list nodes where all VM images are available (used by has_feature API)
2303 sub shared_nodes {
2304     my ($conf, $storecfg) = @_;
2305
2306     my $nodelist = PVE::Cluster::get_nodelist();
2307     my $nodehash = { map { $_ => 1 } @$nodelist };
2308     my $nodename = PVE::INotify::nodename();
2309
2310     foreach_drive($conf, sub {
2311         my ($ds, $drive) = @_;
2312
2313         my $volid = $drive->{file};
2314         return if !$volid;
2315
2316         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2317         if ($storeid) {
2318             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2319             if ($scfg->{disable}) {
2320                 $nodehash = {};
2321             } elsif (my $avail = $scfg->{nodes}) {
2322                 foreach my $node (keys %$nodehash) {
2323                     delete $nodehash->{$node} if !$avail->{$node};
2324                 }
2325             } elsif (!$scfg->{shared}) {
2326                 foreach my $node (keys %$nodehash) {
2327                     delete $nodehash->{$node} if $node ne $nodename
2328                 }
2329             }
2330         }
2331     });
2332
2333     return $nodehash
2334 }
2335
2336 sub check_cmdline {
2337     my ($pidfile, $pid) = @_;
2338
2339     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2340     if (defined($fh)) {
2341         my $line = <$fh>;
2342         $fh->close;
2343         return undef if !$line;
2344         my @param = split(/\0/, $line);
2345
2346         my $cmd = $param[0];
2347         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2348
2349         for (my $i = 0; $i < scalar (@param); $i++) {
2350             my $p = $param[$i];
2351             next if !$p;
2352             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2353                 my $p = $param[$i+1];
2354                 return 1 if $p && ($p eq $pidfile);
2355                 return undef;
2356             }
2357         }
2358     }
2359     return undef;
2360 }
2361
2362 sub check_running {
2363     my ($vmid, $nocheck, $node) = @_;
2364
2365     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2366
2367     die "unable to find configuration file for VM $vmid - no such machine\n"
2368         if !$nocheck && ! -f $filename;
2369
2370     my $pidfile = pidfile_name($vmid);
2371
2372     if (my $fd = IO::File->new("<$pidfile")) {
2373         my $st = stat($fd);
2374         my $line = <$fd>;
2375         close($fd);
2376
2377         my $mtime = $st->mtime;
2378         if ($mtime > time()) {
2379             warn "file '$filename' modified in future\n";
2380         }
2381
2382         if ($line =~ m/^(\d+)$/) {
2383             my $pid = $1;
2384             if (check_cmdline($pidfile, $pid)) {
2385                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2386                     return $pid;
2387                 }
2388             }
2389         }
2390     }
2391
2392     return undef;
2393 }
2394
2395 sub vzlist {
2396
2397     my $vzlist = config_list();
2398
2399     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2400
2401     while (defined(my $de = $fd->read)) {
2402         next if $de !~ m/^(\d+)\.pid$/;
2403         my $vmid = $1;
2404         next if !defined($vzlist->{$vmid});
2405         if (my $pid = check_running($vmid)) {
2406             $vzlist->{$vmid}->{pid} = $pid;
2407         }
2408     }
2409
2410     return $vzlist;
2411 }
2412
2413 sub disksize {
2414     my ($storecfg, $conf) = @_;
2415
2416     my $bootdisk = $conf->{bootdisk};
2417     return undef if !$bootdisk;
2418     return undef if !is_valid_drivename($bootdisk);
2419
2420     return undef if !$conf->{$bootdisk};
2421
2422     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2423     return undef if !defined($drive);
2424
2425     return undef if drive_is_cdrom($drive);
2426
2427     my $volid = $drive->{file};
2428     return undef if !$volid;
2429
2430     return $drive->{size};
2431 }
2432
2433 my $last_proc_pid_stat;
2434
2435 # get VM status information
2436 # This must be fast and should not block ($full == false)
2437 # We only query KVM using QMP if $full == true (this can be slow)
2438 sub vmstatus {
2439     my ($opt_vmid, $full) = @_;
2440
2441     my $res = {};
2442
2443     my $storecfg = PVE::Storage::config();
2444
2445     my $list = vzlist();
2446     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2447
2448     my $cpucount = $cpuinfo->{cpus} || 1;
2449
2450     foreach my $vmid (keys %$list) {
2451         next if $opt_vmid && ($vmid ne $opt_vmid);
2452
2453         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
2454         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2455
2456         my $d = {};
2457         $d->{pid} = $list->{$vmid}->{pid};
2458
2459         # fixme: better status?
2460         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2461
2462         my $size = disksize($storecfg, $conf);
2463         if (defined($size)) {
2464             $d->{disk} = 0; # no info available
2465             $d->{maxdisk} = $size;
2466         } else {
2467             $d->{disk} = 0;
2468             $d->{maxdisk} = 0;
2469         }
2470
2471         $d->{cpus} = ($conf->{sockets} || 1) * ($conf->{cores} || 1);
2472         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2473         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2474
2475         $d->{name} = $conf->{name} || "VM $vmid";
2476         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024) : 0;
2477
2478         if ($conf->{balloon}) {
2479             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2480             $d->{shares} = defined($conf->{shares}) ? $conf->{shares} : 1000;
2481         }
2482
2483         $d->{uptime} = 0;
2484         $d->{cpu} = 0;
2485         $d->{mem} = 0;
2486
2487         $d->{netout} = 0;
2488         $d->{netin} = 0;
2489
2490         $d->{diskread} = 0;
2491         $d->{diskwrite} = 0;
2492
2493         $d->{template} = PVE::QemuConfig->is_template($conf);
2494
2495         $res->{$vmid} = $d;
2496     }
2497
2498     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2499     foreach my $dev (keys %$netdev) {
2500         next if $dev !~ m/^tap([1-9]\d*)i/;
2501         my $vmid = $1;
2502         my $d = $res->{$vmid};
2503         next if !$d;
2504
2505         $d->{netout} += $netdev->{$dev}->{receive};
2506         $d->{netin} += $netdev->{$dev}->{transmit};
2507
2508         if ($full) {
2509             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2510             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2511         }
2512
2513     }
2514
2515     my $ctime = gettimeofday;
2516
2517     foreach my $vmid (keys %$list) {
2518
2519         my $d = $res->{$vmid};
2520         my $pid = $d->{pid};
2521         next if !$pid;
2522
2523         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2524         next if !$pstat; # not running
2525
2526         my $used = $pstat->{utime} + $pstat->{stime};
2527
2528         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2529
2530         if ($pstat->{vsize}) {
2531             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2532         }
2533
2534         my $old = $last_proc_pid_stat->{$pid};
2535         if (!$old) {
2536             $last_proc_pid_stat->{$pid} = {
2537                 time => $ctime,
2538                 used => $used,
2539                 cpu => 0,
2540             };
2541             next;
2542         }
2543
2544         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2545
2546         if ($dtime > 1000) {
2547             my $dutime = $used -  $old->{used};
2548
2549             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2550             $last_proc_pid_stat->{$pid} = {
2551                 time => $ctime,
2552                 used => $used,
2553                 cpu => $d->{cpu},
2554             };
2555         } else {
2556             $d->{cpu} = $old->{cpu};
2557         }
2558     }
2559
2560     return $res if !$full;
2561
2562     my $qmpclient = PVE::QMPClient->new();
2563
2564     my $ballooncb = sub {
2565         my ($vmid, $resp) = @_;
2566
2567         my $info = $resp->{'return'};
2568         return if !$info->{max_mem};
2569
2570         my $d = $res->{$vmid};
2571
2572         # use memory assigned to VM
2573         $d->{maxmem} = $info->{max_mem};
2574         $d->{balloon} = $info->{actual};
2575
2576         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2577             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2578             $d->{freemem} = $info->{free_mem};
2579         }
2580
2581         $d->{ballooninfo} = $info;
2582     };
2583
2584     my $blockstatscb = sub {
2585         my ($vmid, $resp) = @_;
2586         my $data = $resp->{'return'} || [];
2587         my $totalrdbytes = 0;
2588         my $totalwrbytes = 0;
2589
2590         for my $blockstat (@$data) {
2591             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2592             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2593
2594             $blockstat->{device} =~ s/drive-//;
2595             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2596         }
2597         $res->{$vmid}->{diskread} = $totalrdbytes;
2598         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2599     };
2600
2601     my $statuscb = sub {
2602         my ($vmid, $resp) = @_;
2603
2604         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2605         # this fails if ballon driver is not loaded, so this must be
2606         # the last commnand (following command are aborted if this fails).
2607         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2608
2609         my $status = 'unknown';
2610         if (!defined($status = $resp->{'return'}->{status})) {
2611             warn "unable to get VM status\n";
2612             return;
2613         }
2614
2615         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2616     };
2617
2618     foreach my $vmid (keys %$list) {
2619         next if $opt_vmid && ($vmid ne $opt_vmid);
2620         next if !$res->{$vmid}->{pid}; # not running
2621         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2622     }
2623
2624     $qmpclient->queue_execute(undef, 1);
2625
2626     foreach my $vmid (keys %$list) {
2627         next if $opt_vmid && ($vmid ne $opt_vmid);
2628         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2629     }
2630
2631     return $res;
2632 }
2633
2634 sub foreach_drive {
2635     my ($conf, $func, @param) = @_;
2636
2637     foreach my $ds (valid_drive_names()) {
2638         next if !defined($conf->{$ds});
2639
2640         my $drive = parse_drive($ds, $conf->{$ds});
2641         next if !$drive;
2642
2643         &$func($ds, $drive, @param);
2644     }
2645 }
2646
2647 sub foreach_volid {
2648     my ($conf, $func, @param) = @_;
2649
2650     my $volhash = {};
2651
2652     my $test_volid = sub {
2653         my ($volid, $is_cdrom) = @_;
2654
2655         return if !$volid;
2656
2657         $volhash->{$volid} = $is_cdrom || 0;
2658     };
2659
2660     foreach_drive($conf, sub {
2661         my ($ds, $drive) = @_;
2662         &$test_volid($drive->{file}, drive_is_cdrom($drive));
2663     });
2664
2665     foreach my $snapname (keys %{$conf->{snapshots}}) {
2666         my $snap = $conf->{snapshots}->{$snapname};
2667         &$test_volid($snap->{vmstate}, 0);
2668         foreach_drive($snap, sub {
2669             my ($ds, $drive) = @_;
2670             &$test_volid($drive->{file}, drive_is_cdrom($drive));
2671         });
2672     }
2673
2674     foreach my $volid (keys %$volhash) {
2675         &$func($volid, $volhash->{$volid}, @param);
2676     }
2677 }
2678
2679 sub vga_conf_has_spice {
2680     my ($vga) = @_;
2681
2682     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
2683
2684     return $1 || 1;
2685 }
2686
2687 sub config_to_command {
2688     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2689
2690     my $cmd = [];
2691     my $globalFlags = [];
2692     my $machineFlags = [];
2693     my $rtcFlags = [];
2694     my $cpuFlags = [];
2695     my $devices = [];
2696     my $pciaddr = '';
2697     my $bridges = {};
2698     my $kvmver = kvm_user_version();
2699     my $vernum = 0; # unknown
2700     my $ostype = $conf->{ostype};
2701     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
2702         $vernum = $1*1000000+$2*1000;
2703     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
2704         $vernum = $1*1000000+$2*1000+$3;
2705     }
2706
2707     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
2708
2709     my $have_ovz = -f '/proc/vz/vestat';
2710
2711     my $q35 = machine_type_is_q35($conf);
2712     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2713     my $machine_type = $forcemachine || $conf->{machine};
2714     my $use_old_bios_files = undef;
2715     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
2716
2717     my $cpuunits = defined($conf->{cpuunits}) ?
2718             $conf->{cpuunits} : $defaults->{cpuunits};
2719
2720     push @$cmd, '/usr/bin/kvm';
2721
2722     push @$cmd, '-id', $vmid;
2723
2724     my $use_virtio = 0;
2725
2726     my $qmpsocket = qmp_socket($vmid);
2727     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2728     push @$cmd, '-mon', "chardev=qmp,mode=control";
2729
2730
2731     push @$cmd, '-pidfile' , pidfile_name($vmid);
2732
2733     push @$cmd, '-daemonize';
2734
2735     if ($conf->{smbios1}) {
2736         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
2737     }
2738
2739     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2740         my $ovmfvar = "OVMF_VARS-pure-efi.fd";
2741         my $ovmfvar_src = "/usr/share/kvm/$ovmfvar";
2742         my $ovmfvar_dst = "/tmp/$vmid-$ovmfvar";
2743         PVE::Tools::file_copy($ovmfvar_src, $ovmfvar_dst, 256*1024);
2744         push @$cmd, '-drive', "if=pflash,format=raw,readonly,file=/usr/share/kvm/OVMF-pure-efi.fd";
2745         push @$cmd, '-drive', "if=pflash,format=raw,file=$ovmfvar_dst";
2746     }
2747
2748
2749     # add usb controllers
2750     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $q35, $usbdesc->{format}, $MAX_USB_DEVICES);
2751     push @$devices, @usbcontrollers if @usbcontrollers;
2752     my $vga = $conf->{vga};
2753
2754     my $qxlnum = vga_conf_has_spice($vga);
2755     $vga = 'qxl' if $qxlnum;
2756
2757     if (!$vga) {
2758         if ($conf->{ostype} && ($conf->{ostype} eq 'win8' ||
2759                                 $conf->{ostype} eq 'win7' ||
2760                                 $conf->{ostype} eq 'w2k8')) {
2761             $vga = 'std';
2762         } else {
2763             $vga = 'cirrus';
2764         }
2765     }
2766
2767     # enable absolute mouse coordinates (needed by vnc)
2768     my $tablet;
2769     if (defined($conf->{tablet})) {
2770         $tablet = $conf->{tablet};
2771     } else {
2772         $tablet = $defaults->{tablet};
2773         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
2774         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
2775     }
2776
2777     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
2778
2779     my $kvm_off = 0;
2780     # host pci devices
2781     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
2782         my $d = parse_hostpci($conf->{"hostpci$i"});
2783         next if !$d;
2784
2785         my $pcie = $d->{pcie};
2786         if($pcie){
2787             die "q35 machine model is not enabled" if !$q35;
2788             $pciaddr = print_pcie_addr("hostpci$i");
2789         }else{
2790             $pciaddr = print_pci_addr("hostpci$i", $bridges);
2791         }
2792
2793         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
2794         my $xvga = '';
2795         if ($d->{'x-vga'}) {
2796             $xvga = ',x-vga=on';
2797             $kvm_off = 1;
2798             $vga = 'none';
2799             if ($ostype eq 'win7' || $ostype eq 'win8' || $ostype eq 'w2k8') {
2800                 push @$cpuFlags , 'hv_vendor_id=proxmox';
2801             }
2802             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2803                 $xvga = "";
2804             }
2805         }
2806         my $pcidevices = $d->{pciid};
2807         my $multifunction = 1 if @$pcidevices > 1;
2808
2809         my $j=0;
2810         foreach my $pcidevice (@$pcidevices) {
2811
2812             my $id = "hostpci$i";
2813             $id .= ".$j" if $multifunction;
2814             my $addr = $pciaddr;
2815             $addr .= ".$j" if $multifunction;
2816             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
2817
2818             if($j == 0){
2819                 $devicestr .= "$rombar$xvga";
2820                 $devicestr .= ",multifunction=on" if $multifunction;
2821             }
2822
2823             push @$devices, '-device', $devicestr;
2824             $j++;
2825         }
2826     }
2827
2828     # usb devices
2829     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES);
2830     push @$devices, @usbdevices if @usbdevices;
2831     # serial devices
2832     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2833         if (my $path = $conf->{"serial$i"}) {
2834             if ($path eq 'socket') {
2835                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
2836                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
2837                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2838             } else {
2839                 die "no such serial device\n" if ! -c $path;
2840                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
2841                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2842             }
2843         }
2844     }
2845
2846     # parallel devices
2847     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
2848         if (my $path = $conf->{"parallel$i"}) {
2849             die "no such parallel device\n" if ! -c $path;
2850             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
2851             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
2852             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
2853         }
2854     }
2855
2856     my $vmname = $conf->{name} || "vm$vmid";
2857
2858     push @$cmd, '-name', $vmname;
2859
2860     my $sockets = 1;
2861     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
2862     $sockets = $conf->{sockets} if  $conf->{sockets};
2863
2864     my $cores = $conf->{cores} || 1;
2865
2866     my $maxcpus = $sockets * $cores;
2867
2868     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
2869
2870     my $allowed_vcpus = $cpuinfo->{cpus};
2871
2872     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
2873         if ($allowed_vcpus < $maxcpus);
2874
2875     push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
2876
2877     push @$cmd, '-nodefaults';
2878
2879     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2880
2881     my $bootindex_hash = {};
2882     my $i = 1;
2883     foreach my $o (split(//, $bootorder)) {
2884         $bootindex_hash->{$o} = $i*100;
2885         $i++;
2886     }
2887
2888     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000";
2889
2890     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
2891
2892     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
2893
2894     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
2895
2896     if ($vga && $vga !~ m/^serial\d+$/ && $vga ne 'none'){
2897         my $socket = vnc_socket($vmid);
2898         push @$cmd,  '-vnc', "unix:$socket,x509,password";
2899     } else {
2900         push @$cmd, '-nographic';
2901     }
2902
2903     # time drift fix
2904     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
2905
2906     my $nokvm = defined($conf->{kvm}) && $conf->{kvm} == 0 ? 1 : 0;
2907     my $useLocaltime = $conf->{localtime};
2908
2909     if ($ostype) {
2910         # other, wxp, w2k, w2k3, w2k8, wvista, win7, win8, l24, l26, solaris
2911
2912         if ($ostype =~ m/^w/) { # windows
2913             $useLocaltime = 1 if !defined($conf->{localtime});
2914
2915             # use time drift fix when acpi is enabled
2916             if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
2917                 $tdf = 1 if !defined($conf->{tdf});
2918             }
2919         }
2920
2921         if ($ostype eq 'win7' || $ostype eq 'win8' || $ostype eq 'w2k8' ||
2922             $ostype eq 'wvista') {
2923             push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
2924             push @$cmd, '-no-hpet';
2925             if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
2926                 push @$cpuFlags , 'hv_spinlocks=0x1fff' if !$nokvm;
2927                 push @$cpuFlags , 'hv_vapic' if !$nokvm;
2928                 push @$cpuFlags , 'hv_time' if !$nokvm;
2929
2930                 if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
2931                     push @$cpuFlags , 'hv_reset' if !$nokvm;
2932                     push @$cpuFlags , 'hv_vpindex' if !$nokvm;
2933                     push @$cpuFlags , 'hv_runtime' if !$nokvm;
2934                 }
2935
2936             } else {
2937                 push @$cpuFlags , 'hv_spinlocks=0xffff' if !$nokvm;
2938             }
2939         }
2940
2941         if ($ostype eq 'win7' || $ostype eq 'win8') {
2942             push @$cpuFlags , 'hv_relaxed' if !$nokvm;
2943         }
2944     }
2945
2946     push @$rtcFlags, 'driftfix=slew' if $tdf;
2947
2948     if ($nokvm) {
2949         push @$machineFlags, 'accel=tcg';
2950     } else {
2951         die "No accelerator found!\n" if !$cpuinfo->{hvm};
2952     }
2953
2954     if ($machine_type) {
2955         push @$machineFlags, "type=${machine_type}";
2956     }
2957
2958     if ($conf->{startdate}) {
2959         push @$rtcFlags, "base=$conf->{startdate}";
2960     } elsif ($useLocaltime) {
2961         push @$rtcFlags, 'base=localtime';
2962     }
2963
2964     my $cpu = $nokvm ? "qemu64" : "kvm64";
2965     if (my $cputype = $conf->{cpu}) {
2966         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
2967             or die "Cannot parse cpu description: $cputype\n";
2968         $cpu = $cpuconf->{cputype};
2969         $kvm_off = 1 if $cpuconf->{hidden};
2970     }
2971
2972     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
2973
2974     push @$cpuFlags , '-x2apic'
2975         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
2976
2977     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
2978
2979     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
2980
2981     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
2982
2983         push @$cpuFlags , '+kvm_pv_unhalt' if !$nokvm;
2984         push @$cpuFlags , '+kvm_pv_eoi' if !$nokvm;
2985     }
2986
2987     push @$cpuFlags, 'enforce' if $cpu ne 'host' && !$nokvm;
2988
2989     push @$cpuFlags, 'kvm=off' if $kvm_off;
2990
2991     my $cpu_vendor = $cpu_vendor_list->{$cpu} ||
2992         die "internal error"; # should not happen
2993
2994     push @$cpuFlags, "vendor=${cpu_vendor}"
2995         if $cpu_vendor ne 'default';
2996
2997     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
2998
2999     push @$cmd, '-cpu', $cpu;
3000
3001     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3002
3003     push @$cmd, '-S' if $conf->{freeze};
3004
3005     # set keyboard layout
3006     my $kb = $conf->{keyboard} || $defaults->{keyboard};
3007     push @$cmd, '-k', $kb if $kb;
3008
3009     # enable sound
3010     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3011     #push @$cmd, '-soundhw', 'es1370';
3012     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3013
3014     if($conf->{agent}) {
3015         my $qgasocket = qmp_socket($vmid, 1);
3016         my $pciaddr = print_pci_addr("qga0", $bridges);
3017         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3018         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3019         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3020     }
3021
3022     my $spice_port;
3023
3024     if ($qxlnum) {
3025         if ($qxlnum > 1) {
3026             if ($conf->{ostype} && $conf->{ostype} =~ m/^w/){
3027                 for(my $i = 1; $i < $qxlnum; $i++){
3028                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3029                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3030                 }
3031             } else {
3032                 # assume other OS works like Linux
3033                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3034                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3035             }
3036         }
3037
3038         my $pciaddr = print_pci_addr("spice", $bridges);
3039
3040         my $nodename = PVE::INotify::nodename();
3041         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3042         $spice_port = PVE::Tools::next_spice_port($pfamily);
3043
3044         push @$devices, '-spice', "tls-port=${spice_port},addr=localhost,tls-ciphers=DES-CBC3-SHA,seamless-migration=on";
3045
3046         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3047         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3048         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3049     }
3050
3051     # enable balloon by default, unless explicitly disabled
3052     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3053         $pciaddr = print_pci_addr("balloon0", $bridges);
3054         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3055     }
3056
3057     if ($conf->{watchdog}) {
3058         my $wdopts = parse_watchdog($conf->{watchdog});
3059         $pciaddr = print_pci_addr("watchdog", $bridges);
3060         my $watchdog = $wdopts->{model} || 'i6300esb';
3061         push @$devices, '-device', "$watchdog$pciaddr";
3062         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3063     }
3064
3065     my $vollist = [];
3066     my $scsicontroller = {};
3067     my $ahcicontroller = {};
3068     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3069
3070     # Add iscsi initiator name if available
3071     if (my $initiator = get_initiator_name()) {
3072         push @$devices, '-iscsi', "initiator-name=$initiator";
3073     }
3074
3075     foreach_drive($conf, sub {
3076         my ($ds, $drive) = @_;
3077
3078         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3079             push @$vollist, $drive->{file};
3080         }
3081
3082         $use_virtio = 1 if $ds =~ m/^virtio/;
3083
3084         if (drive_is_cdrom ($drive)) {
3085             if ($bootindex_hash->{d}) {
3086                 $drive->{bootindex} = $bootindex_hash->{d};
3087                 $bootindex_hash->{d} += 1;
3088             }
3089         } else {
3090             if ($bootindex_hash->{c}) {
3091                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3092                 $bootindex_hash->{c} += 1;
3093             }
3094         }
3095
3096         if($drive->{interface} eq 'virtio'){
3097            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3098         }
3099
3100         if ($drive->{interface} eq 'scsi') {
3101
3102             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3103
3104             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3105             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3106
3107             my $iothread = '';
3108             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3109                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3110                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3111             } elsif ($drive->{iothread}) {
3112                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3113             }
3114
3115             my $queues = '';
3116             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3117                 $queues = ",num_queues=$drive->{queues}";
3118             }
3119
3120             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3121             $scsicontroller->{$controller}=1;
3122         }
3123
3124         if ($drive->{interface} eq 'sata') {
3125            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3126            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3127            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3128            $ahcicontroller->{$controller}=1;
3129         }
3130
3131         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3132         push @$devices, '-drive',$drive_cmd;
3133         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3134     });
3135
3136     for (my $i = 0; $i < $MAX_NETS; $i++) {
3137          next if !$conf->{"net$i"};
3138          my $d = parse_net($conf->{"net$i"});
3139          next if !$d;
3140
3141          $use_virtio = 1 if $d->{model} eq 'virtio';
3142
3143          if ($bootindex_hash->{n}) {
3144             $d->{bootindex} = $bootindex_hash->{n};
3145             $bootindex_hash->{n} += 1;
3146          }
3147
3148          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3149          push @$devices, '-netdev', $netdevfull;
3150
3151          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3152          push @$devices, '-device', $netdevicefull;
3153     }
3154
3155     if (!$q35) {
3156         # add pci bridges
3157         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3158            $bridges->{1} = 1;
3159            $bridges->{2} = 1;
3160         }
3161
3162         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3163
3164         while (my ($k, $v) = each %$bridges) {
3165             $pciaddr = print_pci_addr("pci.$k");
3166             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3167         }
3168     }
3169
3170     # add custom args
3171     if ($conf->{args}) {
3172         my $aa = PVE::Tools::split_args($conf->{args});
3173         push @$cmd, @$aa;
3174     }
3175
3176     push @$cmd, @$devices;
3177     push @$cmd, '-rtc', join(',', @$rtcFlags)
3178         if scalar(@$rtcFlags);
3179     push @$cmd, '-machine', join(',', @$machineFlags)
3180         if scalar(@$machineFlags);
3181     push @$cmd, '-global', join(',', @$globalFlags)
3182         if scalar(@$globalFlags);
3183
3184     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3185 }
3186
3187 sub vnc_socket {
3188     my ($vmid) = @_;
3189     return "${var_run_tmpdir}/$vmid.vnc";
3190 }
3191
3192 sub spice_port {
3193     my ($vmid) = @_;
3194
3195     my $res = vm_mon_cmd($vmid, 'query-spice');
3196
3197     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3198 }
3199
3200 sub qmp_socket {
3201     my ($vmid, $qga) = @_;
3202     my $sockettype = $qga ? 'qga' : 'qmp';
3203     return "${var_run_tmpdir}/$vmid.$sockettype";
3204 }
3205
3206 sub pidfile_name {
3207     my ($vmid) = @_;
3208     return "${var_run_tmpdir}/$vmid.pid";
3209 }
3210
3211 sub vm_devices_list {
3212     my ($vmid) = @_;
3213
3214     my $res = vm_mon_cmd($vmid, 'query-pci');
3215     my $devices = {};
3216     foreach my $pcibus (@$res) {
3217         foreach my $device (@{$pcibus->{devices}}) {
3218             next if !$device->{'qdev_id'};
3219             if ($device->{'pci_bridge'}) {
3220                 $devices->{$device->{'qdev_id'}} = 1;
3221                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3222                     next if !$bridge_device->{'qdev_id'};
3223                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3224                     $devices->{$device->{'qdev_id'}}++;
3225                 }
3226             } else {
3227                 $devices->{$device->{'qdev_id'}} = 1;
3228             }
3229         }
3230     }
3231
3232     my $resblock = vm_mon_cmd($vmid, 'query-block');
3233     foreach my $block (@$resblock) {
3234         if($block->{device} =~ m/^drive-(\S+)/){
3235                 $devices->{$1} = 1;
3236         }
3237     }
3238
3239     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3240     foreach my $mice (@$resmice) {
3241         if ($mice->{name} eq 'QEMU HID Tablet') {
3242             $devices->{tablet} = 1;
3243             last;
3244         }
3245     }
3246
3247     # for usb devices there is no query-usb
3248     # but we can iterate over the entries in
3249     # qom-list path=/machine/peripheral
3250     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3251     foreach my $per (@$resperipheral) {
3252         if ($per->{name} =~ m/^usb\d+$/) {
3253             $devices->{$per->{name}} = 1;
3254         }
3255     }
3256
3257     return $devices;
3258 }
3259
3260 sub vm_deviceplug {
3261     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3262
3263     my $q35 = machine_type_is_q35($conf);
3264
3265     my $devices_list = vm_devices_list($vmid);
3266     return 1 if defined($devices_list->{$deviceid});
3267
3268     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3269
3270     if ($deviceid eq 'tablet') {
3271
3272         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3273
3274     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3275
3276         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3277
3278     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3279
3280         qemu_iothread_add($vmid, $deviceid, $device);
3281
3282         qemu_driveadd($storecfg, $vmid, $device);
3283         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3284
3285         qemu_deviceadd($vmid, $devicefull);
3286         eval { qemu_deviceaddverify($vmid, $deviceid); };
3287         if (my $err = $@) {
3288             eval { qemu_drivedel($vmid, $deviceid); };
3289             warn $@ if $@;
3290             die $err;
3291         }
3292
3293     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3294
3295
3296         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3297         my $pciaddr = print_pci_addr($deviceid);
3298         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3299
3300         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3301
3302         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3303             qemu_iothread_add($vmid, $deviceid, $device);
3304             $devicefull .= ",iothread=iothread-$deviceid";
3305         }
3306
3307         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3308             $devicefull .= ",num_queues=$device->{queues}";
3309         }
3310
3311         qemu_deviceadd($vmid, $devicefull);
3312         qemu_deviceaddverify($vmid, $deviceid);
3313
3314     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3315
3316         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3317         qemu_driveadd($storecfg, $vmid, $device);
3318
3319         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3320         eval { qemu_deviceadd($vmid, $devicefull); };
3321         if (my $err = $@) {
3322             eval { qemu_drivedel($vmid, $deviceid); };
3323             warn $@ if $@;
3324             die $err;
3325         }
3326
3327     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3328
3329         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3330
3331         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
3332         my $use_old_bios_files = undef;
3333         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3334
3335         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3336         qemu_deviceadd($vmid, $netdevicefull);
3337         eval { qemu_deviceaddverify($vmid, $deviceid); };
3338         if (my $err = $@) {
3339             eval { qemu_netdevdel($vmid, $deviceid); };
3340             warn $@ if $@;
3341             die $err;
3342         }
3343
3344     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3345
3346         my $bridgeid = $2;
3347         my $pciaddr = print_pci_addr($deviceid);
3348         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3349
3350         qemu_deviceadd($vmid, $devicefull);
3351         qemu_deviceaddverify($vmid, $deviceid);
3352
3353     } else {
3354         die "can't hotplug device '$deviceid'\n";
3355     }
3356
3357     return 1;
3358 }
3359
3360 # fixme: this should raise exceptions on error!
3361 sub vm_deviceunplug {
3362     my ($vmid, $conf, $deviceid) = @_;
3363
3364     my $devices_list = vm_devices_list($vmid);
3365     return 1 if !defined($devices_list->{$deviceid});
3366
3367     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3368
3369     if ($deviceid eq 'tablet') {
3370
3371         qemu_devicedel($vmid, $deviceid);
3372
3373     } elsif ($deviceid =~ m/^usb\d+$/) {
3374
3375         qemu_devicedel($vmid, $deviceid);
3376         qemu_devicedelverify($vmid, $deviceid);
3377
3378     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3379
3380         qemu_devicedel($vmid, $deviceid);
3381         qemu_devicedelverify($vmid, $deviceid);
3382         qemu_drivedel($vmid, $deviceid);
3383         qemu_iothread_del($conf, $vmid, $deviceid);
3384
3385     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3386
3387         qemu_devicedel($vmid, $deviceid);
3388         qemu_devicedelverify($vmid, $deviceid);
3389         qemu_iothread_del($conf, $vmid, $deviceid);
3390
3391     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3392
3393         #qemu 2.3 segfault on drive_del with virtioscsi + iothread
3394         my $device = parse_drive($deviceid, $conf->{$deviceid});
3395         die "virtioscsi with iothread is not hot-unplugglable currently" if $device->{iothread};
3396
3397         qemu_devicedel($vmid, $deviceid);
3398         qemu_drivedel($vmid, $deviceid);
3399         qemu_deletescsihw($conf, $vmid, $deviceid);
3400
3401     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3402
3403         qemu_devicedel($vmid, $deviceid);
3404         qemu_devicedelverify($vmid, $deviceid);
3405         qemu_netdevdel($vmid, $deviceid);
3406
3407     } else {
3408         die "can't unplug device '$deviceid'\n";
3409     }
3410
3411     return 1;
3412 }
3413
3414 sub qemu_deviceadd {
3415     my ($vmid, $devicefull) = @_;
3416
3417     $devicefull = "driver=".$devicefull;
3418     my %options =  split(/[=,]/, $devicefull);
3419
3420     vm_mon_cmd($vmid, "device_add" , %options);
3421 }
3422
3423 sub qemu_devicedel {
3424     my ($vmid, $deviceid) = @_;
3425
3426     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3427 }
3428
3429 sub qemu_iothread_add {
3430     my($vmid, $deviceid, $device) = @_;
3431
3432     if ($device->{iothread}) {
3433         my $iothreads = vm_iothreads_list($vmid);
3434         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3435     }
3436 }
3437
3438 sub qemu_iothread_del {
3439     my($conf, $vmid, $deviceid) = @_;
3440
3441     my $device = parse_drive($deviceid, $conf->{$deviceid});
3442     if ($device->{iothread}) {
3443         my $iothreads = vm_iothreads_list($vmid);
3444         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3445     }
3446 }
3447
3448 sub qemu_objectadd {
3449     my($vmid, $objectid, $qomtype) = @_;
3450
3451     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3452
3453     return 1;
3454 }
3455
3456 sub qemu_objectdel {
3457     my($vmid, $objectid) = @_;
3458
3459     vm_mon_cmd($vmid, "object-del", id => $objectid);
3460
3461     return 1;
3462 }
3463
3464 sub qemu_driveadd {
3465     my ($storecfg, $vmid, $device) = @_;
3466
3467     my $drive = print_drive_full($storecfg, $vmid, $device);
3468     $drive =~ s/\\/\\\\/g;
3469     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3470
3471     # If the command succeeds qemu prints: "OK"
3472     return 1 if $ret =~ m/OK/s;
3473
3474     die "adding drive failed: $ret\n";
3475 }
3476
3477 sub qemu_drivedel {
3478     my($vmid, $deviceid) = @_;
3479
3480     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3481     $ret =~ s/^\s+//;
3482
3483     return 1 if $ret eq "";
3484
3485     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3486     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3487
3488     die "deleting drive $deviceid failed : $ret\n";
3489 }
3490
3491 sub qemu_deviceaddverify {
3492     my ($vmid, $deviceid) = @_;
3493
3494     for (my $i = 0; $i <= 5; $i++) {
3495          my $devices_list = vm_devices_list($vmid);
3496          return 1 if defined($devices_list->{$deviceid});
3497          sleep 1;
3498     }
3499
3500     die "error on hotplug device '$deviceid'\n";
3501 }
3502
3503
3504 sub qemu_devicedelverify {
3505     my ($vmid, $deviceid) = @_;
3506
3507     # need to verify that the device is correctly removed as device_del
3508     # is async and empty return is not reliable
3509
3510     for (my $i = 0; $i <= 5; $i++) {
3511          my $devices_list = vm_devices_list($vmid);
3512          return 1 if !defined($devices_list->{$deviceid});
3513          sleep 1;
3514     }
3515
3516     die "error on hot-unplugging device '$deviceid'\n";
3517 }
3518
3519 sub qemu_findorcreatescsihw {
3520     my ($storecfg, $conf, $vmid, $device) = @_;
3521
3522     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3523
3524     my $scsihwid="$controller_prefix$controller";
3525     my $devices_list = vm_devices_list($vmid);
3526
3527     if(!defined($devices_list->{$scsihwid})) {
3528         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3529     }
3530
3531     return 1;
3532 }
3533
3534 sub qemu_deletescsihw {
3535     my ($conf, $vmid, $opt) = @_;
3536
3537     my $device = parse_drive($opt, $conf->{$opt});
3538
3539     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3540         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3541         return 1;
3542     }
3543
3544     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3545
3546     my $devices_list = vm_devices_list($vmid);
3547     foreach my $opt (keys %{$devices_list}) {
3548         if (PVE::QemuServer::is_valid_drivename($opt)) {
3549             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3550             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3551                 return 1;
3552             }
3553         }
3554     }
3555
3556     my $scsihwid="scsihw$controller";
3557
3558     vm_deviceunplug($vmid, $conf, $scsihwid);
3559
3560     return 1;
3561 }
3562
3563 sub qemu_add_pci_bridge {
3564     my ($storecfg, $conf, $vmid, $device) = @_;
3565
3566     my $bridges = {};
3567
3568     my $bridgeid;
3569
3570     print_pci_addr($device, $bridges);
3571
3572     while (my ($k, $v) = each %$bridges) {
3573         $bridgeid = $k;
3574     }
3575     return 1 if !defined($bridgeid) || $bridgeid < 1;
3576
3577     my $bridge = "pci.$bridgeid";
3578     my $devices_list = vm_devices_list($vmid);
3579
3580     if (!defined($devices_list->{$bridge})) {
3581         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3582     }
3583
3584     return 1;
3585 }
3586
3587 sub qemu_set_link_status {
3588     my ($vmid, $device, $up) = @_;
3589
3590     vm_mon_cmd($vmid, "set_link", name => $device,
3591                up => $up ? JSON::true : JSON::false);
3592 }
3593
3594 sub qemu_netdevadd {
3595     my ($vmid, $conf, $device, $deviceid) = @_;
3596
3597     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
3598     my %options =  split(/[=,]/, $netdev);
3599
3600     vm_mon_cmd($vmid, "netdev_add",  %options);
3601     return 1;
3602 }
3603
3604 sub qemu_netdevdel {
3605     my ($vmid, $deviceid) = @_;
3606
3607     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3608 }
3609
3610 sub qemu_usb_hotplug {
3611     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3612
3613     return if !$device;
3614
3615     # remove the old one first
3616     vm_deviceunplug($vmid, $conf, $deviceid);
3617
3618     # check if xhci controller is necessary and available
3619     if ($device->{usb3}) {
3620
3621         my $devicelist = vm_devices_list($vmid);
3622
3623         if (!$devicelist->{xhci}) {
3624             my $pciaddr = print_pci_addr("xhci");
3625             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
3626         }
3627     }
3628     my $d = parse_usb_device($device->{host});
3629     $d->{usb3} = $device->{usb3};
3630
3631     # add the new one
3632     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d);
3633 }
3634
3635 sub qemu_cpu_hotplug {
3636     my ($vmid, $conf, $vcpus) = @_;
3637
3638     my $sockets = 1;
3639     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3640     $sockets = $conf->{sockets} if  $conf->{sockets};
3641     my $cores = $conf->{cores} || 1;
3642     my $maxcpus = $sockets * $cores;
3643
3644     $vcpus = $maxcpus if !$vcpus;
3645
3646     die "you can't add more vcpus than maxcpus\n"
3647         if $vcpus > $maxcpus;
3648
3649     my $currentvcpus = $conf->{vcpus} || $maxcpus;
3650     die "online cpu unplug is not yet possible\n"
3651         if $vcpus < $currentvcpus;
3652
3653     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3654     die "vcpus in running vm is different than configuration\n"
3655         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
3656
3657     for (my $i = $currentvcpus; $i < $vcpus; $i++) {
3658         vm_mon_cmd($vmid, "cpu-add", id => int($i));
3659     }
3660 }
3661
3662 sub qemu_block_set_io_throttle {
3663     my ($vmid, $deviceid,
3664         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
3665         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max) = @_;
3666
3667     return if !check_running($vmid) ;
3668
3669     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
3670         bps => int($bps),
3671         bps_rd => int($bps_rd),
3672         bps_wr => int($bps_wr),
3673         iops => int($iops),
3674         iops_rd => int($iops_rd),
3675         iops_wr => int($iops_wr),
3676         bps_max => int($bps_max),
3677         bps_rd_max => int($bps_rd_max),
3678         bps_wr_max => int($bps_wr_max),
3679         iops_max => int($iops_max),
3680         iops_rd_max => int($iops_rd_max),
3681         iops_wr_max => int($iops_wr_max)
3682     );
3683
3684 }
3685
3686 # old code, only used to shutdown old VM after update
3687 sub __read_avail {
3688     my ($fh, $timeout) = @_;
3689
3690     my $sel = new IO::Select;
3691     $sel->add($fh);
3692
3693     my $res = '';
3694     my $buf;
3695
3696     my @ready;
3697     while (scalar (@ready = $sel->can_read($timeout))) {
3698         my $count;
3699         if ($count = $fh->sysread($buf, 8192)) {
3700             if ($buf =~ /^(.*)\(qemu\) $/s) {
3701                 $res .= $1;
3702                 last;
3703             } else {
3704                 $res .= $buf;
3705             }
3706         } else {
3707             if (!defined($count)) {
3708                 die "$!\n";
3709             }
3710             last;
3711         }
3712     }
3713
3714     die "monitor read timeout\n" if !scalar(@ready);
3715
3716     return $res;
3717 }
3718
3719 # old code, only used to shutdown old VM after update
3720 sub vm_monitor_command {
3721     my ($vmid, $cmdstr, $nocheck) = @_;
3722
3723     my $res;
3724
3725     eval {
3726         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
3727
3728         my $sname = "${var_run_tmpdir}/$vmid.mon";
3729
3730         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
3731             die "unable to connect to VM $vmid socket - $!\n";
3732
3733         my $timeout = 3;
3734
3735         # hack: migrate sometime blocks the monitor (when migrate_downtime
3736         # is set)
3737         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3738             $timeout = 60*60; # 1 hour
3739         }
3740
3741         # read banner;
3742         my $data = __read_avail($sock, $timeout);
3743
3744         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
3745             die "got unexpected qemu monitor banner\n";
3746         }
3747
3748         my $sel = new IO::Select;
3749         $sel->add($sock);
3750
3751         if (!scalar(my @ready = $sel->can_write($timeout))) {
3752             die "monitor write error - timeout";
3753         }
3754
3755         my $fullcmd = "$cmdstr\r";
3756
3757         # syslog('info', "VM $vmid monitor command: $cmdstr");
3758
3759         my $b;
3760         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
3761             die "monitor write error - $!";
3762         }
3763
3764         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
3765
3766         $timeout = 20;
3767
3768         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3769             $timeout = 60*60; # 1 hour
3770         } elsif ($cmdstr =~ m/^(eject|change)/) {
3771             $timeout = 60; # note: cdrom mount command is slow
3772         }
3773         if ($res = __read_avail($sock, $timeout)) {
3774
3775             my @lines = split("\r?\n", $res);
3776
3777             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
3778
3779             $res = join("\n", @lines);
3780             $res .= "\n";
3781         }
3782     };
3783
3784     my $err = $@;
3785
3786     if ($err) {
3787         syslog("err", "VM $vmid monitor command failed - $err");
3788         die $err;
3789     }
3790
3791     return $res;
3792 }
3793
3794 sub qemu_block_resize {
3795     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
3796
3797     my $running = check_running($vmid);
3798
3799     return if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
3800
3801     return if !$running;
3802
3803     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
3804
3805 }
3806
3807 sub qemu_volume_snapshot {
3808     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3809
3810     my $running = check_running($vmid);
3811
3812     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
3813         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
3814     } else {
3815         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
3816     }
3817 }
3818
3819 sub qemu_volume_snapshot_delete {
3820     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3821
3822     my $running = check_running($vmid);
3823
3824     return if !PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
3825
3826     return if !$running;
3827
3828     vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
3829 }
3830
3831 sub set_migration_caps {
3832     my ($vmid) = @_;
3833
3834     my $cap_ref = [];
3835
3836     my $enabled_cap = {
3837         "auto-converge" => 1,
3838         "xbzrle" => 1,
3839         "x-rdma-pin-all" => 0,
3840         "zero-blocks" => 0,
3841         "compress" => 0
3842     };
3843
3844     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
3845
3846     for my $supported_capability (@$supported_capabilities) {
3847         push @$cap_ref, {
3848             capability => $supported_capability->{capability},
3849             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
3850         };
3851     }
3852
3853     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
3854 }
3855
3856 my $fast_plug_option = {
3857     'lock' => 1,
3858     'name' => 1,
3859     'onboot' => 1,
3860     'shares' => 1,
3861     'startup' => 1,
3862     'description' => 1,
3863 };
3864
3865 # hotplug changes in [PENDING]
3866 # $selection hash can be used to only apply specified options, for
3867 # example: { cores => 1 } (only apply changed 'cores')
3868 # $errors ref is used to return error messages
3869 sub vmconfig_hotplug_pending {
3870     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
3871
3872     my $defaults = load_defaults();
3873
3874     # commit values which do not have any impact on running VM first
3875     # Note: those option cannot raise errors, we we do not care about
3876     # $selection and always apply them.
3877
3878     my $add_error = sub {
3879         my ($opt, $msg) = @_;
3880         $errors->{$opt} = "hotplug problem - $msg";
3881     };
3882
3883     my $changes = 0;
3884     foreach my $opt (keys %{$conf->{pending}}) { # add/change
3885         if ($fast_plug_option->{$opt}) {
3886             $conf->{$opt} = $conf->{pending}->{$opt};
3887             delete $conf->{pending}->{$opt};
3888             $changes = 1;
3889         }
3890     }
3891
3892     if ($changes) {
3893         PVE::QemuConfig->write_config($vmid, $conf);
3894         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
3895     }
3896
3897     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3898
3899     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
3900     while (my ($opt, $force) = each %$pending_delete_hash) {
3901         next if $selection && !$selection->{$opt};
3902         eval {
3903             if ($opt eq 'hotplug') {
3904                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
3905             } elsif ($opt eq 'tablet') {
3906                 die "skip\n" if !$hotplug_features->{usb};
3907                 if ($defaults->{tablet}) {
3908                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
3909                 } else {
3910                     vm_deviceunplug($vmid, $conf, $opt);
3911                 }
3912             } elsif ($opt =~ m/^usb\d+/) {
3913                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
3914                 vm_deviceunplug($vmid, $conf, $opt);
3915             } elsif ($opt eq 'vcpus') {
3916                 die "skip\n" if !$hotplug_features->{cpu};
3917                 qemu_cpu_hotplug($vmid, $conf, undef);
3918             } elsif ($opt eq 'balloon') {
3919                 # enable balloon device is not hotpluggable
3920                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
3921             } elsif ($fast_plug_option->{$opt}) {
3922                 # do nothing
3923             } elsif ($opt =~ m/^net(\d+)$/) {
3924                 die "skip\n" if !$hotplug_features->{network};
3925                 vm_deviceunplug($vmid, $conf, $opt);
3926             } elsif (is_valid_drivename($opt)) {
3927                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
3928                 vm_deviceunplug($vmid, $conf, $opt);
3929                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
3930             } elsif ($opt =~ m/^memory$/) {
3931                 die "skip\n" if !$hotplug_features->{memory};
3932                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
3933             } elsif ($opt eq 'cpuunits') {
3934                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
3935             } elsif ($opt eq 'cpulimit') {
3936                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
3937             } else {
3938                 die "skip\n";
3939             }
3940         };
3941         if (my $err = $@) {
3942             &$add_error($opt, $err) if $err ne "skip\n";
3943         } else {
3944             # save new config if hotplug was successful
3945             delete $conf->{$opt};
3946             vmconfig_undelete_pending_option($conf, $opt);
3947             PVE::QemuConfig->write_config($vmid, $conf);
3948             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
3949         }
3950     }
3951
3952     foreach my $opt (keys %{$conf->{pending}}) {
3953         next if $selection && !$selection->{$opt};
3954         my $value = $conf->{pending}->{$opt};
3955         eval {
3956             if ($opt eq 'hotplug') {
3957                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
3958             } elsif ($opt eq 'tablet') {
3959                 die "skip\n" if !$hotplug_features->{usb};
3960                 if ($value == 1) {
3961                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
3962                 } elsif ($value == 0) {
3963                     vm_deviceunplug($vmid, $conf, $opt);
3964                 }
3965             } elsif ($opt =~ m/^usb\d+$/) {
3966                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
3967                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
3968                 die "skip\n" if !$d;
3969                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d);
3970             } elsif ($opt eq 'vcpus') {
3971                 die "skip\n" if !$hotplug_features->{cpu};
3972                 qemu_cpu_hotplug($vmid, $conf, $value);
3973             } elsif ($opt eq 'balloon') {
3974                 # enable/disable balloning device is not hotpluggable
3975                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
3976                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
3977                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
3978
3979                 # allow manual ballooning if shares is set to zero
3980                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
3981                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
3982                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
3983                 }
3984             } elsif ($opt =~ m/^net(\d+)$/) {
3985                 # some changes can be done without hotplug
3986                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
3987                                     $vmid, $opt, $value);
3988             } elsif (is_valid_drivename($opt)) {
3989                 # some changes can be done without hotplug
3990                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
3991                                      $vmid, $opt, $value, 1);
3992             } elsif ($opt =~ m/^memory$/) { #dimms
3993                 die "skip\n" if !$hotplug_features->{memory};
3994                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
3995             } elsif ($opt eq 'cpuunits') {
3996                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
3997             } elsif ($opt eq 'cpulimit') {
3998                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
3999                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4000             } else {
4001                 die "skip\n";  # skip non-hot-pluggable options
4002             }
4003         };
4004         if (my $err = $@) {
4005             &$add_error($opt, $err) if $err ne "skip\n";
4006         } else {
4007             # save new config if hotplug was successful
4008             $conf->{$opt} = $value;
4009             delete $conf->{pending}->{$opt};
4010             PVE::QemuConfig->write_config($vmid, $conf);
4011             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4012         }
4013     }
4014 }
4015
4016 sub try_deallocate_drive {
4017     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4018
4019     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4020         my $volid = $drive->{file};
4021         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4022             my $sid = PVE::Storage::parse_volume_id($volid);
4023             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4024
4025             # check if the disk is really unused
4026             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4027                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4028             PVE::Storage::vdisk_free($storecfg, $volid);
4029             return 1;
4030         } else {
4031             # If vm is not owner of this disk remove from config
4032             return 1;
4033         }
4034     }
4035
4036     return undef;
4037 }
4038
4039 sub vmconfig_delete_or_detach_drive {
4040     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4041
4042     my $drive = parse_drive($opt, $conf->{$opt});
4043
4044     my $rpcenv = PVE::RPCEnvironment::get();
4045     my $authuser = $rpcenv->get_user();
4046
4047     if ($force) {
4048         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4049         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4050     } else {
4051         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4052     }
4053 }
4054
4055 sub vmconfig_apply_pending {
4056     my ($vmid, $conf, $storecfg) = @_;
4057
4058     # cold plug
4059
4060     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4061     while (my ($opt, $force) = each %$pending_delete_hash) {
4062         die "internal error" if $opt =~ m/^unused/;
4063         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4064         if (!defined($conf->{$opt})) {
4065             vmconfig_undelete_pending_option($conf, $opt);
4066             PVE::QemuConfig->write_config($vmid, $conf);
4067         } elsif (is_valid_drivename($opt)) {
4068             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4069             vmconfig_undelete_pending_option($conf, $opt);
4070             delete $conf->{$opt};
4071             PVE::QemuConfig->write_config($vmid, $conf);
4072         } else {
4073             vmconfig_undelete_pending_option($conf, $opt);
4074             delete $conf->{$opt};
4075             PVE::QemuConfig->write_config($vmid, $conf);
4076         }
4077     }
4078
4079     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4080
4081     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4082         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4083
4084         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4085             # skip if nothing changed
4086         } elsif (is_valid_drivename($opt)) {
4087             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4088                 if defined($conf->{$opt});
4089             $conf->{$opt} = $conf->{pending}->{$opt};
4090         } else {
4091             $conf->{$opt} = $conf->{pending}->{$opt};
4092         }
4093
4094         delete $conf->{pending}->{$opt};
4095         PVE::QemuConfig->write_config($vmid, $conf);
4096     }
4097 }
4098
4099 my $safe_num_ne = sub {
4100     my ($a, $b) = @_;
4101
4102     return 0 if !defined($a) && !defined($b);
4103     return 1 if !defined($a);
4104     return 1 if !defined($b);
4105
4106     return $a != $b;
4107 };
4108
4109 my $safe_string_ne = sub {
4110     my ($a, $b) = @_;
4111
4112     return 0 if !defined($a) && !defined($b);
4113     return 1 if !defined($a);
4114     return 1 if !defined($b);
4115
4116     return $a ne $b;
4117 };
4118
4119 sub vmconfig_update_net {
4120     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4121
4122     my $newnet = parse_net($value);
4123
4124     if ($conf->{$opt}) {
4125         my $oldnet = parse_net($conf->{$opt});
4126
4127         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4128             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4129             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4130             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4131
4132             # for non online change, we try to hot-unplug
4133             die "skip\n" if !$hotplug;
4134             vm_deviceunplug($vmid, $conf, $opt);
4135         } else {
4136
4137             die "internal error" if $opt !~ m/net(\d+)/;
4138             my $iface = "tap${vmid}i$1";
4139
4140             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4141                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4142                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4143                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4144                 PVE::Network::tap_unplug($iface);
4145                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4146             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4147                 # Rate can be applied on its own but any change above needs to
4148                 # include the rate in tap_plug since OVS resets everything.
4149                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4150             }
4151
4152             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4153                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4154             }
4155
4156             return 1;
4157         }
4158     }
4159
4160     if ($hotplug) {
4161         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4162     } else {
4163         die "skip\n";
4164     }
4165 }
4166
4167 sub vmconfig_update_disk {
4168     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4169
4170     # fixme: do we need force?
4171
4172     my $drive = parse_drive($opt, $value);
4173
4174     if ($conf->{$opt}) {
4175
4176         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4177
4178             my $media = $drive->{media} || 'disk';
4179             my $oldmedia = $old_drive->{media} || 'disk';
4180             die "unable to change media type\n" if $media ne $oldmedia;
4181
4182             if (!drive_is_cdrom($old_drive)) {
4183
4184                 if ($drive->{file} ne $old_drive->{file}) {
4185
4186                     die "skip\n" if !$hotplug;
4187
4188                     # unplug and register as unused
4189                     vm_deviceunplug($vmid, $conf, $opt);
4190                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4191
4192                 } else {
4193                     # update existing disk
4194
4195                     # skip non hotpluggable value
4196                     if (&$safe_num_ne($drive->{discard}, $old_drive->{discard}) ||
4197                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4198                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4199                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4200                         die "skip\n";
4201                     }
4202
4203                     # apply throttle
4204                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4205                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4206                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4207                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4208                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4209                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4210                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4211                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4212                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4213                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4214                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4215                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max})) {
4216
4217                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4218                                                    ($drive->{mbps} || 0)*1024*1024,
4219                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4220                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4221                                                    $drive->{iops} || 0,
4222                                                    $drive->{iops_rd} || 0,
4223                                                    $drive->{iops_wr} || 0,
4224                                                    ($drive->{mbps_max} || 0)*1024*1024,
4225                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4226                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4227                                                    $drive->{iops_max} || 0,
4228                                                    $drive->{iops_rd_max} || 0,
4229                                                    $drive->{iops_wr_max} || 0);
4230
4231                     }
4232
4233                     return 1;
4234                 }
4235
4236             } else { # cdrom
4237
4238                 if ($drive->{file} eq 'none') {
4239                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4240                 } else {
4241                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4242                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4243                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4244                 }
4245
4246                 return 1;
4247             }
4248         }
4249     }
4250
4251     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4252     # hotplug new disks
4253     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4254     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4255 }
4256
4257 sub vm_start {
4258     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4259         $forcemachine, $spice_ticket) = @_;
4260
4261     PVE::QemuConfig->lock_config($vmid, sub {
4262         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4263
4264         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
4265
4266         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4267
4268         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4269
4270         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4271             vmconfig_apply_pending($vmid, $conf, $storecfg);
4272             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4273         }
4274
4275         my $defaults = load_defaults();
4276
4277         # set environment variable useful inside network script
4278         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4279
4280         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4281
4282         my $migrate_port = 0;
4283         my $migrate_uri;
4284         if ($statefile) {
4285             if ($statefile eq 'tcp') {
4286                 my $localip = "localhost";
4287                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4288                 my $nodename = PVE::INotify::nodename();
4289                 if ($datacenterconf->{migration_unsecure}) {
4290                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4291                         $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4292                 }
4293                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4294                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4295                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4296                 push @$cmd, '-incoming', $migrate_uri;
4297                 push @$cmd, '-S';
4298
4299             } elsif ($statefile eq 'unix') {
4300                 # should be default for secure migrations as a ssh TCP forward
4301                 # tunnel is not deterministic reliable ready and fails regurarly
4302                 # to set up in time, so use UNIX socket forwards
4303                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
4304                 unlink $socket_addr;
4305
4306                 $migrate_uri = "unix:$socket_addr";
4307
4308                 push @$cmd, '-incoming', $migrate_uri;
4309                 push @$cmd, '-S';
4310
4311             } else {
4312                 push @$cmd, '-loadstate', $statefile;
4313             }
4314         } elsif ($paused) {
4315             push @$cmd, '-S';
4316         }
4317
4318         # host pci devices
4319         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4320           my $d = parse_hostpci($conf->{"hostpci$i"});
4321           next if !$d;
4322           my $pcidevices = $d->{pciid};
4323           foreach my $pcidevice (@$pcidevices) {
4324                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4325
4326                 my $info = pci_device_info("0000:$pciid");
4327                 die "IOMMU not present\n" if !check_iommu_support();
4328                 die "no pci device info for device '$pciid'\n" if !$info;
4329                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4330                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4331           }
4332         }
4333
4334         PVE::Storage::activate_volumes($storecfg, $vollist);
4335
4336         if (!check_running($vmid, 1) && -d "/sys/fs/cgroup/systemd/qemu.slice/$vmid.scope") {
4337             my $cmd = [];
4338             push @$cmd, '/bin/systemctl', 'stop', "$vmid.scope";
4339             eval  { run_command($cmd); };
4340         }
4341
4342         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
4343                                                   : $defaults->{cpuunits};
4344
4345         my %run_params = (timeout => $statefile ? undef : 30, umask => 0077);
4346
4347         my %properties = (
4348             Slice => 'qemu.slice',
4349             KillMode => 'none',
4350             CPUShares => $cpuunits
4351         );
4352
4353         if (my $cpulimit = $conf->{cpulimit}) {
4354             $properties{CPUQuota} = int($cpulimit * 100);
4355         }
4356         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
4357
4358         if ($conf->{hugepages}) {
4359
4360             my $code = sub {
4361                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
4362                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
4363
4364                 PVE::QemuServer::Memory::hugepages_mount();
4365                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
4366
4367                 eval  {
4368                     PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4369                     run_command($cmd, %run_params);
4370                 };
4371
4372                 if (my $err = $@) {
4373                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
4374                     die $err;
4375                 }
4376
4377                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
4378             };
4379             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
4380
4381         } else {
4382             eval  {
4383                 PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4384                 run_command($cmd, %run_params);
4385             };
4386         }
4387
4388         if (my $err = $@) {
4389             # deactivate volumes if start fails
4390             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
4391             die "start failed: $err";
4392         }
4393
4394         print "migration listens on $migrate_uri\n" if $migrate_uri;
4395
4396         if ($statefile && $statefile ne 'tcp')  {
4397             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4398             warn $@ if $@;
4399         }
4400
4401         if ($migratedfrom) {
4402
4403             eval {
4404                 set_migration_caps($vmid);
4405             };
4406             warn $@ if $@;
4407
4408             if ($spice_port) {
4409                 print "spice listens on port $spice_port\n";
4410                 if ($spice_ticket) {
4411                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4412                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4413                 }
4414             }
4415
4416         } else {
4417
4418             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4419                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4420                     if $conf->{balloon};
4421             }
4422
4423             foreach my $opt (keys %$conf) {
4424                 next if $opt !~  m/^net\d+$/;
4425                 my $nicconf = parse_net($conf->{$opt});
4426                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4427             }
4428         }
4429
4430         vm_mon_cmd_nocheck($vmid, 'qom-set',
4431                     path => "machine/peripheral/balloon0",
4432                     property => "guest-stats-polling-interval",
4433                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4434
4435     });
4436 }
4437
4438 sub vm_mon_cmd {
4439     my ($vmid, $execute, %params) = @_;
4440
4441     my $cmd = { execute => $execute, arguments => \%params };
4442     vm_qmp_command($vmid, $cmd);
4443 }
4444
4445 sub vm_mon_cmd_nocheck {
4446     my ($vmid, $execute, %params) = @_;
4447
4448     my $cmd = { execute => $execute, arguments => \%params };
4449     vm_qmp_command($vmid, $cmd, 1);
4450 }
4451
4452 sub vm_qmp_command {
4453     my ($vmid, $cmd, $nocheck) = @_;
4454
4455     my $res;
4456
4457     my $timeout;
4458     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4459         $timeout = $cmd->{arguments}->{timeout};
4460         delete $cmd->{arguments}->{timeout};
4461     }
4462
4463     eval {
4464         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4465         my $sname = qmp_socket($vmid);
4466         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4467             my $qmpclient = PVE::QMPClient->new();
4468
4469             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4470         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
4471             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
4472                 if scalar(%{$cmd->{arguments}});
4473             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
4474         } else {
4475             die "unable to open monitor socket\n";
4476         }
4477     };
4478     if (my $err = $@) {
4479         syslog("err", "VM $vmid qmp command failed - $err");
4480         die $err;
4481     }
4482
4483     return $res;
4484 }
4485
4486 sub vm_human_monitor_command {
4487     my ($vmid, $cmdline) = @_;
4488
4489     my $res;
4490
4491     my $cmd = {
4492         execute => 'human-monitor-command',
4493         arguments => { 'command-line' => $cmdline},
4494     };
4495
4496     return vm_qmp_command($vmid, $cmd);
4497 }
4498
4499 sub vm_commandline {
4500     my ($storecfg, $vmid) = @_;
4501
4502     my $conf = PVE::QemuConfig->load_config($vmid);
4503
4504     my $defaults = load_defaults();
4505
4506     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
4507
4508     return join(' ', @$cmd);
4509 }
4510
4511 sub vm_reset {
4512     my ($vmid, $skiplock) = @_;
4513
4514     PVE::QemuConfig->lock_config($vmid, sub {
4515
4516         my $conf = PVE::QemuConfig->load_config($vmid);
4517
4518         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4519
4520         vm_mon_cmd($vmid, "system_reset");
4521     });
4522 }
4523
4524 sub get_vm_volumes {
4525     my ($conf) = @_;
4526
4527     my $vollist = [];
4528     foreach_volid($conf, sub {
4529         my ($volid, $is_cdrom) = @_;
4530
4531         return if $volid =~ m|^/|;
4532
4533         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4534         return if !$sid;
4535
4536         push @$vollist, $volid;
4537     });
4538
4539     return $vollist;
4540 }
4541
4542 sub vm_stop_cleanup {
4543     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
4544
4545     eval {
4546
4547         if (!$keepActive) {
4548             my $vollist = get_vm_volumes($conf);
4549             PVE::Storage::deactivate_volumes($storecfg, $vollist);
4550         }
4551
4552         foreach my $ext (qw(mon qmp pid vnc qga)) {
4553             unlink "/var/run/qemu-server/${vmid}.$ext";
4554         }
4555
4556         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
4557     };
4558     warn $@ if $@; # avoid errors - just warn
4559 }
4560
4561 # Note: use $nockeck to skip tests if VM configuration file exists.
4562 # We need that when migration VMs to other nodes (files already moved)
4563 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
4564 sub vm_stop {
4565     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
4566
4567     $force = 1 if !defined($force) && !$shutdown;
4568
4569     if ($migratedfrom){
4570         my $pid = check_running($vmid, $nocheck, $migratedfrom);
4571         kill 15, $pid if $pid;
4572         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4573         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
4574         return;
4575     }
4576
4577     PVE::QemuConfig->lock_config($vmid, sub {
4578
4579         my $pid = check_running($vmid, $nocheck);
4580         return if !$pid;
4581
4582         my $conf;
4583         if (!$nocheck) {
4584             $conf = PVE::QemuConfig->load_config($vmid);
4585             PVE::QemuConfig->check_lock($conf) if !$skiplock;
4586             if (!defined($timeout) && $shutdown && $conf->{startup}) {
4587                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
4588                 $timeout = $opts->{down} if $opts->{down};
4589             }
4590         }
4591
4592         $timeout = 60 if !defined($timeout);
4593
4594         eval {
4595             if ($shutdown) {
4596                 if (defined($conf) && $conf->{agent}) {
4597                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
4598                 } else {
4599                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
4600                 }
4601             } else {
4602                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
4603             }
4604         };
4605         my $err = $@;
4606
4607         if (!$err) {
4608             my $count = 0;
4609             while (($count < $timeout) && check_running($vmid, $nocheck)) {
4610                 $count++;
4611                 sleep 1;
4612             }
4613
4614             if ($count >= $timeout) {
4615                 if ($force) {
4616                     warn "VM still running - terminating now with SIGTERM\n";
4617                     kill 15, $pid;
4618                 } else {
4619                     die "VM quit/powerdown failed - got timeout\n";
4620                 }
4621             } else {
4622                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4623                 return;
4624             }
4625         } else {
4626             if ($force) {
4627                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
4628                 kill 15, $pid;
4629             } else {
4630                 die "VM quit/powerdown failed\n";
4631             }
4632         }
4633
4634         # wait again
4635         $timeout = 10;
4636
4637         my $count = 0;
4638         while (($count < $timeout) && check_running($vmid, $nocheck)) {
4639             $count++;
4640             sleep 1;
4641         }
4642
4643         if ($count >= $timeout) {
4644             warn "VM still running - terminating now with SIGKILL\n";
4645             kill 9, $pid;
4646             sleep 1;
4647         }
4648
4649         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4650    });
4651 }
4652
4653 sub vm_suspend {
4654     my ($vmid, $skiplock) = @_;
4655
4656     PVE::QemuConfig->lock_config($vmid, sub {
4657
4658         my $conf = PVE::QemuConfig->load_config($vmid);
4659
4660         PVE::QemuConfig->check_lock($conf)
4661             if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
4662
4663         vm_mon_cmd($vmid, "stop");
4664     });
4665 }
4666
4667 sub vm_resume {
4668     my ($vmid, $skiplock, $nocheck) = @_;
4669
4670     PVE::QemuConfig->lock_config($vmid, sub {
4671
4672         if (!$nocheck) {
4673
4674             my $conf = PVE::QemuConfig->load_config($vmid);
4675
4676             PVE::QemuConfig->check_lock($conf)
4677                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
4678
4679             vm_mon_cmd($vmid, "cont");
4680
4681         } else {
4682             vm_mon_cmd_nocheck($vmid, "cont");
4683         }
4684     });
4685 }
4686
4687 sub vm_sendkey {
4688     my ($vmid, $skiplock, $key) = @_;
4689
4690     PVE::QemuConfig->lock_config($vmid, sub {
4691
4692         my $conf = PVE::QemuConfig->load_config($vmid);
4693
4694         # there is no qmp command, so we use the human monitor command
4695         vm_human_monitor_command($vmid, "sendkey $key");
4696     });
4697 }
4698
4699 sub vm_destroy {
4700     my ($storecfg, $vmid, $skiplock) = @_;
4701
4702     PVE::QemuConfig->lock_config($vmid, sub {
4703
4704         my $conf = PVE::QemuConfig->load_config($vmid);
4705
4706         if (!check_running($vmid)) {
4707             destroy_vm($storecfg, $vmid, undef, $skiplock);
4708         } else {
4709             die "VM $vmid is running - destroy failed\n";
4710         }
4711     });
4712 }
4713
4714 # pci helpers
4715
4716 sub file_write {
4717     my ($filename, $buf) = @_;
4718
4719     my $fh = IO::File->new($filename, "w");
4720     return undef if !$fh;
4721
4722     my $res = print $fh $buf;
4723
4724     $fh->close();
4725
4726     return $res;
4727 }
4728
4729 sub pci_device_info {
4730     my ($name) = @_;
4731
4732     my $res;
4733
4734     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
4735     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
4736
4737     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
4738     return undef if !defined($irq) || $irq !~ m/^\d+$/;
4739
4740     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
4741     return undef if !defined($vendor) || $vendor !~ s/^0x//;
4742
4743     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
4744     return undef if !defined($product) || $product !~ s/^0x//;
4745
4746     $res = {
4747         name => $name,
4748         vendor => $vendor,
4749         product => $product,
4750         domain => $domain,
4751         bus => $bus,
4752         slot => $slot,
4753         func => $func,
4754         irq => $irq,
4755         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
4756     };
4757
4758     return $res;
4759 }
4760
4761 sub pci_dev_reset {
4762     my ($dev) = @_;
4763
4764     my $name = $dev->{name};
4765
4766     my $fn = "$pcisysfs/devices/$name/reset";
4767
4768     return file_write($fn, "1");
4769 }
4770
4771 sub pci_dev_bind_to_vfio {
4772     my ($dev) = @_;
4773
4774     my $name = $dev->{name};
4775
4776     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4777
4778     if (!-d $vfio_basedir) {
4779         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4780     }
4781     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4782
4783     my $testdir = "$vfio_basedir/$name";
4784     return 1 if -d $testdir;
4785
4786     my $data = "$dev->{vendor} $dev->{product}";
4787     return undef if !file_write("$vfio_basedir/new_id", $data);
4788
4789     my $fn = "$pcisysfs/devices/$name/driver/unbind";
4790     if (!file_write($fn, $name)) {
4791         return undef if -f $fn;
4792     }
4793
4794     $fn = "$vfio_basedir/bind";
4795     if (! -d $testdir) {
4796         return undef if !file_write($fn, $name);
4797     }
4798
4799     return -d $testdir;
4800 }
4801
4802 sub pci_dev_group_bind_to_vfio {
4803     my ($pciid) = @_;
4804
4805     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4806
4807     if (!-d $vfio_basedir) {
4808         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4809     }
4810     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4811
4812     # get IOMMU group devices
4813     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
4814       my @devs = grep /^0000:/, readdir($D);
4815     closedir($D);
4816
4817     foreach my $pciid (@devs) {
4818         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
4819
4820         # pci bridges, switches or root ports are not supported
4821         # they have a pci_bus subdirectory so skip them
4822         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
4823
4824         my $info = pci_device_info($1);
4825         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
4826     }
4827
4828     return 1;
4829 }
4830
4831 # vzdump restore implementaion
4832
4833 sub tar_archive_read_firstfile {
4834     my $archive = shift;
4835
4836     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
4837
4838     # try to detect archive type first
4839     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
4840         die "unable to open file '$archive'\n";
4841     my $firstfile = <$fh>;
4842     kill 15, $pid;
4843     close $fh;
4844
4845     die "ERROR: archive contaions no data\n" if !$firstfile;
4846     chomp $firstfile;
4847
4848     return $firstfile;
4849 }
4850
4851 sub tar_restore_cleanup {
4852     my ($storecfg, $statfile) = @_;
4853
4854     print STDERR "starting cleanup\n";
4855
4856     if (my $fd = IO::File->new($statfile, "r")) {
4857         while (defined(my $line = <$fd>)) {
4858             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
4859                 my $volid = $2;
4860                 eval {
4861                     if ($volid =~ m|^/|) {
4862                         unlink $volid || die 'unlink failed\n';
4863                     } else {
4864                         PVE::Storage::vdisk_free($storecfg, $volid);
4865                     }
4866                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
4867                 };
4868                 print STDERR "unable to cleanup '$volid' - $@" if $@;
4869             } else {
4870                 print STDERR "unable to parse line in statfile - $line";
4871             }
4872         }
4873         $fd->close();
4874     }
4875 }
4876
4877 sub restore_archive {
4878     my ($archive, $vmid, $user, $opts) = @_;
4879
4880     my $format = $opts->{format};
4881     my $comp;
4882
4883     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
4884         $format = 'tar' if !$format;
4885         $comp = 'gzip';
4886     } elsif ($archive =~ m/\.tar$/) {
4887         $format = 'tar' if !$format;
4888     } elsif ($archive =~ m/.tar.lzo$/) {
4889         $format = 'tar' if !$format;
4890         $comp = 'lzop';
4891     } elsif ($archive =~ m/\.vma$/) {
4892         $format = 'vma' if !$format;
4893     } elsif ($archive =~ m/\.vma\.gz$/) {
4894         $format = 'vma' if !$format;
4895         $comp = 'gzip';
4896     } elsif ($archive =~ m/\.vma\.lzo$/) {
4897         $format = 'vma' if !$format;
4898         $comp = 'lzop';
4899     } else {
4900         $format = 'vma' if !$format; # default
4901     }
4902
4903     # try to detect archive format
4904     if ($format eq 'tar') {
4905         return restore_tar_archive($archive, $vmid, $user, $opts);
4906     } else {
4907         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
4908     }
4909 }
4910
4911 sub restore_update_config_line {
4912     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
4913
4914     return if $line =~ m/^\#qmdump\#/;
4915     return if $line =~ m/^\#vzdump\#/;
4916     return if $line =~ m/^lock:/;
4917     return if $line =~ m/^unused\d+:/;
4918     return if $line =~ m/^parent:/;
4919     return if $line =~ m/^template:/; # restored VM is never a template
4920
4921     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
4922         # try to convert old 1.X settings
4923         my ($id, $ind, $ethcfg) = ($1, $2, $3);
4924         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
4925             my ($model, $macaddr) = split(/\=/, $devconfig);
4926             $macaddr = PVE::Tools::random_ether_addr() if !$macaddr || $unique;
4927             my $net = {
4928                 model => $model,
4929                 bridge => "vmbr$ind",
4930                 macaddr => $macaddr,
4931             };
4932             my $netstr = print_net($net);
4933
4934             print $outfd "net$cookie->{netcount}: $netstr\n";
4935             $cookie->{netcount}++;
4936         }
4937     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
4938         my ($id, $netstr) = ($1, $2);
4939         my $net = parse_net($netstr);
4940         $net->{macaddr} = PVE::Tools::random_ether_addr() if $net->{macaddr};
4941         $netstr = print_net($net);
4942         print $outfd "$id: $netstr\n";
4943     } elsif ($line =~ m/^((ide|scsi|virtio|sata)\d+):\s*(\S+)\s*$/) {
4944         my $virtdev = $1;
4945         my $value = $3;
4946         my $di = parse_drive($virtdev, $value);
4947         if (defined($di->{backup}) && !$di->{backup}) {
4948             print $outfd "#$line";
4949         } elsif ($map->{$virtdev}) {
4950             delete $di->{format}; # format can change on restore
4951             $di->{file} = $map->{$virtdev};
4952             $value = print_drive($vmid, $di);
4953             print $outfd "$virtdev: $value\n";
4954         } else {
4955             print $outfd $line;
4956         }
4957     } else {
4958         print $outfd $line;
4959     }
4960 }
4961
4962 sub scan_volids {
4963     my ($cfg, $vmid) = @_;
4964
4965     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
4966
4967     my $volid_hash = {};
4968     foreach my $storeid (keys %$info) {
4969         foreach my $item (@{$info->{$storeid}}) {
4970             next if !($item->{volid} && $item->{size});
4971             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
4972             $volid_hash->{$item->{volid}} = $item;
4973         }
4974     }
4975
4976     return $volid_hash;
4977 }
4978
4979 sub is_volume_in_use {
4980     my ($storecfg, $conf, $skip_drive, $volid) = @_;
4981
4982     my $path = PVE::Storage::path($storecfg, $volid);
4983
4984     my $scan_config = sub {
4985         my ($cref, $snapname) = @_;
4986
4987         foreach my $key (keys %$cref) {
4988             my $value = $cref->{$key};
4989             if (is_valid_drivename($key)) {
4990                 next if $skip_drive && $key eq $skip_drive;
4991                 my $drive = parse_drive($key, $value);
4992                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
4993                 return 1 if $volid eq $drive->{file};
4994                 if ($drive->{file} =~ m!^/!) {
4995                     return 1 if $drive->{file} eq $path;
4996                 } else {
4997                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
4998                     next if !$storeid;
4999                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5000                     next if !$scfg;
5001                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5002                 }
5003             }
5004         }
5005
5006         return 0;
5007     };
5008
5009     return 1 if &$scan_config($conf);
5010
5011     undef $skip_drive;
5012
5013     foreach my $snapname (keys %{$conf->{snapshots}}) {
5014         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5015     }
5016
5017     return 0;
5018 }
5019
5020 sub update_disksize {
5021     my ($vmid, $conf, $volid_hash) = @_;
5022
5023     my $changes;
5024
5025     my $used = {};
5026
5027     # Note: it is allowed to define multiple storages with same path (alias), so
5028     # we need to check both 'volid' and real 'path' (two different volid can point
5029     # to the same path).
5030
5031     my $usedpath = {};
5032
5033     # update size info
5034     foreach my $opt (keys %$conf) {
5035         if (is_valid_drivename($opt)) {
5036             my $drive = parse_drive($opt, $conf->{$opt});
5037             my $volid = $drive->{file};
5038             next if !$volid;
5039
5040             $used->{$volid} = 1;
5041             if ($volid_hash->{$volid} &&
5042                 (my $path = $volid_hash->{$volid}->{path})) {
5043                 $usedpath->{$path} = 1;
5044             }
5045
5046             next if drive_is_cdrom($drive);
5047             next if !$volid_hash->{$volid};
5048
5049             $drive->{size} = $volid_hash->{$volid}->{size};
5050             my $new = print_drive($vmid, $drive);
5051             if ($new ne $conf->{$opt}) {
5052                 $changes = 1;
5053                 $conf->{$opt} = $new;
5054             }
5055         }
5056     }
5057
5058     # remove 'unusedX' entry if volume is used
5059     foreach my $opt (keys %$conf) {
5060         next if $opt !~ m/^unused\d+$/;
5061         my $volid = $conf->{$opt};
5062         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5063         if ($used->{$volid} || ($path && $usedpath->{$path})) {
5064             $changes = 1;
5065             delete $conf->{$opt};
5066         }
5067     }
5068
5069     foreach my $volid (sort keys %$volid_hash) {
5070         next if $volid =~ m/vm-$vmid-state-/;
5071         next if $used->{$volid};
5072         my $path = $volid_hash->{$volid}->{path};
5073         next if !$path; # just to be sure
5074         next if $usedpath->{$path};
5075         $changes = 1;
5076         PVE::QemuConfig->add_unused_volume($conf, $volid);
5077         $usedpath->{$path} = 1; # avoid to add more than once (aliases)
5078     }
5079
5080     return $changes;
5081 }
5082
5083 sub rescan {
5084     my ($vmid, $nolock) = @_;
5085
5086     my $cfg = PVE::Storage::config();
5087
5088     my $volid_hash = scan_volids($cfg, $vmid);
5089
5090     my $updatefn =  sub {
5091         my ($vmid) = @_;
5092
5093         my $conf = PVE::QemuConfig->load_config($vmid);
5094
5095         PVE::QemuConfig->check_lock($conf);
5096
5097         my $vm_volids = {};
5098         foreach my $volid (keys %$volid_hash) {
5099             my $info = $volid_hash->{$volid};
5100             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5101         }
5102
5103         my $changes = update_disksize($vmid, $conf, $vm_volids);
5104
5105         PVE::QemuConfig->write_config($vmid, $conf) if $changes;
5106     };
5107
5108     if (defined($vmid)) {
5109         if ($nolock) {
5110             &$updatefn($vmid);
5111         } else {
5112             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5113         }
5114     } else {
5115         my $vmlist = config_list();
5116         foreach my $vmid (keys %$vmlist) {
5117             if ($nolock) {
5118                 &$updatefn($vmid);
5119             } else {
5120                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5121             }
5122         }
5123     }
5124 }
5125
5126 sub restore_vma_archive {
5127     my ($archive, $vmid, $user, $opts, $comp) = @_;
5128
5129     my $input = $archive eq '-' ? "<&STDIN" : undef;
5130     my $readfrom = $archive;
5131
5132     my $uncomp = '';
5133     if ($comp) {
5134         $readfrom = '-';
5135         my $qarchive = PVE::Tools::shellquote($archive);
5136         if ($comp eq 'gzip') {
5137             $uncomp = "zcat $qarchive|";
5138         } elsif ($comp eq 'lzop') {
5139             $uncomp = "lzop -d -c $qarchive|";
5140         } else {
5141             die "unknown compression method '$comp'\n";
5142         }
5143
5144     }
5145
5146     my $tmpdir = "/var/tmp/vzdumptmp$$";
5147     rmtree $tmpdir;
5148
5149     # disable interrupts (always do cleanups)
5150     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5151         warn "got interrupt - ignored\n";
5152     };
5153
5154     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5155     POSIX::mkfifo($mapfifo, 0600);
5156     my $fifofh;
5157
5158     my $openfifo = sub {
5159         open($fifofh, '>', $mapfifo) || die $!;
5160     };
5161
5162     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5163
5164     my $oldtimeout;
5165     my $timeout = 5;
5166
5167     my $devinfo = {};
5168
5169     my $rpcenv = PVE::RPCEnvironment::get();
5170
5171     my $conffile = PVE::QemuConfig->config_file($vmid);
5172     my $tmpfn = "$conffile.$$.tmp";
5173
5174     # Note: $oldconf is undef if VM does not exists
5175     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
5176     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
5177
5178     my $print_devmap = sub {
5179         my $virtdev_hash = {};
5180
5181         my $cfgfn = "$tmpdir/qemu-server.conf";
5182
5183         # we can read the config - that is already extracted
5184         my $fh = IO::File->new($cfgfn, "r") ||
5185             "unable to read qemu-server.conf - $!\n";
5186
5187         my $fwcfgfn = "$tmpdir/qemu-server.fw";
5188         if (-f $fwcfgfn) {
5189             my $pve_firewall_dir = '/etc/pve/firewall';
5190             mkdir $pve_firewall_dir; # make sure the dir exists
5191             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
5192         }
5193
5194         while (defined(my $line = <$fh>)) {
5195             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5196                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5197                 die "archive does not contain data for drive '$virtdev'\n"
5198                     if !$devinfo->{$devname};
5199                 if (defined($opts->{storage})) {
5200                     $storeid = $opts->{storage} || 'local';
5201                 } elsif (!$storeid) {
5202                     $storeid = 'local';
5203                 }
5204                 $format = 'raw' if !$format;
5205                 $devinfo->{$devname}->{devname} = $devname;
5206                 $devinfo->{$devname}->{virtdev} = $virtdev;
5207                 $devinfo->{$devname}->{format} = $format;
5208                 $devinfo->{$devname}->{storeid} = $storeid;
5209
5210                 # check permission on storage
5211                 my $pool = $opts->{pool}; # todo: do we need that?
5212                 if ($user ne 'root@pam') {
5213                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5214                 }
5215
5216                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5217             }
5218         }
5219
5220         foreach my $devname (keys %$devinfo) {
5221             die "found no device mapping information for device '$devname'\n"
5222                 if !$devinfo->{$devname}->{virtdev};
5223         }
5224
5225         my $cfg = PVE::Storage::config();
5226
5227         # create empty/temp config
5228         if ($oldconf) {
5229             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5230             foreach_drive($oldconf, sub {
5231                 my ($ds, $drive) = @_;
5232
5233                 return if drive_is_cdrom($drive);
5234
5235                 my $volid = $drive->{file};
5236
5237                 return if !$volid || $volid =~ m|^/|;
5238
5239                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5240                 return if !$path || !$owner || ($owner != $vmid);
5241
5242                 # Note: only delete disk we want to restore
5243                 # other volumes will become unused
5244                 if ($virtdev_hash->{$ds}) {
5245                     PVE::Storage::vdisk_free($cfg, $volid);
5246                 }
5247             });
5248
5249             # delete vmstate files
5250             # since after the restore we have no snapshots anymore
5251             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5252                 my $snap = $oldconf->{snapshots}->{$snapname};
5253                 if ($snap->{vmstate}) {
5254                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
5255                     if (my $err = $@) {
5256                         warn $err;
5257                     }
5258                 }
5259             }
5260         }
5261
5262         my $map = {};
5263         foreach my $virtdev (sort keys %$virtdev_hash) {
5264             my $d = $virtdev_hash->{$virtdev};
5265             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5266             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5267
5268             # test if requested format is supported
5269             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5270             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5271             $d->{format} = $defFormat if !$supported;
5272
5273             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5274                                                   $d->{format}, undef, $alloc_size);
5275             print STDERR "new volume ID is '$volid'\n";
5276             $d->{volid} = $volid;
5277             my $path = PVE::Storage::path($cfg, $volid);
5278
5279             PVE::Storage::activate_volumes($cfg,[$volid]);
5280
5281             my $write_zeros = 1;
5282             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
5283                 $write_zeros = 0;
5284             }
5285
5286             print $fifofh "format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
5287
5288             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5289             $map->{$virtdev} = $volid;
5290         }
5291
5292         $fh->seek(0, 0) || die "seek failed - $!\n";
5293
5294         my $outfd = new IO::File ($tmpfn, "w") ||
5295             die "unable to write config for VM $vmid\n";
5296
5297         my $cookie = { netcount => 0 };
5298         while (defined(my $line = <$fh>)) {
5299             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5300         }
5301
5302         $fh->close();
5303         $outfd->close();
5304     };
5305
5306     eval {
5307         # enable interrupts
5308         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5309             die "interrupted by signal\n";
5310         };
5311         local $SIG{ALRM} = sub { die "got timeout\n"; };
5312
5313         $oldtimeout = alarm($timeout);
5314
5315         my $parser = sub {
5316             my $line = shift;
5317
5318             print "$line\n";
5319
5320             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5321                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5322                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5323             } elsif ($line =~ m/^CTIME: /) {
5324                 # we correctly received the vma config, so we can disable
5325                 # the timeout now for disk allocation (set to 10 minutes, so
5326                 # that we always timeout if something goes wrong)
5327                 alarm(600);
5328                 &$print_devmap();
5329                 print $fifofh "done\n";
5330                 my $tmp = $oldtimeout || 0;
5331                 $oldtimeout = undef;
5332                 alarm($tmp);
5333                 close($fifofh);
5334             }
5335         };
5336
5337         print "restore vma archive: $cmd\n";
5338         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5339     };
5340     my $err = $@;
5341
5342     alarm($oldtimeout) if $oldtimeout;
5343
5344     my $vollist = [];
5345     foreach my $devname (keys %$devinfo) {
5346         my $volid = $devinfo->{$devname}->{volid};
5347         push @$vollist, $volid if $volid;
5348     }
5349
5350     my $cfg = PVE::Storage::config();
5351     PVE::Storage::deactivate_volumes($cfg, $vollist);
5352
5353     unlink $mapfifo;
5354
5355     if ($err) {
5356         rmtree $tmpdir;
5357         unlink $tmpfn;
5358
5359         foreach my $devname (keys %$devinfo) {
5360             my $volid = $devinfo->{$devname}->{volid};
5361             next if !$volid;
5362             eval {
5363                 if ($volid =~ m|^/|) {
5364                     unlink $volid || die 'unlink failed\n';
5365                 } else {
5366                     PVE::Storage::vdisk_free($cfg, $volid);
5367                 }
5368                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5369             };
5370             print STDERR "unable to cleanup '$volid' - $@" if $@;
5371         }
5372         die $err;
5373     }
5374
5375     rmtree $tmpdir;
5376
5377     rename($tmpfn, $conffile) ||
5378         die "unable to commit configuration file '$conffile'\n";
5379
5380     PVE::Cluster::cfs_update(); # make sure we read new file
5381
5382     eval { rescan($vmid, 1); };
5383     warn $@ if $@;
5384 }
5385
5386 sub restore_tar_archive {
5387     my ($archive, $vmid, $user, $opts) = @_;
5388
5389     if ($archive ne '-') {
5390         my $firstfile = tar_archive_read_firstfile($archive);
5391         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5392             if $firstfile ne 'qemu-server.conf';
5393     }
5394
5395     my $storecfg = PVE::Storage::config();
5396
5397     # destroy existing data - keep empty config
5398     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
5399     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5400
5401     my $tocmd = "/usr/lib/qemu-server/qmextract";
5402
5403     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5404     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5405     $tocmd .= ' --prealloc' if $opts->{prealloc};
5406     $tocmd .= ' --info' if $opts->{info};
5407
5408     # tar option "xf" does not autodetect compression when read from STDIN,
5409     # so we pipe to zcat
5410     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5411         PVE::Tools::shellquote("--to-command=$tocmd");
5412
5413     my $tmpdir = "/var/tmp/vzdumptmp$$";
5414     mkpath $tmpdir;
5415
5416     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5417     local $ENV{VZDUMP_VMID} = $vmid;
5418     local $ENV{VZDUMP_USER} = $user;
5419
5420     my $conffile = PVE::QemuConfig->config_file($vmid);
5421     my $tmpfn = "$conffile.$$.tmp";
5422
5423     # disable interrupts (always do cleanups)
5424     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5425         print STDERR "got interrupt - ignored\n";
5426     };
5427
5428     eval {
5429         # enable interrupts
5430         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5431             die "interrupted by signal\n";
5432         };
5433
5434         if ($archive eq '-') {
5435             print "extracting archive from STDIN\n";
5436             run_command($cmd, input => "<&STDIN");
5437         } else {
5438             print "extracting archive '$archive'\n";
5439             run_command($cmd);
5440         }
5441
5442         return if $opts->{info};
5443
5444         # read new mapping
5445         my $map = {};
5446         my $statfile = "$tmpdir/qmrestore.stat";
5447         if (my $fd = IO::File->new($statfile, "r")) {
5448             while (defined (my $line = <$fd>)) {
5449                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5450                     $map->{$1} = $2 if $1;
5451                 } else {
5452                     print STDERR "unable to parse line in statfile - $line\n";
5453                 }
5454             }
5455             $fd->close();
5456         }
5457
5458         my $confsrc = "$tmpdir/qemu-server.conf";
5459
5460         my $srcfd = new IO::File($confsrc, "r") ||
5461             die "unable to open file '$confsrc'\n";
5462
5463         my $outfd = new IO::File ($tmpfn, "w") ||
5464             die "unable to write config for VM $vmid\n";
5465
5466         my $cookie = { netcount => 0 };
5467         while (defined (my $line = <$srcfd>)) {
5468             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5469         }
5470
5471         $srcfd->close();
5472         $outfd->close();
5473     };
5474     my $err = $@;
5475
5476     if ($err) {
5477
5478         unlink $tmpfn;
5479
5480         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
5481
5482         die $err;
5483     }
5484
5485     rmtree $tmpdir;
5486
5487     rename $tmpfn, $conffile ||
5488         die "unable to commit configuration file '$conffile'\n";
5489
5490     PVE::Cluster::cfs_update(); # make sure we read new file
5491
5492     eval { rescan($vmid, 1); };
5493     warn $@ if $@;
5494 };
5495
5496 sub foreach_writable_storage {
5497     my ($conf, $func) = @_;
5498
5499     my $sidhash = {};
5500
5501     foreach my $ds (keys %$conf) {
5502         next if !is_valid_drivename($ds);
5503
5504         my $drive = parse_drive($ds, $conf->{$ds});
5505         next if !$drive;
5506         next if drive_is_cdrom($drive);
5507
5508         my $volid = $drive->{file};
5509
5510         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5511         $sidhash->{$sid} = $sid if $sid;
5512     }
5513
5514     foreach my $sid (sort keys %$sidhash) {
5515         &$func($sid);
5516     }
5517 }
5518
5519 sub do_snapshots_with_qemu {
5520     my ($storecfg, $volid) = @_;
5521
5522     my $storage_name = PVE::Storage::parse_volume_id($volid);
5523
5524     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
5525         && !$storecfg->{ids}->{$storage_name}->{krbd}){
5526         return 1;
5527     }
5528
5529     if ($volid =~ m/\.(qcow2|qed)$/){
5530         return 1;
5531     }
5532
5533     return undef;
5534 }
5535
5536 sub qga_check_running {
5537     my ($vmid) = @_;
5538
5539     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
5540     if ($@) {
5541         warn "Qemu Guest Agent are not running - $@";
5542         return 0;
5543     }
5544     return 1;
5545 }
5546
5547 sub template_create {
5548     my ($vmid, $conf, $disk) = @_;
5549
5550     my $storecfg = PVE::Storage::config();
5551
5552     foreach_drive($conf, sub {
5553         my ($ds, $drive) = @_;
5554
5555         return if drive_is_cdrom($drive);
5556         return if $disk && $ds ne $disk;
5557
5558         my $volid = $drive->{file};
5559         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
5560
5561         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
5562         $drive->{file} = $voliddst;
5563         $conf->{$ds} = print_drive($vmid, $drive);
5564         PVE::QemuConfig->write_config($vmid, $conf);
5565     });
5566 }
5567
5568 sub qemu_img_convert {
5569     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
5570
5571     my $storecfg = PVE::Storage::config();
5572     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
5573     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
5574
5575     if ($src_storeid && $dst_storeid) {
5576
5577         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
5578
5579         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
5580         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5581
5582         my $src_format = qemu_img_format($src_scfg, $src_volname);
5583         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
5584
5585         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
5586         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5587
5588         my $cmd = [];
5589         push @$cmd, '/usr/bin/qemu-img', 'convert', '-t', 'writeback', '-p', '-n';
5590         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
5591         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path;
5592         if ($is_zero_initialized) {
5593             push @$cmd, "zeroinit:$dst_path";
5594         } else {
5595             push @$cmd, $dst_path;
5596         }
5597
5598         my $parser = sub {
5599             my $line = shift;
5600             if($line =~ m/\((\S+)\/100\%\)/){
5601                 my $percent = $1;
5602                 my $transferred = int($size * $percent / 100);
5603                 my $remaining = $size - $transferred;
5604
5605                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
5606             }
5607
5608         };
5609
5610         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
5611         my $err = $@;
5612         die "copy failed: $err" if $err;
5613     }
5614 }
5615
5616 sub qemu_img_format {
5617     my ($scfg, $volname) = @_;
5618
5619     if ($scfg->{path} && $volname =~ m/\.(raw|cow|qcow|qcow2|qed|vmdk|cloop)$/) {
5620         return $1;
5621     } else {
5622         return "raw";
5623     }
5624 }
5625
5626 sub qemu_drive_mirror {
5627     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized) = @_;
5628
5629     my $storecfg = PVE::Storage::config();
5630     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
5631
5632     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5633
5634     my $format = qemu_img_format($dst_scfg, $dst_volname);
5635
5636     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5637
5638     my $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
5639
5640     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
5641     $opts->{format} = $format if $format;
5642
5643     print "drive mirror is starting (scanning bitmap) : this step can take some minutes/hours, depend of disk size and storage speed\n";
5644
5645     my $finish_job = sub {
5646         while (1) {
5647             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
5648             my $stat = @$stats[0];
5649             last if !$stat;
5650             sleep 1;
5651         }
5652     };
5653
5654     eval {
5655     vm_mon_cmd($vmid, "drive-mirror", %$opts);
5656         while (1) {
5657             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
5658             my $stat = @$stats[0];
5659             die "mirroring job seem to have die. Maybe do you have bad sectors?" if !$stat;
5660             die "error job is not mirroring" if $stat->{type} ne "mirror";
5661
5662             my $busy = $stat->{busy};
5663             my $ready = $stat->{ready};
5664
5665             if (my $total = $stat->{len}) {
5666                 my $transferred = $stat->{offset} || 0;
5667                 my $remaining = $total - $transferred;
5668                 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
5669
5670                 print "transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
5671             }
5672
5673
5674             if ($stat->{ready} eq 'true') {
5675
5676                 last if $vmiddst != $vmid;
5677
5678                 # try to switch the disk if source and destination are on the same guest
5679                 eval { vm_mon_cmd($vmid, "block-job-complete", device => "drive-$drive") };
5680                 if (!$@) {
5681                     &$finish_job();
5682                     last;
5683                 }
5684                 die $@ if $@ !~ m/cannot be completed/;
5685             }
5686             sleep 1;
5687         }
5688
5689
5690     };
5691     my $err = $@;
5692
5693     my $cancel_job = sub {
5694         vm_mon_cmd($vmid, "block-job-cancel", device => "drive-$drive");
5695         &$finish_job();
5696     };
5697
5698     if ($err) {
5699         eval { &$cancel_job(); };
5700         die "mirroring error: $err";
5701     }
5702
5703     if ($vmiddst != $vmid) {
5704         # if we clone a disk for a new target vm, we don't switch the disk
5705         &$cancel_job(); # so we call block-job-cancel
5706     }
5707 }
5708
5709 sub clone_disk {
5710     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
5711         $newvmid, $storage, $format, $full, $newvollist) = @_;
5712
5713     my $newvolid;
5714
5715     if (!$full) {
5716         print "create linked clone of drive $drivename ($drive->{file})\n";
5717         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
5718         push @$newvollist, $newvolid;
5719     } else {
5720         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
5721         $storeid = $storage if $storage;
5722
5723         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5724         if (!$format) {
5725             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5726             $format = qemu_img_format($scfg, $volname);
5727         }
5728
5729         # test if requested format is supported - else use default
5730         my $supported = grep { $_ eq $format } @$validFormats;
5731         $format = $defFormat if !$supported;
5732
5733         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
5734
5735         print "create full clone of drive $drivename ($drive->{file})\n";
5736         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $format, undef, ($size/1024));
5737         push @$newvollist, $newvolid;
5738
5739         PVE::Storage::activate_volumes($storecfg, $newvollist);
5740
5741         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
5742         if (!$running || $snapname) {
5743             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
5744         } else {
5745             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit);
5746         }
5747     }
5748
5749     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
5750
5751     my $disk = $drive;
5752     $disk->{format} = undef;
5753     $disk->{file} = $newvolid;
5754     $disk->{size} = $size;
5755
5756     return $disk;
5757 }
5758
5759 # this only works if VM is running
5760 sub get_current_qemu_machine {
5761     my ($vmid) = @_;
5762
5763     my $cmd = { execute => 'query-machines', arguments => {} };
5764     my $res = vm_qmp_command($vmid, $cmd);
5765
5766     my ($current, $default);
5767     foreach my $e (@$res) {
5768         $default = $e->{name} if $e->{'is-default'};
5769         $current = $e->{name} if $e->{'is-current'};
5770     }
5771
5772     # fallback to the default machine if current is not supported by qemu
5773     return $current || $default || 'pc';
5774 }
5775
5776 sub qemu_machine_feature_enabled {
5777     my ($machine, $kvmver, $version_major, $version_minor) = @_;
5778
5779     my $current_major;
5780     my $current_minor;
5781
5782     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
5783
5784         $current_major = $3;
5785         $current_minor = $4;
5786
5787     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
5788
5789         $current_major = $1;
5790         $current_minor = $2;
5791     }
5792
5793     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
5794
5795
5796 }
5797
5798 sub qemu_machine_pxe {
5799     my ($vmid, $conf, $machine) = @_;
5800
5801     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
5802
5803     foreach my $opt (keys %$conf) {
5804         next if $opt !~ m/^net(\d+)$/;
5805         my $net = PVE::QemuServer::parse_net($conf->{$opt});
5806         next if !$net;
5807         my $romfile = PVE::QemuServer::vm_mon_cmd_nocheck($vmid, 'qom-get', path => $opt, property => 'romfile');
5808         return $machine.".pxe" if $romfile =~ m/pxe/;
5809         last;
5810     }
5811
5812     return $machine;
5813 }
5814
5815 sub qemu_use_old_bios_files {
5816     my ($machine_type) = @_;
5817
5818     return if !$machine_type;
5819
5820     my $use_old_bios_files = undef;
5821
5822     if ($machine_type =~ m/^(\S+)\.pxe$/) {
5823         $machine_type = $1;
5824         $use_old_bios_files = 1;
5825     } else {
5826         my $kvmver = kvm_user_version();
5827         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
5828         # load new efi bios files on migration. So this hack is required to allow
5829         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
5830         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
5831         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
5832     }
5833
5834     return ($use_old_bios_files, $machine_type);
5835 }
5836
5837 sub lspci {
5838
5839     my $devices = {};
5840
5841     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
5842             my (undef, $id, $function) = @_;
5843             my $res = { id => $id, function => $function};
5844             push @{$devices->{$id}}, $res;
5845     });
5846
5847     return $devices;
5848 }
5849
5850 sub vm_iothreads_list {
5851     my ($vmid) = @_;
5852
5853     my $res = vm_mon_cmd($vmid, 'query-iothreads');
5854
5855     my $iothreads = {};
5856     foreach my $iothread (@$res) {
5857         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
5858     }
5859
5860     return $iothreads;
5861 }
5862
5863 sub scsihw_infos {
5864     my ($conf, $drive) = @_;
5865
5866     my $maxdev = 0;
5867
5868     if ($conf->{scsihw} && ($conf->{scsihw} =~ m/^lsi/)) {
5869         $maxdev = 7;
5870     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
5871         $maxdev = 1;
5872     } else {
5873         $maxdev = 256;
5874     }
5875
5876     my $controller = int($drive->{index} / $maxdev);
5877     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
5878
5879     return ($maxdev, $controller, $controller_prefix);
5880 }
5881
5882 # bash completion helper
5883
5884 sub complete_backup_archives {
5885     my ($cmdname, $pname, $cvalue) = @_;
5886
5887     my $cfg = PVE::Storage::config();
5888
5889     my $storeid;
5890
5891     if ($cvalue =~ m/^([^:]+):/) {
5892         $storeid = $1;
5893     }
5894
5895     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
5896
5897     my $res = [];
5898     foreach my $id (keys %$data) {
5899         foreach my $item (@{$data->{$id}}) {
5900             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
5901             push @$res, $item->{volid} if defined($item->{volid});
5902         }
5903     }
5904
5905     return $res;
5906 }
5907
5908 my $complete_vmid_full = sub {
5909     my ($running) = @_;
5910
5911     my $idlist = vmstatus();
5912
5913     my $res = [];
5914
5915     foreach my $id (keys %$idlist) {
5916         my $d = $idlist->{$id};
5917         if (defined($running)) {
5918             next if $d->{template};
5919             next if $running && $d->{status} ne 'running';
5920             next if !$running && $d->{status} eq 'running';
5921         }
5922         push @$res, $id;
5923
5924     }
5925     return $res;
5926 };
5927
5928 sub complete_vmid {
5929     return &$complete_vmid_full();
5930 }
5931
5932 sub complete_vmid_stopped {
5933     return &$complete_vmid_full(0);
5934 }
5935
5936 sub complete_vmid_running {
5937     return &$complete_vmid_full(1);
5938 }
5939
5940 sub complete_storage {
5941
5942     my $cfg = PVE::Storage::config();
5943     my $ids = $cfg->{ids};
5944
5945     my $res = [];
5946     foreach my $sid (keys %$ids) {
5947         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
5948         next if !$ids->{$sid}->{content}->{images};
5949         push @$res, $sid;
5950     }
5951
5952     return $res;
5953 }
5954
5955 1;