PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QemuConfig;
  31 use PVE::QMPClient;
  32 use PVE::RPCEnvironment;
  33 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
  34 use PVE::QemuServer::Memory;
  35 use PVE::QemuServer::USB qw(parse_usb_device);
  36 use Time::HiRes qw(gettimeofday);
  37 use File::Copy qw(copy);
  38 use URI::Escape;
  39
  40 my $OVMF_CODE = '/usr/share/kvm/OVMF_CODE-pure-efi.fd';
  41 my $OVMF_VARS = '/usr/share/kvm/OVMF_VARS-pure-efi.fd';
  42 my $OVMF_IMG = '/usr/share/kvm/OVMF-pure-efi.fd';
  43
  44 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  45
  46 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  47
  48 # Note about locking: we use flock on the config file protect
  49 # against concurent actions.
  50 # Aditionaly, we have a 'lock' setting in the config file. This
  51 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  52 # allowed when such lock is set. But you can ignore this kind of
  53 # lock with the --skiplock flag.
  54
  55 cfs_register_file('/qemu-server/',
  56                   \&parse_vm_config,
  57                   \&write_vm_config);
  58
  59 PVE::JSONSchema::register_standard_option('skiplock', {
  60     description => "Ignore locks - only root is allowed to use this option.",
  61     type => 'boolean',
  62     optional => 1,
  63 });
  64
  65 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  66     description => "Some command save/restore state from this location.",
  67     type => 'string',
  68     maxLength => 128,
  69     optional => 1,
  70 });
  71
  72 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  73     description => "The name of the snapshot.",
  74     type => 'string', format => 'pve-configid',
  75     maxLength => 40,
  76 });
  77
  78 #no warnings 'redefine';
  79
  80 sub cgroups_write {
  81    my ($controller, $vmid, $option, $value) = @_;
  82
  83    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  84    PVE::ProcFSTools::write_proc_entry($path, $value);
  85
  86 }
  87
  88 my $nodename = PVE::INotify::nodename();
  89
  90 mkdir "/etc/pve/nodes/$nodename";
  91 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  92 mkdir $confdir;
  93
  94 my $var_run_tmpdir = "/var/run/qemu-server";
  95 mkdir $var_run_tmpdir;
  96
  97 my $lock_dir = "/var/lock/qemu-server";
  98 mkdir $lock_dir;
  99
 100 my $pcisysfs = "/sys/bus/pci";
 101
 102 my $cpu_vendor_list = {
 103     # Intel CPUs
 104     486 => 'GenuineIntel',
 105     pentium => 'GenuineIntel',
 106     pentium2  => 'GenuineIntel',
 107     pentium3  => 'GenuineIntel',
 108     coreduo => 'GenuineIntel',
 109     core2duo => 'GenuineIntel',
 110     Conroe  => 'GenuineIntel',
 111     Penryn  => 'GenuineIntel',
 112     Nehalem  => 'GenuineIntel',
 113     Westmere => 'GenuineIntel',
 114     SandyBridge => 'GenuineIntel',
 115     IvyBridge => 'GenuineIntel',
 116     Haswell => 'GenuineIntel',
 117     'Haswell-noTSX' => 'GenuineIntel',
 118     Broadwell => 'GenuineIntel',
 119     'Broadwell-noTSX' => 'GenuineIntel',
 120
 121     # AMD CPUs
 122     athlon => 'AuthenticAMD',
 123     phenom  => 'AuthenticAMD',
 124     Opteron_G1  => 'AuthenticAMD',
 125     Opteron_G2  => 'AuthenticAMD',
 126     Opteron_G3  => 'AuthenticAMD',
 127     Opteron_G4  => 'AuthenticAMD',
 128     Opteron_G5  => 'AuthenticAMD',
 129
 130     # generic types, use vendor from host node
 131     host => 'default',
 132     kvm32 => 'default',
 133     kvm64 => 'default',
 134     qemu32 => 'default',
 135     qemu64 => 'default',
 136 };
 137
 138 my $cpu_fmt = {
 139     cputype => {
 140         description => "Emulated CPU type.",
 141         type => 'string',
 142         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 143         format_description => 'cputype',
 144         default => 'kvm64',
 145         default_key => 1,
 146     },
 147     hidden => {
 148         description => "Do not identify as a KVM virtual machine.",
 149         type => 'boolean',
 150         optional => 1,
 151         default => 0
 152     },
 153 };
 154
 155 my $watchdog_fmt = {
 156     model => {
 157         default_key => 1,
 158         type => 'string',
 159         enum => [qw(i6300esb ib700)],
 160         description => "Watchdog type to emulate.",
 161         default => 'i6300esb',
 162         optional => 1,
 163     },
 164     action => {
 165         type => 'string',
 166         enum => [qw(reset shutdown poweroff pause debug none)],
 167         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 168         optional => 1,
 169     },
 170 };
 171 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 172
 173 my $confdesc = {
 174     onboot => {
 175         optional => 1,
 176         type => 'boolean',
 177         description => "Specifies whether a VM will be started during system bootup.",
 178         default => 0,
 179     },
 180     autostart => {
 181         optional => 1,
 182         type => 'boolean',
 183         description => "Automatic restart after crash (currently ignored).",
 184         default => 0,
 185     },
 186     hotplug => {
 187         optional => 1,
 188         type => 'string', format => 'pve-hotplug-features',
 189         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 190         default => 'network,disk,usb',
 191     },
 192     reboot => {
 193         optional => 1,
 194         type => 'boolean',
 195         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 196         default => 1,
 197     },
 198     lock => {
 199         optional => 1,
 200         type => 'string',
 201         description => "Lock/unlock the VM.",
 202         enum => [qw(migrate backup snapshot rollback)],
 203     },
 204     cpulimit => {
 205         optional => 1,
 206         type => 'number',
 207         description => "Limit of CPU usage.",
 208         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 209         minimum => 0,
 210         maximum => 128,
 211         default => 0,
 212     },
 213     cpuunits => {
 214         optional => 1,
 215         type => 'integer',
 216         description => "CPU weight for a VM.",
 217         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 218         minimum => 0,
 219         maximum => 500000,
 220         default => 1000,
 221     },
 222     memory => {
 223         optional => 1,
 224         type => 'integer',
 225         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 226         minimum => 16,
 227         default => 512,
 228     },
 229     balloon => {
 230         optional => 1,
 231         type => 'integer',
 232         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 233         minimum => 0,
 234     },
 235     shares => {
 236         optional => 1,
 237         type => 'integer',
 238         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 239         minimum => 0,
 240         maximum => 50000,
 241         default => 1000,
 242     },
 243     keyboard => {
 244         optional => 1,
 245         type => 'string',
 246         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.conf' configuration file.",
 247         enum => PVE::Tools::kvmkeymaplist(),
 248         default => 'en-us',
 249     },
 250     name => {
 251         optional => 1,
 252         type => 'string', format => 'dns-name',
 253         description => "Set a name for the VM. Only used on the configuration web interface.",
 254     },
 255     scsihw => {
 256         optional => 1,
 257         type => 'string',
 258         description => "SCSI controller model",
 259         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 260         default => 'lsi',
 261     },
 262     description => {
 263         optional => 1,
 264         type => 'string',
 265         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 266     },
 267     ostype => {
 268         optional => 1,
 269         type => 'string',
 270         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 l24 l26 solaris)],
 271         description => "Specify guest operating system.",
 272         verbose_description => <<EODESC,
 273 Specify guest operating system. This is used to enable special
 274 optimization/features for specific operating systems:
 275
 276 [horizontal]
 277 other;; unspecified OS
 278 wxp;; Microsoft Windows XP
 279 w2k;; Microsoft Windows 2000
 280 w2k3;; Microsoft Windows 2003
 281 w2k8;; Microsoft Windows 2008
 282 wvista;; Microsoft Windows Vista
 283 win7;; Microsoft Windows 7
 284 win8;; Microsoft Windows 8/2012
 285 l24;; Linux 2.4 Kernel
 286 l26;; Linux 2.6/3.X Kernel
 287 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 288 EODESC
 289     },
 290     boot => {
 291         optional => 1,
 292         type => 'string',
 293         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 294         pattern => '[acdn]{1,4}',
 295         default => 'cdn',
 296     },
 297     bootdisk => {
 298         optional => 1,
 299         type => 'string', format => 'pve-qm-bootdisk',
 300         description => "Enable booting from specified disk.",
 301         pattern => '(ide|sata|scsi|virtio)\d+',
 302     },
 303     smp => {
 304         optional => 1,
 305         type => 'integer',
 306         description => "The number of CPUs. Please use option -sockets instead.",
 307         minimum => 1,
 308         default => 1,
 309     },
 310     sockets => {
 311         optional => 1,
 312         type => 'integer',
 313         description => "The number of CPU sockets.",
 314         minimum => 1,
 315         default => 1,
 316     },
 317     cores => {
 318         optional => 1,
 319         type => 'integer',
 320         description => "The number of cores per socket.",
 321         minimum => 1,
 322         default => 1,
 323     },
 324     numa => {
 325         optional => 1,
 326         type => 'boolean',
 327         description => "Enable/disable NUMA.",
 328         default => 0,
 329     },
 330     hugepages => {
 331         optional => 1,
 332         type => 'string',
 333         description => "Enable/disable hugepages memory.",
 334         enum => [qw(any 2 1024)],
 335     },
 336     vcpus => {
 337         optional => 1,
 338         type => 'integer',
 339         description => "Number of hotplugged vcpus.",
 340         minimum => 1,
 341         default => 0,
 342     },
 343     acpi => {
 344         optional => 1,
 345         type => 'boolean',
 346         description => "Enable/disable ACPI.",
 347         default => 1,
 348     },
 349     agent => {
 350         optional => 1,
 351         type => 'boolean',
 352         description => "Enable/disable Qemu GuestAgent.",
 353         default => 0,
 354     },
 355     kvm => {
 356         optional => 1,
 357         type => 'boolean',
 358         description => "Enable/disable KVM hardware virtualization.",
 359         default => 1,
 360     },
 361     tdf => {
 362         optional => 1,
 363         type => 'boolean',
 364         description => "Enable/disable time drift fix.",
 365         default => 0,
 366     },
 367     localtime => {
 368         optional => 1,
 369         type => 'boolean',
 370         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 371     },
 372     freeze => {
 373         optional => 1,
 374         type => 'boolean',
 375         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 376     },
 377     vga => {
 378         optional => 1,
 379         type => 'string',
 380         description => "Select the VGA type.",
 381         verbose_description => "Select the VGA type. If you want to use high resolution" .
 382             " modes (>= 1280x1024x16) then you should use the options " .
 383             "'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and " .
 384             "'cirrus' for other OS types. The 'qxl' option enables the SPICE " .
 385             "display sever. For win* OS you can select how many independent " .
 386             "displays you want, Linux guests can add displays them self. " .
 387             "You can also run without any graphic card, using a serial device" .
 388             " as terminal.",
 389         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 390     },
 391     watchdog => {
 392         optional => 1,
 393         type => 'string', format => 'pve-qm-watchdog',
 394         description => "Create a virtual hardware watchdog device.",
 395         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 396             " (by a guest action), the watchdog must be periodically polled " .
 397             "by an agent inside the guest or else the watchdog will reset " .
 398             "the guest (or execute the respective action specified)",
 399     },
 400     startdate => {
 401         optional => 1,
 402         type => 'string',
 403         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 404         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 405         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 406         default => 'now',
 407     },
 408     startup =>  get_standard_option('pve-startup-order'),
 409     template => {
 410         optional => 1,
 411         type => 'boolean',
 412         description => "Enable/disable Template.",
 413         default => 0,
 414     },
 415     args => {
 416         optional => 1,
 417         type => 'string',
 418         description => "Arbitrary arguments passed to kvm.",
 419         verbose_description => <<EODESCR,
 420 Arbitrary arguments passed to kvm, for example:
 421
 422 args: -no-reboot -no-hpet
 423
 424 NOTE: this option is for experts only.
 425 EODESCR
 426     },
 427     tablet => {
 428         optional => 1,
 429         type => 'boolean',
 430         default => 1,
 431         description => "Enable/disable the USB tablet device.",
 432         verbose_description => "Enable/disable the USB tablet device. This device is " .
 433             "usually needed to allow absolute mouse positioning with VNC. " .
 434             "Else the mouse runs out of sync with normal VNC clients. " .
 435             "If you're running lots of console-only guests on one host, " .
 436             "you may consider disabling this to save some context switches. " .
 437             "This is turned off by default if you use spice (-vga=qxl).",
 438     },
 439     migrate_speed => {
 440         optional => 1,
 441         type => 'integer',
 442         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 443         minimum => 0,
 444         default => 0,
 445     },
 446     migrate_downtime => {
 447         optional => 1,
 448         type => 'number',
 449         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 450         minimum => 0,
 451         default => 0.1,
 452     },
 453     cdrom => {
 454         optional => 1,
 455         type => 'string', format => 'pve-qm-drive',
 456         typetext => 'volume',
 457         description => "This is an alias for option -ide2",
 458     },
 459     cpu => {
 460         optional => 1,
 461         description => "Emulated CPU type.",
 462         type => 'string',
 463         format => $cpu_fmt,
 464     },
 465     parent => get_standard_option('pve-snapshot-name', {
 466         optional => 1,
 467         description => "Parent snapshot name. This is used internally, and should not be modified.",
 468     }),
 469     snaptime => {
 470         optional => 1,
 471         description => "Timestamp for snapshots.",
 472         type => 'integer',
 473         minimum => 0,
 474     },
 475     vmstate => {
 476         optional => 1,
 477         type => 'string', format => 'pve-volume-id',
 478         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 479     },
 480     machine => {
 481         description => "Specific the Qemu machine type.",
 482         type => 'string',
 483         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
 484         maxLength => 40,
 485         optional => 1,
 486     },
 487     smbios1 => {
 488         description => "Specify SMBIOS type 1 fields.",
 489         type => 'string', format => 'pve-qm-smbios1',
 490         maxLength => 256,
 491         optional => 1,
 492     },
 493     protection => {
 494         optional => 1,
 495         type => 'boolean',
 496         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 497         default => 0,
 498     },
 499     bios => {
 500         optional => 1,
 501         type => 'string',
 502         enum => [ qw(seabios ovmf) ],
 503         description => "Select BIOS implementation.",
 504         default => 'seabios',
 505     },
 506 };
 507
 508 # what about other qemu settings ?
 509 #cpu => 'string',
 510 #machine => 'string',
 511 #fda => 'file',
 512 #fdb => 'file',
 513 #mtdblock => 'file',
 514 #sd => 'file',
 515 #pflash => 'file',
 516 #snapshot => 'bool',
 517 #bootp => 'file',
 518 ##tftp => 'dir',
 519 ##smb => 'dir',
 520 #kernel => 'file',
 521 #append => 'string',
 522 #initrd => 'file',
 523 ##soundhw => 'string',
 524
 525 while (my ($k, $v) = each %$confdesc) {
 526     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 527 }
 528
 529 my $MAX_IDE_DISKS = 4;
 530 my $MAX_SCSI_DISKS = 14;
 531 my $MAX_VIRTIO_DISKS = 16;
 532 my $MAX_SATA_DISKS = 6;
 533 my $MAX_USB_DEVICES = 5;
 534 my $MAX_NETS = 32;
 535 my $MAX_UNUSED_DISKS = 8;
 536 my $MAX_HOSTPCI_DEVICES = 4;
 537 my $MAX_SERIAL_PORTS = 4;
 538 my $MAX_PARALLEL_PORTS = 3;
 539 my $MAX_NUMA = 8;
 540
 541 my $numa_fmt = {
 542     cpus => {
 543         type => "string",
 544         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 545         description => "CPUs accessing this NUMA node.",
 546         format_description => "id[-id];...",
 547     },
 548     memory => {
 549         type => "number",
 550         description => "Amount of memory this NUMA node provides.",
 551         optional => 1,
 552     },
 553     hostnodes => {
 554         type => "string",
 555         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 556         description => "Host NUMA nodes to use.",
 557         format_description => "id[-id];...",
 558         optional => 1,
 559     },
 560     policy => {
 561         type => 'string',
 562         enum => [qw(preferred bind interleave)],
 563         description => "NUMA allocation policy.",
 564         optional => 1,
 565     },
 566 };
 567 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 568 my $numadesc = {
 569     optional => 1,
 570     type => 'string', format => $numa_fmt,
 571     description => "NUMA topology.",
 572 };
 573 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 574
 575 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 576     $confdesc->{"numa$i"} = $numadesc;
 577 }
 578
 579 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 580                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 581                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 582 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 583
 584 my $net_fmt_bridge_descr = <<__EOD__;
 585 Bridge to attach the network device to. The Proxmox VE standard bridge
 586 is called 'vmbr0'.
 587
 588 If you do not specify a bridge, we create a kvm user (NATed) network
 589 device, which provides DHCP and DNS services. The following addresses
 590 are used:
 591
 592  10.0.2.2   Gateway
 593  10.0.2.3   DNS Server
 594  10.0.2.4   SMB Server
 595
 596 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 597 __EOD__
 598
 599 my $net_fmt = {
 600     macaddr => {
 601         type => 'string',
 602         pattern => qr/[0-9a-f]{2}(?::[0-9a-f]{2}){5}/i,
 603         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 604         format_description => "XX:XX:XX:XX:XX:XX",
 605         optional => 1,
 606     },
 607     model => {
 608         type => 'string',
 609         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 610         format_description => 'model',
 611         enum => $nic_model_list,
 612         default_key => 1,
 613     },
 614     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 615     bridge => {
 616         type => 'string',
 617         description => $net_fmt_bridge_descr,
 618         format_description => 'bridge',
 619         optional => 1,
 620     },
 621     queues => {
 622         type => 'integer',
 623         minimum => 0, maximum => 16,
 624         description => 'Number of packet queues to be used on the device.',
 625         optional => 1,
 626     },
 627     rate => {
 628         type => 'number',
 629         minimum => 0,
 630         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 631         optional => 1,
 632     },
 633     tag => {
 634         type => 'integer',
 635         minimum => 1, maximum => 4094,
 636         description => 'VLAN tag to apply to packets on this interface.',
 637         optional => 1,
 638     },
 639     trunks => {
 640         type => 'string',
 641         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 642         description => 'VLAN trunks to pass through this interface.',
 643         format_description => 'vlanid[;vlanid...]',
 644         optional => 1,
 645     },
 646     firewall => {
 647         type => 'boolean',
 648         description => 'Whether this interface should be protected by the firewall.',
 649         optional => 1,
 650     },
 651     link_down => {
 652         type => 'boolean',
 653         description => 'Whether this interface should be disconnected (like pulling the plug).',
 654         optional => 1,
 655     },
 656 };
 657
 658 my $netdesc = {
 659     optional => 1,
 660     type => 'string', format => $net_fmt,
 661     description => "Specify network devices.",
 662 };
 663
 664 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 665
 666 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 667     $confdesc->{"net$i"} = $netdesc;
 668 }
 669
 670 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 671 sub verify_volume_id_or_qm_path {
 672     my ($volid, $noerr) = @_;
 673
 674     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 675         return $volid;
 676     }
 677
 678     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 679     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 680     if ($@) {
 681         return undef if $noerr;
 682         die $@;
 683     }
 684     return $volid;
 685 }
 686
 687 my $drivename_hash;
 688
 689 my %drivedesc_base = (
 690     volume => { alias => 'file' },
 691     file => {
 692         type => 'string',
 693         format => 'pve-volume-id-or-qm-path',
 694         default_key => 1,
 695         format_description => 'volume',
 696         description => "The drive's backing volume.",
 697     },
 698     media => {
 699         type => 'string',
 700         enum => [qw(cdrom disk)],
 701         description => "The drive's media type.",
 702         default => 'disk',
 703         optional => 1
 704     },
 705     cyls => {
 706         type => 'integer',
 707         description => "Force the drive's physical geometry to have a specific cylinder count.",
 708         optional => 1
 709     },
 710     heads => {
 711         type => 'integer',
 712         description => "Force the drive's physical geometry to have a specific head count.",
 713         optional => 1
 714     },
 715     secs => {
 716         type => 'integer',
 717         description => "Force the drive's physical geometry to have a specific sector count.",
 718         optional => 1
 719     },
 720     trans => {
 721         type => 'string',
 722         enum => [qw(none lba auto)],
 723         description => "Force disk geometry bios translation mode.",
 724         optional => 1,
 725     },
 726     snapshot => {
 727         type => 'boolean',
 728         description => "Whether the drive should be included when making snapshots.",
 729         optional => 1,
 730     },
 731     cache => {
 732         type => 'string',
 733         enum => [qw(none writethrough writeback unsafe directsync)],
 734         description => "The drive's cache mode",
 735         optional => 1,
 736     },
 737     format => {
 738         type => 'string',
 739         format_description => 'image format',
 740         enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
 741         description => "The drive's backing file's data format.",
 742         optional => 1,
 743     },
 744     size => {
 745         type => 'string',
 746         format => 'disk-size',
 747         format_description => 'DiskSize',
 748         description => "Disk size. This is purely informational and has no effect.",
 749         optional => 1,
 750     },
 751     backup => {
 752         type => 'boolean',
 753         description => "Whether the drive should be included when making backups.",
 754         optional => 1,
 755     },
 756     werror => {
 757         type => 'string',
 758         enum => [qw(enospc ignore report stop)],
 759         description => 'Write error action.',
 760         optional => 1,
 761     },
 762     aio => {
 763         type => 'string',
 764         enum => [qw(native threads)],
 765         description => 'AIO type to use.',
 766         optional => 1,
 767     },
 768     discard => {
 769         type => 'string',
 770         enum => [qw(ignore on)],
 771         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 772         optional => 1,
 773     },
 774     detect_zeroes => {
 775         type => 'boolean',
 776         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 777         optional => 1,
 778     },
 779     serial => {
 780         type => 'string',
 781         format => 'urlencoded',
 782         format_description => 'serial',
 783         maxLength => 20*3, # *3 since it's %xx url enoded
 784         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
 785         optional => 1,
 786     }
 787 );
 788
 789 my %rerror_fmt = (
 790     rerror => {
 791         type => 'string',
 792         enum => [qw(ignore report stop)],
 793         description => 'Read error action.',
 794         optional => 1,
 795     },
 796 );
 797
 798 my %iothread_fmt = ( iothread => {
 799         type => 'boolean',
 800         description => "Whether to use iothreads for this drive",
 801         optional => 1,
 802 });
 803
 804 my %model_fmt = (
 805     model => {
 806         type => 'string',
 807         format => 'urlencoded',
 808         format_description => 'model',
 809         maxLength => 40*3, # *3 since it's %xx url enoded
 810         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
 811         optional => 1,
 812     },
 813 );
 814
 815 my %queues_fmt = (
 816     queues => {
 817         type => 'integer',
 818         description => "Number of queues.",
 819         minimum => 2,
 820         optional => 1
 821     }
 822 );
 823
 824 my $add_throttle_desc = sub {
 825     my ($key, $type, $what, $unit, $longunit) = @_;
 826     $drivedesc_base{$key} = {
 827         type => $type,
 828         format_description => $unit,
 829         description => "Maximum $what speed in $longunit per second.",
 830         optional => 1,
 831     };
 832 };
 833 # throughput: (leaky bucket)
 834 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes');
 835 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes');
 836 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes');
 837 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes');
 838 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes');
 839 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes');
 840 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations');
 841 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations');
 842 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations');
 843
 844 # pools: (pool of IO before throttling starts taking effect)
 845 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes');
 846 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes');
 847 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes');
 848 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations');
 849 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations');
 850 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations');
 851
 852 my $ide_fmt = {
 853     %drivedesc_base,
 854     %rerror_fmt,
 855     %model_fmt,
 856 };
 857
 858 my $idedesc = {
 859     optional => 1,
 860     type => 'string', format => $ide_fmt,
 861     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
 862 };
 863 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
 864
 865 my $scsi_fmt = {
 866     %drivedesc_base,
 867     %iothread_fmt,
 868     %queues_fmt,
 869 };
 870 my $scsidesc = {
 871     optional => 1,
 872     type => 'string', format => $scsi_fmt,
 873     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
 874 };
 875 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 876
 877 my $sata_fmt = {
 878     %drivedesc_base,
 879     %rerror_fmt,
 880 };
 881 my $satadesc = {
 882     optional => 1,
 883     type => 'string', format => $sata_fmt,
 884     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
 885 };
 886 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
 887
 888 my $virtio_fmt = {
 889     %drivedesc_base,
 890     %iothread_fmt,
 891     %rerror_fmt,
 892 };
 893 my $virtiodesc = {
 894     optional => 1,
 895     type => 'string', format => $virtio_fmt,
 896     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
 897 };
 898 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
 899
 900 my $alldrive_fmt = {
 901     %drivedesc_base,
 902     %rerror_fmt,
 903     %iothread_fmt,
 904     %model_fmt,
 905     %queues_fmt,
 906 };
 907
 908 my $efidisk_fmt = {
 909     volume => { alias => 'file' },
 910     file => {
 911         type => 'string',
 912         format => 'pve-volume-id-or-qm-path',
 913         default_key => 1,
 914         format_description => 'volume',
 915         description => "The drive's backing volume.",
 916     },
 917     format => {
 918         type => 'string',
 919         format_description => 'image format',
 920         enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
 921         description => "The drive's backing file's data format.",
 922         optional => 1,
 923     },
 924     size => {
 925         type => 'string',
 926         format => 'disk-size',
 927         format_description => 'DiskSize',
 928         description => "Disk size. This is purely informational and has no effect.",
 929         optional => 1,
 930     },
 931 };
 932
 933 my $efidisk_desc = {
 934     optional => 1,
 935     type => 'string', format => $efidisk_fmt,
 936     description => "Configure a Disk for storing EFI vars",
 937 };
 938
 939 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
 940
 941 my $usb_fmt = {
 942     host => {
 943         default_key => 1,
 944         type => 'string', format => 'pve-qm-usb-device',
 945         format_description => 'HOSTUSBDEVICE|spice',
 946         description => <<EODESCR,
 947 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
 948
 949  'bus-port(.port)*' (decimal numbers) or
 950  'vendor_id:product_id' (hexadeciaml numbers) or
 951  'spice'
 952
 953 You can use the 'lsusb -t' command to list existing usb devices.
 954
 955 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 956
 957 The value 'spice' can be used to add a usb redirection devices for spice.
 958 EODESCR
 959     },
 960     usb3 => {
 961         optional => 1,
 962         type => 'boolean',
 963         description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
 964         default => 0,
 965     },
 966 };
 967
 968 my $usbdesc = {
 969     optional => 1,
 970     type => 'string', format => $usb_fmt,
 971     description => "Configure an USB device (n is 0 to 4).",
 972 };
 973 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
 974
 975 # NOTE: the match-groups of this regex are used in parse_hostpci
 976 my $PCIRE = qr/([a-f0-9]{2}:[a-f0-9]{2})(?:\.([a-f0-9]))?/;
 977 my $hostpci_fmt = {
 978     host => {
 979         default_key => 1,
 980         type => 'string',
 981         pattern => qr/$PCIRE(;$PCIRE)*/,
 982         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
 983         description => <<EODESCR,
 984 Host PCI device pass through. The PCI ID of a host's PCI device or a list
 985 of PCI virtual functions of the host. HOSTPCIID syntax is:
 986
 987 'bus:dev.func' (hexadecimal numbers)
 988
 989 You can us the 'lspci' command to list existing PCI devices.
 990 EODESCR
 991     },
 992     rombar => {
 993         type => 'boolean',
 994         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
 995         optional => 1,
 996         default => 1,
 997     },
 998     pcie => {
 999         type => 'boolean',
1000         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1001         optional => 1,
1002         default => 0,
1003     },
1004     'x-vga' => {
1005         type => 'boolean',
1006         description =>  "Enable vfio-vga device support.",
1007         optional => 1,
1008         default => 0,
1009     },
1010 };
1011 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1012
1013 my $hostpcidesc = {
1014         optional => 1,
1015         type => 'string', format => 'pve-qm-hostpci',
1016         description => "Map host PCI devices into guest.",
1017         verbose_description =>  <<EODESCR,
1018 Map host PCI devices into guest.
1019
1020 NOTE: This option allows direct access to host hardware. So it is no longer
1021 possible to migrate such machines - use with special care.
1022
1023 CAUTION: Experimental! User reported problems with this option.
1024 EODESCR
1025 };
1026 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1027
1028 my $serialdesc = {
1029         optional => 1,
1030         type => 'string',
1031         pattern => '(/dev/.+|socket)',
1032         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1033         verbose_description =>  <<EODESCR,
1034 Create a serial device inside the VM (n is 0 to 3), and pass through a
1035 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1036 host side (use 'qm terminal' to open a terminal connection).
1037
1038 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1039
1040 CAUTION: Experimental! User reported problems with this option.
1041 EODESCR
1042 };
1043
1044 my $paralleldesc= {
1045         optional => 1,
1046         type => 'string',
1047         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1048         description =>  "Map host parallel devices (n is 0 to 2).",
1049         verbose_description =>  <<EODESCR,
1050 Map host parallel devices (n is 0 to 2).
1051
1052 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1053
1054 CAUTION: Experimental! User reported problems with this option.
1055 EODESCR
1056 };
1057
1058 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1059     $confdesc->{"parallel$i"} = $paralleldesc;
1060 }
1061
1062 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1063     $confdesc->{"serial$i"} = $serialdesc;
1064 }
1065
1066 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1067     $confdesc->{"hostpci$i"} = $hostpcidesc;
1068 }
1069
1070 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1071     $drivename_hash->{"ide$i"} = 1;
1072     $confdesc->{"ide$i"} = $idedesc;
1073 }
1074
1075 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1076     $drivename_hash->{"sata$i"} = 1;
1077     $confdesc->{"sata$i"} = $satadesc;
1078 }
1079
1080 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1081     $drivename_hash->{"scsi$i"} = 1;
1082     $confdesc->{"scsi$i"} = $scsidesc ;
1083 }
1084
1085 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1086     $drivename_hash->{"virtio$i"} = 1;
1087     $confdesc->{"virtio$i"} = $virtiodesc;
1088 }
1089
1090 $drivename_hash->{efidisk0} = 1;
1091 $confdesc->{efidisk0} = $efidisk_desc;
1092
1093 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1094     $confdesc->{"usb$i"} = $usbdesc;
1095 }
1096
1097 my $unuseddesc = {
1098     optional => 1,
1099     type => 'string', format => 'pve-volume-id',
1100     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1101 };
1102
1103 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1104     $confdesc->{"unused$i"} = $unuseddesc;
1105 }
1106
1107 my $kvm_api_version = 0;
1108
1109 sub kvm_version {
1110
1111     return $kvm_api_version if $kvm_api_version;
1112
1113     my $fh = IO::File->new("</dev/kvm") ||
1114         return 0;
1115
1116     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
1117         $kvm_api_version = $v;
1118     }
1119
1120     $fh->close();
1121
1122     return  $kvm_api_version;
1123 }
1124
1125 my $kvm_user_version;
1126
1127 sub kvm_user_version {
1128
1129     return $kvm_user_version if $kvm_user_version;
1130
1131     $kvm_user_version = 'unknown';
1132
1133     my $code = sub {
1134         my $line = shift;
1135         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1136             $kvm_user_version = $2;
1137         }
1138     };
1139
1140     eval { run_command("kvm -version", outfunc => $code); };
1141     warn $@ if $@;
1142
1143     return $kvm_user_version;
1144
1145 }
1146
1147 my $kernel_has_vhost_net = -c '/dev/vhost-net';
1148
1149 sub valid_drive_names {
1150     # order is important - used to autoselect boot disk
1151     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1152             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1153             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1154             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1155             'efidisk0');
1156 }
1157
1158 sub is_valid_drivename {
1159     my $dev = shift;
1160
1161     return defined($drivename_hash->{$dev});
1162 }
1163
1164 sub option_exists {
1165     my $key = shift;
1166     return defined($confdesc->{$key});
1167 }
1168
1169 sub nic_models {
1170     return $nic_model_list;
1171 }
1172
1173 sub os_list_description {
1174
1175     return {
1176         other => 'Other',
1177         wxp => 'Windows XP',
1178         w2k => 'Windows 2000',
1179         w2k3 =>, 'Windows 2003',
1180         w2k8 => 'Windows 2008',
1181         wvista => 'Windows Vista',
1182         win7 => 'Windows 7',
1183         win8 => 'Windows 8/2012',
1184         l24 => 'Linux 2.4',
1185         l26 => 'Linux 2.6',
1186     };
1187 }
1188
1189 my $cdrom_path;
1190
1191 sub get_cdrom_path {
1192
1193     return  $cdrom_path if $cdrom_path;
1194
1195     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1196     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1197     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1198 }
1199
1200 sub get_iso_path {
1201     my ($storecfg, $vmid, $cdrom) = @_;
1202
1203     if ($cdrom eq 'cdrom') {
1204         return get_cdrom_path();
1205     } elsif ($cdrom eq 'none') {
1206         return '';
1207     } elsif ($cdrom =~ m|^/|) {
1208         return $cdrom;
1209     } else {
1210         return PVE::Storage::path($storecfg, $cdrom);
1211     }
1212 }
1213
1214 # try to convert old style file names to volume IDs
1215 sub filename_to_volume_id {
1216     my ($vmid, $file, $media) = @_;
1217
1218     if (!($file eq 'none' || $file eq 'cdrom' ||
1219           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1220
1221         return undef if $file =~ m|/|;
1222
1223         if ($media && $media eq 'cdrom') {
1224             $file = "local:iso/$file";
1225         } else {
1226             $file = "local:$vmid/$file";
1227         }
1228     }
1229
1230     return $file;
1231 }
1232
1233 sub verify_media_type {
1234     my ($opt, $vtype, $media) = @_;
1235
1236     return if !$media;
1237
1238     my $etype;
1239     if ($media eq 'disk') {
1240         $etype = 'images';
1241     } elsif ($media eq 'cdrom') {
1242         $etype = 'iso';
1243     } else {
1244         die "internal error";
1245     }
1246
1247     return if ($vtype eq $etype);
1248
1249     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1250 }
1251
1252 sub cleanup_drive_path {
1253     my ($opt, $storecfg, $drive) = @_;
1254
1255     # try to convert filesystem paths to volume IDs
1256
1257     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1258         ($drive->{file} !~ m|^/dev/.+|) &&
1259         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1260         ($drive->{file} !~ m/^\d+$/)) {
1261         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1262         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1263         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1264         verify_media_type($opt, $vtype, $drive->{media});
1265         $drive->{file} = $volid;
1266     }
1267
1268     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1269 }
1270
1271 sub parse_hotplug_features {
1272     my ($data) = @_;
1273
1274     my $res = {};
1275
1276     return $res if $data eq '0';
1277
1278     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1279
1280     foreach my $feature (PVE::Tools::split_list($data)) {
1281         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1282             $res->{$1} = 1;
1283         } else {
1284             die "invalid hotplug feature '$feature'\n";
1285         }
1286     }
1287     return $res;
1288 }
1289
1290 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1291 sub pve_verify_hotplug_features {
1292     my ($value, $noerr) = @_;
1293
1294     return $value if parse_hotplug_features($value);
1295
1296     return undef if $noerr;
1297
1298     die "unable to parse hotplug option\n";
1299 }
1300
1301 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1302 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1303 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1304 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1305 #        [,iothread=on][,serial=serial][,model=model]
1306
1307 sub parse_drive {
1308     my ($key, $data) = @_;
1309
1310     my ($interface, $index);
1311
1312     if ($key =~ m/^([^\d]+)(\d+)$/) {
1313         $interface = $1;
1314         $index = $2;
1315     } else {
1316         return undef;
1317     }
1318
1319     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1320                                      : $confdesc->{$key}->{format};
1321     if (!$desc) {
1322         warn "invalid drive key: $key\n";
1323         return undef;
1324     }
1325     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1326     return undef if !$res;
1327     $res->{interface} = $interface;
1328     $res->{index} = $index;
1329
1330     my $error = 0;
1331     foreach my $opt (qw(bps bps_rd bps_wr)) {
1332         if (my $bps = defined(delete $res->{$opt})) {
1333             if (defined($res->{"m$opt"})) {
1334                 warn "both $opt and m$opt specified\n";
1335                 ++$error;
1336                 next;
1337             }
1338             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1339         }
1340     }
1341     return undef if $error;
1342
1343     return undef if $res->{mbps_rd} && $res->{mbps};
1344     return undef if $res->{mbps_wr} && $res->{mbps};
1345     return undef if $res->{iops_rd} && $res->{iops};
1346     return undef if $res->{iops_wr} && $res->{iops};
1347
1348     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1349         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1350         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1351         return undef if $res->{interface} eq 'virtio';
1352     }
1353
1354     if (my $size = $res->{size}) {
1355         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1356     }
1357
1358     return $res;
1359 }
1360
1361 sub print_drive {
1362     my ($vmid, $drive) = @_;
1363     my $data = { %$drive };
1364     delete $data->{$_} for qw(index interface);
1365     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1366 }
1367
1368 sub scsi_inquiry {
1369     my($fh, $noerr) = @_;
1370
1371     my $SG_IO = 0x2285;
1372     my $SG_GET_VERSION_NUM = 0x2282;
1373
1374     my $versionbuf = "\x00" x 8;
1375     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1376     if (!$ret) {
1377         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1378         return undef;
1379     }
1380     my $version = unpack("I", $versionbuf);
1381     if ($version < 30000) {
1382         die "scsi generic interface too old\n"  if !$noerr;
1383         return undef;
1384     }
1385
1386     my $buf = "\x00" x 36;
1387     my $sensebuf = "\x00" x 8;
1388     my $cmd = pack("C x3 C x1", 0x12, 36);
1389
1390     # see /usr/include/scsi/sg.h
1391     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1392
1393     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1394                       length($sensebuf), 0, length($buf), $buf,
1395                       $cmd, $sensebuf, 6000);
1396
1397     $ret = ioctl($fh, $SG_IO, $packet);
1398     if (!$ret) {
1399         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1400         return undef;
1401     }
1402
1403     my @res = unpack($sg_io_hdr_t, $packet);
1404     if ($res[17] || $res[18]) {
1405         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1406         return undef;
1407     }
1408
1409     my $res = {};
1410     (my $byte0, my $byte1, $res->{vendor},
1411      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1412
1413     $res->{removable} = $byte1 & 128 ? 1 : 0;
1414     $res->{type} = $byte0 & 31;
1415
1416     return $res;
1417 }
1418
1419 sub path_is_scsi {
1420     my ($path) = @_;
1421
1422     my $fh = IO::File->new("+<$path") || return undef;
1423     my $res = scsi_inquiry($fh, 1);
1424     close($fh);
1425
1426     return $res;
1427 }
1428
1429 sub machine_type_is_q35 {
1430     my ($conf) = @_;
1431
1432     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1433 }
1434
1435 sub print_tabletdevice_full {
1436     my ($conf) = @_;
1437
1438     my $q35 = machine_type_is_q35($conf);
1439
1440     # we use uhci for old VMs because tablet driver was buggy in older qemu
1441     my $usbbus = $q35 ? "ehci" : "uhci";
1442
1443     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1444 }
1445
1446 sub print_drivedevice_full {
1447     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1448
1449     my $device = '';
1450     my $maxdev = 0;
1451
1452     if ($drive->{interface} eq 'virtio') {
1453         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1454         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1455         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1456     } elsif ($drive->{interface} eq 'scsi') {
1457
1458         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1459         my $unit = $drive->{index} % $maxdev;
1460         my $devicetype = 'hd';
1461         my $path = '';
1462         if (drive_is_cdrom($drive)) {
1463             $devicetype = 'cd';
1464         } else {
1465             if ($drive->{file} =~ m|^/|) {
1466                 $path = $drive->{file};
1467                 if (my $info = path_is_scsi($path)) {
1468                     if ($info->{type} == 0) {
1469                         $devicetype = 'block';
1470                     } elsif ($info->{type} == 1) { # tape
1471                         $devicetype = 'generic';
1472                     }
1473                 }
1474             } else {
1475                  $path = PVE::Storage::path($storecfg, $drive->{file});
1476             }
1477
1478             if($path =~ m/^iscsi\:\/\//){
1479                 $devicetype = 'generic';
1480             }
1481         }
1482
1483         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1484             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1485         } else {
1486             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1487         }
1488
1489     } elsif ($drive->{interface} eq 'ide'){
1490         $maxdev = 2;
1491         my $controller = int($drive->{index} / $maxdev);
1492         my $unit = $drive->{index} % $maxdev;
1493         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1494
1495         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1496         if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
1497             $model = URI::Escape::uri_unescape($model);
1498             $device .= ",model=$model";
1499         }
1500     } elsif ($drive->{interface} eq 'sata'){
1501         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1502         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1503         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1504     } elsif ($drive->{interface} eq 'usb') {
1505         die "implement me";
1506         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1507     } else {
1508         die "unsupported interface type";
1509     }
1510
1511     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1512
1513     return $device;
1514 }
1515
1516 sub get_initiator_name {
1517     my $initiator;
1518
1519     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1520     while (defined(my $line = <$fh>)) {
1521         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1522         $initiator = $1;
1523         last;
1524     }
1525     $fh->close();
1526
1527     return $initiator;
1528 }
1529
1530 sub print_drive_full {
1531     my ($storecfg, $vmid, $drive) = @_;
1532
1533     my $path;
1534     my $volid = $drive->{file};
1535     my $format;
1536
1537     if (drive_is_cdrom($drive)) {
1538         $path = get_iso_path($storecfg, $vmid, $volid);
1539     } else {
1540         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1541         if ($storeid) {
1542             $path = PVE::Storage::path($storecfg, $volid);
1543             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1544             $format = qemu_img_format($scfg, $volname);
1545         } else {
1546             $path = $volid;
1547             $format = "raw";
1548         }
1549    }
1550
1551     my $opts = '';
1552     my @qemu_drive_options = qw(heads secs cyls trans media format cache snapshot rerror werror aio discard iops iops_rd iops_wr iops_max iops_rd_max iops_wr_max);
1553     foreach my $o (@qemu_drive_options) {
1554         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1555     }
1556     if (my $serial = $drive->{serial}) {
1557         $serial = URI::Escape::uri_unescape($serial);
1558         $opts .= ",serial=$serial";
1559     }
1560
1561     $opts .= ",format=$format" if $format && !$drive->{format};
1562
1563     foreach my $o (qw(bps bps_rd bps_wr)) {
1564         my $v = $drive->{"m$o"};
1565         $opts .= ",$o=" . int($v*1024*1024) if $v;
1566     }
1567
1568     my $cache_direct = 0;
1569
1570     if (my $cache = $drive->{cache}) {
1571         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1572     } elsif (!drive_is_cdrom($drive)) {
1573         $opts .= ",cache=none";
1574         $cache_direct = 1;
1575     }
1576
1577     # aio native works only with O_DIRECT
1578     if (!$drive->{aio}) {
1579         if($cache_direct) {
1580             $opts .= ",aio=native";
1581         } else {
1582             $opts .= ",aio=threads";
1583         }
1584     }
1585
1586     if (!drive_is_cdrom($drive)) {
1587         my $detectzeroes;
1588         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1589             $detectzeroes = 'off';
1590         } elsif ($drive->{discard}) {
1591             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1592         } else {
1593             # This used to be our default with discard not being specified:
1594             $detectzeroes = 'on';
1595         }
1596         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1597     }
1598
1599     my $pathinfo = $path ? "file=$path," : '';
1600
1601     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1602 }
1603
1604 sub print_netdevice_full {
1605     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1606
1607     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1608
1609     my $device = $net->{model};
1610     if ($net->{model} eq 'virtio') {
1611          $device = 'virtio-net-pci';
1612      };
1613
1614     my $pciaddr = print_pci_addr("$netid", $bridges);
1615     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1616     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1617         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1618         my $vectors = $net->{queues} * 2 + 2;
1619         $tmpstr .= ",vectors=$vectors,mq=on";
1620     }
1621     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1622
1623     if ($use_old_bios_files) {
1624         my $romfile;
1625         if ($device eq 'virtio-net-pci') {
1626             $romfile = 'pxe-virtio.rom';
1627         } elsif ($device eq 'e1000') {
1628             $romfile = 'pxe-e1000.rom';
1629         } elsif ($device eq 'ne2k') {
1630             $romfile = 'pxe-ne2k_pci.rom';
1631         } elsif ($device eq 'pcnet') {
1632             $romfile = 'pxe-pcnet.rom';
1633         } elsif ($device eq 'rtl8139') {
1634             $romfile = 'pxe-rtl8139.rom';
1635         }
1636         $tmpstr .= ",romfile=$romfile" if $romfile;
1637     }
1638
1639     return $tmpstr;
1640 }
1641
1642 sub print_netdev_full {
1643     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1644
1645     my $i = '';
1646     if ($netid =~ m/^net(\d+)$/) {
1647         $i = int($1);
1648     }
1649
1650     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1651
1652     my $ifname = "tap${vmid}i$i";
1653
1654     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1655     die "interface name '$ifname' is too long (max 15 character)\n"
1656         if length($ifname) >= 16;
1657
1658     my $vhostparam = '';
1659     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1660
1661     my $vmname = $conf->{name} || "vm$vmid";
1662
1663     my $netdev = "";
1664     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1665
1666     if ($net->{bridge}) {
1667         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1668     } else {
1669         $netdev = "type=user,id=$netid,hostname=$vmname";
1670     }
1671
1672     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1673
1674     return $netdev;
1675 }
1676
1677
1678 sub print_cpu_device {
1679     my ($conf, $id) = @_;
1680
1681     my $nokvm = defined($conf->{kvm}) && $conf->{kvm} == 0 ? 1 : 0;
1682     my $cpu = $nokvm ? "qemu64" : "kvm64";
1683     if (my $cputype = $conf->{cpu}) {
1684         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
1685             or die "Cannot parse cpu description: $cputype\n";
1686         $cpu = $cpuconf->{cputype};
1687     }
1688
1689     my $sockets = 1;
1690     $sockets = $conf->{sockets} if  $conf->{sockets};
1691     my $cores = $conf->{cores} || 1;
1692
1693     my $current_core = ($id - 1) % $cores;
1694     my $current_socket = int(($id - $current_core)/$cores);
1695
1696     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
1697 }
1698
1699 sub drive_is_cdrom {
1700     my ($drive) = @_;
1701
1702     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1703
1704 }
1705
1706 sub parse_number_sets {
1707     my ($set) = @_;
1708     my $res = [];
1709     foreach my $part (split(/;/, $set)) {
1710         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1711             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1712             push @$res, [ $1, $2 ];
1713         } else {
1714             die "invalid range: $part\n";
1715         }
1716     }
1717     return $res;
1718 }
1719
1720 sub parse_numa {
1721     my ($data) = @_;
1722
1723     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
1724     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1725     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1726     return $res;
1727 }
1728
1729 sub parse_hostpci {
1730     my ($value) = @_;
1731
1732     return undef if !$value;
1733
1734     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
1735
1736     my @idlist = split(/;/, $res->{host});
1737     delete $res->{host};
1738     foreach my $id (@idlist) {
1739         if ($id =~ /^$PCIRE$/) {
1740             if (defined($2)) {
1741                 push @{$res->{pciid}}, { id => $1, function => $2 };
1742             } else {
1743                 my $pcidevices = lspci($1);
1744                 $res->{pciid} = $pcidevices->{$1};
1745             }
1746         } else {
1747             # should have been caught by parse_property_string already
1748             die "failed to parse PCI id: $id\n";
1749         }
1750     }
1751     return $res;
1752 }
1753
1754 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1755 sub parse_net {
1756     my ($data) = @_;
1757
1758     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
1759     if ($@) {
1760         warn $@;
1761         return undef;
1762     }
1763     if (!defined($res->{macaddr})) {
1764         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1765         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1766     }
1767     return $res;
1768 }
1769
1770 sub print_net {
1771     my $net = shift;
1772
1773     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1774 }
1775
1776 sub add_random_macs {
1777     my ($settings) = @_;
1778
1779     foreach my $opt (keys %$settings) {
1780         next if $opt !~ m/^net(\d+)$/;
1781         my $net = parse_net($settings->{$opt});
1782         next if !$net;
1783         $settings->{$opt} = print_net($net);
1784     }
1785 }
1786
1787 sub vm_is_volid_owner {
1788     my ($storecfg, $vmid, $volid) = @_;
1789
1790     if ($volid !~  m|^/|) {
1791         my ($path, $owner);
1792         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1793         if ($owner && ($owner == $vmid)) {
1794             return 1;
1795         }
1796     }
1797
1798     return undef;
1799 }
1800
1801 sub split_flagged_list {
1802     my $text = shift || '';
1803     $text =~ s/[,;]/ /g;
1804     $text =~ s/^\s+//;
1805     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
1806 }
1807
1808 sub join_flagged_list {
1809     my ($how, $lst) = @_;
1810     join $how, map { $lst->{$_} . $_ } keys %$lst;
1811 }
1812
1813 sub vmconfig_delete_pending_option {
1814     my ($conf, $key, $force) = @_;
1815
1816     delete $conf->{pending}->{$key};
1817     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1818     $pending_delete_hash->{$key} = $force ? '!' : '';
1819     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1820 }
1821
1822 sub vmconfig_undelete_pending_option {
1823     my ($conf, $key) = @_;
1824
1825     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1826     delete $pending_delete_hash->{$key};
1827
1828     if (%$pending_delete_hash) {
1829         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1830     } else {
1831         delete $conf->{pending}->{delete};
1832     }
1833 }
1834
1835 sub vmconfig_register_unused_drive {
1836     my ($storecfg, $vmid, $conf, $drive) = @_;
1837
1838     if (!drive_is_cdrom($drive)) {
1839         my $volid = $drive->{file};
1840         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1841             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1842         }
1843     }
1844 }
1845
1846 sub vmconfig_cleanup_pending {
1847     my ($conf) = @_;
1848
1849     # remove pending changes when nothing changed
1850     my $changes;
1851     foreach my $opt (keys %{$conf->{pending}}) {
1852         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
1853             $changes = 1;
1854             delete $conf->{pending}->{$opt};
1855         }
1856     }
1857
1858     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
1859     my $pending_delete_hash = {};
1860     while (my ($opt, $force) = each %$current_delete_hash) {
1861         if (defined($conf->{$opt})) {
1862             $pending_delete_hash->{$opt} = $force;
1863         } else {
1864             $changes = 1;
1865         }
1866     }
1867
1868     if (%$pending_delete_hash) {
1869         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1870     } else {
1871         delete $conf->{pending}->{delete};
1872     }
1873
1874     return $changes;
1875 }
1876
1877 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
1878 my $smbios1_fmt = {
1879     uuid => {
1880         type => 'string',
1881         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1882         format_description => 'UUID',
1883         description => "Set SMBIOS1 UUID.",
1884         optional => 1,
1885     },
1886     version => {
1887         type => 'string',
1888         pattern => '\S+',
1889         format_description => 'string',
1890         description => "Set SMBIOS1 version.",
1891         optional => 1,
1892     },
1893     serial => {
1894         type => 'string',
1895         pattern => '\S+',
1896         format_description => 'string',
1897         description => "Set SMBIOS1 serial number.",
1898         optional => 1,
1899     },
1900     manufacturer => {
1901         type => 'string',
1902         pattern => '\S+',
1903         format_description => 'string',
1904         description => "Set SMBIOS1 manufacturer.",
1905         optional => 1,
1906     },
1907     product => {
1908         type => 'string',
1909         pattern => '\S+',
1910         format_description => 'string',
1911         description => "Set SMBIOS1 product ID.",
1912         optional => 1,
1913     },
1914     sku => {
1915         type => 'string',
1916         pattern => '\S+',
1917         format_description => 'string',
1918         description => "Set SMBIOS1 SKU string.",
1919         optional => 1,
1920     },
1921     family => {
1922         type => 'string',
1923         pattern => '\S+',
1924         format_description => 'string',
1925         description => "Set SMBIOS1 family string.",
1926         optional => 1,
1927     },
1928 };
1929
1930 sub parse_smbios1 {
1931     my ($data) = @_;
1932
1933     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
1934     warn $@ if $@;
1935     return $res;
1936 }
1937
1938 sub print_smbios1 {
1939     my ($smbios1) = @_;
1940     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
1941 }
1942
1943 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
1944
1945 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
1946 sub verify_bootdisk {
1947     my ($value, $noerr) = @_;
1948
1949     return $value if is_valid_drivename($value);
1950
1951     return undef if $noerr;
1952
1953     die "invalid boot disk '$value'\n";
1954 }
1955
1956 sub parse_watchdog {
1957     my ($value) = @_;
1958
1959     return undef if !$value;
1960
1961     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
1962     warn $@ if $@;
1963     return $res;
1964 }
1965
1966 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
1967 sub verify_usb_device {
1968     my ($value, $noerr) = @_;
1969
1970     return $value if parse_usb_device($value);
1971
1972     return undef if $noerr;
1973
1974     die "unable to parse usb device\n";
1975 }
1976
1977 # add JSON properties for create and set function
1978 sub json_config_properties {
1979     my $prop = shift;
1980
1981     foreach my $opt (keys %$confdesc) {
1982         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
1983         $prop->{$opt} = $confdesc->{$opt};
1984     }
1985
1986     return $prop;
1987 }
1988
1989 sub check_type {
1990     my ($key, $value) = @_;
1991
1992     die "unknown setting '$key'\n" if !$confdesc->{$key};
1993
1994     my $type = $confdesc->{$key}->{type};
1995
1996     if (!defined($value)) {
1997         die "got undefined value\n";
1998     }
1999
2000     if ($value =~ m/[\n\r]/) {
2001         die "property contains a line feed\n";
2002     }
2003
2004     if ($type eq 'boolean') {
2005         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2006         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2007         die "type check ('boolean') failed - got '$value'\n";
2008     } elsif ($type eq 'integer') {
2009         return int($1) if $value =~ m/^(\d+)$/;
2010         die "type check ('integer') failed - got '$value'\n";
2011     } elsif ($type eq 'number') {
2012         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2013         die "type check ('number') failed - got '$value'\n";
2014     } elsif ($type eq 'string') {
2015         if (my $fmt = $confdesc->{$key}->{format}) {
2016             if ($fmt eq 'pve-qm-drive') {
2017                 # special case - we need to pass $key to parse_drive()
2018                 my $drive = parse_drive($key, $value);
2019                 return $value if $drive;
2020                 die "unable to parse drive options\n";
2021             }
2022             PVE::JSONSchema::check_format($fmt, $value);
2023             return $value;
2024         }
2025         $value =~ s/^\"(.*)\"$/$1/;
2026         return $value;
2027     } else {
2028         die "internal error"
2029     }
2030 }
2031
2032 sub check_iommu_support{
2033     #fixme : need to check IOMMU support
2034     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
2035
2036     my $iommu=1;
2037     return $iommu;
2038
2039 }
2040
2041 sub touch_config {
2042     my ($vmid) = @_;
2043
2044     my $conf = PVE::QemuConfig->config_file($vmid);
2045     utime undef, undef, $conf;
2046 }
2047
2048 sub destroy_vm {
2049     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2050
2051     my $conffile = PVE::QemuConfig->config_file($vmid);
2052
2053     my $conf = PVE::QemuConfig->load_config($vmid);
2054
2055     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2056
2057     # only remove disks owned by this VM
2058     foreach_drive($conf, sub {
2059         my ($ds, $drive) = @_;
2060
2061         return if drive_is_cdrom($drive);
2062
2063         my $volid = $drive->{file};
2064
2065         return if !$volid || $volid =~ m|^/|;
2066
2067         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2068         return if !$path || !$owner || ($owner != $vmid);
2069
2070         PVE::Storage::vdisk_free($storecfg, $volid);
2071     });
2072
2073     if ($keep_empty_config) {
2074         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2075     } else {
2076         unlink $conffile;
2077     }
2078
2079     # also remove unused disk
2080     eval {
2081         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2082
2083         eval {
2084             PVE::Storage::foreach_volid($dl, sub {
2085                 my ($volid, $sid, $volname, $d) = @_;
2086                 PVE::Storage::vdisk_free($storecfg, $volid);
2087             });
2088         };
2089         warn $@ if $@;
2090
2091     };
2092     warn $@ if $@;
2093 }
2094
2095 sub parse_vm_config {
2096     my ($filename, $raw) = @_;
2097
2098     return undef if !defined($raw);
2099
2100     my $res = {
2101         digest => Digest::SHA::sha1_hex($raw),
2102         snapshots => {},
2103         pending => {},
2104     };
2105
2106     $filename =~ m|/qemu-server/(\d+)\.conf$|
2107         || die "got strange filename '$filename'";
2108
2109     my $vmid = $1;
2110
2111     my $conf = $res;
2112     my $descr;
2113     my $section = '';
2114
2115     my @lines = split(/\n/, $raw);
2116     foreach my $line (@lines) {
2117         next if $line =~ m/^\s*$/;
2118
2119         if ($line =~ m/^\[PENDING\]\s*$/i) {
2120             $section = 'pending';
2121             if (defined($descr)) {
2122                 $descr =~ s/\s+$//;
2123                 $conf->{description} = $descr;
2124             }
2125             $descr = undef;
2126             $conf = $res->{$section} = {};
2127             next;
2128
2129         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2130             $section = $1;
2131             if (defined($descr)) {
2132                 $descr =~ s/\s+$//;
2133                 $conf->{description} = $descr;
2134             }
2135             $descr = undef;
2136             $conf = $res->{snapshots}->{$section} = {};
2137             next;
2138         }
2139
2140         if ($line =~ m/^\#(.*)\s*$/) {
2141             $descr = '' if !defined($descr);
2142             $descr .= PVE::Tools::decode_text($1) . "\n";
2143             next;
2144         }
2145
2146         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2147             $descr = '' if !defined($descr);
2148             $descr .= PVE::Tools::decode_text($2);
2149         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2150             $conf->{snapstate} = $1;
2151         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2152             my $key = $1;
2153             my $value = $2;
2154             $conf->{$key} = $value;
2155         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2156             my $value = $1;
2157             if ($section eq 'pending') {
2158                 $conf->{delete} = $value; # we parse this later
2159             } else {
2160                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2161             }
2162         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
2163             my $key = $1;
2164             my $value = $2;
2165             eval { $value = check_type($key, $value); };
2166             if ($@) {
2167                 warn "vm $vmid - unable to parse value of '$key' - $@";
2168             } else {
2169                 my $fmt = $confdesc->{$key}->{format};
2170                 if ($fmt && $fmt eq 'pve-qm-drive') {
2171                     my $v = parse_drive($key, $value);
2172                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2173                         $v->{file} = $volid;
2174                         $value = print_drive($vmid, $v);
2175                     } else {
2176                         warn "vm $vmid - unable to parse value of '$key'\n";
2177                         next;
2178                     }
2179                 }
2180
2181                 if ($key eq 'cdrom') {
2182                     $conf->{ide2} = $value;
2183                 } else {
2184                     $conf->{$key} = $value;
2185                 }
2186             }
2187         }
2188     }
2189
2190     if (defined($descr)) {
2191         $descr =~ s/\s+$//;
2192         $conf->{description} = $descr;
2193     }
2194     delete $res->{snapstate}; # just to be sure
2195
2196     return $res;
2197 }
2198
2199 sub write_vm_config {
2200     my ($filename, $conf) = @_;
2201
2202     delete $conf->{snapstate}; # just to be sure
2203
2204     if ($conf->{cdrom}) {
2205         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2206         $conf->{ide2} = $conf->{cdrom};
2207         delete $conf->{cdrom};
2208     }
2209
2210     # we do not use 'smp' any longer
2211     if ($conf->{sockets}) {
2212         delete $conf->{smp};
2213     } elsif ($conf->{smp}) {
2214         $conf->{sockets} = $conf->{smp};
2215         delete $conf->{cores};
2216         delete $conf->{smp};
2217     }
2218
2219     my $used_volids = {};
2220
2221     my $cleanup_config = sub {
2222         my ($cref, $pending, $snapname) = @_;
2223
2224         foreach my $key (keys %$cref) {
2225             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2226                 $key eq 'snapstate' || $key eq 'pending';
2227             my $value = $cref->{$key};
2228             if ($key eq 'delete') {
2229                 die "propertry 'delete' is only allowed in [PENDING]\n"
2230                     if !$pending;
2231                 # fixme: check syntax?
2232                 next;
2233             }
2234             eval { $value = check_type($key, $value); };
2235             die "unable to parse value of '$key' - $@" if $@;
2236
2237             $cref->{$key} = $value;
2238
2239             if (!$snapname && is_valid_drivename($key)) {
2240                 my $drive = parse_drive($key, $value);
2241                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2242             }
2243         }
2244     };
2245
2246     &$cleanup_config($conf);
2247
2248     &$cleanup_config($conf->{pending}, 1);
2249
2250     foreach my $snapname (keys %{$conf->{snapshots}}) {
2251         die "internal error" if $snapname eq 'pending';
2252         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2253     }
2254
2255     # remove 'unusedX' settings if we re-add a volume
2256     foreach my $key (keys %$conf) {
2257         my $value = $conf->{$key};
2258         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2259             delete $conf->{$key};
2260         }
2261     }
2262
2263     my $generate_raw_config = sub {
2264         my ($conf, $pending) = @_;
2265
2266         my $raw = '';
2267
2268         # add description as comment to top of file
2269         if (defined(my $descr = $conf->{description})) {
2270             if ($descr) {
2271                 foreach my $cl (split(/\n/, $descr)) {
2272                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2273                 }
2274             } else {
2275                 $raw .= "#\n" if $pending;
2276             }
2277         }
2278
2279         foreach my $key (sort keys %$conf) {
2280             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2281             $raw .= "$key: $conf->{$key}\n";
2282         }
2283         return $raw;
2284     };
2285
2286     my $raw = &$generate_raw_config($conf);
2287
2288     if (scalar(keys %{$conf->{pending}})){
2289         $raw .= "\n[PENDING]\n";
2290         $raw .= &$generate_raw_config($conf->{pending}, 1);
2291     }
2292
2293     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2294         $raw .= "\n[$snapname]\n";
2295         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2296     }
2297
2298     return $raw;
2299 }
2300
2301 sub load_defaults {
2302
2303     my $res = {};
2304
2305     # we use static defaults from our JSON schema configuration
2306     foreach my $key (keys %$confdesc) {
2307         if (defined(my $default = $confdesc->{$key}->{default})) {
2308             $res->{$key} = $default;
2309         }
2310     }
2311
2312     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2313     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2314
2315     return $res;
2316 }
2317
2318 sub config_list {
2319     my $vmlist = PVE::Cluster::get_vmlist();
2320     my $res = {};
2321     return $res if !$vmlist || !$vmlist->{ids};
2322     my $ids = $vmlist->{ids};
2323
2324     foreach my $vmid (keys %$ids) {
2325         my $d = $ids->{$vmid};
2326         next if !$d->{node} || $d->{node} ne $nodename;
2327         next if !$d->{type} || $d->{type} ne 'qemu';
2328         $res->{$vmid}->{exists} = 1;
2329     }
2330     return $res;
2331 }
2332
2333 # test if VM uses local resources (to prevent migration)
2334 sub check_local_resources {
2335     my ($conf, $noerr) = @_;
2336
2337     my $loc_res = 0;
2338
2339     $loc_res = 1 if $conf->{hostusb}; # old syntax
2340     $loc_res = 1 if $conf->{hostpci}; # old syntax
2341
2342     foreach my $k (keys %$conf) {
2343         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2344         # sockets are safe: they will recreated be on the target side post-migrate
2345         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2346         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2347     }
2348
2349     die "VM uses local resources\n" if $loc_res && !$noerr;
2350
2351     return $loc_res;
2352 }
2353
2354 # check if used storages are available on all nodes (use by migrate)
2355 sub check_storage_availability {
2356     my ($storecfg, $conf, $node) = @_;
2357
2358     foreach_drive($conf, sub {
2359         my ($ds, $drive) = @_;
2360
2361         my $volid = $drive->{file};
2362         return if !$volid;
2363
2364         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2365         return if !$sid;
2366
2367         # check if storage is available on both nodes
2368         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2369         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2370    });
2371 }
2372
2373 # list nodes where all VM images are available (used by has_feature API)
2374 sub shared_nodes {
2375     my ($conf, $storecfg) = @_;
2376
2377     my $nodelist = PVE::Cluster::get_nodelist();
2378     my $nodehash = { map { $_ => 1 } @$nodelist };
2379     my $nodename = PVE::INotify::nodename();
2380
2381     foreach_drive($conf, sub {
2382         my ($ds, $drive) = @_;
2383
2384         my $volid = $drive->{file};
2385         return if !$volid;
2386
2387         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2388         if ($storeid) {
2389             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2390             if ($scfg->{disable}) {
2391                 $nodehash = {};
2392             } elsif (my $avail = $scfg->{nodes}) {
2393                 foreach my $node (keys %$nodehash) {
2394                     delete $nodehash->{$node} if !$avail->{$node};
2395                 }
2396             } elsif (!$scfg->{shared}) {
2397                 foreach my $node (keys %$nodehash) {
2398                     delete $nodehash->{$node} if $node ne $nodename
2399                 }
2400             }
2401         }
2402     });
2403
2404     return $nodehash
2405 }
2406
2407 sub check_cmdline {
2408     my ($pidfile, $pid) = @_;
2409
2410     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2411     if (defined($fh)) {
2412         my $line = <$fh>;
2413         $fh->close;
2414         return undef if !$line;
2415         my @param = split(/\0/, $line);
2416
2417         my $cmd = $param[0];
2418         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2419
2420         for (my $i = 0; $i < scalar (@param); $i++) {
2421             my $p = $param[$i];
2422             next if !$p;
2423             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2424                 my $p = $param[$i+1];
2425                 return 1 if $p && ($p eq $pidfile);
2426                 return undef;
2427             }
2428         }
2429     }
2430     return undef;
2431 }
2432
2433 sub check_running {
2434     my ($vmid, $nocheck, $node) = @_;
2435
2436     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2437
2438     die "unable to find configuration file for VM $vmid - no such machine\n"
2439         if !$nocheck && ! -f $filename;
2440
2441     my $pidfile = pidfile_name($vmid);
2442
2443     if (my $fd = IO::File->new("<$pidfile")) {
2444         my $st = stat($fd);
2445         my $line = <$fd>;
2446         close($fd);
2447
2448         my $mtime = $st->mtime;
2449         if ($mtime > time()) {
2450             warn "file '$filename' modified in future\n";
2451         }
2452
2453         if ($line =~ m/^(\d+)$/) {
2454             my $pid = $1;
2455             if (check_cmdline($pidfile, $pid)) {
2456                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2457                     return $pid;
2458                 }
2459             }
2460         }
2461     }
2462
2463     return undef;
2464 }
2465
2466 sub vzlist {
2467
2468     my $vzlist = config_list();
2469
2470     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2471
2472     while (defined(my $de = $fd->read)) {
2473         next if $de !~ m/^(\d+)\.pid$/;
2474         my $vmid = $1;
2475         next if !defined($vzlist->{$vmid});
2476         if (my $pid = check_running($vmid)) {
2477             $vzlist->{$vmid}->{pid} = $pid;
2478         }
2479     }
2480
2481     return $vzlist;
2482 }
2483
2484 sub disksize {
2485     my ($storecfg, $conf) = @_;
2486
2487     my $bootdisk = $conf->{bootdisk};
2488     return undef if !$bootdisk;
2489     return undef if !is_valid_drivename($bootdisk);
2490
2491     return undef if !$conf->{$bootdisk};
2492
2493     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2494     return undef if !defined($drive);
2495
2496     return undef if drive_is_cdrom($drive);
2497
2498     my $volid = $drive->{file};
2499     return undef if !$volid;
2500
2501     return $drive->{size};
2502 }
2503
2504 my $last_proc_pid_stat;
2505
2506 # get VM status information
2507 # This must be fast and should not block ($full == false)
2508 # We only query KVM using QMP if $full == true (this can be slow)
2509 sub vmstatus {
2510     my ($opt_vmid, $full) = @_;
2511
2512     my $res = {};
2513
2514     my $storecfg = PVE::Storage::config();
2515
2516     my $list = vzlist();
2517     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2518
2519     my $cpucount = $cpuinfo->{cpus} || 1;
2520
2521     foreach my $vmid (keys %$list) {
2522         next if $opt_vmid && ($vmid ne $opt_vmid);
2523
2524         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
2525         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2526
2527         my $d = {};
2528         $d->{pid} = $list->{$vmid}->{pid};
2529
2530         # fixme: better status?
2531         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2532
2533         my $size = disksize($storecfg, $conf);
2534         if (defined($size)) {
2535             $d->{disk} = 0; # no info available
2536             $d->{maxdisk} = $size;
2537         } else {
2538             $d->{disk} = 0;
2539             $d->{maxdisk} = 0;
2540         }
2541
2542         $d->{cpus} = ($conf->{sockets} || 1) * ($conf->{cores} || 1);
2543         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2544         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2545
2546         $d->{name} = $conf->{name} || "VM $vmid";
2547         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024) : 0;
2548
2549         if ($conf->{balloon}) {
2550             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2551             $d->{shares} = defined($conf->{shares}) ? $conf->{shares} : 1000;
2552         }
2553
2554         $d->{uptime} = 0;
2555         $d->{cpu} = 0;
2556         $d->{mem} = 0;
2557
2558         $d->{netout} = 0;
2559         $d->{netin} = 0;
2560
2561         $d->{diskread} = 0;
2562         $d->{diskwrite} = 0;
2563
2564         $d->{template} = PVE::QemuConfig->is_template($conf);
2565
2566         $res->{$vmid} = $d;
2567     }
2568
2569     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2570     foreach my $dev (keys %$netdev) {
2571         next if $dev !~ m/^tap([1-9]\d*)i/;
2572         my $vmid = $1;
2573         my $d = $res->{$vmid};
2574         next if !$d;
2575
2576         $d->{netout} += $netdev->{$dev}->{receive};
2577         $d->{netin} += $netdev->{$dev}->{transmit};
2578
2579         if ($full) {
2580             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2581             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2582         }
2583
2584     }
2585
2586     my $ctime = gettimeofday;
2587
2588     foreach my $vmid (keys %$list) {
2589
2590         my $d = $res->{$vmid};
2591         my $pid = $d->{pid};
2592         next if !$pid;
2593
2594         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2595         next if !$pstat; # not running
2596
2597         my $used = $pstat->{utime} + $pstat->{stime};
2598
2599         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2600
2601         if ($pstat->{vsize}) {
2602             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2603         }
2604
2605         my $old = $last_proc_pid_stat->{$pid};
2606         if (!$old) {
2607             $last_proc_pid_stat->{$pid} = {
2608                 time => $ctime,
2609                 used => $used,
2610                 cpu => 0,
2611             };
2612             next;
2613         }
2614
2615         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2616
2617         if ($dtime > 1000) {
2618             my $dutime = $used -  $old->{used};
2619
2620             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2621             $last_proc_pid_stat->{$pid} = {
2622                 time => $ctime,
2623                 used => $used,
2624                 cpu => $d->{cpu},
2625             };
2626         } else {
2627             $d->{cpu} = $old->{cpu};
2628         }
2629     }
2630
2631     return $res if !$full;
2632
2633     my $qmpclient = PVE::QMPClient->new();
2634
2635     my $ballooncb = sub {
2636         my ($vmid, $resp) = @_;
2637
2638         my $info = $resp->{'return'};
2639         return if !$info->{max_mem};
2640
2641         my $d = $res->{$vmid};
2642
2643         # use memory assigned to VM
2644         $d->{maxmem} = $info->{max_mem};
2645         $d->{balloon} = $info->{actual};
2646
2647         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2648             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2649             $d->{freemem} = $info->{free_mem};
2650         }
2651
2652         $d->{ballooninfo} = $info;
2653     };
2654
2655     my $blockstatscb = sub {
2656         my ($vmid, $resp) = @_;
2657         my $data = $resp->{'return'} || [];
2658         my $totalrdbytes = 0;
2659         my $totalwrbytes = 0;
2660
2661         for my $blockstat (@$data) {
2662             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2663             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2664
2665             $blockstat->{device} =~ s/drive-//;
2666             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2667         }
2668         $res->{$vmid}->{diskread} = $totalrdbytes;
2669         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2670     };
2671
2672     my $statuscb = sub {
2673         my ($vmid, $resp) = @_;
2674
2675         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2676         # this fails if ballon driver is not loaded, so this must be
2677         # the last commnand (following command are aborted if this fails).
2678         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2679
2680         my $status = 'unknown';
2681         if (!defined($status = $resp->{'return'}->{status})) {
2682             warn "unable to get VM status\n";
2683             return;
2684         }
2685
2686         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2687     };
2688
2689     foreach my $vmid (keys %$list) {
2690         next if $opt_vmid && ($vmid ne $opt_vmid);
2691         next if !$res->{$vmid}->{pid}; # not running
2692         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2693     }
2694
2695     $qmpclient->queue_execute(undef, 1);
2696
2697     foreach my $vmid (keys %$list) {
2698         next if $opt_vmid && ($vmid ne $opt_vmid);
2699         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2700     }
2701
2702     return $res;
2703 }
2704
2705 sub foreach_drive {
2706     my ($conf, $func, @param) = @_;
2707
2708     foreach my $ds (valid_drive_names()) {
2709         next if !defined($conf->{$ds});
2710
2711         my $drive = parse_drive($ds, $conf->{$ds});
2712         next if !$drive;
2713
2714         &$func($ds, $drive, @param);
2715     }
2716 }
2717
2718 sub foreach_volid {
2719     my ($conf, $func, @param) = @_;
2720
2721     my $volhash = {};
2722
2723     my $test_volid = sub {
2724         my ($volid, $is_cdrom) = @_;
2725
2726         return if !$volid;
2727
2728         $volhash->{$volid} = $is_cdrom || 0;
2729     };
2730
2731     foreach_drive($conf, sub {
2732         my ($ds, $drive) = @_;
2733         &$test_volid($drive->{file}, drive_is_cdrom($drive));
2734     });
2735
2736     foreach my $snapname (keys %{$conf->{snapshots}}) {
2737         my $snap = $conf->{snapshots}->{$snapname};
2738         &$test_volid($snap->{vmstate}, 0);
2739         foreach_drive($snap, sub {
2740             my ($ds, $drive) = @_;
2741             &$test_volid($drive->{file}, drive_is_cdrom($drive));
2742         });
2743     }
2744
2745     foreach my $volid (keys %$volhash) {
2746         &$func($volid, $volhash->{$volid}, @param);
2747     }
2748 }
2749
2750 sub vga_conf_has_spice {
2751     my ($vga) = @_;
2752
2753     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
2754
2755     return $1 || 1;
2756 }
2757
2758 sub config_to_command {
2759     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2760
2761     my $cmd = [];
2762     my $globalFlags = [];
2763     my $machineFlags = [];
2764     my $rtcFlags = [];
2765     my $cpuFlags = [];
2766     my $devices = [];
2767     my $pciaddr = '';
2768     my $bridges = {};
2769     my $kvmver = kvm_user_version();
2770     my $vernum = 0; # unknown
2771     my $ostype = $conf->{ostype};
2772     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
2773         $vernum = $1*1000000+$2*1000;
2774     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
2775         $vernum = $1*1000000+$2*1000+$3;
2776     }
2777
2778     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
2779
2780     my $have_ovz = -f '/proc/vz/vestat';
2781
2782     my $q35 = machine_type_is_q35($conf);
2783     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2784     my $machine_type = $forcemachine || $conf->{machine};
2785     my $use_old_bios_files = undef;
2786     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
2787
2788     my $cpuunits = defined($conf->{cpuunits}) ?
2789             $conf->{cpuunits} : $defaults->{cpuunits};
2790
2791     push @$cmd, '/usr/bin/kvm';
2792
2793     push @$cmd, '-id', $vmid;
2794
2795     my $use_virtio = 0;
2796
2797     my $qmpsocket = qmp_socket($vmid);
2798     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2799     push @$cmd, '-mon', "chardev=qmp,mode=control";
2800
2801
2802     push @$cmd, '-pidfile' , pidfile_name($vmid);
2803
2804     push @$cmd, '-daemonize';
2805
2806     if ($conf->{smbios1}) {
2807         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
2808     }
2809
2810     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2811         my $ovmfbase;
2812
2813         # prefer the OVMF_CODE variant
2814         if (-f $OVMF_CODE) {
2815             $ovmfbase = $OVMF_CODE;
2816         } elsif (-f $OVMF_IMG) {
2817             $ovmfbase = $OVMF_IMG;
2818         }
2819
2820         die "no uefi base img found\n" if !$ovmfbase;
2821         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmfbase";
2822
2823         if (defined($conf->{efidisk0}) && ($ovmfbase eq $OVMF_CODE)) {
2824             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $conf->{efidisk0});
2825             my $format = $d->{format} // 'raw';
2826             my $path;
2827             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
2828             if ($storeid) {
2829                 $path = PVE::Storage::path($storecfg, $d->{file});
2830                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2831                 $format = qemu_img_format($scfg, $volname);
2832             } else {
2833                 $path = $d->{file};
2834                 $format = "raw";
2835             }
2836             push @$cmd, '-drive', "if=pflash,unit=1,id=drive-efidisk0,format=$format,file=$path";
2837         } elsif ($ovmfbase eq $OVMF_CODE) {
2838             warn "using uefi without permanent efivars disk\n";
2839             my $ovmfvar_dst = "/tmp/$vmid-ovmf.fd";
2840             PVE::Tools::file_copy($OVMF_VARS, $ovmfvar_dst, 256*1024);
2841             push @$cmd, '-drive', "if=pflash,unit=1,format=raw,file=$ovmfvar_dst";
2842         } else {
2843             # if the base img is not OVMF_CODE, we do not have to bother
2844             # to create/use a vars image, since it will not be used anyway
2845             # this can only happen if someone manually deletes the OVMF_CODE image
2846             # or has an old pve-qemu-kvm version installed.
2847             # both should not happen, but we ignore it here
2848         }
2849     }
2850
2851
2852     # add usb controllers
2853     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $q35, $usbdesc->{format}, $MAX_USB_DEVICES);
2854     push @$devices, @usbcontrollers if @usbcontrollers;
2855     my $vga = $conf->{vga};
2856
2857     my $qxlnum = vga_conf_has_spice($vga);
2858     $vga = 'qxl' if $qxlnum;
2859
2860     if (!$vga) {
2861         if ($conf->{ostype} && ($conf->{ostype} eq 'win8' ||
2862                                 $conf->{ostype} eq 'win7' ||
2863                                 $conf->{ostype} eq 'w2k8')) {
2864             $vga = 'std';
2865         } else {
2866             $vga = 'cirrus';
2867         }
2868     }
2869
2870     # enable absolute mouse coordinates (needed by vnc)
2871     my $tablet;
2872     if (defined($conf->{tablet})) {
2873         $tablet = $conf->{tablet};
2874     } else {
2875         $tablet = $defaults->{tablet};
2876         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
2877         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
2878     }
2879
2880     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
2881
2882     my $kvm_off = 0;
2883     # host pci devices
2884     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
2885         my $d = parse_hostpci($conf->{"hostpci$i"});
2886         next if !$d;
2887
2888         my $pcie = $d->{pcie};
2889         if($pcie){
2890             die "q35 machine model is not enabled" if !$q35;
2891             $pciaddr = print_pcie_addr("hostpci$i");
2892         }else{
2893             $pciaddr = print_pci_addr("hostpci$i", $bridges);
2894         }
2895
2896         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
2897         my $xvga = '';
2898         if ($d->{'x-vga'}) {
2899             $xvga = ',x-vga=on';
2900             $kvm_off = 1;
2901             $vga = 'none';
2902             if ($ostype eq 'win7' || $ostype eq 'win8' || $ostype eq 'w2k8') {
2903                 push @$cpuFlags , 'hv_vendor_id=proxmox';
2904             }
2905             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2906                 $xvga = "";
2907             }
2908         }
2909         my $pcidevices = $d->{pciid};
2910         my $multifunction = 1 if @$pcidevices > 1;
2911
2912         my $j=0;
2913         foreach my $pcidevice (@$pcidevices) {
2914
2915             my $id = "hostpci$i";
2916             $id .= ".$j" if $multifunction;
2917             my $addr = $pciaddr;
2918             $addr .= ".$j" if $multifunction;
2919             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
2920
2921             if($j == 0){
2922                 $devicestr .= "$rombar$xvga";
2923                 $devicestr .= ",multifunction=on" if $multifunction;
2924             }
2925
2926             push @$devices, '-device', $devicestr;
2927             $j++;
2928         }
2929     }
2930
2931     # usb devices
2932     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES);
2933     push @$devices, @usbdevices if @usbdevices;
2934     # serial devices
2935     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2936         if (my $path = $conf->{"serial$i"}) {
2937             if ($path eq 'socket') {
2938                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
2939                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
2940                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2941             } else {
2942                 die "no such serial device\n" if ! -c $path;
2943                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
2944                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2945             }
2946         }
2947     }
2948
2949     # parallel devices
2950     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
2951         if (my $path = $conf->{"parallel$i"}) {
2952             die "no such parallel device\n" if ! -c $path;
2953             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
2954             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
2955             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
2956         }
2957     }
2958
2959     my $vmname = $conf->{name} || "vm$vmid";
2960
2961     push @$cmd, '-name', $vmname;
2962
2963     my $sockets = 1;
2964     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
2965     $sockets = $conf->{sockets} if  $conf->{sockets};
2966
2967     my $cores = $conf->{cores} || 1;
2968
2969     my $maxcpus = $sockets * $cores;
2970
2971     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
2972
2973     my $allowed_vcpus = $cpuinfo->{cpus};
2974
2975     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
2976         if ($allowed_vcpus < $maxcpus);
2977
2978     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
2979
2980         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
2981         for (my $i = 2; $i <= $vcpus; $i++)  {
2982             my $cpustr = print_cpu_device($conf,$i);
2983             push @$cmd, '-device', $cpustr;
2984         }
2985
2986     } else {
2987
2988         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
2989     }
2990     push @$cmd, '-nodefaults';
2991
2992     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2993
2994     my $bootindex_hash = {};
2995     my $i = 1;
2996     foreach my $o (split(//, $bootorder)) {
2997         $bootindex_hash->{$o} = $i*100;
2998         $i++;
2999     }
3000
3001     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3002
3003     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3004
3005     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3006
3007     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
3008
3009     if ($vga && $vga !~ m/^serial\d+$/ && $vga ne 'none'){
3010         my $socket = vnc_socket($vmid);
3011         push @$cmd,  '-vnc', "unix:$socket,x509,password";
3012     } else {
3013         push @$cmd, '-nographic';
3014     }
3015
3016     # time drift fix
3017     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3018
3019     my $nokvm = defined($conf->{kvm}) && $conf->{kvm} == 0 ? 1 : 0;
3020     my $useLocaltime = $conf->{localtime};
3021
3022     if ($ostype) {
3023         # other, wxp, w2k, w2k3, w2k8, wvista, win7, win8, l24, l26, solaris
3024
3025         if ($ostype =~ m/^w/) { # windows
3026             $useLocaltime = 1 if !defined($conf->{localtime});
3027
3028             # use time drift fix when acpi is enabled
3029             if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3030                 $tdf = 1 if !defined($conf->{tdf});
3031             }
3032         }
3033
3034         if ($ostype eq 'win7' || $ostype eq 'win8' || $ostype eq 'w2k8' ||
3035             $ostype eq 'wvista') {
3036             push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3037             push @$cmd, '-no-hpet';
3038             if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3039                 push @$cpuFlags , 'hv_spinlocks=0x1fff' if !$nokvm;
3040                 push @$cpuFlags , 'hv_vapic' if !$nokvm;
3041                 push @$cpuFlags , 'hv_time' if !$nokvm;
3042
3043                 if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
3044                     push @$cpuFlags , 'hv_reset' if !$nokvm;
3045                     push @$cpuFlags , 'hv_vpindex' if !$nokvm;
3046                     push @$cpuFlags , 'hv_runtime' if !$nokvm;
3047                 }
3048
3049             } else {
3050                 push @$cpuFlags , 'hv_spinlocks=0xffff' if !$nokvm;
3051             }
3052         }
3053
3054         if ($ostype eq 'win7' || $ostype eq 'win8') {
3055             push @$cpuFlags , 'hv_relaxed' if !$nokvm;
3056         }
3057     }
3058
3059     push @$rtcFlags, 'driftfix=slew' if $tdf;
3060
3061     if ($nokvm) {
3062         push @$machineFlags, 'accel=tcg';
3063     } else {
3064         die "No accelerator found!\n" if !$cpuinfo->{hvm};
3065     }
3066
3067     if ($machine_type) {
3068         push @$machineFlags, "type=${machine_type}";
3069     }
3070
3071     if ($conf->{startdate}) {
3072         push @$rtcFlags, "base=$conf->{startdate}";
3073     } elsif ($useLocaltime) {
3074         push @$rtcFlags, 'base=localtime';
3075     }
3076
3077     my $cpu = $nokvm ? "qemu64" : "kvm64";
3078     if (my $cputype = $conf->{cpu}) {
3079         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3080             or die "Cannot parse cpu description: $cputype\n";
3081         $cpu = $cpuconf->{cputype};
3082         $kvm_off = 1 if $cpuconf->{hidden};
3083     }
3084
3085     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
3086
3087     push @$cpuFlags , '-x2apic'
3088         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3089
3090     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3091
3092     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3093
3094     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3095
3096         push @$cpuFlags , '+kvm_pv_unhalt' if !$nokvm;
3097         push @$cpuFlags , '+kvm_pv_eoi' if !$nokvm;
3098     }
3099
3100     push @$cpuFlags, 'enforce' if $cpu ne 'host' && !$nokvm;
3101
3102     push @$cpuFlags, 'kvm=off' if $kvm_off;
3103
3104     my $cpu_vendor = $cpu_vendor_list->{$cpu} ||
3105         die "internal error"; # should not happen
3106
3107     push @$cpuFlags, "vendor=${cpu_vendor}"
3108         if $cpu_vendor ne 'default';
3109
3110     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3111
3112     push @$cmd, '-cpu', $cpu;
3113
3114     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3115
3116     push @$cmd, '-S' if $conf->{freeze};
3117
3118     # set keyboard layout
3119     my $kb = $conf->{keyboard} || $defaults->{keyboard};
3120     push @$cmd, '-k', $kb if $kb;
3121
3122     # enable sound
3123     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3124     #push @$cmd, '-soundhw', 'es1370';
3125     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3126
3127     if($conf->{agent}) {
3128         my $qgasocket = qmp_socket($vmid, 1);
3129         my $pciaddr = print_pci_addr("qga0", $bridges);
3130         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3131         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3132         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3133     }
3134
3135     my $spice_port;
3136
3137     if ($qxlnum) {
3138         if ($qxlnum > 1) {
3139             if ($conf->{ostype} && $conf->{ostype} =~ m/^w/){
3140                 for(my $i = 1; $i < $qxlnum; $i++){
3141                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3142                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3143                 }
3144             } else {
3145                 # assume other OS works like Linux
3146                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3147                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3148             }
3149         }
3150
3151         my $pciaddr = print_pci_addr("spice", $bridges);
3152
3153         my $nodename = PVE::INotify::nodename();
3154         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3155         $spice_port = PVE::Tools::next_spice_port($pfamily);
3156
3157         push @$devices, '-spice', "tls-port=${spice_port},addr=localhost,tls-ciphers=DES-CBC3-SHA,seamless-migration=on";
3158
3159         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3160         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3161         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3162     }
3163
3164     # enable balloon by default, unless explicitly disabled
3165     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3166         $pciaddr = print_pci_addr("balloon0", $bridges);
3167         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3168     }
3169
3170     if ($conf->{watchdog}) {
3171         my $wdopts = parse_watchdog($conf->{watchdog});
3172         $pciaddr = print_pci_addr("watchdog", $bridges);
3173         my $watchdog = $wdopts->{model} || 'i6300esb';
3174         push @$devices, '-device', "$watchdog$pciaddr";
3175         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3176     }
3177
3178     my $vollist = [];
3179     my $scsicontroller = {};
3180     my $ahcicontroller = {};
3181     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3182
3183     # Add iscsi initiator name if available
3184     if (my $initiator = get_initiator_name()) {
3185         push @$devices, '-iscsi', "initiator-name=$initiator";
3186     }
3187
3188     foreach_drive($conf, sub {
3189         my ($ds, $drive) = @_;
3190
3191         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3192             push @$vollist, $drive->{file};
3193         }
3194
3195         $use_virtio = 1 if $ds =~ m/^virtio/;
3196
3197         if (drive_is_cdrom ($drive)) {
3198             if ($bootindex_hash->{d}) {
3199                 $drive->{bootindex} = $bootindex_hash->{d};
3200                 $bootindex_hash->{d} += 1;
3201             }
3202         } else {
3203             if ($bootindex_hash->{c}) {
3204                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3205                 $bootindex_hash->{c} += 1;
3206             }
3207         }
3208
3209         if($drive->{interface} eq 'virtio'){
3210            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3211         }
3212
3213         if ($drive->{interface} eq 'scsi') {
3214
3215             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3216
3217             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3218             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3219
3220             my $iothread = '';
3221             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3222                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3223                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3224             } elsif ($drive->{iothread}) {
3225                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3226             }
3227
3228             my $queues = '';
3229             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3230                 $queues = ",num_queues=$drive->{queues}";
3231             }
3232
3233             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3234             $scsicontroller->{$controller}=1;
3235         }
3236
3237         if ($drive->{interface} eq 'sata') {
3238            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3239            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3240            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3241            $ahcicontroller->{$controller}=1;
3242         }
3243
3244         if ($drive->{interface} eq 'efidisk') {
3245             # this will be added somewhere else
3246             return;
3247         }
3248
3249         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3250         push @$devices, '-drive',$drive_cmd;
3251         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3252     });
3253
3254     for (my $i = 0; $i < $MAX_NETS; $i++) {
3255          next if !$conf->{"net$i"};
3256          my $d = parse_net($conf->{"net$i"});
3257          next if !$d;
3258
3259          $use_virtio = 1 if $d->{model} eq 'virtio';
3260
3261          if ($bootindex_hash->{n}) {
3262             $d->{bootindex} = $bootindex_hash->{n};
3263             $bootindex_hash->{n} += 1;
3264          }
3265
3266          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3267          push @$devices, '-netdev', $netdevfull;
3268
3269          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3270          push @$devices, '-device', $netdevicefull;
3271     }
3272
3273     if (!$q35) {
3274         # add pci bridges
3275         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3276            $bridges->{1} = 1;
3277            $bridges->{2} = 1;
3278         }
3279
3280         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3281
3282         while (my ($k, $v) = each %$bridges) {
3283             $pciaddr = print_pci_addr("pci.$k");
3284             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3285         }
3286     }
3287
3288     # add custom args
3289     if ($conf->{args}) {
3290         my $aa = PVE::Tools::split_args($conf->{args});
3291         push @$cmd, @$aa;
3292     }
3293
3294     push @$cmd, @$devices;
3295     push @$cmd, '-rtc', join(',', @$rtcFlags)
3296         if scalar(@$rtcFlags);
3297     push @$cmd, '-machine', join(',', @$machineFlags)
3298         if scalar(@$machineFlags);
3299     push @$cmd, '-global', join(',', @$globalFlags)
3300         if scalar(@$globalFlags);
3301
3302     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3303 }
3304
3305 sub vnc_socket {
3306     my ($vmid) = @_;
3307     return "${var_run_tmpdir}/$vmid.vnc";
3308 }
3309
3310 sub spice_port {
3311     my ($vmid) = @_;
3312
3313     my $res = vm_mon_cmd($vmid, 'query-spice');
3314
3315     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3316 }
3317
3318 sub qmp_socket {
3319     my ($vmid, $qga) = @_;
3320     my $sockettype = $qga ? 'qga' : 'qmp';
3321     return "${var_run_tmpdir}/$vmid.$sockettype";
3322 }
3323
3324 sub pidfile_name {
3325     my ($vmid) = @_;
3326     return "${var_run_tmpdir}/$vmid.pid";
3327 }
3328
3329 sub vm_devices_list {
3330     my ($vmid) = @_;
3331
3332     my $res = vm_mon_cmd($vmid, 'query-pci');
3333     my $devices = {};
3334     foreach my $pcibus (@$res) {
3335         foreach my $device (@{$pcibus->{devices}}) {
3336             next if !$device->{'qdev_id'};
3337             if ($device->{'pci_bridge'}) {
3338                 $devices->{$device->{'qdev_id'}} = 1;
3339                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3340                     next if !$bridge_device->{'qdev_id'};
3341                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3342                     $devices->{$device->{'qdev_id'}}++;
3343                 }
3344             } else {
3345                 $devices->{$device->{'qdev_id'}} = 1;
3346             }
3347         }
3348     }
3349
3350     my $resblock = vm_mon_cmd($vmid, 'query-block');
3351     foreach my $block (@$resblock) {
3352         if($block->{device} =~ m/^drive-(\S+)/){
3353                 $devices->{$1} = 1;
3354         }
3355     }
3356
3357     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3358     foreach my $mice (@$resmice) {
3359         if ($mice->{name} eq 'QEMU HID Tablet') {
3360             $devices->{tablet} = 1;
3361             last;
3362         }
3363     }
3364
3365     # for usb devices there is no query-usb
3366     # but we can iterate over the entries in
3367     # qom-list path=/machine/peripheral
3368     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3369     foreach my $per (@$resperipheral) {
3370         if ($per->{name} =~ m/^usb\d+$/) {
3371             $devices->{$per->{name}} = 1;
3372         }
3373     }
3374
3375     return $devices;
3376 }
3377
3378 sub vm_deviceplug {
3379     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3380
3381     my $q35 = machine_type_is_q35($conf);
3382
3383     my $devices_list = vm_devices_list($vmid);
3384     return 1 if defined($devices_list->{$deviceid});
3385
3386     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3387
3388     if ($deviceid eq 'tablet') {
3389
3390         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3391
3392     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3393
3394         die "usb hotplug currently not reliable\n";
3395         # since we can't reliably hot unplug all added usb devices
3396         # and usb passthrough disables live migration
3397         # we disable usb hotplugging for now
3398         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3399
3400     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3401
3402         qemu_iothread_add($vmid, $deviceid, $device);
3403
3404         qemu_driveadd($storecfg, $vmid, $device);
3405         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3406
3407         qemu_deviceadd($vmid, $devicefull);
3408         eval { qemu_deviceaddverify($vmid, $deviceid); };
3409         if (my $err = $@) {
3410             eval { qemu_drivedel($vmid, $deviceid); };
3411             warn $@ if $@;
3412             die $err;
3413         }
3414
3415     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3416
3417
3418         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3419         my $pciaddr = print_pci_addr($deviceid);
3420         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3421
3422         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3423
3424         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3425             qemu_iothread_add($vmid, $deviceid, $device);
3426             $devicefull .= ",iothread=iothread-$deviceid";
3427         }
3428
3429         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3430             $devicefull .= ",num_queues=$device->{queues}";
3431         }
3432
3433         qemu_deviceadd($vmid, $devicefull);
3434         qemu_deviceaddverify($vmid, $deviceid);
3435
3436     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3437
3438         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3439         qemu_driveadd($storecfg, $vmid, $device);
3440
3441         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3442         eval { qemu_deviceadd($vmid, $devicefull); };
3443         if (my $err = $@) {
3444             eval { qemu_drivedel($vmid, $deviceid); };
3445             warn $@ if $@;
3446             die $err;
3447         }
3448
3449     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3450
3451         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3452
3453         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
3454         my $use_old_bios_files = undef;
3455         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3456
3457         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3458         qemu_deviceadd($vmid, $netdevicefull);
3459         eval { qemu_deviceaddverify($vmid, $deviceid); };
3460         if (my $err = $@) {
3461             eval { qemu_netdevdel($vmid, $deviceid); };
3462             warn $@ if $@;
3463             die $err;
3464         }
3465
3466     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3467
3468         my $bridgeid = $2;
3469         my $pciaddr = print_pci_addr($deviceid);
3470         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3471
3472         qemu_deviceadd($vmid, $devicefull);
3473         qemu_deviceaddverify($vmid, $deviceid);
3474
3475     } else {
3476         die "can't hotplug device '$deviceid'\n";
3477     }
3478
3479     return 1;
3480 }
3481
3482 # fixme: this should raise exceptions on error!
3483 sub vm_deviceunplug {
3484     my ($vmid, $conf, $deviceid) = @_;
3485
3486     my $devices_list = vm_devices_list($vmid);
3487     return 1 if !defined($devices_list->{$deviceid});
3488
3489     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3490
3491     if ($deviceid eq 'tablet') {
3492
3493         qemu_devicedel($vmid, $deviceid);
3494
3495     } elsif ($deviceid =~ m/^usb\d+$/) {
3496
3497         die "usb hotplug currently not reliable\n";
3498         # when unplugging usb devices this way,
3499         # there may be remaining usb controllers/hubs
3500         # so we disable it for now
3501         qemu_devicedel($vmid, $deviceid);
3502         qemu_devicedelverify($vmid, $deviceid);
3503
3504     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3505
3506         qemu_devicedel($vmid, $deviceid);
3507         qemu_devicedelverify($vmid, $deviceid);
3508         qemu_drivedel($vmid, $deviceid);
3509         qemu_iothread_del($conf, $vmid, $deviceid);
3510
3511     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3512
3513         qemu_devicedel($vmid, $deviceid);
3514         qemu_devicedelverify($vmid, $deviceid);
3515         qemu_iothread_del($conf, $vmid, $deviceid);
3516
3517     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3518
3519         #qemu 2.3 segfault on drive_del with virtioscsi + iothread
3520         my $device = parse_drive($deviceid, $conf->{$deviceid});
3521         die "virtioscsi with iothread is not hot-unplugglable currently" if $device->{iothread};
3522
3523         qemu_devicedel($vmid, $deviceid);
3524         qemu_drivedel($vmid, $deviceid);
3525         qemu_deletescsihw($conf, $vmid, $deviceid);
3526
3527     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3528
3529         qemu_devicedel($vmid, $deviceid);
3530         qemu_devicedelverify($vmid, $deviceid);
3531         qemu_netdevdel($vmid, $deviceid);
3532
3533     } else {
3534         die "can't unplug device '$deviceid'\n";
3535     }
3536
3537     return 1;
3538 }
3539
3540 sub qemu_deviceadd {
3541     my ($vmid, $devicefull) = @_;
3542
3543     $devicefull = "driver=".$devicefull;
3544     my %options =  split(/[=,]/, $devicefull);
3545
3546     vm_mon_cmd($vmid, "device_add" , %options);
3547 }
3548
3549 sub qemu_devicedel {
3550     my ($vmid, $deviceid) = @_;
3551
3552     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3553 }
3554
3555 sub qemu_iothread_add {
3556     my($vmid, $deviceid, $device) = @_;
3557
3558     if ($device->{iothread}) {
3559         my $iothreads = vm_iothreads_list($vmid);
3560         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3561     }
3562 }
3563
3564 sub qemu_iothread_del {
3565     my($conf, $vmid, $deviceid) = @_;
3566
3567     my $device = parse_drive($deviceid, $conf->{$deviceid});
3568     if ($device->{iothread}) {
3569         my $iothreads = vm_iothreads_list($vmid);
3570         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3571     }
3572 }
3573
3574 sub qemu_objectadd {
3575     my($vmid, $objectid, $qomtype) = @_;
3576
3577     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3578
3579     return 1;
3580 }
3581
3582 sub qemu_objectdel {
3583     my($vmid, $objectid) = @_;
3584
3585     vm_mon_cmd($vmid, "object-del", id => $objectid);
3586
3587     return 1;
3588 }
3589
3590 sub qemu_driveadd {
3591     my ($storecfg, $vmid, $device) = @_;
3592
3593     my $drive = print_drive_full($storecfg, $vmid, $device);
3594     $drive =~ s/\\/\\\\/g;
3595     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3596
3597     # If the command succeeds qemu prints: "OK"
3598     return 1 if $ret =~ m/OK/s;
3599
3600     die "adding drive failed: $ret\n";
3601 }
3602
3603 sub qemu_drivedel {
3604     my($vmid, $deviceid) = @_;
3605
3606     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3607     $ret =~ s/^\s+//;
3608
3609     return 1 if $ret eq "";
3610
3611     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3612     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3613
3614     die "deleting drive $deviceid failed : $ret\n";
3615 }
3616
3617 sub qemu_deviceaddverify {
3618     my ($vmid, $deviceid) = @_;
3619
3620     for (my $i = 0; $i <= 5; $i++) {
3621          my $devices_list = vm_devices_list($vmid);
3622          return 1 if defined($devices_list->{$deviceid});
3623          sleep 1;
3624     }
3625
3626     die "error on hotplug device '$deviceid'\n";
3627 }
3628
3629
3630 sub qemu_devicedelverify {
3631     my ($vmid, $deviceid) = @_;
3632
3633     # need to verify that the device is correctly removed as device_del
3634     # is async and empty return is not reliable
3635
3636     for (my $i = 0; $i <= 5; $i++) {
3637          my $devices_list = vm_devices_list($vmid);
3638          return 1 if !defined($devices_list->{$deviceid});
3639          sleep 1;
3640     }
3641
3642     die "error on hot-unplugging device '$deviceid'\n";
3643 }
3644
3645 sub qemu_findorcreatescsihw {
3646     my ($storecfg, $conf, $vmid, $device) = @_;
3647
3648     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3649
3650     my $scsihwid="$controller_prefix$controller";
3651     my $devices_list = vm_devices_list($vmid);
3652
3653     if(!defined($devices_list->{$scsihwid})) {
3654         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3655     }
3656
3657     return 1;
3658 }
3659
3660 sub qemu_deletescsihw {
3661     my ($conf, $vmid, $opt) = @_;
3662
3663     my $device = parse_drive($opt, $conf->{$opt});
3664
3665     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3666         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3667         return 1;
3668     }
3669
3670     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3671
3672     my $devices_list = vm_devices_list($vmid);
3673     foreach my $opt (keys %{$devices_list}) {
3674         if (PVE::QemuServer::is_valid_drivename($opt)) {
3675             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3676             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3677                 return 1;
3678             }
3679         }
3680     }
3681
3682     my $scsihwid="scsihw$controller";
3683
3684     vm_deviceunplug($vmid, $conf, $scsihwid);
3685
3686     return 1;
3687 }
3688
3689 sub qemu_add_pci_bridge {
3690     my ($storecfg, $conf, $vmid, $device) = @_;
3691
3692     my $bridges = {};
3693
3694     my $bridgeid;
3695
3696     print_pci_addr($device, $bridges);
3697
3698     while (my ($k, $v) = each %$bridges) {
3699         $bridgeid = $k;
3700     }
3701     return 1 if !defined($bridgeid) || $bridgeid < 1;
3702
3703     my $bridge = "pci.$bridgeid";
3704     my $devices_list = vm_devices_list($vmid);
3705
3706     if (!defined($devices_list->{$bridge})) {
3707         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3708     }
3709
3710     return 1;
3711 }
3712
3713 sub qemu_set_link_status {
3714     my ($vmid, $device, $up) = @_;
3715
3716     vm_mon_cmd($vmid, "set_link", name => $device,
3717                up => $up ? JSON::true : JSON::false);
3718 }
3719
3720 sub qemu_netdevadd {
3721     my ($vmid, $conf, $device, $deviceid) = @_;
3722
3723     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
3724     my %options =  split(/[=,]/, $netdev);
3725
3726     vm_mon_cmd($vmid, "netdev_add",  %options);
3727     return 1;
3728 }
3729
3730 sub qemu_netdevdel {
3731     my ($vmid, $deviceid) = @_;
3732
3733     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3734 }
3735
3736 sub qemu_usb_hotplug {
3737     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3738
3739     return if !$device;
3740
3741     # remove the old one first
3742     vm_deviceunplug($vmid, $conf, $deviceid);
3743
3744     # check if xhci controller is necessary and available
3745     if ($device->{usb3}) {
3746
3747         my $devicelist = vm_devices_list($vmid);
3748
3749         if (!$devicelist->{xhci}) {
3750             my $pciaddr = print_pci_addr("xhci");
3751             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
3752         }
3753     }
3754     my $d = parse_usb_device($device->{host});
3755     $d->{usb3} = $device->{usb3};
3756
3757     # add the new one
3758     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d);
3759 }
3760
3761 sub qemu_cpu_hotplug {
3762     my ($vmid, $conf, $vcpus) = @_;
3763
3764     my $sockets = 1;
3765     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3766     $sockets = $conf->{sockets} if  $conf->{sockets};
3767     my $cores = $conf->{cores} || 1;
3768     my $maxcpus = $sockets * $cores;
3769
3770     $vcpus = $maxcpus if !$vcpus;
3771
3772     die "you can't add more vcpus than maxcpus\n"
3773         if $vcpus > $maxcpus;
3774
3775     my $currentvcpus = $conf->{vcpus} || $maxcpus;
3776     die "online cpu unplug is not yet possible\n"
3777         if $vcpus < $currentvcpus;
3778
3779     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3780     die "vcpus in running vm is different than configuration\n"
3781         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
3782
3783     for (my $i = $currentvcpus; $i < $vcpus; $i++) {
3784         vm_mon_cmd($vmid, "cpu-add", id => int($i));
3785     }
3786 }
3787
3788 sub qemu_block_set_io_throttle {
3789     my ($vmid, $deviceid,
3790         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
3791         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max) = @_;
3792
3793     return if !check_running($vmid) ;
3794
3795     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
3796         bps => int($bps),
3797         bps_rd => int($bps_rd),
3798         bps_wr => int($bps_wr),
3799         iops => int($iops),
3800         iops_rd => int($iops_rd),
3801         iops_wr => int($iops_wr),
3802         bps_max => int($bps_max),
3803         bps_rd_max => int($bps_rd_max),
3804         bps_wr_max => int($bps_wr_max),
3805         iops_max => int($iops_max),
3806         iops_rd_max => int($iops_rd_max),
3807         iops_wr_max => int($iops_wr_max)
3808     );
3809
3810 }
3811
3812 # old code, only used to shutdown old VM after update
3813 sub __read_avail {
3814     my ($fh, $timeout) = @_;
3815
3816     my $sel = new IO::Select;
3817     $sel->add($fh);
3818
3819     my $res = '';
3820     my $buf;
3821
3822     my @ready;
3823     while (scalar (@ready = $sel->can_read($timeout))) {
3824         my $count;
3825         if ($count = $fh->sysread($buf, 8192)) {
3826             if ($buf =~ /^(.*)\(qemu\) $/s) {
3827                 $res .= $1;
3828                 last;
3829             } else {
3830                 $res .= $buf;
3831             }
3832         } else {
3833             if (!defined($count)) {
3834                 die "$!\n";
3835             }
3836             last;
3837         }
3838     }
3839
3840     die "monitor read timeout\n" if !scalar(@ready);
3841
3842     return $res;
3843 }
3844
3845 # old code, only used to shutdown old VM after update
3846 sub vm_monitor_command {
3847     my ($vmid, $cmdstr, $nocheck) = @_;
3848
3849     my $res;
3850
3851     eval {
3852         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
3853
3854         my $sname = "${var_run_tmpdir}/$vmid.mon";
3855
3856         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
3857             die "unable to connect to VM $vmid socket - $!\n";
3858
3859         my $timeout = 3;
3860
3861         # hack: migrate sometime blocks the monitor (when migrate_downtime
3862         # is set)
3863         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3864             $timeout = 60*60; # 1 hour
3865         }
3866
3867         # read banner;
3868         my $data = __read_avail($sock, $timeout);
3869
3870         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
3871             die "got unexpected qemu monitor banner\n";
3872         }
3873
3874         my $sel = new IO::Select;
3875         $sel->add($sock);
3876
3877         if (!scalar(my @ready = $sel->can_write($timeout))) {
3878             die "monitor write error - timeout";
3879         }
3880
3881         my $fullcmd = "$cmdstr\r";
3882
3883         # syslog('info', "VM $vmid monitor command: $cmdstr");
3884
3885         my $b;
3886         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
3887             die "monitor write error - $!";
3888         }
3889
3890         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
3891
3892         $timeout = 20;
3893
3894         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3895             $timeout = 60*60; # 1 hour
3896         } elsif ($cmdstr =~ m/^(eject|change)/) {
3897             $timeout = 60; # note: cdrom mount command is slow
3898         }
3899         if ($res = __read_avail($sock, $timeout)) {
3900
3901             my @lines = split("\r?\n", $res);
3902
3903             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
3904
3905             $res = join("\n", @lines);
3906             $res .= "\n";
3907         }
3908     };
3909
3910     my $err = $@;
3911
3912     if ($err) {
3913         syslog("err", "VM $vmid monitor command failed - $err");
3914         die $err;
3915     }
3916
3917     return $res;
3918 }
3919
3920 sub qemu_block_resize {
3921     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
3922
3923     my $running = check_running($vmid);
3924
3925     return if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
3926
3927     return if !$running;
3928
3929     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
3930
3931 }
3932
3933 sub qemu_volume_snapshot {
3934     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3935
3936     my $running = check_running($vmid);
3937
3938     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
3939         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
3940     } else {
3941         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
3942     }
3943 }
3944
3945 sub qemu_volume_snapshot_delete {
3946     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3947
3948     my $running = check_running($vmid);
3949
3950     return if !PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
3951
3952     return if !$running;
3953
3954     vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
3955 }
3956
3957 sub set_migration_caps {
3958     my ($vmid) = @_;
3959
3960     my $cap_ref = [];
3961
3962     my $enabled_cap = {
3963         "auto-converge" => 1,
3964         "xbzrle" => 1,
3965         "x-rdma-pin-all" => 0,
3966         "zero-blocks" => 0,
3967         "compress" => 0
3968     };
3969
3970     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
3971
3972     for my $supported_capability (@$supported_capabilities) {
3973         push @$cap_ref, {
3974             capability => $supported_capability->{capability},
3975             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
3976         };
3977     }
3978
3979     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
3980 }
3981
3982 my $fast_plug_option = {
3983     'lock' => 1,
3984     'name' => 1,
3985     'onboot' => 1,
3986     'shares' => 1,
3987     'startup' => 1,
3988     'description' => 1,
3989     'protection' => 1,
3990 };
3991
3992 # hotplug changes in [PENDING]
3993 # $selection hash can be used to only apply specified options, for
3994 # example: { cores => 1 } (only apply changed 'cores')
3995 # $errors ref is used to return error messages
3996 sub vmconfig_hotplug_pending {
3997     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
3998
3999     my $defaults = load_defaults();
4000
4001     # commit values which do not have any impact on running VM first
4002     # Note: those option cannot raise errors, we we do not care about
4003     # $selection and always apply them.
4004
4005     my $add_error = sub {
4006         my ($opt, $msg) = @_;
4007         $errors->{$opt} = "hotplug problem - $msg";
4008     };
4009
4010     my $changes = 0;
4011     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4012         if ($fast_plug_option->{$opt}) {
4013             $conf->{$opt} = $conf->{pending}->{$opt};
4014             delete $conf->{pending}->{$opt};
4015             $changes = 1;
4016         }
4017     }
4018
4019     if ($changes) {
4020         PVE::QemuConfig->write_config($vmid, $conf);
4021         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4022     }
4023
4024     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4025
4026     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4027     while (my ($opt, $force) = each %$pending_delete_hash) {
4028         next if $selection && !$selection->{$opt};
4029         eval {
4030             if ($opt eq 'hotplug') {
4031                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4032             } elsif ($opt eq 'tablet') {
4033                 die "skip\n" if !$hotplug_features->{usb};
4034                 if ($defaults->{tablet}) {
4035                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4036                 } else {
4037                     vm_deviceunplug($vmid, $conf, $opt);
4038                 }
4039             } elsif ($opt =~ m/^usb\d+/) {
4040                 die "skip\n";
4041                 # since we cannot reliably hot unplug usb devices
4042                 # we are disabling it
4043                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4044                 vm_deviceunplug($vmid, $conf, $opt);
4045             } elsif ($opt eq 'vcpus') {
4046                 die "skip\n" if !$hotplug_features->{cpu};
4047                 qemu_cpu_hotplug($vmid, $conf, undef);
4048             } elsif ($opt eq 'balloon') {
4049                 # enable balloon device is not hotpluggable
4050                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
4051             } elsif ($fast_plug_option->{$opt}) {
4052                 # do nothing
4053             } elsif ($opt =~ m/^net(\d+)$/) {
4054                 die "skip\n" if !$hotplug_features->{network};
4055                 vm_deviceunplug($vmid, $conf, $opt);
4056             } elsif (is_valid_drivename($opt)) {
4057                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4058                 vm_deviceunplug($vmid, $conf, $opt);
4059                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4060             } elsif ($opt =~ m/^memory$/) {
4061                 die "skip\n" if !$hotplug_features->{memory};
4062                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4063             } elsif ($opt eq 'cpuunits') {
4064                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4065             } elsif ($opt eq 'cpulimit') {
4066                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4067             } else {
4068                 die "skip\n";
4069             }
4070         };
4071         if (my $err = $@) {
4072             &$add_error($opt, $err) if $err ne "skip\n";
4073         } else {
4074             # save new config if hotplug was successful
4075             delete $conf->{$opt};
4076             vmconfig_undelete_pending_option($conf, $opt);
4077             PVE::QemuConfig->write_config($vmid, $conf);
4078             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4079         }
4080     }
4081
4082     foreach my $opt (keys %{$conf->{pending}}) {
4083         next if $selection && !$selection->{$opt};
4084         my $value = $conf->{pending}->{$opt};
4085         eval {
4086             if ($opt eq 'hotplug') {
4087                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4088             } elsif ($opt eq 'tablet') {
4089                 die "skip\n" if !$hotplug_features->{usb};
4090                 if ($value == 1) {
4091                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4092                 } elsif ($value == 0) {
4093                     vm_deviceunplug($vmid, $conf, $opt);
4094                 }
4095             } elsif ($opt =~ m/^usb\d+$/) {
4096                 die "skip\n";
4097                 # since we cannot reliably hot unplug usb devices
4098                 # we are disabling it
4099                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4100                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4101                 die "skip\n" if !$d;
4102                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d);
4103             } elsif ($opt eq 'vcpus') {
4104                 die "skip\n" if !$hotplug_features->{cpu};
4105                 qemu_cpu_hotplug($vmid, $conf, $value);
4106             } elsif ($opt eq 'balloon') {
4107                 # enable/disable balloning device is not hotpluggable
4108                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4109                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4110                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4111
4112                 # allow manual ballooning if shares is set to zero
4113                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4114                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4115                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4116                 }
4117             } elsif ($opt =~ m/^net(\d+)$/) {
4118                 # some changes can be done without hotplug
4119                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4120                                     $vmid, $opt, $value);
4121             } elsif (is_valid_drivename($opt)) {
4122                 # some changes can be done without hotplug
4123                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4124                                      $vmid, $opt, $value, 1);
4125             } elsif ($opt =~ m/^memory$/) { #dimms
4126                 die "skip\n" if !$hotplug_features->{memory};
4127                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4128             } elsif ($opt eq 'cpuunits') {
4129                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4130             } elsif ($opt eq 'cpulimit') {
4131                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4132                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4133             } else {
4134                 die "skip\n";  # skip non-hot-pluggable options
4135             }
4136         };
4137         if (my $err = $@) {
4138             &$add_error($opt, $err) if $err ne "skip\n";
4139         } else {
4140             # save new config if hotplug was successful
4141             $conf->{$opt} = $value;
4142             delete $conf->{pending}->{$opt};
4143             PVE::QemuConfig->write_config($vmid, $conf);
4144             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4145         }
4146     }
4147 }
4148
4149 sub try_deallocate_drive {
4150     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4151
4152     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4153         my $volid = $drive->{file};
4154         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4155             my $sid = PVE::Storage::parse_volume_id($volid);
4156             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4157
4158             # check if the disk is really unused
4159             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4160                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4161             PVE::Storage::vdisk_free($storecfg, $volid);
4162             return 1;
4163         } else {
4164             # If vm is not owner of this disk remove from config
4165             return 1;
4166         }
4167     }
4168
4169     return undef;
4170 }
4171
4172 sub vmconfig_delete_or_detach_drive {
4173     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4174
4175     my $drive = parse_drive($opt, $conf->{$opt});
4176
4177     my $rpcenv = PVE::RPCEnvironment::get();
4178     my $authuser = $rpcenv->get_user();
4179
4180     if ($force) {
4181         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4182         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4183     } else {
4184         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4185     }
4186 }
4187
4188 sub vmconfig_apply_pending {
4189     my ($vmid, $conf, $storecfg) = @_;
4190
4191     # cold plug
4192
4193     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4194     while (my ($opt, $force) = each %$pending_delete_hash) {
4195         die "internal error" if $opt =~ m/^unused/;
4196         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4197         if (!defined($conf->{$opt})) {
4198             vmconfig_undelete_pending_option($conf, $opt);
4199             PVE::QemuConfig->write_config($vmid, $conf);
4200         } elsif (is_valid_drivename($opt)) {
4201             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4202             vmconfig_undelete_pending_option($conf, $opt);
4203             delete $conf->{$opt};
4204             PVE::QemuConfig->write_config($vmid, $conf);
4205         } else {
4206             vmconfig_undelete_pending_option($conf, $opt);
4207             delete $conf->{$opt};
4208             PVE::QemuConfig->write_config($vmid, $conf);
4209         }
4210     }
4211
4212     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4213
4214     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4215         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4216
4217         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4218             # skip if nothing changed
4219         } elsif (is_valid_drivename($opt)) {
4220             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4221                 if defined($conf->{$opt});
4222             $conf->{$opt} = $conf->{pending}->{$opt};
4223         } else {
4224             $conf->{$opt} = $conf->{pending}->{$opt};
4225         }
4226
4227         delete $conf->{pending}->{$opt};
4228         PVE::QemuConfig->write_config($vmid, $conf);
4229     }
4230 }
4231
4232 my $safe_num_ne = sub {
4233     my ($a, $b) = @_;
4234
4235     return 0 if !defined($a) && !defined($b);
4236     return 1 if !defined($a);
4237     return 1 if !defined($b);
4238
4239     return $a != $b;
4240 };
4241
4242 my $safe_string_ne = sub {
4243     my ($a, $b) = @_;
4244
4245     return 0 if !defined($a) && !defined($b);
4246     return 1 if !defined($a);
4247     return 1 if !defined($b);
4248
4249     return $a ne $b;
4250 };
4251
4252 sub vmconfig_update_net {
4253     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4254
4255     my $newnet = parse_net($value);
4256
4257     if ($conf->{$opt}) {
4258         my $oldnet = parse_net($conf->{$opt});
4259
4260         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4261             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4262             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4263             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4264
4265             # for non online change, we try to hot-unplug
4266             die "skip\n" if !$hotplug;
4267             vm_deviceunplug($vmid, $conf, $opt);
4268         } else {
4269
4270             die "internal error" if $opt !~ m/net(\d+)/;
4271             my $iface = "tap${vmid}i$1";
4272
4273             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4274                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4275                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4276                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4277                 PVE::Network::tap_unplug($iface);
4278                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4279             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4280                 # Rate can be applied on its own but any change above needs to
4281                 # include the rate in tap_plug since OVS resets everything.
4282                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4283             }
4284
4285             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4286                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4287             }
4288
4289             return 1;
4290         }
4291     }
4292
4293     if ($hotplug) {
4294         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4295     } else {
4296         die "skip\n";
4297     }
4298 }
4299
4300 sub vmconfig_update_disk {
4301     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4302
4303     # fixme: do we need force?
4304
4305     my $drive = parse_drive($opt, $value);
4306
4307     if ($conf->{$opt}) {
4308
4309         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4310
4311             my $media = $drive->{media} || 'disk';
4312             my $oldmedia = $old_drive->{media} || 'disk';
4313             die "unable to change media type\n" if $media ne $oldmedia;
4314
4315             if (!drive_is_cdrom($old_drive)) {
4316
4317                 if ($drive->{file} ne $old_drive->{file}) {
4318
4319                     die "skip\n" if !$hotplug;
4320
4321                     # unplug and register as unused
4322                     vm_deviceunplug($vmid, $conf, $opt);
4323                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4324
4325                 } else {
4326                     # update existing disk
4327
4328                     # skip non hotpluggable value
4329                     if (&$safe_num_ne($drive->{discard}, $old_drive->{discard}) ||
4330                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4331                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4332                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4333                         die "skip\n";
4334                     }
4335
4336                     # apply throttle
4337                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4338                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4339                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4340                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4341                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4342                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4343                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4344                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4345                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4346                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4347                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4348                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max})) {
4349
4350                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4351                                                    ($drive->{mbps} || 0)*1024*1024,
4352                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4353                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4354                                                    $drive->{iops} || 0,
4355                                                    $drive->{iops_rd} || 0,
4356                                                    $drive->{iops_wr} || 0,
4357                                                    ($drive->{mbps_max} || 0)*1024*1024,
4358                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4359                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4360                                                    $drive->{iops_max} || 0,
4361                                                    $drive->{iops_rd_max} || 0,
4362                                                    $drive->{iops_wr_max} || 0);
4363
4364                     }
4365
4366                     return 1;
4367                 }
4368
4369             } else { # cdrom
4370
4371                 if ($drive->{file} eq 'none') {
4372                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4373                 } else {
4374                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4375                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4376                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4377                 }
4378
4379                 return 1;
4380             }
4381         }
4382     }
4383
4384     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4385     # hotplug new disks
4386     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4387     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4388 }
4389
4390 sub vm_start {
4391     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4392         $forcemachine, $spice_ticket) = @_;
4393
4394     PVE::QemuConfig->lock_config($vmid, sub {
4395         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4396
4397         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
4398
4399         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4400
4401         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4402
4403         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4404             vmconfig_apply_pending($vmid, $conf, $storecfg);
4405             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4406         }
4407
4408         my $defaults = load_defaults();
4409
4410         # set environment variable useful inside network script
4411         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4412
4413         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4414
4415         my $migrate_port = 0;
4416         my $migrate_uri;
4417         if ($statefile) {
4418             if ($statefile eq 'tcp') {
4419                 my $localip = "localhost";
4420                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4421                 my $nodename = PVE::INotify::nodename();
4422                 if ($datacenterconf->{migration_unsecure}) {
4423                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4424                         $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4425                 }
4426                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4427                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4428                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4429                 push @$cmd, '-incoming', $migrate_uri;
4430                 push @$cmd, '-S';
4431
4432             } elsif ($statefile eq 'unix') {
4433                 # should be default for secure migrations as a ssh TCP forward
4434                 # tunnel is not deterministic reliable ready and fails regurarly
4435                 # to set up in time, so use UNIX socket forwards
4436                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
4437                 unlink $socket_addr;
4438
4439                 $migrate_uri = "unix:$socket_addr";
4440
4441                 push @$cmd, '-incoming', $migrate_uri;
4442                 push @$cmd, '-S';
4443
4444             } else {
4445                 push @$cmd, '-loadstate', $statefile;
4446             }
4447         } elsif ($paused) {
4448             push @$cmd, '-S';
4449         }
4450
4451         # host pci devices
4452         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4453           my $d = parse_hostpci($conf->{"hostpci$i"});
4454           next if !$d;
4455           my $pcidevices = $d->{pciid};
4456           foreach my $pcidevice (@$pcidevices) {
4457                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4458
4459                 my $info = pci_device_info("0000:$pciid");
4460                 die "IOMMU not present\n" if !check_iommu_support();
4461                 die "no pci device info for device '$pciid'\n" if !$info;
4462                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4463                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4464           }
4465         }
4466
4467         PVE::Storage::activate_volumes($storecfg, $vollist);
4468
4469         if (!check_running($vmid, 1) && -d "/sys/fs/cgroup/systemd/qemu.slice/$vmid.scope") {
4470             my $cmd = [];
4471             push @$cmd, '/bin/systemctl', 'stop', "$vmid.scope";
4472             eval  { run_command($cmd); };
4473         }
4474
4475         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
4476                                                   : $defaults->{cpuunits};
4477
4478         my %run_params = (timeout => $statefile ? undef : 30, umask => 0077);
4479
4480         my %properties = (
4481             Slice => 'qemu.slice',
4482             KillMode => 'none',
4483             CPUShares => $cpuunits
4484         );
4485
4486         if (my $cpulimit = $conf->{cpulimit}) {
4487             $properties{CPUQuota} = int($cpulimit * 100);
4488         }
4489         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
4490
4491         if ($conf->{hugepages}) {
4492
4493             my $code = sub {
4494                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
4495                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
4496
4497                 PVE::QemuServer::Memory::hugepages_mount();
4498                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
4499
4500                 eval  {
4501                     PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4502                     run_command($cmd, %run_params);
4503                 };
4504
4505                 if (my $err = $@) {
4506                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
4507                     die $err;
4508                 }
4509
4510                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
4511             };
4512             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
4513
4514         } else {
4515             eval  {
4516                 PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4517                 run_command($cmd, %run_params);
4518             };
4519         }
4520
4521         if (my $err = $@) {
4522             # deactivate volumes if start fails
4523             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
4524             die "start failed: $err";
4525         }
4526
4527         print "migration listens on $migrate_uri\n" if $migrate_uri;
4528
4529         if ($statefile && $statefile ne 'tcp')  {
4530             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4531             warn $@ if $@;
4532         }
4533
4534         if ($migratedfrom) {
4535
4536             eval {
4537                 set_migration_caps($vmid);
4538             };
4539             warn $@ if $@;
4540
4541             if ($spice_port) {
4542                 print "spice listens on port $spice_port\n";
4543                 if ($spice_ticket) {
4544                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4545                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4546                 }
4547             }
4548
4549         } else {
4550
4551             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4552                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4553                     if $conf->{balloon};
4554             }
4555
4556             foreach my $opt (keys %$conf) {
4557                 next if $opt !~  m/^net\d+$/;
4558                 my $nicconf = parse_net($conf->{$opt});
4559                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4560             }
4561         }
4562
4563         vm_mon_cmd_nocheck($vmid, 'qom-set',
4564                     path => "machine/peripheral/balloon0",
4565                     property => "guest-stats-polling-interval",
4566                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4567
4568     });
4569 }
4570
4571 sub vm_mon_cmd {
4572     my ($vmid, $execute, %params) = @_;
4573
4574     my $cmd = { execute => $execute, arguments => \%params };
4575     vm_qmp_command($vmid, $cmd);
4576 }
4577
4578 sub vm_mon_cmd_nocheck {
4579     my ($vmid, $execute, %params) = @_;
4580
4581     my $cmd = { execute => $execute, arguments => \%params };
4582     vm_qmp_command($vmid, $cmd, 1);
4583 }
4584
4585 sub vm_qmp_command {
4586     my ($vmid, $cmd, $nocheck) = @_;
4587
4588     my $res;
4589
4590     my $timeout;
4591     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4592         $timeout = $cmd->{arguments}->{timeout};
4593         delete $cmd->{arguments}->{timeout};
4594     }
4595
4596     eval {
4597         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4598         my $sname = qmp_socket($vmid);
4599         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4600             my $qmpclient = PVE::QMPClient->new();
4601
4602             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4603         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
4604             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
4605                 if scalar(%{$cmd->{arguments}});
4606             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
4607         } else {
4608             die "unable to open monitor socket\n";
4609         }
4610     };
4611     if (my $err = $@) {
4612         syslog("err", "VM $vmid qmp command failed - $err");
4613         die $err;
4614     }
4615
4616     return $res;
4617 }
4618
4619 sub vm_human_monitor_command {
4620     my ($vmid, $cmdline) = @_;
4621
4622     my $res;
4623
4624     my $cmd = {
4625         execute => 'human-monitor-command',
4626         arguments => { 'command-line' => $cmdline},
4627     };
4628
4629     return vm_qmp_command($vmid, $cmd);
4630 }
4631
4632 sub vm_commandline {
4633     my ($storecfg, $vmid) = @_;
4634
4635     my $conf = PVE::QemuConfig->load_config($vmid);
4636
4637     my $defaults = load_defaults();
4638
4639     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
4640
4641     return PVE::Tools::cmd2string($cmd);
4642 }
4643
4644 sub vm_reset {
4645     my ($vmid, $skiplock) = @_;
4646
4647     PVE::QemuConfig->lock_config($vmid, sub {
4648
4649         my $conf = PVE::QemuConfig->load_config($vmid);
4650
4651         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4652
4653         vm_mon_cmd($vmid, "system_reset");
4654     });
4655 }
4656
4657 sub get_vm_volumes {
4658     my ($conf) = @_;
4659
4660     my $vollist = [];
4661     foreach_volid($conf, sub {
4662         my ($volid, $is_cdrom) = @_;
4663
4664         return if $volid =~ m|^/|;
4665
4666         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4667         return if !$sid;
4668
4669         push @$vollist, $volid;
4670     });
4671
4672     return $vollist;
4673 }
4674
4675 sub vm_stop_cleanup {
4676     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
4677
4678     eval {
4679
4680         if (!$keepActive) {
4681             my $vollist = get_vm_volumes($conf);
4682             PVE::Storage::deactivate_volumes($storecfg, $vollist);
4683         }
4684
4685         foreach my $ext (qw(mon qmp pid vnc qga)) {
4686             unlink "/var/run/qemu-server/${vmid}.$ext";
4687         }
4688
4689         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
4690     };
4691     warn $@ if $@; # avoid errors - just warn
4692 }
4693
4694 # Note: use $nockeck to skip tests if VM configuration file exists.
4695 # We need that when migration VMs to other nodes (files already moved)
4696 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
4697 sub vm_stop {
4698     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
4699
4700     $force = 1 if !defined($force) && !$shutdown;
4701
4702     if ($migratedfrom){
4703         my $pid = check_running($vmid, $nocheck, $migratedfrom);
4704         kill 15, $pid if $pid;
4705         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4706         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
4707         return;
4708     }
4709
4710     PVE::QemuConfig->lock_config($vmid, sub {
4711
4712         my $pid = check_running($vmid, $nocheck);
4713         return if !$pid;
4714
4715         my $conf;
4716         if (!$nocheck) {
4717             $conf = PVE::QemuConfig->load_config($vmid);
4718             PVE::QemuConfig->check_lock($conf) if !$skiplock;
4719             if (!defined($timeout) && $shutdown && $conf->{startup}) {
4720                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
4721                 $timeout = $opts->{down} if $opts->{down};
4722             }
4723         }
4724
4725         $timeout = 60 if !defined($timeout);
4726
4727         eval {
4728             if ($shutdown) {
4729                 if (defined($conf) && $conf->{agent}) {
4730                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
4731                 } else {
4732                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
4733                 }
4734             } else {
4735                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
4736             }
4737         };
4738         my $err = $@;
4739
4740         if (!$err) {
4741             my $count = 0;
4742             while (($count < $timeout) && check_running($vmid, $nocheck)) {
4743                 $count++;
4744                 sleep 1;
4745             }
4746
4747             if ($count >= $timeout) {
4748                 if ($force) {
4749                     warn "VM still running - terminating now with SIGTERM\n";
4750                     kill 15, $pid;
4751                 } else {
4752                     die "VM quit/powerdown failed - got timeout\n";
4753                 }
4754             } else {
4755                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4756                 return;
4757             }
4758         } else {
4759             if ($force) {
4760                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
4761                 kill 15, $pid;
4762             } else {
4763                 die "VM quit/powerdown failed\n";
4764             }
4765         }
4766
4767         # wait again
4768         $timeout = 10;
4769
4770         my $count = 0;
4771         while (($count < $timeout) && check_running($vmid, $nocheck)) {
4772             $count++;
4773             sleep 1;
4774         }
4775
4776         if ($count >= $timeout) {
4777             warn "VM still running - terminating now with SIGKILL\n";
4778             kill 9, $pid;
4779             sleep 1;
4780         }
4781
4782         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4783    });
4784 }
4785
4786 sub vm_suspend {
4787     my ($vmid, $skiplock) = @_;
4788
4789     PVE::QemuConfig->lock_config($vmid, sub {
4790
4791         my $conf = PVE::QemuConfig->load_config($vmid);
4792
4793         PVE::QemuConfig->check_lock($conf)
4794             if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
4795
4796         vm_mon_cmd($vmid, "stop");
4797     });
4798 }
4799
4800 sub vm_resume {
4801     my ($vmid, $skiplock, $nocheck) = @_;
4802
4803     PVE::QemuConfig->lock_config($vmid, sub {
4804
4805         if (!$nocheck) {
4806
4807             my $conf = PVE::QemuConfig->load_config($vmid);
4808
4809             PVE::QemuConfig->check_lock($conf)
4810                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
4811
4812             vm_mon_cmd($vmid, "cont");
4813
4814         } else {
4815             vm_mon_cmd_nocheck($vmid, "cont");
4816         }
4817     });
4818 }
4819
4820 sub vm_sendkey {
4821     my ($vmid, $skiplock, $key) = @_;
4822
4823     PVE::QemuConfig->lock_config($vmid, sub {
4824
4825         my $conf = PVE::QemuConfig->load_config($vmid);
4826
4827         # there is no qmp command, so we use the human monitor command
4828         vm_human_monitor_command($vmid, "sendkey $key");
4829     });
4830 }
4831
4832 sub vm_destroy {
4833     my ($storecfg, $vmid, $skiplock) = @_;
4834
4835     PVE::QemuConfig->lock_config($vmid, sub {
4836
4837         my $conf = PVE::QemuConfig->load_config($vmid);
4838
4839         if (!check_running($vmid)) {
4840             destroy_vm($storecfg, $vmid, undef, $skiplock);
4841         } else {
4842             die "VM $vmid is running - destroy failed\n";
4843         }
4844     });
4845 }
4846
4847 # pci helpers
4848
4849 sub file_write {
4850     my ($filename, $buf) = @_;
4851
4852     my $fh = IO::File->new($filename, "w");
4853     return undef if !$fh;
4854
4855     my $res = print $fh $buf;
4856
4857     $fh->close();
4858
4859     return $res;
4860 }
4861
4862 sub pci_device_info {
4863     my ($name) = @_;
4864
4865     my $res;
4866
4867     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
4868     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
4869
4870     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
4871     return undef if !defined($irq) || $irq !~ m/^\d+$/;
4872
4873     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
4874     return undef if !defined($vendor) || $vendor !~ s/^0x//;
4875
4876     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
4877     return undef if !defined($product) || $product !~ s/^0x//;
4878
4879     $res = {
4880         name => $name,
4881         vendor => $vendor,
4882         product => $product,
4883         domain => $domain,
4884         bus => $bus,
4885         slot => $slot,
4886         func => $func,
4887         irq => $irq,
4888         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
4889     };
4890
4891     return $res;
4892 }
4893
4894 sub pci_dev_reset {
4895     my ($dev) = @_;
4896
4897     my $name = $dev->{name};
4898
4899     my $fn = "$pcisysfs/devices/$name/reset";
4900
4901     return file_write($fn, "1");
4902 }
4903
4904 sub pci_dev_bind_to_vfio {
4905     my ($dev) = @_;
4906
4907     my $name = $dev->{name};
4908
4909     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4910
4911     if (!-d $vfio_basedir) {
4912         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4913     }
4914     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4915
4916     my $testdir = "$vfio_basedir/$name";
4917     return 1 if -d $testdir;
4918
4919     my $data = "$dev->{vendor} $dev->{product}";
4920     return undef if !file_write("$vfio_basedir/new_id", $data);
4921
4922     my $fn = "$pcisysfs/devices/$name/driver/unbind";
4923     if (!file_write($fn, $name)) {
4924         return undef if -f $fn;
4925     }
4926
4927     $fn = "$vfio_basedir/bind";
4928     if (! -d $testdir) {
4929         return undef if !file_write($fn, $name);
4930     }
4931
4932     return -d $testdir;
4933 }
4934
4935 sub pci_dev_group_bind_to_vfio {
4936     my ($pciid) = @_;
4937
4938     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4939
4940     if (!-d $vfio_basedir) {
4941         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4942     }
4943     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4944
4945     # get IOMMU group devices
4946     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
4947       my @devs = grep /^0000:/, readdir($D);
4948     closedir($D);
4949
4950     foreach my $pciid (@devs) {
4951         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
4952
4953         # pci bridges, switches or root ports are not supported
4954         # they have a pci_bus subdirectory so skip them
4955         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
4956
4957         my $info = pci_device_info($1);
4958         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
4959     }
4960
4961     return 1;
4962 }
4963
4964 # vzdump restore implementaion
4965
4966 sub tar_archive_read_firstfile {
4967     my $archive = shift;
4968
4969     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
4970
4971     # try to detect archive type first
4972     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
4973         die "unable to open file '$archive'\n";
4974     my $firstfile = <$fh>;
4975     kill 15, $pid;
4976     close $fh;
4977
4978     die "ERROR: archive contaions no data\n" if !$firstfile;
4979     chomp $firstfile;
4980
4981     return $firstfile;
4982 }
4983
4984 sub tar_restore_cleanup {
4985     my ($storecfg, $statfile) = @_;
4986
4987     print STDERR "starting cleanup\n";
4988
4989     if (my $fd = IO::File->new($statfile, "r")) {
4990         while (defined(my $line = <$fd>)) {
4991             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
4992                 my $volid = $2;
4993                 eval {
4994                     if ($volid =~ m|^/|) {
4995                         unlink $volid || die 'unlink failed\n';
4996                     } else {
4997                         PVE::Storage::vdisk_free($storecfg, $volid);
4998                     }
4999                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5000                 };
5001                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5002             } else {
5003                 print STDERR "unable to parse line in statfile - $line";
5004             }
5005         }
5006         $fd->close();
5007     }
5008 }
5009
5010 sub restore_archive {
5011     my ($archive, $vmid, $user, $opts) = @_;
5012
5013     my $format = $opts->{format};
5014     my $comp;
5015
5016     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5017         $format = 'tar' if !$format;
5018         $comp = 'gzip';
5019     } elsif ($archive =~ m/\.tar$/) {
5020         $format = 'tar' if !$format;
5021     } elsif ($archive =~ m/.tar.lzo$/) {
5022         $format = 'tar' if !$format;
5023         $comp = 'lzop';
5024     } elsif ($archive =~ m/\.vma$/) {
5025         $format = 'vma' if !$format;
5026     } elsif ($archive =~ m/\.vma\.gz$/) {
5027         $format = 'vma' if !$format;
5028         $comp = 'gzip';
5029     } elsif ($archive =~ m/\.vma\.lzo$/) {
5030         $format = 'vma' if !$format;
5031         $comp = 'lzop';
5032     } else {
5033         $format = 'vma' if !$format; # default
5034     }
5035
5036     # try to detect archive format
5037     if ($format eq 'tar') {
5038         return restore_tar_archive($archive, $vmid, $user, $opts);
5039     } else {
5040         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5041     }
5042 }
5043
5044 sub restore_update_config_line {
5045     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5046
5047     return if $line =~ m/^\#qmdump\#/;
5048     return if $line =~ m/^\#vzdump\#/;
5049     return if $line =~ m/^lock:/;
5050     return if $line =~ m/^unused\d+:/;
5051     return if $line =~ m/^parent:/;
5052     return if $line =~ m/^template:/; # restored VM is never a template
5053
5054     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5055     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5056         # try to convert old 1.X settings
5057         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5058         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5059             my ($model, $macaddr) = split(/\=/, $devconfig);
5060             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5061             my $net = {
5062                 model => $model,
5063                 bridge => "vmbr$ind",
5064                 macaddr => $macaddr,
5065             };
5066             my $netstr = print_net($net);
5067
5068             print $outfd "net$cookie->{netcount}: $netstr\n";
5069             $cookie->{netcount}++;
5070         }
5071     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5072         my ($id, $netstr) = ($1, $2);
5073         my $net = parse_net($netstr);
5074         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5075         $netstr = print_net($net);
5076         print $outfd "$id: $netstr\n";
5077     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5078         my $virtdev = $1;
5079         my $value = $3;
5080         my $di = parse_drive($virtdev, $value);
5081         if (defined($di->{backup}) && !$di->{backup}) {
5082             print $outfd "#$line";
5083         } elsif ($map->{$virtdev}) {
5084             delete $di->{format}; # format can change on restore
5085             $di->{file} = $map->{$virtdev};
5086             $value = print_drive($vmid, $di);
5087             print $outfd "$virtdev: $value\n";
5088         } else {
5089             print $outfd $line;
5090         }
5091     } else {
5092         print $outfd $line;
5093     }
5094 }
5095
5096 sub scan_volids {
5097     my ($cfg, $vmid) = @_;
5098
5099     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5100
5101     my $volid_hash = {};
5102     foreach my $storeid (keys %$info) {
5103         foreach my $item (@{$info->{$storeid}}) {
5104             next if !($item->{volid} && $item->{size});
5105             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5106             $volid_hash->{$item->{volid}} = $item;
5107         }
5108     }
5109
5110     return $volid_hash;
5111 }
5112
5113 sub is_volume_in_use {
5114     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5115
5116     my $path = PVE::Storage::path($storecfg, $volid);
5117
5118     my $scan_config = sub {
5119         my ($cref, $snapname) = @_;
5120
5121         foreach my $key (keys %$cref) {
5122             my $value = $cref->{$key};
5123             if (is_valid_drivename($key)) {
5124                 next if $skip_drive && $key eq $skip_drive;
5125                 my $drive = parse_drive($key, $value);
5126                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5127                 return 1 if $volid eq $drive->{file};
5128                 if ($drive->{file} =~ m!^/!) {
5129                     return 1 if $drive->{file} eq $path;
5130                 } else {
5131                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5132                     next if !$storeid;
5133                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5134                     next if !$scfg;
5135                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5136                 }
5137             }
5138         }
5139
5140         return 0;
5141     };
5142
5143     return 1 if &$scan_config($conf);
5144
5145     undef $skip_drive;
5146
5147     foreach my $snapname (keys %{$conf->{snapshots}}) {
5148         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5149     }
5150
5151     return 0;
5152 }
5153
5154 sub update_disksize {
5155     my ($vmid, $conf, $volid_hash) = @_;
5156
5157     my $changes;
5158
5159     my $used = {};
5160
5161     # Note: it is allowed to define multiple storages with same path (alias), so
5162     # we need to check both 'volid' and real 'path' (two different volid can point
5163     # to the same path).
5164
5165     my $usedpath = {};
5166
5167     # update size info
5168     foreach my $opt (keys %$conf) {
5169         if (is_valid_drivename($opt)) {
5170             my $drive = parse_drive($opt, $conf->{$opt});
5171             my $volid = $drive->{file};
5172             next if !$volid;
5173
5174             $used->{$volid} = 1;
5175             if ($volid_hash->{$volid} &&
5176                 (my $path = $volid_hash->{$volid}->{path})) {
5177                 $usedpath->{$path} = 1;
5178             }
5179
5180             next if drive_is_cdrom($drive);
5181             next if !$volid_hash->{$volid};
5182
5183             $drive->{size} = $volid_hash->{$volid}->{size};
5184             my $new = print_drive($vmid, $drive);
5185             if ($new ne $conf->{$opt}) {
5186                 $changes = 1;
5187                 $conf->{$opt} = $new;
5188             }
5189         }
5190     }
5191
5192     # remove 'unusedX' entry if volume is used
5193     foreach my $opt (keys %$conf) {
5194         next if $opt !~ m/^unused\d+$/;
5195         my $volid = $conf->{$opt};
5196         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5197         if ($used->{$volid} || ($path && $usedpath->{$path})) {
5198             $changes = 1;
5199             delete $conf->{$opt};
5200         }
5201     }
5202
5203     foreach my $volid (sort keys %$volid_hash) {
5204         next if $volid =~ m/vm-$vmid-state-/;
5205         next if $used->{$volid};
5206         my $path = $volid_hash->{$volid}->{path};
5207         next if !$path; # just to be sure
5208         next if $usedpath->{$path};
5209         $changes = 1;
5210         PVE::QemuConfig->add_unused_volume($conf, $volid);
5211         $usedpath->{$path} = 1; # avoid to add more than once (aliases)
5212     }
5213
5214     return $changes;
5215 }
5216
5217 sub rescan {
5218     my ($vmid, $nolock) = @_;
5219
5220     my $cfg = PVE::Storage::config();
5221
5222     my $volid_hash = scan_volids($cfg, $vmid);
5223
5224     my $updatefn =  sub {
5225         my ($vmid) = @_;
5226
5227         my $conf = PVE::QemuConfig->load_config($vmid);
5228
5229         PVE::QemuConfig->check_lock($conf);
5230
5231         my $vm_volids = {};
5232         foreach my $volid (keys %$volid_hash) {
5233             my $info = $volid_hash->{$volid};
5234             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5235         }
5236
5237         my $changes = update_disksize($vmid, $conf, $vm_volids);
5238
5239         PVE::QemuConfig->write_config($vmid, $conf) if $changes;
5240     };
5241
5242     if (defined($vmid)) {
5243         if ($nolock) {
5244             &$updatefn($vmid);
5245         } else {
5246             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5247         }
5248     } else {
5249         my $vmlist = config_list();
5250         foreach my $vmid (keys %$vmlist) {
5251             if ($nolock) {
5252                 &$updatefn($vmid);
5253             } else {
5254                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5255             }
5256         }
5257     }
5258 }
5259
5260 sub restore_vma_archive {
5261     my ($archive, $vmid, $user, $opts, $comp) = @_;
5262
5263     my $input = $archive eq '-' ? "<&STDIN" : undef;
5264     my $readfrom = $archive;
5265
5266     my $uncomp = '';
5267     if ($comp) {
5268         $readfrom = '-';
5269         my $qarchive = PVE::Tools::shellquote($archive);
5270         if ($comp eq 'gzip') {
5271             $uncomp = "zcat $qarchive|";
5272         } elsif ($comp eq 'lzop') {
5273             $uncomp = "lzop -d -c $qarchive|";
5274         } else {
5275             die "unknown compression method '$comp'\n";
5276         }
5277
5278     }
5279
5280     my $tmpdir = "/var/tmp/vzdumptmp$$";
5281     rmtree $tmpdir;
5282
5283     # disable interrupts (always do cleanups)
5284     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5285         warn "got interrupt - ignored\n";
5286     };
5287
5288     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5289     POSIX::mkfifo($mapfifo, 0600);
5290     my $fifofh;
5291
5292     my $openfifo = sub {
5293         open($fifofh, '>', $mapfifo) || die $!;
5294     };
5295
5296     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5297
5298     my $oldtimeout;
5299     my $timeout = 5;
5300
5301     my $devinfo = {};
5302
5303     my $rpcenv = PVE::RPCEnvironment::get();
5304
5305     my $conffile = PVE::QemuConfig->config_file($vmid);
5306     my $tmpfn = "$conffile.$$.tmp";
5307
5308     # Note: $oldconf is undef if VM does not exists
5309     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
5310     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
5311
5312     my $print_devmap = sub {
5313         my $virtdev_hash = {};
5314
5315         my $cfgfn = "$tmpdir/qemu-server.conf";
5316
5317         # we can read the config - that is already extracted
5318         my $fh = IO::File->new($cfgfn, "r") ||
5319             "unable to read qemu-server.conf - $!\n";
5320
5321         my $fwcfgfn = "$tmpdir/qemu-server.fw";
5322         if (-f $fwcfgfn) {
5323             my $pve_firewall_dir = '/etc/pve/firewall';
5324             mkdir $pve_firewall_dir; # make sure the dir exists
5325             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
5326         }
5327
5328         while (defined(my $line = <$fh>)) {
5329             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5330                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5331                 die "archive does not contain data for drive '$virtdev'\n"
5332                     if !$devinfo->{$devname};
5333                 if (defined($opts->{storage})) {
5334                     $storeid = $opts->{storage} || 'local';
5335                 } elsif (!$storeid) {
5336                     $storeid = 'local';
5337                 }
5338                 $format = 'raw' if !$format;
5339                 $devinfo->{$devname}->{devname} = $devname;
5340                 $devinfo->{$devname}->{virtdev} = $virtdev;
5341                 $devinfo->{$devname}->{format} = $format;
5342                 $devinfo->{$devname}->{storeid} = $storeid;
5343
5344                 # check permission on storage
5345                 my $pool = $opts->{pool}; # todo: do we need that?
5346                 if ($user ne 'root@pam') {
5347                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5348                 }
5349
5350                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5351             }
5352         }
5353
5354         foreach my $devname (keys %$devinfo) {
5355             die "found no device mapping information for device '$devname'\n"
5356                 if !$devinfo->{$devname}->{virtdev};
5357         }
5358
5359         my $cfg = PVE::Storage::config();
5360
5361         # create empty/temp config
5362         if ($oldconf) {
5363             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5364             foreach_drive($oldconf, sub {
5365                 my ($ds, $drive) = @_;
5366
5367                 return if drive_is_cdrom($drive);
5368
5369                 my $volid = $drive->{file};
5370
5371                 return if !$volid || $volid =~ m|^/|;
5372
5373                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5374                 return if !$path || !$owner || ($owner != $vmid);
5375
5376                 # Note: only delete disk we want to restore
5377                 # other volumes will become unused
5378                 if ($virtdev_hash->{$ds}) {
5379                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
5380                     if (my $err = $@) {
5381                         warn $err;
5382                     }
5383                 }
5384             });
5385
5386             # delete vmstate files
5387             # since after the restore we have no snapshots anymore
5388             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5389                 my $snap = $oldconf->{snapshots}->{$snapname};
5390                 if ($snap->{vmstate}) {
5391                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
5392                     if (my $err = $@) {
5393                         warn $err;
5394                     }
5395                 }
5396             }
5397         }
5398
5399         my $map = {};
5400         foreach my $virtdev (sort keys %$virtdev_hash) {
5401             my $d = $virtdev_hash->{$virtdev};
5402             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5403             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5404
5405             # test if requested format is supported
5406             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5407             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5408             $d->{format} = $defFormat if !$supported;
5409
5410             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5411                                                   $d->{format}, undef, $alloc_size);
5412             print STDERR "new volume ID is '$volid'\n";
5413             $d->{volid} = $volid;
5414             my $path = PVE::Storage::path($cfg, $volid);
5415
5416             PVE::Storage::activate_volumes($cfg,[$volid]);
5417
5418             my $write_zeros = 1;
5419             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
5420                 $write_zeros = 0;
5421             }
5422
5423             print $fifofh "format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
5424
5425             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5426             $map->{$virtdev} = $volid;
5427         }
5428
5429         $fh->seek(0, 0) || die "seek failed - $!\n";
5430
5431         my $outfd = new IO::File ($tmpfn, "w") ||
5432             die "unable to write config for VM $vmid\n";
5433
5434         my $cookie = { netcount => 0 };
5435         while (defined(my $line = <$fh>)) {
5436             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5437         }
5438
5439         $fh->close();
5440         $outfd->close();
5441     };
5442
5443     eval {
5444         # enable interrupts
5445         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5446             die "interrupted by signal\n";
5447         };
5448         local $SIG{ALRM} = sub { die "got timeout\n"; };
5449
5450         $oldtimeout = alarm($timeout);
5451
5452         my $parser = sub {
5453             my $line = shift;
5454
5455             print "$line\n";
5456
5457             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5458                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5459                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5460             } elsif ($line =~ m/^CTIME: /) {
5461                 # we correctly received the vma config, so we can disable
5462                 # the timeout now for disk allocation (set to 10 minutes, so
5463                 # that we always timeout if something goes wrong)
5464                 alarm(600);
5465                 &$print_devmap();
5466                 print $fifofh "done\n";
5467                 my $tmp = $oldtimeout || 0;
5468                 $oldtimeout = undef;
5469                 alarm($tmp);
5470                 close($fifofh);
5471             }
5472         };
5473
5474         print "restore vma archive: $cmd\n";
5475         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5476     };
5477     my $err = $@;
5478
5479     alarm($oldtimeout) if $oldtimeout;
5480
5481     my $vollist = [];
5482     foreach my $devname (keys %$devinfo) {
5483         my $volid = $devinfo->{$devname}->{volid};
5484         push @$vollist, $volid if $volid;
5485     }
5486
5487     my $cfg = PVE::Storage::config();
5488     PVE::Storage::deactivate_volumes($cfg, $vollist);
5489
5490     unlink $mapfifo;
5491
5492     if ($err) {
5493         rmtree $tmpdir;
5494         unlink $tmpfn;
5495
5496         foreach my $devname (keys %$devinfo) {
5497             my $volid = $devinfo->{$devname}->{volid};
5498             next if !$volid;
5499             eval {
5500                 if ($volid =~ m|^/|) {
5501                     unlink $volid || die 'unlink failed\n';
5502                 } else {
5503                     PVE::Storage::vdisk_free($cfg, $volid);
5504                 }
5505                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5506             };
5507             print STDERR "unable to cleanup '$volid' - $@" if $@;
5508         }
5509         die $err;
5510     }
5511
5512     rmtree $tmpdir;
5513
5514     rename($tmpfn, $conffile) ||
5515         die "unable to commit configuration file '$conffile'\n";
5516
5517     PVE::Cluster::cfs_update(); # make sure we read new file
5518
5519     eval { rescan($vmid, 1); };
5520     warn $@ if $@;
5521 }
5522
5523 sub restore_tar_archive {
5524     my ($archive, $vmid, $user, $opts) = @_;
5525
5526     if ($archive ne '-') {
5527         my $firstfile = tar_archive_read_firstfile($archive);
5528         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5529             if $firstfile ne 'qemu-server.conf';
5530     }
5531
5532     my $storecfg = PVE::Storage::config();
5533
5534     # destroy existing data - keep empty config
5535     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
5536     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5537
5538     my $tocmd = "/usr/lib/qemu-server/qmextract";
5539
5540     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5541     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5542     $tocmd .= ' --prealloc' if $opts->{prealloc};
5543     $tocmd .= ' --info' if $opts->{info};
5544
5545     # tar option "xf" does not autodetect compression when read from STDIN,
5546     # so we pipe to zcat
5547     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5548         PVE::Tools::shellquote("--to-command=$tocmd");
5549
5550     my $tmpdir = "/var/tmp/vzdumptmp$$";
5551     mkpath $tmpdir;
5552
5553     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5554     local $ENV{VZDUMP_VMID} = $vmid;
5555     local $ENV{VZDUMP_USER} = $user;
5556
5557     my $conffile = PVE::QemuConfig->config_file($vmid);
5558     my $tmpfn = "$conffile.$$.tmp";
5559
5560     # disable interrupts (always do cleanups)
5561     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5562         print STDERR "got interrupt - ignored\n";
5563     };
5564
5565     eval {
5566         # enable interrupts
5567         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5568             die "interrupted by signal\n";
5569         };
5570
5571         if ($archive eq '-') {
5572             print "extracting archive from STDIN\n";
5573             run_command($cmd, input => "<&STDIN");
5574         } else {
5575             print "extracting archive '$archive'\n";
5576             run_command($cmd);
5577         }
5578
5579         return if $opts->{info};
5580
5581         # read new mapping
5582         my $map = {};
5583         my $statfile = "$tmpdir/qmrestore.stat";
5584         if (my $fd = IO::File->new($statfile, "r")) {
5585             while (defined (my $line = <$fd>)) {
5586                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5587                     $map->{$1} = $2 if $1;
5588                 } else {
5589                     print STDERR "unable to parse line in statfile - $line\n";
5590                 }
5591             }
5592             $fd->close();
5593         }
5594
5595         my $confsrc = "$tmpdir/qemu-server.conf";
5596
5597         my $srcfd = new IO::File($confsrc, "r") ||
5598             die "unable to open file '$confsrc'\n";
5599
5600         my $outfd = new IO::File ($tmpfn, "w") ||
5601             die "unable to write config for VM $vmid\n";
5602
5603         my $cookie = { netcount => 0 };
5604         while (defined (my $line = <$srcfd>)) {
5605             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5606         }
5607
5608         $srcfd->close();
5609         $outfd->close();
5610     };
5611     my $err = $@;
5612
5613     if ($err) {
5614
5615         unlink $tmpfn;
5616
5617         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
5618
5619         die $err;
5620     }
5621
5622     rmtree $tmpdir;
5623
5624     rename $tmpfn, $conffile ||
5625         die "unable to commit configuration file '$conffile'\n";
5626
5627     PVE::Cluster::cfs_update(); # make sure we read new file
5628
5629     eval { rescan($vmid, 1); };
5630     warn $@ if $@;
5631 };
5632
5633 sub foreach_writable_storage {
5634     my ($conf, $func) = @_;
5635
5636     my $sidhash = {};
5637
5638     foreach my $ds (keys %$conf) {
5639         next if !is_valid_drivename($ds);
5640
5641         my $drive = parse_drive($ds, $conf->{$ds});
5642         next if !$drive;
5643         next if drive_is_cdrom($drive);
5644
5645         my $volid = $drive->{file};
5646
5647         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5648         $sidhash->{$sid} = $sid if $sid;
5649     }
5650
5651     foreach my $sid (sort keys %$sidhash) {
5652         &$func($sid);
5653     }
5654 }
5655
5656 sub do_snapshots_with_qemu {
5657     my ($storecfg, $volid) = @_;
5658
5659     my $storage_name = PVE::Storage::parse_volume_id($volid);
5660
5661     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
5662         && !$storecfg->{ids}->{$storage_name}->{krbd}){
5663         return 1;
5664     }
5665
5666     if ($volid =~ m/\.(qcow2|qed)$/){
5667         return 1;
5668     }
5669
5670     return undef;
5671 }
5672
5673 sub qga_check_running {
5674     my ($vmid) = @_;
5675
5676     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
5677     if ($@) {
5678         warn "Qemu Guest Agent are not running - $@";
5679         return 0;
5680     }
5681     return 1;
5682 }
5683
5684 sub template_create {
5685     my ($vmid, $conf, $disk) = @_;
5686
5687     my $storecfg = PVE::Storage::config();
5688
5689     foreach_drive($conf, sub {
5690         my ($ds, $drive) = @_;
5691
5692         return if drive_is_cdrom($drive);
5693         return if $disk && $ds ne $disk;
5694
5695         my $volid = $drive->{file};
5696         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
5697
5698         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
5699         $drive->{file} = $voliddst;
5700         $conf->{$ds} = print_drive($vmid, $drive);
5701         PVE::QemuConfig->write_config($vmid, $conf);
5702     });
5703 }
5704
5705 sub qemu_img_convert {
5706     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
5707
5708     my $storecfg = PVE::Storage::config();
5709     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
5710     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
5711
5712     if ($src_storeid && $dst_storeid) {
5713
5714         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
5715
5716         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
5717         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5718
5719         my $src_format = qemu_img_format($src_scfg, $src_volname);
5720         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
5721
5722         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
5723         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5724
5725         my $cmd = [];
5726         push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
5727         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
5728         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path;
5729         if ($is_zero_initialized) {
5730             push @$cmd, "zeroinit:$dst_path";
5731         } else {
5732             push @$cmd, $dst_path;
5733         }
5734
5735         my $parser = sub {
5736             my $line = shift;
5737             if($line =~ m/\((\S+)\/100\%\)/){
5738                 my $percent = $1;
5739                 my $transferred = int($size * $percent / 100);
5740                 my $remaining = $size - $transferred;
5741
5742                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
5743             }
5744
5745         };
5746
5747         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
5748         my $err = $@;
5749         die "copy failed: $err" if $err;
5750     }
5751 }
5752
5753 sub qemu_img_format {
5754     my ($scfg, $volname) = @_;
5755
5756     if ($scfg->{path} && $volname =~ m/\.(raw|cow|qcow|qcow2|qed|vmdk|cloop)$/) {
5757         return $1;
5758     } else {
5759         return "raw";
5760     }
5761 }
5762
5763 sub qemu_drive_mirror {
5764     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized) = @_;
5765
5766     my $storecfg = PVE::Storage::config();
5767     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
5768
5769     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5770
5771     my $format = qemu_img_format($dst_scfg, $dst_volname);
5772
5773     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5774
5775     my $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
5776
5777     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
5778     $opts->{format} = $format if $format;
5779
5780     print "drive mirror is starting (scanning bitmap) : this step can take some minutes/hours, depend of disk size and storage speed\n";
5781
5782     my $finish_job = sub {
5783         while (1) {
5784             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
5785             my $stat = @$stats[0];
5786             last if !$stat;
5787             sleep 1;
5788         }
5789     };
5790
5791     eval {
5792     vm_mon_cmd($vmid, "drive-mirror", %$opts);
5793         while (1) {
5794             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
5795             my $stat = @$stats[0];
5796             die "mirroring job seem to have die. Maybe do you have bad sectors?" if !$stat;
5797             die "error job is not mirroring" if $stat->{type} ne "mirror";
5798
5799             my $busy = $stat->{busy};
5800             my $ready = $stat->{ready};
5801
5802             if (my $total = $stat->{len}) {
5803                 my $transferred = $stat->{offset} || 0;
5804                 my $remaining = $total - $transferred;
5805                 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
5806
5807                 print "transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
5808             }
5809
5810
5811             if ($stat->{ready} eq 'true') {
5812
5813                 last if $vmiddst != $vmid;
5814
5815                 # try to switch the disk if source and destination are on the same guest
5816                 eval { vm_mon_cmd($vmid, "block-job-complete", device => "drive-$drive") };
5817                 if (!$@) {
5818                     &$finish_job();
5819                     last;
5820                 }
5821                 die $@ if $@ !~ m/cannot be completed/;
5822             }
5823             sleep 1;
5824         }
5825
5826
5827     };
5828     my $err = $@;
5829
5830     my $cancel_job = sub {
5831         vm_mon_cmd($vmid, "block-job-cancel", device => "drive-$drive");
5832         &$finish_job();
5833     };
5834
5835     if ($err) {
5836         eval { &$cancel_job(); };
5837         die "mirroring error: $err";
5838     }
5839
5840     if ($vmiddst != $vmid) {
5841         # if we clone a disk for a new target vm, we don't switch the disk
5842         &$cancel_job(); # so we call block-job-cancel
5843     }
5844 }
5845
5846 sub clone_disk {
5847     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
5848         $newvmid, $storage, $format, $full, $newvollist) = @_;
5849
5850     my $newvolid;
5851
5852     if (!$full) {
5853         print "create linked clone of drive $drivename ($drive->{file})\n";
5854         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
5855         push @$newvollist, $newvolid;
5856     } else {
5857         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
5858         $storeid = $storage if $storage;
5859
5860         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5861         if (!$format) {
5862             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5863             $format = qemu_img_format($scfg, $volname);
5864         }
5865
5866         # test if requested format is supported - else use default
5867         my $supported = grep { $_ eq $format } @$validFormats;
5868         $format = $defFormat if !$supported;
5869
5870         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
5871
5872         print "create full clone of drive $drivename ($drive->{file})\n";
5873         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $format, undef, ($size/1024));
5874         push @$newvollist, $newvolid;
5875
5876         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
5877
5878         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
5879         if (!$running || $snapname) {
5880             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
5881         } else {
5882             #qemu 2.6
5883             die "drive-mirror is not working currently when iothread is enabled" if $drive->{iothread};
5884
5885             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit);
5886         }
5887     }
5888
5889     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
5890
5891     my $disk = $drive;
5892     $disk->{format} = undef;
5893     $disk->{file} = $newvolid;
5894     $disk->{size} = $size;
5895
5896     return $disk;
5897 }
5898
5899 # this only works if VM is running
5900 sub get_current_qemu_machine {
5901     my ($vmid) = @_;
5902
5903     my $cmd = { execute => 'query-machines', arguments => {} };
5904     my $res = vm_qmp_command($vmid, $cmd);
5905
5906     my ($current, $default);
5907     foreach my $e (@$res) {
5908         $default = $e->{name} if $e->{'is-default'};
5909         $current = $e->{name} if $e->{'is-current'};
5910     }
5911
5912     # fallback to the default machine if current is not supported by qemu
5913     return $current || $default || 'pc';
5914 }
5915
5916 sub qemu_machine_feature_enabled {
5917     my ($machine, $kvmver, $version_major, $version_minor) = @_;
5918
5919     my $current_major;
5920     my $current_minor;
5921
5922     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
5923
5924         $current_major = $3;
5925         $current_minor = $4;
5926
5927     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
5928
5929         $current_major = $1;
5930         $current_minor = $2;
5931     }
5932
5933     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
5934
5935
5936 }
5937
5938 sub qemu_machine_pxe {
5939     my ($vmid, $conf, $machine) = @_;
5940
5941     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
5942
5943     foreach my $opt (keys %$conf) {
5944         next if $opt !~ m/^net(\d+)$/;
5945         my $net = PVE::QemuServer::parse_net($conf->{$opt});
5946         next if !$net;
5947         my $romfile = PVE::QemuServer::vm_mon_cmd_nocheck($vmid, 'qom-get', path => $opt, property => 'romfile');
5948         return $machine.".pxe" if $romfile =~ m/pxe/;
5949         last;
5950     }
5951
5952     return $machine;
5953 }
5954
5955 sub qemu_use_old_bios_files {
5956     my ($machine_type) = @_;
5957
5958     return if !$machine_type;
5959
5960     my $use_old_bios_files = undef;
5961
5962     if ($machine_type =~ m/^(\S+)\.pxe$/) {
5963         $machine_type = $1;
5964         $use_old_bios_files = 1;
5965     } else {
5966         my $kvmver = kvm_user_version();
5967         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
5968         # load new efi bios files on migration. So this hack is required to allow
5969         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
5970         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
5971         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
5972     }
5973
5974     return ($use_old_bios_files, $machine_type);
5975 }
5976
5977 sub lspci {
5978
5979     my $devices = {};
5980
5981     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
5982             my (undef, $id, $function) = @_;
5983             my $res = { id => $id, function => $function};
5984             push @{$devices->{$id}}, $res;
5985     });
5986
5987     return $devices;
5988 }
5989
5990 sub vm_iothreads_list {
5991     my ($vmid) = @_;
5992
5993     my $res = vm_mon_cmd($vmid, 'query-iothreads');
5994
5995     my $iothreads = {};
5996     foreach my $iothread (@$res) {
5997         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
5998     }
5999
6000     return $iothreads;
6001 }
6002
6003 sub scsihw_infos {
6004     my ($conf, $drive) = @_;
6005
6006     my $maxdev = 0;
6007
6008     if ($conf->{scsihw} && ($conf->{scsihw} =~ m/^lsi/)) {
6009         $maxdev = 7;
6010     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6011         $maxdev = 1;
6012     } else {
6013         $maxdev = 256;
6014     }
6015
6016     my $controller = int($drive->{index} / $maxdev);
6017     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6018
6019     return ($maxdev, $controller, $controller_prefix);
6020 }
6021
6022 # bash completion helper
6023
6024 sub complete_backup_archives {
6025     my ($cmdname, $pname, $cvalue) = @_;
6026
6027     my $cfg = PVE::Storage::config();
6028
6029     my $storeid;
6030
6031     if ($cvalue =~ m/^([^:]+):/) {
6032         $storeid = $1;
6033     }
6034
6035     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6036
6037     my $res = [];
6038     foreach my $id (keys %$data) {
6039         foreach my $item (@{$data->{$id}}) {
6040             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6041             push @$res, $item->{volid} if defined($item->{volid});
6042         }
6043     }
6044
6045     return $res;
6046 }
6047
6048 my $complete_vmid_full = sub {
6049     my ($running) = @_;
6050
6051     my $idlist = vmstatus();
6052
6053     my $res = [];
6054
6055     foreach my $id (keys %$idlist) {
6056         my $d = $idlist->{$id};
6057         if (defined($running)) {
6058             next if $d->{template};
6059             next if $running && $d->{status} ne 'running';
6060             next if !$running && $d->{status} eq 'running';
6061         }
6062         push @$res, $id;
6063
6064     }
6065     return $res;
6066 };
6067
6068 sub complete_vmid {
6069     return &$complete_vmid_full();
6070 }
6071
6072 sub complete_vmid_stopped {
6073     return &$complete_vmid_full(0);
6074 }
6075
6076 sub complete_vmid_running {
6077     return &$complete_vmid_full(1);
6078 }
6079
6080 sub complete_storage {
6081
6082     my $cfg = PVE::Storage::config();
6083     my $ids = $cfg->{ids};
6084
6085     my $res = [];
6086     foreach my $sid (keys %$ids) {
6087         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6088         next if !$ids->{$sid}->{content}->{images};
6089         push @$res, $sid;
6090     }
6091
6092     return $res;
6093 }
6094
6095 1;