PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QMPClient;
  31 use PVE::RPCEnvironment;
  32 use Time::HiRes qw(gettimeofday);
  33
  34 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  35
  36 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  37
  38 # Note about locking: we use flock on the config file protect
  39 # against concurent actions.
  40 # Aditionaly, we have a 'lock' setting in the config file. This
  41 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  42 # allowed when such lock is set. But you can ignore this kind of
  43 # lock with the --skiplock flag.
  44
  45 cfs_register_file('/qemu-server/',
  46                   \&parse_vm_config,
  47                   \&write_vm_config);
  48
  49 PVE::JSONSchema::register_standard_option('skiplock', {
  50     description => "Ignore locks - only root is allowed to use this option.",
  51     type => 'boolean',
  52     optional => 1,
  53 });
  54
  55 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  56     description => "Some command save/restore state from this location.",
  57     type => 'string',
  58     maxLength => 128,
  59     optional => 1,
  60 });
  61
  62 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  63     description => "The name of the snapshot.",
  64     type => 'string', format => 'pve-configid',
  65     maxLength => 40,
  66 });
  67
  68 #no warnings 'redefine';
  69
  70 sub cgroups_write {
  71    my ($controller, $vmid, $option, $value) = @_;
  72
  73    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  74    PVE::ProcFSTools::write_proc_entry($path, $value);
  75
  76 }
  77
  78 my $nodename = PVE::INotify::nodename();
  79
  80 mkdir "/etc/pve/nodes/$nodename";
  81 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  82 mkdir $confdir;
  83
  84 my $var_run_tmpdir = "/var/run/qemu-server";
  85 mkdir $var_run_tmpdir;
  86
  87 my $lock_dir = "/var/lock/qemu-server";
  88 mkdir $lock_dir;
  89
  90 my $pcisysfs = "/sys/bus/pci";
  91
  92 my $confdesc = {
  93     onboot => {
  94         optional => 1,
  95         type => 'boolean',
  96         description => "Specifies whether a VM will be started during system bootup.",
  97         default => 0,
  98     },
  99     autostart => {
 100         optional => 1,
 101         type => 'boolean',
 102         description => "Automatic restart after crash (currently ignored).",
 103         default => 0,
 104     },
 105     hotplug => {
 106         optional => 1,
 107         type => 'string', format => 'pve-hotplug-features',
 108         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 109         default => 'network,disk,usb',
 110     },
 111     reboot => {
 112         optional => 1,
 113         type => 'boolean',
 114         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 115         default => 1,
 116     },
 117     lock => {
 118         optional => 1,
 119         type => 'string',
 120         description => "Lock/unlock the VM.",
 121         enum => [qw(migrate backup snapshot rollback)],
 122     },
 123     cpulimit => {
 124         optional => 1,
 125         type => 'number',
 126         description => "Limit of CPU usage. Note if the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 127         minimum => 0,
 128         maximum => 128,
 129         default => 0,
 130     },
 131     cpuunits => {
 132         optional => 1,
 133         type => 'integer',
 134         description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 135         minimum => 0,
 136         maximum => 500000,
 137         default => 1000,
 138     },
 139     memory => {
 140         optional => 1,
 141         type => 'integer',
 142         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 143         minimum => 16,
 144         default => 512,
 145     },
 146     balloon => {
 147         optional => 1,
 148         type => 'integer',
 149         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 150         minimum => 0,
 151     },
 152     shares => {
 153         optional => 1,
 154         type => 'integer',
 155         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 156         minimum => 0,
 157         maximum => 50000,
 158         default => 1000,
 159     },
 160     keyboard => {
 161         optional => 1,
 162         type => 'string',
 163         description => "Keybord layout for vnc server. Default is read from the datacenter configuration file.",
 164         enum => PVE::Tools::kvmkeymaplist(),
 165         default => 'en-us',
 166     },
 167     name => {
 168         optional => 1,
 169         type => 'string', format => 'dns-name',
 170         description => "Set a name for the VM. Only used on the configuration web interface.",
 171     },
 172     scsihw => {
 173         optional => 1,
 174         type => 'string',
 175         description => "scsi controller model",
 176         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 177         default => 'lsi',
 178     },
 179     description => {
 180         optional => 1,
 181         type => 'string',
 182         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 183     },
 184     ostype => {
 185         optional => 1,
 186         type => 'string',
 187         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 l24 l26 solaris)],
 188         description => <<EODESC,
 189 Used to enable special optimization/features for specific
 190 operating systems:
 191
 192 other  => unspecified OS
 193 wxp    => Microsoft Windows XP
 194 w2k    => Microsoft Windows 2000
 195 w2k3   => Microsoft Windows 2003
 196 w2k8   => Microsoft Windows 2008
 197 wvista => Microsoft Windows Vista
 198 win7   => Microsoft Windows 7
 199 win8   => Microsoft Windows 8/2012
 200 l24    => Linux 2.4 Kernel
 201 l26    => Linux 2.6/3.X Kernel
 202 solaris => solaris/opensolaris/openindiania kernel
 203
 204 other|l24|l26|solaris                       ... no special behaviour
 205 wxp|w2k|w2k3|w2k8|wvista|win7|win8  ... use --localtime switch
 206 EODESC
 207     },
 208     boot => {
 209         optional => 1,
 210         type => 'string',
 211         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 212         pattern => '[acdn]{1,4}',
 213         default => 'cdn',
 214     },
 215     bootdisk => {
 216         optional => 1,
 217         type => 'string', format => 'pve-qm-bootdisk',
 218         description => "Enable booting from specified disk.",
 219         pattern => '(ide|sata|scsi|virtio)\d+',
 220     },
 221     smp => {
 222         optional => 1,
 223         type => 'integer',
 224         description => "The number of CPUs. Please use option -sockets instead.",
 225         minimum => 1,
 226         default => 1,
 227     },
 228     sockets => {
 229         optional => 1,
 230         type => 'integer',
 231         description => "The number of CPU sockets.",
 232         minimum => 1,
 233         default => 1,
 234     },
 235     cores => {
 236         optional => 1,
 237         type => 'integer',
 238         description => "The number of cores per socket.",
 239         minimum => 1,
 240         default => 1,
 241     },
 242     numa => {
 243         optional => 1,
 244         type => 'boolean',
 245         description => "Enable/disable Numa.",
 246         default => 0,
 247     },
 248     vcpus => {
 249         optional => 1,
 250         type => 'integer',
 251         description => "Number of hotplugged vcpus.",
 252         minimum => 1,
 253         default => 0,
 254     },
 255     acpi => {
 256         optional => 1,
 257         type => 'boolean',
 258         description => "Enable/disable ACPI.",
 259         default => 1,
 260     },
 261     agent => {
 262         optional => 1,
 263         type => 'boolean',
 264         description => "Enable/disable Qemu GuestAgent.",
 265         default => 0,
 266     },
 267     kvm => {
 268         optional => 1,
 269         type => 'boolean',
 270         description => "Enable/disable KVM hardware virtualization.",
 271         default => 1,
 272     },
 273     tdf => {
 274         optional => 1,
 275         type => 'boolean',
 276         description => "Enable/disable time drift fix.",
 277         default => 0,
 278     },
 279     localtime => {
 280         optional => 1,
 281         type => 'boolean',
 282         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 283     },
 284     freeze => {
 285         optional => 1,
 286         type => 'boolean',
 287         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 288     },
 289     vga => {
 290         optional => 1,
 291         type => 'string',
 292         description => "Select VGA type. If you want to use high resolution modes (>= 1280x1024x16) then you should use option 'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and 'cirrur' for other OS types. Option 'qxl' enables the SPICE display sever. You can also run without any graphic card using a serial devive as terminal.",
 293         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 294     },
 295     watchdog => {
 296         optional => 1,
 297         type => 'string', format => 'pve-qm-watchdog',
 298         typetext => '[[model=]i6300esb|ib700] [,[action=]reset|shutdown|poweroff|pause|debug|none]',
 299         description => "Create a virtual hardware watchdog device.  Once enabled (by a guest action), the watchdog must be periodically polled by an agent inside the guest or else the guest will be restarted (or execute the action specified)",
 300     },
 301     startdate => {
 302         optional => 1,
 303         type => 'string',
 304         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 305         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 306         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 307         default => 'now',
 308     },
 309     startup =>  get_standard_option('pve-startup-order'),
 310     template => {
 311         optional => 1,
 312         type => 'boolean',
 313         description => "Enable/disable Template.",
 314         default => 0,
 315     },
 316     args => {
 317         optional => 1,
 318         type => 'string',
 319         description => <<EODESCR,
 320 Note: this option is for experts only. It allows you to pass arbitrary arguments to kvm, for example:
 321
 322 args: -no-reboot -no-hpet
 323 EODESCR
 324     },
 325     tablet => {
 326         optional => 1,
 327         type => 'boolean',
 328         default => 1,
 329         description => "Enable/disable the usb tablet device. This device is usually needed to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with normal VNC clients. If you're running lots of console-only guests on one host, you may consider disabling this to save some context switches. This is turned of by default if you use spice (vga=qxl).",
 330     },
 331     migrate_speed => {
 332         optional => 1,
 333         type => 'integer',
 334         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 335         minimum => 0,
 336         default => 0,
 337     },
 338     migrate_downtime => {
 339         optional => 1,
 340         type => 'number',
 341         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 342         minimum => 0,
 343         default => 0.1,
 344     },
 345     cdrom => {
 346         optional => 1,
 347         type => 'string', format => 'pve-qm-drive',
 348         typetext => 'volume',
 349         description => "This is an alias for option -ide2",
 350     },
 351     cpu => {
 352         optional => 1,
 353         description => "Emulated CPU type.",
 354         type => 'string',
 355         enum => [ qw(486 athlon pentium pentium2 pentium3 coreduo core2duo kvm32 kvm64 qemu32 qemu64 phenom Conroe Penryn Nehalem Westmere SandyBridge IvyBridge Haswell Broadwell Opteron_G1 Opteron_G2 Opteron_G3 Opteron_G4 Opteron_G5 host) ],
 356         default => 'kvm64',
 357     },
 358     parent => get_standard_option('pve-snapshot-name', {
 359         optional => 1,
 360         description => "Parent snapshot name. This is used internally, and should not be modified.",
 361     }),
 362     snaptime => {
 363         optional => 1,
 364         description => "Timestamp for snapshots.",
 365         type => 'integer',
 366         minimum => 0,
 367     },
 368     vmstate => {
 369         optional => 1,
 370         type => 'string', format => 'pve-volume-id',
 371         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 372     },
 373     machine => {
 374         description => "Specific the Qemu machine type.",
 375         type => 'string',
 376         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
 377         maxLength => 40,
 378         optional => 1,
 379     },
 380     smbios1 => {
 381         description => "Specify SMBIOS type 1 fields.",
 382         type => 'string', format => 'pve-qm-smbios1',
 383         typetext => "[manufacturer=str][,product=str][,version=str][,serial=str] [,uuid=uuid][,sku=str][,family=str]",
 384         maxLength => 256,
 385         optional => 1,
 386     },
 387     protection => {
 388         optional => 1,
 389         type => 'boolean',
 390         description => "Sets the protection flag of the VM. This will prevent the remove operation.",
 391         default => 0,
 392     },
 393 };
 394
 395 # what about other qemu settings ?
 396 #cpu => 'string',
 397 #machine => 'string',
 398 #fda => 'file',
 399 #fdb => 'file',
 400 #mtdblock => 'file',
 401 #sd => 'file',
 402 #pflash => 'file',
 403 #snapshot => 'bool',
 404 #bootp => 'file',
 405 ##tftp => 'dir',
 406 ##smb => 'dir',
 407 #kernel => 'file',
 408 #append => 'string',
 409 #initrd => 'file',
 410 ##soundhw => 'string',
 411
 412 while (my ($k, $v) = each %$confdesc) {
 413     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 414 }
 415
 416 my $MAX_IDE_DISKS = 4;
 417 my $MAX_SCSI_DISKS = 14;
 418 my $MAX_VIRTIO_DISKS = 16;
 419 my $MAX_SATA_DISKS = 6;
 420 my $MAX_USB_DEVICES = 5;
 421 my $MAX_NETS = 32;
 422 my $MAX_UNUSED_DISKS = 8;
 423 my $MAX_HOSTPCI_DEVICES = 4;
 424 my $MAX_SERIAL_PORTS = 4;
 425 my $MAX_PARALLEL_PORTS = 3;
 426 my $MAX_NUMA = 8;
 427 my $MAX_MEM = 4194304;
 428 my $STATICMEM = 1024;
 429
 430 my $numadesc = {
 431     optional => 1,
 432     type => 'string', format => 'pve-qm-numanode',
 433     typetext => "cpus=<id[-id],memory=<mb>[[,hostnodes=<id[-id]>] [,policy=<preferred|bind|interleave>]]",
 434     description => "numa topology",
 435 };
 436 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 437
 438 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 439     $confdesc->{"numa$i"} = $numadesc;
 440 }
 441
 442 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 443                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 444                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 445 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 446
 447 my $netdesc = {
 448     optional => 1,
 449     type => 'string', format => 'pve-qm-net',
 450     typetext => "MODEL=XX:XX:XX:XX:XX:XX [,bridge=<dev>][,queues=<nbqueues>][,rate=<mbps>] [,tag=<vlanid>][,firewall=0|1],link_down=0|1]",
 451     description => <<EODESCR,
 452 Specify network devices.
 453
 454 MODEL is one of: $nic_model_list_txt
 455
 456 XX:XX:XX:XX:XX:XX should be an unique MAC address. This is
 457 automatically generated if not specified.
 458
 459 The bridge parameter can be used to automatically add the interface to a bridge device. The Proxmox VE standard bridge is called 'vmbr0'.
 460
 461 Option 'rate' is used to limit traffic bandwidth from and to this interface. It is specified as floating point number, unit is 'Megabytes per second'.
 462
 463 If you specify no bridge, we create a kvm 'user' (NATed) network device, which provides DHCP and DNS services. The following addresses are used:
 464
 465 10.0.2.2   Gateway
 466 10.0.2.3   DNS Server
 467 10.0.2.4   SMB Server
 468
 469 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 470
 471 EODESCR
 472 };
 473 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 474
 475 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 476     $confdesc->{"net$i"} = $netdesc;
 477 }
 478
 479 my $drivename_hash;
 480
 481 my $idedesc = {
 482     optional => 1,
 483     type => 'string', format => 'pve-qm-drive',
 484     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads] [,discard=ignore|on] [,serial=serial][,model=model]',
 485     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
 486 };
 487 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
 488
 489 my $scsidesc = {
 490     optional => 1,
 491     type => 'string', format => 'pve-qm-drive',
 492     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads] [,discard=ignore|on] [,iothread=on] [,queues=<nbqueues>] [,serial=serial]',
 493     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
 494 };
 495 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 496
 497 my $satadesc = {
 498     optional => 1,
 499     type => 'string', format => 'pve-qm-drive',
 500     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads]  [,discard=ignore|on] [,serial=serial]',
 501     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
 502 };
 503 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
 504
 505 my $virtiodesc = {
 506     optional => 1,
 507     type => 'string', format => 'pve-qm-drive',
 508     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads]  [,discard=ignore|on] [,iothread=on] [,serial=serial]',
 509     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
 510 };
 511 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
 512
 513 my $usbdesc = {
 514     optional => 1,
 515     type => 'string', format => 'pve-qm-usb-device',
 516     typetext => 'host=HOSTUSBDEVICE|spice',
 517     description => <<EODESCR,
 518 Configure an USB device (n is 0 to 4). This can be used to
 519 pass-through usb devices to the guest. HOSTUSBDEVICE syntax is:
 520
 521 'bus-port(.port)*' (decimal numbers) or
 522 'vendor_id:product_id' (hexadeciaml numbers)
 523
 524 You can use the 'lsusb -t' command to list existing usb devices.
 525
 526 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 527
 528 The value 'spice' can be used to add a usb redirection devices for spice.
 529
 530 EODESCR
 531 };
 532 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
 533
 534 my $hostpcidesc = {
 535         optional => 1,
 536         type => 'string', format => 'pve-qm-hostpci',
 537         typetext => "[host=]HOSTPCIDEVICE [,rombar=on|off] [,pcie=0|1] [,x-vga=on|off]",
 538         description => <<EODESCR,
 539 Map host pci devices. HOSTPCIDEVICE syntax is:
 540
 541 'bus:dev.func' (hexadecimal numbers)
 542
 543 You can us the 'lspci' command to list existing pci devices.
 544
 545 The 'rombar' option determines whether or not the device's ROM will be visible in the guest's memory map (default is 'on').
 546
 547 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 548
 549 Experimental: user reported problems with this option.
 550 EODESCR
 551 };
 552 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
 553
 554 my $serialdesc = {
 555         optional => 1,
 556         type => 'string',
 557         pattern => '(/dev/.+|socket)',
 558         description =>  <<EODESCR,
 559 Create a serial device inside the VM (n is 0 to 3), and pass through a host serial device (i.e. /dev/ttyS0), or create a unix socket on the host side (use 'qm terminal' to open a terminal connection).
 560
 561 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 562
 563 Experimental: user reported problems with this option.
 564 EODESCR
 565 };
 566
 567 my $paralleldesc= {
 568         optional => 1,
 569         type => 'string',
 570         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
 571         description =>  <<EODESCR,
 572 Map host parallel devices (n is 0 to 2).
 573
 574 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 575
 576 Experimental: user reported problems with this option.
 577 EODESCR
 578 };
 579
 580 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
 581     $confdesc->{"parallel$i"} = $paralleldesc;
 582 }
 583
 584 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
 585     $confdesc->{"serial$i"} = $serialdesc;
 586 }
 587
 588 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
 589     $confdesc->{"hostpci$i"} = $hostpcidesc;
 590 }
 591
 592 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
 593     $drivename_hash->{"ide$i"} = 1;
 594     $confdesc->{"ide$i"} = $idedesc;
 595 }
 596
 597 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
 598     $drivename_hash->{"sata$i"} = 1;
 599     $confdesc->{"sata$i"} = $satadesc;
 600 }
 601
 602 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
 603     $drivename_hash->{"scsi$i"} = 1;
 604     $confdesc->{"scsi$i"} = $scsidesc ;
 605 }
 606
 607 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
 608     $drivename_hash->{"virtio$i"} = 1;
 609     $confdesc->{"virtio$i"} = $virtiodesc;
 610 }
 611
 612 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
 613     $confdesc->{"usb$i"} = $usbdesc;
 614 }
 615
 616 my $unuseddesc = {
 617     optional => 1,
 618     type => 'string', format => 'pve-volume-id',
 619     description => "Reference to unused volumes.",
 620 };
 621
 622 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
 623     $confdesc->{"unused$i"} = $unuseddesc;
 624 }
 625
 626 my $kvm_api_version = 0;
 627
 628 sub kvm_version {
 629
 630     return $kvm_api_version if $kvm_api_version;
 631
 632     my $fh = IO::File->new("</dev/kvm") ||
 633         return 0;
 634
 635     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
 636         $kvm_api_version = $v;
 637     }
 638
 639     $fh->close();
 640
 641     return  $kvm_api_version;
 642 }
 643
 644 my $kvm_user_version;
 645
 646 sub kvm_user_version {
 647
 648     return $kvm_user_version if $kvm_user_version;
 649
 650     $kvm_user_version = 'unknown';
 651
 652     my $tmp = `kvm -help 2>/dev/null`;
 653
 654     if ($tmp =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
 655         $kvm_user_version = $2;
 656     }
 657
 658     return $kvm_user_version;
 659
 660 }
 661
 662 my $kernel_has_vhost_net = -c '/dev/vhost-net';
 663
 664 sub disknames {
 665     # order is important - used to autoselect boot disk
 666     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
 667             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
 668             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
 669             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))));
 670 }
 671
 672 sub valid_drivename {
 673     my $dev = shift;
 674
 675     return defined($drivename_hash->{$dev});
 676 }
 677
 678 sub option_exists {
 679     my $key = shift;
 680     return defined($confdesc->{$key});
 681 }
 682
 683 sub nic_models {
 684     return $nic_model_list;
 685 }
 686
 687 sub os_list_description {
 688
 689     return {
 690         other => 'Other',
 691         wxp => 'Windows XP',
 692         w2k => 'Windows 2000',
 693         w2k3 =>, 'Windows 2003',
 694         w2k8 => 'Windows 2008',
 695         wvista => 'Windows Vista',
 696         win7 => 'Windows 7',
 697         win8 => 'Windows 8/2012',
 698         l24 => 'Linux 2.4',
 699         l26 => 'Linux 2.6',
 700     };
 701 }
 702
 703 my $cdrom_path;
 704
 705 sub get_cdrom_path {
 706
 707     return  $cdrom_path if $cdrom_path;
 708
 709     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
 710     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
 711     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
 712 }
 713
 714 sub get_iso_path {
 715     my ($storecfg, $vmid, $cdrom) = @_;
 716
 717     if ($cdrom eq 'cdrom') {
 718         return get_cdrom_path();
 719     } elsif ($cdrom eq 'none') {
 720         return '';
 721     } elsif ($cdrom =~ m|^/|) {
 722         return $cdrom;
 723     } else {
 724         return PVE::Storage::path($storecfg, $cdrom);
 725     }
 726 }
 727
 728 # try to convert old style file names to volume IDs
 729 sub filename_to_volume_id {
 730     my ($vmid, $file, $media) = @_;
 731
 732     if (!($file eq 'none' || $file eq 'cdrom' ||
 733           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
 734
 735         return undef if $file =~ m|/|;
 736
 737         if ($media && $media eq 'cdrom') {
 738             $file = "local:iso/$file";
 739         } else {
 740             $file = "local:$vmid/$file";
 741         }
 742     }
 743
 744     return $file;
 745 }
 746
 747 sub verify_media_type {
 748     my ($opt, $vtype, $media) = @_;
 749
 750     return if !$media;
 751
 752     my $etype;
 753     if ($media eq 'disk') {
 754         $etype = 'images';
 755     } elsif ($media eq 'cdrom') {
 756         $etype = 'iso';
 757     } else {
 758         die "internal error";
 759     }
 760
 761     return if ($vtype eq $etype);
 762
 763     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
 764 }
 765
 766 sub cleanup_drive_path {
 767     my ($opt, $storecfg, $drive) = @_;
 768
 769     # try to convert filesystem paths to volume IDs
 770
 771     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
 772         ($drive->{file} !~ m|^/dev/.+|) &&
 773         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
 774         ($drive->{file} !~ m/^\d+$/)) {
 775         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
 776         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
 777         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
 778         verify_media_type($opt, $vtype, $drive->{media});
 779         $drive->{file} = $volid;
 780     }
 781
 782     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
 783 }
 784
 785 sub create_conf_nolock {
 786     my ($vmid, $settings) = @_;
 787
 788     my $filename = config_file($vmid);
 789
 790     die "configuration file '$filename' already exists\n" if -f $filename;
 791
 792     my $defaults = load_defaults();
 793
 794     $settings->{name} = "vm$vmid" if !$settings->{name};
 795     $settings->{memory} = $defaults->{memory} if !$settings->{memory};
 796
 797     my $data = '';
 798     foreach my $opt (keys %$settings) {
 799         next if !$confdesc->{$opt};
 800
 801         my $value = $settings->{$opt};
 802         next if !$value;
 803
 804         $data .= "$opt: $value\n";
 805     }
 806
 807     PVE::Tools::file_set_contents($filename, $data);
 808 }
 809
 810 sub parse_hotplug_features {
 811     my ($data) = @_;
 812
 813     my $res = {};
 814
 815     return $res if $data eq '0';
 816
 817     $data = $confdesc->{hotplug}->{default} if $data eq '1';
 818
 819     foreach my $feature (PVE::Tools::split_list($data)) {
 820         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
 821             $res->{$1} = 1;
 822         } else {
 823             warn "ignoring unknown hotplug feature '$feature'\n";
 824         }
 825     }
 826     return $res;
 827 }
 828
 829 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
 830 sub pve_verify_hotplug_features {
 831     my ($value, $noerr) = @_;
 832
 833     return $value if parse_hotplug_features($value);
 834
 835     return undef if $noerr;
 836
 837     die "unable to parse hotplug option\n";
 838 }
 839
 840 my $parse_size = sub {
 841     my ($value) = @_;
 842
 843     return undef if $value !~ m/^(\d+(\.\d+)?)([KMG])?$/;
 844     my ($size, $unit) = ($1, $3);
 845     if ($unit) {
 846         if ($unit eq 'K') {
 847             $size = $size * 1024;
 848         } elsif ($unit eq 'M') {
 849             $size = $size * 1024 * 1024;
 850         } elsif ($unit eq 'G') {
 851             $size = $size * 1024 * 1024 * 1024;
 852         }
 853     }
 854     return int($size);
 855 };
 856
 857 my $format_size = sub {
 858     my ($size) = @_;
 859
 860     $size = int($size);
 861
 862     my $kb = int($size/1024);
 863     return $size if $kb*1024 != $size;
 864
 865     my $mb = int($kb/1024);
 866     return "${kb}K" if $mb*1024 != $kb;
 867
 868     my $gb = int($mb/1024);
 869     return "${mb}M" if $gb*1024 != $mb;
 870
 871     return "${gb}G";
 872 };
 873
 874 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
 875 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
 876 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
 877 #        [,aio=native|threads][,discard=ignore|on][,iothread=on]
 878 #        [,serial=serial][,model=model]
 879
 880 sub parse_drive {
 881     my ($key, $data) = @_;
 882
 883     my $res = {};
 884
 885     # $key may be undefined - used to verify JSON parameters
 886     if (!defined($key)) {
 887         $res->{interface} = 'unknown'; # should not harm when used to verify parameters
 888         $res->{index} = 0;
 889     } elsif ($key =~ m/^([^\d]+)(\d+)$/) {
 890         $res->{interface} = $1;
 891         $res->{index} = $2;
 892     } else {
 893         return undef;
 894     }
 895
 896     foreach my $p (split (/,/, $data)) {
 897         next if $p =~ m/^\s*$/;
 898
 899         if ($p =~ m/^(file|volume|cyls|heads|secs|trans|media|snapshot|cache|format|rerror|werror|backup|aio|bps|mbps|mbps_max|bps_rd|mbps_rd|mbps_rd_max|bps_wr|mbps_wr|mbps_wr_max|iops|iops_max|iops_rd|iops_rd_max|iops_wr|iops_wr_max|size|discard|iothread|queues|serial|model)=(.+)$/) {
 900             my ($k, $v) = ($1, $2);
 901
 902             $k = 'file' if $k eq 'volume';
 903
 904             return undef if defined $res->{$k};
 905
 906             if ($k eq 'bps' || $k eq 'bps_rd' || $k eq 'bps_wr') {
 907                 return undef if !$v || $v !~ m/^\d+/;
 908                 $k = "m$k";
 909                 $v = sprintf("%.3f", $v / (1024*1024));
 910             }
 911             $res->{$k} = $v;
 912         } else {
 913             if (!$res->{file} && $p !~ m/=/) {
 914                 $res->{file} = $p;
 915             } else {
 916                 return undef;
 917             }
 918         }
 919     }
 920
 921     return undef if !$res->{file};
 922
 923     return undef if $res->{cache} &&
 924         $res->{cache} !~ m/^(off|none|writethrough|writeback|unsafe|directsync)$/;
 925     return undef if $res->{snapshot} && $res->{snapshot} !~ m/^(on|off)$/;
 926     return undef if $res->{cyls} && $res->{cyls} !~ m/^\d+$/;
 927     return undef if $res->{heads} && $res->{heads} !~ m/^\d+$/;
 928     return undef if $res->{secs} && $res->{secs} !~ m/^\d+$/;
 929     return undef if $res->{media} && $res->{media} !~ m/^(disk|cdrom)$/;
 930     return undef if $res->{trans} && $res->{trans} !~ m/^(none|lba|auto)$/;
 931     return undef if $res->{format} && $res->{format} !~ m/^(raw|cow|qcow|qed|qcow2|vmdk|cloop)$/;
 932     return undef if $res->{rerror} && $res->{rerror} !~ m/^(ignore|report|stop)$/;
 933     return undef if $res->{werror} && $res->{werror} !~ m/^(enospc|ignore|report|stop)$/;
 934     return undef if $res->{backup} && $res->{backup} !~ m/^(yes|no)$/;
 935     return undef if $res->{aio} && $res->{aio} !~ m/^(native|threads)$/;
 936     return undef if $res->{discard} && $res->{discard} !~ m/^(ignore|on)$/;
 937     return undef if $res->{iothread} && $res->{iothread} !~ m/^(on)$/;
 938     return undef if $res->{queues} && ($res->{queues} !~ m/^\d+$/ || $res->{queues} < 2);
 939
 940     return undef if $res->{mbps_rd} && $res->{mbps};
 941     return undef if $res->{mbps_wr} && $res->{mbps};
 942
 943     return undef if $res->{mbps} && $res->{mbps} !~ m/^\d+(\.\d+)?$/;
 944     return undef if $res->{mbps_max} && $res->{mbps_max} !~ m/^\d+(\.\d+)?$/;
 945     return undef if $res->{mbps_rd} && $res->{mbps_rd} !~ m/^\d+(\.\d+)?$/;
 946     return undef if $res->{mbps_rd_max} && $res->{mbps_rd_max} !~ m/^\d+(\.\d+)?$/;
 947     return undef if $res->{mbps_wr} && $res->{mbps_wr} !~ m/^\d+(\.\d+)?$/;
 948     return undef if $res->{mbps_wr_max} && $res->{mbps_wr_max} !~ m/^\d+(\.\d+)?$/;
 949
 950     return undef if $res->{iops_rd} && $res->{iops};
 951     return undef if $res->{iops_wr} && $res->{iops};
 952
 953
 954     return undef if $res->{iops} && $res->{iops} !~ m/^\d+$/;
 955     return undef if $res->{iops_max} && $res->{iops_max} !~ m/^\d+$/;
 956     return undef if $res->{iops_rd} && $res->{iops_rd} !~ m/^\d+$/;
 957     return undef if $res->{iops_rd_max} && $res->{iops_rd_max} !~ m/^\d+$/;
 958     return undef if $res->{iops_wr} && $res->{iops_wr} !~ m/^\d+$/;
 959     return undef if $res->{iops_wr_max} && $res->{iops_wr_max} !~ m/^\d+$/;
 960
 961     if ($res->{size}) {
 962         return undef if !defined($res->{size} = &$parse_size($res->{size}));
 963     }
 964
 965     if ($res->{media} && ($res->{media} eq 'cdrom')) {
 966         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
 967         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
 968         return undef if $res->{interface} eq 'virtio';
 969     }
 970
 971     # rerror does not work with scsi drives
 972     if ($res->{rerror}) {
 973         return undef if $res->{interface} eq 'scsi';
 974     }
 975
 976     return $res;
 977 }
 978
 979 my @qemu_drive_options = qw(heads secs cyls trans media format cache snapshot rerror werror aio discard iops iops_rd iops_wr iops_max iops_rd_max iops_wr_max serial);
 980
 981 sub print_drive {
 982     my ($vmid, $drive) = @_;
 983
 984     my $opts = '';
 985     foreach my $o (@qemu_drive_options, 'mbps', 'mbps_rd', 'mbps_wr', 'mbps_max', 'mbps_rd_max', 'mbps_wr_max', 'backup', 'iothread', 'queues') {
 986         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
 987     }
 988
 989     if ($drive->{size}) {
 990         $opts .= ",size=" . &$format_size($drive->{size});
 991     }
 992
 993     if (my $model = $drive->{model}) {
 994         $opts .= ",model=$model";
 995     }
 996
 997     return "$drive->{file}$opts";
 998 }
 999
1000 sub scsi_inquiry {
1001     my($fh, $noerr) = @_;
1002
1003     my $SG_IO = 0x2285;
1004     my $SG_GET_VERSION_NUM = 0x2282;
1005
1006     my $versionbuf = "\x00" x 8;
1007     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1008     if (!$ret) {
1009         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1010         return undef;
1011     }
1012     my $version = unpack("I", $versionbuf);
1013     if ($version < 30000) {
1014         die "scsi generic interface too old\n"  if !$noerr;
1015         return undef;
1016     }
1017
1018     my $buf = "\x00" x 36;
1019     my $sensebuf = "\x00" x 8;
1020     my $cmd = pack("C x3 C x1", 0x12, 36);
1021
1022     # see /usr/include/scsi/sg.h
1023     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1024
1025     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1026                       length($sensebuf), 0, length($buf), $buf,
1027                       $cmd, $sensebuf, 6000);
1028
1029     $ret = ioctl($fh, $SG_IO, $packet);
1030     if (!$ret) {
1031         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1032         return undef;
1033     }
1034
1035     my @res = unpack($sg_io_hdr_t, $packet);
1036     if ($res[17] || $res[18]) {
1037         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1038         return undef;
1039     }
1040
1041     my $res = {};
1042     (my $byte0, my $byte1, $res->{vendor},
1043      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1044
1045     $res->{removable} = $byte1 & 128 ? 1 : 0;
1046     $res->{type} = $byte0 & 31;
1047
1048     return $res;
1049 }
1050
1051 sub path_is_scsi {
1052     my ($path) = @_;
1053
1054     my $fh = IO::File->new("+<$path") || return undef;
1055     my $res = scsi_inquiry($fh, 1);
1056     close($fh);
1057
1058     return $res;
1059 }
1060
1061 sub machine_type_is_q35 {
1062     my ($conf) = @_;
1063
1064     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1065 }
1066
1067 sub print_tabletdevice_full {
1068     my ($conf) = @_;
1069
1070     my $q35 = machine_type_is_q35($conf);
1071
1072     # we use uhci for old VMs because tablet driver was buggy in older qemu
1073     my $usbbus = $q35 ? "ehci" : "uhci";
1074
1075     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1076 }
1077
1078 sub print_drivedevice_full {
1079     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1080
1081     my $device = '';
1082     my $maxdev = 0;
1083
1084     if ($drive->{interface} eq 'virtio') {
1085         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1086         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1087         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1088     } elsif ($drive->{interface} eq 'scsi') {
1089
1090         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1091         my $unit = $drive->{index} % $maxdev;
1092         my $devicetype = 'hd';
1093         my $path = '';
1094         if (drive_is_cdrom($drive)) {
1095               $devicetype = 'cd';
1096         } else {
1097               if ($drive->{file} =~ m|^/|) {
1098                   $path = $drive->{file};
1099               } else {
1100                   $path = PVE::Storage::path($storecfg, $drive->{file});
1101               }
1102
1103               if($path =~ m/^iscsi\:\/\//){
1104                   $devicetype = 'generic';
1105               } else {
1106                   if (my $info = path_is_scsi($path)) {
1107                       if ($info->{type} == 0) {
1108                           $devicetype = 'block';
1109                       } elsif ($info->{type} == 1) { # tape
1110                           $devicetype = 'generic';
1111                       }
1112                   }
1113               }
1114          }
1115
1116         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1117             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1118         } else {
1119             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1120         }
1121
1122     } elsif ($drive->{interface} eq 'ide'){
1123         $maxdev = 2;
1124         my $controller = int($drive->{index} / $maxdev);
1125         my $unit = $drive->{index} % $maxdev;
1126         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1127
1128         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1129         if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
1130             $device .= ",model=$model";
1131         }
1132     } elsif ($drive->{interface} eq 'sata'){
1133         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1134         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1135         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1136     } elsif ($drive->{interface} eq 'usb') {
1137         die "implement me";
1138         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1139     } else {
1140         die "unsupported interface type";
1141     }
1142
1143     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1144
1145     return $device;
1146 }
1147
1148 sub get_initiator_name {
1149     my $initiator;
1150
1151     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1152     while (defined(my $line = <$fh>)) {
1153         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1154         $initiator = $1;
1155         last;
1156     }
1157     $fh->close();
1158
1159     return $initiator;
1160 }
1161
1162 sub print_drive_full {
1163     my ($storecfg, $vmid, $drive) = @_;
1164
1165     my $path;
1166     my $volid = $drive->{file};
1167     my $format;
1168
1169     if (drive_is_cdrom($drive)) {
1170         $path = get_iso_path($storecfg, $vmid, $volid);
1171     } else {
1172         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1173         if ($storeid) {
1174             $path = PVE::Storage::path($storecfg, $volid);
1175             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1176             $format = qemu_img_format($scfg, $volname);
1177         } else {
1178             $path = $volid;
1179         }
1180    }
1181
1182     my $opts = '';
1183     foreach my $o (@qemu_drive_options) {
1184         next if $o eq 'bootindex';
1185         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1186     }
1187
1188     $opts .= ",format=$format" if $format && !$drive->{format};
1189
1190     foreach my $o (qw(bps bps_rd bps_wr)) {
1191         my $v = $drive->{"m$o"};
1192         $opts .= ",$o=" . int($v*1024*1024) if $v;
1193     }
1194
1195     my $cache_direct = 0;
1196
1197     if (my $cache = $drive->{cache}) {
1198         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1199     } elsif (!drive_is_cdrom($drive)) {
1200         $opts .= ",cache=none";
1201         $cache_direct = 1;
1202     }
1203
1204     # aio native works only with O_DIRECT
1205     if (!$drive->{aio}) {
1206         if($cache_direct) {
1207             $opts .= ",aio=native";
1208         } else {
1209             $opts .= ",aio=threads";
1210         }
1211     }
1212
1213     my $detectzeroes = $drive->{discard} ? "unmap" : "on";
1214     $opts .= ",detect-zeroes=$detectzeroes" if !drive_is_cdrom($drive);
1215
1216     my $pathinfo = $path ? "file=$path," : '';
1217
1218     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1219 }
1220
1221 sub print_netdevice_full {
1222     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1223
1224     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1225
1226     my $device = $net->{model};
1227     if ($net->{model} eq 'virtio') {
1228          $device = 'virtio-net-pci';
1229      };
1230
1231     my $pciaddr = print_pci_addr("$netid", $bridges);
1232     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1233     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1234         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1235         my $vectors = $net->{queues} * 2 + 2;
1236         $tmpstr .= ",vectors=$vectors,mq=on";
1237     }
1238     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1239
1240     if ($use_old_bios_files) {
1241         my $romfile;
1242         if ($device eq 'virtio-net-pci') {
1243             $romfile = 'pxe-virtio.rom';
1244         } elsif ($device eq 'e1000') {
1245             $romfile = 'pxe-e1000.rom';
1246         } elsif ($device eq 'ne2k') {
1247             $romfile = 'pxe-ne2k_pci.rom';
1248         } elsif ($device eq 'pcnet') {
1249             $romfile = 'pxe-pcnet.rom';
1250         } elsif ($device eq 'rtl8139') {
1251             $romfile = 'pxe-rtl8139.rom';
1252         }
1253         $tmpstr .= ",romfile=$romfile" if $romfile;
1254     }
1255
1256     return $tmpstr;
1257 }
1258
1259 sub print_netdev_full {
1260     my ($vmid, $conf, $net, $netid) = @_;
1261
1262     my $i = '';
1263     if ($netid =~ m/^net(\d+)$/) {
1264         $i = int($1);
1265     }
1266
1267     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1268
1269     my $ifname = "tap${vmid}i$i";
1270
1271     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1272     die "interface name '$ifname' is too long (max 15 character)\n"
1273         if length($ifname) >= 16;
1274
1275     my $vhostparam = '';
1276     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1277
1278     my $vmname = $conf->{name} || "vm$vmid";
1279
1280     my $netdev = "";
1281
1282     if ($net->{bridge}) {
1283         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/pve-bridge,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1284     } else {
1285         $netdev = "type=user,id=$netid,hostname=$vmname";
1286     }
1287
1288     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1289
1290     return $netdev;
1291 }
1292
1293 sub drive_is_cdrom {
1294     my ($drive) = @_;
1295
1296     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1297
1298 }
1299
1300 sub parse_numa {
1301     my ($data) = @_;
1302
1303     my $res = {};
1304
1305     foreach my $kvp (split(/,/, $data)) {
1306
1307         if ($kvp =~ m/^memory=(\S+)$/) {
1308             $res->{memory} = $1;
1309         } elsif ($kvp =~ m/^policy=(preferred|bind|interleave)$/) {
1310             $res->{policy} = $1;
1311         } elsif ($kvp =~ m/^cpus=(\d+)(-(\d+))?$/) {
1312             $res->{cpus}->{start} = $1;
1313             $res->{cpus}->{end} = $3;
1314         } elsif ($kvp =~ m/^hostnodes=(\d+)(-(\d+))?$/) {
1315             $res->{hostnodes}->{start} = $1;
1316             $res->{hostnodes}->{end} = $3;
1317         } else {
1318             return undef;
1319         }
1320     }
1321
1322     return $res;
1323 }
1324
1325 sub parse_hostpci {
1326     my ($value) = @_;
1327
1328     return undef if !$value;
1329
1330
1331     my @list = split(/,/, $value);
1332     my $found;
1333
1334     my $res = {};
1335     foreach my $kv (@list) {
1336
1337         if ($kv =~ m/^(host=)?([a-f0-9]{2}:[a-f0-9]{2})(\.([a-f0-9]))?$/) {
1338             $found = 1;
1339             if(defined($4)){
1340                 push @{$res->{pciid}}, { id => $2 , function => $4};
1341
1342             }else{
1343                 my $pcidevices = lspci($2);
1344                 $res->{pciid} = $pcidevices->{$2};
1345             }
1346         } elsif ($kv =~ m/^rombar=(on|off)$/) {
1347             $res->{rombar} = $1;
1348         } elsif ($kv =~ m/^x-vga=(on|off)$/) {
1349             $res->{'x-vga'} = $1;
1350         } elsif ($kv =~ m/^pcie=(\d+)$/) {
1351             $res->{pcie} = 1 if $1 == 1;
1352         } else {
1353             warn "unknown hostpci setting '$kv'\n";
1354         }
1355     }
1356
1357     return undef if !$found;
1358
1359     return $res;
1360 }
1361
1362 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1363 sub parse_net {
1364     my ($data) = @_;
1365
1366     my $res = {};
1367
1368     foreach my $kvp (split(/,/, $data)) {
1369
1370         if ($kvp =~ m/^(ne2k_pci|e1000|e1000-82540em|e1000-82544gc|e1000-82545em|rtl8139|pcnet|virtio|ne2k_isa|i82551|i82557b|i82559er|vmxnet3)(=([0-9a-f]{2}(:[0-9a-f]{2}){5}))?$/i) {
1371             my $model = lc($1);
1372             my $mac = defined($3) ? uc($3) : PVE::Tools::random_ether_addr();
1373             $res->{model} = $model;
1374             $res->{macaddr} = $mac;
1375         } elsif ($kvp =~ m/^bridge=(\S+)$/) {
1376             $res->{bridge} = $1;
1377         } elsif ($kvp =~ m/^queues=(\d+)$/) {
1378             $res->{queues} = $1;
1379         } elsif ($kvp =~ m/^rate=(\d+(\.\d+)?)$/) {
1380             $res->{rate} = $1;
1381         } elsif ($kvp =~ m/^tag=(\d+)$/) {
1382             $res->{tag} = $1;
1383         } elsif ($kvp =~ m/^firewall=([01])$/) {
1384             $res->{firewall} = $1;
1385         } elsif ($kvp =~ m/^link_down=([01])$/) {
1386             $res->{link_down} = $1;
1387         } else {
1388             return undef;
1389         }
1390
1391     }
1392
1393     return undef if !$res->{model};
1394
1395     return $res;
1396 }
1397
1398 sub print_net {
1399     my $net = shift;
1400
1401     my $res = "$net->{model}";
1402     $res .= "=$net->{macaddr}" if $net->{macaddr};
1403     $res .= ",bridge=$net->{bridge}" if $net->{bridge};
1404     $res .= ",rate=$net->{rate}" if $net->{rate};
1405     $res .= ",tag=$net->{tag}" if $net->{tag};
1406     $res .= ",firewall=1" if $net->{firewall};
1407     $res .= ",link_down=1" if $net->{link_down};
1408     $res .= ",queues=$net->{queues}" if $net->{queues};
1409
1410     return $res;
1411 }
1412
1413 sub add_random_macs {
1414     my ($settings) = @_;
1415
1416     foreach my $opt (keys %$settings) {
1417         next if $opt !~ m/^net(\d+)$/;
1418         my $net = parse_net($settings->{$opt});
1419         next if !$net;
1420         $settings->{$opt} = print_net($net);
1421     }
1422 }
1423
1424 sub add_unused_volume {
1425     my ($config, $volid) = @_;
1426
1427     my $key;
1428     for (my $ind = $MAX_UNUSED_DISKS - 1; $ind >= 0; $ind--) {
1429         my $test = "unused$ind";
1430         if (my $vid = $config->{$test}) {
1431             return if $vid eq $volid; # do not add duplicates
1432         } else {
1433             $key = $test;
1434         }
1435     }
1436
1437     die "To many unused volume - please delete them first.\n" if !$key;
1438
1439     $config->{$key} = $volid;
1440
1441     return $key;
1442 }
1443
1444 sub vm_is_volid_owner {
1445     my ($storecfg, $vmid, $volid) = @_;
1446
1447     if ($volid !~  m|^/|) {
1448         my ($path, $owner);
1449         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1450         if ($owner && ($owner == $vmid)) {
1451             return 1;
1452         }
1453     }
1454
1455     return undef;
1456 }
1457
1458 sub split_flagged_list {
1459     my $text = shift || '';
1460     $text =~ s/[,;]/ /g;
1461     $text =~ s/^\s+//;
1462     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
1463 }
1464
1465 sub join_flagged_list {
1466     my ($how, $lst) = @_;
1467     join $how, map { $lst->{$_} . $_ } keys %$lst;
1468 }
1469
1470 sub vmconfig_delete_pending_option {
1471     my ($conf, $key, $force) = @_;
1472
1473     delete $conf->{pending}->{$key};
1474     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1475     $pending_delete_hash->{$key} = $force ? '!' : '';
1476     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1477 }
1478
1479 sub vmconfig_undelete_pending_option {
1480     my ($conf, $key) = @_;
1481
1482     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1483     delete $pending_delete_hash->{$key};
1484
1485     if (%$pending_delete_hash) {
1486         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1487     } else {
1488         delete $conf->{pending}->{delete};
1489     }
1490 }
1491
1492 sub vmconfig_register_unused_drive {
1493     my ($storecfg, $vmid, $conf, $drive) = @_;
1494
1495     if (!drive_is_cdrom($drive)) {
1496         my $volid = $drive->{file};
1497         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1498             add_unused_volume($conf, $volid, $vmid);
1499         }
1500     }
1501 }
1502
1503 sub vmconfig_cleanup_pending {
1504     my ($conf) = @_;
1505
1506     # remove pending changes when nothing changed
1507     my $changes;
1508     foreach my $opt (keys %{$conf->{pending}}) {
1509         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
1510             $changes = 1;
1511             delete $conf->{pending}->{$opt};
1512         }
1513     }
1514
1515     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
1516     my $pending_delete_hash = {};
1517     while (my ($opt, $force) = each %$current_delete_hash) {
1518         if (defined($conf->{$opt})) {
1519             $pending_delete_hash->{$opt} = $force;
1520         } else {
1521             $changes = 1;
1522         }
1523     }
1524
1525     if (%$pending_delete_hash) {
1526         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1527     } else {
1528         delete $conf->{pending}->{delete};
1529     }
1530
1531     return $changes;
1532 }
1533
1534 my $valid_smbios1_options = {
1535     manufacturer => '\S+',
1536     product => '\S+',
1537     version => '\S+',
1538     serial => '\S+',
1539     uuid => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1540     sku => '\S+',
1541     family => '\S+',
1542 };
1543
1544 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
1545 sub parse_smbios1 {
1546     my ($data) = @_;
1547
1548     my $res = {};
1549
1550     foreach my $kvp (split(/,/, $data)) {
1551         return undef if $kvp !~ m/^(\S+)=(.+)$/;
1552         my ($k, $v) = split(/=/, $kvp);
1553         return undef if !defined($k) || !defined($v);
1554         return undef if !$valid_smbios1_options->{$k};
1555         return undef if $v !~ m/^$valid_smbios1_options->{$k}$/;
1556         $res->{$k} = $v;
1557     }
1558
1559     return $res;
1560 }
1561
1562 sub print_smbios1 {
1563     my ($smbios1) = @_;
1564
1565     my $data = '';
1566     foreach my $k (keys %$smbios1) {
1567         next if !defined($smbios1->{$k});
1568         next if !$valid_smbios1_options->{$k};
1569         $data .= ',' if $data;
1570         $data .= "$k=$smbios1->{$k}";
1571     }
1572     return $data;
1573 }
1574
1575 PVE::JSONSchema::register_format('pve-qm-smbios1', \&verify_smbios1);
1576 sub verify_smbios1 {
1577     my ($value, $noerr) = @_;
1578
1579     return $value if parse_smbios1($value);
1580
1581     return undef if $noerr;
1582
1583     die "unable to parse smbios (type 1) options\n";
1584 }
1585
1586 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
1587 sub verify_bootdisk {
1588     my ($value, $noerr) = @_;
1589
1590     return $value if valid_drivename($value);
1591
1592     return undef if $noerr;
1593
1594     die "invalid boot disk '$value'\n";
1595 }
1596
1597 PVE::JSONSchema::register_format('pve-qm-numanode', \&verify_numa);
1598 sub verify_numa {
1599     my ($value, $noerr) = @_;
1600
1601     return $value if parse_numa($value);
1602
1603     return undef if $noerr;
1604
1605     die "unable to parse numa options\n";
1606 }
1607
1608 PVE::JSONSchema::register_format('pve-qm-net', \&verify_net);
1609 sub verify_net {
1610     my ($value, $noerr) = @_;
1611
1612     return $value if parse_net($value);
1613
1614     return undef if $noerr;
1615
1616     die "unable to parse network options\n";
1617 }
1618
1619 PVE::JSONSchema::register_format('pve-qm-drive', \&verify_drive);
1620 sub verify_drive {
1621     my ($value, $noerr) = @_;
1622
1623     return $value if parse_drive(undef, $value);
1624
1625     return undef if $noerr;
1626
1627     die "unable to parse drive options\n";
1628 }
1629
1630 PVE::JSONSchema::register_format('pve-qm-hostpci', \&verify_hostpci);
1631 sub verify_hostpci {
1632     my ($value, $noerr) = @_;
1633
1634     return $value if parse_hostpci($value);
1635
1636     return undef if $noerr;
1637
1638     die "unable to parse pci id\n";
1639 }
1640
1641 PVE::JSONSchema::register_format('pve-qm-watchdog', \&verify_watchdog);
1642 sub verify_watchdog {
1643     my ($value, $noerr) = @_;
1644
1645     return $value if parse_watchdog($value);
1646
1647     return undef if $noerr;
1648
1649     die "unable to parse watchdog options\n";
1650 }
1651
1652 sub parse_watchdog {
1653     my ($value) = @_;
1654
1655     return undef if !$value;
1656
1657     my $res = {};
1658
1659     foreach my $p (split(/,/, $value)) {
1660         next if $p =~ m/^\s*$/;
1661
1662         if ($p =~ m/^(model=)?(i6300esb|ib700)$/) {
1663             $res->{model} = $2;
1664         } elsif ($p =~ m/^(action=)?(reset|shutdown|poweroff|pause|debug|none)$/) {
1665             $res->{action} = $2;
1666         } else {
1667             return undef;
1668         }
1669     }
1670
1671     return $res;
1672 }
1673
1674 sub parse_usb_device {
1675     my ($value) = @_;
1676
1677     return undef if !$value;
1678
1679     my @dl = split(/,/, $value);
1680     my $found;
1681
1682     my $res = {};
1683     foreach my $v (@dl) {
1684         if ($v =~ m/^host=(0x)?([0-9A-Fa-f]{4}):(0x)?([0-9A-Fa-f]{4})$/) {
1685             $found = 1;
1686             $res->{vendorid} = $2;
1687             $res->{productid} = $4;
1688         } elsif ($v =~ m/^host=(\d+)\-(\d+(\.\d+)*)$/) {
1689             $found = 1;
1690             $res->{hostbus} = $1;
1691             $res->{hostport} = $2;
1692         } elsif ($v =~ m/^spice$/) {
1693             $found = 1;
1694             $res->{spice} = 1;
1695         } else {
1696             return undef;
1697         }
1698     }
1699     return undef if !$found;
1700
1701     return $res;
1702 }
1703
1704 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
1705 sub verify_usb_device {
1706     my ($value, $noerr) = @_;
1707
1708     return $value if parse_usb_device($value);
1709
1710     return undef if $noerr;
1711
1712     die "unable to parse usb device\n";
1713 }
1714
1715 # add JSON properties for create and set function
1716 sub json_config_properties {
1717     my $prop = shift;
1718
1719     foreach my $opt (keys %$confdesc) {
1720         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
1721         $prop->{$opt} = $confdesc->{$opt};
1722     }
1723
1724     return $prop;
1725 }
1726
1727 sub check_type {
1728     my ($key, $value) = @_;
1729
1730     die "unknown setting '$key'\n" if !$confdesc->{$key};
1731
1732     my $type = $confdesc->{$key}->{type};
1733
1734     if (!defined($value)) {
1735         die "got undefined value\n";
1736     }
1737
1738     if ($value =~ m/[\n\r]/) {
1739         die "property contains a line feed\n";
1740     }
1741
1742     if ($type eq 'boolean') {
1743         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
1744         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
1745         die "type check ('boolean') failed - got '$value'\n";
1746     } elsif ($type eq 'integer') {
1747         return int($1) if $value =~ m/^(\d+)$/;
1748         die "type check ('integer') failed - got '$value'\n";
1749     } elsif ($type eq 'number') {
1750         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
1751         die "type check ('number') failed - got '$value'\n";
1752     } elsif ($type eq 'string') {
1753         if (my $fmt = $confdesc->{$key}->{format}) {
1754             if ($fmt eq 'pve-qm-drive') {
1755                 # special case - we need to pass $key to parse_drive()
1756                 my $drive = parse_drive($key, $value);
1757                 return $value if $drive;
1758                 die "unable to parse drive options\n";
1759             }
1760             PVE::JSONSchema::check_format($fmt, $value);
1761             return $value;
1762         }
1763         $value =~ s/^\"(.*)\"$/$1/;
1764         return $value;
1765     } else {
1766         die "internal error"
1767     }
1768 }
1769
1770 sub lock_config_full {
1771     my ($vmid, $timeout, $code, @param) = @_;
1772
1773     my $filename = config_file_lock($vmid);
1774
1775     my $res = lock_file($filename, $timeout, $code, @param);
1776
1777     die $@ if $@;
1778
1779     return $res;
1780 }
1781
1782 sub lock_config_mode {
1783     my ($vmid, $timeout, $shared, $code, @param) = @_;
1784
1785     my $filename = config_file_lock($vmid);
1786
1787     my $res = lock_file_full($filename, $timeout, $shared, $code, @param);
1788
1789     die $@ if $@;
1790
1791     return $res;
1792 }
1793
1794 sub lock_config {
1795     my ($vmid, $code, @param) = @_;
1796
1797     return lock_config_full($vmid, 10, $code, @param);
1798 }
1799
1800 sub cfs_config_path {
1801     my ($vmid, $node) = @_;
1802
1803     $node = $nodename if !$node;
1804     return "nodes/$node/qemu-server/$vmid.conf";
1805 }
1806
1807 sub check_iommu_support{
1808     #fixme : need to check IOMMU support
1809     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
1810
1811     my $iommu=1;
1812     return $iommu;
1813
1814 }
1815
1816 sub config_file {
1817     my ($vmid, $node) = @_;
1818
1819     my $cfspath = cfs_config_path($vmid, $node);
1820     return "/etc/pve/$cfspath";
1821 }
1822
1823 sub config_file_lock {
1824     my ($vmid) = @_;
1825
1826     return "$lock_dir/lock-$vmid.conf";
1827 }
1828
1829 sub touch_config {
1830     my ($vmid) = @_;
1831
1832     my $conf = config_file($vmid);
1833     utime undef, undef, $conf;
1834 }
1835
1836 sub destroy_vm {
1837     my ($storecfg, $vmid, $keep_empty_config) = @_;
1838
1839     my $conffile = config_file($vmid);
1840
1841     my $conf = load_config($vmid);
1842
1843     check_lock($conf);
1844
1845     # only remove disks owned by this VM
1846     foreach_drive($conf, sub {
1847         my ($ds, $drive) = @_;
1848
1849         return if drive_is_cdrom($drive);
1850
1851         my $volid = $drive->{file};
1852
1853         return if !$volid || $volid =~ m|^/|;
1854
1855         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
1856         return if !$path || !$owner || ($owner != $vmid);
1857
1858         PVE::Storage::vdisk_free($storecfg, $volid);
1859     });
1860
1861     if ($keep_empty_config) {
1862         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
1863     } else {
1864         unlink $conffile;
1865     }
1866
1867     # also remove unused disk
1868     eval {
1869         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
1870
1871         eval {
1872             PVE::Storage::foreach_volid($dl, sub {
1873                 my ($volid, $sid, $volname, $d) = @_;
1874                 PVE::Storage::vdisk_free($storecfg, $volid);
1875             });
1876         };
1877         warn $@ if $@;
1878
1879     };
1880     warn $@ if $@;
1881 }
1882
1883 sub load_config {
1884     my ($vmid, $node) = @_;
1885
1886     my $cfspath = cfs_config_path($vmid, $node);
1887
1888     my $conf = PVE::Cluster::cfs_read_file($cfspath);
1889
1890     die "no such VM ('$vmid')\n" if !defined($conf);
1891
1892     return $conf;
1893 }
1894
1895 sub parse_vm_config {
1896     my ($filename, $raw) = @_;
1897
1898     return undef if !defined($raw);
1899
1900     my $res = {
1901         digest => Digest::SHA::sha1_hex($raw),
1902         snapshots => {},
1903         pending => {},
1904     };
1905
1906     $filename =~ m|/qemu-server/(\d+)\.conf$|
1907         || die "got strange filename '$filename'";
1908
1909     my $vmid = $1;
1910
1911     my $conf = $res;
1912     my $descr;
1913     my $section = '';
1914
1915     my @lines = split(/\n/, $raw);
1916     foreach my $line (@lines) {
1917         next if $line =~ m/^\s*$/;
1918
1919         if ($line =~ m/^\[PENDING\]\s*$/i) {
1920             $section = 'pending';
1921             if (defined($descr)) {
1922                 $descr =~ s/\s+$//;
1923                 $conf->{description} = $descr;
1924             }
1925             $descr = undef;
1926             $conf = $res->{$section} = {};
1927             next;
1928
1929         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
1930             $section = $1;
1931             if (defined($descr)) {
1932                 $descr =~ s/\s+$//;
1933                 $conf->{description} = $descr;
1934             }
1935             $descr = undef;
1936             $conf = $res->{snapshots}->{$section} = {};
1937             next;
1938         }
1939
1940         if ($line =~ m/^\#(.*)\s*$/) {
1941             $descr = '' if !defined($descr);
1942             $descr .= PVE::Tools::decode_text($1) . "\n";
1943             next;
1944         }
1945
1946         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
1947             $descr = '' if !defined($descr);
1948             $descr .= PVE::Tools::decode_text($2);
1949         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
1950             $conf->{snapstate} = $1;
1951         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
1952             my $key = $1;
1953             my $value = $2;
1954             $conf->{$key} = $value;
1955         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
1956             my $value = $1;
1957             if ($section eq 'pending') {
1958                 $conf->{delete} = $value; # we parse this later
1959             } else {
1960                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
1961             }
1962         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
1963             my $key = $1;
1964             my $value = $2;
1965             eval { $value = check_type($key, $value); };
1966             if ($@) {
1967                 warn "vm $vmid - unable to parse value of '$key' - $@";
1968             } else {
1969                 my $fmt = $confdesc->{$key}->{format};
1970                 if ($fmt && $fmt eq 'pve-qm-drive') {
1971                     my $v = parse_drive($key, $value);
1972                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
1973                         $v->{file} = $volid;
1974                         $value = print_drive($vmid, $v);
1975                     } else {
1976                         warn "vm $vmid - unable to parse value of '$key'\n";
1977                         next;
1978                     }
1979                 }
1980
1981                 if ($key eq 'cdrom') {
1982                     $conf->{ide2} = $value;
1983                 } else {
1984                     $conf->{$key} = $value;
1985                 }
1986             }
1987         }
1988     }
1989
1990     if (defined($descr)) {
1991         $descr =~ s/\s+$//;
1992         $conf->{description} = $descr;
1993     }
1994     delete $res->{snapstate}; # just to be sure
1995
1996     return $res;
1997 }
1998
1999 sub write_vm_config {
2000     my ($filename, $conf) = @_;
2001
2002     delete $conf->{snapstate}; # just to be sure
2003
2004     if ($conf->{cdrom}) {
2005         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2006         $conf->{ide2} = $conf->{cdrom};
2007         delete $conf->{cdrom};
2008     }
2009
2010     # we do not use 'smp' any longer
2011     if ($conf->{sockets}) {
2012         delete $conf->{smp};
2013     } elsif ($conf->{smp}) {
2014         $conf->{sockets} = $conf->{smp};
2015         delete $conf->{cores};
2016         delete $conf->{smp};
2017     }
2018
2019     my $used_volids = {};
2020
2021     my $cleanup_config = sub {
2022         my ($cref, $pending, $snapname) = @_;
2023
2024         foreach my $key (keys %$cref) {
2025             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2026                 $key eq 'snapstate' || $key eq 'pending';
2027             my $value = $cref->{$key};
2028             if ($key eq 'delete') {
2029                 die "propertry 'delete' is only allowed in [PENDING]\n"
2030                     if !$pending;
2031                 # fixme: check syntax?
2032                 next;
2033             }
2034             eval { $value = check_type($key, $value); };
2035             die "unable to parse value of '$key' - $@" if $@;
2036
2037             $cref->{$key} = $value;
2038
2039             if (!$snapname && valid_drivename($key)) {
2040                 my $drive = parse_drive($key, $value);
2041                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2042             }
2043         }
2044     };
2045
2046     &$cleanup_config($conf);
2047
2048     &$cleanup_config($conf->{pending}, 1);
2049
2050     foreach my $snapname (keys %{$conf->{snapshots}}) {
2051         die "internal error" if $snapname eq 'pending';
2052         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2053     }
2054
2055     # remove 'unusedX' settings if we re-add a volume
2056     foreach my $key (keys %$conf) {
2057         my $value = $conf->{$key};
2058         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2059             delete $conf->{$key};
2060         }
2061     }
2062
2063     my $generate_raw_config = sub {
2064         my ($conf, $pending) = @_;
2065
2066         my $raw = '';
2067
2068         # add description as comment to top of file
2069         if (defined(my $descr = $conf->{description})) {
2070             if ($descr) {
2071                 foreach my $cl (split(/\n/, $descr)) {
2072                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2073                 }
2074             } else {
2075                 $raw .= "#\n" if $pending;
2076             }
2077         }
2078
2079         foreach my $key (sort keys %$conf) {
2080             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2081             $raw .= "$key: $conf->{$key}\n";
2082         }
2083         return $raw;
2084     };
2085
2086     my $raw = &$generate_raw_config($conf);
2087
2088     if (scalar(keys %{$conf->{pending}})){
2089         $raw .= "\n[PENDING]\n";
2090         $raw .= &$generate_raw_config($conf->{pending}, 1);
2091     }
2092
2093     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2094         $raw .= "\n[$snapname]\n";
2095         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2096     }
2097
2098     return $raw;
2099 }
2100
2101 sub update_config_nolock {
2102     my ($vmid, $conf, $skiplock) = @_;
2103
2104     check_lock($conf) if !$skiplock;
2105
2106     my $cfspath = cfs_config_path($vmid);
2107
2108     PVE::Cluster::cfs_write_file($cfspath, $conf);
2109 }
2110
2111 sub update_config {
2112     my ($vmid, $conf, $skiplock) = @_;
2113
2114     lock_config($vmid, &update_config_nolock, $conf, $skiplock);
2115 }
2116
2117 sub load_defaults {
2118
2119     my $res = {};
2120
2121     # we use static defaults from our JSON schema configuration
2122     foreach my $key (keys %$confdesc) {
2123         if (defined(my $default = $confdesc->{$key}->{default})) {
2124             $res->{$key} = $default;
2125         }
2126     }
2127
2128     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2129     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2130
2131     return $res;
2132 }
2133
2134 sub config_list {
2135     my $vmlist = PVE::Cluster::get_vmlist();
2136     my $res = {};
2137     return $res if !$vmlist || !$vmlist->{ids};
2138     my $ids = $vmlist->{ids};
2139
2140     foreach my $vmid (keys %$ids) {
2141         my $d = $ids->{$vmid};
2142         next if !$d->{node} || $d->{node} ne $nodename;
2143         next if !$d->{type} || $d->{type} ne 'qemu';
2144         $res->{$vmid}->{exists} = 1;
2145     }
2146     return $res;
2147 }
2148
2149 # test if VM uses local resources (to prevent migration)
2150 sub check_local_resources {
2151     my ($conf, $noerr) = @_;
2152
2153     my $loc_res = 0;
2154
2155     $loc_res = 1 if $conf->{hostusb}; # old syntax
2156     $loc_res = 1 if $conf->{hostpci}; # old syntax
2157
2158     foreach my $k (keys %$conf) {
2159         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2160         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2161     }
2162
2163     die "VM uses local resources\n" if $loc_res && !$noerr;
2164
2165     return $loc_res;
2166 }
2167
2168 # check if used storages are available on all nodes (use by migrate)
2169 sub check_storage_availability {
2170     my ($storecfg, $conf, $node) = @_;
2171
2172     foreach_drive($conf, sub {
2173         my ($ds, $drive) = @_;
2174
2175         my $volid = $drive->{file};
2176         return if !$volid;
2177
2178         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2179         return if !$sid;
2180
2181         # check if storage is available on both nodes
2182         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2183         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2184    });
2185 }
2186
2187 # list nodes where all VM images are available (used by has_feature API)
2188 sub shared_nodes {
2189     my ($conf, $storecfg) = @_;
2190
2191     my $nodelist = PVE::Cluster::get_nodelist();
2192     my $nodehash = { map { $_ => 1 } @$nodelist };
2193     my $nodename = PVE::INotify::nodename();
2194
2195     foreach_drive($conf, sub {
2196         my ($ds, $drive) = @_;
2197
2198         my $volid = $drive->{file};
2199         return if !$volid;
2200
2201         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2202         if ($storeid) {
2203             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2204             if ($scfg->{disable}) {
2205                 $nodehash = {};
2206             } elsif (my $avail = $scfg->{nodes}) {
2207                 foreach my $node (keys %$nodehash) {
2208                     delete $nodehash->{$node} if !$avail->{$node};
2209                 }
2210             } elsif (!$scfg->{shared}) {
2211                 foreach my $node (keys %$nodehash) {
2212                     delete $nodehash->{$node} if $node ne $nodename
2213                 }
2214             }
2215         }
2216     });
2217
2218     return $nodehash
2219 }
2220
2221 sub check_lock {
2222     my ($conf) = @_;
2223
2224     die "VM is locked ($conf->{lock})\n" if $conf->{lock};
2225 }
2226
2227 sub check_cmdline {
2228     my ($pidfile, $pid) = @_;
2229
2230     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2231     if (defined($fh)) {
2232         my $line = <$fh>;
2233         $fh->close;
2234         return undef if !$line;
2235         my @param = split(/\0/, $line);
2236
2237         my $cmd = $param[0];
2238         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2239
2240         for (my $i = 0; $i < scalar (@param); $i++) {
2241             my $p = $param[$i];
2242             next if !$p;
2243             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2244                 my $p = $param[$i+1];
2245                 return 1 if $p && ($p eq $pidfile);
2246                 return undef;
2247             }
2248         }
2249     }
2250     return undef;
2251 }
2252
2253 sub check_running {
2254     my ($vmid, $nocheck, $node) = @_;
2255
2256     my $filename = config_file($vmid, $node);
2257
2258     die "unable to find configuration file for VM $vmid - no such machine\n"
2259         if !$nocheck && ! -f $filename;
2260
2261     my $pidfile = pidfile_name($vmid);
2262
2263     if (my $fd = IO::File->new("<$pidfile")) {
2264         my $st = stat($fd);
2265         my $line = <$fd>;
2266         close($fd);
2267
2268         my $mtime = $st->mtime;
2269         if ($mtime > time()) {
2270             warn "file '$filename' modified in future\n";
2271         }
2272
2273         if ($line =~ m/^(\d+)$/) {
2274             my $pid = $1;
2275             if (check_cmdline($pidfile, $pid)) {
2276                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2277                     return $pid;
2278                 }
2279             }
2280         }
2281     }
2282
2283     return undef;
2284 }
2285
2286 sub vzlist {
2287
2288     my $vzlist = config_list();
2289
2290     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2291
2292     while (defined(my $de = $fd->read)) {
2293         next if $de !~ m/^(\d+)\.pid$/;
2294         my $vmid = $1;
2295         next if !defined($vzlist->{$vmid});
2296         if (my $pid = check_running($vmid)) {
2297             $vzlist->{$vmid}->{pid} = $pid;
2298         }
2299     }
2300
2301     return $vzlist;
2302 }
2303
2304 sub disksize {
2305     my ($storecfg, $conf) = @_;
2306
2307     my $bootdisk = $conf->{bootdisk};
2308     return undef if !$bootdisk;
2309     return undef if !valid_drivename($bootdisk);
2310
2311     return undef if !$conf->{$bootdisk};
2312
2313     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2314     return undef if !defined($drive);
2315
2316     return undef if drive_is_cdrom($drive);
2317
2318     my $volid = $drive->{file};
2319     return undef if !$volid;
2320
2321     return $drive->{size};
2322 }
2323
2324 my $last_proc_pid_stat;
2325
2326 # get VM status information
2327 # This must be fast and should not block ($full == false)
2328 # We only query KVM using QMP if $full == true (this can be slow)
2329 sub vmstatus {
2330     my ($opt_vmid, $full) = @_;
2331
2332     my $res = {};
2333
2334     my $storecfg = PVE::Storage::config();
2335
2336     my $list = vzlist();
2337     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2338
2339     my $cpucount = $cpuinfo->{cpus} || 1;
2340
2341     foreach my $vmid (keys %$list) {
2342         next if $opt_vmid && ($vmid ne $opt_vmid);
2343
2344         my $cfspath = cfs_config_path($vmid);
2345         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2346
2347         my $d = {};
2348         $d->{pid} = $list->{$vmid}->{pid};
2349
2350         # fixme: better status?
2351         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2352
2353         my $size = disksize($storecfg, $conf);
2354         if (defined($size)) {
2355             $d->{disk} = 0; # no info available
2356             $d->{maxdisk} = $size;
2357         } else {
2358             $d->{disk} = 0;
2359             $d->{maxdisk} = 0;
2360         }
2361
2362         $d->{cpus} = ($conf->{sockets} || 1) * ($conf->{cores} || 1);
2363         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2364         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2365
2366         $d->{name} = $conf->{name} || "VM $vmid";
2367         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024) : 0;
2368
2369         if ($conf->{balloon}) {
2370             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2371             $d->{shares} = defined($conf->{shares}) ? $conf->{shares} : 1000;
2372         }
2373
2374         $d->{uptime} = 0;
2375         $d->{cpu} = 0;
2376         $d->{mem} = 0;
2377
2378         $d->{netout} = 0;
2379         $d->{netin} = 0;
2380
2381         $d->{diskread} = 0;
2382         $d->{diskwrite} = 0;
2383
2384         $d->{template} = is_template($conf);
2385
2386         $res->{$vmid} = $d;
2387     }
2388
2389     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2390     foreach my $dev (keys %$netdev) {
2391         next if $dev !~ m/^tap([1-9]\d*)i/;
2392         my $vmid = $1;
2393         my $d = $res->{$vmid};
2394         next if !$d;
2395
2396         $d->{netout} += $netdev->{$dev}->{receive};
2397         $d->{netin} += $netdev->{$dev}->{transmit};
2398
2399         if ($full) {
2400             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2401             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2402         }
2403
2404     }
2405
2406     my $ctime = gettimeofday;
2407
2408     foreach my $vmid (keys %$list) {
2409
2410         my $d = $res->{$vmid};
2411         my $pid = $d->{pid};
2412         next if !$pid;
2413
2414         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2415         next if !$pstat; # not running
2416
2417         my $used = $pstat->{utime} + $pstat->{stime};
2418
2419         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2420
2421         if ($pstat->{vsize}) {
2422             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2423         }
2424
2425         my $old = $last_proc_pid_stat->{$pid};
2426         if (!$old) {
2427             $last_proc_pid_stat->{$pid} = {
2428                 time => $ctime,
2429                 used => $used,
2430                 cpu => 0,
2431             };
2432             next;
2433         }
2434
2435         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2436
2437         if ($dtime > 1000) {
2438             my $dutime = $used -  $old->{used};
2439
2440             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2441             $last_proc_pid_stat->{$pid} = {
2442                 time => $ctime,
2443                 used => $used,
2444                 cpu => $d->{cpu},
2445             };
2446         } else {
2447             $d->{cpu} = $old->{cpu};
2448         }
2449     }
2450
2451     return $res if !$full;
2452
2453     my $qmpclient = PVE::QMPClient->new();
2454
2455     my $ballooncb = sub {
2456         my ($vmid, $resp) = @_;
2457
2458         my $info = $resp->{'return'};
2459         return if !$info->{max_mem};
2460
2461         my $d = $res->{$vmid};
2462
2463         # use memory assigned to VM
2464         $d->{maxmem} = $info->{max_mem};
2465         $d->{balloon} = $info->{actual};
2466
2467         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2468             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2469             $d->{freemem} = $info->{free_mem};
2470         }
2471
2472         $d->{ballooninfo} = $info;
2473     };
2474
2475     my $blockstatscb = sub {
2476         my ($vmid, $resp) = @_;
2477         my $data = $resp->{'return'} || [];
2478         my $totalrdbytes = 0;
2479         my $totalwrbytes = 0;
2480
2481         for my $blockstat (@$data) {
2482             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2483             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2484
2485             $blockstat->{device} =~ s/drive-//;
2486             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2487         }
2488         $res->{$vmid}->{diskread} = $totalrdbytes;
2489         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2490     };
2491
2492     my $statuscb = sub {
2493         my ($vmid, $resp) = @_;
2494
2495         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2496         # this fails if ballon driver is not loaded, so this must be
2497         # the last commnand (following command are aborted if this fails).
2498         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2499
2500         my $status = 'unknown';
2501         if (!defined($status = $resp->{'return'}->{status})) {
2502             warn "unable to get VM status\n";
2503             return;
2504         }
2505
2506         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2507     };
2508
2509     foreach my $vmid (keys %$list) {
2510         next if $opt_vmid && ($vmid ne $opt_vmid);
2511         next if !$res->{$vmid}->{pid}; # not running
2512         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2513     }
2514
2515     $qmpclient->queue_execute(undef, 1);
2516
2517     foreach my $vmid (keys %$list) {
2518         next if $opt_vmid && ($vmid ne $opt_vmid);
2519         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2520     }
2521
2522     return $res;
2523 }
2524
2525 sub foreach_dimm {
2526     my ($conf, $vmid, $memory, $sockets, $func) = @_;
2527
2528     my $dimm_id = 0;
2529     my $current_size = 1024;
2530     my $dimm_size = 512;
2531     return if $current_size == $memory;
2532
2533     for (my $j = 0; $j < 8; $j++) {
2534         for (my $i = 0; $i < 32; $i++) {
2535             my $name = "dimm${dimm_id}";
2536             $dimm_id++;
2537             my $numanode = $i % $sockets;
2538             $current_size += $dimm_size;
2539             &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
2540             return  $current_size if $current_size >= $memory;
2541         }
2542         $dimm_size *= 2;
2543     }
2544 }
2545
2546 sub foreach_reverse_dimm {
2547     my ($conf, $vmid, $memory, $sockets, $func) = @_;
2548
2549     my $dimm_id = 253;
2550     my $current_size = 4177920;
2551     my $dimm_size = 65536;
2552     return if $current_size == $memory;
2553
2554     for (my $j = 0; $j < 8; $j++) {
2555         for (my $i = 0; $i < 32; $i++) {
2556             my $name = "dimm${dimm_id}";
2557             $dimm_id--;
2558             my $numanode = $i % $sockets;
2559             $current_size -= $dimm_size;
2560             &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
2561             return  $current_size if $current_size <= $memory;
2562         }
2563         $dimm_size /= 2;
2564     }
2565 }
2566
2567 sub foreach_drive {
2568     my ($conf, $func) = @_;
2569
2570     foreach my $ds (keys %$conf) {
2571         next if !valid_drivename($ds);
2572
2573         my $drive = parse_drive($ds, $conf->{$ds});
2574         next if !$drive;
2575
2576         &$func($ds, $drive);
2577     }
2578 }
2579
2580 sub foreach_volid {
2581     my ($conf, $func) = @_;
2582
2583     my $volhash = {};
2584
2585     my $test_volid = sub {
2586         my ($volid, $is_cdrom) = @_;
2587
2588         return if !$volid;
2589
2590         $volhash->{$volid} = $is_cdrom || 0;
2591     };
2592
2593     foreach_drive($conf, sub {
2594         my ($ds, $drive) = @_;
2595         &$test_volid($drive->{file}, drive_is_cdrom($drive));
2596     });
2597
2598     foreach my $snapname (keys %{$conf->{snapshots}}) {
2599         my $snap = $conf->{snapshots}->{$snapname};
2600         &$test_volid($snap->{vmstate}, 0);
2601         foreach_drive($snap, sub {
2602             my ($ds, $drive) = @_;
2603             &$test_volid($drive->{file}, drive_is_cdrom($drive));
2604         });
2605     }
2606
2607     foreach my $volid (keys %$volhash) {
2608         &$func($volid, $volhash->{$volid});
2609     }
2610 }
2611
2612 sub vga_conf_has_spice {
2613     my ($vga) = @_;
2614
2615     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
2616
2617     return $1 || 1;
2618 }
2619
2620 sub config_to_command {
2621     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2622
2623     my $cmd = [];
2624     my $globalFlags = [];
2625     my $machineFlags = [];
2626     my $rtcFlags = [];
2627     my $cpuFlags = [];
2628     my $devices = [];
2629     my $pciaddr = '';
2630     my $bridges = {};
2631     my $kvmver = kvm_user_version();
2632     my $vernum = 0; # unknown
2633     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
2634         $vernum = $1*1000000+$2*1000;
2635     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
2636         $vernum = $1*1000000+$2*1000+$3;
2637     }
2638
2639     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
2640
2641     my $have_ovz = -f '/proc/vz/vestat';
2642
2643     my $q35 = machine_type_is_q35($conf);
2644     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2645     my $machine_type = $forcemachine || $conf->{machine};
2646     my $use_old_bios_files = undef;
2647     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
2648
2649     my $cpuunits = defined($conf->{cpuunits}) ?
2650             $conf->{cpuunits} : $defaults->{cpuunits};
2651
2652     push @$cmd, '/usr/bin/systemd-run';
2653     push @$cmd, '--scope';
2654     push @$cmd, '--slice', "qemu";
2655     push @$cmd, '--unit', $vmid;
2656     push @$cmd, '-p', "CPUShares=$cpuunits";
2657     if ($conf->{cpulimit}) {
2658         my $cpulimit = int($conf->{cpulimit} * 100);
2659         push @$cmd, '-p', "CPUQuota=$cpulimit\%";
2660     }
2661
2662     push @$cmd, '/usr/bin/kvm';
2663
2664     push @$cmd, '-id', $vmid;
2665
2666     my $use_virtio = 0;
2667
2668     my $qmpsocket = qmp_socket($vmid);
2669     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2670     push @$cmd, '-mon', "chardev=qmp,mode=control";
2671
2672     my $socket = vnc_socket($vmid);
2673     push @$cmd,  '-vnc', "unix:$socket,x509,password";
2674
2675     push @$cmd, '-pidfile' , pidfile_name($vmid);
2676
2677     push @$cmd, '-daemonize';
2678
2679     if ($conf->{smbios1}) {
2680         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
2681     }
2682
2683     if ($q35) {
2684         # the q35 chipset support native usb2, so we enable usb controller
2685         # by default for this machine type
2686         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
2687     } else {
2688         $pciaddr = print_pci_addr("piix3", $bridges);
2689         push @$devices, '-device', "piix3-usb-uhci,id=uhci$pciaddr.0x2";
2690
2691         my $use_usb2 = 0;
2692         for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2693             next if !$conf->{"usb$i"};
2694             $use_usb2 = 1;
2695         }
2696         # include usb device config
2697         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-usb.cfg' if $use_usb2;
2698     }
2699
2700     my $vga = $conf->{vga};
2701
2702     my $qxlnum = vga_conf_has_spice($vga);
2703     $vga = 'qxl' if $qxlnum;
2704
2705     if (!$vga) {
2706         if ($conf->{ostype} && ($conf->{ostype} eq 'win8' ||
2707                                 $conf->{ostype} eq 'win7' ||
2708                                 $conf->{ostype} eq 'w2k8')) {
2709             $vga = 'std';
2710         } else {
2711             $vga = 'cirrus';
2712         }
2713     }
2714
2715     # enable absolute mouse coordinates (needed by vnc)
2716     my $tablet;
2717     if (defined($conf->{tablet})) {
2718         $tablet = $conf->{tablet};
2719     } else {
2720         $tablet = $defaults->{tablet};
2721         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
2722         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
2723     }
2724
2725     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
2726
2727     # host pci devices
2728     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
2729         my $d = parse_hostpci($conf->{"hostpci$i"});
2730         next if !$d;
2731
2732         my $pcie = $d->{pcie};
2733         if($pcie){
2734             die "q35 machine model is not enabled" if !$q35;
2735             $pciaddr = print_pcie_addr("hostpci$i");
2736         }else{
2737             $pciaddr = print_pci_addr("hostpci$i", $bridges);
2738         }
2739
2740         my $rombar = $d->{rombar} && $d->{rombar} eq 'off' ? ",rombar=0" : "";
2741         my $xvga = $d->{'x-vga'} && $d->{'x-vga'} eq 'on' ? ",x-vga=on" : "";
2742         if ($xvga && $xvga ne '') {
2743             push @$cpuFlags, 'kvm=off';
2744             $vga = 'none';
2745         }
2746         my $pcidevices = $d->{pciid};
2747         my $multifunction = 1 if @$pcidevices > 1;
2748
2749         my $j=0;
2750         foreach my $pcidevice (@$pcidevices) {
2751
2752             my $id = "hostpci$i";
2753             $id .= ".$j" if $multifunction;
2754             my $addr = $pciaddr;
2755             $addr .= ".$j" if $multifunction;
2756             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
2757
2758             if($j == 0){
2759                 $devicestr .= "$rombar$xvga";
2760                 $devicestr .= ",multifunction=on" if $multifunction;
2761             }
2762
2763             push @$devices, '-device', $devicestr;
2764             $j++;
2765         }
2766     }
2767
2768     # usb devices
2769     for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2770         my $d = parse_usb_device($conf->{"usb$i"});
2771         next if !$d;
2772         if ($d->{vendorid} && $d->{productid}) {
2773             push @$devices, '-device', "usb-host,vendorid=0x$d->{vendorid},productid=0x$d->{productid}";
2774         } elsif (defined($d->{hostbus}) && defined($d->{hostport})) {
2775             push @$devices, '-device', "usb-host,hostbus=$d->{hostbus},hostport=$d->{hostport}";
2776         } elsif ($d->{spice}) {
2777             # usb redir support for spice
2778             push @$devices, '-chardev', "spicevmc,id=usbredirchardev$i,name=usbredir";
2779             push @$devices, '-device', "usb-redir,chardev=usbredirchardev$i,id=usbredirdev$i,bus=ehci.0";
2780         }
2781     }
2782
2783     # serial devices
2784     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2785         if (my $path = $conf->{"serial$i"}) {
2786             if ($path eq 'socket') {
2787                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
2788                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
2789                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2790             } else {
2791                 die "no such serial device\n" if ! -c $path;
2792                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
2793                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2794             }
2795         }
2796     }
2797
2798     # parallel devices
2799     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
2800         if (my $path = $conf->{"parallel$i"}) {
2801             die "no such parallel device\n" if ! -c $path;
2802             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
2803             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
2804             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
2805         }
2806     }
2807
2808     my $vmname = $conf->{name} || "vm$vmid";
2809
2810     push @$cmd, '-name', $vmname;
2811
2812     my $sockets = 1;
2813     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
2814     $sockets = $conf->{sockets} if  $conf->{sockets};
2815
2816     my $cores = $conf->{cores} || 1;
2817
2818     my $maxcpus = $sockets * $cores;
2819
2820     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
2821
2822     my $allowed_vcpus = $cpuinfo->{cpus};
2823
2824     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
2825         if ($allowed_vcpus < $maxcpus);
2826
2827     push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
2828
2829     push @$cmd, '-nodefaults';
2830
2831     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2832
2833     my $bootindex_hash = {};
2834     my $i = 1;
2835     foreach my $o (split(//, $bootorder)) {
2836         $bootindex_hash->{$o} = $i*100;
2837         $i++;
2838     }
2839
2840     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000";
2841
2842     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
2843
2844     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
2845
2846     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
2847
2848     # time drift fix
2849     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
2850
2851     my $nokvm = defined($conf->{kvm}) && $conf->{kvm} == 0 ? 1 : 0;
2852     my $useLocaltime = $conf->{localtime};
2853
2854     if (my $ost = $conf->{ostype}) {
2855         # other, wxp, w2k, w2k3, w2k8, wvista, win7, win8, l24, l26, solaris
2856
2857         if ($ost =~ m/^w/) { # windows
2858             $useLocaltime = 1 if !defined($conf->{localtime});
2859
2860             # use time drift fix when acpi is enabled
2861             if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
2862                 $tdf = 1 if !defined($conf->{tdf});
2863             }
2864         }
2865
2866         if ($ost eq 'win7' || $ost eq 'win8' || $ost eq 'w2k8' ||
2867             $ost eq 'wvista') {
2868             push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
2869             push @$cmd, '-no-hpet';
2870             if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
2871                 push @$cpuFlags , 'hv_spinlocks=0x1fff' if !$nokvm;
2872                 push @$cpuFlags , 'hv_vapic' if !$nokvm;
2873                 push @$cpuFlags , 'hv_time' if !$nokvm;
2874
2875             } else {
2876                 push @$cpuFlags , 'hv_spinlocks=0xffff' if !$nokvm;
2877             }
2878         }
2879
2880         if ($ost eq 'win7' || $ost eq 'win8') {
2881             push @$cpuFlags , 'hv_relaxed' if !$nokvm;
2882         }
2883     }
2884
2885     push @$rtcFlags, 'driftfix=slew' if $tdf;
2886
2887     if ($nokvm) {
2888         push @$machineFlags, 'accel=tcg';
2889     } else {
2890         die "No accelerator found!\n" if !$cpuinfo->{hvm};
2891     }
2892
2893     if ($machine_type) {
2894         push @$machineFlags, "type=${machine_type}";
2895     }
2896
2897     if ($conf->{startdate}) {
2898         push @$rtcFlags, "base=$conf->{startdate}";
2899     } elsif ($useLocaltime) {
2900         push @$rtcFlags, 'base=localtime';
2901     }
2902
2903     my $cpu = $nokvm ? "qemu64" : "kvm64";
2904     $cpu = $conf->{cpu} if $conf->{cpu};
2905
2906     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
2907
2908     push @$cpuFlags , '-x2apic'
2909         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
2910
2911     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
2912
2913     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
2914
2915     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
2916
2917         push @$cpuFlags , '+kvm_pv_unhalt' if !$nokvm;
2918         push @$cpuFlags , '+kvm_pv_eoi' if !$nokvm;
2919         push @$cpuFlags , '-kvm_steal_time' if !$nokvm;
2920     }
2921
2922     push @$cpuFlags, 'enforce' if $cpu ne 'host' && !$nokvm;
2923
2924     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
2925
2926     push @$cmd, '-cpu', $cpu;
2927
2928     my $memory = $conf->{memory} || $defaults->{memory};
2929     my $static_memory = 0;
2930     my $dimm_memory = 0;
2931
2932     if ($hotplug_features->{memory}) {
2933         die "Numa need to be enabled for memory hotplug\n" if !$conf->{numa};
2934         die "Total memory is bigger than ${MAX_MEM}MB\n" if $memory > $MAX_MEM;
2935         $static_memory = $STATICMEM;
2936         die "minimum memory must be ${static_memory}MB\n" if($memory < $static_memory);
2937         $dimm_memory = $memory - $static_memory;
2938         push @$cmd, '-m', "size=${static_memory},slots=255,maxmem=${MAX_MEM}M";
2939
2940     } else {
2941
2942         $static_memory = $memory;
2943         push @$cmd, '-m', $static_memory;
2944     }
2945
2946     if ($conf->{numa}) {
2947
2948         my $numa_totalmemory = undef;
2949         for (my $i = 0; $i < $MAX_NUMA; $i++) {
2950             next if !$conf->{"numa$i"};
2951             my $numa = parse_numa($conf->{"numa$i"});
2952             next if !$numa;
2953             # memory
2954             die "missing numa node$i memory value\n" if !$numa->{memory};
2955             my $numa_memory = $numa->{memory};
2956             $numa_totalmemory += $numa_memory;
2957             my $numa_object = "memory-backend-ram,id=ram-node$i,size=${numa_memory}M";
2958
2959             # cpus
2960             my $cpus_start = $numa->{cpus}->{start};
2961             die "missing numa node$i cpus\n" if !defined($cpus_start);
2962             my $cpus_end = $numa->{cpus}->{end} if defined($numa->{cpus}->{end});
2963             my $cpus = $cpus_start;
2964             if (defined($cpus_end)) {
2965                 $cpus .= "-$cpus_end";
2966                 die "numa node$i :  cpu range $cpus is incorrect\n" if $cpus_end <= $cpus_start;
2967             }
2968
2969             # hostnodes
2970             my $hostnodes_start = $numa->{hostnodes}->{start};
2971             if (defined($hostnodes_start)) {
2972                 my $hostnodes_end = $numa->{hostnodes}->{end} if defined($numa->{hostnodes}->{end});
2973                 my $hostnodes = $hostnodes_start;
2974                 if (defined($hostnodes_end)) {
2975                     $hostnodes .= "-$hostnodes_end";
2976                     die "host node $hostnodes range is incorrect\n" if $hostnodes_end <= $hostnodes_start;
2977                 }
2978
2979                 my $hostnodes_end_range = defined($hostnodes_end) ? $hostnodes_end : $hostnodes_start;
2980                 for (my $i = $hostnodes_start; $i <= $hostnodes_end_range; $i++ ) {
2981                     die "host numa node$i don't exist\n" if ! -d "/sys/devices/system/node/node$i/";
2982                 }
2983
2984                 # policy
2985                 my $policy = $numa->{policy};
2986                 die "you need to define a policy for hostnode $hostnodes\n" if !$policy;
2987                 $numa_object .= ",host-nodes=$hostnodes,policy=$policy";
2988             }
2989
2990             push @$cmd, '-object', $numa_object;
2991             push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
2992         }
2993
2994         die "total memory for NUMA nodes must be equal to vm static memory\n"
2995             if $numa_totalmemory && $numa_totalmemory != $static_memory;
2996
2997         #if no custom tology, we split memory and cores across numa nodes
2998         if(!$numa_totalmemory) {
2999
3000             my $numa_memory = ($static_memory / $sockets) . "M";
3001
3002             for (my $i = 0; $i < $sockets; $i++)  {
3003
3004                 my $cpustart = ($cores * $i);
3005                 my $cpuend = ($cpustart + $cores - 1) if $cores && $cores > 1;
3006                 my $cpus = $cpustart;
3007                 $cpus .= "-$cpuend" if $cpuend;
3008
3009                 push @$cmd, '-object', "memory-backend-ram,size=$numa_memory,id=ram-node$i";
3010                 push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
3011             }
3012         }
3013     }
3014
3015     if ($hotplug_features->{memory}) {
3016         foreach_dimm($conf, $vmid, $memory, $sockets, sub {
3017             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3018             push @$cmd, "-object" , "memory-backend-ram,id=mem-$name,size=${dimm_size}M";
3019             push @$cmd, "-device", "pc-dimm,id=$name,memdev=mem-$name,node=$numanode";
3020
3021             #if dimm_memory is not aligned to dimm map
3022             if($current_size > $memory) {
3023                  $conf->{memory} = $current_size;
3024                  update_config_nolock($vmid, $conf, 1);
3025             }
3026         });
3027     }
3028
3029     push @$cmd, '-S' if $conf->{freeze};
3030
3031     # set keyboard layout
3032     my $kb = $conf->{keyboard} || $defaults->{keyboard};
3033     push @$cmd, '-k', $kb if $kb;
3034
3035     # enable sound
3036     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3037     #push @$cmd, '-soundhw', 'es1370';
3038     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3039
3040     if($conf->{agent}) {
3041         my $qgasocket = qmp_socket($vmid, 1);
3042         my $pciaddr = print_pci_addr("qga0", $bridges);
3043         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3044         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3045         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3046     }
3047
3048     my $spice_port;
3049
3050     if ($qxlnum) {
3051         if ($qxlnum > 1) {
3052             if ($conf->{ostype} && $conf->{ostype} =~ m/^w/){
3053                 for(my $i = 1; $i < $qxlnum; $i++){
3054                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3055                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3056                 }
3057             } else {
3058                 # assume other OS works like Linux
3059                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3060                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3061             }
3062         }
3063
3064         my $pciaddr = print_pci_addr("spice", $bridges);
3065
3066         my $nodename = PVE::INotify::nodename();
3067         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3068         $spice_port = PVE::Tools::next_spice_port($pfamily);
3069
3070         push @$devices, '-spice', "tls-port=${spice_port},addr=localhost,tls-ciphers=DES-CBC3-SHA,seamless-migration=on";
3071
3072         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3073         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3074         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3075     }
3076
3077     # enable balloon by default, unless explicitly disabled
3078     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3079         $pciaddr = print_pci_addr("balloon0", $bridges);
3080         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3081     }
3082
3083     if ($conf->{watchdog}) {
3084         my $wdopts = parse_watchdog($conf->{watchdog});
3085         $pciaddr = print_pci_addr("watchdog", $bridges);
3086         my $watchdog = $wdopts->{model} || 'i6300esb';
3087         push @$devices, '-device', "$watchdog$pciaddr";
3088         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3089     }
3090
3091     my $vollist = [];
3092     my $scsicontroller = {};
3093     my $ahcicontroller = {};
3094     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3095
3096     # Add iscsi initiator name if available
3097     if (my $initiator = get_initiator_name()) {
3098         push @$devices, '-iscsi', "initiator-name=$initiator";
3099     }
3100
3101     foreach_drive($conf, sub {
3102         my ($ds, $drive) = @_;
3103
3104         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3105             push @$vollist, $drive->{file};
3106         }
3107
3108         $use_virtio = 1 if $ds =~ m/^virtio/;
3109
3110         if (drive_is_cdrom ($drive)) {
3111             if ($bootindex_hash->{d}) {
3112                 $drive->{bootindex} = $bootindex_hash->{d};
3113                 $bootindex_hash->{d} += 1;
3114             }
3115         } else {
3116             if ($bootindex_hash->{c}) {
3117                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3118                 $bootindex_hash->{c} += 1;
3119             }
3120         }
3121
3122         if($drive->{interface} eq 'virtio'){
3123            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3124         }
3125
3126         if ($drive->{interface} eq 'scsi') {
3127
3128             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3129
3130             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3131             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3132
3133             my $iothread = '';
3134             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3135                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3136                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3137             }
3138
3139             my $queues = '';
3140             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3141                 $queues = ",num_queues=$drive->{queues}";
3142             }
3143
3144             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3145             $scsicontroller->{$controller}=1;
3146         }
3147
3148         if ($drive->{interface} eq 'sata') {
3149            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3150            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3151            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3152            $ahcicontroller->{$controller}=1;
3153         }
3154
3155         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3156         push @$devices, '-drive',$drive_cmd;
3157         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3158     });
3159
3160     for (my $i = 0; $i < $MAX_NETS; $i++) {
3161          next if !$conf->{"net$i"};
3162          my $d = parse_net($conf->{"net$i"});
3163          next if !$d;
3164
3165          $use_virtio = 1 if $d->{model} eq 'virtio';
3166
3167          if ($bootindex_hash->{n}) {
3168             $d->{bootindex} = $bootindex_hash->{n};
3169             $bootindex_hash->{n} += 1;
3170          }
3171
3172          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3173          push @$devices, '-netdev', $netdevfull;
3174
3175          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3176          push @$devices, '-device', $netdevicefull;
3177     }
3178
3179     if (!$q35) {
3180         # add pci bridges
3181         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3182            $bridges->{1} = 1;
3183            $bridges->{2} = 1;
3184         }
3185
3186         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3187
3188         while (my ($k, $v) = each %$bridges) {
3189             $pciaddr = print_pci_addr("pci.$k");
3190             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3191         }
3192     }
3193
3194     # add custom args
3195     if ($conf->{args}) {
3196         my $aa = PVE::Tools::split_args($conf->{args});
3197         push @$cmd, @$aa;
3198     }
3199
3200     push @$cmd, @$devices;
3201     push @$cmd, '-rtc', join(',', @$rtcFlags)
3202         if scalar(@$rtcFlags);
3203     push @$cmd, '-machine', join(',', @$machineFlags)
3204         if scalar(@$machineFlags);
3205     push @$cmd, '-global', join(',', @$globalFlags)
3206         if scalar(@$globalFlags);
3207
3208     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3209 }
3210
3211 sub vnc_socket {
3212     my ($vmid) = @_;
3213     return "${var_run_tmpdir}/$vmid.vnc";
3214 }
3215
3216 sub spice_port {
3217     my ($vmid) = @_;
3218
3219     my $res = vm_mon_cmd($vmid, 'query-spice');
3220
3221     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3222 }
3223
3224 sub qmp_socket {
3225     my ($vmid, $qga) = @_;
3226     my $sockettype = $qga ? 'qga' : 'qmp';
3227     return "${var_run_tmpdir}/$vmid.$sockettype";
3228 }
3229
3230 sub pidfile_name {
3231     my ($vmid) = @_;
3232     return "${var_run_tmpdir}/$vmid.pid";
3233 }
3234
3235 sub vm_devices_list {
3236     my ($vmid) = @_;
3237
3238     my $res = vm_mon_cmd($vmid, 'query-pci');
3239     my $devices = {};
3240     foreach my $pcibus (@$res) {
3241         foreach my $device (@{$pcibus->{devices}}) {
3242             next if !$device->{'qdev_id'};
3243             if ($device->{'pci_bridge'}) {
3244                 $devices->{$device->{'qdev_id'}} = 1;
3245                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3246                     next if !$bridge_device->{'qdev_id'};
3247                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3248                     $devices->{$device->{'qdev_id'}}++;
3249                 }
3250             } else {
3251                 $devices->{$device->{'qdev_id'}} = 1;
3252             }
3253         }
3254     }
3255
3256     my $resblock = vm_mon_cmd($vmid, 'query-block');
3257     foreach my $block (@$resblock) {
3258         if($block->{device} =~ m/^drive-(\S+)/){
3259                 $devices->{$1} = 1;
3260         }
3261     }
3262
3263     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3264     foreach my $mice (@$resmice) {
3265         if ($mice->{name} eq 'QEMU HID Tablet') {
3266             $devices->{tablet} = 1;
3267             last;
3268         }
3269     }
3270
3271     return $devices;
3272 }
3273
3274 sub vm_deviceplug {
3275     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3276
3277     my $q35 = machine_type_is_q35($conf);
3278
3279     my $devices_list = vm_devices_list($vmid);
3280     return 1 if defined($devices_list->{$deviceid});
3281
3282     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3283
3284     if ($deviceid eq 'tablet') {
3285
3286         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3287
3288     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3289
3290         qemu_iothread_add($vmid, $deviceid, $device);
3291
3292         qemu_driveadd($storecfg, $vmid, $device);
3293         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3294
3295         qemu_deviceadd($vmid, $devicefull);
3296         eval { qemu_deviceaddverify($vmid, $deviceid); };
3297         if (my $err = $@) {
3298             eval { qemu_drivedel($vmid, $deviceid); };
3299             warn $@ if $@;
3300             die $err;
3301         }
3302
3303     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3304
3305
3306         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3307         my $pciaddr = print_pci_addr($deviceid);
3308         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3309
3310         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3311
3312         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3313             qemu_iothread_add($vmid, $deviceid, $device);
3314             $devicefull .= ",iothread=iothread-$deviceid";
3315         }
3316
3317         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3318             $devicefull .= ",num_queues=$device->{queues}";
3319         }
3320
3321         qemu_deviceadd($vmid, $devicefull);
3322         qemu_deviceaddverify($vmid, $deviceid);
3323
3324     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3325
3326         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3327         qemu_driveadd($storecfg, $vmid, $device);
3328
3329         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3330         eval { qemu_deviceadd($vmid, $devicefull); };
3331         if (my $err = $@) {
3332             eval { qemu_drivedel($vmid, $deviceid); };
3333             warn $@ if $@;
3334             die $err;
3335         }
3336
3337     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3338
3339         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3340
3341         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
3342         my $use_old_bios_files = undef;
3343         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3344
3345         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3346         qemu_deviceadd($vmid, $netdevicefull);
3347         eval { qemu_deviceaddverify($vmid, $deviceid); };
3348         if (my $err = $@) {
3349             eval { qemu_netdevdel($vmid, $deviceid); };
3350             warn $@ if $@;
3351             die $err;
3352         }
3353
3354     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3355
3356         my $bridgeid = $2;
3357         my $pciaddr = print_pci_addr($deviceid);
3358         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3359
3360         qemu_deviceadd($vmid, $devicefull);
3361         qemu_deviceaddverify($vmid, $deviceid);
3362
3363     } else {
3364         die "can't hotplug device '$deviceid'\n";
3365     }
3366
3367     return 1;
3368 }
3369
3370 # fixme: this should raise exceptions on error!
3371 sub vm_deviceunplug {
3372     my ($vmid, $conf, $deviceid) = @_;
3373
3374     my $devices_list = vm_devices_list($vmid);
3375     return 1 if !defined($devices_list->{$deviceid});
3376
3377     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3378
3379     if ($deviceid eq 'tablet') {
3380
3381         qemu_devicedel($vmid, $deviceid);
3382
3383     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3384
3385         qemu_devicedel($vmid, $deviceid);
3386         qemu_devicedelverify($vmid, $deviceid);
3387         qemu_drivedel($vmid, $deviceid);
3388         qemu_iothread_del($conf, $vmid, $deviceid);
3389
3390     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3391
3392         qemu_devicedel($vmid, $deviceid);
3393         qemu_devicedelverify($vmid, $deviceid);
3394         qemu_iothread_del($conf, $vmid, $deviceid);
3395
3396     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3397
3398         #qemu 2.3 segfault on drive_del with virtioscsi + iothread
3399         my $device = parse_drive($deviceid, $conf->{$deviceid});
3400         die "virtioscsi with iothread is not hot-unplugglable currently" if $device->{iothread};
3401
3402         qemu_devicedel($vmid, $deviceid);
3403         qemu_drivedel($vmid, $deviceid);
3404         qemu_deletescsihw($conf, $vmid, $deviceid);
3405
3406     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3407
3408         qemu_devicedel($vmid, $deviceid);
3409         qemu_devicedelverify($vmid, $deviceid);
3410         qemu_netdevdel($vmid, $deviceid);
3411
3412     } else {
3413         die "can't unplug device '$deviceid'\n";
3414     }
3415
3416     return 1;
3417 }
3418
3419 sub qemu_deviceadd {
3420     my ($vmid, $devicefull) = @_;
3421
3422     $devicefull = "driver=".$devicefull;
3423     my %options =  split(/[=,]/, $devicefull);
3424
3425     vm_mon_cmd($vmid, "device_add" , %options);
3426 }
3427
3428 sub qemu_devicedel {
3429     my ($vmid, $deviceid) = @_;
3430
3431     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3432 }
3433
3434 sub qemu_iothread_add {
3435     my($vmid, $deviceid, $device) = @_;
3436
3437     if ($device->{iothread}) {
3438         my $iothreads = vm_iothreads_list($vmid);
3439         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3440     }
3441 }
3442
3443 sub qemu_iothread_del {
3444     my($conf, $vmid, $deviceid) = @_;
3445
3446     my $device = parse_drive($deviceid, $conf->{$deviceid});
3447     if ($device->{iothread}) {
3448         my $iothreads = vm_iothreads_list($vmid);
3449         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3450     }
3451 }
3452
3453 sub qemu_objectadd {
3454     my($vmid, $objectid, $qomtype) = @_;
3455
3456     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3457
3458     return 1;
3459 }
3460
3461 sub qemu_objectdel {
3462     my($vmid, $objectid) = @_;
3463
3464     vm_mon_cmd($vmid, "object-del", id => $objectid);
3465
3466     return 1;
3467 }
3468
3469 sub qemu_driveadd {
3470     my ($storecfg, $vmid, $device) = @_;
3471
3472     my $drive = print_drive_full($storecfg, $vmid, $device);
3473     $drive =~ s/\\/\\\\/g;
3474     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3475
3476     # If the command succeeds qemu prints: "OK"
3477     return 1 if $ret =~ m/OK/s;
3478
3479     die "adding drive failed: $ret\n";
3480 }
3481
3482 sub qemu_drivedel {
3483     my($vmid, $deviceid) = @_;
3484
3485     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3486     $ret =~ s/^\s+//;
3487
3488     return 1 if $ret eq "";
3489
3490     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3491     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3492
3493     die "deleting drive $deviceid failed : $ret\n";
3494 }
3495
3496 sub qemu_deviceaddverify {
3497     my ($vmid, $deviceid) = @_;
3498
3499     for (my $i = 0; $i <= 5; $i++) {
3500          my $devices_list = vm_devices_list($vmid);
3501          return 1 if defined($devices_list->{$deviceid});
3502          sleep 1;
3503     }
3504
3505     die "error on hotplug device '$deviceid'\n";
3506 }
3507
3508
3509 sub qemu_devicedelverify {
3510     my ($vmid, $deviceid) = @_;
3511
3512     # need to verify that the device is correctly removed as device_del
3513     # is async and empty return is not reliable
3514
3515     for (my $i = 0; $i <= 5; $i++) {
3516          my $devices_list = vm_devices_list($vmid);
3517          return 1 if !defined($devices_list->{$deviceid});
3518          sleep 1;
3519     }
3520
3521     die "error on hot-unplugging device '$deviceid'\n";
3522 }
3523
3524 sub qemu_findorcreatescsihw {
3525     my ($storecfg, $conf, $vmid, $device) = @_;
3526
3527     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3528
3529     my $scsihwid="$controller_prefix$controller";
3530     my $devices_list = vm_devices_list($vmid);
3531
3532     if(!defined($devices_list->{$scsihwid})) {
3533         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3534     }
3535
3536     return 1;
3537 }
3538
3539 sub qemu_deletescsihw {
3540     my ($conf, $vmid, $opt) = @_;
3541
3542     my $device = parse_drive($opt, $conf->{$opt});
3543
3544     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3545         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3546         return 1;
3547     }
3548
3549     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3550
3551     my $devices_list = vm_devices_list($vmid);
3552     foreach my $opt (keys %{$devices_list}) {
3553         if (PVE::QemuServer::valid_drivename($opt)) {
3554             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3555             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3556                 return 1;
3557             }
3558         }
3559     }
3560
3561     my $scsihwid="scsihw$controller";
3562
3563     vm_deviceunplug($vmid, $conf, $scsihwid);
3564
3565     return 1;
3566 }
3567
3568 sub qemu_add_pci_bridge {
3569     my ($storecfg, $conf, $vmid, $device) = @_;
3570
3571     my $bridges = {};
3572
3573     my $bridgeid;
3574
3575     print_pci_addr($device, $bridges);
3576
3577     while (my ($k, $v) = each %$bridges) {
3578         $bridgeid = $k;
3579     }
3580     return 1 if !defined($bridgeid) || $bridgeid < 1;
3581
3582     my $bridge = "pci.$bridgeid";
3583     my $devices_list = vm_devices_list($vmid);
3584
3585     if (!defined($devices_list->{$bridge})) {
3586         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3587     }
3588
3589     return 1;
3590 }
3591
3592 sub qemu_set_link_status {
3593     my ($vmid, $device, $up) = @_;
3594
3595     vm_mon_cmd($vmid, "set_link", name => $device,
3596                up => $up ? JSON::true : JSON::false);
3597 }
3598
3599 sub qemu_netdevadd {
3600     my ($vmid, $conf, $device, $deviceid) = @_;
3601
3602     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid);
3603     my %options =  split(/[=,]/, $netdev);
3604
3605     vm_mon_cmd($vmid, "netdev_add",  %options);
3606     return 1;
3607 }
3608
3609 sub qemu_netdevdel {
3610     my ($vmid, $deviceid) = @_;
3611
3612     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3613 }
3614
3615 sub qemu_cpu_hotplug {
3616     my ($vmid, $conf, $vcpus) = @_;
3617
3618     my $sockets = 1;
3619     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3620     $sockets = $conf->{sockets} if  $conf->{sockets};
3621     my $cores = $conf->{cores} || 1;
3622     my $maxcpus = $sockets * $cores;
3623
3624     $vcpus = $maxcpus if !$vcpus;
3625
3626     die "you can't add more vcpus than maxcpus\n"
3627         if $vcpus > $maxcpus;
3628
3629     my $currentvcpus = $conf->{vcpus} || $maxcpus;
3630     die "online cpu unplug is not yet possible\n"
3631         if $vcpus < $currentvcpus;
3632
3633     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3634     die "vcpus in running vm is different than configuration\n"
3635         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
3636
3637     for (my $i = $currentvcpus; $i < $vcpus; $i++) {
3638         vm_mon_cmd($vmid, "cpu-add", id => int($i));
3639     }
3640 }
3641
3642 sub qemu_memory_hotplug {
3643     my ($vmid, $conf, $defaults, $opt, $value) = @_;
3644
3645     return $value if !check_running($vmid);
3646
3647     my $memory = $conf->{memory} || $defaults->{memory};
3648     $value = $defaults->{memory} if !$value;
3649     return $value if $value == $memory;
3650
3651     my $static_memory = $STATICMEM;
3652     my $dimm_memory = $memory - $static_memory;
3653
3654     die "memory can't be lower than $static_memory MB" if $value < $static_memory;
3655     die "you cannot add more memory than $MAX_MEM MB!\n" if $memory > $MAX_MEM;
3656
3657
3658     my $sockets = 1;
3659     $sockets = $conf->{sockets} if $conf->{sockets};
3660
3661     if($value > $memory) {
3662
3663         foreach_dimm($conf, $vmid, $value, $sockets, sub {
3664             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3665
3666                 return if $current_size <= $conf->{memory};
3667
3668                 eval { vm_mon_cmd($vmid, "object-add", 'qom-type' => "memory-backend-ram", id => "mem-$name", props => { size => int($dimm_size*1024*1024) } ) };
3669                 if (my $err = $@) {
3670                     eval { qemu_objectdel($vmid, "mem-$name"); };
3671                     die $err;
3672                 }
3673
3674                 eval { vm_mon_cmd($vmid, "device_add", driver => "pc-dimm", id => "$name", memdev => "mem-$name", node => $numanode) };
3675                 if (my $err = $@) {
3676                     eval { qemu_objectdel($vmid, "mem-$name"); };
3677                     die $err;
3678                 }
3679                 #update conf after each succesful module hotplug
3680                 $conf->{memory} = $current_size;
3681                 update_config_nolock($vmid, $conf, 1);
3682         });
3683
3684     } else {
3685
3686         foreach_reverse_dimm($conf, $vmid, $value, $sockets, sub {
3687             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3688
3689                 return if $current_size >= $conf->{memory};
3690                 print "try to unplug memory dimm $name\n";
3691
3692                 my $retry = 0;
3693                 while (1) {
3694                     eval { qemu_devicedel($vmid, $name) };
3695                     sleep 3;
3696                     my $dimm_list = qemu_dimm_list($vmid);
3697                     last if !$dimm_list->{$name};
3698                     raise_param_exc({ $name => "error unplug memory module" }) if $retry > 5;
3699                     $retry++;
3700                 }
3701
3702                 #update conf after each succesful module unplug
3703                 $conf->{memory} = $current_size;
3704
3705                 eval { qemu_objectdel($vmid, "mem-$name"); };
3706                 update_config_nolock($vmid, $conf, 1);
3707         });
3708     }
3709 }
3710
3711 sub qemu_dimm_list {
3712     my ($vmid) = @_;
3713
3714     my $dimmarray = vm_mon_cmd_nocheck($vmid, "query-memory-devices");
3715     my $dimms = {};
3716
3717     foreach my $dimm (@$dimmarray) {
3718
3719         $dimms->{$dimm->{data}->{id}}->{id} = $dimm->{data}->{id};
3720         $dimms->{$dimm->{data}->{id}}->{node} = $dimm->{data}->{node};
3721         $dimms->{$dimm->{data}->{id}}->{addr} = $dimm->{data}->{addr};
3722         $dimms->{$dimm->{data}->{id}}->{size} = $dimm->{data}->{size};
3723         $dimms->{$dimm->{data}->{id}}->{slot} = $dimm->{data}->{slot};
3724     }
3725     return $dimms;
3726 }
3727
3728 sub qemu_block_set_io_throttle {
3729     my ($vmid, $deviceid, $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr) = @_;
3730
3731     return if !check_running($vmid) ;
3732
3733     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid, bps => int($bps), bps_rd => int($bps_rd), bps_wr => int($bps_wr), iops => int($iops), iops_rd => int($iops_rd), iops_wr => int($iops_wr));
3734
3735 }
3736
3737 # old code, only used to shutdown old VM after update
3738 sub __read_avail {
3739     my ($fh, $timeout) = @_;
3740
3741     my $sel = new IO::Select;
3742     $sel->add($fh);
3743
3744     my $res = '';
3745     my $buf;
3746
3747     my @ready;
3748     while (scalar (@ready = $sel->can_read($timeout))) {
3749         my $count;
3750         if ($count = $fh->sysread($buf, 8192)) {
3751             if ($buf =~ /^(.*)\(qemu\) $/s) {
3752                 $res .= $1;
3753                 last;
3754             } else {
3755                 $res .= $buf;
3756             }
3757         } else {
3758             if (!defined($count)) {
3759                 die "$!\n";
3760             }
3761             last;
3762         }
3763     }
3764
3765     die "monitor read timeout\n" if !scalar(@ready);
3766
3767     return $res;
3768 }
3769
3770 # old code, only used to shutdown old VM after update
3771 sub vm_monitor_command {
3772     my ($vmid, $cmdstr, $nocheck) = @_;
3773
3774     my $res;
3775
3776     eval {
3777         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
3778
3779         my $sname = "${var_run_tmpdir}/$vmid.mon";
3780
3781         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
3782             die "unable to connect to VM $vmid socket - $!\n";
3783
3784         my $timeout = 3;
3785
3786         # hack: migrate sometime blocks the monitor (when migrate_downtime
3787         # is set)
3788         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3789             $timeout = 60*60; # 1 hour
3790         }
3791
3792         # read banner;
3793         my $data = __read_avail($sock, $timeout);
3794
3795         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
3796             die "got unexpected qemu monitor banner\n";
3797         }
3798
3799         my $sel = new IO::Select;
3800         $sel->add($sock);
3801
3802         if (!scalar(my @ready = $sel->can_write($timeout))) {
3803             die "monitor write error - timeout";
3804         }
3805
3806         my $fullcmd = "$cmdstr\r";
3807
3808         # syslog('info', "VM $vmid monitor command: $cmdstr");
3809
3810         my $b;
3811         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
3812             die "monitor write error - $!";
3813         }
3814
3815         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
3816
3817         $timeout = 20;
3818
3819         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3820             $timeout = 60*60; # 1 hour
3821         } elsif ($cmdstr =~ m/^(eject|change)/) {
3822             $timeout = 60; # note: cdrom mount command is slow
3823         }
3824         if ($res = __read_avail($sock, $timeout)) {
3825
3826             my @lines = split("\r?\n", $res);
3827
3828             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
3829
3830             $res = join("\n", @lines);
3831             $res .= "\n";
3832         }
3833     };
3834
3835     my $err = $@;
3836
3837     if ($err) {
3838         syslog("err", "VM $vmid monitor command failed - $err");
3839         die $err;
3840     }
3841
3842     return $res;
3843 }
3844
3845 sub qemu_block_resize {
3846     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
3847
3848     my $running = check_running($vmid);
3849
3850     return if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
3851
3852     return if !$running;
3853
3854     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
3855
3856 }
3857
3858 sub qemu_volume_snapshot {
3859     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3860
3861     my $running = check_running($vmid);
3862
3863     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
3864         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
3865     } else {
3866         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
3867     }
3868 }
3869
3870 sub qemu_volume_snapshot_delete {
3871     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3872
3873     my $running = check_running($vmid);
3874
3875     return if !PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
3876
3877     return if !$running;
3878
3879     vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
3880 }
3881
3882 sub set_migration_caps {
3883     my ($vmid) = @_;
3884
3885     my $cap_ref = [];
3886
3887     my $enabled_cap = {
3888         "auto-converge" => 1,
3889         "xbzrle" => 1,
3890         "x-rdma-pin-all" => 0,
3891         "zero-blocks" => 0,
3892         "compress" => 0
3893     };
3894
3895     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
3896
3897     for my $supported_capability (@$supported_capabilities) {
3898         push @$cap_ref, {
3899             capability => $supported_capability->{capability},
3900             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
3901         };
3902     }
3903
3904     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
3905 }
3906
3907 my $fast_plug_option = {
3908     'lock' => 1,
3909     'name' => 1,
3910     'onboot' => 1,
3911     'shares' => 1,
3912     'startup' => 1,
3913     'description' => 1,
3914 };
3915
3916 # hotplug changes in [PENDING]
3917 # $selection hash can be used to only apply specified options, for
3918 # example: { cores => 1 } (only apply changed 'cores')
3919 # $errors ref is used to return error messages
3920 sub vmconfig_hotplug_pending {
3921     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
3922
3923     my $defaults = load_defaults();
3924
3925     # commit values which do not have any impact on running VM first
3926     # Note: those option cannot raise errors, we we do not care about
3927     # $selection and always apply them.
3928
3929     my $add_error = sub {
3930         my ($opt, $msg) = @_;
3931         $errors->{$opt} = "hotplug problem - $msg";
3932     };
3933
3934     my $changes = 0;
3935     foreach my $opt (keys %{$conf->{pending}}) { # add/change
3936         if ($fast_plug_option->{$opt}) {
3937             $conf->{$opt} = $conf->{pending}->{$opt};
3938             delete $conf->{pending}->{$opt};
3939             $changes = 1;
3940         }
3941     }
3942
3943     if ($changes) {
3944         update_config_nolock($vmid, $conf, 1);
3945         $conf = load_config($vmid); # update/reload
3946     }
3947
3948     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3949
3950     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
3951     while (my ($opt, $force) = each %$pending_delete_hash) {
3952         next if $selection && !$selection->{$opt};
3953         eval {
3954             if ($opt eq 'hotplug') {
3955                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
3956             } elsif ($opt eq 'tablet') {
3957                 die "skip\n" if !$hotplug_features->{usb};
3958                 if ($defaults->{tablet}) {
3959                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
3960                 } else {
3961                     vm_deviceunplug($vmid, $conf, $opt);
3962                 }
3963             } elsif ($opt eq 'vcpus') {
3964                 die "skip\n" if !$hotplug_features->{cpu};
3965                 qemu_cpu_hotplug($vmid, $conf, undef);
3966             } elsif ($opt eq 'balloon') {
3967                 # enable balloon device is not hotpluggable
3968                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
3969             } elsif ($fast_plug_option->{$opt}) {
3970                 # do nothing
3971             } elsif ($opt =~ m/^net(\d+)$/) {
3972                 die "skip\n" if !$hotplug_features->{network};
3973                 vm_deviceunplug($vmid, $conf, $opt);
3974             } elsif (valid_drivename($opt)) {
3975                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
3976                 vm_deviceunplug($vmid, $conf, $opt);
3977                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
3978             } elsif ($opt =~ m/^memory$/) {
3979                 die "skip\n" if !$hotplug_features->{memory};
3980                 qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
3981             } elsif ($opt eq 'cpuunits') {
3982                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
3983             } elsif ($opt eq 'cpulimit') {
3984                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
3985             } else {
3986                 die "skip\n";
3987             }
3988         };
3989         if (my $err = $@) {
3990             &$add_error($opt, $err) if $err ne "skip\n";
3991         } else {
3992             # save new config if hotplug was successful
3993             delete $conf->{$opt};
3994             vmconfig_undelete_pending_option($conf, $opt);
3995             update_config_nolock($vmid, $conf, 1);
3996             $conf = load_config($vmid); # update/reload
3997         }
3998     }
3999
4000     foreach my $opt (keys %{$conf->{pending}}) {
4001         next if $selection && !$selection->{$opt};
4002         my $value = $conf->{pending}->{$opt};
4003         eval {
4004             if ($opt eq 'hotplug') {
4005                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4006             } elsif ($opt eq 'tablet') {
4007                 die "skip\n" if !$hotplug_features->{usb};
4008                 if ($value == 1) {
4009                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4010                 } elsif ($value == 0) {
4011                     vm_deviceunplug($vmid, $conf, $opt);
4012                 }
4013             } elsif ($opt eq 'vcpus') {
4014                 die "skip\n" if !$hotplug_features->{cpu};
4015                 qemu_cpu_hotplug($vmid, $conf, $value);
4016             } elsif ($opt eq 'balloon') {
4017                 # enable/disable balloning device is not hotpluggable
4018                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4019                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4020                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4021
4022                 # allow manual ballooning if shares is set to zero
4023                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4024                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4025                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4026                 }
4027             } elsif ($opt =~ m/^net(\d+)$/) {
4028                 # some changes can be done without hotplug
4029                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4030                                     $vmid, $opt, $value);
4031             } elsif (valid_drivename($opt)) {
4032                 # some changes can be done without hotplug
4033                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4034                                      $vmid, $opt, $value, 1);
4035             } elsif ($opt =~ m/^memory$/) { #dimms
4036                 die "skip\n" if !$hotplug_features->{memory};
4037                 $value = qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4038             } elsif ($opt eq 'cpuunits') {
4039                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4040             } elsif ($opt eq 'cpulimit') {
4041                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4042                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4043             } else {
4044                 die "skip\n";  # skip non-hot-pluggable options
4045             }
4046         };
4047         if (my $err = $@) {
4048             &$add_error($opt, $err) if $err ne "skip\n";
4049         } else {
4050             # save new config if hotplug was successful
4051             $conf->{$opt} = $value;
4052             delete $conf->{pending}->{$opt};
4053             update_config_nolock($vmid, $conf, 1);
4054             $conf = load_config($vmid); # update/reload
4055         }
4056     }
4057 }
4058
4059 sub try_deallocate_drive {
4060     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4061
4062     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4063         my $volid = $drive->{file};
4064         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4065             my $sid = PVE::Storage::parse_volume_id($volid);
4066             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4067
4068             # check if the disk is really unused
4069             my $used_paths = PVE::QemuServer::get_used_paths($vmid, $storecfg, $conf, 1, $key);
4070             my $path = PVE::Storage::path($storecfg, $volid);
4071             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4072                    if $used_paths->{$path};
4073             PVE::Storage::vdisk_free($storecfg, $volid);
4074             return 1;
4075         } else {
4076             # If vm is not owner of this disk remove from config
4077             return 1;
4078         }
4079     }
4080
4081     return undef;
4082 }
4083
4084 sub vmconfig_delete_or_detach_drive {
4085     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4086
4087     my $drive = parse_drive($opt, $conf->{$opt});
4088
4089     my $rpcenv = PVE::RPCEnvironment::get();
4090     my $authuser = $rpcenv->get_user();
4091
4092     if ($force) {
4093         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4094         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4095     } else {
4096         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4097     }
4098 }
4099
4100 sub vmconfig_apply_pending {
4101     my ($vmid, $conf, $storecfg) = @_;
4102
4103     # cold plug
4104
4105     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4106     while (my ($opt, $force) = each %$pending_delete_hash) {
4107         die "internal error" if $opt =~ m/^unused/;
4108         $conf = load_config($vmid); # update/reload
4109         if (!defined($conf->{$opt})) {
4110             vmconfig_undelete_pending_option($conf, $opt);
4111             update_config_nolock($vmid, $conf, 1);
4112         } elsif (valid_drivename($opt)) {
4113             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4114             vmconfig_undelete_pending_option($conf, $opt);
4115             delete $conf->{$opt};
4116             update_config_nolock($vmid, $conf, 1);
4117         } else {
4118             vmconfig_undelete_pending_option($conf, $opt);
4119             delete $conf->{$opt};
4120             update_config_nolock($vmid, $conf, 1);
4121         }
4122     }
4123
4124     $conf = load_config($vmid); # update/reload
4125
4126     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4127         $conf = load_config($vmid); # update/reload
4128
4129         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4130             # skip if nothing changed
4131         } elsif (valid_drivename($opt)) {
4132             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4133                 if defined($conf->{$opt});
4134             $conf->{$opt} = $conf->{pending}->{$opt};
4135         } else {
4136             $conf->{$opt} = $conf->{pending}->{$opt};
4137         }
4138
4139         delete $conf->{pending}->{$opt};
4140         update_config_nolock($vmid, $conf, 1);
4141     }
4142 }
4143
4144 my $safe_num_ne = sub {
4145     my ($a, $b) = @_;
4146
4147     return 0 if !defined($a) && !defined($b);
4148     return 1 if !defined($a);
4149     return 1 if !defined($b);
4150
4151     return $a != $b;
4152 };
4153
4154 my $safe_string_ne = sub {
4155     my ($a, $b) = @_;
4156
4157     return 0 if !defined($a) && !defined($b);
4158     return 1 if !defined($a);
4159     return 1 if !defined($b);
4160
4161     return $a ne $b;
4162 };
4163
4164 sub vmconfig_update_net {
4165     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4166
4167     my $newnet = parse_net($value);
4168
4169     if ($conf->{$opt}) {
4170         my $oldnet = parse_net($conf->{$opt});
4171
4172         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4173             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4174             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4175             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4176
4177             # for non online change, we try to hot-unplug
4178             die "skip\n" if !$hotplug;
4179             vm_deviceunplug($vmid, $conf, $opt);
4180         } else {
4181
4182             die "internal error" if $opt !~ m/net(\d+)/;
4183             my $iface = "tap${vmid}i$1";
4184
4185             if (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4186                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4187             }
4188
4189             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4190                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4191                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4192                 PVE::Network::tap_unplug($iface);
4193                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall});
4194             }
4195
4196             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4197                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4198             }
4199
4200             return 1;
4201         }
4202     }
4203
4204     if ($hotplug) {
4205         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4206     } else {
4207         die "skip\n";
4208     }
4209 }
4210
4211 sub vmconfig_update_disk {
4212     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4213
4214     # fixme: do we need force?
4215
4216     my $drive = parse_drive($opt, $value);
4217
4218     if ($conf->{$opt}) {
4219
4220         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4221
4222             my $media = $drive->{media} || 'disk';
4223             my $oldmedia = $old_drive->{media} || 'disk';
4224             die "unable to change media type\n" if $media ne $oldmedia;
4225
4226             if (!drive_is_cdrom($old_drive)) {
4227
4228                 if ($drive->{file} ne $old_drive->{file}) {
4229
4230                     die "skip\n" if !$hotplug;
4231
4232                     # unplug and register as unused
4233                     vm_deviceunplug($vmid, $conf, $opt);
4234                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4235
4236                 } else {
4237                     # update existing disk
4238
4239                     # skip non hotpluggable value
4240                     if (&$safe_num_ne($drive->{discard}, $old_drive->{discard}) ||
4241                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4242                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4243                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4244                         die "skip\n";
4245                     }
4246
4247                     # apply throttle
4248                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4249                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4250                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4251                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4252                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4253                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4254                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4255                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4256                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4257                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4258                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4259                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max})) {
4260
4261                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4262                                                    ($drive->{mbps} || 0)*1024*1024,
4263                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4264                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4265                                                    $drive->{iops} || 0,
4266                                                    $drive->{iops_rd} || 0,
4267                                                    $drive->{iops_wr} || 0,
4268                                                    ($drive->{mbps_max} || 0)*1024*1024,
4269                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4270                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4271                                                    $drive->{iops_max} || 0,
4272                                                    $drive->{iops_rd_max} || 0,
4273                                                    $drive->{iops_wr_max} || 0);
4274
4275                     }
4276
4277                     return 1;
4278                 }
4279
4280             } else { # cdrom
4281
4282                 if ($drive->{file} eq 'none') {
4283                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4284                 } else {
4285                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4286                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4287                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4288                 }
4289
4290                 return 1;
4291             }
4292         }
4293     }
4294
4295     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4296     # hotplug new disks
4297     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4298 }
4299
4300 sub vm_start {
4301     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4302         $forcemachine, $spice_ticket) = @_;
4303
4304     lock_config($vmid, sub {
4305         my $conf = load_config($vmid, $migratedfrom);
4306
4307         die "you can't start a vm if it's a template\n" if is_template($conf);
4308
4309         check_lock($conf) if !$skiplock;
4310
4311         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4312
4313         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4314             vmconfig_apply_pending($vmid, $conf, $storecfg);
4315             $conf = load_config($vmid); # update/reload
4316         }
4317
4318         my $defaults = load_defaults();
4319
4320         # set environment variable useful inside network script
4321         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4322
4323         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4324
4325         my $migrate_port = 0;
4326         my $migrate_uri;
4327         if ($statefile) {
4328             if ($statefile eq 'tcp') {
4329                 my $localip = "localhost";
4330                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4331                 my $nodename = PVE::INotify::nodename();
4332                 if ($datacenterconf->{migration_unsecure}) {
4333                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4334                         $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4335                 }
4336                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4337                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4338                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4339                 push @$cmd, '-incoming', $migrate_uri;
4340                 push @$cmd, '-S';
4341             } else {
4342                 push @$cmd, '-loadstate', $statefile;
4343             }
4344         } elsif ($paused) {
4345             push @$cmd, '-S';
4346         }
4347
4348         # host pci devices
4349         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4350           my $d = parse_hostpci($conf->{"hostpci$i"});
4351           next if !$d;
4352           my $pcidevices = $d->{pciid};
4353           foreach my $pcidevice (@$pcidevices) {
4354                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4355
4356                 my $info = pci_device_info("0000:$pciid");
4357                 die "IOMMU not present\n" if !check_iommu_support();
4358                 die "no pci device info for device '$pciid'\n" if !$info;
4359                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4360                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4361           }
4362         }
4363
4364         PVE::Storage::activate_volumes($storecfg, $vollist);
4365
4366         eval  { run_command($cmd, timeout => $statefile ? undef : 30,
4367                     umask => 0077); };
4368         my $err = $@;
4369         die "start failed: $err" if $err;
4370
4371         print "migration listens on $migrate_uri\n" if $migrate_uri;
4372
4373         if ($statefile && $statefile ne 'tcp')  {
4374             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4375             warn $@ if $@;
4376         }
4377
4378         if ($migratedfrom) {
4379
4380             eval {
4381                 set_migration_caps($vmid);
4382             };
4383             warn $@ if $@;
4384
4385             if ($spice_port) {
4386                 print "spice listens on port $spice_port\n";
4387                 if ($spice_ticket) {
4388                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4389                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4390                 }
4391             }
4392
4393         } else {
4394
4395             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4396                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4397                     if $conf->{balloon};
4398             }
4399
4400             foreach my $opt (keys %$conf) {
4401                 next if $opt !~  m/^net\d+$/;
4402                 my $nicconf = parse_net($conf->{$opt});
4403                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4404             }
4405         }
4406
4407         vm_mon_cmd_nocheck($vmid, 'qom-set',
4408                     path => "machine/peripheral/balloon0",
4409                     property => "guest-stats-polling-interval",
4410                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4411
4412     });
4413 }
4414
4415 sub vm_mon_cmd {
4416     my ($vmid, $execute, %params) = @_;
4417
4418     my $cmd = { execute => $execute, arguments => \%params };
4419     vm_qmp_command($vmid, $cmd);
4420 }
4421
4422 sub vm_mon_cmd_nocheck {
4423     my ($vmid, $execute, %params) = @_;
4424
4425     my $cmd = { execute => $execute, arguments => \%params };
4426     vm_qmp_command($vmid, $cmd, 1);
4427 }
4428
4429 sub vm_qmp_command {
4430     my ($vmid, $cmd, $nocheck) = @_;
4431
4432     my $res;
4433
4434     my $timeout;
4435     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4436         $timeout = $cmd->{arguments}->{timeout};
4437         delete $cmd->{arguments}->{timeout};
4438     }
4439
4440     eval {
4441         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4442         my $sname = qmp_socket($vmid);
4443         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4444             my $qmpclient = PVE::QMPClient->new();
4445
4446             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4447         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
4448             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
4449                 if scalar(%{$cmd->{arguments}});
4450             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
4451         } else {
4452             die "unable to open monitor socket\n";
4453         }
4454     };
4455     if (my $err = $@) {
4456         syslog("err", "VM $vmid qmp command failed - $err");
4457         die $err;
4458     }
4459
4460     return $res;
4461 }
4462
4463 sub vm_human_monitor_command {
4464     my ($vmid, $cmdline) = @_;
4465
4466     my $res;
4467
4468     my $cmd = {
4469         execute => 'human-monitor-command',
4470         arguments => { 'command-line' => $cmdline},
4471     };
4472
4473     return vm_qmp_command($vmid, $cmd);
4474 }
4475
4476 sub vm_commandline {
4477     my ($storecfg, $vmid) = @_;
4478
4479     my $conf = load_config($vmid);
4480
4481     my $defaults = load_defaults();
4482
4483     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
4484
4485     return join(' ', @$cmd);
4486 }
4487
4488 sub vm_reset {
4489     my ($vmid, $skiplock) = @_;
4490
4491     lock_config($vmid, sub {
4492
4493         my $conf = load_config($vmid);
4494
4495         check_lock($conf) if !$skiplock;
4496
4497         vm_mon_cmd($vmid, "system_reset");
4498     });
4499 }
4500
4501 sub get_vm_volumes {
4502     my ($conf) = @_;
4503
4504     my $vollist = [];
4505     foreach_volid($conf, sub {
4506         my ($volid, $is_cdrom) = @_;
4507
4508         return if $volid =~ m|^/|;
4509
4510         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4511         return if !$sid;
4512
4513         push @$vollist, $volid;
4514     });
4515
4516     return $vollist;
4517 }
4518
4519 sub vm_stop_cleanup {
4520     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
4521
4522     eval {
4523
4524         if (!$keepActive) {
4525             my $vollist = get_vm_volumes($conf);
4526             PVE::Storage::deactivate_volumes($storecfg, $vollist);
4527         }
4528
4529         foreach my $ext (qw(mon qmp pid vnc qga)) {
4530             unlink "/var/run/qemu-server/${vmid}.$ext";
4531         }
4532
4533         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
4534     };
4535     warn $@ if $@; # avoid errors - just warn
4536 }
4537
4538 # Note: use $nockeck to skip tests if VM configuration file exists.
4539 # We need that when migration VMs to other nodes (files already moved)
4540 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
4541 sub vm_stop {
4542     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
4543
4544     $force = 1 if !defined($force) && !$shutdown;
4545
4546     if ($migratedfrom){
4547         my $pid = check_running($vmid, $nocheck, $migratedfrom);
4548         kill 15, $pid if $pid;
4549         my $conf = load_config($vmid, $migratedfrom);
4550         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
4551         return;
4552     }
4553
4554     lock_config($vmid, sub {
4555
4556         my $pid = check_running($vmid, $nocheck);
4557         return if !$pid;
4558
4559         my $conf;
4560         if (!$nocheck) {
4561             $conf = load_config($vmid);
4562             check_lock($conf) if !$skiplock;
4563             if (!defined($timeout) && $shutdown && $conf->{startup}) {
4564                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
4565                 $timeout = $opts->{down} if $opts->{down};
4566             }
4567         }
4568
4569         $timeout = 60 if !defined($timeout);
4570
4571         eval {
4572             if ($shutdown) {
4573                 if (defined($conf) && $conf->{agent}) {
4574                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
4575                 } else {
4576                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
4577                 }
4578             } else {
4579                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
4580             }
4581         };
4582         my $err = $@;
4583
4584         if (!$err) {
4585             my $count = 0;
4586             while (($count < $timeout) && check_running($vmid, $nocheck)) {
4587                 $count++;
4588                 sleep 1;
4589             }
4590
4591             if ($count >= $timeout) {
4592                 if ($force) {
4593                     warn "VM still running - terminating now with SIGTERM\n";
4594                     kill 15, $pid;
4595                 } else {
4596                     die "VM quit/powerdown failed - got timeout\n";
4597                 }
4598             } else {
4599                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4600                 return;
4601             }
4602         } else {
4603             if ($force) {
4604                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
4605                 kill 15, $pid;
4606             } else {
4607                 die "VM quit/powerdown failed\n";
4608             }
4609         }
4610
4611         # wait again
4612         $timeout = 10;
4613
4614         my $count = 0;
4615         while (($count < $timeout) && check_running($vmid, $nocheck)) {
4616             $count++;
4617             sleep 1;
4618         }
4619
4620         if ($count >= $timeout) {
4621             warn "VM still running - terminating now with SIGKILL\n";
4622             kill 9, $pid;
4623             sleep 1;
4624         }
4625
4626         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4627    });
4628 }
4629
4630 sub vm_suspend {
4631     my ($vmid, $skiplock) = @_;
4632
4633     lock_config($vmid, sub {
4634
4635         my $conf = load_config($vmid);
4636
4637         check_lock($conf) if !($skiplock || ($conf->{lock} && $conf->{lock} eq 'backup'));
4638
4639         vm_mon_cmd($vmid, "stop");
4640     });
4641 }
4642
4643 sub vm_resume {
4644     my ($vmid, $skiplock, $nocheck) = @_;
4645
4646     lock_config($vmid, sub {
4647
4648         if (!$nocheck) {
4649
4650             my $conf = load_config($vmid);
4651
4652             check_lock($conf) if !($skiplock || ($conf->{lock} && $conf->{lock} eq 'backup'));
4653
4654             vm_mon_cmd($vmid, "cont");
4655
4656         } else {
4657             vm_mon_cmd_nocheck($vmid, "cont");
4658         }
4659     });
4660 }
4661
4662 sub vm_sendkey {
4663     my ($vmid, $skiplock, $key) = @_;
4664
4665     lock_config($vmid, sub {
4666
4667         my $conf = load_config($vmid);
4668
4669         # there is no qmp command, so we use the human monitor command
4670         vm_human_monitor_command($vmid, "sendkey $key");
4671     });
4672 }
4673
4674 sub vm_destroy {
4675     my ($storecfg, $vmid, $skiplock) = @_;
4676
4677     lock_config($vmid, sub {
4678
4679         my $conf = load_config($vmid);
4680
4681         check_lock($conf) if !$skiplock;
4682
4683         if (!check_running($vmid)) {
4684             destroy_vm($storecfg, $vmid);
4685         } else {
4686             die "VM $vmid is running - destroy failed\n";
4687         }
4688     });
4689 }
4690
4691 # pci helpers
4692
4693 sub file_write {
4694     my ($filename, $buf) = @_;
4695
4696     my $fh = IO::File->new($filename, "w");
4697     return undef if !$fh;
4698
4699     my $res = print $fh $buf;
4700
4701     $fh->close();
4702
4703     return $res;
4704 }
4705
4706 sub pci_device_info {
4707     my ($name) = @_;
4708
4709     my $res;
4710
4711     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
4712     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
4713
4714     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
4715     return undef if !defined($irq) || $irq !~ m/^\d+$/;
4716
4717     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
4718     return undef if !defined($vendor) || $vendor !~ s/^0x//;
4719
4720     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
4721     return undef if !defined($product) || $product !~ s/^0x//;
4722
4723     $res = {
4724         name => $name,
4725         vendor => $vendor,
4726         product => $product,
4727         domain => $domain,
4728         bus => $bus,
4729         slot => $slot,
4730         func => $func,
4731         irq => $irq,
4732         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
4733     };
4734
4735     return $res;
4736 }
4737
4738 sub pci_dev_reset {
4739     my ($dev) = @_;
4740
4741     my $name = $dev->{name};
4742
4743     my $fn = "$pcisysfs/devices/$name/reset";
4744
4745     return file_write($fn, "1");
4746 }
4747
4748 sub pci_dev_bind_to_vfio {
4749     my ($dev) = @_;
4750
4751     my $name = $dev->{name};
4752
4753     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4754
4755     if (!-d $vfio_basedir) {
4756         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4757     }
4758     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4759
4760     my $testdir = "$vfio_basedir/$name";
4761     return 1 if -d $testdir;
4762
4763     my $data = "$dev->{vendor} $dev->{product}";
4764     return undef if !file_write("$vfio_basedir/new_id", $data);
4765
4766     my $fn = "$pcisysfs/devices/$name/driver/unbind";
4767     if (!file_write($fn, $name)) {
4768         return undef if -f $fn;
4769     }
4770
4771     $fn = "$vfio_basedir/bind";
4772     if (! -d $testdir) {
4773         return undef if !file_write($fn, $name);
4774     }
4775
4776     return -d $testdir;
4777 }
4778
4779 sub pci_dev_group_bind_to_vfio {
4780     my ($pciid) = @_;
4781
4782     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4783
4784     if (!-d $vfio_basedir) {
4785         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4786     }
4787     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4788
4789     # get IOMMU group devices
4790     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
4791       my @devs = grep /^0000:/, readdir($D);
4792     closedir($D);
4793
4794     foreach my $pciid (@devs) {
4795         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
4796
4797         # pci bridges, switches or root ports are not supported
4798         # they have a pci_bus subdirectory so skip them
4799         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
4800
4801         my $info = pci_device_info($1);
4802         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
4803     }
4804
4805     return 1;
4806 }
4807
4808 sub print_pci_addr {
4809     my ($id, $bridges) = @_;
4810
4811     my $res = '';
4812     my $devices = {
4813         piix3 => { bus => 0, addr => 1 },
4814         #addr2 : first videocard
4815         balloon0 => { bus => 0, addr => 3 },
4816         watchdog => { bus => 0, addr => 4 },
4817         scsihw0 => { bus => 0, addr => 5 },
4818         'pci.3' => { bus => 0, addr => 5 }, #can also be used for virtio-scsi-single bridge
4819         scsihw1 => { bus => 0, addr => 6 },
4820         ahci0 => { bus => 0, addr => 7 },
4821         qga0 => { bus => 0, addr => 8 },
4822         spice => { bus => 0, addr => 9 },
4823         virtio0 => { bus => 0, addr => 10 },
4824         virtio1 => { bus => 0, addr => 11 },
4825         virtio2 => { bus => 0, addr => 12 },
4826         virtio3 => { bus => 0, addr => 13 },
4827         virtio4 => { bus => 0, addr => 14 },
4828         virtio5 => { bus => 0, addr => 15 },
4829         hostpci0 => { bus => 0, addr => 16 },
4830         hostpci1 => { bus => 0, addr => 17 },
4831         net0 => { bus => 0, addr => 18 },
4832         net1 => { bus => 0, addr => 19 },
4833         net2 => { bus => 0, addr => 20 },
4834         net3 => { bus => 0, addr => 21 },
4835         net4 => { bus => 0, addr => 22 },
4836         net5 => { bus => 0, addr => 23 },
4837         vga1 => { bus => 0, addr => 24 },
4838         vga2 => { bus => 0, addr => 25 },
4839         vga3 => { bus => 0, addr => 26 },
4840         hostpci2 => { bus => 0, addr => 27 },
4841         hostpci3 => { bus => 0, addr => 28 },
4842         #addr29 : usb-host (pve-usb.cfg)
4843         'pci.1' => { bus => 0, addr => 30 },
4844         'pci.2' => { bus => 0, addr => 31 },
4845         'net6' => { bus => 1, addr => 1 },
4846         'net7' => { bus => 1, addr => 2 },
4847         'net8' => { bus => 1, addr => 3 },
4848         'net9' => { bus => 1, addr => 4 },
4849         'net10' => { bus => 1, addr => 5 },
4850         'net11' => { bus => 1, addr => 6 },
4851         'net12' => { bus => 1, addr => 7 },
4852         'net13' => { bus => 1, addr => 8 },
4853         'net14' => { bus => 1, addr => 9 },
4854         'net15' => { bus => 1, addr => 10 },
4855         'net16' => { bus => 1, addr => 11 },
4856         'net17' => { bus => 1, addr => 12 },
4857         'net18' => { bus => 1, addr => 13 },
4858         'net19' => { bus => 1, addr => 14 },
4859         'net20' => { bus => 1, addr => 15 },
4860         'net21' => { bus => 1, addr => 16 },
4861         'net22' => { bus => 1, addr => 17 },
4862         'net23' => { bus => 1, addr => 18 },
4863         'net24' => { bus => 1, addr => 19 },
4864         'net25' => { bus => 1, addr => 20 },
4865         'net26' => { bus => 1, addr => 21 },
4866         'net27' => { bus => 1, addr => 22 },
4867         'net28' => { bus => 1, addr => 23 },
4868         'net29' => { bus => 1, addr => 24 },
4869         'net30' => { bus => 1, addr => 25 },
4870         'net31' => { bus => 1, addr => 26 },
4871         'virtio6' => { bus => 2, addr => 1 },
4872         'virtio7' => { bus => 2, addr => 2 },
4873         'virtio8' => { bus => 2, addr => 3 },
4874         'virtio9' => { bus => 2, addr => 4 },
4875         'virtio10' => { bus => 2, addr => 5 },
4876         'virtio11' => { bus => 2, addr => 6 },
4877         'virtio12' => { bus => 2, addr => 7 },
4878         'virtio13' => { bus => 2, addr => 8 },
4879         'virtio14' => { bus => 2, addr => 9 },
4880         'virtio15' => { bus => 2, addr => 10 },
4881         'virtioscsi0' => { bus => 3, addr => 1 },
4882         'virtioscsi1' => { bus => 3, addr => 2 },
4883         'virtioscsi2' => { bus => 3, addr => 3 },
4884         'virtioscsi3' => { bus => 3, addr => 4 },
4885         'virtioscsi4' => { bus => 3, addr => 5 },
4886         'virtioscsi5' => { bus => 3, addr => 6 },
4887         'virtioscsi6' => { bus => 3, addr => 7 },
4888         'virtioscsi7' => { bus => 3, addr => 8 },
4889         'virtioscsi8' => { bus => 3, addr => 9 },
4890         'virtioscsi9' => { bus => 3, addr => 10 },
4891         'virtioscsi10' => { bus => 3, addr => 11 },
4892         'virtioscsi11' => { bus => 3, addr => 12 },
4893         'virtioscsi12' => { bus => 3, addr => 13 },
4894         'virtioscsi13' => { bus => 3, addr => 14 },
4895         'virtioscsi14' => { bus => 3, addr => 15 },
4896         'virtioscsi15' => { bus => 3, addr => 16 },
4897         'virtioscsi16' => { bus => 3, addr => 17 },
4898         'virtioscsi17' => { bus => 3, addr => 18 },
4899         'virtioscsi18' => { bus => 3, addr => 19 },
4900         'virtioscsi19' => { bus => 3, addr => 20 },
4901         'virtioscsi20' => { bus => 3, addr => 21 },
4902         'virtioscsi21' => { bus => 3, addr => 22 },
4903         'virtioscsi22' => { bus => 3, addr => 23 },
4904         'virtioscsi23' => { bus => 3, addr => 24 },
4905         'virtioscsi24' => { bus => 3, addr => 25 },
4906         'virtioscsi25' => { bus => 3, addr => 26 },
4907         'virtioscsi26' => { bus => 3, addr => 27 },
4908         'virtioscsi27' => { bus => 3, addr => 28 },
4909         'virtioscsi28' => { bus => 3, addr => 29 },
4910         'virtioscsi29' => { bus => 3, addr => 30 },
4911         'virtioscsi30' => { bus => 3, addr => 31 },
4912
4913     };
4914
4915     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
4916            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
4917            my $bus = $devices->{$id}->{bus};
4918            $res = ",bus=pci.$bus,addr=$addr";
4919            $bridges->{$bus} = 1 if $bridges;
4920     }
4921     return $res;
4922
4923 }
4924
4925 sub print_pcie_addr {
4926     my ($id) = @_;
4927
4928     my $res = '';
4929     my $devices = {
4930         hostpci0 => { bus => "ich9-pcie-port-1", addr => 0 },
4931         hostpci1 => { bus => "ich9-pcie-port-2", addr => 0 },
4932         hostpci2 => { bus => "ich9-pcie-port-3", addr => 0 },
4933         hostpci3 => { bus => "ich9-pcie-port-4", addr => 0 },
4934     };
4935
4936     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
4937            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
4938            my $bus = $devices->{$id}->{bus};
4939            $res = ",bus=$bus,addr=$addr";
4940     }
4941     return $res;
4942
4943 }
4944
4945 # vzdump restore implementaion
4946
4947 sub tar_archive_read_firstfile {
4948     my $archive = shift;
4949
4950     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
4951
4952     # try to detect archive type first
4953     my $pid = open (TMP, "tar tf '$archive'|") ||
4954         die "unable to open file '$archive'\n";
4955     my $firstfile = <TMP>;
4956     kill 15, $pid;
4957     close TMP;
4958
4959     die "ERROR: archive contaions no data\n" if !$firstfile;
4960     chomp $firstfile;
4961
4962     return $firstfile;
4963 }
4964
4965 sub tar_restore_cleanup {
4966     my ($storecfg, $statfile) = @_;
4967
4968     print STDERR "starting cleanup\n";
4969
4970     if (my $fd = IO::File->new($statfile, "r")) {
4971         while (defined(my $line = <$fd>)) {
4972             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
4973                 my $volid = $2;
4974                 eval {
4975                     if ($volid =~ m|^/|) {
4976                         unlink $volid || die 'unlink failed\n';
4977                     } else {
4978                         PVE::Storage::vdisk_free($storecfg, $volid);
4979                     }
4980                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
4981                 };
4982                 print STDERR "unable to cleanup '$volid' - $@" if $@;
4983             } else {
4984                 print STDERR "unable to parse line in statfile - $line";
4985             }
4986         }
4987         $fd->close();
4988     }
4989 }
4990
4991 sub restore_archive {
4992     my ($archive, $vmid, $user, $opts) = @_;
4993
4994     my $format = $opts->{format};
4995     my $comp;
4996
4997     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
4998         $format = 'tar' if !$format;
4999         $comp = 'gzip';
5000     } elsif ($archive =~ m/\.tar$/) {
5001         $format = 'tar' if !$format;
5002     } elsif ($archive =~ m/.tar.lzo$/) {
5003         $format = 'tar' if !$format;
5004         $comp = 'lzop';
5005     } elsif ($archive =~ m/\.vma$/) {
5006         $format = 'vma' if !$format;
5007     } elsif ($archive =~ m/\.vma\.gz$/) {
5008         $format = 'vma' if !$format;
5009         $comp = 'gzip';
5010     } elsif ($archive =~ m/\.vma\.lzo$/) {
5011         $format = 'vma' if !$format;
5012         $comp = 'lzop';
5013     } else {
5014         $format = 'vma' if !$format; # default
5015     }
5016
5017     # try to detect archive format
5018     if ($format eq 'tar') {
5019         return restore_tar_archive($archive, $vmid, $user, $opts);
5020     } else {
5021         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5022     }
5023 }
5024
5025 sub restore_update_config_line {
5026     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5027
5028     return if $line =~ m/^\#qmdump\#/;
5029     return if $line =~ m/^\#vzdump\#/;
5030     return if $line =~ m/^lock:/;
5031     return if $line =~ m/^unused\d+:/;
5032     return if $line =~ m/^parent:/;
5033     return if $line =~ m/^template:/; # restored VM is never a template
5034
5035     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5036         # try to convert old 1.X settings
5037         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5038         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5039             my ($model, $macaddr) = split(/\=/, $devconfig);
5040             $macaddr = PVE::Tools::random_ether_addr() if !$macaddr || $unique;
5041             my $net = {
5042                 model => $model,
5043                 bridge => "vmbr$ind",
5044                 macaddr => $macaddr,
5045             };
5046             my $netstr = print_net($net);
5047
5048             print $outfd "net$cookie->{netcount}: $netstr\n";
5049             $cookie->{netcount}++;
5050         }
5051     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5052         my ($id, $netstr) = ($1, $2);
5053         my $net = parse_net($netstr);
5054         $net->{macaddr} = PVE::Tools::random_ether_addr() if $net->{macaddr};
5055         $netstr = print_net($net);
5056         print $outfd "$id: $netstr\n";
5057     } elsif ($line =~ m/^((ide|scsi|virtio|sata)\d+):\s*(\S+)\s*$/) {
5058         my $virtdev = $1;
5059         my $value = $3;
5060         if ($line =~ m/backup=no/) {
5061             print $outfd "#$line";
5062         } elsif ($virtdev && $map->{$virtdev}) {
5063             my $di = parse_drive($virtdev, $value);
5064             delete $di->{format}; # format can change on restore
5065             $di->{file} = $map->{$virtdev};
5066             $value = print_drive($vmid, $di);
5067             print $outfd "$virtdev: $value\n";
5068         } else {
5069             print $outfd $line;
5070         }
5071     } else {
5072         print $outfd $line;
5073     }
5074 }
5075
5076 sub scan_volids {
5077     my ($cfg, $vmid) = @_;
5078
5079     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5080
5081     my $volid_hash = {};
5082     foreach my $storeid (keys %$info) {
5083         foreach my $item (@{$info->{$storeid}}) {
5084             next if !($item->{volid} && $item->{size});
5085             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5086             $volid_hash->{$item->{volid}} = $item;
5087         }
5088     }
5089
5090     return $volid_hash;
5091 }
5092
5093 sub get_used_paths {
5094     my ($vmid, $storecfg, $conf, $scan_snapshots, $skip_drive) = @_;
5095
5096     my $used_path = {};
5097
5098     my $scan_config = sub {
5099         my ($cref, $snapname) = @_;
5100
5101         foreach my $key (keys %$cref) {
5102             my $value = $cref->{$key};
5103             if (valid_drivename($key)) {
5104                 next if $skip_drive && $key eq $skip_drive;
5105                 my $drive = parse_drive($key, $value);
5106                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5107                 if ($drive->{file} =~ m!^/!) {
5108                     $used_path->{$drive->{file}}++; # = 1;
5109                 } else {
5110                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5111                     next if !$storeid;
5112                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5113                     next if !$scfg;
5114                     my $path = PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5115                     $used_path->{$path}++; # = 1;
5116                 }
5117             }
5118         }
5119     };
5120
5121     &$scan_config($conf);
5122
5123     undef $skip_drive;
5124
5125     if ($scan_snapshots) {
5126         foreach my $snapname (keys %{$conf->{snapshots}}) {
5127             &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5128         }
5129     }
5130
5131     return $used_path;
5132 }
5133
5134 sub update_disksize {
5135     my ($vmid, $conf, $volid_hash) = @_;
5136
5137     my $changes;
5138
5139     my $used = {};
5140
5141     # Note: it is allowed to define multiple storages with same path (alias), so
5142     # we need to check both 'volid' and real 'path' (two different volid can point
5143     # to the same path).
5144
5145     my $usedpath = {};
5146
5147     # update size info
5148     foreach my $opt (keys %$conf) {
5149         if (valid_drivename($opt)) {
5150             my $drive = parse_drive($opt, $conf->{$opt});
5151             my $volid = $drive->{file};
5152             next if !$volid;
5153
5154             $used->{$volid} = 1;
5155             if ($volid_hash->{$volid} &&
5156                 (my $path = $volid_hash->{$volid}->{path})) {
5157                 $usedpath->{$path} = 1;
5158             }
5159
5160             next if drive_is_cdrom($drive);
5161             next if !$volid_hash->{$volid};
5162
5163             $drive->{size} = $volid_hash->{$volid}->{size};
5164             my $new = print_drive($vmid, $drive);
5165             if ($new ne $conf->{$opt}) {
5166                 $changes = 1;
5167                 $conf->{$opt} = $new;
5168             }
5169         }
5170     }
5171
5172     # remove 'unusedX' entry if volume is used
5173     foreach my $opt (keys %$conf) {
5174         next if $opt !~ m/^unused\d+$/;
5175         my $volid = $conf->{$opt};
5176         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5177         if ($used->{$volid} || ($path && $usedpath->{$path})) {
5178             $changes = 1;
5179             delete $conf->{$opt};
5180         }
5181     }
5182
5183     foreach my $volid (sort keys %$volid_hash) {
5184         next if $volid =~ m/vm-$vmid-state-/;
5185         next if $used->{$volid};
5186         my $path = $volid_hash->{$volid}->{path};
5187         next if !$path; # just to be sure
5188         next if $usedpath->{$path};
5189         $changes = 1;
5190         add_unused_volume($conf, $volid);
5191         $usedpath->{$path} = 1; # avoid to add more than once (aliases)
5192     }
5193
5194     return $changes;
5195 }
5196
5197 sub rescan {
5198     my ($vmid, $nolock) = @_;
5199
5200     my $cfg = PVE::Cluster::cfs_read_file("storage.cfg");
5201
5202     my $volid_hash = scan_volids($cfg, $vmid);
5203
5204     my $updatefn =  sub {
5205         my ($vmid) = @_;
5206
5207         my $conf = load_config($vmid);
5208
5209         check_lock($conf);
5210
5211         my $vm_volids = {};
5212         foreach my $volid (keys %$volid_hash) {
5213             my $info = $volid_hash->{$volid};
5214             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5215         }
5216
5217         my $changes = update_disksize($vmid, $conf, $vm_volids);
5218
5219         update_config_nolock($vmid, $conf, 1) if $changes;
5220     };
5221
5222     if (defined($vmid)) {
5223         if ($nolock) {
5224             &$updatefn($vmid);
5225         } else {
5226             lock_config($vmid, $updatefn, $vmid);
5227         }
5228     } else {
5229         my $vmlist = config_list();
5230         foreach my $vmid (keys %$vmlist) {
5231             if ($nolock) {
5232                 &$updatefn($vmid);
5233             } else {
5234                 lock_config($vmid, $updatefn, $vmid);
5235             }
5236         }
5237     }
5238 }
5239
5240 sub restore_vma_archive {
5241     my ($archive, $vmid, $user, $opts, $comp) = @_;
5242
5243     my $input = $archive eq '-' ? "<&STDIN" : undef;
5244     my $readfrom = $archive;
5245
5246     my $uncomp = '';
5247     if ($comp) {
5248         $readfrom = '-';
5249         my $qarchive = PVE::Tools::shellquote($archive);
5250         if ($comp eq 'gzip') {
5251             $uncomp = "zcat $qarchive|";
5252         } elsif ($comp eq 'lzop') {
5253             $uncomp = "lzop -d -c $qarchive|";
5254         } else {
5255             die "unknown compression method '$comp'\n";
5256         }
5257
5258     }
5259
5260     my $tmpdir = "/var/tmp/vzdumptmp$$";
5261     rmtree $tmpdir;
5262
5263     # disable interrupts (always do cleanups)
5264     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5265         warn "got interrupt - ignored\n";
5266     };
5267
5268     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5269     POSIX::mkfifo($mapfifo, 0600);
5270     my $fifofh;
5271
5272     my $openfifo = sub {
5273         open($fifofh, '>', $mapfifo) || die $!;
5274     };
5275
5276     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5277
5278     my $oldtimeout;
5279     my $timeout = 5;
5280
5281     my $devinfo = {};
5282
5283     my $rpcenv = PVE::RPCEnvironment::get();
5284
5285     my $conffile = config_file($vmid);
5286     my $tmpfn = "$conffile.$$.tmp";
5287
5288     # Note: $oldconf is undef if VM does not exists
5289     my $oldconf = PVE::Cluster::cfs_read_file(cfs_config_path($vmid));
5290
5291     my $print_devmap = sub {
5292         my $virtdev_hash = {};
5293
5294         my $cfgfn = "$tmpdir/qemu-server.conf";
5295
5296         # we can read the config - that is already extracted
5297         my $fh = IO::File->new($cfgfn, "r") ||
5298             "unable to read qemu-server.conf - $!\n";
5299
5300         while (defined(my $line = <$fh>)) {
5301             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5302                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5303                 die "archive does not contain data for drive '$virtdev'\n"
5304                     if !$devinfo->{$devname};
5305                 if (defined($opts->{storage})) {
5306                     $storeid = $opts->{storage} || 'local';
5307                 } elsif (!$storeid) {
5308                     $storeid = 'local';
5309                 }
5310                 $format = 'raw' if !$format;
5311                 $devinfo->{$devname}->{devname} = $devname;
5312                 $devinfo->{$devname}->{virtdev} = $virtdev;
5313                 $devinfo->{$devname}->{format} = $format;
5314                 $devinfo->{$devname}->{storeid} = $storeid;
5315
5316                 # check permission on storage
5317                 my $pool = $opts->{pool}; # todo: do we need that?
5318                 if ($user ne 'root@pam') {
5319                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5320                 }
5321
5322                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5323             }
5324         }
5325
5326         foreach my $devname (keys %$devinfo) {
5327             die "found no device mapping information for device '$devname'\n"
5328                 if !$devinfo->{$devname}->{virtdev};
5329         }
5330
5331         my $cfg = cfs_read_file('storage.cfg');
5332
5333         # create empty/temp config
5334         if ($oldconf) {
5335             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5336             foreach_drive($oldconf, sub {
5337                 my ($ds, $drive) = @_;
5338
5339                 return if drive_is_cdrom($drive);
5340
5341                 my $volid = $drive->{file};
5342
5343                 return if !$volid || $volid =~ m|^/|;
5344
5345                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5346                 return if !$path || !$owner || ($owner != $vmid);
5347
5348                 # Note: only delete disk we want to restore
5349                 # other volumes will become unused
5350                 if ($virtdev_hash->{$ds}) {
5351                     PVE::Storage::vdisk_free($cfg, $volid);
5352                 }
5353             });
5354         }
5355
5356         my $map = {};
5357         foreach my $virtdev (sort keys %$virtdev_hash) {
5358             my $d = $virtdev_hash->{$virtdev};
5359             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5360             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5361
5362             # test if requested format is supported
5363             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5364             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5365             $d->{format} = $defFormat if !$supported;
5366
5367             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5368                                                   $d->{format}, undef, $alloc_size);
5369             print STDERR "new volume ID is '$volid'\n";
5370             $d->{volid} = $volid;
5371             my $path = PVE::Storage::path($cfg, $volid);
5372
5373             PVE::Storage::activate_volumes($cfg,[$volid]);
5374
5375             my $write_zeros = 1;
5376             # fixme: what other storages types initialize volumes with zero?
5377             if ($scfg->{type} eq 'dir' || $scfg->{type} eq 'nfs' || $scfg->{type} eq 'glusterfs' ||
5378                 $scfg->{type} eq 'sheepdog' || $scfg->{type} eq 'rbd') {
5379                 $write_zeros = 0;
5380             }
5381
5382             print $fifofh "${write_zeros}:$d->{devname}=$path\n";
5383
5384             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5385             $map->{$virtdev} = $volid;
5386         }
5387
5388         $fh->seek(0, 0) || die "seek failed - $!\n";
5389
5390         my $outfd = new IO::File ($tmpfn, "w") ||
5391             die "unable to write config for VM $vmid\n";
5392
5393         my $cookie = { netcount => 0 };
5394         while (defined(my $line = <$fh>)) {
5395             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5396         }
5397
5398         $fh->close();
5399         $outfd->close();
5400     };
5401
5402     eval {
5403         # enable interrupts
5404         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5405             die "interrupted by signal\n";
5406         };
5407         local $SIG{ALRM} = sub { die "got timeout\n"; };
5408
5409         $oldtimeout = alarm($timeout);
5410
5411         my $parser = sub {
5412             my $line = shift;
5413
5414             print "$line\n";
5415
5416             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5417                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5418                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5419             } elsif ($line =~ m/^CTIME: /) {
5420                 # we correctly received the vma config, so we can disable
5421                 # the timeout now for disk allocation (set to 10 minutes, so
5422                 # that we always timeout if something goes wrong)
5423                 alarm(600);
5424                 &$print_devmap();
5425                 print $fifofh "done\n";
5426                 my $tmp = $oldtimeout || 0;
5427                 $oldtimeout = undef;
5428                 alarm($tmp);
5429                 close($fifofh);
5430             }
5431         };
5432
5433         print "restore vma archive: $cmd\n";
5434         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5435     };
5436     my $err = $@;
5437
5438     alarm($oldtimeout) if $oldtimeout;
5439
5440     my $vollist = [];
5441     foreach my $devname (keys %$devinfo) {
5442         my $volid = $devinfo->{$devname}->{volid};
5443         push @$vollist, $volid if $volid;
5444     }
5445
5446     my $cfg = cfs_read_file('storage.cfg');
5447     PVE::Storage::deactivate_volumes($cfg, $vollist);
5448
5449     unlink $mapfifo;
5450
5451     if ($err) {
5452         rmtree $tmpdir;
5453         unlink $tmpfn;
5454
5455         foreach my $devname (keys %$devinfo) {
5456             my $volid = $devinfo->{$devname}->{volid};
5457             next if !$volid;
5458             eval {
5459                 if ($volid =~ m|^/|) {
5460                     unlink $volid || die 'unlink failed\n';
5461                 } else {
5462                     PVE::Storage::vdisk_free($cfg, $volid);
5463                 }
5464                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5465             };
5466             print STDERR "unable to cleanup '$volid' - $@" if $@;
5467         }
5468         die $err;
5469     }
5470
5471     rmtree $tmpdir;
5472
5473     rename($tmpfn, $conffile) ||
5474         die "unable to commit configuration file '$conffile'\n";
5475
5476     PVE::Cluster::cfs_update(); # make sure we read new file
5477
5478     eval { rescan($vmid, 1); };
5479     warn $@ if $@;
5480 }
5481
5482 sub restore_tar_archive {
5483     my ($archive, $vmid, $user, $opts) = @_;
5484
5485     if ($archive ne '-') {
5486         my $firstfile = tar_archive_read_firstfile($archive);
5487         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5488             if $firstfile ne 'qemu-server.conf';
5489     }
5490
5491     my $storecfg = cfs_read_file('storage.cfg');
5492
5493     # destroy existing data - keep empty config
5494     my $vmcfgfn = config_file($vmid);
5495     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5496
5497     my $tocmd = "/usr/lib/qemu-server/qmextract";
5498
5499     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5500     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5501     $tocmd .= ' --prealloc' if $opts->{prealloc};
5502     $tocmd .= ' --info' if $opts->{info};
5503
5504     # tar option "xf" does not autodetect compression when read from STDIN,
5505     # so we pipe to zcat
5506     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5507         PVE::Tools::shellquote("--to-command=$tocmd");
5508
5509     my $tmpdir = "/var/tmp/vzdumptmp$$";
5510     mkpath $tmpdir;
5511
5512     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5513     local $ENV{VZDUMP_VMID} = $vmid;
5514     local $ENV{VZDUMP_USER} = $user;
5515
5516     my $conffile = config_file($vmid);
5517     my $tmpfn = "$conffile.$$.tmp";
5518
5519     # disable interrupts (always do cleanups)
5520     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5521         print STDERR "got interrupt - ignored\n";
5522     };
5523
5524     eval {
5525         # enable interrupts
5526         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5527             die "interrupted by signal\n";
5528         };
5529
5530         if ($archive eq '-') {
5531             print "extracting archive from STDIN\n";
5532             run_command($cmd, input => "<&STDIN");
5533         } else {
5534             print "extracting archive '$archive'\n";
5535             run_command($cmd);
5536         }
5537
5538         return if $opts->{info};
5539
5540         # read new mapping
5541         my $map = {};
5542         my $statfile = "$tmpdir/qmrestore.stat";
5543         if (my $fd = IO::File->new($statfile, "r")) {
5544             while (defined (my $line = <$fd>)) {
5545                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5546                     $map->{$1} = $2 if $1;
5547                 } else {
5548                     print STDERR "unable to parse line in statfile - $line\n";
5549                 }
5550             }
5551             $fd->close();
5552         }
5553
5554         my $confsrc = "$tmpdir/qemu-server.conf";
5555
5556         my $srcfd = new IO::File($confsrc, "r") ||
5557             die "unable to open file '$confsrc'\n";
5558
5559         my $outfd = new IO::File ($tmpfn, "w") ||
5560             die "unable to write config for VM $vmid\n";
5561
5562         my $cookie = { netcount => 0 };
5563         while (defined (my $line = <$srcfd>)) {
5564             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5565         }
5566
5567         $srcfd->close();
5568         $outfd->close();
5569     };
5570     my $err = $@;
5571
5572     if ($err) {
5573
5574         unlink $tmpfn;
5575
5576         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
5577
5578         die $err;
5579     }
5580
5581     rmtree $tmpdir;
5582
5583     rename $tmpfn, $conffile ||
5584         die "unable to commit configuration file '$conffile'\n";
5585
5586     PVE::Cluster::cfs_update(); # make sure we read new file
5587
5588     eval { rescan($vmid, 1); };
5589     warn $@ if $@;
5590 };
5591
5592
5593 # Internal snapshots
5594
5595 # NOTE: Snapshot create/delete involves several non-atomic
5596 # action, and can take a long time.
5597 # So we try to avoid locking the file and use 'lock' variable
5598 # inside the config file instead.
5599
5600 my $snapshot_copy_config = sub {
5601     my ($source, $dest) = @_;
5602
5603     foreach my $k (keys %$source) {
5604         next if $k eq 'snapshots';
5605         next if $k eq 'snapstate';
5606         next if $k eq 'snaptime';
5607         next if $k eq 'vmstate';
5608         next if $k eq 'lock';
5609         next if $k eq 'digest';
5610         next if $k eq 'description';
5611         next if $k =~ m/^unused\d+$/;
5612
5613         $dest->{$k} = $source->{$k};
5614     }
5615 };
5616
5617 my $snapshot_apply_config = sub {
5618     my ($conf, $snap) = @_;
5619
5620     # copy snapshot list
5621     my $newconf = {
5622         snapshots => $conf->{snapshots},
5623     };
5624
5625     # keep description and list of unused disks
5626     foreach my $k (keys %$conf) {
5627         next if !($k =~ m/^unused\d+$/ || $k eq 'description');
5628         $newconf->{$k} = $conf->{$k};
5629     }
5630
5631     &$snapshot_copy_config($snap, $newconf);
5632
5633     return $newconf;
5634 };
5635
5636 sub foreach_writable_storage {
5637     my ($conf, $func) = @_;
5638
5639     my $sidhash = {};
5640
5641     foreach my $ds (keys %$conf) {
5642         next if !valid_drivename($ds);
5643
5644         my $drive = parse_drive($ds, $conf->{$ds});
5645         next if !$drive;
5646         next if drive_is_cdrom($drive);
5647
5648         my $volid = $drive->{file};
5649
5650         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5651         $sidhash->{$sid} = $sid if $sid;
5652     }
5653
5654     foreach my $sid (sort keys %$sidhash) {
5655         &$func($sid);
5656     }
5657 }
5658
5659 my $alloc_vmstate_volid = sub {
5660     my ($storecfg, $vmid, $conf, $snapname) = @_;
5661
5662     # Note: we try to be smart when selecting a $target storage
5663
5664     my $target;
5665
5666     # search shared storage first
5667     foreach_writable_storage($conf, sub {
5668         my ($sid) = @_;
5669         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
5670         return if !$scfg->{shared};
5671
5672         $target = $sid if !$target || $scfg->{path}; # prefer file based storage
5673     });
5674
5675     if (!$target) {
5676         # now search local storage
5677         foreach_writable_storage($conf, sub {
5678             my ($sid) = @_;
5679             my $scfg = PVE::Storage::storage_config($storecfg, $sid);
5680             return if $scfg->{shared};
5681
5682             $target = $sid if !$target || $scfg->{path}; # prefer file based storage;
5683         });
5684     }
5685
5686     $target = 'local' if !$target;
5687
5688     my $driver_state_size = 500; # assume 32MB is enough to safe all driver state;
5689     # we abort live save after $conf->{memory}, so we need at max twice that space
5690     my $size = $conf->{memory}*2 + $driver_state_size;
5691
5692     my $name = "vm-$vmid-state-$snapname";
5693     my $scfg = PVE::Storage::storage_config($storecfg, $target);
5694     $name .= ".raw" if $scfg->{path}; # add filename extension for file base storage
5695     my $volid = PVE::Storage::vdisk_alloc($storecfg, $target, $vmid, 'raw', $name, $size*1024);
5696
5697     return $volid;
5698 };
5699
5700 my $snapshot_prepare = sub {
5701     my ($vmid, $snapname, $save_vmstate, $comment) = @_;
5702
5703     my $snap;
5704
5705     my $updatefn =  sub {
5706
5707         my $conf = load_config($vmid);
5708
5709         die "you can't take a snapshot if it's a template\n"
5710             if is_template($conf);
5711
5712         check_lock($conf);
5713
5714         $conf->{lock} = 'snapshot';
5715
5716         die "snapshot name '$snapname' already used\n"
5717             if defined($conf->{snapshots}->{$snapname});
5718
5719         my $storecfg = PVE::Storage::config();
5720         die "snapshot feature is not available" if !has_feature('snapshot', $conf, $storecfg);
5721
5722         $snap = $conf->{snapshots}->{$snapname} = {};
5723
5724         if ($save_vmstate && check_running($vmid)) {
5725             $snap->{vmstate} = &$alloc_vmstate_volid($storecfg, $vmid, $conf, $snapname);
5726         }
5727
5728         &$snapshot_copy_config($conf, $snap);
5729
5730         $snap->{snapstate} = "prepare";
5731         $snap->{snaptime} = time();
5732         $snap->{description} = $comment if $comment;
5733
5734         # always overwrite machine if we save vmstate. This makes sure we
5735         # can restore it later using correct machine type
5736         $snap->{machine} = get_current_qemu_machine($vmid) if $snap->{vmstate};
5737
5738         update_config_nolock($vmid, $conf, 1);
5739     };
5740
5741     lock_config($vmid, $updatefn);
5742
5743     return $snap;
5744 };
5745
5746 my $snapshot_commit = sub {
5747     my ($vmid, $snapname) = @_;
5748
5749     my $updatefn = sub {
5750
5751         my $conf = load_config($vmid);
5752
5753         die "missing snapshot lock\n"
5754             if !($conf->{lock} && $conf->{lock} eq 'snapshot');
5755
5756         my $has_machine_config = defined($conf->{machine});
5757
5758         my $snap = $conf->{snapshots}->{$snapname};
5759
5760         die "snapshot '$snapname' does not exist\n" if !defined($snap);
5761
5762         die "wrong snapshot state\n"
5763             if !($snap->{snapstate} && $snap->{snapstate} eq "prepare");
5764
5765         delete $snap->{snapstate};
5766         delete $conf->{lock};
5767
5768         my $newconf = &$snapshot_apply_config($conf, $snap);
5769
5770         delete $newconf->{machine} if !$has_machine_config;
5771
5772         $newconf->{parent} = $snapname;
5773
5774         update_config_nolock($vmid, $newconf, 1);
5775     };
5776
5777     lock_config($vmid, $updatefn);
5778 };
5779
5780 sub snapshot_rollback {
5781     my ($vmid, $snapname) = @_;
5782
5783     my $prepare = 1;
5784
5785     my $storecfg = PVE::Storage::config();
5786
5787     my $conf = load_config($vmid);
5788
5789     my $get_snapshot_config = sub {
5790
5791         die "you can't rollback if vm is a template\n" if is_template($conf);
5792
5793         my $res = $conf->{snapshots}->{$snapname};
5794
5795         die "snapshot '$snapname' does not exist\n" if !defined($res);
5796
5797         return $res;
5798     };
5799
5800     my $snap = &$get_snapshot_config();
5801
5802     foreach_drive($snap, sub {
5803         my ($ds, $drive) = @_;
5804
5805         return if drive_is_cdrom($drive);
5806
5807         my $volid = $drive->{file};
5808
5809         PVE::Storage::volume_rollback_is_possible($storecfg, $volid, $snapname);
5810     });
5811
5812     my $updatefn = sub {
5813
5814         $conf = load_config($vmid);
5815
5816         $snap = &$get_snapshot_config();
5817
5818         die "unable to rollback to incomplete snapshot (snapstate = $snap->{snapstate})\n"
5819             if $snap->{snapstate};
5820
5821         if ($prepare) {
5822             check_lock($conf);
5823             vm_stop($storecfg, $vmid, undef, undef, 5, undef, undef);
5824         }
5825
5826         die "unable to rollback vm $vmid: vm is running\n"
5827             if check_running($vmid);
5828
5829         if ($prepare) {
5830             $conf->{lock} = 'rollback';
5831         } else {
5832             die "got wrong lock\n" if !($conf->{lock} && $conf->{lock} eq 'rollback');
5833             delete $conf->{lock};
5834         }
5835
5836         my $forcemachine;
5837
5838         if (!$prepare) {
5839             my $has_machine_config = defined($conf->{machine});
5840
5841             # copy snapshot config to current config
5842             $conf = &$snapshot_apply_config($conf, $snap);
5843             $conf->{parent} = $snapname;
5844
5845             # Note: old code did not store 'machine', so we try to be smart
5846             # and guess the snapshot was generated with kvm 1.4 (pc-i440fx-1.4).
5847             $forcemachine = $conf->{machine} || 'pc-i440fx-1.4';
5848             # we remove the 'machine' configuration if not explicitly specified
5849             # in the original config.
5850             delete $conf->{machine} if $snap->{vmstate} && !$has_machine_config;
5851         }
5852
5853         update_config_nolock($vmid, $conf, 1);
5854
5855         if (!$prepare && $snap->{vmstate}) {
5856             my $statefile = PVE::Storage::path($storecfg, $snap->{vmstate});
5857             vm_start($storecfg, $vmid, $statefile, undef, undef, undef, $forcemachine);
5858         }
5859     };
5860
5861     lock_config($vmid, $updatefn);
5862
5863     foreach_drive($snap, sub {
5864         my ($ds, $drive) = @_;
5865
5866         return if drive_is_cdrom($drive);
5867
5868         my $volid = $drive->{file};
5869         my $device = "drive-$ds";
5870
5871         PVE::Storage::volume_snapshot_rollback($storecfg, $volid, $snapname);
5872     });
5873
5874     $prepare = 0;
5875     lock_config($vmid, $updatefn);
5876 }
5877
5878 my $savevm_wait = sub {
5879     my ($vmid) = @_;
5880
5881     for(;;) {
5882         my $stat = vm_mon_cmd_nocheck($vmid, "query-savevm");
5883         if (!$stat->{status}) {
5884             die "savevm not active\n";
5885         } elsif ($stat->{status} eq 'active') {
5886             sleep(1);
5887             next;
5888         } elsif ($stat->{status} eq 'completed') {
5889             last;
5890         } else {
5891             die "query-savevm returned status '$stat->{status}'\n";
5892         }
5893     }
5894 };
5895
5896 sub do_snapshots_with_qemu {
5897     my ($storecfg, $volid) = @_;
5898
5899     my $storage_name = PVE::Storage::parse_volume_id($volid);
5900
5901     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
5902         && !$storecfg->{ids}->{$storage_name}->{krbd}){
5903         return 1;
5904     }
5905
5906     if ($volid =~ m/\.(qcow2|qed)$/){
5907         return 1;
5908     }
5909
5910     return undef;
5911 }
5912
5913 sub snapshot_create {
5914     my ($vmid, $snapname, $save_vmstate, $comment) = @_;
5915
5916     my $snap = &$snapshot_prepare($vmid, $snapname, $save_vmstate, $comment);
5917
5918     $save_vmstate = 0 if !$snap->{vmstate}; # vm is not running
5919
5920     my $config = load_config($vmid);
5921
5922     my $running = check_running($vmid);
5923
5924     my $freezefs = $running && $config->{agent};
5925     $freezefs = 0 if $snap->{vmstate}; # not needed if we save RAM
5926
5927     my $drivehash = {};
5928
5929     if ($freezefs) {
5930         eval { vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
5931         warn "guest-fsfreeze-freeze problems - $@" if $@;
5932     }
5933
5934     eval {
5935         # create internal snapshots of all drives
5936
5937         my $storecfg = PVE::Storage::config();
5938
5939         if ($running) {
5940             if ($snap->{vmstate}) {
5941                 my $path = PVE::Storage::path($storecfg, $snap->{vmstate});
5942                 vm_mon_cmd($vmid, "savevm-start", statefile => $path);
5943                 &$savevm_wait($vmid);
5944             } else {
5945                 vm_mon_cmd($vmid, "savevm-start");
5946             }
5947         };
5948
5949         foreach_drive($snap, sub {
5950             my ($ds, $drive) = @_;
5951
5952             return if drive_is_cdrom($drive);
5953
5954             my $volid = $drive->{file};
5955             my $device = "drive-$ds";
5956
5957             qemu_volume_snapshot($vmid, $device, $storecfg, $volid, $snapname);
5958             $drivehash->{$ds} = 1;
5959        });
5960     };
5961     my $err = $@;
5962
5963     if ($running) {
5964         eval { vm_mon_cmd($vmid, "savevm-end")  };
5965         warn $@ if $@;
5966
5967         if ($freezefs) {
5968             eval { vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
5969             warn "guest-fsfreeze-thaw problems - $@" if $@;
5970         }
5971
5972         # savevm-end is async, we need to wait
5973         for (;;) {
5974             my $stat = vm_mon_cmd_nocheck($vmid, "query-savevm");
5975             if (!$stat->{bytes}) {
5976                 last;
5977             } else {
5978                 print "savevm not yet finished\n";
5979                 sleep(1);
5980                 next;
5981             }
5982         }
5983     }
5984
5985     if ($err) {
5986         warn "snapshot create failed: starting cleanup\n";
5987         eval { snapshot_delete($vmid, $snapname, 0, $drivehash); };
5988         warn $@ if $@;
5989         die $err;
5990     }
5991
5992     &$snapshot_commit($vmid, $snapname);
5993 }
5994
5995 # Note: $drivehash is only set when called from snapshot_create.
5996 sub snapshot_delete {
5997     my ($vmid, $snapname, $force, $drivehash) = @_;
5998
5999     my $prepare = 1;
6000
6001     my $snap;
6002     my $unused = [];
6003
6004     my $unlink_parent = sub {
6005         my ($confref, $new_parent) = @_;
6006
6007         if ($confref->{parent} && $confref->{parent} eq $snapname) {
6008             if ($new_parent) {
6009                 $confref->{parent} = $new_parent;
6010             } else {
6011                 delete $confref->{parent};
6012             }
6013         }
6014     };
6015
6016     my $updatefn =  sub {
6017         my ($remove_drive) = @_;
6018
6019         my $conf = load_config($vmid);
6020
6021         if (!$drivehash) {
6022             check_lock($conf);
6023             die "you can't delete a snapshot if vm is a template\n"
6024                 if is_template($conf);
6025         }
6026
6027         $snap = $conf->{snapshots}->{$snapname};
6028
6029         die "snapshot '$snapname' does not exist\n" if !defined($snap);
6030
6031         # remove parent refs
6032         if (!$prepare) {
6033             &$unlink_parent($conf, $snap->{parent});
6034             foreach my $sn (keys %{$conf->{snapshots}}) {
6035                 next if $sn eq $snapname;
6036                 &$unlink_parent($conf->{snapshots}->{$sn}, $snap->{parent});
6037             }
6038         }
6039
6040         if ($remove_drive) {
6041             if ($remove_drive eq 'vmstate') {
6042                 delete $snap->{$remove_drive};
6043             } else {
6044                 my $drive = parse_drive($remove_drive, $snap->{$remove_drive});
6045                 my $volid = $drive->{file};
6046                 delete $snap->{$remove_drive};
6047                 add_unused_volume($conf, $volid);
6048             }
6049         }
6050
6051         if ($prepare) {
6052             $snap->{snapstate} = 'delete';
6053         } else {
6054             delete $conf->{snapshots}->{$snapname};
6055             delete $conf->{lock} if $drivehash;
6056             foreach my $volid (@$unused) {
6057                 add_unused_volume($conf, $volid);
6058             }
6059         }
6060
6061         update_config_nolock($vmid, $conf, 1);
6062     };
6063
6064     lock_config($vmid, $updatefn);
6065
6066     # now remove vmstate file
6067
6068     my $storecfg = PVE::Storage::config();
6069
6070     if ($snap->{vmstate}) {
6071         eval {  PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
6072         if (my $err = $@) {
6073             die $err if !$force;
6074             warn $err;
6075         }
6076         # save changes (remove vmstate from snapshot)
6077         lock_config($vmid, $updatefn, 'vmstate') if !$force;
6078     };
6079
6080     # now remove all internal snapshots
6081     foreach_drive($snap, sub {
6082         my ($ds, $drive) = @_;
6083
6084         return if drive_is_cdrom($drive);
6085
6086         my $volid = $drive->{file};
6087         my $device = "drive-$ds";
6088
6089         if (!$drivehash || $drivehash->{$ds}) {
6090             eval { qemu_volume_snapshot_delete($vmid, $device, $storecfg, $volid, $snapname); };
6091             if (my $err = $@) {
6092                 die $err if !$force;
6093                 warn $err;
6094             }
6095         }
6096
6097         # save changes (remove drive fron snapshot)
6098         lock_config($vmid, $updatefn, $ds) if !$force;
6099         push @$unused, $volid;
6100     });
6101
6102     # now cleanup config
6103     $prepare = 0;
6104     lock_config($vmid, $updatefn);
6105 }
6106
6107 sub has_feature {
6108     my ($feature, $conf, $storecfg, $snapname, $running) = @_;
6109
6110     my $err;
6111     foreach_drive($conf, sub {
6112         my ($ds, $drive) = @_;
6113
6114         return if drive_is_cdrom($drive);
6115         my $volid = $drive->{file};
6116         $err = 1 if !PVE::Storage::volume_has_feature($storecfg, $feature, $volid, $snapname, $running);
6117     });
6118
6119     return $err ? 0 : 1;
6120 }
6121
6122 sub template_create {
6123     my ($vmid, $conf, $disk) = @_;
6124
6125     my $storecfg = PVE::Storage::config();
6126
6127     foreach_drive($conf, sub {
6128         my ($ds, $drive) = @_;
6129
6130         return if drive_is_cdrom($drive);
6131         return if $disk && $ds ne $disk;
6132
6133         my $volid = $drive->{file};
6134         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6135
6136         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6137         $drive->{file} = $voliddst;
6138         $conf->{$ds} = print_drive($vmid, $drive);
6139         update_config_nolock($vmid, $conf, 1);
6140     });
6141 }
6142
6143 sub is_template {
6144     my ($conf) = @_;
6145
6146     return 1 if defined $conf->{template} && $conf->{template} == 1;
6147 }
6148
6149 sub qemu_img_convert {
6150     my ($src_volid, $dst_volid, $size, $snapname) = @_;
6151
6152     my $storecfg = PVE::Storage::config();
6153     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6154     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6155
6156     if ($src_storeid && $dst_storeid) {
6157         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6158         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6159
6160         my $src_format = qemu_img_format($src_scfg, $src_volname);
6161         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6162
6163         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6164         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6165
6166         my $cmd = [];
6167         push @$cmd, '/usr/bin/qemu-img', 'convert', '-t', 'writeback', '-p', '-n';
6168         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
6169         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path, $dst_path;
6170
6171         my $parser = sub {
6172             my $line = shift;
6173             if($line =~ m/\((\S+)\/100\%\)/){
6174                 my $percent = $1;
6175                 my $transferred = int($size * $percent / 100);
6176                 my $remaining = $size - $transferred;
6177
6178                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6179             }
6180
6181         };
6182
6183         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6184         my $err = $@;
6185         die "copy failed: $err" if $err;
6186     }
6187 }
6188
6189 sub qemu_img_format {
6190     my ($scfg, $volname) = @_;
6191
6192     if ($scfg->{path} && $volname =~ m/\.(raw|cow|qcow|qcow2|qed|vmdk|cloop)$/) {
6193         return $1;
6194     } else {
6195         return "raw";
6196     }
6197 }
6198
6199 sub qemu_drive_mirror {
6200     my ($vmid, $drive, $dst_volid, $vmiddst) = @_;
6201
6202     my $storecfg = PVE::Storage::config();
6203     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6204
6205     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6206
6207     my $format = qemu_img_format($dst_scfg, $dst_volname);
6208
6209     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6210
6211     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $dst_path };
6212     $opts->{format} = $format if $format;
6213
6214     print "drive mirror is starting (scanning bitmap) : this step can take some minutes/hours, depend of disk size and storage speed\n";
6215
6216     eval {
6217     vm_mon_cmd($vmid, "drive-mirror", %$opts);
6218         while (1) {
6219             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6220             my $stat = @$stats[0];
6221             die "mirroring job seem to have die. Maybe do you have bad sectors?" if !$stat;
6222             die "error job is not mirroring" if $stat->{type} ne "mirror";
6223
6224             my $busy = $stat->{busy};
6225             my $ready = $stat->{ready};
6226
6227             if (my $total = $stat->{len}) {
6228                 my $transferred = $stat->{offset} || 0;
6229                 my $remaining = $total - $transferred;
6230                 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6231
6232                 print "transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6233             }
6234
6235
6236             if ($stat->{ready} eq 'true') {
6237
6238                 last if $vmiddst != $vmid;
6239
6240                 # try to switch the disk if source and destination are on the same guest
6241                 eval { vm_mon_cmd($vmid, "block-job-complete", device => "drive-$drive") };
6242                 last if !$@;
6243                 die $@ if $@ !~ m/cannot be completed/;
6244             }
6245             sleep 1;
6246         }
6247
6248
6249     };
6250     my $err = $@;
6251
6252     my $cancel_job = sub {
6253         vm_mon_cmd($vmid, "block-job-cancel", device => "drive-$drive");
6254         while (1) {
6255             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6256             my $stat = @$stats[0];
6257             last if !$stat;
6258             sleep 1;
6259         }
6260     };
6261
6262     if ($err) {
6263         eval { &$cancel_job(); };
6264         die "mirroring error: $err";
6265     }
6266
6267     if ($vmiddst != $vmid) {
6268         # if we clone a disk for a new target vm, we don't switch the disk
6269         &$cancel_job(); # so we call block-job-cancel
6270     }
6271 }
6272
6273 sub clone_disk {
6274     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6275         $newvmid, $storage, $format, $full, $newvollist) = @_;
6276
6277     my $newvolid;
6278
6279     if (!$full) {
6280         print "create linked clone of drive $drivename ($drive->{file})\n";
6281         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6282         push @$newvollist, $newvolid;
6283     } else {
6284         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6285         $storeid = $storage if $storage;
6286
6287         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6288         if (!$format) {
6289             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6290             $format = qemu_img_format($scfg, $volname);
6291         }
6292
6293         # test if requested format is supported - else use default
6294         my $supported = grep { $_ eq $format } @$validFormats;
6295         $format = $defFormat if !$supported;
6296
6297         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6298
6299         print "create full clone of drive $drivename ($drive->{file})\n";
6300         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $format, undef, ($size/1024));
6301         push @$newvollist, $newvolid;
6302
6303         PVE::Storage::activate_volumes($storecfg, $newvollist);
6304
6305         if (!$running || $snapname) {
6306             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname);
6307         } else {
6308             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid);
6309         }
6310     }
6311
6312     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6313
6314     my $disk = $drive;
6315     $disk->{format} = undef;
6316     $disk->{file} = $newvolid;
6317     $disk->{size} = $size;
6318
6319     return $disk;
6320 }
6321
6322 # this only works if VM is running
6323 sub get_current_qemu_machine {
6324     my ($vmid) = @_;
6325
6326     my $cmd = { execute => 'query-machines', arguments => {} };
6327     my $res = vm_qmp_command($vmid, $cmd);
6328
6329     my ($current, $default);
6330     foreach my $e (@$res) {
6331         $default = $e->{name} if $e->{'is-default'};
6332         $current = $e->{name} if $e->{'is-current'};
6333     }
6334
6335     # fallback to the default machine if current is not supported by qemu
6336     return $current || $default || 'pc';
6337 }
6338
6339 sub qemu_machine_feature_enabled {
6340     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6341
6342     my $current_major;
6343     my $current_minor;
6344
6345     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6346
6347         $current_major = $3;
6348         $current_minor = $4;
6349
6350     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6351
6352         $current_major = $1;
6353         $current_minor = $2;
6354     }
6355
6356     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
6357
6358
6359 }
6360
6361 sub qemu_machine_pxe {
6362     my ($vmid, $conf, $machine) = @_;
6363
6364     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6365
6366     foreach my $opt (keys %$conf) {
6367         next if $opt !~ m/^net(\d+)$/;
6368         my $net = PVE::QemuServer::parse_net($conf->{$opt});
6369         next if !$net;
6370         my $romfile = PVE::QemuServer::vm_mon_cmd_nocheck($vmid, 'qom-get', path => $opt, property => 'romfile');
6371         return $machine.".pxe" if $romfile =~ m/pxe/;
6372         last;
6373     }
6374
6375 }
6376
6377 sub qemu_use_old_bios_files {
6378     my ($machine_type) = @_;
6379
6380     return if !$machine_type;
6381
6382     my $use_old_bios_files = undef;
6383
6384     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6385         $machine_type = $1;
6386         $use_old_bios_files = 1;
6387     } else {
6388         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6389         # load new efi bios files on migration. So this hack is required to allow
6390         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6391         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6392         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, undef, 2, 4);
6393     }
6394
6395     return ($use_old_bios_files, $machine_type);
6396 }
6397
6398 sub lspci {
6399
6400     my $devices = {};
6401
6402     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6403             my (undef, $id, $function) = @_;
6404             my $res = { id => $id, function => $function};
6405             push @{$devices->{$id}}, $res;
6406     });
6407
6408     return $devices;
6409 }
6410
6411 sub vm_iothreads_list {
6412     my ($vmid) = @_;
6413
6414     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6415
6416     my $iothreads = {};
6417     foreach my $iothread (@$res) {
6418         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6419     }
6420
6421     return $iothreads;
6422 }
6423
6424 sub scsihw_infos {
6425     my ($conf, $drive) = @_;
6426
6427     my $maxdev = 0;
6428
6429     if ($conf->{scsihw} && ($conf->{scsihw} =~ m/^lsi/)) {
6430         $maxdev = 7;
6431     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6432         $maxdev = 1;
6433     } else {
6434         $maxdev = 256;
6435     }
6436
6437     my $controller = int($drive->{index} / $maxdev);
6438     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6439
6440     return ($maxdev, $controller, $controller_prefix);
6441 }
6442
6443 # bash completion helper
6444
6445 sub complete_backup_archives {
6446     my ($cmdname, $pname, $cvalue) = @_;
6447
6448     my $cfg = PVE::Storage::config();
6449
6450     my $storeid;
6451
6452     if ($cvalue =~ m/^([^:]+):/) {
6453         $storeid = $1;
6454     }
6455
6456     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6457
6458     my $res = [];
6459     foreach my $id (keys %$data) {
6460         foreach my $item (@{$data->{$id}}) {
6461             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6462             push @$res, $item->{volid} if defined($item->{volid});
6463         }
6464     }
6465
6466     return $res;
6467 }
6468
6469 my $complete_vmid_full = sub {
6470     my ($running) = @_;
6471
6472     my $idlist = vmstatus();
6473
6474     my $res = [];
6475
6476     foreach my $id (keys %$idlist) {
6477         my $d = $idlist->{$id};
6478         if (defined($running)) {
6479             next if $d->{template};
6480             next if $running && $d->{status} ne 'running';
6481             next if !$running && $d->{status} eq 'running';
6482         }
6483         push @$res, $id;
6484
6485     }
6486     return $res;
6487 };
6488
6489 sub complete_vmid {
6490     return &$complete_vmid_full();
6491 }
6492
6493 sub complete_vmid_stopped {
6494     return &$complete_vmid_full(0);
6495 }
6496
6497 sub complete_vmid_running {
6498     return &$complete_vmid_full(1);
6499 }
6500
6501 sub complete_storage {
6502
6503     my $cfg = PVE::Storage::config();
6504     my $ids = $cfg->{ids};
6505
6506     my $res = [];
6507     foreach my $sid (keys %$ids) {
6508         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6509         push @$res, $sid;
6510     }
6511
6512     return $res;
6513 }
6514
6515 1;