PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QMPClient;
  31 use PVE::RPCEnvironment;
  32 use Time::HiRes qw(gettimeofday);
  33
  34 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  35
  36 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  37
  38 # Note about locking: we use flock on the config file protect
  39 # against concurent actions.
  40 # Aditionaly, we have a 'lock' setting in the config file. This
  41 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  42 # allowed when such lock is set. But you can ignore this kind of
  43 # lock with the --skiplock flag.
  44
  45 cfs_register_file('/qemu-server/',
  46                   \&parse_vm_config,
  47                   \&write_vm_config);
  48
  49 PVE::JSONSchema::register_standard_option('skiplock', {
  50     description => "Ignore locks - only root is allowed to use this option.",
  51     type => 'boolean',
  52     optional => 1,
  53 });
  54
  55 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  56     description => "Some command save/restore state from this location.",
  57     type => 'string',
  58     maxLength => 128,
  59     optional => 1,
  60 });
  61
  62 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  63     description => "The name of the snapshot.",
  64     type => 'string', format => 'pve-configid',
  65     maxLength => 40,
  66 });
  67
  68 #no warnings 'redefine';
  69
  70 sub cgroups_write {
  71    my ($controller, $vmid, $option, $value) = @_;
  72
  73    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  74    PVE::ProcFSTools::write_proc_entry($path, $value);
  75
  76 }
  77
  78 my $nodename = PVE::INotify::nodename();
  79
  80 mkdir "/etc/pve/nodes/$nodename";
  81 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  82 mkdir $confdir;
  83
  84 my $var_run_tmpdir = "/var/run/qemu-server";
  85 mkdir $var_run_tmpdir;
  86
  87 my $lock_dir = "/var/lock/qemu-server";
  88 mkdir $lock_dir;
  89
  90 my $pcisysfs = "/sys/bus/pci";
  91
  92 my $confdesc = {
  93     onboot => {
  94         optional => 1,
  95         type => 'boolean',
  96         description => "Specifies whether a VM will be started during system bootup.",
  97         default => 0,
  98     },
  99     autostart => {
 100         optional => 1,
 101         type => 'boolean',
 102         description => "Automatic restart after crash (currently ignored).",
 103         default => 0,
 104     },
 105     hotplug => {
 106         optional => 1,
 107         type => 'string', format => 'pve-hotplug-features',
 108         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 109         default => 'network,disk,usb',
 110     },
 111     reboot => {
 112         optional => 1,
 113         type => 'boolean',
 114         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 115         default => 1,
 116     },
 117     lock => {
 118         optional => 1,
 119         type => 'string',
 120         description => "Lock/unlock the VM.",
 121         enum => [qw(migrate backup snapshot rollback)],
 122     },
 123     cpulimit => {
 124         optional => 1,
 125         type => 'number',
 126         description => "Limit of CPU usage. Note if the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 127         minimum => 0,
 128         maximum => 128,
 129         default => 0,
 130     },
 131     cpuunits => {
 132         optional => 1,
 133         type => 'integer',
 134         description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 135         minimum => 0,
 136         maximum => 500000,
 137         default => 1000,
 138     },
 139     memory => {
 140         optional => 1,
 141         type => 'integer',
 142         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 143         minimum => 16,
 144         default => 512,
 145     },
 146     balloon => {
 147         optional => 1,
 148         type => 'integer',
 149         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 150         minimum => 0,
 151     },
 152     shares => {
 153         optional => 1,
 154         type => 'integer',
 155         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 156         minimum => 0,
 157         maximum => 50000,
 158         default => 1000,
 159     },
 160     keyboard => {
 161         optional => 1,
 162         type => 'string',
 163         description => "Keybord layout for vnc server. Default is read from the datacenter configuration file.",
 164         enum => PVE::Tools::kvmkeymaplist(),
 165         default => 'en-us',
 166     },
 167     name => {
 168         optional => 1,
 169         type => 'string', format => 'dns-name',
 170         description => "Set a name for the VM. Only used on the configuration web interface.",
 171     },
 172     scsihw => {
 173         optional => 1,
 174         type => 'string',
 175         description => "scsi controller model",
 176         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 177         default => 'lsi',
 178     },
 179     description => {
 180         optional => 1,
 181         type => 'string',
 182         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 183     },
 184     ostype => {
 185         optional => 1,
 186         type => 'string',
 187         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 l24 l26 solaris)],
 188         description => <<EODESC,
 189 Used to enable special optimization/features for specific
 190 operating systems:
 191
 192 other  => unspecified OS
 193 wxp    => Microsoft Windows XP
 194 w2k    => Microsoft Windows 2000
 195 w2k3   => Microsoft Windows 2003
 196 w2k8   => Microsoft Windows 2008
 197 wvista => Microsoft Windows Vista
 198 win7   => Microsoft Windows 7
 199 win8   => Microsoft Windows 8/2012
 200 l24    => Linux 2.4 Kernel
 201 l26    => Linux 2.6/3.X Kernel
 202 solaris => solaris/opensolaris/openindiania kernel
 203
 204 other|l24|l26|solaris                       ... no special behaviour
 205 wxp|w2k|w2k3|w2k8|wvista|win7|win8  ... use --localtime switch
 206 EODESC
 207     },
 208     boot => {
 209         optional => 1,
 210         type => 'string',
 211         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 212         pattern => '[acdn]{1,4}',
 213         default => 'cdn',
 214     },
 215     bootdisk => {
 216         optional => 1,
 217         type => 'string', format => 'pve-qm-bootdisk',
 218         description => "Enable booting from specified disk.",
 219         pattern => '(ide|sata|scsi|virtio)\d+',
 220     },
 221     smp => {
 222         optional => 1,
 223         type => 'integer',
 224         description => "The number of CPUs. Please use option -sockets instead.",
 225         minimum => 1,
 226         default => 1,
 227     },
 228     sockets => {
 229         optional => 1,
 230         type => 'integer',
 231         description => "The number of CPU sockets.",
 232         minimum => 1,
 233         default => 1,
 234     },
 235     cores => {
 236         optional => 1,
 237         type => 'integer',
 238         description => "The number of cores per socket.",
 239         minimum => 1,
 240         default => 1,
 241     },
 242     numa => {
 243         optional => 1,
 244         type => 'boolean',
 245         description => "Enable/disable Numa.",
 246         default => 0,
 247     },
 248     vcpus => {
 249         optional => 1,
 250         type => 'integer',
 251         description => "Number of hotplugged vcpus.",
 252         minimum => 1,
 253         default => 0,
 254     },
 255     acpi => {
 256         optional => 1,
 257         type => 'boolean',
 258         description => "Enable/disable ACPI.",
 259         default => 1,
 260     },
 261     agent => {
 262         optional => 1,
 263         type => 'boolean',
 264         description => "Enable/disable Qemu GuestAgent.",
 265         default => 0,
 266     },
 267     kvm => {
 268         optional => 1,
 269         type => 'boolean',
 270         description => "Enable/disable KVM hardware virtualization.",
 271         default => 1,
 272     },
 273     tdf => {
 274         optional => 1,
 275         type => 'boolean',
 276         description => "Enable/disable time drift fix.",
 277         default => 0,
 278     },
 279     localtime => {
 280         optional => 1,
 281         type => 'boolean',
 282         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 283     },
 284     freeze => {
 285         optional => 1,
 286         type => 'boolean',
 287         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 288     },
 289     vga => {
 290         optional => 1,
 291         type => 'string',
 292         description => "Select VGA type. If you want to use high resolution modes (>= 1280x1024x16) then you should use option 'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and 'cirrur' for other OS types. Option 'qxl' enables the SPICE display sever. You can also run without any graphic card using a serial devive as terminal.",
 293         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 294     },
 295     watchdog => {
 296         optional => 1,
 297         type => 'string', format => 'pve-qm-watchdog',
 298         typetext => '[[model=]i6300esb|ib700] [,[action=]reset|shutdown|poweroff|pause|debug|none]',
 299         description => "Create a virtual hardware watchdog device.  Once enabled (by a guest action), the watchdog must be periodically polled by an agent inside the guest or else the guest will be restarted (or execute the action specified)",
 300     },
 301     startdate => {
 302         optional => 1,
 303         type => 'string',
 304         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 305         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 306         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 307         default => 'now',
 308     },
 309     startup =>  get_standard_option('pve-startup-order'),
 310     template => {
 311         optional => 1,
 312         type => 'boolean',
 313         description => "Enable/disable Template.",
 314         default => 0,
 315     },
 316     args => {
 317         optional => 1,
 318         type => 'string',
 319         description => <<EODESCR,
 320 Note: this option is for experts only. It allows you to pass arbitrary arguments to kvm, for example:
 321
 322 args: -no-reboot -no-hpet
 323 EODESCR
 324     },
 325     tablet => {
 326         optional => 1,
 327         type => 'boolean',
 328         default => 1,
 329         description => "Enable/disable the usb tablet device. This device is usually needed to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with normal VNC clients. If you're running lots of console-only guests on one host, you may consider disabling this to save some context switches. This is turned of by default if you use spice (vga=qxl).",
 330     },
 331     migrate_speed => {
 332         optional => 1,
 333         type => 'integer',
 334         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 335         minimum => 0,
 336         default => 0,
 337     },
 338     migrate_downtime => {
 339         optional => 1,
 340         type => 'number',
 341         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 342         minimum => 0,
 343         default => 0.1,
 344     },
 345     cdrom => {
 346         optional => 1,
 347         type => 'string', format => 'pve-qm-drive',
 348         typetext => 'volume',
 349         description => "This is an alias for option -ide2",
 350     },
 351     cpu => {
 352         optional => 1,
 353         description => "Emulated CPU type.",
 354         type => 'string',
 355         enum => [ qw(486 athlon pentium pentium2 pentium3 coreduo core2duo kvm32 kvm64 qemu32 qemu64 phenom Conroe Penryn Nehalem Westmere SandyBridge IvyBridge Haswell Broadwell Opteron_G1 Opteron_G2 Opteron_G3 Opteron_G4 Opteron_G5 host) ],
 356         default => 'kvm64',
 357     },
 358     parent => get_standard_option('pve-snapshot-name', {
 359         optional => 1,
 360         description => "Parent snapshot name. This is used internally, and should not be modified.",
 361     }),
 362     snaptime => {
 363         optional => 1,
 364         description => "Timestamp for snapshots.",
 365         type => 'integer',
 366         minimum => 0,
 367     },
 368     vmstate => {
 369         optional => 1,
 370         type => 'string', format => 'pve-volume-id',
 371         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 372     },
 373     machine => {
 374         description => "Specific the Qemu machine type.",
 375         type => 'string',
 376         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
 377         maxLength => 40,
 378         optional => 1,
 379     },
 380     smbios1 => {
 381         description => "Specify SMBIOS type 1 fields.",
 382         type => 'string', format => 'pve-qm-smbios1',
 383         typetext => "[manufacturer=str][,product=str][,version=str][,serial=str] [,uuid=uuid][,sku=str][,family=str]",
 384         maxLength => 256,
 385         optional => 1,
 386     },
 387     protection => {
 388         optional => 1,
 389         type => 'boolean',
 390         description => "Sets the protection flag of the VM. This will prevent the remove operation.",
 391         default => 0,
 392     },
 393 };
 394
 395 # what about other qemu settings ?
 396 #cpu => 'string',
 397 #machine => 'string',
 398 #fda => 'file',
 399 #fdb => 'file',
 400 #mtdblock => 'file',
 401 #sd => 'file',
 402 #pflash => 'file',
 403 #snapshot => 'bool',
 404 #bootp => 'file',
 405 ##tftp => 'dir',
 406 ##smb => 'dir',
 407 #kernel => 'file',
 408 #append => 'string',
 409 #initrd => 'file',
 410 ##soundhw => 'string',
 411
 412 while (my ($k, $v) = each %$confdesc) {
 413     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 414 }
 415
 416 my $MAX_IDE_DISKS = 4;
 417 my $MAX_SCSI_DISKS = 14;
 418 my $MAX_VIRTIO_DISKS = 16;
 419 my $MAX_SATA_DISKS = 6;
 420 my $MAX_USB_DEVICES = 5;
 421 my $MAX_NETS = 32;
 422 my $MAX_UNUSED_DISKS = 8;
 423 my $MAX_HOSTPCI_DEVICES = 4;
 424 my $MAX_SERIAL_PORTS = 4;
 425 my $MAX_PARALLEL_PORTS = 3;
 426 my $MAX_NUMA = 8;
 427 my $MAX_MEM = 4194304;
 428 my $STATICMEM = 1024;
 429
 430 my $numadesc = {
 431     optional => 1,
 432     type => 'string', format => 'pve-qm-numanode',
 433     typetext => "cpus=<id[-id],memory=<mb>[[,hostnodes=<id[-id]>] [,policy=<preferred|bind|interleave>]]",
 434     description => "numa topology",
 435 };
 436 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 437
 438 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 439     $confdesc->{"numa$i"} = $numadesc;
 440 }
 441
 442 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 443                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 444                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 445 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 446
 447 my $netdesc = {
 448     optional => 1,
 449     type => 'string', format => 'pve-qm-net',
 450     typetext => "MODEL=XX:XX:XX:XX:XX:XX [,bridge=<dev>][,queues=<nbqueues>][,rate=<mbps>] [,tag=<vlanid>][,firewall=0|1],link_down=0|1]",
 451     description => <<EODESCR,
 452 Specify network devices.
 453
 454 MODEL is one of: $nic_model_list_txt
 455
 456 XX:XX:XX:XX:XX:XX should be an unique MAC address. This is
 457 automatically generated if not specified.
 458
 459 The bridge parameter can be used to automatically add the interface to a bridge device. The Proxmox VE standard bridge is called 'vmbr0'.
 460
 461 Option 'rate' is used to limit traffic bandwidth from and to this interface. It is specified as floating point number, unit is 'Megabytes per second'.
 462
 463 If you specify no bridge, we create a kvm 'user' (NATed) network device, which provides DHCP and DNS services. The following addresses are used:
 464
 465 10.0.2.2   Gateway
 466 10.0.2.3   DNS Server
 467 10.0.2.4   SMB Server
 468
 469 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 470
 471 EODESCR
 472 };
 473 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 474
 475 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 476     $confdesc->{"net$i"} = $netdesc;
 477 }
 478
 479 my $drivename_hash;
 480
 481 my $idedesc = {
 482     optional => 1,
 483     type => 'string', format => 'pve-qm-drive',
 484     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads] [,discard=ignore|on] [,serial=serial][,model=model]',
 485     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
 486 };
 487 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
 488
 489 my $scsidesc = {
 490     optional => 1,
 491     type => 'string', format => 'pve-qm-drive',
 492     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads] [,discard=ignore|on] [,iothread=on] [,queues=<nbqueues>] [,serial=serial]',
 493     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
 494 };
 495 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 496
 497 my $satadesc = {
 498     optional => 1,
 499     type => 'string', format => 'pve-qm-drive',
 500     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads]  [,discard=ignore|on] [,serial=serial]',
 501     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
 502 };
 503 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
 504
 505 my $virtiodesc = {
 506     optional => 1,
 507     type => 'string', format => 'pve-qm-drive',
 508     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads]  [,discard=ignore|on] [,iothread=on] [,serial=serial]',
 509     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
 510 };
 511 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
 512
 513 my $usbdesc = {
 514     optional => 1,
 515     type => 'string', format => 'pve-qm-usb-device',
 516     typetext => 'host=HOSTUSBDEVICE|spice',
 517     description => <<EODESCR,
 518 Configure an USB device (n is 0 to 4). This can be used to
 519 pass-through usb devices to the guest. HOSTUSBDEVICE syntax is:
 520
 521 'bus-port(.port)*' (decimal numbers) or
 522 'vendor_id:product_id' (hexadeciaml numbers)
 523
 524 You can use the 'lsusb -t' command to list existing usb devices.
 525
 526 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 527
 528 The value 'spice' can be used to add a usb redirection devices for spice.
 529
 530 EODESCR
 531 };
 532 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
 533
 534 my $hostpcidesc = {
 535         optional => 1,
 536         type => 'string', format => 'pve-qm-hostpci',
 537         typetext => "[host=]HOSTPCIDEVICE [,rombar=on|off] [,pcie=0|1] [,x-vga=on|off]",
 538         description => <<EODESCR,
 539 Map host pci devices. HOSTPCIDEVICE syntax is:
 540
 541 'bus:dev.func' (hexadecimal numbers)
 542
 543 You can us the 'lspci' command to list existing pci devices.
 544
 545 The 'rombar' option determines whether or not the device's ROM will be visible in the guest's memory map (default is 'on').
 546
 547 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 548
 549 Experimental: user reported problems with this option.
 550 EODESCR
 551 };
 552 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
 553
 554 my $serialdesc = {
 555         optional => 1,
 556         type => 'string',
 557         pattern => '(/dev/.+|socket)',
 558         description =>  <<EODESCR,
 559 Create a serial device inside the VM (n is 0 to 3), and pass through a host serial device (i.e. /dev/ttyS0), or create a unix socket on the host side (use 'qm terminal' to open a terminal connection).
 560
 561 Note: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
 562
 563 Experimental: user reported problems with this option.
 564 EODESCR
 565 };
 566
 567 my $paralleldesc= {
 568         optional => 1,
 569         type => 'string',
 570         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
 571         description =>  <<EODESCR,
 572 Map host parallel devices (n is 0 to 2).
 573
 574 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 575
 576 Experimental: user reported problems with this option.
 577 EODESCR
 578 };
 579
 580 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
 581     $confdesc->{"parallel$i"} = $paralleldesc;
 582 }
 583
 584 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
 585     $confdesc->{"serial$i"} = $serialdesc;
 586 }
 587
 588 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
 589     $confdesc->{"hostpci$i"} = $hostpcidesc;
 590 }
 591
 592 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
 593     $drivename_hash->{"ide$i"} = 1;
 594     $confdesc->{"ide$i"} = $idedesc;
 595 }
 596
 597 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
 598     $drivename_hash->{"sata$i"} = 1;
 599     $confdesc->{"sata$i"} = $satadesc;
 600 }
 601
 602 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
 603     $drivename_hash->{"scsi$i"} = 1;
 604     $confdesc->{"scsi$i"} = $scsidesc ;
 605 }
 606
 607 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
 608     $drivename_hash->{"virtio$i"} = 1;
 609     $confdesc->{"virtio$i"} = $virtiodesc;
 610 }
 611
 612 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
 613     $confdesc->{"usb$i"} = $usbdesc;
 614 }
 615
 616 my $unuseddesc = {
 617     optional => 1,
 618     type => 'string', format => 'pve-volume-id',
 619     description => "Reference to unused volumes.",
 620 };
 621
 622 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
 623     $confdesc->{"unused$i"} = $unuseddesc;
 624 }
 625
 626 my $kvm_api_version = 0;
 627
 628 sub kvm_version {
 629
 630     return $kvm_api_version if $kvm_api_version;
 631
 632     my $fh = IO::File->new("</dev/kvm") ||
 633         return 0;
 634
 635     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
 636         $kvm_api_version = $v;
 637     }
 638
 639     $fh->close();
 640
 641     return  $kvm_api_version;
 642 }
 643
 644 my $kvm_user_version;
 645
 646 sub kvm_user_version {
 647
 648     return $kvm_user_version if $kvm_user_version;
 649
 650     $kvm_user_version = 'unknown';
 651
 652     my $tmp = `kvm -help 2>/dev/null`;
 653
 654     if ($tmp =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
 655         $kvm_user_version = $2;
 656     }
 657
 658     return $kvm_user_version;
 659
 660 }
 661
 662 my $kernel_has_vhost_net = -c '/dev/vhost-net';
 663
 664 sub disknames {
 665     # order is important - used to autoselect boot disk
 666     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
 667             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
 668             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
 669             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))));
 670 }
 671
 672 sub valid_drivename {
 673     my $dev = shift;
 674
 675     return defined($drivename_hash->{$dev});
 676 }
 677
 678 sub option_exists {
 679     my $key = shift;
 680     return defined($confdesc->{$key});
 681 }
 682
 683 sub nic_models {
 684     return $nic_model_list;
 685 }
 686
 687 sub os_list_description {
 688
 689     return {
 690         other => 'Other',
 691         wxp => 'Windows XP',
 692         w2k => 'Windows 2000',
 693         w2k3 =>, 'Windows 2003',
 694         w2k8 => 'Windows 2008',
 695         wvista => 'Windows Vista',
 696         win7 => 'Windows 7',
 697         win8 => 'Windows 8/2012',
 698         l24 => 'Linux 2.4',
 699         l26 => 'Linux 2.6',
 700     };
 701 }
 702
 703 my $cdrom_path;
 704
 705 sub get_cdrom_path {
 706
 707     return  $cdrom_path if $cdrom_path;
 708
 709     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
 710     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
 711     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
 712 }
 713
 714 sub get_iso_path {
 715     my ($storecfg, $vmid, $cdrom) = @_;
 716
 717     if ($cdrom eq 'cdrom') {
 718         return get_cdrom_path();
 719     } elsif ($cdrom eq 'none') {
 720         return '';
 721     } elsif ($cdrom =~ m|^/|) {
 722         return $cdrom;
 723     } else {
 724         return PVE::Storage::path($storecfg, $cdrom);
 725     }
 726 }
 727
 728 # try to convert old style file names to volume IDs
 729 sub filename_to_volume_id {
 730     my ($vmid, $file, $media) = @_;
 731
 732     if (!($file eq 'none' || $file eq 'cdrom' ||
 733           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
 734
 735         return undef if $file =~ m|/|;
 736
 737         if ($media && $media eq 'cdrom') {
 738             $file = "local:iso/$file";
 739         } else {
 740             $file = "local:$vmid/$file";
 741         }
 742     }
 743
 744     return $file;
 745 }
 746
 747 sub verify_media_type {
 748     my ($opt, $vtype, $media) = @_;
 749
 750     return if !$media;
 751
 752     my $etype;
 753     if ($media eq 'disk') {
 754         $etype = 'images';
 755     } elsif ($media eq 'cdrom') {
 756         $etype = 'iso';
 757     } else {
 758         die "internal error";
 759     }
 760
 761     return if ($vtype eq $etype);
 762
 763     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
 764 }
 765
 766 sub cleanup_drive_path {
 767     my ($opt, $storecfg, $drive) = @_;
 768
 769     # try to convert filesystem paths to volume IDs
 770
 771     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
 772         ($drive->{file} !~ m|^/dev/.+|) &&
 773         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
 774         ($drive->{file} !~ m/^\d+$/)) {
 775         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
 776         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
 777         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
 778         verify_media_type($opt, $vtype, $drive->{media});
 779         $drive->{file} = $volid;
 780     }
 781
 782     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
 783 }
 784
 785 sub create_conf_nolock {
 786     my ($vmid, $settings) = @_;
 787
 788     my $filename = config_file($vmid);
 789
 790     die "configuration file '$filename' already exists\n" if -f $filename;
 791
 792     my $defaults = load_defaults();
 793
 794     $settings->{name} = "vm$vmid" if !$settings->{name};
 795     $settings->{memory} = $defaults->{memory} if !$settings->{memory};
 796
 797     my $data = '';
 798     foreach my $opt (keys %$settings) {
 799         next if !$confdesc->{$opt};
 800
 801         my $value = $settings->{$opt};
 802         next if !$value;
 803
 804         $data .= "$opt: $value\n";
 805     }
 806
 807     PVE::Tools::file_set_contents($filename, $data);
 808 }
 809
 810 sub parse_hotplug_features {
 811     my ($data) = @_;
 812
 813     my $res = {};
 814
 815     return $res if $data eq '0';
 816
 817     $data = $confdesc->{hotplug}->{default} if $data eq '1';
 818
 819     foreach my $feature (PVE::Tools::split_list($data)) {
 820         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
 821             $res->{$1} = 1;
 822         } else {
 823             warn "ignoring unknown hotplug feature '$feature'\n";
 824         }
 825     }
 826     return $res;
 827 }
 828
 829 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
 830 sub pve_verify_hotplug_features {
 831     my ($value, $noerr) = @_;
 832
 833     return $value if parse_hotplug_features($value);
 834
 835     return undef if $noerr;
 836
 837     die "unable to parse hotplug option\n";
 838 }
 839
 840 my $parse_size = sub {
 841     my ($value) = @_;
 842
 843     return undef if $value !~ m/^(\d+(\.\d+)?)([KMG])?$/;
 844     my ($size, $unit) = ($1, $3);
 845     if ($unit) {
 846         if ($unit eq 'K') {
 847             $size = $size * 1024;
 848         } elsif ($unit eq 'M') {
 849             $size = $size * 1024 * 1024;
 850         } elsif ($unit eq 'G') {
 851             $size = $size * 1024 * 1024 * 1024;
 852         }
 853     }
 854     return int($size);
 855 };
 856
 857 my $format_size = sub {
 858     my ($size) = @_;
 859
 860     $size = int($size);
 861
 862     my $kb = int($size/1024);
 863     return $size if $kb*1024 != $size;
 864
 865     my $mb = int($kb/1024);
 866     return "${kb}K" if $mb*1024 != $kb;
 867
 868     my $gb = int($mb/1024);
 869     return "${mb}M" if $gb*1024 != $mb;
 870
 871     return "${gb}G";
 872 };
 873
 874 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
 875 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
 876 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
 877 #        [,aio=native|threads][,discard=ignore|on][,iothread=on]
 878 #        [,serial=serial][,model=model]
 879
 880 sub parse_drive {
 881     my ($key, $data) = @_;
 882
 883     my $res = {};
 884
 885     # $key may be undefined - used to verify JSON parameters
 886     if (!defined($key)) {
 887         $res->{interface} = 'unknown'; # should not harm when used to verify parameters
 888         $res->{index} = 0;
 889     } elsif ($key =~ m/^([^\d]+)(\d+)$/) {
 890         $res->{interface} = $1;
 891         $res->{index} = $2;
 892     } else {
 893         return undef;
 894     }
 895
 896     foreach my $p (split (/,/, $data)) {
 897         next if $p =~ m/^\s*$/;
 898
 899         if ($p =~ m/^(file|volume|cyls|heads|secs|trans|media|snapshot|cache|format|rerror|werror|backup|aio|bps|mbps|mbps_max|bps_rd|mbps_rd|mbps_rd_max|bps_wr|mbps_wr|mbps_wr_max|iops|iops_max|iops_rd|iops_rd_max|iops_wr|iops_wr_max|size|discard|iothread|queues|serial|model)=(.+)$/) {
 900             my ($k, $v) = ($1, $2);
 901
 902             $k = 'file' if $k eq 'volume';
 903
 904             return undef if defined $res->{$k};
 905
 906             if ($k eq 'bps' || $k eq 'bps_rd' || $k eq 'bps_wr') {
 907                 return undef if !$v || $v !~ m/^\d+/;
 908                 $k = "m$k";
 909                 $v = sprintf("%.3f", $v / (1024*1024));
 910             }
 911             $res->{$k} = $v;
 912         } else {
 913             if (!$res->{file} && $p !~ m/=/) {
 914                 $res->{file} = $p;
 915             } else {
 916                 return undef;
 917             }
 918         }
 919     }
 920
 921     return undef if !$res->{file};
 922
 923     return undef if $res->{cache} &&
 924         $res->{cache} !~ m/^(off|none|writethrough|writeback|unsafe|directsync)$/;
 925     return undef if $res->{snapshot} && $res->{snapshot} !~ m/^(on|off)$/;
 926     return undef if $res->{cyls} && $res->{cyls} !~ m/^\d+$/;
 927     return undef if $res->{heads} && $res->{heads} !~ m/^\d+$/;
 928     return undef if $res->{secs} && $res->{secs} !~ m/^\d+$/;
 929     return undef if $res->{media} && $res->{media} !~ m/^(disk|cdrom)$/;
 930     return undef if $res->{trans} && $res->{trans} !~ m/^(none|lba|auto)$/;
 931     return undef if $res->{format} && $res->{format} !~ m/^(raw|cow|qcow|qed|qcow2|vmdk|cloop)$/;
 932     return undef if $res->{rerror} && $res->{rerror} !~ m/^(ignore|report|stop)$/;
 933     return undef if $res->{werror} && $res->{werror} !~ m/^(enospc|ignore|report|stop)$/;
 934     return undef if $res->{backup} && $res->{backup} !~ m/^(yes|no)$/;
 935     return undef if $res->{aio} && $res->{aio} !~ m/^(native|threads)$/;
 936     return undef if $res->{discard} && $res->{discard} !~ m/^(ignore|on)$/;
 937     return undef if $res->{iothread} && $res->{iothread} !~ m/^(on)$/;
 938     return undef if $res->{queues} && ($res->{queues} !~ m/^\d+$/ || $res->{queues} < 2);
 939
 940     return undef if $res->{mbps_rd} && $res->{mbps};
 941     return undef if $res->{mbps_wr} && $res->{mbps};
 942
 943     return undef if $res->{mbps} && $res->{mbps} !~ m/^\d+(\.\d+)?$/;
 944     return undef if $res->{mbps_max} && $res->{mbps_max} !~ m/^\d+(\.\d+)?$/;
 945     return undef if $res->{mbps_rd} && $res->{mbps_rd} !~ m/^\d+(\.\d+)?$/;
 946     return undef if $res->{mbps_rd_max} && $res->{mbps_rd_max} !~ m/^\d+(\.\d+)?$/;
 947     return undef if $res->{mbps_wr} && $res->{mbps_wr} !~ m/^\d+(\.\d+)?$/;
 948     return undef if $res->{mbps_wr_max} && $res->{mbps_wr_max} !~ m/^\d+(\.\d+)?$/;
 949
 950     return undef if $res->{iops_rd} && $res->{iops};
 951     return undef if $res->{iops_wr} && $res->{iops};
 952
 953
 954     return undef if $res->{iops} && $res->{iops} !~ m/^\d+$/;
 955     return undef if $res->{iops_max} && $res->{iops_max} !~ m/^\d+$/;
 956     return undef if $res->{iops_rd} && $res->{iops_rd} !~ m/^\d+$/;
 957     return undef if $res->{iops_rd_max} && $res->{iops_rd_max} !~ m/^\d+$/;
 958     return undef if $res->{iops_wr} && $res->{iops_wr} !~ m/^\d+$/;
 959     return undef if $res->{iops_wr_max} && $res->{iops_wr_max} !~ m/^\d+$/;
 960
 961     if ($res->{size}) {
 962         return undef if !defined($res->{size} = &$parse_size($res->{size}));
 963     }
 964
 965     if ($res->{media} && ($res->{media} eq 'cdrom')) {
 966         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
 967         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
 968         return undef if $res->{interface} eq 'virtio';
 969     }
 970
 971     # rerror does not work with scsi drives
 972     if ($res->{rerror}) {
 973         return undef if $res->{interface} eq 'scsi';
 974     }
 975
 976     return $res;
 977 }
 978
 979 my @qemu_drive_options = qw(heads secs cyls trans media format cache snapshot rerror werror aio discard iops iops_rd iops_wr iops_max iops_rd_max iops_wr_max serial);
 980
 981 sub print_drive {
 982     my ($vmid, $drive) = @_;
 983
 984     my $opts = '';
 985     foreach my $o (@qemu_drive_options, 'mbps', 'mbps_rd', 'mbps_wr', 'mbps_max', 'mbps_rd_max', 'mbps_wr_max', 'backup', 'iothread', 'queues') {
 986         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
 987     }
 988
 989     if ($drive->{size}) {
 990         $opts .= ",size=" . &$format_size($drive->{size});
 991     }
 992
 993     if (my $model = $drive->{model}) {
 994         $opts .= ",model=$model";
 995     }
 996
 997     return "$drive->{file}$opts";
 998 }
 999
1000 sub scsi_inquiry {
1001     my($fh, $noerr) = @_;
1002
1003     my $SG_IO = 0x2285;
1004     my $SG_GET_VERSION_NUM = 0x2282;
1005
1006     my $versionbuf = "\x00" x 8;
1007     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1008     if (!$ret) {
1009         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1010         return undef;
1011     }
1012     my $version = unpack("I", $versionbuf);
1013     if ($version < 30000) {
1014         die "scsi generic interface too old\n"  if !$noerr;
1015         return undef;
1016     }
1017
1018     my $buf = "\x00" x 36;
1019     my $sensebuf = "\x00" x 8;
1020     my $cmd = pack("C x3 C x1", 0x12, 36);
1021
1022     # see /usr/include/scsi/sg.h
1023     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1024
1025     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1026                       length($sensebuf), 0, length($buf), $buf,
1027                       $cmd, $sensebuf, 6000);
1028
1029     $ret = ioctl($fh, $SG_IO, $packet);
1030     if (!$ret) {
1031         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1032         return undef;
1033     }
1034
1035     my @res = unpack($sg_io_hdr_t, $packet);
1036     if ($res[17] || $res[18]) {
1037         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1038         return undef;
1039     }
1040
1041     my $res = {};
1042     (my $byte0, my $byte1, $res->{vendor},
1043      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1044
1045     $res->{removable} = $byte1 & 128 ? 1 : 0;
1046     $res->{type} = $byte0 & 31;
1047
1048     return $res;
1049 }
1050
1051 sub path_is_scsi {
1052     my ($path) = @_;
1053
1054     my $fh = IO::File->new("+<$path") || return undef;
1055     my $res = scsi_inquiry($fh, 1);
1056     close($fh);
1057
1058     return $res;
1059 }
1060
1061 sub machine_type_is_q35 {
1062     my ($conf) = @_;
1063
1064     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1065 }
1066
1067 sub print_tabletdevice_full {
1068     my ($conf) = @_;
1069
1070     my $q35 = machine_type_is_q35($conf);
1071
1072     # we use uhci for old VMs because tablet driver was buggy in older qemu
1073     my $usbbus = $q35 ? "ehci" : "uhci";
1074
1075     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1076 }
1077
1078 sub print_drivedevice_full {
1079     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1080
1081     my $device = '';
1082     my $maxdev = 0;
1083
1084     if ($drive->{interface} eq 'virtio') {
1085         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1086         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1087         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1088     } elsif ($drive->{interface} eq 'scsi') {
1089
1090         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1091         my $unit = $drive->{index} % $maxdev;
1092         my $devicetype = 'hd';
1093         my $path = '';
1094         if (drive_is_cdrom($drive)) {
1095               $devicetype = 'cd';
1096         } else {
1097               if ($drive->{file} =~ m|^/|) {
1098                   $path = $drive->{file};
1099               } else {
1100                   $path = PVE::Storage::path($storecfg, $drive->{file});
1101               }
1102
1103               if($path =~ m/^iscsi\:\/\//){
1104                   $devicetype = 'generic';
1105               } else {
1106                   if (my $info = path_is_scsi($path)) {
1107                       if ($info->{type} == 0) {
1108                           $devicetype = 'block';
1109                       } elsif ($info->{type} == 1) { # tape
1110                           $devicetype = 'generic';
1111                       }
1112                   }
1113               }
1114          }
1115
1116         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1117             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1118         } else {
1119             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1120         }
1121
1122     } elsif ($drive->{interface} eq 'ide'){
1123         $maxdev = 2;
1124         my $controller = int($drive->{index} / $maxdev);
1125         my $unit = $drive->{index} % $maxdev;
1126         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1127
1128         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1129         if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
1130             $device .= ",model=$model";
1131         }
1132     } elsif ($drive->{interface} eq 'sata'){
1133         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1134         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1135         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1136     } elsif ($drive->{interface} eq 'usb') {
1137         die "implement me";
1138         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1139     } else {
1140         die "unsupported interface type";
1141     }
1142
1143     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1144
1145     return $device;
1146 }
1147
1148 sub get_initiator_name {
1149     my $initiator;
1150
1151     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1152     while (defined(my $line = <$fh>)) {
1153         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1154         $initiator = $1;
1155         last;
1156     }
1157     $fh->close();
1158
1159     return $initiator;
1160 }
1161
1162 sub print_drive_full {
1163     my ($storecfg, $vmid, $drive) = @_;
1164
1165     my $path;
1166     my $volid = $drive->{file};
1167     my $format;
1168
1169     if (drive_is_cdrom($drive)) {
1170         $path = get_iso_path($storecfg, $vmid, $volid);
1171     } else {
1172         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1173         if ($storeid) {
1174             $path = PVE::Storage::path($storecfg, $volid);
1175             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1176             $format = qemu_img_format($scfg, $volname);
1177         } else {
1178             $path = $volid;
1179         }
1180    }
1181
1182     my $opts = '';
1183     foreach my $o (@qemu_drive_options) {
1184         next if $o eq 'bootindex';
1185         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1186     }
1187
1188     $opts .= ",format=$format" if $format && !$drive->{format};
1189
1190     foreach my $o (qw(bps bps_rd bps_wr)) {
1191         my $v = $drive->{"m$o"};
1192         $opts .= ",$o=" . int($v*1024*1024) if $v;
1193     }
1194
1195     my $cache_direct = 0;
1196
1197     if (my $cache = $drive->{cache}) {
1198         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1199     } elsif (!drive_is_cdrom($drive)) {
1200         $opts .= ",cache=none";
1201         $cache_direct = 1;
1202     }
1203
1204     # aio native works only with O_DIRECT
1205     if (!$drive->{aio}) {
1206         if($cache_direct) {
1207             $opts .= ",aio=native";
1208         } else {
1209             $opts .= ",aio=threads";
1210         }
1211     }
1212
1213     my $detectzeroes = $drive->{discard} ? "unmap" : "on";
1214     $opts .= ",detect-zeroes=$detectzeroes" if !drive_is_cdrom($drive);
1215
1216     my $pathinfo = $path ? "file=$path," : '';
1217
1218     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1219 }
1220
1221 sub print_netdevice_full {
1222     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1223
1224     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1225
1226     my $device = $net->{model};
1227     if ($net->{model} eq 'virtio') {
1228          $device = 'virtio-net-pci';
1229      };
1230
1231     my $pciaddr = print_pci_addr("$netid", $bridges);
1232     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1233     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1234         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1235         my $vectors = $net->{queues} * 2 + 2;
1236         $tmpstr .= ",vectors=$vectors,mq=on";
1237     }
1238     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1239
1240     if ($use_old_bios_files) {
1241         my $romfile;
1242         if ($device eq 'virtio-net-pci') {
1243             $romfile = 'pxe-virtio.rom';
1244         } elsif ($device eq 'e1000') {
1245             $romfile = 'pxe-e1000.rom';
1246         } elsif ($device eq 'ne2k') {
1247             $romfile = 'pxe-ne2k_pci.rom';
1248         } elsif ($device eq 'pcnet') {
1249             $romfile = 'pxe-pcnet.rom';
1250         } elsif ($device eq 'rtl8139') {
1251             $romfile = 'pxe-rtl8139.rom';
1252         }
1253         $tmpstr .= ",romfile=$romfile" if $romfile;
1254     }
1255
1256     return $tmpstr;
1257 }
1258
1259 sub print_netdev_full {
1260     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1261
1262     my $i = '';
1263     if ($netid =~ m/^net(\d+)$/) {
1264         $i = int($1);
1265     }
1266
1267     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1268
1269     my $ifname = "tap${vmid}i$i";
1270
1271     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1272     die "interface name '$ifname' is too long (max 15 character)\n"
1273         if length($ifname) >= 16;
1274
1275     my $vhostparam = '';
1276     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1277
1278     my $vmname = $conf->{name} || "vm$vmid";
1279
1280     my $netdev = "";
1281     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1282
1283     if ($net->{bridge}) {
1284         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1285     } else {
1286         $netdev = "type=user,id=$netid,hostname=$vmname";
1287     }
1288
1289     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1290
1291     return $netdev;
1292 }
1293
1294 sub drive_is_cdrom {
1295     my ($drive) = @_;
1296
1297     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1298
1299 }
1300
1301 sub parse_numa {
1302     my ($data) = @_;
1303
1304     my $res = {};
1305
1306     foreach my $kvp (split(/,/, $data)) {
1307
1308         if ($kvp =~ m/^memory=(\S+)$/) {
1309             $res->{memory} = $1;
1310         } elsif ($kvp =~ m/^policy=(preferred|bind|interleave)$/) {
1311             $res->{policy} = $1;
1312         } elsif ($kvp =~ m/^cpus=(\d+)(-(\d+))?$/) {
1313             $res->{cpus}->{start} = $1;
1314             $res->{cpus}->{end} = $3;
1315         } elsif ($kvp =~ m/^hostnodes=(\d+)(-(\d+))?$/) {
1316             $res->{hostnodes}->{start} = $1;
1317             $res->{hostnodes}->{end} = $3;
1318         } else {
1319             return undef;
1320         }
1321     }
1322
1323     return $res;
1324 }
1325
1326 sub parse_hostpci {
1327     my ($value) = @_;
1328
1329     return undef if !$value;
1330
1331
1332     my @list = split(/,/, $value);
1333     my $found;
1334
1335     my $res = {};
1336     foreach my $kv (@list) {
1337
1338         if ($kv =~ m/^(host=)?([a-f0-9]{2}:[a-f0-9]{2})(\.([a-f0-9]))?$/) {
1339             $found = 1;
1340             if(defined($4)){
1341                 push @{$res->{pciid}}, { id => $2 , function => $4};
1342
1343             }else{
1344                 my $pcidevices = lspci($2);
1345                 $res->{pciid} = $pcidevices->{$2};
1346             }
1347         } elsif ($kv =~ m/^rombar=(on|off)$/) {
1348             $res->{rombar} = $1;
1349         } elsif ($kv =~ m/^x-vga=(on|off)$/) {
1350             $res->{'x-vga'} = $1;
1351         } elsif ($kv =~ m/^pcie=(\d+)$/) {
1352             $res->{pcie} = 1 if $1 == 1;
1353         } else {
1354             warn "unknown hostpci setting '$kv'\n";
1355         }
1356     }
1357
1358     return undef if !$found;
1359
1360     return $res;
1361 }
1362
1363 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1364 sub parse_net {
1365     my ($data) = @_;
1366
1367     my $res = {};
1368
1369     foreach my $kvp (split(/,/, $data)) {
1370
1371         if ($kvp =~ m/^(ne2k_pci|e1000|e1000-82540em|e1000-82544gc|e1000-82545em|rtl8139|pcnet|virtio|ne2k_isa|i82551|i82557b|i82559er|vmxnet3)(=([0-9a-f]{2}(:[0-9a-f]{2}){5}))?$/i) {
1372             my $model = lc($1);
1373             my $mac = defined($3) ? uc($3) : PVE::Tools::random_ether_addr();
1374             $res->{model} = $model;
1375             $res->{macaddr} = $mac;
1376         } elsif ($kvp =~ m/^bridge=(\S+)$/) {
1377             $res->{bridge} = $1;
1378         } elsif ($kvp =~ m/^queues=(\d+)$/) {
1379             $res->{queues} = $1;
1380         } elsif ($kvp =~ m/^rate=(\d+(\.\d+)?)$/) {
1381             $res->{rate} = $1;
1382         } elsif ($kvp =~ m/^tag=(\d+)$/) {
1383             $res->{tag} = $1;
1384         } elsif ($kvp =~ m/^firewall=([01])$/) {
1385             $res->{firewall} = $1;
1386         } elsif ($kvp =~ m/^link_down=([01])$/) {
1387             $res->{link_down} = $1;
1388         } else {
1389             return undef;
1390         }
1391
1392     }
1393
1394     return undef if !$res->{model};
1395
1396     return $res;
1397 }
1398
1399 sub print_net {
1400     my $net = shift;
1401
1402     my $res = "$net->{model}";
1403     $res .= "=$net->{macaddr}" if $net->{macaddr};
1404     $res .= ",bridge=$net->{bridge}" if $net->{bridge};
1405     $res .= ",rate=$net->{rate}" if $net->{rate};
1406     $res .= ",tag=$net->{tag}" if $net->{tag};
1407     $res .= ",firewall=1" if $net->{firewall};
1408     $res .= ",link_down=1" if $net->{link_down};
1409     $res .= ",queues=$net->{queues}" if $net->{queues};
1410
1411     return $res;
1412 }
1413
1414 sub add_random_macs {
1415     my ($settings) = @_;
1416
1417     foreach my $opt (keys %$settings) {
1418         next if $opt !~ m/^net(\d+)$/;
1419         my $net = parse_net($settings->{$opt});
1420         next if !$net;
1421         $settings->{$opt} = print_net($net);
1422     }
1423 }
1424
1425 sub add_unused_volume {
1426     my ($config, $volid) = @_;
1427
1428     my $key;
1429     for (my $ind = $MAX_UNUSED_DISKS - 1; $ind >= 0; $ind--) {
1430         my $test = "unused$ind";
1431         if (my $vid = $config->{$test}) {
1432             return if $vid eq $volid; # do not add duplicates
1433         } else {
1434             $key = $test;
1435         }
1436     }
1437
1438     die "To many unused volume - please delete them first.\n" if !$key;
1439
1440     $config->{$key} = $volid;
1441
1442     return $key;
1443 }
1444
1445 sub vm_is_volid_owner {
1446     my ($storecfg, $vmid, $volid) = @_;
1447
1448     if ($volid !~  m|^/|) {
1449         my ($path, $owner);
1450         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1451         if ($owner && ($owner == $vmid)) {
1452             return 1;
1453         }
1454     }
1455
1456     return undef;
1457 }
1458
1459 sub split_flagged_list {
1460     my $text = shift || '';
1461     $text =~ s/[,;]/ /g;
1462     $text =~ s/^\s+//;
1463     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
1464 }
1465
1466 sub join_flagged_list {
1467     my ($how, $lst) = @_;
1468     join $how, map { $lst->{$_} . $_ } keys %$lst;
1469 }
1470
1471 sub vmconfig_delete_pending_option {
1472     my ($conf, $key, $force) = @_;
1473
1474     delete $conf->{pending}->{$key};
1475     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1476     $pending_delete_hash->{$key} = $force ? '!' : '';
1477     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1478 }
1479
1480 sub vmconfig_undelete_pending_option {
1481     my ($conf, $key) = @_;
1482
1483     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1484     delete $pending_delete_hash->{$key};
1485
1486     if (%$pending_delete_hash) {
1487         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1488     } else {
1489         delete $conf->{pending}->{delete};
1490     }
1491 }
1492
1493 sub vmconfig_register_unused_drive {
1494     my ($storecfg, $vmid, $conf, $drive) = @_;
1495
1496     if (!drive_is_cdrom($drive)) {
1497         my $volid = $drive->{file};
1498         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1499             add_unused_volume($conf, $volid, $vmid);
1500         }
1501     }
1502 }
1503
1504 sub vmconfig_cleanup_pending {
1505     my ($conf) = @_;
1506
1507     # remove pending changes when nothing changed
1508     my $changes;
1509     foreach my $opt (keys %{$conf->{pending}}) {
1510         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
1511             $changes = 1;
1512             delete $conf->{pending}->{$opt};
1513         }
1514     }
1515
1516     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
1517     my $pending_delete_hash = {};
1518     while (my ($opt, $force) = each %$current_delete_hash) {
1519         if (defined($conf->{$opt})) {
1520             $pending_delete_hash->{$opt} = $force;
1521         } else {
1522             $changes = 1;
1523         }
1524     }
1525
1526     if (%$pending_delete_hash) {
1527         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1528     } else {
1529         delete $conf->{pending}->{delete};
1530     }
1531
1532     return $changes;
1533 }
1534
1535 my $valid_smbios1_options = {
1536     manufacturer => '\S+',
1537     product => '\S+',
1538     version => '\S+',
1539     serial => '\S+',
1540     uuid => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1541     sku => '\S+',
1542     family => '\S+',
1543 };
1544
1545 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
1546 sub parse_smbios1 {
1547     my ($data) = @_;
1548
1549     my $res = {};
1550
1551     foreach my $kvp (split(/,/, $data)) {
1552         return undef if $kvp !~ m/^(\S+)=(.+)$/;
1553         my ($k, $v) = split(/=/, $kvp);
1554         return undef if !defined($k) || !defined($v);
1555         return undef if !$valid_smbios1_options->{$k};
1556         return undef if $v !~ m/^$valid_smbios1_options->{$k}$/;
1557         $res->{$k} = $v;
1558     }
1559
1560     return $res;
1561 }
1562
1563 sub print_smbios1 {
1564     my ($smbios1) = @_;
1565
1566     my $data = '';
1567     foreach my $k (keys %$smbios1) {
1568         next if !defined($smbios1->{$k});
1569         next if !$valid_smbios1_options->{$k};
1570         $data .= ',' if $data;
1571         $data .= "$k=$smbios1->{$k}";
1572     }
1573     return $data;
1574 }
1575
1576 PVE::JSONSchema::register_format('pve-qm-smbios1', \&verify_smbios1);
1577 sub verify_smbios1 {
1578     my ($value, $noerr) = @_;
1579
1580     return $value if parse_smbios1($value);
1581
1582     return undef if $noerr;
1583
1584     die "unable to parse smbios (type 1) options\n";
1585 }
1586
1587 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
1588 sub verify_bootdisk {
1589     my ($value, $noerr) = @_;
1590
1591     return $value if valid_drivename($value);
1592
1593     return undef if $noerr;
1594
1595     die "invalid boot disk '$value'\n";
1596 }
1597
1598 PVE::JSONSchema::register_format('pve-qm-numanode', \&verify_numa);
1599 sub verify_numa {
1600     my ($value, $noerr) = @_;
1601
1602     return $value if parse_numa($value);
1603
1604     return undef if $noerr;
1605
1606     die "unable to parse numa options\n";
1607 }
1608
1609 PVE::JSONSchema::register_format('pve-qm-net', \&verify_net);
1610 sub verify_net {
1611     my ($value, $noerr) = @_;
1612
1613     return $value if parse_net($value);
1614
1615     return undef if $noerr;
1616
1617     die "unable to parse network options\n";
1618 }
1619
1620 PVE::JSONSchema::register_format('pve-qm-drive', \&verify_drive);
1621 sub verify_drive {
1622     my ($value, $noerr) = @_;
1623
1624     return $value if parse_drive(undef, $value);
1625
1626     return undef if $noerr;
1627
1628     die "unable to parse drive options\n";
1629 }
1630
1631 PVE::JSONSchema::register_format('pve-qm-hostpci', \&verify_hostpci);
1632 sub verify_hostpci {
1633     my ($value, $noerr) = @_;
1634
1635     return $value if parse_hostpci($value);
1636
1637     return undef if $noerr;
1638
1639     die "unable to parse pci id\n";
1640 }
1641
1642 PVE::JSONSchema::register_format('pve-qm-watchdog', \&verify_watchdog);
1643 sub verify_watchdog {
1644     my ($value, $noerr) = @_;
1645
1646     return $value if parse_watchdog($value);
1647
1648     return undef if $noerr;
1649
1650     die "unable to parse watchdog options\n";
1651 }
1652
1653 sub parse_watchdog {
1654     my ($value) = @_;
1655
1656     return undef if !$value;
1657
1658     my $res = {};
1659
1660     foreach my $p (split(/,/, $value)) {
1661         next if $p =~ m/^\s*$/;
1662
1663         if ($p =~ m/^(model=)?(i6300esb|ib700)$/) {
1664             $res->{model} = $2;
1665         } elsif ($p =~ m/^(action=)?(reset|shutdown|poweroff|pause|debug|none)$/) {
1666             $res->{action} = $2;
1667         } else {
1668             return undef;
1669         }
1670     }
1671
1672     return $res;
1673 }
1674
1675 sub parse_usb_device {
1676     my ($value) = @_;
1677
1678     return undef if !$value;
1679
1680     my @dl = split(/,/, $value);
1681     my $found;
1682
1683     my $res = {};
1684     foreach my $v (@dl) {
1685         if ($v =~ m/^host=(0x)?([0-9A-Fa-f]{4}):(0x)?([0-9A-Fa-f]{4})$/) {
1686             $found = 1;
1687             $res->{vendorid} = $2;
1688             $res->{productid} = $4;
1689         } elsif ($v =~ m/^host=(\d+)\-(\d+(\.\d+)*)$/) {
1690             $found = 1;
1691             $res->{hostbus} = $1;
1692             $res->{hostport} = $2;
1693         } elsif ($v =~ m/^spice$/) {
1694             $found = 1;
1695             $res->{spice} = 1;
1696         } else {
1697             return undef;
1698         }
1699     }
1700     return undef if !$found;
1701
1702     return $res;
1703 }
1704
1705 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
1706 sub verify_usb_device {
1707     my ($value, $noerr) = @_;
1708
1709     return $value if parse_usb_device($value);
1710
1711     return undef if $noerr;
1712
1713     die "unable to parse usb device\n";
1714 }
1715
1716 # add JSON properties for create and set function
1717 sub json_config_properties {
1718     my $prop = shift;
1719
1720     foreach my $opt (keys %$confdesc) {
1721         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
1722         $prop->{$opt} = $confdesc->{$opt};
1723     }
1724
1725     return $prop;
1726 }
1727
1728 sub check_type {
1729     my ($key, $value) = @_;
1730
1731     die "unknown setting '$key'\n" if !$confdesc->{$key};
1732
1733     my $type = $confdesc->{$key}->{type};
1734
1735     if (!defined($value)) {
1736         die "got undefined value\n";
1737     }
1738
1739     if ($value =~ m/[\n\r]/) {
1740         die "property contains a line feed\n";
1741     }
1742
1743     if ($type eq 'boolean') {
1744         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
1745         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
1746         die "type check ('boolean') failed - got '$value'\n";
1747     } elsif ($type eq 'integer') {
1748         return int($1) if $value =~ m/^(\d+)$/;
1749         die "type check ('integer') failed - got '$value'\n";
1750     } elsif ($type eq 'number') {
1751         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
1752         die "type check ('number') failed - got '$value'\n";
1753     } elsif ($type eq 'string') {
1754         if (my $fmt = $confdesc->{$key}->{format}) {
1755             if ($fmt eq 'pve-qm-drive') {
1756                 # special case - we need to pass $key to parse_drive()
1757                 my $drive = parse_drive($key, $value);
1758                 return $value if $drive;
1759                 die "unable to parse drive options\n";
1760             }
1761             PVE::JSONSchema::check_format($fmt, $value);
1762             return $value;
1763         }
1764         $value =~ s/^\"(.*)\"$/$1/;
1765         return $value;
1766     } else {
1767         die "internal error"
1768     }
1769 }
1770
1771 sub lock_config_full {
1772     my ($vmid, $timeout, $code, @param) = @_;
1773
1774     my $filename = config_file_lock($vmid);
1775
1776     my $res = lock_file($filename, $timeout, $code, @param);
1777
1778     die $@ if $@;
1779
1780     return $res;
1781 }
1782
1783 sub lock_config_mode {
1784     my ($vmid, $timeout, $shared, $code, @param) = @_;
1785
1786     my $filename = config_file_lock($vmid);
1787
1788     my $res = lock_file_full($filename, $timeout, $shared, $code, @param);
1789
1790     die $@ if $@;
1791
1792     return $res;
1793 }
1794
1795 sub lock_config {
1796     my ($vmid, $code, @param) = @_;
1797
1798     return lock_config_full($vmid, 10, $code, @param);
1799 }
1800
1801 sub cfs_config_path {
1802     my ($vmid, $node) = @_;
1803
1804     $node = $nodename if !$node;
1805     return "nodes/$node/qemu-server/$vmid.conf";
1806 }
1807
1808 sub check_iommu_support{
1809     #fixme : need to check IOMMU support
1810     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
1811
1812     my $iommu=1;
1813     return $iommu;
1814
1815 }
1816
1817 sub config_file {
1818     my ($vmid, $node) = @_;
1819
1820     my $cfspath = cfs_config_path($vmid, $node);
1821     return "/etc/pve/$cfspath";
1822 }
1823
1824 sub config_file_lock {
1825     my ($vmid) = @_;
1826
1827     return "$lock_dir/lock-$vmid.conf";
1828 }
1829
1830 sub touch_config {
1831     my ($vmid) = @_;
1832
1833     my $conf = config_file($vmid);
1834     utime undef, undef, $conf;
1835 }
1836
1837 sub destroy_vm {
1838     my ($storecfg, $vmid, $keep_empty_config) = @_;
1839
1840     my $conffile = config_file($vmid);
1841
1842     my $conf = load_config($vmid);
1843
1844     check_lock($conf);
1845
1846     # only remove disks owned by this VM
1847     foreach_drive($conf, sub {
1848         my ($ds, $drive) = @_;
1849
1850         return if drive_is_cdrom($drive);
1851
1852         my $volid = $drive->{file};
1853
1854         return if !$volid || $volid =~ m|^/|;
1855
1856         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
1857         return if !$path || !$owner || ($owner != $vmid);
1858
1859         PVE::Storage::vdisk_free($storecfg, $volid);
1860     });
1861
1862     if ($keep_empty_config) {
1863         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
1864     } else {
1865         unlink $conffile;
1866     }
1867
1868     # also remove unused disk
1869     eval {
1870         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
1871
1872         eval {
1873             PVE::Storage::foreach_volid($dl, sub {
1874                 my ($volid, $sid, $volname, $d) = @_;
1875                 PVE::Storage::vdisk_free($storecfg, $volid);
1876             });
1877         };
1878         warn $@ if $@;
1879
1880     };
1881     warn $@ if $@;
1882 }
1883
1884 sub load_config {
1885     my ($vmid, $node) = @_;
1886
1887     my $cfspath = cfs_config_path($vmid, $node);
1888
1889     my $conf = PVE::Cluster::cfs_read_file($cfspath);
1890
1891     die "no such VM ('$vmid')\n" if !defined($conf);
1892
1893     return $conf;
1894 }
1895
1896 sub parse_vm_config {
1897     my ($filename, $raw) = @_;
1898
1899     return undef if !defined($raw);
1900
1901     my $res = {
1902         digest => Digest::SHA::sha1_hex($raw),
1903         snapshots => {},
1904         pending => {},
1905     };
1906
1907     $filename =~ m|/qemu-server/(\d+)\.conf$|
1908         || die "got strange filename '$filename'";
1909
1910     my $vmid = $1;
1911
1912     my $conf = $res;
1913     my $descr;
1914     my $section = '';
1915
1916     my @lines = split(/\n/, $raw);
1917     foreach my $line (@lines) {
1918         next if $line =~ m/^\s*$/;
1919
1920         if ($line =~ m/^\[PENDING\]\s*$/i) {
1921             $section = 'pending';
1922             if (defined($descr)) {
1923                 $descr =~ s/\s+$//;
1924                 $conf->{description} = $descr;
1925             }
1926             $descr = undef;
1927             $conf = $res->{$section} = {};
1928             next;
1929
1930         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
1931             $section = $1;
1932             if (defined($descr)) {
1933                 $descr =~ s/\s+$//;
1934                 $conf->{description} = $descr;
1935             }
1936             $descr = undef;
1937             $conf = $res->{snapshots}->{$section} = {};
1938             next;
1939         }
1940
1941         if ($line =~ m/^\#(.*)\s*$/) {
1942             $descr = '' if !defined($descr);
1943             $descr .= PVE::Tools::decode_text($1) . "\n";
1944             next;
1945         }
1946
1947         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
1948             $descr = '' if !defined($descr);
1949             $descr .= PVE::Tools::decode_text($2);
1950         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
1951             $conf->{snapstate} = $1;
1952         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
1953             my $key = $1;
1954             my $value = $2;
1955             $conf->{$key} = $value;
1956         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
1957             my $value = $1;
1958             if ($section eq 'pending') {
1959                 $conf->{delete} = $value; # we parse this later
1960             } else {
1961                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
1962             }
1963         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
1964             my $key = $1;
1965             my $value = $2;
1966             eval { $value = check_type($key, $value); };
1967             if ($@) {
1968                 warn "vm $vmid - unable to parse value of '$key' - $@";
1969             } else {
1970                 my $fmt = $confdesc->{$key}->{format};
1971                 if ($fmt && $fmt eq 'pve-qm-drive') {
1972                     my $v = parse_drive($key, $value);
1973                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
1974                         $v->{file} = $volid;
1975                         $value = print_drive($vmid, $v);
1976                     } else {
1977                         warn "vm $vmid - unable to parse value of '$key'\n";
1978                         next;
1979                     }
1980                 }
1981
1982                 if ($key eq 'cdrom') {
1983                     $conf->{ide2} = $value;
1984                 } else {
1985                     $conf->{$key} = $value;
1986                 }
1987             }
1988         }
1989     }
1990
1991     if (defined($descr)) {
1992         $descr =~ s/\s+$//;
1993         $conf->{description} = $descr;
1994     }
1995     delete $res->{snapstate}; # just to be sure
1996
1997     return $res;
1998 }
1999
2000 sub write_vm_config {
2001     my ($filename, $conf) = @_;
2002
2003     delete $conf->{snapstate}; # just to be sure
2004
2005     if ($conf->{cdrom}) {
2006         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2007         $conf->{ide2} = $conf->{cdrom};
2008         delete $conf->{cdrom};
2009     }
2010
2011     # we do not use 'smp' any longer
2012     if ($conf->{sockets}) {
2013         delete $conf->{smp};
2014     } elsif ($conf->{smp}) {
2015         $conf->{sockets} = $conf->{smp};
2016         delete $conf->{cores};
2017         delete $conf->{smp};
2018     }
2019
2020     my $used_volids = {};
2021
2022     my $cleanup_config = sub {
2023         my ($cref, $pending, $snapname) = @_;
2024
2025         foreach my $key (keys %$cref) {
2026             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2027                 $key eq 'snapstate' || $key eq 'pending';
2028             my $value = $cref->{$key};
2029             if ($key eq 'delete') {
2030                 die "propertry 'delete' is only allowed in [PENDING]\n"
2031                     if !$pending;
2032                 # fixme: check syntax?
2033                 next;
2034             }
2035             eval { $value = check_type($key, $value); };
2036             die "unable to parse value of '$key' - $@" if $@;
2037
2038             $cref->{$key} = $value;
2039
2040             if (!$snapname && valid_drivename($key)) {
2041                 my $drive = parse_drive($key, $value);
2042                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2043             }
2044         }
2045     };
2046
2047     &$cleanup_config($conf);
2048
2049     &$cleanup_config($conf->{pending}, 1);
2050
2051     foreach my $snapname (keys %{$conf->{snapshots}}) {
2052         die "internal error" if $snapname eq 'pending';
2053         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2054     }
2055
2056     # remove 'unusedX' settings if we re-add a volume
2057     foreach my $key (keys %$conf) {
2058         my $value = $conf->{$key};
2059         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2060             delete $conf->{$key};
2061         }
2062     }
2063
2064     my $generate_raw_config = sub {
2065         my ($conf, $pending) = @_;
2066
2067         my $raw = '';
2068
2069         # add description as comment to top of file
2070         if (defined(my $descr = $conf->{description})) {
2071             if ($descr) {
2072                 foreach my $cl (split(/\n/, $descr)) {
2073                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2074                 }
2075             } else {
2076                 $raw .= "#\n" if $pending;
2077             }
2078         }
2079
2080         foreach my $key (sort keys %$conf) {
2081             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2082             $raw .= "$key: $conf->{$key}\n";
2083         }
2084         return $raw;
2085     };
2086
2087     my $raw = &$generate_raw_config($conf);
2088
2089     if (scalar(keys %{$conf->{pending}})){
2090         $raw .= "\n[PENDING]\n";
2091         $raw .= &$generate_raw_config($conf->{pending}, 1);
2092     }
2093
2094     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2095         $raw .= "\n[$snapname]\n";
2096         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2097     }
2098
2099     return $raw;
2100 }
2101
2102 sub update_config_nolock {
2103     my ($vmid, $conf, $skiplock) = @_;
2104
2105     check_lock($conf) if !$skiplock;
2106
2107     my $cfspath = cfs_config_path($vmid);
2108
2109     PVE::Cluster::cfs_write_file($cfspath, $conf);
2110 }
2111
2112 sub update_config {
2113     my ($vmid, $conf, $skiplock) = @_;
2114
2115     lock_config($vmid, &update_config_nolock, $conf, $skiplock);
2116 }
2117
2118 sub load_defaults {
2119
2120     my $res = {};
2121
2122     # we use static defaults from our JSON schema configuration
2123     foreach my $key (keys %$confdesc) {
2124         if (defined(my $default = $confdesc->{$key}->{default})) {
2125             $res->{$key} = $default;
2126         }
2127     }
2128
2129     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2130     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2131
2132     return $res;
2133 }
2134
2135 sub config_list {
2136     my $vmlist = PVE::Cluster::get_vmlist();
2137     my $res = {};
2138     return $res if !$vmlist || !$vmlist->{ids};
2139     my $ids = $vmlist->{ids};
2140
2141     foreach my $vmid (keys %$ids) {
2142         my $d = $ids->{$vmid};
2143         next if !$d->{node} || $d->{node} ne $nodename;
2144         next if !$d->{type} || $d->{type} ne 'qemu';
2145         $res->{$vmid}->{exists} = 1;
2146     }
2147     return $res;
2148 }
2149
2150 # test if VM uses local resources (to prevent migration)
2151 sub check_local_resources {
2152     my ($conf, $noerr) = @_;
2153
2154     my $loc_res = 0;
2155
2156     $loc_res = 1 if $conf->{hostusb}; # old syntax
2157     $loc_res = 1 if $conf->{hostpci}; # old syntax
2158
2159     foreach my $k (keys %$conf) {
2160         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2161         # sockets are safe: they will recreated be on the target side post-migrate
2162         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2163         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2164     }
2165
2166     die "VM uses local resources\n" if $loc_res && !$noerr;
2167
2168     return $loc_res;
2169 }
2170
2171 # check if used storages are available on all nodes (use by migrate)
2172 sub check_storage_availability {
2173     my ($storecfg, $conf, $node) = @_;
2174
2175     foreach_drive($conf, sub {
2176         my ($ds, $drive) = @_;
2177
2178         my $volid = $drive->{file};
2179         return if !$volid;
2180
2181         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2182         return if !$sid;
2183
2184         # check if storage is available on both nodes
2185         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2186         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2187    });
2188 }
2189
2190 # list nodes where all VM images are available (used by has_feature API)
2191 sub shared_nodes {
2192     my ($conf, $storecfg) = @_;
2193
2194     my $nodelist = PVE::Cluster::get_nodelist();
2195     my $nodehash = { map { $_ => 1 } @$nodelist };
2196     my $nodename = PVE::INotify::nodename();
2197
2198     foreach_drive($conf, sub {
2199         my ($ds, $drive) = @_;
2200
2201         my $volid = $drive->{file};
2202         return if !$volid;
2203
2204         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2205         if ($storeid) {
2206             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2207             if ($scfg->{disable}) {
2208                 $nodehash = {};
2209             } elsif (my $avail = $scfg->{nodes}) {
2210                 foreach my $node (keys %$nodehash) {
2211                     delete $nodehash->{$node} if !$avail->{$node};
2212                 }
2213             } elsif (!$scfg->{shared}) {
2214                 foreach my $node (keys %$nodehash) {
2215                     delete $nodehash->{$node} if $node ne $nodename
2216                 }
2217             }
2218         }
2219     });
2220
2221     return $nodehash
2222 }
2223
2224 sub check_lock {
2225     my ($conf) = @_;
2226
2227     die "VM is locked ($conf->{lock})\n" if $conf->{lock};
2228 }
2229
2230 sub check_cmdline {
2231     my ($pidfile, $pid) = @_;
2232
2233     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2234     if (defined($fh)) {
2235         my $line = <$fh>;
2236         $fh->close;
2237         return undef if !$line;
2238         my @param = split(/\0/, $line);
2239
2240         my $cmd = $param[0];
2241         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2242
2243         for (my $i = 0; $i < scalar (@param); $i++) {
2244             my $p = $param[$i];
2245             next if !$p;
2246             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2247                 my $p = $param[$i+1];
2248                 return 1 if $p && ($p eq $pidfile);
2249                 return undef;
2250             }
2251         }
2252     }
2253     return undef;
2254 }
2255
2256 sub check_running {
2257     my ($vmid, $nocheck, $node) = @_;
2258
2259     my $filename = config_file($vmid, $node);
2260
2261     die "unable to find configuration file for VM $vmid - no such machine\n"
2262         if !$nocheck && ! -f $filename;
2263
2264     my $pidfile = pidfile_name($vmid);
2265
2266     if (my $fd = IO::File->new("<$pidfile")) {
2267         my $st = stat($fd);
2268         my $line = <$fd>;
2269         close($fd);
2270
2271         my $mtime = $st->mtime;
2272         if ($mtime > time()) {
2273             warn "file '$filename' modified in future\n";
2274         }
2275
2276         if ($line =~ m/^(\d+)$/) {
2277             my $pid = $1;
2278             if (check_cmdline($pidfile, $pid)) {
2279                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2280                     return $pid;
2281                 }
2282             }
2283         }
2284     }
2285
2286     return undef;
2287 }
2288
2289 sub vzlist {
2290
2291     my $vzlist = config_list();
2292
2293     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2294
2295     while (defined(my $de = $fd->read)) {
2296         next if $de !~ m/^(\d+)\.pid$/;
2297         my $vmid = $1;
2298         next if !defined($vzlist->{$vmid});
2299         if (my $pid = check_running($vmid)) {
2300             $vzlist->{$vmid}->{pid} = $pid;
2301         }
2302     }
2303
2304     return $vzlist;
2305 }
2306
2307 sub disksize {
2308     my ($storecfg, $conf) = @_;
2309
2310     my $bootdisk = $conf->{bootdisk};
2311     return undef if !$bootdisk;
2312     return undef if !valid_drivename($bootdisk);
2313
2314     return undef if !$conf->{$bootdisk};
2315
2316     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2317     return undef if !defined($drive);
2318
2319     return undef if drive_is_cdrom($drive);
2320
2321     my $volid = $drive->{file};
2322     return undef if !$volid;
2323
2324     return $drive->{size};
2325 }
2326
2327 my $last_proc_pid_stat;
2328
2329 # get VM status information
2330 # This must be fast and should not block ($full == false)
2331 # We only query KVM using QMP if $full == true (this can be slow)
2332 sub vmstatus {
2333     my ($opt_vmid, $full) = @_;
2334
2335     my $res = {};
2336
2337     my $storecfg = PVE::Storage::config();
2338
2339     my $list = vzlist();
2340     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2341
2342     my $cpucount = $cpuinfo->{cpus} || 1;
2343
2344     foreach my $vmid (keys %$list) {
2345         next if $opt_vmid && ($vmid ne $opt_vmid);
2346
2347         my $cfspath = cfs_config_path($vmid);
2348         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2349
2350         my $d = {};
2351         $d->{pid} = $list->{$vmid}->{pid};
2352
2353         # fixme: better status?
2354         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2355
2356         my $size = disksize($storecfg, $conf);
2357         if (defined($size)) {
2358             $d->{disk} = 0; # no info available
2359             $d->{maxdisk} = $size;
2360         } else {
2361             $d->{disk} = 0;
2362             $d->{maxdisk} = 0;
2363         }
2364
2365         $d->{cpus} = ($conf->{sockets} || 1) * ($conf->{cores} || 1);
2366         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2367         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2368
2369         $d->{name} = $conf->{name} || "VM $vmid";
2370         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024) : 0;
2371
2372         if ($conf->{balloon}) {
2373             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2374             $d->{shares} = defined($conf->{shares}) ? $conf->{shares} : 1000;
2375         }
2376
2377         $d->{uptime} = 0;
2378         $d->{cpu} = 0;
2379         $d->{mem} = 0;
2380
2381         $d->{netout} = 0;
2382         $d->{netin} = 0;
2383
2384         $d->{diskread} = 0;
2385         $d->{diskwrite} = 0;
2386
2387         $d->{template} = is_template($conf);
2388
2389         $res->{$vmid} = $d;
2390     }
2391
2392     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2393     foreach my $dev (keys %$netdev) {
2394         next if $dev !~ m/^tap([1-9]\d*)i/;
2395         my $vmid = $1;
2396         my $d = $res->{$vmid};
2397         next if !$d;
2398
2399         $d->{netout} += $netdev->{$dev}->{receive};
2400         $d->{netin} += $netdev->{$dev}->{transmit};
2401
2402         if ($full) {
2403             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2404             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2405         }
2406
2407     }
2408
2409     my $ctime = gettimeofday;
2410
2411     foreach my $vmid (keys %$list) {
2412
2413         my $d = $res->{$vmid};
2414         my $pid = $d->{pid};
2415         next if !$pid;
2416
2417         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2418         next if !$pstat; # not running
2419
2420         my $used = $pstat->{utime} + $pstat->{stime};
2421
2422         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2423
2424         if ($pstat->{vsize}) {
2425             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2426         }
2427
2428         my $old = $last_proc_pid_stat->{$pid};
2429         if (!$old) {
2430             $last_proc_pid_stat->{$pid} = {
2431                 time => $ctime,
2432                 used => $used,
2433                 cpu => 0,
2434             };
2435             next;
2436         }
2437
2438         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2439
2440         if ($dtime > 1000) {
2441             my $dutime = $used -  $old->{used};
2442
2443             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2444             $last_proc_pid_stat->{$pid} = {
2445                 time => $ctime,
2446                 used => $used,
2447                 cpu => $d->{cpu},
2448             };
2449         } else {
2450             $d->{cpu} = $old->{cpu};
2451         }
2452     }
2453
2454     return $res if !$full;
2455
2456     my $qmpclient = PVE::QMPClient->new();
2457
2458     my $ballooncb = sub {
2459         my ($vmid, $resp) = @_;
2460
2461         my $info = $resp->{'return'};
2462         return if !$info->{max_mem};
2463
2464         my $d = $res->{$vmid};
2465
2466         # use memory assigned to VM
2467         $d->{maxmem} = $info->{max_mem};
2468         $d->{balloon} = $info->{actual};
2469
2470         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2471             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2472             $d->{freemem} = $info->{free_mem};
2473         }
2474
2475         $d->{ballooninfo} = $info;
2476     };
2477
2478     my $blockstatscb = sub {
2479         my ($vmid, $resp) = @_;
2480         my $data = $resp->{'return'} || [];
2481         my $totalrdbytes = 0;
2482         my $totalwrbytes = 0;
2483
2484         for my $blockstat (@$data) {
2485             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2486             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2487
2488             $blockstat->{device} =~ s/drive-//;
2489             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2490         }
2491         $res->{$vmid}->{diskread} = $totalrdbytes;
2492         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2493     };
2494
2495     my $statuscb = sub {
2496         my ($vmid, $resp) = @_;
2497
2498         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2499         # this fails if ballon driver is not loaded, so this must be
2500         # the last commnand (following command are aborted if this fails).
2501         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2502
2503         my $status = 'unknown';
2504         if (!defined($status = $resp->{'return'}->{status})) {
2505             warn "unable to get VM status\n";
2506             return;
2507         }
2508
2509         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2510     };
2511
2512     foreach my $vmid (keys %$list) {
2513         next if $opt_vmid && ($vmid ne $opt_vmid);
2514         next if !$res->{$vmid}->{pid}; # not running
2515         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2516     }
2517
2518     $qmpclient->queue_execute(undef, 1);
2519
2520     foreach my $vmid (keys %$list) {
2521         next if $opt_vmid && ($vmid ne $opt_vmid);
2522         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2523     }
2524
2525     return $res;
2526 }
2527
2528 sub foreach_dimm {
2529     my ($conf, $vmid, $memory, $sockets, $func) = @_;
2530
2531     my $dimm_id = 0;
2532     my $current_size = 1024;
2533     my $dimm_size = 512;
2534     return if $current_size == $memory;
2535
2536     for (my $j = 0; $j < 8; $j++) {
2537         for (my $i = 0; $i < 32; $i++) {
2538             my $name = "dimm${dimm_id}";
2539             $dimm_id++;
2540             my $numanode = $i % $sockets;
2541             $current_size += $dimm_size;
2542             &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
2543             return  $current_size if $current_size >= $memory;
2544         }
2545         $dimm_size *= 2;
2546     }
2547 }
2548
2549 sub foreach_reverse_dimm {
2550     my ($conf, $vmid, $memory, $sockets, $func) = @_;
2551
2552     my $dimm_id = 253;
2553     my $current_size = 4177920;
2554     my $dimm_size = 65536;
2555     return if $current_size == $memory;
2556
2557     for (my $j = 0; $j < 8; $j++) {
2558         for (my $i = 0; $i < 32; $i++) {
2559             my $name = "dimm${dimm_id}";
2560             $dimm_id--;
2561             my $numanode = $i % $sockets;
2562             $current_size -= $dimm_size;
2563             &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
2564             return  $current_size if $current_size <= $memory;
2565         }
2566         $dimm_size /= 2;
2567     }
2568 }
2569
2570 sub foreach_drive {
2571     my ($conf, $func) = @_;
2572
2573     foreach my $ds (keys %$conf) {
2574         next if !valid_drivename($ds);
2575
2576         my $drive = parse_drive($ds, $conf->{$ds});
2577         next if !$drive;
2578
2579         &$func($ds, $drive);
2580     }
2581 }
2582
2583 sub foreach_volid {
2584     my ($conf, $func) = @_;
2585
2586     my $volhash = {};
2587
2588     my $test_volid = sub {
2589         my ($volid, $is_cdrom) = @_;
2590
2591         return if !$volid;
2592
2593         $volhash->{$volid} = $is_cdrom || 0;
2594     };
2595
2596     foreach_drive($conf, sub {
2597         my ($ds, $drive) = @_;
2598         &$test_volid($drive->{file}, drive_is_cdrom($drive));
2599     });
2600
2601     foreach my $snapname (keys %{$conf->{snapshots}}) {
2602         my $snap = $conf->{snapshots}->{$snapname};
2603         &$test_volid($snap->{vmstate}, 0);
2604         foreach_drive($snap, sub {
2605             my ($ds, $drive) = @_;
2606             &$test_volid($drive->{file}, drive_is_cdrom($drive));
2607         });
2608     }
2609
2610     foreach my $volid (keys %$volhash) {
2611         &$func($volid, $volhash->{$volid});
2612     }
2613 }
2614
2615 sub vga_conf_has_spice {
2616     my ($vga) = @_;
2617
2618     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
2619
2620     return $1 || 1;
2621 }
2622
2623 sub config_to_command {
2624     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2625
2626     my $cmd = [];
2627     my $globalFlags = [];
2628     my $machineFlags = [];
2629     my $rtcFlags = [];
2630     my $cpuFlags = [];
2631     my $devices = [];
2632     my $pciaddr = '';
2633     my $bridges = {};
2634     my $kvmver = kvm_user_version();
2635     my $vernum = 0; # unknown
2636     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
2637         $vernum = $1*1000000+$2*1000;
2638     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
2639         $vernum = $1*1000000+$2*1000+$3;
2640     }
2641
2642     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
2643
2644     my $have_ovz = -f '/proc/vz/vestat';
2645
2646     my $q35 = machine_type_is_q35($conf);
2647     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2648     my $machine_type = $forcemachine || $conf->{machine};
2649     my $use_old_bios_files = undef;
2650     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
2651
2652     my $cpuunits = defined($conf->{cpuunits}) ?
2653             $conf->{cpuunits} : $defaults->{cpuunits};
2654
2655     push @$cmd, '/usr/bin/systemd-run';
2656     push @$cmd, '--scope';
2657     push @$cmd, '--slice', "qemu";
2658     push @$cmd, '--unit', $vmid;
2659     # set KillMode=none, so that systemd don't kill those scopes
2660     # at shutdown (pve-manager service should stop the VMs instead)
2661     push @$cmd, '-p', "KillMode=none";
2662     push @$cmd, '-p', "CPUShares=$cpuunits";
2663     if ($conf->{cpulimit}) {
2664         my $cpulimit = int($conf->{cpulimit} * 100);
2665         push @$cmd, '-p', "CPUQuota=$cpulimit\%";
2666     }
2667
2668     push @$cmd, '/usr/bin/kvm';
2669
2670     push @$cmd, '-id', $vmid;
2671
2672     my $use_virtio = 0;
2673
2674     my $qmpsocket = qmp_socket($vmid);
2675     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2676     push @$cmd, '-mon', "chardev=qmp,mode=control";
2677
2678     my $socket = vnc_socket($vmid);
2679     push @$cmd,  '-vnc', "unix:$socket,x509,password";
2680
2681     push @$cmd, '-pidfile' , pidfile_name($vmid);
2682
2683     push @$cmd, '-daemonize';
2684
2685     if ($conf->{smbios1}) {
2686         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
2687     }
2688
2689     if ($q35) {
2690         # the q35 chipset support native usb2, so we enable usb controller
2691         # by default for this machine type
2692         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
2693     } else {
2694         $pciaddr = print_pci_addr("piix3", $bridges);
2695         push @$devices, '-device', "piix3-usb-uhci,id=uhci$pciaddr.0x2";
2696
2697         my $use_usb2 = 0;
2698         for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2699             next if !$conf->{"usb$i"};
2700             $use_usb2 = 1;
2701         }
2702         # include usb device config
2703         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-usb.cfg' if $use_usb2;
2704     }
2705
2706     my $vga = $conf->{vga};
2707
2708     my $qxlnum = vga_conf_has_spice($vga);
2709     $vga = 'qxl' if $qxlnum;
2710
2711     if (!$vga) {
2712         if ($conf->{ostype} && ($conf->{ostype} eq 'win8' ||
2713                                 $conf->{ostype} eq 'win7' ||
2714                                 $conf->{ostype} eq 'w2k8')) {
2715             $vga = 'std';
2716         } else {
2717             $vga = 'cirrus';
2718         }
2719     }
2720
2721     # enable absolute mouse coordinates (needed by vnc)
2722     my $tablet;
2723     if (defined($conf->{tablet})) {
2724         $tablet = $conf->{tablet};
2725     } else {
2726         $tablet = $defaults->{tablet};
2727         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
2728         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
2729     }
2730
2731     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
2732
2733     # host pci devices
2734     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
2735         my $d = parse_hostpci($conf->{"hostpci$i"});
2736         next if !$d;
2737
2738         my $pcie = $d->{pcie};
2739         if($pcie){
2740             die "q35 machine model is not enabled" if !$q35;
2741             $pciaddr = print_pcie_addr("hostpci$i");
2742         }else{
2743             $pciaddr = print_pci_addr("hostpci$i", $bridges);
2744         }
2745
2746         my $rombar = $d->{rombar} && $d->{rombar} eq 'off' ? ",rombar=0" : "";
2747         my $xvga = $d->{'x-vga'} && $d->{'x-vga'} eq 'on' ? ",x-vga=on" : "";
2748         if ($xvga && $xvga ne '') {
2749             push @$cpuFlags, 'kvm=off';
2750             $vga = 'none';
2751         }
2752         my $pcidevices = $d->{pciid};
2753         my $multifunction = 1 if @$pcidevices > 1;
2754
2755         my $j=0;
2756         foreach my $pcidevice (@$pcidevices) {
2757
2758             my $id = "hostpci$i";
2759             $id .= ".$j" if $multifunction;
2760             my $addr = $pciaddr;
2761             $addr .= ".$j" if $multifunction;
2762             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
2763
2764             if($j == 0){
2765                 $devicestr .= "$rombar$xvga";
2766                 $devicestr .= ",multifunction=on" if $multifunction;
2767             }
2768
2769             push @$devices, '-device', $devicestr;
2770             $j++;
2771         }
2772     }
2773
2774     # usb devices
2775     for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2776         my $d = parse_usb_device($conf->{"usb$i"});
2777         next if !$d;
2778         if ($d->{vendorid} && $d->{productid}) {
2779             push @$devices, '-device', "usb-host,vendorid=0x$d->{vendorid},productid=0x$d->{productid}";
2780         } elsif (defined($d->{hostbus}) && defined($d->{hostport})) {
2781             push @$devices, '-device', "usb-host,hostbus=$d->{hostbus},hostport=$d->{hostport}";
2782         } elsif ($d->{spice}) {
2783             # usb redir support for spice
2784             push @$devices, '-chardev', "spicevmc,id=usbredirchardev$i,name=usbredir";
2785             push @$devices, '-device', "usb-redir,chardev=usbredirchardev$i,id=usbredirdev$i,bus=ehci.0";
2786         }
2787     }
2788
2789     # serial devices
2790     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2791         if (my $path = $conf->{"serial$i"}) {
2792             if ($path eq 'socket') {
2793                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
2794                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
2795                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2796             } else {
2797                 die "no such serial device\n" if ! -c $path;
2798                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
2799                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2800             }
2801         }
2802     }
2803
2804     # parallel devices
2805     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
2806         if (my $path = $conf->{"parallel$i"}) {
2807             die "no such parallel device\n" if ! -c $path;
2808             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
2809             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
2810             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
2811         }
2812     }
2813
2814     my $vmname = $conf->{name} || "vm$vmid";
2815
2816     push @$cmd, '-name', $vmname;
2817
2818     my $sockets = 1;
2819     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
2820     $sockets = $conf->{sockets} if  $conf->{sockets};
2821
2822     my $cores = $conf->{cores} || 1;
2823
2824     my $maxcpus = $sockets * $cores;
2825
2826     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
2827
2828     my $allowed_vcpus = $cpuinfo->{cpus};
2829
2830     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
2831         if ($allowed_vcpus < $maxcpus);
2832
2833     push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
2834
2835     push @$cmd, '-nodefaults';
2836
2837     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2838
2839     my $bootindex_hash = {};
2840     my $i = 1;
2841     foreach my $o (split(//, $bootorder)) {
2842         $bootindex_hash->{$o} = $i*100;
2843         $i++;
2844     }
2845
2846     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000";
2847
2848     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
2849
2850     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
2851
2852     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
2853
2854     # time drift fix
2855     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
2856
2857     my $nokvm = defined($conf->{kvm}) && $conf->{kvm} == 0 ? 1 : 0;
2858     my $useLocaltime = $conf->{localtime};
2859
2860     if (my $ost = $conf->{ostype}) {
2861         # other, wxp, w2k, w2k3, w2k8, wvista, win7, win8, l24, l26, solaris
2862
2863         if ($ost =~ m/^w/) { # windows
2864             $useLocaltime = 1 if !defined($conf->{localtime});
2865
2866             # use time drift fix when acpi is enabled
2867             if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
2868                 $tdf = 1 if !defined($conf->{tdf});
2869             }
2870         }
2871
2872         if ($ost eq 'win7' || $ost eq 'win8' || $ost eq 'w2k8' ||
2873             $ost eq 'wvista') {
2874             push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
2875             push @$cmd, '-no-hpet';
2876             if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
2877                 push @$cpuFlags , 'hv_spinlocks=0x1fff' if !$nokvm;
2878                 push @$cpuFlags , 'hv_vapic' if !$nokvm;
2879                 push @$cpuFlags , 'hv_time' if !$nokvm;
2880
2881             } else {
2882                 push @$cpuFlags , 'hv_spinlocks=0xffff' if !$nokvm;
2883             }
2884         }
2885
2886         if ($ost eq 'win7' || $ost eq 'win8') {
2887             push @$cpuFlags , 'hv_relaxed' if !$nokvm;
2888         }
2889     }
2890
2891     push @$rtcFlags, 'driftfix=slew' if $tdf;
2892
2893     if ($nokvm) {
2894         push @$machineFlags, 'accel=tcg';
2895     } else {
2896         die "No accelerator found!\n" if !$cpuinfo->{hvm};
2897     }
2898
2899     if ($machine_type) {
2900         push @$machineFlags, "type=${machine_type}";
2901     }
2902
2903     if ($conf->{startdate}) {
2904         push @$rtcFlags, "base=$conf->{startdate}";
2905     } elsif ($useLocaltime) {
2906         push @$rtcFlags, 'base=localtime';
2907     }
2908
2909     my $cpu = $nokvm ? "qemu64" : "kvm64";
2910     $cpu = $conf->{cpu} if $conf->{cpu};
2911
2912     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
2913
2914     push @$cpuFlags , '-x2apic'
2915         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
2916
2917     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
2918
2919     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
2920
2921     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
2922
2923         push @$cpuFlags , '+kvm_pv_unhalt' if !$nokvm;
2924         push @$cpuFlags , '+kvm_pv_eoi' if !$nokvm;
2925     }
2926
2927     push @$cpuFlags, 'enforce' if $cpu ne 'host' && !$nokvm;
2928
2929     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
2930
2931     push @$cmd, '-cpu', $cpu;
2932
2933     my $memory = $conf->{memory} || $defaults->{memory};
2934     my $static_memory = 0;
2935     my $dimm_memory = 0;
2936
2937     if ($hotplug_features->{memory}) {
2938         die "Numa need to be enabled for memory hotplug\n" if !$conf->{numa};
2939         die "Total memory is bigger than ${MAX_MEM}MB\n" if $memory > $MAX_MEM;
2940         $static_memory = $STATICMEM;
2941         die "minimum memory must be ${static_memory}MB\n" if($memory < $static_memory);
2942         $dimm_memory = $memory - $static_memory;
2943         push @$cmd, '-m', "size=${static_memory},slots=255,maxmem=${MAX_MEM}M";
2944
2945     } else {
2946
2947         $static_memory = $memory;
2948         push @$cmd, '-m', $static_memory;
2949     }
2950
2951     if ($conf->{numa}) {
2952
2953         my $numa_totalmemory = undef;
2954         for (my $i = 0; $i < $MAX_NUMA; $i++) {
2955             next if !$conf->{"numa$i"};
2956             my $numa = parse_numa($conf->{"numa$i"});
2957             next if !$numa;
2958             # memory
2959             die "missing numa node$i memory value\n" if !$numa->{memory};
2960             my $numa_memory = $numa->{memory};
2961             $numa_totalmemory += $numa_memory;
2962             my $numa_object = "memory-backend-ram,id=ram-node$i,size=${numa_memory}M";
2963
2964             # cpus
2965             my $cpus_start = $numa->{cpus}->{start};
2966             die "missing numa node$i cpus\n" if !defined($cpus_start);
2967             my $cpus_end = $numa->{cpus}->{end} if defined($numa->{cpus}->{end});
2968             my $cpus = $cpus_start;
2969             if (defined($cpus_end)) {
2970                 $cpus .= "-$cpus_end";
2971                 die "numa node$i :  cpu range $cpus is incorrect\n" if $cpus_end <= $cpus_start;
2972             }
2973
2974             # hostnodes
2975             my $hostnodes_start = $numa->{hostnodes}->{start};
2976             if (defined($hostnodes_start)) {
2977                 my $hostnodes_end = $numa->{hostnodes}->{end} if defined($numa->{hostnodes}->{end});
2978                 my $hostnodes = $hostnodes_start;
2979                 if (defined($hostnodes_end)) {
2980                     $hostnodes .= "-$hostnodes_end";
2981                     die "host node $hostnodes range is incorrect\n" if $hostnodes_end <= $hostnodes_start;
2982                 }
2983
2984                 my $hostnodes_end_range = defined($hostnodes_end) ? $hostnodes_end : $hostnodes_start;
2985                 for (my $i = $hostnodes_start; $i <= $hostnodes_end_range; $i++ ) {
2986                     die "host numa node$i don't exist\n" if ! -d "/sys/devices/system/node/node$i/";
2987                 }
2988
2989                 # policy
2990                 my $policy = $numa->{policy};
2991                 die "you need to define a policy for hostnode $hostnodes\n" if !$policy;
2992                 $numa_object .= ",host-nodes=$hostnodes,policy=$policy";
2993             }
2994
2995             push @$cmd, '-object', $numa_object;
2996             push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
2997         }
2998
2999         die "total memory for NUMA nodes must be equal to vm static memory\n"
3000             if $numa_totalmemory && $numa_totalmemory != $static_memory;
3001
3002         #if no custom tology, we split memory and cores across numa nodes
3003         if(!$numa_totalmemory) {
3004
3005             my $numa_memory = ($static_memory / $sockets) . "M";
3006
3007             for (my $i = 0; $i < $sockets; $i++)  {
3008
3009                 my $cpustart = ($cores * $i);
3010                 my $cpuend = ($cpustart + $cores - 1) if $cores && $cores > 1;
3011                 my $cpus = $cpustart;
3012                 $cpus .= "-$cpuend" if $cpuend;
3013
3014                 push @$cmd, '-object', "memory-backend-ram,size=$numa_memory,id=ram-node$i";
3015                 push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
3016             }
3017         }
3018     }
3019
3020     if ($hotplug_features->{memory}) {
3021         foreach_dimm($conf, $vmid, $memory, $sockets, sub {
3022             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3023             push @$cmd, "-object" , "memory-backend-ram,id=mem-$name,size=${dimm_size}M";
3024             push @$cmd, "-device", "pc-dimm,id=$name,memdev=mem-$name,node=$numanode";
3025
3026             #if dimm_memory is not aligned to dimm map
3027             if($current_size > $memory) {
3028                  $conf->{memory} = $current_size;
3029                  update_config_nolock($vmid, $conf, 1);
3030             }
3031         });
3032     }
3033
3034     push @$cmd, '-S' if $conf->{freeze};
3035
3036     # set keyboard layout
3037     my $kb = $conf->{keyboard} || $defaults->{keyboard};
3038     push @$cmd, '-k', $kb if $kb;
3039
3040     # enable sound
3041     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3042     #push @$cmd, '-soundhw', 'es1370';
3043     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3044
3045     if($conf->{agent}) {
3046         my $qgasocket = qmp_socket($vmid, 1);
3047         my $pciaddr = print_pci_addr("qga0", $bridges);
3048         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3049         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3050         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3051     }
3052
3053     my $spice_port;
3054
3055     if ($qxlnum) {
3056         if ($qxlnum > 1) {
3057             if ($conf->{ostype} && $conf->{ostype} =~ m/^w/){
3058                 for(my $i = 1; $i < $qxlnum; $i++){
3059                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3060                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3061                 }
3062             } else {
3063                 # assume other OS works like Linux
3064                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3065                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3066             }
3067         }
3068
3069         my $pciaddr = print_pci_addr("spice", $bridges);
3070
3071         my $nodename = PVE::INotify::nodename();
3072         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3073         $spice_port = PVE::Tools::next_spice_port($pfamily);
3074
3075         push @$devices, '-spice', "tls-port=${spice_port},addr=localhost,tls-ciphers=DES-CBC3-SHA,seamless-migration=on";
3076
3077         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3078         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3079         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3080     }
3081
3082     # enable balloon by default, unless explicitly disabled
3083     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3084         $pciaddr = print_pci_addr("balloon0", $bridges);
3085         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3086     }
3087
3088     if ($conf->{watchdog}) {
3089         my $wdopts = parse_watchdog($conf->{watchdog});
3090         $pciaddr = print_pci_addr("watchdog", $bridges);
3091         my $watchdog = $wdopts->{model} || 'i6300esb';
3092         push @$devices, '-device', "$watchdog$pciaddr";
3093         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3094     }
3095
3096     my $vollist = [];
3097     my $scsicontroller = {};
3098     my $ahcicontroller = {};
3099     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3100
3101     # Add iscsi initiator name if available
3102     if (my $initiator = get_initiator_name()) {
3103         push @$devices, '-iscsi', "initiator-name=$initiator";
3104     }
3105
3106     foreach_drive($conf, sub {
3107         my ($ds, $drive) = @_;
3108
3109         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3110             push @$vollist, $drive->{file};
3111         }
3112
3113         $use_virtio = 1 if $ds =~ m/^virtio/;
3114
3115         if (drive_is_cdrom ($drive)) {
3116             if ($bootindex_hash->{d}) {
3117                 $drive->{bootindex} = $bootindex_hash->{d};
3118                 $bootindex_hash->{d} += 1;
3119             }
3120         } else {
3121             if ($bootindex_hash->{c}) {
3122                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3123                 $bootindex_hash->{c} += 1;
3124             }
3125         }
3126
3127         if($drive->{interface} eq 'virtio'){
3128            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3129         }
3130
3131         if ($drive->{interface} eq 'scsi') {
3132
3133             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3134
3135             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3136             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3137
3138             my $iothread = '';
3139             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3140                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3141                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3142             }
3143
3144             my $queues = '';
3145             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3146                 $queues = ",num_queues=$drive->{queues}";
3147             }
3148
3149             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3150             $scsicontroller->{$controller}=1;
3151         }
3152
3153         if ($drive->{interface} eq 'sata') {
3154            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3155            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3156            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3157            $ahcicontroller->{$controller}=1;
3158         }
3159
3160         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3161         push @$devices, '-drive',$drive_cmd;
3162         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3163     });
3164
3165     for (my $i = 0; $i < $MAX_NETS; $i++) {
3166          next if !$conf->{"net$i"};
3167          my $d = parse_net($conf->{"net$i"});
3168          next if !$d;
3169
3170          $use_virtio = 1 if $d->{model} eq 'virtio';
3171
3172          if ($bootindex_hash->{n}) {
3173             $d->{bootindex} = $bootindex_hash->{n};
3174             $bootindex_hash->{n} += 1;
3175          }
3176
3177          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3178          push @$devices, '-netdev', $netdevfull;
3179
3180          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3181          push @$devices, '-device', $netdevicefull;
3182     }
3183
3184     if (!$q35) {
3185         # add pci bridges
3186         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3187            $bridges->{1} = 1;
3188            $bridges->{2} = 1;
3189         }
3190
3191         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3192
3193         while (my ($k, $v) = each %$bridges) {
3194             $pciaddr = print_pci_addr("pci.$k");
3195             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3196         }
3197     }
3198
3199     # add custom args
3200     if ($conf->{args}) {
3201         my $aa = PVE::Tools::split_args($conf->{args});
3202         push @$cmd, @$aa;
3203     }
3204
3205     push @$cmd, @$devices;
3206     push @$cmd, '-rtc', join(',', @$rtcFlags)
3207         if scalar(@$rtcFlags);
3208     push @$cmd, '-machine', join(',', @$machineFlags)
3209         if scalar(@$machineFlags);
3210     push @$cmd, '-global', join(',', @$globalFlags)
3211         if scalar(@$globalFlags);
3212
3213     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3214 }
3215
3216 sub vnc_socket {
3217     my ($vmid) = @_;
3218     return "${var_run_tmpdir}/$vmid.vnc";
3219 }
3220
3221 sub spice_port {
3222     my ($vmid) = @_;
3223
3224     my $res = vm_mon_cmd($vmid, 'query-spice');
3225
3226     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3227 }
3228
3229 sub qmp_socket {
3230     my ($vmid, $qga) = @_;
3231     my $sockettype = $qga ? 'qga' : 'qmp';
3232     return "${var_run_tmpdir}/$vmid.$sockettype";
3233 }
3234
3235 sub pidfile_name {
3236     my ($vmid) = @_;
3237     return "${var_run_tmpdir}/$vmid.pid";
3238 }
3239
3240 sub vm_devices_list {
3241     my ($vmid) = @_;
3242
3243     my $res = vm_mon_cmd($vmid, 'query-pci');
3244     my $devices = {};
3245     foreach my $pcibus (@$res) {
3246         foreach my $device (@{$pcibus->{devices}}) {
3247             next if !$device->{'qdev_id'};
3248             if ($device->{'pci_bridge'}) {
3249                 $devices->{$device->{'qdev_id'}} = 1;
3250                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3251                     next if !$bridge_device->{'qdev_id'};
3252                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3253                     $devices->{$device->{'qdev_id'}}++;
3254                 }
3255             } else {
3256                 $devices->{$device->{'qdev_id'}} = 1;
3257             }
3258         }
3259     }
3260
3261     my $resblock = vm_mon_cmd($vmid, 'query-block');
3262     foreach my $block (@$resblock) {
3263         if($block->{device} =~ m/^drive-(\S+)/){
3264                 $devices->{$1} = 1;
3265         }
3266     }
3267
3268     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3269     foreach my $mice (@$resmice) {
3270         if ($mice->{name} eq 'QEMU HID Tablet') {
3271             $devices->{tablet} = 1;
3272             last;
3273         }
3274     }
3275
3276     return $devices;
3277 }
3278
3279 sub vm_deviceplug {
3280     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3281
3282     my $q35 = machine_type_is_q35($conf);
3283
3284     my $devices_list = vm_devices_list($vmid);
3285     return 1 if defined($devices_list->{$deviceid});
3286
3287     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3288
3289     if ($deviceid eq 'tablet') {
3290
3291         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3292
3293     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3294
3295         qemu_iothread_add($vmid, $deviceid, $device);
3296
3297         qemu_driveadd($storecfg, $vmid, $device);
3298         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3299
3300         qemu_deviceadd($vmid, $devicefull);
3301         eval { qemu_deviceaddverify($vmid, $deviceid); };
3302         if (my $err = $@) {
3303             eval { qemu_drivedel($vmid, $deviceid); };
3304             warn $@ if $@;
3305             die $err;
3306         }
3307
3308     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3309
3310
3311         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3312         my $pciaddr = print_pci_addr($deviceid);
3313         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3314
3315         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3316
3317         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3318             qemu_iothread_add($vmid, $deviceid, $device);
3319             $devicefull .= ",iothread=iothread-$deviceid";
3320         }
3321
3322         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3323             $devicefull .= ",num_queues=$device->{queues}";
3324         }
3325
3326         qemu_deviceadd($vmid, $devicefull);
3327         qemu_deviceaddverify($vmid, $deviceid);
3328
3329     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3330
3331         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3332         qemu_driveadd($storecfg, $vmid, $device);
3333
3334         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3335         eval { qemu_deviceadd($vmid, $devicefull); };
3336         if (my $err = $@) {
3337             eval { qemu_drivedel($vmid, $deviceid); };
3338             warn $@ if $@;
3339             die $err;
3340         }
3341
3342     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3343
3344         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3345
3346         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
3347         my $use_old_bios_files = undef;
3348         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3349
3350         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3351         qemu_deviceadd($vmid, $netdevicefull);
3352         eval { qemu_deviceaddverify($vmid, $deviceid); };
3353         if (my $err = $@) {
3354             eval { qemu_netdevdel($vmid, $deviceid); };
3355             warn $@ if $@;
3356             die $err;
3357         }
3358
3359     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3360
3361         my $bridgeid = $2;
3362         my $pciaddr = print_pci_addr($deviceid);
3363         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3364
3365         qemu_deviceadd($vmid, $devicefull);
3366         qemu_deviceaddverify($vmid, $deviceid);
3367
3368     } else {
3369         die "can't hotplug device '$deviceid'\n";
3370     }
3371
3372     return 1;
3373 }
3374
3375 # fixme: this should raise exceptions on error!
3376 sub vm_deviceunplug {
3377     my ($vmid, $conf, $deviceid) = @_;
3378
3379     my $devices_list = vm_devices_list($vmid);
3380     return 1 if !defined($devices_list->{$deviceid});
3381
3382     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3383
3384     if ($deviceid eq 'tablet') {
3385
3386         qemu_devicedel($vmid, $deviceid);
3387
3388     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3389
3390         qemu_devicedel($vmid, $deviceid);
3391         qemu_devicedelverify($vmid, $deviceid);
3392         qemu_drivedel($vmid, $deviceid);
3393         qemu_iothread_del($conf, $vmid, $deviceid);
3394
3395     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3396
3397         qemu_devicedel($vmid, $deviceid);
3398         qemu_devicedelverify($vmid, $deviceid);
3399         qemu_iothread_del($conf, $vmid, $deviceid);
3400
3401     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3402
3403         #qemu 2.3 segfault on drive_del with virtioscsi + iothread
3404         my $device = parse_drive($deviceid, $conf->{$deviceid});
3405         die "virtioscsi with iothread is not hot-unplugglable currently" if $device->{iothread};
3406
3407         qemu_devicedel($vmid, $deviceid);
3408         qemu_drivedel($vmid, $deviceid);
3409         qemu_deletescsihw($conf, $vmid, $deviceid);
3410
3411     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3412
3413         qemu_devicedel($vmid, $deviceid);
3414         qemu_devicedelverify($vmid, $deviceid);
3415         qemu_netdevdel($vmid, $deviceid);
3416
3417     } else {
3418         die "can't unplug device '$deviceid'\n";
3419     }
3420
3421     return 1;
3422 }
3423
3424 sub qemu_deviceadd {
3425     my ($vmid, $devicefull) = @_;
3426
3427     $devicefull = "driver=".$devicefull;
3428     my %options =  split(/[=,]/, $devicefull);
3429
3430     vm_mon_cmd($vmid, "device_add" , %options);
3431 }
3432
3433 sub qemu_devicedel {
3434     my ($vmid, $deviceid) = @_;
3435
3436     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3437 }
3438
3439 sub qemu_iothread_add {
3440     my($vmid, $deviceid, $device) = @_;
3441
3442     if ($device->{iothread}) {
3443         my $iothreads = vm_iothreads_list($vmid);
3444         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3445     }
3446 }
3447
3448 sub qemu_iothread_del {
3449     my($conf, $vmid, $deviceid) = @_;
3450
3451     my $device = parse_drive($deviceid, $conf->{$deviceid});
3452     if ($device->{iothread}) {
3453         my $iothreads = vm_iothreads_list($vmid);
3454         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3455     }
3456 }
3457
3458 sub qemu_objectadd {
3459     my($vmid, $objectid, $qomtype) = @_;
3460
3461     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3462
3463     return 1;
3464 }
3465
3466 sub qemu_objectdel {
3467     my($vmid, $objectid) = @_;
3468
3469     vm_mon_cmd($vmid, "object-del", id => $objectid);
3470
3471     return 1;
3472 }
3473
3474 sub qemu_driveadd {
3475     my ($storecfg, $vmid, $device) = @_;
3476
3477     my $drive = print_drive_full($storecfg, $vmid, $device);
3478     $drive =~ s/\\/\\\\/g;
3479     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3480
3481     # If the command succeeds qemu prints: "OK"
3482     return 1 if $ret =~ m/OK/s;
3483
3484     die "adding drive failed: $ret\n";
3485 }
3486
3487 sub qemu_drivedel {
3488     my($vmid, $deviceid) = @_;
3489
3490     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3491     $ret =~ s/^\s+//;
3492
3493     return 1 if $ret eq "";
3494
3495     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3496     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3497
3498     die "deleting drive $deviceid failed : $ret\n";
3499 }
3500
3501 sub qemu_deviceaddverify {
3502     my ($vmid, $deviceid) = @_;
3503
3504     for (my $i = 0; $i <= 5; $i++) {
3505          my $devices_list = vm_devices_list($vmid);
3506          return 1 if defined($devices_list->{$deviceid});
3507          sleep 1;
3508     }
3509
3510     die "error on hotplug device '$deviceid'\n";
3511 }
3512
3513
3514 sub qemu_devicedelverify {
3515     my ($vmid, $deviceid) = @_;
3516
3517     # need to verify that the device is correctly removed as device_del
3518     # is async and empty return is not reliable
3519
3520     for (my $i = 0; $i <= 5; $i++) {
3521          my $devices_list = vm_devices_list($vmid);
3522          return 1 if !defined($devices_list->{$deviceid});
3523          sleep 1;
3524     }
3525
3526     die "error on hot-unplugging device '$deviceid'\n";
3527 }
3528
3529 sub qemu_findorcreatescsihw {
3530     my ($storecfg, $conf, $vmid, $device) = @_;
3531
3532     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3533
3534     my $scsihwid="$controller_prefix$controller";
3535     my $devices_list = vm_devices_list($vmid);
3536
3537     if(!defined($devices_list->{$scsihwid})) {
3538         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3539     }
3540
3541     return 1;
3542 }
3543
3544 sub qemu_deletescsihw {
3545     my ($conf, $vmid, $opt) = @_;
3546
3547     my $device = parse_drive($opt, $conf->{$opt});
3548
3549     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3550         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3551         return 1;
3552     }
3553
3554     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3555
3556     my $devices_list = vm_devices_list($vmid);
3557     foreach my $opt (keys %{$devices_list}) {
3558         if (PVE::QemuServer::valid_drivename($opt)) {
3559             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3560             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3561                 return 1;
3562             }
3563         }
3564     }
3565
3566     my $scsihwid="scsihw$controller";
3567
3568     vm_deviceunplug($vmid, $conf, $scsihwid);
3569
3570     return 1;
3571 }
3572
3573 sub qemu_add_pci_bridge {
3574     my ($storecfg, $conf, $vmid, $device) = @_;
3575
3576     my $bridges = {};
3577
3578     my $bridgeid;
3579
3580     print_pci_addr($device, $bridges);
3581
3582     while (my ($k, $v) = each %$bridges) {
3583         $bridgeid = $k;
3584     }
3585     return 1 if !defined($bridgeid) || $bridgeid < 1;
3586
3587     my $bridge = "pci.$bridgeid";
3588     my $devices_list = vm_devices_list($vmid);
3589
3590     if (!defined($devices_list->{$bridge})) {
3591         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3592     }
3593
3594     return 1;
3595 }
3596
3597 sub qemu_set_link_status {
3598     my ($vmid, $device, $up) = @_;
3599
3600     vm_mon_cmd($vmid, "set_link", name => $device,
3601                up => $up ? JSON::true : JSON::false);
3602 }
3603
3604 sub qemu_netdevadd {
3605     my ($vmid, $conf, $device, $deviceid) = @_;
3606
3607     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
3608     my %options =  split(/[=,]/, $netdev);
3609
3610     vm_mon_cmd($vmid, "netdev_add",  %options);
3611     return 1;
3612 }
3613
3614 sub qemu_netdevdel {
3615     my ($vmid, $deviceid) = @_;
3616
3617     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3618 }
3619
3620 sub qemu_cpu_hotplug {
3621     my ($vmid, $conf, $vcpus) = @_;
3622
3623     my $sockets = 1;
3624     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3625     $sockets = $conf->{sockets} if  $conf->{sockets};
3626     my $cores = $conf->{cores} || 1;
3627     my $maxcpus = $sockets * $cores;
3628
3629     $vcpus = $maxcpus if !$vcpus;
3630
3631     die "you can't add more vcpus than maxcpus\n"
3632         if $vcpus > $maxcpus;
3633
3634     my $currentvcpus = $conf->{vcpus} || $maxcpus;
3635     die "online cpu unplug is not yet possible\n"
3636         if $vcpus < $currentvcpus;
3637
3638     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3639     die "vcpus in running vm is different than configuration\n"
3640         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
3641
3642     for (my $i = $currentvcpus; $i < $vcpus; $i++) {
3643         vm_mon_cmd($vmid, "cpu-add", id => int($i));
3644     }
3645 }
3646
3647 sub qemu_memory_hotplug {
3648     my ($vmid, $conf, $defaults, $opt, $value) = @_;
3649
3650     return $value if !check_running($vmid);
3651
3652     my $memory = $conf->{memory} || $defaults->{memory};
3653     $value = $defaults->{memory} if !$value;
3654     return $value if $value == $memory;
3655
3656     my $static_memory = $STATICMEM;
3657     my $dimm_memory = $memory - $static_memory;
3658
3659     die "memory can't be lower than $static_memory MB" if $value < $static_memory;
3660     die "you cannot add more memory than $MAX_MEM MB!\n" if $memory > $MAX_MEM;
3661
3662
3663     my $sockets = 1;
3664     $sockets = $conf->{sockets} if $conf->{sockets};
3665
3666     if($value > $memory) {
3667
3668         foreach_dimm($conf, $vmid, $value, $sockets, sub {
3669             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3670
3671                 return if $current_size <= $conf->{memory};
3672
3673                 eval { vm_mon_cmd($vmid, "object-add", 'qom-type' => "memory-backend-ram", id => "mem-$name", props => { size => int($dimm_size*1024*1024) } ) };
3674                 if (my $err = $@) {
3675                     eval { qemu_objectdel($vmid, "mem-$name"); };
3676                     die $err;
3677                 }
3678
3679                 eval { vm_mon_cmd($vmid, "device_add", driver => "pc-dimm", id => "$name", memdev => "mem-$name", node => $numanode) };
3680                 if (my $err = $@) {
3681                     eval { qemu_objectdel($vmid, "mem-$name"); };
3682                     die $err;
3683                 }
3684                 #update conf after each succesful module hotplug
3685                 $conf->{memory} = $current_size;
3686                 update_config_nolock($vmid, $conf, 1);
3687         });
3688
3689     } else {
3690
3691         foreach_reverse_dimm($conf, $vmid, $value, $sockets, sub {
3692             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3693
3694                 return if $current_size >= $conf->{memory};
3695                 print "try to unplug memory dimm $name\n";
3696
3697                 my $retry = 0;
3698                 while (1) {
3699                     eval { qemu_devicedel($vmid, $name) };
3700                     sleep 3;
3701                     my $dimm_list = qemu_dimm_list($vmid);
3702                     last if !$dimm_list->{$name};
3703                     raise_param_exc({ $name => "error unplug memory module" }) if $retry > 5;
3704                     $retry++;
3705                 }
3706
3707                 #update conf after each succesful module unplug
3708                 $conf->{memory} = $current_size;
3709
3710                 eval { qemu_objectdel($vmid, "mem-$name"); };
3711                 update_config_nolock($vmid, $conf, 1);
3712         });
3713     }
3714 }
3715
3716 sub qemu_dimm_list {
3717     my ($vmid) = @_;
3718
3719     my $dimmarray = vm_mon_cmd_nocheck($vmid, "query-memory-devices");
3720     my $dimms = {};
3721
3722     foreach my $dimm (@$dimmarray) {
3723
3724         $dimms->{$dimm->{data}->{id}}->{id} = $dimm->{data}->{id};
3725         $dimms->{$dimm->{data}->{id}}->{node} = $dimm->{data}->{node};
3726         $dimms->{$dimm->{data}->{id}}->{addr} = $dimm->{data}->{addr};
3727         $dimms->{$dimm->{data}->{id}}->{size} = $dimm->{data}->{size};
3728         $dimms->{$dimm->{data}->{id}}->{slot} = $dimm->{data}->{slot};
3729     }
3730     return $dimms;
3731 }
3732
3733 sub qemu_block_set_io_throttle {
3734     my ($vmid, $deviceid, $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr) = @_;
3735
3736     return if !check_running($vmid) ;
3737
3738     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid, bps => int($bps), bps_rd => int($bps_rd), bps_wr => int($bps_wr), iops => int($iops), iops_rd => int($iops_rd), iops_wr => int($iops_wr));
3739
3740 }
3741
3742 # old code, only used to shutdown old VM after update
3743 sub __read_avail {
3744     my ($fh, $timeout) = @_;
3745
3746     my $sel = new IO::Select;
3747     $sel->add($fh);
3748
3749     my $res = '';
3750     my $buf;
3751
3752     my @ready;
3753     while (scalar (@ready = $sel->can_read($timeout))) {
3754         my $count;
3755         if ($count = $fh->sysread($buf, 8192)) {
3756             if ($buf =~ /^(.*)\(qemu\) $/s) {
3757                 $res .= $1;
3758                 last;
3759             } else {
3760                 $res .= $buf;
3761             }
3762         } else {
3763             if (!defined($count)) {
3764                 die "$!\n";
3765             }
3766             last;
3767         }
3768     }
3769
3770     die "monitor read timeout\n" if !scalar(@ready);
3771
3772     return $res;
3773 }
3774
3775 # old code, only used to shutdown old VM after update
3776 sub vm_monitor_command {
3777     my ($vmid, $cmdstr, $nocheck) = @_;
3778
3779     my $res;
3780
3781     eval {
3782         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
3783
3784         my $sname = "${var_run_tmpdir}/$vmid.mon";
3785
3786         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
3787             die "unable to connect to VM $vmid socket - $!\n";
3788
3789         my $timeout = 3;
3790
3791         # hack: migrate sometime blocks the monitor (when migrate_downtime
3792         # is set)
3793         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3794             $timeout = 60*60; # 1 hour
3795         }
3796
3797         # read banner;
3798         my $data = __read_avail($sock, $timeout);
3799
3800         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
3801             die "got unexpected qemu monitor banner\n";
3802         }
3803
3804         my $sel = new IO::Select;
3805         $sel->add($sock);
3806
3807         if (!scalar(my @ready = $sel->can_write($timeout))) {
3808             die "monitor write error - timeout";
3809         }
3810
3811         my $fullcmd = "$cmdstr\r";
3812
3813         # syslog('info', "VM $vmid monitor command: $cmdstr");
3814
3815         my $b;
3816         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
3817             die "monitor write error - $!";
3818         }
3819
3820         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
3821
3822         $timeout = 20;
3823
3824         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3825             $timeout = 60*60; # 1 hour
3826         } elsif ($cmdstr =~ m/^(eject|change)/) {
3827             $timeout = 60; # note: cdrom mount command is slow
3828         }
3829         if ($res = __read_avail($sock, $timeout)) {
3830
3831             my @lines = split("\r?\n", $res);
3832
3833             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
3834
3835             $res = join("\n", @lines);
3836             $res .= "\n";
3837         }
3838     };
3839
3840     my $err = $@;
3841
3842     if ($err) {
3843         syslog("err", "VM $vmid monitor command failed - $err");
3844         die $err;
3845     }
3846
3847     return $res;
3848 }
3849
3850 sub qemu_block_resize {
3851     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
3852
3853     my $running = check_running($vmid);
3854
3855     return if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
3856
3857     return if !$running;
3858
3859     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
3860
3861 }
3862
3863 sub qemu_volume_snapshot {
3864     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3865
3866     my $running = check_running($vmid);
3867
3868     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
3869         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
3870     } else {
3871         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
3872     }
3873 }
3874
3875 sub qemu_volume_snapshot_delete {
3876     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3877
3878     my $running = check_running($vmid);
3879
3880     return if !PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
3881
3882     return if !$running;
3883
3884     vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
3885 }
3886
3887 sub set_migration_caps {
3888     my ($vmid) = @_;
3889
3890     my $cap_ref = [];
3891
3892     my $enabled_cap = {
3893         "auto-converge" => 1,
3894         "xbzrle" => 1,
3895         "x-rdma-pin-all" => 0,
3896         "zero-blocks" => 0,
3897         "compress" => 0
3898     };
3899
3900     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
3901
3902     for my $supported_capability (@$supported_capabilities) {
3903         push @$cap_ref, {
3904             capability => $supported_capability->{capability},
3905             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
3906         };
3907     }
3908
3909     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
3910 }
3911
3912 my $fast_plug_option = {
3913     'lock' => 1,
3914     'name' => 1,
3915     'onboot' => 1,
3916     'shares' => 1,
3917     'startup' => 1,
3918     'description' => 1,
3919 };
3920
3921 # hotplug changes in [PENDING]
3922 # $selection hash can be used to only apply specified options, for
3923 # example: { cores => 1 } (only apply changed 'cores')
3924 # $errors ref is used to return error messages
3925 sub vmconfig_hotplug_pending {
3926     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
3927
3928     my $defaults = load_defaults();
3929
3930     # commit values which do not have any impact on running VM first
3931     # Note: those option cannot raise errors, we we do not care about
3932     # $selection and always apply them.
3933
3934     my $add_error = sub {
3935         my ($opt, $msg) = @_;
3936         $errors->{$opt} = "hotplug problem - $msg";
3937     };
3938
3939     my $changes = 0;
3940     foreach my $opt (keys %{$conf->{pending}}) { # add/change
3941         if ($fast_plug_option->{$opt}) {
3942             $conf->{$opt} = $conf->{pending}->{$opt};
3943             delete $conf->{pending}->{$opt};
3944             $changes = 1;
3945         }
3946     }
3947
3948     if ($changes) {
3949         update_config_nolock($vmid, $conf, 1);
3950         $conf = load_config($vmid); # update/reload
3951     }
3952
3953     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3954
3955     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
3956     while (my ($opt, $force) = each %$pending_delete_hash) {
3957         next if $selection && !$selection->{$opt};
3958         eval {
3959             if ($opt eq 'hotplug') {
3960                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
3961             } elsif ($opt eq 'tablet') {
3962                 die "skip\n" if !$hotplug_features->{usb};
3963                 if ($defaults->{tablet}) {
3964                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
3965                 } else {
3966                     vm_deviceunplug($vmid, $conf, $opt);
3967                 }
3968             } elsif ($opt eq 'vcpus') {
3969                 die "skip\n" if !$hotplug_features->{cpu};
3970                 qemu_cpu_hotplug($vmid, $conf, undef);
3971             } elsif ($opt eq 'balloon') {
3972                 # enable balloon device is not hotpluggable
3973                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
3974             } elsif ($fast_plug_option->{$opt}) {
3975                 # do nothing
3976             } elsif ($opt =~ m/^net(\d+)$/) {
3977                 die "skip\n" if !$hotplug_features->{network};
3978                 vm_deviceunplug($vmid, $conf, $opt);
3979             } elsif (valid_drivename($opt)) {
3980                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
3981                 vm_deviceunplug($vmid, $conf, $opt);
3982                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
3983             } elsif ($opt =~ m/^memory$/) {
3984                 die "skip\n" if !$hotplug_features->{memory};
3985                 qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
3986             } elsif ($opt eq 'cpuunits') {
3987                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
3988             } elsif ($opt eq 'cpulimit') {
3989                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
3990             } else {
3991                 die "skip\n";
3992             }
3993         };
3994         if (my $err = $@) {
3995             &$add_error($opt, $err) if $err ne "skip\n";
3996         } else {
3997             # save new config if hotplug was successful
3998             delete $conf->{$opt};
3999             vmconfig_undelete_pending_option($conf, $opt);
4000             update_config_nolock($vmid, $conf, 1);
4001             $conf = load_config($vmid); # update/reload
4002         }
4003     }
4004
4005     foreach my $opt (keys %{$conf->{pending}}) {
4006         next if $selection && !$selection->{$opt};
4007         my $value = $conf->{pending}->{$opt};
4008         eval {
4009             if ($opt eq 'hotplug') {
4010                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4011             } elsif ($opt eq 'tablet') {
4012                 die "skip\n" if !$hotplug_features->{usb};
4013                 if ($value == 1) {
4014                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4015                 } elsif ($value == 0) {
4016                     vm_deviceunplug($vmid, $conf, $opt);
4017                 }
4018             } elsif ($opt eq 'vcpus') {
4019                 die "skip\n" if !$hotplug_features->{cpu};
4020                 qemu_cpu_hotplug($vmid, $conf, $value);
4021             } elsif ($opt eq 'balloon') {
4022                 # enable/disable balloning device is not hotpluggable
4023                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4024                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4025                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4026
4027                 # allow manual ballooning if shares is set to zero
4028                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4029                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4030                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4031                 }
4032             } elsif ($opt =~ m/^net(\d+)$/) {
4033                 # some changes can be done without hotplug
4034                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4035                                     $vmid, $opt, $value);
4036             } elsif (valid_drivename($opt)) {
4037                 # some changes can be done without hotplug
4038                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4039                                      $vmid, $opt, $value, 1);
4040             } elsif ($opt =~ m/^memory$/) { #dimms
4041                 die "skip\n" if !$hotplug_features->{memory};
4042                 $value = qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4043             } elsif ($opt eq 'cpuunits') {
4044                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4045             } elsif ($opt eq 'cpulimit') {
4046                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4047                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4048             } else {
4049                 die "skip\n";  # skip non-hot-pluggable options
4050             }
4051         };
4052         if (my $err = $@) {
4053             &$add_error($opt, $err) if $err ne "skip\n";
4054         } else {
4055             # save new config if hotplug was successful
4056             $conf->{$opt} = $value;
4057             delete $conf->{pending}->{$opt};
4058             update_config_nolock($vmid, $conf, 1);
4059             $conf = load_config($vmid); # update/reload
4060         }
4061     }
4062 }
4063
4064 sub try_deallocate_drive {
4065     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4066
4067     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4068         my $volid = $drive->{file};
4069         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4070             my $sid = PVE::Storage::parse_volume_id($volid);
4071             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4072
4073             # check if the disk is really unused
4074             my $used_paths = PVE::QemuServer::get_used_paths($vmid, $storecfg, $conf, 1, $key);
4075             my $path = PVE::Storage::path($storecfg, $volid);
4076             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4077                    if $used_paths->{$path};
4078             PVE::Storage::vdisk_free($storecfg, $volid);
4079             return 1;
4080         } else {
4081             # If vm is not owner of this disk remove from config
4082             return 1;
4083         }
4084     }
4085
4086     return undef;
4087 }
4088
4089 sub vmconfig_delete_or_detach_drive {
4090     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4091
4092     my $drive = parse_drive($opt, $conf->{$opt});
4093
4094     my $rpcenv = PVE::RPCEnvironment::get();
4095     my $authuser = $rpcenv->get_user();
4096
4097     if ($force) {
4098         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4099         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4100     } else {
4101         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4102     }
4103 }
4104
4105 sub vmconfig_apply_pending {
4106     my ($vmid, $conf, $storecfg) = @_;
4107
4108     # cold plug
4109
4110     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4111     while (my ($opt, $force) = each %$pending_delete_hash) {
4112         die "internal error" if $opt =~ m/^unused/;
4113         $conf = load_config($vmid); # update/reload
4114         if (!defined($conf->{$opt})) {
4115             vmconfig_undelete_pending_option($conf, $opt);
4116             update_config_nolock($vmid, $conf, 1);
4117         } elsif (valid_drivename($opt)) {
4118             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4119             vmconfig_undelete_pending_option($conf, $opt);
4120             delete $conf->{$opt};
4121             update_config_nolock($vmid, $conf, 1);
4122         } else {
4123             vmconfig_undelete_pending_option($conf, $opt);
4124             delete $conf->{$opt};
4125             update_config_nolock($vmid, $conf, 1);
4126         }
4127     }
4128
4129     $conf = load_config($vmid); # update/reload
4130
4131     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4132         $conf = load_config($vmid); # update/reload
4133
4134         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4135             # skip if nothing changed
4136         } elsif (valid_drivename($opt)) {
4137             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4138                 if defined($conf->{$opt});
4139             $conf->{$opt} = $conf->{pending}->{$opt};
4140         } else {
4141             $conf->{$opt} = $conf->{pending}->{$opt};
4142         }
4143
4144         delete $conf->{pending}->{$opt};
4145         update_config_nolock($vmid, $conf, 1);
4146     }
4147 }
4148
4149 my $safe_num_ne = sub {
4150     my ($a, $b) = @_;
4151
4152     return 0 if !defined($a) && !defined($b);
4153     return 1 if !defined($a);
4154     return 1 if !defined($b);
4155
4156     return $a != $b;
4157 };
4158
4159 my $safe_string_ne = sub {
4160     my ($a, $b) = @_;
4161
4162     return 0 if !defined($a) && !defined($b);
4163     return 1 if !defined($a);
4164     return 1 if !defined($b);
4165
4166     return $a ne $b;
4167 };
4168
4169 sub vmconfig_update_net {
4170     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4171
4172     my $newnet = parse_net($value);
4173
4174     if ($conf->{$opt}) {
4175         my $oldnet = parse_net($conf->{$opt});
4176
4177         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4178             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4179             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4180             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4181
4182             # for non online change, we try to hot-unplug
4183             die "skip\n" if !$hotplug;
4184             vm_deviceunplug($vmid, $conf, $opt);
4185         } else {
4186
4187             die "internal error" if $opt !~ m/net(\d+)/;
4188             my $iface = "tap${vmid}i$1";
4189
4190             if (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4191                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4192             }
4193
4194             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4195                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4196                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4197                 PVE::Network::tap_unplug($iface);
4198                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall});
4199             }
4200
4201             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4202                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4203             }
4204
4205             return 1;
4206         }
4207     }
4208
4209     if ($hotplug) {
4210         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4211     } else {
4212         die "skip\n";
4213     }
4214 }
4215
4216 sub vmconfig_update_disk {
4217     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4218
4219     # fixme: do we need force?
4220
4221     my $drive = parse_drive($opt, $value);
4222
4223     if ($conf->{$opt}) {
4224
4225         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4226
4227             my $media = $drive->{media} || 'disk';
4228             my $oldmedia = $old_drive->{media} || 'disk';
4229             die "unable to change media type\n" if $media ne $oldmedia;
4230
4231             if (!drive_is_cdrom($old_drive)) {
4232
4233                 if ($drive->{file} ne $old_drive->{file}) {
4234
4235                     die "skip\n" if !$hotplug;
4236
4237                     # unplug and register as unused
4238                     vm_deviceunplug($vmid, $conf, $opt);
4239                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4240
4241                 } else {
4242                     # update existing disk
4243
4244                     # skip non hotpluggable value
4245                     if (&$safe_num_ne($drive->{discard}, $old_drive->{discard}) ||
4246                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4247                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4248                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4249                         die "skip\n";
4250                     }
4251
4252                     # apply throttle
4253                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4254                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4255                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4256                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4257                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4258                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4259                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4260                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4261                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4262                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4263                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4264                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max})) {
4265
4266                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4267                                                    ($drive->{mbps} || 0)*1024*1024,
4268                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4269                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4270                                                    $drive->{iops} || 0,
4271                                                    $drive->{iops_rd} || 0,
4272                                                    $drive->{iops_wr} || 0,
4273                                                    ($drive->{mbps_max} || 0)*1024*1024,
4274                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4275                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4276                                                    $drive->{iops_max} || 0,
4277                                                    $drive->{iops_rd_max} || 0,
4278                                                    $drive->{iops_wr_max} || 0);
4279
4280                     }
4281
4282                     return 1;
4283                 }
4284
4285             } else { # cdrom
4286
4287                 if ($drive->{file} eq 'none') {
4288                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4289                 } else {
4290                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4291                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4292                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4293                 }
4294
4295                 return 1;
4296             }
4297         }
4298     }
4299
4300     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4301     # hotplug new disks
4302     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4303 }
4304
4305 sub vm_start {
4306     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4307         $forcemachine, $spice_ticket) = @_;
4308
4309     lock_config($vmid, sub {
4310         my $conf = load_config($vmid, $migratedfrom);
4311
4312         die "you can't start a vm if it's a template\n" if is_template($conf);
4313
4314         check_lock($conf) if !$skiplock;
4315
4316         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4317
4318         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4319             vmconfig_apply_pending($vmid, $conf, $storecfg);
4320             $conf = load_config($vmid); # update/reload
4321         }
4322
4323         my $defaults = load_defaults();
4324
4325         # set environment variable useful inside network script
4326         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4327
4328         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4329
4330         my $migrate_port = 0;
4331         my $migrate_uri;
4332         if ($statefile) {
4333             if ($statefile eq 'tcp') {
4334                 my $localip = "localhost";
4335                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4336                 my $nodename = PVE::INotify::nodename();
4337                 if ($datacenterconf->{migration_unsecure}) {
4338                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4339                         $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4340                 }
4341                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4342                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4343                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4344                 push @$cmd, '-incoming', $migrate_uri;
4345                 push @$cmd, '-S';
4346             } else {
4347                 push @$cmd, '-loadstate', $statefile;
4348             }
4349         } elsif ($paused) {
4350             push @$cmd, '-S';
4351         }
4352
4353         # host pci devices
4354         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4355           my $d = parse_hostpci($conf->{"hostpci$i"});
4356           next if !$d;
4357           my $pcidevices = $d->{pciid};
4358           foreach my $pcidevice (@$pcidevices) {
4359                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4360
4361                 my $info = pci_device_info("0000:$pciid");
4362                 die "IOMMU not present\n" if !check_iommu_support();
4363                 die "no pci device info for device '$pciid'\n" if !$info;
4364                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4365                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4366           }
4367         }
4368
4369         PVE::Storage::activate_volumes($storecfg, $vollist);
4370
4371         eval  { run_command($cmd, timeout => $statefile ? undef : 30,
4372                     umask => 0077); };
4373         my $err = $@;
4374         die "start failed: $err" if $err;
4375
4376         print "migration listens on $migrate_uri\n" if $migrate_uri;
4377
4378         if ($statefile && $statefile ne 'tcp')  {
4379             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4380             warn $@ if $@;
4381         }
4382
4383         if ($migratedfrom) {
4384
4385             eval {
4386                 set_migration_caps($vmid);
4387             };
4388             warn $@ if $@;
4389
4390             if ($spice_port) {
4391                 print "spice listens on port $spice_port\n";
4392                 if ($spice_ticket) {
4393                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4394                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4395                 }
4396             }
4397
4398         } else {
4399
4400             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4401                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4402                     if $conf->{balloon};
4403             }
4404
4405             foreach my $opt (keys %$conf) {
4406                 next if $opt !~  m/^net\d+$/;
4407                 my $nicconf = parse_net($conf->{$opt});
4408                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4409             }
4410         }
4411
4412         vm_mon_cmd_nocheck($vmid, 'qom-set',
4413                     path => "machine/peripheral/balloon0",
4414                     property => "guest-stats-polling-interval",
4415                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4416
4417     });
4418 }
4419
4420 sub vm_mon_cmd {
4421     my ($vmid, $execute, %params) = @_;
4422
4423     my $cmd = { execute => $execute, arguments => \%params };
4424     vm_qmp_command($vmid, $cmd);
4425 }
4426
4427 sub vm_mon_cmd_nocheck {
4428     my ($vmid, $execute, %params) = @_;
4429
4430     my $cmd = { execute => $execute, arguments => \%params };
4431     vm_qmp_command($vmid, $cmd, 1);
4432 }
4433
4434 sub vm_qmp_command {
4435     my ($vmid, $cmd, $nocheck) = @_;
4436
4437     my $res;
4438
4439     my $timeout;
4440     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4441         $timeout = $cmd->{arguments}->{timeout};
4442         delete $cmd->{arguments}->{timeout};
4443     }
4444
4445     eval {
4446         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4447         my $sname = qmp_socket($vmid);
4448         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4449             my $qmpclient = PVE::QMPClient->new();
4450
4451             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4452         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
4453             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
4454                 if scalar(%{$cmd->{arguments}});
4455             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
4456         } else {
4457             die "unable to open monitor socket\n";
4458         }
4459     };
4460     if (my $err = $@) {
4461         syslog("err", "VM $vmid qmp command failed - $err");
4462         die $err;
4463     }
4464
4465     return $res;
4466 }
4467
4468 sub vm_human_monitor_command {
4469     my ($vmid, $cmdline) = @_;
4470
4471     my $res;
4472
4473     my $cmd = {
4474         execute => 'human-monitor-command',
4475         arguments => { 'command-line' => $cmdline},
4476     };
4477
4478     return vm_qmp_command($vmid, $cmd);
4479 }
4480
4481 sub vm_commandline {
4482     my ($storecfg, $vmid) = @_;
4483
4484     my $conf = load_config($vmid);
4485
4486     my $defaults = load_defaults();
4487
4488     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
4489
4490     return join(' ', @$cmd);
4491 }
4492
4493 sub vm_reset {
4494     my ($vmid, $skiplock) = @_;
4495
4496     lock_config($vmid, sub {
4497
4498         my $conf = load_config($vmid);
4499
4500         check_lock($conf) if !$skiplock;
4501
4502         vm_mon_cmd($vmid, "system_reset");
4503     });
4504 }
4505
4506 sub get_vm_volumes {
4507     my ($conf) = @_;
4508
4509     my $vollist = [];
4510     foreach_volid($conf, sub {
4511         my ($volid, $is_cdrom) = @_;
4512
4513         return if $volid =~ m|^/|;
4514
4515         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4516         return if !$sid;
4517
4518         push @$vollist, $volid;
4519     });
4520
4521     return $vollist;
4522 }
4523
4524 sub vm_stop_cleanup {
4525     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
4526
4527     eval {
4528
4529         if (!$keepActive) {
4530             my $vollist = get_vm_volumes($conf);
4531             PVE::Storage::deactivate_volumes($storecfg, $vollist);
4532         }
4533
4534         foreach my $ext (qw(mon qmp pid vnc qga)) {
4535             unlink "/var/run/qemu-server/${vmid}.$ext";
4536         }
4537
4538         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
4539     };
4540     warn $@ if $@; # avoid errors - just warn
4541 }
4542
4543 # Note: use $nockeck to skip tests if VM configuration file exists.
4544 # We need that when migration VMs to other nodes (files already moved)
4545 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
4546 sub vm_stop {
4547     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
4548
4549     $force = 1 if !defined($force) && !$shutdown;
4550
4551     if ($migratedfrom){
4552         my $pid = check_running($vmid, $nocheck, $migratedfrom);
4553         kill 15, $pid if $pid;
4554         my $conf = load_config($vmid, $migratedfrom);
4555         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
4556         return;
4557     }
4558
4559     lock_config($vmid, sub {
4560
4561         my $pid = check_running($vmid, $nocheck);
4562         return if !$pid;
4563
4564         my $conf;
4565         if (!$nocheck) {
4566             $conf = load_config($vmid);
4567             check_lock($conf) if !$skiplock;
4568             if (!defined($timeout) && $shutdown && $conf->{startup}) {
4569                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
4570                 $timeout = $opts->{down} if $opts->{down};
4571             }
4572         }
4573
4574         $timeout = 60 if !defined($timeout);
4575
4576         eval {
4577             if ($shutdown) {
4578                 if (defined($conf) && $conf->{agent}) {
4579                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
4580                 } else {
4581                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
4582                 }
4583             } else {
4584                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
4585             }
4586         };
4587         my $err = $@;
4588
4589         if (!$err) {
4590             my $count = 0;
4591             while (($count < $timeout) && check_running($vmid, $nocheck)) {
4592                 $count++;
4593                 sleep 1;
4594             }
4595
4596             if ($count >= $timeout) {
4597                 if ($force) {
4598                     warn "VM still running - terminating now with SIGTERM\n";
4599                     kill 15, $pid;
4600                 } else {
4601                     die "VM quit/powerdown failed - got timeout\n";
4602                 }
4603             } else {
4604                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4605                 return;
4606             }
4607         } else {
4608             if ($force) {
4609                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
4610                 kill 15, $pid;
4611             } else {
4612                 die "VM quit/powerdown failed\n";
4613             }
4614         }
4615
4616         # wait again
4617         $timeout = 10;
4618
4619         my $count = 0;
4620         while (($count < $timeout) && check_running($vmid, $nocheck)) {
4621             $count++;
4622             sleep 1;
4623         }
4624
4625         if ($count >= $timeout) {
4626             warn "VM still running - terminating now with SIGKILL\n";
4627             kill 9, $pid;
4628             sleep 1;
4629         }
4630
4631         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4632    });
4633 }
4634
4635 sub vm_suspend {
4636     my ($vmid, $skiplock) = @_;
4637
4638     lock_config($vmid, sub {
4639
4640         my $conf = load_config($vmid);
4641
4642         check_lock($conf) if !($skiplock || ($conf->{lock} && $conf->{lock} eq 'backup'));
4643
4644         vm_mon_cmd($vmid, "stop");
4645     });
4646 }
4647
4648 sub vm_resume {
4649     my ($vmid, $skiplock, $nocheck) = @_;
4650
4651     lock_config($vmid, sub {
4652
4653         if (!$nocheck) {
4654
4655             my $conf = load_config($vmid);
4656
4657             check_lock($conf) if !($skiplock || ($conf->{lock} && $conf->{lock} eq 'backup'));
4658
4659             vm_mon_cmd($vmid, "cont");
4660
4661         } else {
4662             vm_mon_cmd_nocheck($vmid, "cont");
4663         }
4664     });
4665 }
4666
4667 sub vm_sendkey {
4668     my ($vmid, $skiplock, $key) = @_;
4669
4670     lock_config($vmid, sub {
4671
4672         my $conf = load_config($vmid);
4673
4674         # there is no qmp command, so we use the human monitor command
4675         vm_human_monitor_command($vmid, "sendkey $key");
4676     });
4677 }
4678
4679 sub vm_destroy {
4680     my ($storecfg, $vmid, $skiplock) = @_;
4681
4682     lock_config($vmid, sub {
4683
4684         my $conf = load_config($vmid);
4685
4686         check_lock($conf) if !$skiplock;
4687
4688         if (!check_running($vmid)) {
4689             destroy_vm($storecfg, $vmid);
4690         } else {
4691             die "VM $vmid is running - destroy failed\n";
4692         }
4693     });
4694 }
4695
4696 # pci helpers
4697
4698 sub file_write {
4699     my ($filename, $buf) = @_;
4700
4701     my $fh = IO::File->new($filename, "w");
4702     return undef if !$fh;
4703
4704     my $res = print $fh $buf;
4705
4706     $fh->close();
4707
4708     return $res;
4709 }
4710
4711 sub pci_device_info {
4712     my ($name) = @_;
4713
4714     my $res;
4715
4716     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
4717     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
4718
4719     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
4720     return undef if !defined($irq) || $irq !~ m/^\d+$/;
4721
4722     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
4723     return undef if !defined($vendor) || $vendor !~ s/^0x//;
4724
4725     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
4726     return undef if !defined($product) || $product !~ s/^0x//;
4727
4728     $res = {
4729         name => $name,
4730         vendor => $vendor,
4731         product => $product,
4732         domain => $domain,
4733         bus => $bus,
4734         slot => $slot,
4735         func => $func,
4736         irq => $irq,
4737         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
4738     };
4739
4740     return $res;
4741 }
4742
4743 sub pci_dev_reset {
4744     my ($dev) = @_;
4745
4746     my $name = $dev->{name};
4747
4748     my $fn = "$pcisysfs/devices/$name/reset";
4749
4750     return file_write($fn, "1");
4751 }
4752
4753 sub pci_dev_bind_to_vfio {
4754     my ($dev) = @_;
4755
4756     my $name = $dev->{name};
4757
4758     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4759
4760     if (!-d $vfio_basedir) {
4761         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4762     }
4763     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4764
4765     my $testdir = "$vfio_basedir/$name";
4766     return 1 if -d $testdir;
4767
4768     my $data = "$dev->{vendor} $dev->{product}";
4769     return undef if !file_write("$vfio_basedir/new_id", $data);
4770
4771     my $fn = "$pcisysfs/devices/$name/driver/unbind";
4772     if (!file_write($fn, $name)) {
4773         return undef if -f $fn;
4774     }
4775
4776     $fn = "$vfio_basedir/bind";
4777     if (! -d $testdir) {
4778         return undef if !file_write($fn, $name);
4779     }
4780
4781     return -d $testdir;
4782 }
4783
4784 sub pci_dev_group_bind_to_vfio {
4785     my ($pciid) = @_;
4786
4787     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4788
4789     if (!-d $vfio_basedir) {
4790         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4791     }
4792     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4793
4794     # get IOMMU group devices
4795     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
4796       my @devs = grep /^0000:/, readdir($D);
4797     closedir($D);
4798
4799     foreach my $pciid (@devs) {
4800         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
4801
4802         # pci bridges, switches or root ports are not supported
4803         # they have a pci_bus subdirectory so skip them
4804         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
4805
4806         my $info = pci_device_info($1);
4807         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
4808     }
4809
4810     return 1;
4811 }
4812
4813 sub print_pci_addr {
4814     my ($id, $bridges) = @_;
4815
4816     my $res = '';
4817     my $devices = {
4818         piix3 => { bus => 0, addr => 1 },
4819         #addr2 : first videocard
4820         balloon0 => { bus => 0, addr => 3 },
4821         watchdog => { bus => 0, addr => 4 },
4822         scsihw0 => { bus => 0, addr => 5 },
4823         'pci.3' => { bus => 0, addr => 5 }, #can also be used for virtio-scsi-single bridge
4824         scsihw1 => { bus => 0, addr => 6 },
4825         ahci0 => { bus => 0, addr => 7 },
4826         qga0 => { bus => 0, addr => 8 },
4827         spice => { bus => 0, addr => 9 },
4828         virtio0 => { bus => 0, addr => 10 },
4829         virtio1 => { bus => 0, addr => 11 },
4830         virtio2 => { bus => 0, addr => 12 },
4831         virtio3 => { bus => 0, addr => 13 },
4832         virtio4 => { bus => 0, addr => 14 },
4833         virtio5 => { bus => 0, addr => 15 },
4834         hostpci0 => { bus => 0, addr => 16 },
4835         hostpci1 => { bus => 0, addr => 17 },
4836         net0 => { bus => 0, addr => 18 },
4837         net1 => { bus => 0, addr => 19 },
4838         net2 => { bus => 0, addr => 20 },
4839         net3 => { bus => 0, addr => 21 },
4840         net4 => { bus => 0, addr => 22 },
4841         net5 => { bus => 0, addr => 23 },
4842         vga1 => { bus => 0, addr => 24 },
4843         vga2 => { bus => 0, addr => 25 },
4844         vga3 => { bus => 0, addr => 26 },
4845         hostpci2 => { bus => 0, addr => 27 },
4846         hostpci3 => { bus => 0, addr => 28 },
4847         #addr29 : usb-host (pve-usb.cfg)
4848         'pci.1' => { bus => 0, addr => 30 },
4849         'pci.2' => { bus => 0, addr => 31 },
4850         'net6' => { bus => 1, addr => 1 },
4851         'net7' => { bus => 1, addr => 2 },
4852         'net8' => { bus => 1, addr => 3 },
4853         'net9' => { bus => 1, addr => 4 },
4854         'net10' => { bus => 1, addr => 5 },
4855         'net11' => { bus => 1, addr => 6 },
4856         'net12' => { bus => 1, addr => 7 },
4857         'net13' => { bus => 1, addr => 8 },
4858         'net14' => { bus => 1, addr => 9 },
4859         'net15' => { bus => 1, addr => 10 },
4860         'net16' => { bus => 1, addr => 11 },
4861         'net17' => { bus => 1, addr => 12 },
4862         'net18' => { bus => 1, addr => 13 },
4863         'net19' => { bus => 1, addr => 14 },
4864         'net20' => { bus => 1, addr => 15 },
4865         'net21' => { bus => 1, addr => 16 },
4866         'net22' => { bus => 1, addr => 17 },
4867         'net23' => { bus => 1, addr => 18 },
4868         'net24' => { bus => 1, addr => 19 },
4869         'net25' => { bus => 1, addr => 20 },
4870         'net26' => { bus => 1, addr => 21 },
4871         'net27' => { bus => 1, addr => 22 },
4872         'net28' => { bus => 1, addr => 23 },
4873         'net29' => { bus => 1, addr => 24 },
4874         'net30' => { bus => 1, addr => 25 },
4875         'net31' => { bus => 1, addr => 26 },
4876         'virtio6' => { bus => 2, addr => 1 },
4877         'virtio7' => { bus => 2, addr => 2 },
4878         'virtio8' => { bus => 2, addr => 3 },
4879         'virtio9' => { bus => 2, addr => 4 },
4880         'virtio10' => { bus => 2, addr => 5 },
4881         'virtio11' => { bus => 2, addr => 6 },
4882         'virtio12' => { bus => 2, addr => 7 },
4883         'virtio13' => { bus => 2, addr => 8 },
4884         'virtio14' => { bus => 2, addr => 9 },
4885         'virtio15' => { bus => 2, addr => 10 },
4886         'virtioscsi0' => { bus => 3, addr => 1 },
4887         'virtioscsi1' => { bus => 3, addr => 2 },
4888         'virtioscsi2' => { bus => 3, addr => 3 },
4889         'virtioscsi3' => { bus => 3, addr => 4 },
4890         'virtioscsi4' => { bus => 3, addr => 5 },
4891         'virtioscsi5' => { bus => 3, addr => 6 },
4892         'virtioscsi6' => { bus => 3, addr => 7 },
4893         'virtioscsi7' => { bus => 3, addr => 8 },
4894         'virtioscsi8' => { bus => 3, addr => 9 },
4895         'virtioscsi9' => { bus => 3, addr => 10 },
4896         'virtioscsi10' => { bus => 3, addr => 11 },
4897         'virtioscsi11' => { bus => 3, addr => 12 },
4898         'virtioscsi12' => { bus => 3, addr => 13 },
4899         'virtioscsi13' => { bus => 3, addr => 14 },
4900         'virtioscsi14' => { bus => 3, addr => 15 },
4901         'virtioscsi15' => { bus => 3, addr => 16 },
4902         'virtioscsi16' => { bus => 3, addr => 17 },
4903         'virtioscsi17' => { bus => 3, addr => 18 },
4904         'virtioscsi18' => { bus => 3, addr => 19 },
4905         'virtioscsi19' => { bus => 3, addr => 20 },
4906         'virtioscsi20' => { bus => 3, addr => 21 },
4907         'virtioscsi21' => { bus => 3, addr => 22 },
4908         'virtioscsi22' => { bus => 3, addr => 23 },
4909         'virtioscsi23' => { bus => 3, addr => 24 },
4910         'virtioscsi24' => { bus => 3, addr => 25 },
4911         'virtioscsi25' => { bus => 3, addr => 26 },
4912         'virtioscsi26' => { bus => 3, addr => 27 },
4913         'virtioscsi27' => { bus => 3, addr => 28 },
4914         'virtioscsi28' => { bus => 3, addr => 29 },
4915         'virtioscsi29' => { bus => 3, addr => 30 },
4916         'virtioscsi30' => { bus => 3, addr => 31 },
4917
4918     };
4919
4920     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
4921            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
4922            my $bus = $devices->{$id}->{bus};
4923            $res = ",bus=pci.$bus,addr=$addr";
4924            $bridges->{$bus} = 1 if $bridges;
4925     }
4926     return $res;
4927
4928 }
4929
4930 sub print_pcie_addr {
4931     my ($id) = @_;
4932
4933     my $res = '';
4934     my $devices = {
4935         hostpci0 => { bus => "ich9-pcie-port-1", addr => 0 },
4936         hostpci1 => { bus => "ich9-pcie-port-2", addr => 0 },
4937         hostpci2 => { bus => "ich9-pcie-port-3", addr => 0 },
4938         hostpci3 => { bus => "ich9-pcie-port-4", addr => 0 },
4939     };
4940
4941     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
4942            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
4943            my $bus = $devices->{$id}->{bus};
4944            $res = ",bus=$bus,addr=$addr";
4945     }
4946     return $res;
4947
4948 }
4949
4950 # vzdump restore implementaion
4951
4952 sub tar_archive_read_firstfile {
4953     my $archive = shift;
4954
4955     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
4956
4957     # try to detect archive type first
4958     my $pid = open (TMP, "tar tf '$archive'|") ||
4959         die "unable to open file '$archive'\n";
4960     my $firstfile = <TMP>;
4961     kill 15, $pid;
4962     close TMP;
4963
4964     die "ERROR: archive contaions no data\n" if !$firstfile;
4965     chomp $firstfile;
4966
4967     return $firstfile;
4968 }
4969
4970 sub tar_restore_cleanup {
4971     my ($storecfg, $statfile) = @_;
4972
4973     print STDERR "starting cleanup\n";
4974
4975     if (my $fd = IO::File->new($statfile, "r")) {
4976         while (defined(my $line = <$fd>)) {
4977             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
4978                 my $volid = $2;
4979                 eval {
4980                     if ($volid =~ m|^/|) {
4981                         unlink $volid || die 'unlink failed\n';
4982                     } else {
4983                         PVE::Storage::vdisk_free($storecfg, $volid);
4984                     }
4985                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
4986                 };
4987                 print STDERR "unable to cleanup '$volid' - $@" if $@;
4988             } else {
4989                 print STDERR "unable to parse line in statfile - $line";
4990             }
4991         }
4992         $fd->close();
4993     }
4994 }
4995
4996 sub restore_archive {
4997     my ($archive, $vmid, $user, $opts) = @_;
4998
4999     my $format = $opts->{format};
5000     my $comp;
5001
5002     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5003         $format = 'tar' if !$format;
5004         $comp = 'gzip';
5005     } elsif ($archive =~ m/\.tar$/) {
5006         $format = 'tar' if !$format;
5007     } elsif ($archive =~ m/.tar.lzo$/) {
5008         $format = 'tar' if !$format;
5009         $comp = 'lzop';
5010     } elsif ($archive =~ m/\.vma$/) {
5011         $format = 'vma' if !$format;
5012     } elsif ($archive =~ m/\.vma\.gz$/) {
5013         $format = 'vma' if !$format;
5014         $comp = 'gzip';
5015     } elsif ($archive =~ m/\.vma\.lzo$/) {
5016         $format = 'vma' if !$format;
5017         $comp = 'lzop';
5018     } else {
5019         $format = 'vma' if !$format; # default
5020     }
5021
5022     # try to detect archive format
5023     if ($format eq 'tar') {
5024         return restore_tar_archive($archive, $vmid, $user, $opts);
5025     } else {
5026         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5027     }
5028 }
5029
5030 sub restore_update_config_line {
5031     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5032
5033     return if $line =~ m/^\#qmdump\#/;
5034     return if $line =~ m/^\#vzdump\#/;
5035     return if $line =~ m/^lock:/;
5036     return if $line =~ m/^unused\d+:/;
5037     return if $line =~ m/^parent:/;
5038     return if $line =~ m/^template:/; # restored VM is never a template
5039
5040     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5041         # try to convert old 1.X settings
5042         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5043         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5044             my ($model, $macaddr) = split(/\=/, $devconfig);
5045             $macaddr = PVE::Tools::random_ether_addr() if !$macaddr || $unique;
5046             my $net = {
5047                 model => $model,
5048                 bridge => "vmbr$ind",
5049                 macaddr => $macaddr,
5050             };
5051             my $netstr = print_net($net);
5052
5053             print $outfd "net$cookie->{netcount}: $netstr\n";
5054             $cookie->{netcount}++;
5055         }
5056     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5057         my ($id, $netstr) = ($1, $2);
5058         my $net = parse_net($netstr);
5059         $net->{macaddr} = PVE::Tools::random_ether_addr() if $net->{macaddr};
5060         $netstr = print_net($net);
5061         print $outfd "$id: $netstr\n";
5062     } elsif ($line =~ m/^((ide|scsi|virtio|sata)\d+):\s*(\S+)\s*$/) {
5063         my $virtdev = $1;
5064         my $value = $3;
5065         if ($line =~ m/backup=no/) {
5066             print $outfd "#$line";
5067         } elsif ($virtdev && $map->{$virtdev}) {
5068             my $di = parse_drive($virtdev, $value);
5069             delete $di->{format}; # format can change on restore
5070             $di->{file} = $map->{$virtdev};
5071             $value = print_drive($vmid, $di);
5072             print $outfd "$virtdev: $value\n";
5073         } else {
5074             print $outfd $line;
5075         }
5076     } else {
5077         print $outfd $line;
5078     }
5079 }
5080
5081 sub scan_volids {
5082     my ($cfg, $vmid) = @_;
5083
5084     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5085
5086     my $volid_hash = {};
5087     foreach my $storeid (keys %$info) {
5088         foreach my $item (@{$info->{$storeid}}) {
5089             next if !($item->{volid} && $item->{size});
5090             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5091             $volid_hash->{$item->{volid}} = $item;
5092         }
5093     }
5094
5095     return $volid_hash;
5096 }
5097
5098 sub get_used_paths {
5099     my ($vmid, $storecfg, $conf, $scan_snapshots, $skip_drive) = @_;
5100
5101     my $used_path = {};
5102
5103     my $scan_config = sub {
5104         my ($cref, $snapname) = @_;
5105
5106         foreach my $key (keys %$cref) {
5107             my $value = $cref->{$key};
5108             if (valid_drivename($key)) {
5109                 next if $skip_drive && $key eq $skip_drive;
5110                 my $drive = parse_drive($key, $value);
5111                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5112                 if ($drive->{file} =~ m!^/!) {
5113                     $used_path->{$drive->{file}}++; # = 1;
5114                 } else {
5115                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5116                     next if !$storeid;
5117                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5118                     next if !$scfg;
5119                     my $path = PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5120                     $used_path->{$path}++; # = 1;
5121                 }
5122             }
5123         }
5124     };
5125
5126     &$scan_config($conf);
5127
5128     undef $skip_drive;
5129
5130     if ($scan_snapshots) {
5131         foreach my $snapname (keys %{$conf->{snapshots}}) {
5132             &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5133         }
5134     }
5135
5136     return $used_path;
5137 }
5138
5139 sub update_disksize {
5140     my ($vmid, $conf, $volid_hash) = @_;
5141
5142     my $changes;
5143
5144     my $used = {};
5145
5146     # Note: it is allowed to define multiple storages with same path (alias), so
5147     # we need to check both 'volid' and real 'path' (two different volid can point
5148     # to the same path).
5149
5150     my $usedpath = {};
5151
5152     # update size info
5153     foreach my $opt (keys %$conf) {
5154         if (valid_drivename($opt)) {
5155             my $drive = parse_drive($opt, $conf->{$opt});
5156             my $volid = $drive->{file};
5157             next if !$volid;
5158
5159             $used->{$volid} = 1;
5160             if ($volid_hash->{$volid} &&
5161                 (my $path = $volid_hash->{$volid}->{path})) {
5162                 $usedpath->{$path} = 1;
5163             }
5164
5165             next if drive_is_cdrom($drive);
5166             next if !$volid_hash->{$volid};
5167
5168             $drive->{size} = $volid_hash->{$volid}->{size};
5169             my $new = print_drive($vmid, $drive);
5170             if ($new ne $conf->{$opt}) {
5171                 $changes = 1;
5172                 $conf->{$opt} = $new;
5173             }
5174         }
5175     }
5176
5177     # remove 'unusedX' entry if volume is used
5178     foreach my $opt (keys %$conf) {
5179         next if $opt !~ m/^unused\d+$/;
5180         my $volid = $conf->{$opt};
5181         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5182         if ($used->{$volid} || ($path && $usedpath->{$path})) {
5183             $changes = 1;
5184             delete $conf->{$opt};
5185         }
5186     }
5187
5188     foreach my $volid (sort keys %$volid_hash) {
5189         next if $volid =~ m/vm-$vmid-state-/;
5190         next if $used->{$volid};
5191         my $path = $volid_hash->{$volid}->{path};
5192         next if !$path; # just to be sure
5193         next if $usedpath->{$path};
5194         $changes = 1;
5195         add_unused_volume($conf, $volid);
5196         $usedpath->{$path} = 1; # avoid to add more than once (aliases)
5197     }
5198
5199     return $changes;
5200 }
5201
5202 sub rescan {
5203     my ($vmid, $nolock) = @_;
5204
5205     my $cfg = PVE::Cluster::cfs_read_file("storage.cfg");
5206
5207     my $volid_hash = scan_volids($cfg, $vmid);
5208
5209     my $updatefn =  sub {
5210         my ($vmid) = @_;
5211
5212         my $conf = load_config($vmid);
5213
5214         check_lock($conf);
5215
5216         my $vm_volids = {};
5217         foreach my $volid (keys %$volid_hash) {
5218             my $info = $volid_hash->{$volid};
5219             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5220         }
5221
5222         my $changes = update_disksize($vmid, $conf, $vm_volids);
5223
5224         update_config_nolock($vmid, $conf, 1) if $changes;
5225     };
5226
5227     if (defined($vmid)) {
5228         if ($nolock) {
5229             &$updatefn($vmid);
5230         } else {
5231             lock_config($vmid, $updatefn, $vmid);
5232         }
5233     } else {
5234         my $vmlist = config_list();
5235         foreach my $vmid (keys %$vmlist) {
5236             if ($nolock) {
5237                 &$updatefn($vmid);
5238             } else {
5239                 lock_config($vmid, $updatefn, $vmid);
5240             }
5241         }
5242     }
5243 }
5244
5245 sub restore_vma_archive {
5246     my ($archive, $vmid, $user, $opts, $comp) = @_;
5247
5248     my $input = $archive eq '-' ? "<&STDIN" : undef;
5249     my $readfrom = $archive;
5250
5251     my $uncomp = '';
5252     if ($comp) {
5253         $readfrom = '-';
5254         my $qarchive = PVE::Tools::shellquote($archive);
5255         if ($comp eq 'gzip') {
5256             $uncomp = "zcat $qarchive|";
5257         } elsif ($comp eq 'lzop') {
5258             $uncomp = "lzop -d -c $qarchive|";
5259         } else {
5260             die "unknown compression method '$comp'\n";
5261         }
5262
5263     }
5264
5265     my $tmpdir = "/var/tmp/vzdumptmp$$";
5266     rmtree $tmpdir;
5267
5268     # disable interrupts (always do cleanups)
5269     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5270         warn "got interrupt - ignored\n";
5271     };
5272
5273     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5274     POSIX::mkfifo($mapfifo, 0600);
5275     my $fifofh;
5276
5277     my $openfifo = sub {
5278         open($fifofh, '>', $mapfifo) || die $!;
5279     };
5280
5281     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5282
5283     my $oldtimeout;
5284     my $timeout = 5;
5285
5286     my $devinfo = {};
5287
5288     my $rpcenv = PVE::RPCEnvironment::get();
5289
5290     my $conffile = config_file($vmid);
5291     my $tmpfn = "$conffile.$$.tmp";
5292
5293     # Note: $oldconf is undef if VM does not exists
5294     my $oldconf = PVE::Cluster::cfs_read_file(cfs_config_path($vmid));
5295
5296     my $print_devmap = sub {
5297         my $virtdev_hash = {};
5298
5299         my $cfgfn = "$tmpdir/qemu-server.conf";
5300
5301         # we can read the config - that is already extracted
5302         my $fh = IO::File->new($cfgfn, "r") ||
5303             "unable to read qemu-server.conf - $!\n";
5304
5305         while (defined(my $line = <$fh>)) {
5306             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5307                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5308                 die "archive does not contain data for drive '$virtdev'\n"
5309                     if !$devinfo->{$devname};
5310                 if (defined($opts->{storage})) {
5311                     $storeid = $opts->{storage} || 'local';
5312                 } elsif (!$storeid) {
5313                     $storeid = 'local';
5314                 }
5315                 $format = 'raw' if !$format;
5316                 $devinfo->{$devname}->{devname} = $devname;
5317                 $devinfo->{$devname}->{virtdev} = $virtdev;
5318                 $devinfo->{$devname}->{format} = $format;
5319                 $devinfo->{$devname}->{storeid} = $storeid;
5320
5321                 # check permission on storage
5322                 my $pool = $opts->{pool}; # todo: do we need that?
5323                 if ($user ne 'root@pam') {
5324                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5325                 }
5326
5327                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5328             }
5329         }
5330
5331         foreach my $devname (keys %$devinfo) {
5332             die "found no device mapping information for device '$devname'\n"
5333                 if !$devinfo->{$devname}->{virtdev};
5334         }
5335
5336         my $cfg = cfs_read_file('storage.cfg');
5337
5338         # create empty/temp config
5339         if ($oldconf) {
5340             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5341             foreach_drive($oldconf, sub {
5342                 my ($ds, $drive) = @_;
5343
5344                 return if drive_is_cdrom($drive);
5345
5346                 my $volid = $drive->{file};
5347
5348                 return if !$volid || $volid =~ m|^/|;
5349
5350                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5351                 return if !$path || !$owner || ($owner != $vmid);
5352
5353                 # Note: only delete disk we want to restore
5354                 # other volumes will become unused
5355                 if ($virtdev_hash->{$ds}) {
5356                     PVE::Storage::vdisk_free($cfg, $volid);
5357                 }
5358             });
5359         }
5360
5361         my $map = {};
5362         foreach my $virtdev (sort keys %$virtdev_hash) {
5363             my $d = $virtdev_hash->{$virtdev};
5364             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5365             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5366
5367             # test if requested format is supported
5368             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5369             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5370             $d->{format} = $defFormat if !$supported;
5371
5372             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5373                                                   $d->{format}, undef, $alloc_size);
5374             print STDERR "new volume ID is '$volid'\n";
5375             $d->{volid} = $volid;
5376             my $path = PVE::Storage::path($cfg, $volid);
5377
5378             PVE::Storage::activate_volumes($cfg,[$volid]);
5379
5380             my $write_zeros = 1;
5381             # fixme: what other storages types initialize volumes with zero?
5382             if ($scfg->{type} eq 'dir' || $scfg->{type} eq 'nfs' || $scfg->{type} eq 'glusterfs' ||
5383                 $scfg->{type} eq 'sheepdog' || $scfg->{type} eq 'rbd') {
5384                 $write_zeros = 0;
5385             }
5386
5387             print $fifofh "${write_zeros}:$d->{devname}=$path\n";
5388
5389             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5390             $map->{$virtdev} = $volid;
5391         }
5392
5393         $fh->seek(0, 0) || die "seek failed - $!\n";
5394
5395         my $outfd = new IO::File ($tmpfn, "w") ||
5396             die "unable to write config for VM $vmid\n";
5397
5398         my $cookie = { netcount => 0 };
5399         while (defined(my $line = <$fh>)) {
5400             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5401         }
5402
5403         $fh->close();
5404         $outfd->close();
5405     };
5406
5407     eval {
5408         # enable interrupts
5409         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5410             die "interrupted by signal\n";
5411         };
5412         local $SIG{ALRM} = sub { die "got timeout\n"; };
5413
5414         $oldtimeout = alarm($timeout);
5415
5416         my $parser = sub {
5417             my $line = shift;
5418
5419             print "$line\n";
5420
5421             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5422                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5423                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5424             } elsif ($line =~ m/^CTIME: /) {
5425                 # we correctly received the vma config, so we can disable
5426                 # the timeout now for disk allocation (set to 10 minutes, so
5427                 # that we always timeout if something goes wrong)
5428                 alarm(600);
5429                 &$print_devmap();
5430                 print $fifofh "done\n";
5431                 my $tmp = $oldtimeout || 0;
5432                 $oldtimeout = undef;
5433                 alarm($tmp);
5434                 close($fifofh);
5435             }
5436         };
5437
5438         print "restore vma archive: $cmd\n";
5439         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5440     };
5441     my $err = $@;
5442
5443     alarm($oldtimeout) if $oldtimeout;
5444
5445     my $vollist = [];
5446     foreach my $devname (keys %$devinfo) {
5447         my $volid = $devinfo->{$devname}->{volid};
5448         push @$vollist, $volid if $volid;
5449     }
5450
5451     my $cfg = cfs_read_file('storage.cfg');
5452     PVE::Storage::deactivate_volumes($cfg, $vollist);
5453
5454     unlink $mapfifo;
5455
5456     if ($err) {
5457         rmtree $tmpdir;
5458         unlink $tmpfn;
5459
5460         foreach my $devname (keys %$devinfo) {
5461             my $volid = $devinfo->{$devname}->{volid};
5462             next if !$volid;
5463             eval {
5464                 if ($volid =~ m|^/|) {
5465                     unlink $volid || die 'unlink failed\n';
5466                 } else {
5467                     PVE::Storage::vdisk_free($cfg, $volid);
5468                 }
5469                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5470             };
5471             print STDERR "unable to cleanup '$volid' - $@" if $@;
5472         }
5473         die $err;
5474     }
5475
5476     rmtree $tmpdir;
5477
5478     rename($tmpfn, $conffile) ||
5479         die "unable to commit configuration file '$conffile'\n";
5480
5481     PVE::Cluster::cfs_update(); # make sure we read new file
5482
5483     eval { rescan($vmid, 1); };
5484     warn $@ if $@;
5485 }
5486
5487 sub restore_tar_archive {
5488     my ($archive, $vmid, $user, $opts) = @_;
5489
5490     if ($archive ne '-') {
5491         my $firstfile = tar_archive_read_firstfile($archive);
5492         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5493             if $firstfile ne 'qemu-server.conf';
5494     }
5495
5496     my $storecfg = cfs_read_file('storage.cfg');
5497
5498     # destroy existing data - keep empty config
5499     my $vmcfgfn = config_file($vmid);
5500     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5501
5502     my $tocmd = "/usr/lib/qemu-server/qmextract";
5503
5504     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5505     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5506     $tocmd .= ' --prealloc' if $opts->{prealloc};
5507     $tocmd .= ' --info' if $opts->{info};
5508
5509     # tar option "xf" does not autodetect compression when read from STDIN,
5510     # so we pipe to zcat
5511     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5512         PVE::Tools::shellquote("--to-command=$tocmd");
5513
5514     my $tmpdir = "/var/tmp/vzdumptmp$$";
5515     mkpath $tmpdir;
5516
5517     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5518     local $ENV{VZDUMP_VMID} = $vmid;
5519     local $ENV{VZDUMP_USER} = $user;
5520
5521     my $conffile = config_file($vmid);
5522     my $tmpfn = "$conffile.$$.tmp";
5523
5524     # disable interrupts (always do cleanups)
5525     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5526         print STDERR "got interrupt - ignored\n";
5527     };
5528
5529     eval {
5530         # enable interrupts
5531         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5532             die "interrupted by signal\n";
5533         };
5534
5535         if ($archive eq '-') {
5536             print "extracting archive from STDIN\n";
5537             run_command($cmd, input => "<&STDIN");
5538         } else {
5539             print "extracting archive '$archive'\n";
5540             run_command($cmd);
5541         }
5542
5543         return if $opts->{info};
5544
5545         # read new mapping
5546         my $map = {};
5547         my $statfile = "$tmpdir/qmrestore.stat";
5548         if (my $fd = IO::File->new($statfile, "r")) {
5549             while (defined (my $line = <$fd>)) {
5550                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5551                     $map->{$1} = $2 if $1;
5552                 } else {
5553                     print STDERR "unable to parse line in statfile - $line\n";
5554                 }
5555             }
5556             $fd->close();
5557         }
5558
5559         my $confsrc = "$tmpdir/qemu-server.conf";
5560
5561         my $srcfd = new IO::File($confsrc, "r") ||
5562             die "unable to open file '$confsrc'\n";
5563
5564         my $outfd = new IO::File ($tmpfn, "w") ||
5565             die "unable to write config for VM $vmid\n";
5566
5567         my $cookie = { netcount => 0 };
5568         while (defined (my $line = <$srcfd>)) {
5569             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5570         }
5571
5572         $srcfd->close();
5573         $outfd->close();
5574     };
5575     my $err = $@;
5576
5577     if ($err) {
5578
5579         unlink $tmpfn;
5580
5581         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
5582
5583         die $err;
5584     }
5585
5586     rmtree $tmpdir;
5587
5588     rename $tmpfn, $conffile ||
5589         die "unable to commit configuration file '$conffile'\n";
5590
5591     PVE::Cluster::cfs_update(); # make sure we read new file
5592
5593     eval { rescan($vmid, 1); };
5594     warn $@ if $@;
5595 };
5596
5597
5598 # Internal snapshots
5599
5600 # NOTE: Snapshot create/delete involves several non-atomic
5601 # action, and can take a long time.
5602 # So we try to avoid locking the file and use 'lock' variable
5603 # inside the config file instead.
5604
5605 my $snapshot_copy_config = sub {
5606     my ($source, $dest) = @_;
5607
5608     foreach my $k (keys %$source) {
5609         next if $k eq 'snapshots';
5610         next if $k eq 'snapstate';
5611         next if $k eq 'snaptime';
5612         next if $k eq 'vmstate';
5613         next if $k eq 'lock';
5614         next if $k eq 'digest';
5615         next if $k eq 'description';
5616         next if $k =~ m/^unused\d+$/;
5617
5618         $dest->{$k} = $source->{$k};
5619     }
5620 };
5621
5622 my $snapshot_apply_config = sub {
5623     my ($conf, $snap) = @_;
5624
5625     # copy snapshot list
5626     my $newconf = {
5627         snapshots => $conf->{snapshots},
5628     };
5629
5630     # keep description and list of unused disks
5631     foreach my $k (keys %$conf) {
5632         next if !($k =~ m/^unused\d+$/ || $k eq 'description');
5633         $newconf->{$k} = $conf->{$k};
5634     }
5635
5636     &$snapshot_copy_config($snap, $newconf);
5637
5638     return $newconf;
5639 };
5640
5641 sub foreach_writable_storage {
5642     my ($conf, $func) = @_;
5643
5644     my $sidhash = {};
5645
5646     foreach my $ds (keys %$conf) {
5647         next if !valid_drivename($ds);
5648
5649         my $drive = parse_drive($ds, $conf->{$ds});
5650         next if !$drive;
5651         next if drive_is_cdrom($drive);
5652
5653         my $volid = $drive->{file};
5654
5655         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5656         $sidhash->{$sid} = $sid if $sid;
5657     }
5658
5659     foreach my $sid (sort keys %$sidhash) {
5660         &$func($sid);
5661     }
5662 }
5663
5664 my $alloc_vmstate_volid = sub {
5665     my ($storecfg, $vmid, $conf, $snapname) = @_;
5666
5667     # Note: we try to be smart when selecting a $target storage
5668
5669     my $target;
5670
5671     # search shared storage first
5672     foreach_writable_storage($conf, sub {
5673         my ($sid) = @_;
5674         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
5675         return if !$scfg->{shared};
5676
5677         $target = $sid if !$target || $scfg->{path}; # prefer file based storage
5678     });
5679
5680     if (!$target) {
5681         # now search local storage
5682         foreach_writable_storage($conf, sub {
5683             my ($sid) = @_;
5684             my $scfg = PVE::Storage::storage_config($storecfg, $sid);
5685             return if $scfg->{shared};
5686
5687             $target = $sid if !$target || $scfg->{path}; # prefer file based storage;
5688         });
5689     }
5690
5691     $target = 'local' if !$target;
5692
5693     my $driver_state_size = 500; # assume 32MB is enough to safe all driver state;
5694     # we abort live save after $conf->{memory}, so we need at max twice that space
5695     my $size = $conf->{memory}*2 + $driver_state_size;
5696
5697     my $name = "vm-$vmid-state-$snapname";
5698     my $scfg = PVE::Storage::storage_config($storecfg, $target);
5699     $name .= ".raw" if $scfg->{path}; # add filename extension for file base storage
5700     my $volid = PVE::Storage::vdisk_alloc($storecfg, $target, $vmid, 'raw', $name, $size*1024);
5701
5702     return $volid;
5703 };
5704
5705 my $snapshot_prepare = sub {
5706     my ($vmid, $snapname, $save_vmstate, $comment) = @_;
5707
5708     my $snap;
5709
5710     my $updatefn =  sub {
5711
5712         my $conf = load_config($vmid);
5713
5714         die "you can't take a snapshot if it's a template\n"
5715             if is_template($conf);
5716
5717         check_lock($conf);
5718
5719         $conf->{lock} = 'snapshot';
5720
5721         die "snapshot name '$snapname' already used\n"
5722             if defined($conf->{snapshots}->{$snapname});
5723
5724         my $storecfg = PVE::Storage::config();
5725         die "snapshot feature is not available" if !has_feature('snapshot', $conf, $storecfg);
5726
5727         $snap = $conf->{snapshots}->{$snapname} = {};
5728
5729         if ($save_vmstate && check_running($vmid)) {
5730             $snap->{vmstate} = &$alloc_vmstate_volid($storecfg, $vmid, $conf, $snapname);
5731         }
5732
5733         &$snapshot_copy_config($conf, $snap);
5734
5735         $snap->{snapstate} = "prepare";
5736         $snap->{snaptime} = time();
5737         $snap->{description} = $comment if $comment;
5738
5739         # always overwrite machine if we save vmstate. This makes sure we
5740         # can restore it later using correct machine type
5741         $snap->{machine} = get_current_qemu_machine($vmid) if $snap->{vmstate};
5742
5743         update_config_nolock($vmid, $conf, 1);
5744     };
5745
5746     lock_config($vmid, $updatefn);
5747
5748     return $snap;
5749 };
5750
5751 my $snapshot_commit = sub {
5752     my ($vmid, $snapname) = @_;
5753
5754     my $updatefn = sub {
5755
5756         my $conf = load_config($vmid);
5757
5758         die "missing snapshot lock\n"
5759             if !($conf->{lock} && $conf->{lock} eq 'snapshot');
5760
5761         my $has_machine_config = defined($conf->{machine});
5762
5763         my $snap = $conf->{snapshots}->{$snapname};
5764
5765         die "snapshot '$snapname' does not exist\n" if !defined($snap);
5766
5767         die "wrong snapshot state\n"
5768             if !($snap->{snapstate} && $snap->{snapstate} eq "prepare");
5769
5770         delete $snap->{snapstate};
5771         delete $conf->{lock};
5772
5773         my $newconf = &$snapshot_apply_config($conf, $snap);
5774
5775         delete $newconf->{machine} if !$has_machine_config;
5776
5777         $newconf->{parent} = $snapname;
5778
5779         update_config_nolock($vmid, $newconf, 1);
5780     };
5781
5782     lock_config($vmid, $updatefn);
5783 };
5784
5785 sub snapshot_rollback {
5786     my ($vmid, $snapname) = @_;
5787
5788     my $prepare = 1;
5789
5790     my $storecfg = PVE::Storage::config();
5791
5792     my $conf = load_config($vmid);
5793
5794     my $get_snapshot_config = sub {
5795
5796         die "you can't rollback if vm is a template\n" if is_template($conf);
5797
5798         my $res = $conf->{snapshots}->{$snapname};
5799
5800         die "snapshot '$snapname' does not exist\n" if !defined($res);
5801
5802         return $res;
5803     };
5804
5805     my $snap = &$get_snapshot_config();
5806
5807     foreach_drive($snap, sub {
5808         my ($ds, $drive) = @_;
5809
5810         return if drive_is_cdrom($drive);
5811
5812         my $volid = $drive->{file};
5813
5814         PVE::Storage::volume_rollback_is_possible($storecfg, $volid, $snapname);
5815     });
5816
5817     my $updatefn = sub {
5818
5819         $conf = load_config($vmid);
5820
5821         $snap = &$get_snapshot_config();
5822
5823         die "unable to rollback to incomplete snapshot (snapstate = $snap->{snapstate})\n"
5824             if $snap->{snapstate};
5825
5826         if ($prepare) {
5827             check_lock($conf);
5828             vm_stop($storecfg, $vmid, undef, undef, 5, undef, undef);
5829         }
5830
5831         die "unable to rollback vm $vmid: vm is running\n"
5832             if check_running($vmid);
5833
5834         if ($prepare) {
5835             $conf->{lock} = 'rollback';
5836         } else {
5837             die "got wrong lock\n" if !($conf->{lock} && $conf->{lock} eq 'rollback');
5838             delete $conf->{lock};
5839         }
5840
5841         my $forcemachine;
5842
5843         if (!$prepare) {
5844             my $has_machine_config = defined($conf->{machine});
5845
5846             # copy snapshot config to current config
5847             $conf = &$snapshot_apply_config($conf, $snap);
5848             $conf->{parent} = $snapname;
5849
5850             # Note: old code did not store 'machine', so we try to be smart
5851             # and guess the snapshot was generated with kvm 1.4 (pc-i440fx-1.4).
5852             $forcemachine = $conf->{machine} || 'pc-i440fx-1.4';
5853             # we remove the 'machine' configuration if not explicitly specified
5854             # in the original config.
5855             delete $conf->{machine} if $snap->{vmstate} && !$has_machine_config;
5856         }
5857
5858         update_config_nolock($vmid, $conf, 1);
5859
5860         if (!$prepare && $snap->{vmstate}) {
5861             my $statefile = PVE::Storage::path($storecfg, $snap->{vmstate});
5862             vm_start($storecfg, $vmid, $statefile, undef, undef, undef, $forcemachine);
5863         }
5864     };
5865
5866     lock_config($vmid, $updatefn);
5867
5868     foreach_drive($snap, sub {
5869         my ($ds, $drive) = @_;
5870
5871         return if drive_is_cdrom($drive);
5872
5873         my $volid = $drive->{file};
5874         my $device = "drive-$ds";
5875
5876         PVE::Storage::volume_snapshot_rollback($storecfg, $volid, $snapname);
5877     });
5878
5879     $prepare = 0;
5880     lock_config($vmid, $updatefn);
5881 }
5882
5883 my $savevm_wait = sub {
5884     my ($vmid) = @_;
5885
5886     for(;;) {
5887         my $stat = vm_mon_cmd_nocheck($vmid, "query-savevm");
5888         if (!$stat->{status}) {
5889             die "savevm not active\n";
5890         } elsif ($stat->{status} eq 'active') {
5891             sleep(1);
5892             next;
5893         } elsif ($stat->{status} eq 'completed') {
5894             last;
5895         } else {
5896             die "query-savevm returned status '$stat->{status}'\n";
5897         }
5898     }
5899 };
5900
5901 sub do_snapshots_with_qemu {
5902     my ($storecfg, $volid) = @_;
5903
5904     my $storage_name = PVE::Storage::parse_volume_id($volid);
5905
5906     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
5907         && !$storecfg->{ids}->{$storage_name}->{krbd}){
5908         return 1;
5909     }
5910
5911     if ($volid =~ m/\.(qcow2|qed)$/){
5912         return 1;
5913     }
5914
5915     return undef;
5916 }
5917
5918 sub snapshot_create {
5919     my ($vmid, $snapname, $save_vmstate, $comment) = @_;
5920
5921     my $snap = &$snapshot_prepare($vmid, $snapname, $save_vmstate, $comment);
5922
5923     $save_vmstate = 0 if !$snap->{vmstate}; # vm is not running
5924
5925     my $config = load_config($vmid);
5926
5927     my $running = check_running($vmid);
5928
5929     my $freezefs = $running && $config->{agent};
5930     $freezefs = 0 if $snap->{vmstate}; # not needed if we save RAM
5931
5932     my $drivehash = {};
5933
5934     if ($freezefs) {
5935         eval { vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
5936         warn "guest-fsfreeze-freeze problems - $@" if $@;
5937     }
5938
5939     eval {
5940         # create internal snapshots of all drives
5941
5942         my $storecfg = PVE::Storage::config();
5943
5944         if ($running) {
5945             if ($snap->{vmstate}) {
5946                 my $path = PVE::Storage::path($storecfg, $snap->{vmstate});
5947                 vm_mon_cmd($vmid, "savevm-start", statefile => $path);
5948                 &$savevm_wait($vmid);
5949             } else {
5950                 vm_mon_cmd($vmid, "savevm-start");
5951             }
5952         };
5953
5954         foreach_drive($snap, sub {
5955             my ($ds, $drive) = @_;
5956
5957             return if drive_is_cdrom($drive);
5958
5959             my $volid = $drive->{file};
5960             my $device = "drive-$ds";
5961
5962             qemu_volume_snapshot($vmid, $device, $storecfg, $volid, $snapname);
5963             $drivehash->{$ds} = 1;
5964        });
5965     };
5966     my $err = $@;
5967
5968     if ($running) {
5969         eval { vm_mon_cmd($vmid, "savevm-end")  };
5970         warn $@ if $@;
5971
5972         if ($freezefs) {
5973             eval { vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
5974             warn "guest-fsfreeze-thaw problems - $@" if $@;
5975         }
5976
5977         # savevm-end is async, we need to wait
5978         for (;;) {
5979             my $stat = vm_mon_cmd_nocheck($vmid, "query-savevm");
5980             if (!$stat->{bytes}) {
5981                 last;
5982             } else {
5983                 print "savevm not yet finished\n";
5984                 sleep(1);
5985                 next;
5986             }
5987         }
5988     }
5989
5990     if ($err) {
5991         warn "snapshot create failed: starting cleanup\n";
5992         eval { snapshot_delete($vmid, $snapname, 0, $drivehash); };
5993         warn $@ if $@;
5994         die $err;
5995     }
5996
5997     &$snapshot_commit($vmid, $snapname);
5998 }
5999
6000 # Note: $drivehash is only set when called from snapshot_create.
6001 sub snapshot_delete {
6002     my ($vmid, $snapname, $force, $drivehash) = @_;
6003
6004     my $prepare = 1;
6005
6006     my $snap;
6007     my $unused = [];
6008
6009     my $unlink_parent = sub {
6010         my ($confref, $new_parent) = @_;
6011
6012         if ($confref->{parent} && $confref->{parent} eq $snapname) {
6013             if ($new_parent) {
6014                 $confref->{parent} = $new_parent;
6015             } else {
6016                 delete $confref->{parent};
6017             }
6018         }
6019     };
6020
6021     my $updatefn =  sub {
6022         my ($remove_drive) = @_;
6023
6024         my $conf = load_config($vmid);
6025
6026         if (!$drivehash) {
6027             check_lock($conf);
6028             die "you can't delete a snapshot if vm is a template\n"
6029                 if is_template($conf);
6030         }
6031
6032         $snap = $conf->{snapshots}->{$snapname};
6033
6034         die "snapshot '$snapname' does not exist\n" if !defined($snap);
6035
6036         # remove parent refs
6037         if (!$prepare) {
6038             &$unlink_parent($conf, $snap->{parent});
6039             foreach my $sn (keys %{$conf->{snapshots}}) {
6040                 next if $sn eq $snapname;
6041                 &$unlink_parent($conf->{snapshots}->{$sn}, $snap->{parent});
6042             }
6043         }
6044
6045         if ($remove_drive) {
6046             if ($remove_drive eq 'vmstate') {
6047                 delete $snap->{$remove_drive};
6048             } else {
6049                 my $drive = parse_drive($remove_drive, $snap->{$remove_drive});
6050                 my $volid = $drive->{file};
6051                 delete $snap->{$remove_drive};
6052                 add_unused_volume($conf, $volid);
6053             }
6054         }
6055
6056         if ($prepare) {
6057             $snap->{snapstate} = 'delete';
6058         } else {
6059             delete $conf->{snapshots}->{$snapname};
6060             delete $conf->{lock} if $drivehash;
6061             foreach my $volid (@$unused) {
6062                 add_unused_volume($conf, $volid);
6063             }
6064         }
6065
6066         update_config_nolock($vmid, $conf, 1);
6067     };
6068
6069     lock_config($vmid, $updatefn);
6070
6071     # now remove vmstate file
6072
6073     my $storecfg = PVE::Storage::config();
6074
6075     if ($snap->{vmstate}) {
6076         eval {  PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
6077         if (my $err = $@) {
6078             die $err if !$force;
6079             warn $err;
6080         }
6081         # save changes (remove vmstate from snapshot)
6082         lock_config($vmid, $updatefn, 'vmstate') if !$force;
6083     };
6084
6085     # now remove all internal snapshots
6086     foreach_drive($snap, sub {
6087         my ($ds, $drive) = @_;
6088
6089         return if drive_is_cdrom($drive);
6090
6091         my $volid = $drive->{file};
6092         my $device = "drive-$ds";
6093
6094         if (!$drivehash || $drivehash->{$ds}) {
6095             eval { qemu_volume_snapshot_delete($vmid, $device, $storecfg, $volid, $snapname); };
6096             if (my $err = $@) {
6097                 die $err if !$force;
6098                 warn $err;
6099             }
6100         }
6101
6102         # save changes (remove drive fron snapshot)
6103         lock_config($vmid, $updatefn, $ds) if !$force;
6104         push @$unused, $volid;
6105     });
6106
6107     # now cleanup config
6108     $prepare = 0;
6109     lock_config($vmid, $updatefn);
6110 }
6111
6112 sub has_feature {
6113     my ($feature, $conf, $storecfg, $snapname, $running) = @_;
6114
6115     my $err;
6116     foreach_drive($conf, sub {
6117         my ($ds, $drive) = @_;
6118
6119         return if drive_is_cdrom($drive);
6120         my $volid = $drive->{file};
6121         $err = 1 if !PVE::Storage::volume_has_feature($storecfg, $feature, $volid, $snapname, $running);
6122     });
6123
6124     return $err ? 0 : 1;
6125 }
6126
6127 sub template_create {
6128     my ($vmid, $conf, $disk) = @_;
6129
6130     my $storecfg = PVE::Storage::config();
6131
6132     foreach_drive($conf, sub {
6133         my ($ds, $drive) = @_;
6134
6135         return if drive_is_cdrom($drive);
6136         return if $disk && $ds ne $disk;
6137
6138         my $volid = $drive->{file};
6139         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6140
6141         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6142         $drive->{file} = $voliddst;
6143         $conf->{$ds} = print_drive($vmid, $drive);
6144         update_config_nolock($vmid, $conf, 1);
6145     });
6146 }
6147
6148 sub is_template {
6149     my ($conf) = @_;
6150
6151     return 1 if defined $conf->{template} && $conf->{template} == 1;
6152 }
6153
6154 sub qemu_img_convert {
6155     my ($src_volid, $dst_volid, $size, $snapname) = @_;
6156
6157     my $storecfg = PVE::Storage::config();
6158     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6159     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6160
6161     if ($src_storeid && $dst_storeid) {
6162
6163         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6164
6165         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6166         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6167
6168         my $src_format = qemu_img_format($src_scfg, $src_volname);
6169         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6170
6171         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6172         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6173
6174         my $cmd = [];
6175         push @$cmd, '/usr/bin/qemu-img', 'convert', '-t', 'writeback', '-p', '-n';
6176         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
6177         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path, $dst_path;
6178
6179         my $parser = sub {
6180             my $line = shift;
6181             if($line =~ m/\((\S+)\/100\%\)/){
6182                 my $percent = $1;
6183                 my $transferred = int($size * $percent / 100);
6184                 my $remaining = $size - $transferred;
6185
6186                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6187             }
6188
6189         };
6190
6191         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6192         my $err = $@;
6193         die "copy failed: $err" if $err;
6194     }
6195 }
6196
6197 sub qemu_img_format {
6198     my ($scfg, $volname) = @_;
6199
6200     if ($scfg->{path} && $volname =~ m/\.(raw|cow|qcow|qcow2|qed|vmdk|cloop)$/) {
6201         return $1;
6202     } else {
6203         return "raw";
6204     }
6205 }
6206
6207 sub qemu_drive_mirror {
6208     my ($vmid, $drive, $dst_volid, $vmiddst) = @_;
6209
6210     my $storecfg = PVE::Storage::config();
6211     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6212
6213     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6214
6215     my $format = qemu_img_format($dst_scfg, $dst_volname);
6216
6217     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6218
6219     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $dst_path };
6220     $opts->{format} = $format if $format;
6221
6222     print "drive mirror is starting (scanning bitmap) : this step can take some minutes/hours, depend of disk size and storage speed\n";
6223
6224     eval {
6225     vm_mon_cmd($vmid, "drive-mirror", %$opts);
6226         while (1) {
6227             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6228             my $stat = @$stats[0];
6229             die "mirroring job seem to have die. Maybe do you have bad sectors?" if !$stat;
6230             die "error job is not mirroring" if $stat->{type} ne "mirror";
6231
6232             my $busy = $stat->{busy};
6233             my $ready = $stat->{ready};
6234
6235             if (my $total = $stat->{len}) {
6236                 my $transferred = $stat->{offset} || 0;
6237                 my $remaining = $total - $transferred;
6238                 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6239
6240                 print "transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6241             }
6242
6243
6244             if ($stat->{ready} eq 'true') {
6245
6246                 last if $vmiddst != $vmid;
6247
6248                 # try to switch the disk if source and destination are on the same guest
6249                 eval { vm_mon_cmd($vmid, "block-job-complete", device => "drive-$drive") };
6250                 last if !$@;
6251                 die $@ if $@ !~ m/cannot be completed/;
6252             }
6253             sleep 1;
6254         }
6255
6256
6257     };
6258     my $err = $@;
6259
6260     my $cancel_job = sub {
6261         vm_mon_cmd($vmid, "block-job-cancel", device => "drive-$drive");
6262         while (1) {
6263             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6264             my $stat = @$stats[0];
6265             last if !$stat;
6266             sleep 1;
6267         }
6268     };
6269
6270     if ($err) {
6271         eval { &$cancel_job(); };
6272         die "mirroring error: $err";
6273     }
6274
6275     if ($vmiddst != $vmid) {
6276         # if we clone a disk for a new target vm, we don't switch the disk
6277         &$cancel_job(); # so we call block-job-cancel
6278     }
6279 }
6280
6281 sub clone_disk {
6282     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6283         $newvmid, $storage, $format, $full, $newvollist) = @_;
6284
6285     my $newvolid;
6286
6287     if (!$full) {
6288         print "create linked clone of drive $drivename ($drive->{file})\n";
6289         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6290         push @$newvollist, $newvolid;
6291     } else {
6292         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6293         $storeid = $storage if $storage;
6294
6295         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6296         if (!$format) {
6297             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6298             $format = qemu_img_format($scfg, $volname);
6299         }
6300
6301         # test if requested format is supported - else use default
6302         my $supported = grep { $_ eq $format } @$validFormats;
6303         $format = $defFormat if !$supported;
6304
6305         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6306
6307         print "create full clone of drive $drivename ($drive->{file})\n";
6308         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $format, undef, ($size/1024));
6309         push @$newvollist, $newvolid;
6310
6311         PVE::Storage::activate_volumes($storecfg, $newvollist);
6312
6313         if (!$running || $snapname) {
6314             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname);
6315         } else {
6316             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid);
6317         }
6318     }
6319
6320     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6321
6322     my $disk = $drive;
6323     $disk->{format} = undef;
6324     $disk->{file} = $newvolid;
6325     $disk->{size} = $size;
6326
6327     return $disk;
6328 }
6329
6330 # this only works if VM is running
6331 sub get_current_qemu_machine {
6332     my ($vmid) = @_;
6333
6334     my $cmd = { execute => 'query-machines', arguments => {} };
6335     my $res = vm_qmp_command($vmid, $cmd);
6336
6337     my ($current, $default);
6338     foreach my $e (@$res) {
6339         $default = $e->{name} if $e->{'is-default'};
6340         $current = $e->{name} if $e->{'is-current'};
6341     }
6342
6343     # fallback to the default machine if current is not supported by qemu
6344     return $current || $default || 'pc';
6345 }
6346
6347 sub qemu_machine_feature_enabled {
6348     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6349
6350     my $current_major;
6351     my $current_minor;
6352
6353     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6354
6355         $current_major = $3;
6356         $current_minor = $4;
6357
6358     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6359
6360         $current_major = $1;
6361         $current_minor = $2;
6362     }
6363
6364     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
6365
6366
6367 }
6368
6369 sub qemu_machine_pxe {
6370     my ($vmid, $conf, $machine) = @_;
6371
6372     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6373
6374     foreach my $opt (keys %$conf) {
6375         next if $opt !~ m/^net(\d+)$/;
6376         my $net = PVE::QemuServer::parse_net($conf->{$opt});
6377         next if !$net;
6378         my $romfile = PVE::QemuServer::vm_mon_cmd_nocheck($vmid, 'qom-get', path => $opt, property => 'romfile');
6379         return $machine.".pxe" if $romfile =~ m/pxe/;
6380         last;
6381     }
6382
6383 }
6384
6385 sub qemu_use_old_bios_files {
6386     my ($machine_type) = @_;
6387
6388     return if !$machine_type;
6389
6390     my $use_old_bios_files = undef;
6391
6392     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6393         $machine_type = $1;
6394         $use_old_bios_files = 1;
6395     } else {
6396         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6397         # load new efi bios files on migration. So this hack is required to allow
6398         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6399         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6400         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, undef, 2, 4);
6401     }
6402
6403     return ($use_old_bios_files, $machine_type);
6404 }
6405
6406 sub lspci {
6407
6408     my $devices = {};
6409
6410     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6411             my (undef, $id, $function) = @_;
6412             my $res = { id => $id, function => $function};
6413             push @{$devices->{$id}}, $res;
6414     });
6415
6416     return $devices;
6417 }
6418
6419 sub vm_iothreads_list {
6420     my ($vmid) = @_;
6421
6422     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6423
6424     my $iothreads = {};
6425     foreach my $iothread (@$res) {
6426         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6427     }
6428
6429     return $iothreads;
6430 }
6431
6432 sub scsihw_infos {
6433     my ($conf, $drive) = @_;
6434
6435     my $maxdev = 0;
6436
6437     if ($conf->{scsihw} && ($conf->{scsihw} =~ m/^lsi/)) {
6438         $maxdev = 7;
6439     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6440         $maxdev = 1;
6441     } else {
6442         $maxdev = 256;
6443     }
6444
6445     my $controller = int($drive->{index} / $maxdev);
6446     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6447
6448     return ($maxdev, $controller, $controller_prefix);
6449 }
6450
6451 # bash completion helper
6452
6453 sub complete_backup_archives {
6454     my ($cmdname, $pname, $cvalue) = @_;
6455
6456     my $cfg = PVE::Storage::config();
6457
6458     my $storeid;
6459
6460     if ($cvalue =~ m/^([^:]+):/) {
6461         $storeid = $1;
6462     }
6463
6464     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6465
6466     my $res = [];
6467     foreach my $id (keys %$data) {
6468         foreach my $item (@{$data->{$id}}) {
6469             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6470             push @$res, $item->{volid} if defined($item->{volid});
6471         }
6472     }
6473
6474     return $res;
6475 }
6476
6477 my $complete_vmid_full = sub {
6478     my ($running) = @_;
6479
6480     my $idlist = vmstatus();
6481
6482     my $res = [];
6483
6484     foreach my $id (keys %$idlist) {
6485         my $d = $idlist->{$id};
6486         if (defined($running)) {
6487             next if $d->{template};
6488             next if $running && $d->{status} ne 'running';
6489             next if !$running && $d->{status} eq 'running';
6490         }
6491         push @$res, $id;
6492
6493     }
6494     return $res;
6495 };
6496
6497 sub complete_vmid {
6498     return &$complete_vmid_full();
6499 }
6500
6501 sub complete_vmid_stopped {
6502     return &$complete_vmid_full(0);
6503 }
6504
6505 sub complete_vmid_running {
6506     return &$complete_vmid_full(1);
6507 }
6508
6509 sub complete_storage {
6510
6511     my $cfg = PVE::Storage::config();
6512     my $ids = $cfg->{ids};
6513
6514     my $res = [];
6515     foreach my $sid (keys %$ids) {
6516         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6517         next if !$ids->{$sid}->{content}->{images};
6518         push @$res, $sid;
6519     }
6520
6521     return $res;
6522 }
6523
6524 1;