PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QMPClient;
  31 use PVE::RPCEnvironment;
  32 use Time::HiRes qw(gettimeofday);
  33
  34 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  35
  36 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  37
  38 # Note about locking: we use flock on the config file protect
  39 # against concurent actions.
  40 # Aditionaly, we have a 'lock' setting in the config file. This
  41 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  42 # allowed when such lock is set. But you can ignore this kind of
  43 # lock with the --skiplock flag.
  44
  45 cfs_register_file('/qemu-server/',
  46                   \&parse_vm_config,
  47                   \&write_vm_config);
  48
  49 PVE::JSONSchema::register_standard_option('skiplock', {
  50     description => "Ignore locks - only root is allowed to use this option.",
  51     type => 'boolean',
  52     optional => 1,
  53 });
  54
  55 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  56     description => "Some command save/restore state from this location.",
  57     type => 'string',
  58     maxLength => 128,
  59     optional => 1,
  60 });
  61
  62 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  63     description => "The name of the snapshot.",
  64     type => 'string', format => 'pve-configid',
  65     maxLength => 40,
  66 });
  67
  68 #no warnings 'redefine';
  69
  70 sub cgroups_write {
  71    my ($controller, $vmid, $option, $value) = @_;
  72
  73    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  74    PVE::ProcFSTools::write_proc_entry($path, $value);
  75
  76 }
  77
  78 my $nodename = PVE::INotify::nodename();
  79
  80 mkdir "/etc/pve/nodes/$nodename";
  81 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  82 mkdir $confdir;
  83
  84 my $var_run_tmpdir = "/var/run/qemu-server";
  85 mkdir $var_run_tmpdir;
  86
  87 my $lock_dir = "/var/lock/qemu-server";
  88 mkdir $lock_dir;
  89
  90 my $pcisysfs = "/sys/bus/pci";
  91
  92 my $confdesc = {
  93     onboot => {
  94         optional => 1,
  95         type => 'boolean',
  96         description => "Specifies whether a VM will be started during system bootup.",
  97         default => 0,
  98     },
  99     autostart => {
 100         optional => 1,
 101         type => 'boolean',
 102         description => "Automatic restart after crash (currently ignored).",
 103         default => 0,
 104     },
 105     hotplug => {
 106         optional => 1,
 107         type => 'string', format => 'pve-hotplug-features',
 108         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 109         default => 'network,disk,usb',
 110     },
 111     reboot => {
 112         optional => 1,
 113         type => 'boolean',
 114         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 115         default => 1,
 116     },
 117     lock => {
 118         optional => 1,
 119         type => 'string',
 120         description => "Lock/unlock the VM.",
 121         enum => [qw(migrate backup snapshot rollback)],
 122     },
 123     cpulimit => {
 124         optional => 1,
 125         type => 'number',
 126         description => "Limit of CPU usage. Note if the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 127         minimum => 0,
 128         maximum => 128,
 129         default => 0,
 130     },
 131     cpuunits => {
 132         optional => 1,
 133         type => 'integer',
 134         description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 135         minimum => 0,
 136         maximum => 500000,
 137         default => 1000,
 138     },
 139     memory => {
 140         optional => 1,
 141         type => 'integer',
 142         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 143         minimum => 16,
 144         default => 512,
 145     },
 146     balloon => {
 147         optional => 1,
 148         type => 'integer',
 149         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 150         minimum => 0,
 151     },
 152     shares => {
 153         optional => 1,
 154         type => 'integer',
 155         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 156         minimum => 0,
 157         maximum => 50000,
 158         default => 1000,
 159     },
 160     keyboard => {
 161         optional => 1,
 162         type => 'string',
 163         description => "Keybord layout for vnc server. Default is read from the datacenter configuration file.",
 164         enum => PVE::Tools::kvmkeymaplist(),
 165         default => 'en-us',
 166     },
 167     name => {
 168         optional => 1,
 169         type => 'string', format => 'dns-name',
 170         description => "Set a name for the VM. Only used on the configuration web interface.",
 171     },
 172     scsihw => {
 173         optional => 1,
 174         type => 'string',
 175         description => "scsi controller model",
 176         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 177         default => 'lsi',
 178     },
 179     description => {
 180         optional => 1,
 181         type => 'string',
 182         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 183     },
 184     ostype => {
 185         optional => 1,
 186         type => 'string',
 187         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 l24 l26 solaris)],
 188         description => <<EODESC,
 189 Used to enable special optimization/features for specific
 190 operating systems:
 191
 192 other  => unspecified OS
 193 wxp    => Microsoft Windows XP
 194 w2k    => Microsoft Windows 2000
 195 w2k3   => Microsoft Windows 2003
 196 w2k8   => Microsoft Windows 2008
 197 wvista => Microsoft Windows Vista
 198 win7   => Microsoft Windows 7
 199 win8   => Microsoft Windows 8/2012
 200 l24    => Linux 2.4 Kernel
 201 l26    => Linux 2.6/3.X Kernel
 202 solaris => solaris/opensolaris/openindiania kernel
 203
 204 other|l24|l26|solaris                       ... no special behaviour
 205 wxp|w2k|w2k3|w2k8|wvista|win7|win8  ... use --localtime switch
 206 EODESC
 207     },
 208     boot => {
 209         optional => 1,
 210         type => 'string',
 211         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 212         pattern => '[acdn]{1,4}',
 213         default => 'cdn',
 214     },
 215     bootdisk => {
 216         optional => 1,
 217         type => 'string', format => 'pve-qm-bootdisk',
 218         description => "Enable booting from specified disk.",
 219         pattern => '(ide|sata|scsi|virtio)\d+',
 220     },
 221     smp => {
 222         optional => 1,
 223         type => 'integer',
 224         description => "The number of CPUs. Please use option -sockets instead.",
 225         minimum => 1,
 226         default => 1,
 227     },
 228     sockets => {
 229         optional => 1,
 230         type => 'integer',
 231         description => "The number of CPU sockets.",
 232         minimum => 1,
 233         default => 1,
 234     },
 235     cores => {
 236         optional => 1,
 237         type => 'integer',
 238         description => "The number of cores per socket.",
 239         minimum => 1,
 240         default => 1,
 241     },
 242     numa => {
 243         optional => 1,
 244         type => 'boolean',
 245         description => "Enable/disable Numa.",
 246         default => 0,
 247     },
 248     vcpus => {
 249         optional => 1,
 250         type => 'integer',
 251         description => "Number of hotplugged vcpus.",
 252         minimum => 1,
 253         default => 0,
 254     },
 255     acpi => {
 256         optional => 1,
 257         type => 'boolean',
 258         description => "Enable/disable ACPI.",
 259         default => 1,
 260     },
 261     agent => {
 262         optional => 1,
 263         type => 'boolean',
 264         description => "Enable/disable Qemu GuestAgent.",
 265         default => 0,
 266     },
 267     kvm => {
 268         optional => 1,
 269         type => 'boolean',
 270         description => "Enable/disable KVM hardware virtualization.",
 271         default => 1,
 272     },
 273     tdf => {
 274         optional => 1,
 275         type => 'boolean',
 276         description => "Enable/disable time drift fix.",
 277         default => 0,
 278     },
 279     localtime => {
 280         optional => 1,
 281         type => 'boolean',
 282         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 283     },
 284     freeze => {
 285         optional => 1,
 286         type => 'boolean',
 287         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 288     },
 289     vga => {
 290         optional => 1,
 291         type => 'string',
 292         description => "Select VGA type. If you want to use high resolution modes (>= 1280x1024x16) then you should use option 'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and 'cirrur' for other OS types. Option 'qxl' enables the SPICE display sever. You can also run without any graphic card using a serial devive as terminal.",
 293         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 294     },
 295     watchdog => {
 296         optional => 1,
 297         type => 'string', format => 'pve-qm-watchdog',
 298         typetext => '[[model=]i6300esb|ib700] [,[action=]reset|shutdown|poweroff|pause|debug|none]',
 299         description => "Create a virtual hardware watchdog device.  Once enabled (by a guest action), the watchdog must be periodically polled by an agent inside the guest or else the guest will be restarted (or execute the action specified)",
 300     },
 301     startdate => {
 302         optional => 1,
 303         type => 'string',
 304         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 305         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 306         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 307         default => 'now',
 308     },
 309     startup =>  get_standard_option('pve-startup-order'),
 310     template => {
 311         optional => 1,
 312         type => 'boolean',
 313         description => "Enable/disable Template.",
 314         default => 0,
 315     },
 316     args => {
 317         optional => 1,
 318         type => 'string',
 319         description => <<EODESCR,
 320 Note: this option is for experts only. It allows you to pass arbitrary arguments to kvm, for example:
 321
 322 args: -no-reboot -no-hpet
 323 EODESCR
 324     },
 325     tablet => {
 326         optional => 1,
 327         type => 'boolean',
 328         default => 1,
 329         description => "Enable/disable the usb tablet device. This device is usually needed to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with normal VNC clients. If you're running lots of console-only guests on one host, you may consider disabling this to save some context switches. This is turned of by default if you use spice (vga=qxl).",
 330     },
 331     migrate_speed => {
 332         optional => 1,
 333         type => 'integer',
 334         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 335         minimum => 0,
 336         default => 0,
 337     },
 338     migrate_downtime => {
 339         optional => 1,
 340         type => 'number',
 341         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 342         minimum => 0,
 343         default => 0.1,
 344     },
 345     cdrom => {
 346         optional => 1,
 347         type => 'string', format => 'pve-qm-drive',
 348         typetext => 'volume',
 349         description => "This is an alias for option -ide2",
 350     },
 351     cpu => {
 352         optional => 1,
 353         description => "Emulated CPU type.",
 354         type => 'string',
 355         enum => [ qw(486 athlon pentium pentium2 pentium3 coreduo core2duo kvm32 kvm64 qemu32 qemu64 phenom Conroe Penryn Nehalem Westmere SandyBridge IvyBridge Haswell Broadwell Opteron_G1 Opteron_G2 Opteron_G3 Opteron_G4 Opteron_G5 host) ],
 356         default => 'kvm64',
 357     },
 358     parent => get_standard_option('pve-snapshot-name', {
 359         optional => 1,
 360         description => "Parent snapshot name. This is used internally, and should not be modified.",
 361     }),
 362     snaptime => {
 363         optional => 1,
 364         description => "Timestamp for snapshots.",
 365         type => 'integer',
 366         minimum => 0,
 367     },
 368     vmstate => {
 369         optional => 1,
 370         type => 'string', format => 'pve-volume-id',
 371         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 372     },
 373     machine => {
 374         description => "Specific the Qemu machine type.",
 375         type => 'string',
 376         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
 377         maxLength => 40,
 378         optional => 1,
 379     },
 380     smbios1 => {
 381         description => "Specify SMBIOS type 1 fields.",
 382         type => 'string', format => 'pve-qm-smbios1',
 383         typetext => "[manufacturer=str][,product=str][,version=str][,serial=str] [,uuid=uuid][,sku=str][,family=str]",
 384         maxLength => 256,
 385         optional => 1,
 386     },
 387     protection => {
 388         optional => 1,
 389         type => 'boolean',
 390         description => "Sets the protection flag of the VM. This will prevent the remove operation.",
 391         default => 0,
 392     },
 393 };
 394
 395 # what about other qemu settings ?
 396 #cpu => 'string',
 397 #machine => 'string',
 398 #fda => 'file',
 399 #fdb => 'file',
 400 #mtdblock => 'file',
 401 #sd => 'file',
 402 #pflash => 'file',
 403 #snapshot => 'bool',
 404 #bootp => 'file',
 405 ##tftp => 'dir',
 406 ##smb => 'dir',
 407 #kernel => 'file',
 408 #append => 'string',
 409 #initrd => 'file',
 410 ##soundhw => 'string',
 411
 412 while (my ($k, $v) = each %$confdesc) {
 413     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 414 }
 415
 416 my $MAX_IDE_DISKS = 4;
 417 my $MAX_SCSI_DISKS = 14;
 418 my $MAX_VIRTIO_DISKS = 16;
 419 my $MAX_SATA_DISKS = 6;
 420 my $MAX_USB_DEVICES = 5;
 421 my $MAX_NETS = 32;
 422 my $MAX_UNUSED_DISKS = 8;
 423 my $MAX_HOSTPCI_DEVICES = 4;
 424 my $MAX_SERIAL_PORTS = 4;
 425 my $MAX_PARALLEL_PORTS = 3;
 426 my $MAX_NUMA = 8;
 427 my $MAX_MEM = 4194304;
 428 my $STATICMEM = 1024;
 429
 430 my $numadesc = {
 431     optional => 1,
 432     type => 'string', format => 'pve-qm-numanode',
 433     typetext => "cpus=<id[-id],memory=<mb>[[,hostnodes=<id[-id]>] [,policy=<preferred|bind|interleave>]]",
 434     description => "numa topology",
 435 };
 436 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 437
 438 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 439     $confdesc->{"numa$i"} = $numadesc;
 440 }
 441
 442 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 443                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 444                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 445 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 446
 447 my $netdesc = {
 448     optional => 1,
 449     type => 'string', format => 'pve-qm-net',
 450     typetext => "MODEL=XX:XX:XX:XX:XX:XX [,bridge=<dev>][,queues=<nbqueues>][,rate=<mbps>] [,tag=<vlanid>][,firewall=0|1],link_down=0|1]",
 451     description => <<EODESCR,
 452 Specify network devices.
 453
 454 MODEL is one of: $nic_model_list_txt
 455
 456 XX:XX:XX:XX:XX:XX should be an unique MAC address. This is
 457 automatically generated if not specified.
 458
 459 The bridge parameter can be used to automatically add the interface to a bridge device. The Proxmox VE standard bridge is called 'vmbr0'.
 460
 461 Option 'rate' is used to limit traffic bandwidth from and to this interface. It is specified as floating point number, unit is 'Megabytes per second'.
 462
 463 If you specify no bridge, we create a kvm 'user' (NATed) network device, which provides DHCP and DNS services. The following addresses are used:
 464
 465 10.0.2.2   Gateway
 466 10.0.2.3   DNS Server
 467 10.0.2.4   SMB Server
 468
 469 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 470
 471 EODESCR
 472 };
 473 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 474
 475 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 476     $confdesc->{"net$i"} = $netdesc;
 477 }
 478
 479 my $drivename_hash;
 480
 481 my $idedesc = {
 482     optional => 1,
 483     type => 'string', format => 'pve-qm-drive',
 484     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads] [,discard=ignore|on] [,serial=serial][,model=model]',
 485     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
 486 };
 487 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
 488
 489 my $scsidesc = {
 490     optional => 1,
 491     type => 'string', format => 'pve-qm-drive',
 492     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads] [,discard=ignore|on] [,iothread=on] [,queues=<nbqueues>] [,serial=serial]',
 493     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
 494 };
 495 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 496
 497 my $satadesc = {
 498     optional => 1,
 499     type => 'string', format => 'pve-qm-drive',
 500     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads]  [,discard=ignore|on] [,serial=serial]',
 501     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
 502 };
 503 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
 504
 505 my $virtiodesc = {
 506     optional => 1,
 507     type => 'string', format => 'pve-qm-drive',
 508     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads]  [,discard=ignore|on] [,iothread=on] [,serial=serial]',
 509     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
 510 };
 511 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
 512
 513 my $usbdesc = {
 514     optional => 1,
 515     type => 'string', format => 'pve-qm-usb-device',
 516     typetext => 'host=HOSTUSBDEVICE|spice',
 517     description => <<EODESCR,
 518 Configure an USB device (n is 0 to 4). This can be used to
 519 pass-through usb devices to the guest. HOSTUSBDEVICE syntax is:
 520
 521 'bus-port(.port)*' (decimal numbers) or
 522 'vendor_id:product_id' (hexadeciaml numbers)
 523
 524 You can use the 'lsusb -t' command to list existing usb devices.
 525
 526 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 527
 528 The value 'spice' can be used to add a usb redirection devices for spice.
 529
 530 EODESCR
 531 };
 532 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
 533
 534 my $hostpcidesc = {
 535         optional => 1,
 536         type => 'string', format => 'pve-qm-hostpci',
 537         typetext => "[host=]HOSTPCIDEVICE [,rombar=on|off] [,pcie=0|1] [,x-vga=on|off]",
 538         description => <<EODESCR,
 539 Map host pci devices. HOSTPCIDEVICE syntax is:
 540
 541 'bus:dev.func' (hexadecimal numbers)
 542
 543 You can us the 'lspci' command to list existing pci devices.
 544
 545 The 'rombar' option determines whether or not the device's ROM will be visible in the guest's memory map (default is 'on').
 546
 547 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 548
 549 Experimental: user reported problems with this option.
 550 EODESCR
 551 };
 552 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
 553
 554 my $serialdesc = {
 555         optional => 1,
 556         type => 'string',
 557         pattern => '(/dev/.+|socket)',
 558         description =>  <<EODESCR,
 559 Create a serial device inside the VM (n is 0 to 3), and pass through a host serial device (i.e. /dev/ttyS0), or create a unix socket on the host side (use 'qm terminal' to open a terminal connection).
 560
 561 Note: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
 562
 563 Experimental: user reported problems with this option.
 564 EODESCR
 565 };
 566
 567 my $paralleldesc= {
 568         optional => 1,
 569         type => 'string',
 570         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
 571         description =>  <<EODESCR,
 572 Map host parallel devices (n is 0 to 2).
 573
 574 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 575
 576 Experimental: user reported problems with this option.
 577 EODESCR
 578 };
 579
 580 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
 581     $confdesc->{"parallel$i"} = $paralleldesc;
 582 }
 583
 584 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
 585     $confdesc->{"serial$i"} = $serialdesc;
 586 }
 587
 588 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
 589     $confdesc->{"hostpci$i"} = $hostpcidesc;
 590 }
 591
 592 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
 593     $drivename_hash->{"ide$i"} = 1;
 594     $confdesc->{"ide$i"} = $idedesc;
 595 }
 596
 597 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
 598     $drivename_hash->{"sata$i"} = 1;
 599     $confdesc->{"sata$i"} = $satadesc;
 600 }
 601
 602 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
 603     $drivename_hash->{"scsi$i"} = 1;
 604     $confdesc->{"scsi$i"} = $scsidesc ;
 605 }
 606
 607 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
 608     $drivename_hash->{"virtio$i"} = 1;
 609     $confdesc->{"virtio$i"} = $virtiodesc;
 610 }
 611
 612 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
 613     $confdesc->{"usb$i"} = $usbdesc;
 614 }
 615
 616 my $unuseddesc = {
 617     optional => 1,
 618     type => 'string', format => 'pve-volume-id',
 619     description => "Reference to unused volumes.",
 620 };
 621
 622 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
 623     $confdesc->{"unused$i"} = $unuseddesc;
 624 }
 625
 626 my $kvm_api_version = 0;
 627
 628 sub kvm_version {
 629
 630     return $kvm_api_version if $kvm_api_version;
 631
 632     my $fh = IO::File->new("</dev/kvm") ||
 633         return 0;
 634
 635     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
 636         $kvm_api_version = $v;
 637     }
 638
 639     $fh->close();
 640
 641     return  $kvm_api_version;
 642 }
 643
 644 my $kvm_user_version;
 645
 646 sub kvm_user_version {
 647
 648     return $kvm_user_version if $kvm_user_version;
 649
 650     $kvm_user_version = 'unknown';
 651
 652     my $tmp = `kvm -help 2>/dev/null`;
 653
 654     if ($tmp =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
 655         $kvm_user_version = $2;
 656     }
 657
 658     return $kvm_user_version;
 659
 660 }
 661
 662 my $kernel_has_vhost_net = -c '/dev/vhost-net';
 663
 664 sub disknames {
 665     # order is important - used to autoselect boot disk
 666     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
 667             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
 668             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
 669             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))));
 670 }
 671
 672 sub valid_drivename {
 673     my $dev = shift;
 674
 675     return defined($drivename_hash->{$dev});
 676 }
 677
 678 sub option_exists {
 679     my $key = shift;
 680     return defined($confdesc->{$key});
 681 }
 682
 683 sub nic_models {
 684     return $nic_model_list;
 685 }
 686
 687 sub os_list_description {
 688
 689     return {
 690         other => 'Other',
 691         wxp => 'Windows XP',
 692         w2k => 'Windows 2000',
 693         w2k3 =>, 'Windows 2003',
 694         w2k8 => 'Windows 2008',
 695         wvista => 'Windows Vista',
 696         win7 => 'Windows 7',
 697         win8 => 'Windows 8/2012',
 698         l24 => 'Linux 2.4',
 699         l26 => 'Linux 2.6',
 700     };
 701 }
 702
 703 my $cdrom_path;
 704
 705 sub get_cdrom_path {
 706
 707     return  $cdrom_path if $cdrom_path;
 708
 709     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
 710     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
 711     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
 712 }
 713
 714 sub get_iso_path {
 715     my ($storecfg, $vmid, $cdrom) = @_;
 716
 717     if ($cdrom eq 'cdrom') {
 718         return get_cdrom_path();
 719     } elsif ($cdrom eq 'none') {
 720         return '';
 721     } elsif ($cdrom =~ m|^/|) {
 722         return $cdrom;
 723     } else {
 724         return PVE::Storage::path($storecfg, $cdrom);
 725     }
 726 }
 727
 728 # try to convert old style file names to volume IDs
 729 sub filename_to_volume_id {
 730     my ($vmid, $file, $media) = @_;
 731
 732     if (!($file eq 'none' || $file eq 'cdrom' ||
 733           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
 734
 735         return undef if $file =~ m|/|;
 736
 737         if ($media && $media eq 'cdrom') {
 738             $file = "local:iso/$file";
 739         } else {
 740             $file = "local:$vmid/$file";
 741         }
 742     }
 743
 744     return $file;
 745 }
 746
 747 sub verify_media_type {
 748     my ($opt, $vtype, $media) = @_;
 749
 750     return if !$media;
 751
 752     my $etype;
 753     if ($media eq 'disk') {
 754         $etype = 'images';
 755     } elsif ($media eq 'cdrom') {
 756         $etype = 'iso';
 757     } else {
 758         die "internal error";
 759     }
 760
 761     return if ($vtype eq $etype);
 762
 763     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
 764 }
 765
 766 sub cleanup_drive_path {
 767     my ($opt, $storecfg, $drive) = @_;
 768
 769     # try to convert filesystem paths to volume IDs
 770
 771     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
 772         ($drive->{file} !~ m|^/dev/.+|) &&
 773         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
 774         ($drive->{file} !~ m/^\d+$/)) {
 775         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
 776         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
 777         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
 778         verify_media_type($opt, $vtype, $drive->{media});
 779         $drive->{file} = $volid;
 780     }
 781
 782     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
 783 }
 784
 785 sub create_conf_nolock {
 786     my ($vmid, $settings) = @_;
 787
 788     my $filename = config_file($vmid);
 789
 790     die "configuration file '$filename' already exists\n" if -f $filename;
 791
 792     my $defaults = load_defaults();
 793
 794     $settings->{name} = "vm$vmid" if !$settings->{name};
 795     $settings->{memory} = $defaults->{memory} if !$settings->{memory};
 796
 797     my $data = '';
 798     foreach my $opt (keys %$settings) {
 799         next if !$confdesc->{$opt};
 800
 801         my $value = $settings->{$opt};
 802         next if !$value;
 803
 804         $data .= "$opt: $value\n";
 805     }
 806
 807     PVE::Tools::file_set_contents($filename, $data);
 808 }
 809
 810 sub parse_hotplug_features {
 811     my ($data) = @_;
 812
 813     my $res = {};
 814
 815     return $res if $data eq '0';
 816
 817     $data = $confdesc->{hotplug}->{default} if $data eq '1';
 818
 819     foreach my $feature (PVE::Tools::split_list($data)) {
 820         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
 821             $res->{$1} = 1;
 822         } else {
 823             warn "ignoring unknown hotplug feature '$feature'\n";
 824         }
 825     }
 826     return $res;
 827 }
 828
 829 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
 830 sub pve_verify_hotplug_features {
 831     my ($value, $noerr) = @_;
 832
 833     return $value if parse_hotplug_features($value);
 834
 835     return undef if $noerr;
 836
 837     die "unable to parse hotplug option\n";
 838 }
 839
 840 my $parse_size = sub {
 841     my ($value) = @_;
 842
 843     return undef if $value !~ m/^(\d+(\.\d+)?)([KMG])?$/;
 844     my ($size, $unit) = ($1, $3);
 845     if ($unit) {
 846         if ($unit eq 'K') {
 847             $size = $size * 1024;
 848         } elsif ($unit eq 'M') {
 849             $size = $size * 1024 * 1024;
 850         } elsif ($unit eq 'G') {
 851             $size = $size * 1024 * 1024 * 1024;
 852         }
 853     }
 854     return int($size);
 855 };
 856
 857 my $format_size = sub {
 858     my ($size) = @_;
 859
 860     $size = int($size);
 861
 862     my $kb = int($size/1024);
 863     return $size if $kb*1024 != $size;
 864
 865     my $mb = int($kb/1024);
 866     return "${kb}K" if $mb*1024 != $kb;
 867
 868     my $gb = int($mb/1024);
 869     return "${mb}M" if $gb*1024 != $mb;
 870
 871     return "${gb}G";
 872 };
 873
 874 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
 875 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
 876 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
 877 #        [,aio=native|threads][,discard=ignore|on][,iothread=on]
 878 #        [,serial=serial][,model=model]
 879
 880 sub parse_drive {
 881     my ($key, $data) = @_;
 882
 883     my $res = {};
 884
 885     # $key may be undefined - used to verify JSON parameters
 886     if (!defined($key)) {
 887         $res->{interface} = 'unknown'; # should not harm when used to verify parameters
 888         $res->{index} = 0;
 889     } elsif ($key =~ m/^([^\d]+)(\d+)$/) {
 890         $res->{interface} = $1;
 891         $res->{index} = $2;
 892     } else {
 893         return undef;
 894     }
 895
 896     foreach my $p (split (/,/, $data)) {
 897         next if $p =~ m/^\s*$/;
 898
 899         if ($p =~ m/^(file|volume|cyls|heads|secs|trans|media|snapshot|cache|format|rerror|werror|backup|aio|bps|mbps|mbps_max|bps_rd|mbps_rd|mbps_rd_max|bps_wr|mbps_wr|mbps_wr_max|iops|iops_max|iops_rd|iops_rd_max|iops_wr|iops_wr_max|size|discard|iothread|queues|serial|model)=(.+)$/) {
 900             my ($k, $v) = ($1, $2);
 901
 902             $k = 'file' if $k eq 'volume';
 903
 904             return undef if defined $res->{$k};
 905
 906             if ($k eq 'bps' || $k eq 'bps_rd' || $k eq 'bps_wr') {
 907                 return undef if !$v || $v !~ m/^\d+/;
 908                 $k = "m$k";
 909                 $v = sprintf("%.3f", $v / (1024*1024));
 910             }
 911             $res->{$k} = $v;
 912         } else {
 913             if (!$res->{file} && $p !~ m/=/) {
 914                 $res->{file} = $p;
 915             } else {
 916                 return undef;
 917             }
 918         }
 919     }
 920
 921     return undef if !$res->{file};
 922
 923     return undef if $res->{cache} &&
 924         $res->{cache} !~ m/^(off|none|writethrough|writeback|unsafe|directsync)$/;
 925     return undef if $res->{snapshot} && $res->{snapshot} !~ m/^(on|off)$/;
 926     return undef if $res->{cyls} && $res->{cyls} !~ m/^\d+$/;
 927     return undef if $res->{heads} && $res->{heads} !~ m/^\d+$/;
 928     return undef if $res->{secs} && $res->{secs} !~ m/^\d+$/;
 929     return undef if $res->{media} && $res->{media} !~ m/^(disk|cdrom)$/;
 930     return undef if $res->{trans} && $res->{trans} !~ m/^(none|lba|auto)$/;
 931     return undef if $res->{format} && $res->{format} !~ m/^(raw|cow|qcow|qed|qcow2|vmdk|cloop)$/;
 932     return undef if $res->{rerror} && $res->{rerror} !~ m/^(ignore|report|stop)$/;
 933     return undef if $res->{werror} && $res->{werror} !~ m/^(enospc|ignore|report|stop)$/;
 934     return undef if $res->{backup} && $res->{backup} !~ m/^(yes|no)$/;
 935     return undef if $res->{aio} && $res->{aio} !~ m/^(native|threads)$/;
 936     return undef if $res->{discard} && $res->{discard} !~ m/^(ignore|on)$/;
 937     return undef if $res->{iothread} && $res->{iothread} !~ m/^(on)$/;
 938     return undef if $res->{queues} && ($res->{queues} !~ m/^\d+$/ || $res->{queues} < 2);
 939
 940     return undef if $res->{mbps_rd} && $res->{mbps};
 941     return undef if $res->{mbps_wr} && $res->{mbps};
 942
 943     return undef if $res->{mbps} && $res->{mbps} !~ m/^\d+(\.\d+)?$/;
 944     return undef if $res->{mbps_max} && $res->{mbps_max} !~ m/^\d+(\.\d+)?$/;
 945     return undef if $res->{mbps_rd} && $res->{mbps_rd} !~ m/^\d+(\.\d+)?$/;
 946     return undef if $res->{mbps_rd_max} && $res->{mbps_rd_max} !~ m/^\d+(\.\d+)?$/;
 947     return undef if $res->{mbps_wr} && $res->{mbps_wr} !~ m/^\d+(\.\d+)?$/;
 948     return undef if $res->{mbps_wr_max} && $res->{mbps_wr_max} !~ m/^\d+(\.\d+)?$/;
 949
 950     return undef if $res->{iops_rd} && $res->{iops};
 951     return undef if $res->{iops_wr} && $res->{iops};
 952
 953
 954     return undef if $res->{iops} && $res->{iops} !~ m/^\d+$/;
 955     return undef if $res->{iops_max} && $res->{iops_max} !~ m/^\d+$/;
 956     return undef if $res->{iops_rd} && $res->{iops_rd} !~ m/^\d+$/;
 957     return undef if $res->{iops_rd_max} && $res->{iops_rd_max} !~ m/^\d+$/;
 958     return undef if $res->{iops_wr} && $res->{iops_wr} !~ m/^\d+$/;
 959     return undef if $res->{iops_wr_max} && $res->{iops_wr_max} !~ m/^\d+$/;
 960
 961     if ($res->{size}) {
 962         return undef if !defined($res->{size} = &$parse_size($res->{size}));
 963     }
 964
 965     if ($res->{media} && ($res->{media} eq 'cdrom')) {
 966         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
 967         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
 968         return undef if $res->{interface} eq 'virtio';
 969     }
 970
 971     # rerror does not work with scsi drives
 972     if ($res->{rerror}) {
 973         return undef if $res->{interface} eq 'scsi';
 974     }
 975
 976     return $res;
 977 }
 978
 979 my @qemu_drive_options = qw(heads secs cyls trans media format cache snapshot rerror werror aio discard iops iops_rd iops_wr iops_max iops_rd_max iops_wr_max serial);
 980
 981 sub print_drive {
 982     my ($vmid, $drive) = @_;
 983
 984     my $opts = '';
 985     foreach my $o (@qemu_drive_options, 'mbps', 'mbps_rd', 'mbps_wr', 'mbps_max', 'mbps_rd_max', 'mbps_wr_max', 'backup', 'iothread', 'queues') {
 986         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
 987     }
 988
 989     if ($drive->{size}) {
 990         $opts .= ",size=" . &$format_size($drive->{size});
 991     }
 992
 993     if (my $model = $drive->{model}) {
 994         $opts .= ",model=$model";
 995     }
 996
 997     return "$drive->{file}$opts";
 998 }
 999
1000 sub scsi_inquiry {
1001     my($fh, $noerr) = @_;
1002
1003     my $SG_IO = 0x2285;
1004     my $SG_GET_VERSION_NUM = 0x2282;
1005
1006     my $versionbuf = "\x00" x 8;
1007     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1008     if (!$ret) {
1009         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1010         return undef;
1011     }
1012     my $version = unpack("I", $versionbuf);
1013     if ($version < 30000) {
1014         die "scsi generic interface too old\n"  if !$noerr;
1015         return undef;
1016     }
1017
1018     my $buf = "\x00" x 36;
1019     my $sensebuf = "\x00" x 8;
1020     my $cmd = pack("C x3 C x1", 0x12, 36);
1021
1022     # see /usr/include/scsi/sg.h
1023     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1024
1025     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1026                       length($sensebuf), 0, length($buf), $buf,
1027                       $cmd, $sensebuf, 6000);
1028
1029     $ret = ioctl($fh, $SG_IO, $packet);
1030     if (!$ret) {
1031         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1032         return undef;
1033     }
1034
1035     my @res = unpack($sg_io_hdr_t, $packet);
1036     if ($res[17] || $res[18]) {
1037         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1038         return undef;
1039     }
1040
1041     my $res = {};
1042     (my $byte0, my $byte1, $res->{vendor},
1043      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1044
1045     $res->{removable} = $byte1 & 128 ? 1 : 0;
1046     $res->{type} = $byte0 & 31;
1047
1048     return $res;
1049 }
1050
1051 sub path_is_scsi {
1052     my ($path) = @_;
1053
1054     my $fh = IO::File->new("+<$path") || return undef;
1055     my $res = scsi_inquiry($fh, 1);
1056     close($fh);
1057
1058     return $res;
1059 }
1060
1061 sub machine_type_is_q35 {
1062     my ($conf) = @_;
1063
1064     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1065 }
1066
1067 sub print_tabletdevice_full {
1068     my ($conf) = @_;
1069
1070     my $q35 = machine_type_is_q35($conf);
1071
1072     # we use uhci for old VMs because tablet driver was buggy in older qemu
1073     my $usbbus = $q35 ? "ehci" : "uhci";
1074
1075     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1076 }
1077
1078 sub print_drivedevice_full {
1079     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1080
1081     my $device = '';
1082     my $maxdev = 0;
1083
1084     if ($drive->{interface} eq 'virtio') {
1085         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1086         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1087         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1088     } elsif ($drive->{interface} eq 'scsi') {
1089
1090         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1091         my $unit = $drive->{index} % $maxdev;
1092         my $devicetype = 'hd';
1093         my $path = '';
1094         if (drive_is_cdrom($drive)) {
1095               $devicetype = 'cd';
1096         } else {
1097               if ($drive->{file} =~ m|^/|) {
1098                   $path = $drive->{file};
1099               } else {
1100                   $path = PVE::Storage::path($storecfg, $drive->{file});
1101               }
1102
1103               if($path =~ m/^iscsi\:\/\//){
1104                   $devicetype = 'generic';
1105               } else {
1106                   if (my $info = path_is_scsi($path)) {
1107                       if ($info->{type} == 0) {
1108                           $devicetype = 'block';
1109                       } elsif ($info->{type} == 1) { # tape
1110                           $devicetype = 'generic';
1111                       }
1112                   }
1113               }
1114          }
1115
1116         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1117             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1118         } else {
1119             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1120         }
1121
1122     } elsif ($drive->{interface} eq 'ide'){
1123         $maxdev = 2;
1124         my $controller = int($drive->{index} / $maxdev);
1125         my $unit = $drive->{index} % $maxdev;
1126         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1127
1128         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1129         if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
1130             $device .= ",model=$model";
1131         }
1132     } elsif ($drive->{interface} eq 'sata'){
1133         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1134         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1135         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1136     } elsif ($drive->{interface} eq 'usb') {
1137         die "implement me";
1138         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1139     } else {
1140         die "unsupported interface type";
1141     }
1142
1143     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1144
1145     return $device;
1146 }
1147
1148 sub get_initiator_name {
1149     my $initiator;
1150
1151     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1152     while (defined(my $line = <$fh>)) {
1153         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1154         $initiator = $1;
1155         last;
1156     }
1157     $fh->close();
1158
1159     return $initiator;
1160 }
1161
1162 sub print_drive_full {
1163     my ($storecfg, $vmid, $drive) = @_;
1164
1165     my $path;
1166     my $volid = $drive->{file};
1167     my $format;
1168
1169     if (drive_is_cdrom($drive)) {
1170         $path = get_iso_path($storecfg, $vmid, $volid);
1171     } else {
1172         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1173         if ($storeid) {
1174             $path = PVE::Storage::path($storecfg, $volid);
1175             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1176             $format = qemu_img_format($scfg, $volname);
1177         } else {
1178             $path = $volid;
1179         }
1180    }
1181
1182     my $opts = '';
1183     foreach my $o (@qemu_drive_options) {
1184         next if $o eq 'bootindex';
1185         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1186     }
1187
1188     $opts .= ",format=$format" if $format && !$drive->{format};
1189
1190     foreach my $o (qw(bps bps_rd bps_wr)) {
1191         my $v = $drive->{"m$o"};
1192         $opts .= ",$o=" . int($v*1024*1024) if $v;
1193     }
1194
1195     my $cache_direct = 0;
1196
1197     if (my $cache = $drive->{cache}) {
1198         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1199     } elsif (!drive_is_cdrom($drive)) {
1200         $opts .= ",cache=none";
1201         $cache_direct = 1;
1202     }
1203
1204     # aio native works only with O_DIRECT
1205     if (!$drive->{aio}) {
1206         if($cache_direct) {
1207             $opts .= ",aio=native";
1208         } else {
1209             $opts .= ",aio=threads";
1210         }
1211     }
1212
1213     my $detectzeroes = $drive->{discard} ? "unmap" : "on";
1214     $opts .= ",detect-zeroes=$detectzeroes" if !drive_is_cdrom($drive);
1215
1216     my $pathinfo = $path ? "file=$path," : '';
1217
1218     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1219 }
1220
1221 sub print_netdevice_full {
1222     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1223
1224     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1225
1226     my $device = $net->{model};
1227     if ($net->{model} eq 'virtio') {
1228          $device = 'virtio-net-pci';
1229      };
1230
1231     my $pciaddr = print_pci_addr("$netid", $bridges);
1232     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1233     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1234         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1235         my $vectors = $net->{queues} * 2 + 2;
1236         $tmpstr .= ",vectors=$vectors,mq=on";
1237     }
1238     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1239
1240     if ($use_old_bios_files) {
1241         my $romfile;
1242         if ($device eq 'virtio-net-pci') {
1243             $romfile = 'pxe-virtio.rom';
1244         } elsif ($device eq 'e1000') {
1245             $romfile = 'pxe-e1000.rom';
1246         } elsif ($device eq 'ne2k') {
1247             $romfile = 'pxe-ne2k_pci.rom';
1248         } elsif ($device eq 'pcnet') {
1249             $romfile = 'pxe-pcnet.rom';
1250         } elsif ($device eq 'rtl8139') {
1251             $romfile = 'pxe-rtl8139.rom';
1252         }
1253         $tmpstr .= ",romfile=$romfile" if $romfile;
1254     }
1255
1256     return $tmpstr;
1257 }
1258
1259 sub print_netdev_full {
1260     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1261
1262     my $i = '';
1263     if ($netid =~ m/^net(\d+)$/) {
1264         $i = int($1);
1265     }
1266
1267     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1268
1269     my $ifname = "tap${vmid}i$i";
1270
1271     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1272     die "interface name '$ifname' is too long (max 15 character)\n"
1273         if length($ifname) >= 16;
1274
1275     my $vhostparam = '';
1276     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1277
1278     my $vmname = $conf->{name} || "vm$vmid";
1279
1280     my $netdev = "";
1281     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1282
1283     if ($net->{bridge}) {
1284         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1285     } else {
1286         $netdev = "type=user,id=$netid,hostname=$vmname";
1287     }
1288
1289     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1290
1291     return $netdev;
1292 }
1293
1294 sub drive_is_cdrom {
1295     my ($drive) = @_;
1296
1297     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1298
1299 }
1300
1301 sub parse_numa {
1302     my ($data) = @_;
1303
1304     my $res = {};
1305
1306     foreach my $kvp (split(/,/, $data)) {
1307
1308         if ($kvp =~ m/^memory=(\S+)$/) {
1309             $res->{memory} = $1;
1310         } elsif ($kvp =~ m/^policy=(preferred|bind|interleave)$/) {
1311             $res->{policy} = $1;
1312         } elsif ($kvp =~ m/^cpus=(\d+)(-(\d+))?$/) {
1313             $res->{cpus}->{start} = $1;
1314             $res->{cpus}->{end} = $3;
1315         } elsif ($kvp =~ m/^hostnodes=(\d+)(-(\d+))?$/) {
1316             $res->{hostnodes}->{start} = $1;
1317             $res->{hostnodes}->{end} = $3;
1318         } else {
1319             return undef;
1320         }
1321     }
1322
1323     return $res;
1324 }
1325
1326 sub parse_hostpci {
1327     my ($value) = @_;
1328
1329     return undef if !$value;
1330
1331
1332     my @list = split(/,/, $value);
1333     my $found;
1334
1335     my $res = {};
1336     foreach my $kv (@list) {
1337
1338         if ($kv =~ m/^(host=)?([a-f0-9]{2}:[a-f0-9]{2})(\.([a-f0-9]))?$/) {
1339             $found = 1;
1340             if(defined($4)){
1341                 push @{$res->{pciid}}, { id => $2 , function => $4};
1342
1343             }else{
1344                 my $pcidevices = lspci($2);
1345                 $res->{pciid} = $pcidevices->{$2};
1346             }
1347         } elsif ($kv =~ m/^rombar=(on|off)$/) {
1348             $res->{rombar} = $1;
1349         } elsif ($kv =~ m/^x-vga=(on|off)$/) {
1350             $res->{'x-vga'} = $1;
1351         } elsif ($kv =~ m/^pcie=(\d+)$/) {
1352             $res->{pcie} = 1 if $1 == 1;
1353         } else {
1354             warn "unknown hostpci setting '$kv'\n";
1355         }
1356     }
1357
1358     return undef if !$found;
1359
1360     return $res;
1361 }
1362
1363 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1364 sub parse_net {
1365     my ($data) = @_;
1366
1367     my $res = {};
1368
1369     foreach my $kvp (split(/,/, $data)) {
1370
1371         if ($kvp =~ m/^(ne2k_pci|e1000|e1000-82540em|e1000-82544gc|e1000-82545em|rtl8139|pcnet|virtio|ne2k_isa|i82551|i82557b|i82559er|vmxnet3)(=([0-9a-f]{2}(:[0-9a-f]{2}){5}))?$/i) {
1372             my $model = lc($1);
1373             my $mac = defined($3) ? uc($3) : PVE::Tools::random_ether_addr();
1374             $res->{model} = $model;
1375             $res->{macaddr} = $mac;
1376         } elsif ($kvp =~ m/^bridge=(\S+)$/) {
1377             $res->{bridge} = $1;
1378         } elsif ($kvp =~ m/^queues=(\d+)$/) {
1379             $res->{queues} = $1;
1380         } elsif ($kvp =~ m/^rate=(\d+(\.\d+)?)$/) {
1381             $res->{rate} = $1;
1382         } elsif ($kvp =~ m/^tag=(\d+)$/) {
1383             $res->{tag} = $1;
1384         } elsif ($kvp =~ m/^firewall=([01])$/) {
1385             $res->{firewall} = $1;
1386         } elsif ($kvp =~ m/^link_down=([01])$/) {
1387             $res->{link_down} = $1;
1388         } else {
1389             return undef;
1390         }
1391
1392     }
1393
1394     return undef if !$res->{model};
1395
1396     return $res;
1397 }
1398
1399 sub print_net {
1400     my $net = shift;
1401
1402     my $res = "$net->{model}";
1403     $res .= "=$net->{macaddr}" if $net->{macaddr};
1404     $res .= ",bridge=$net->{bridge}" if $net->{bridge};
1405     $res .= ",rate=$net->{rate}" if $net->{rate};
1406     $res .= ",tag=$net->{tag}" if $net->{tag};
1407     $res .= ",firewall=1" if $net->{firewall};
1408     $res .= ",link_down=1" if $net->{link_down};
1409     $res .= ",queues=$net->{queues}" if $net->{queues};
1410
1411     return $res;
1412 }
1413
1414 sub add_random_macs {
1415     my ($settings) = @_;
1416
1417     foreach my $opt (keys %$settings) {
1418         next if $opt !~ m/^net(\d+)$/;
1419         my $net = parse_net($settings->{$opt});
1420         next if !$net;
1421         $settings->{$opt} = print_net($net);
1422     }
1423 }
1424
1425 sub add_unused_volume {
1426     my ($config, $volid) = @_;
1427
1428     my $key;
1429     for (my $ind = $MAX_UNUSED_DISKS - 1; $ind >= 0; $ind--) {
1430         my $test = "unused$ind";
1431         if (my $vid = $config->{$test}) {
1432             return if $vid eq $volid; # do not add duplicates
1433         } else {
1434             $key = $test;
1435         }
1436     }
1437
1438     die "To many unused volume - please delete them first.\n" if !$key;
1439
1440     $config->{$key} = $volid;
1441
1442     return $key;
1443 }
1444
1445 sub vm_is_volid_owner {
1446     my ($storecfg, $vmid, $volid) = @_;
1447
1448     if ($volid !~  m|^/|) {
1449         my ($path, $owner);
1450         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1451         if ($owner && ($owner == $vmid)) {
1452             return 1;
1453         }
1454     }
1455
1456     return undef;
1457 }
1458
1459 sub split_flagged_list {
1460     my $text = shift || '';
1461     $text =~ s/[,;]/ /g;
1462     $text =~ s/^\s+//;
1463     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
1464 }
1465
1466 sub join_flagged_list {
1467     my ($how, $lst) = @_;
1468     join $how, map { $lst->{$_} . $_ } keys %$lst;
1469 }
1470
1471 sub vmconfig_delete_pending_option {
1472     my ($conf, $key, $force) = @_;
1473
1474     delete $conf->{pending}->{$key};
1475     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1476     $pending_delete_hash->{$key} = $force ? '!' : '';
1477     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1478 }
1479
1480 sub vmconfig_undelete_pending_option {
1481     my ($conf, $key) = @_;
1482
1483     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1484     delete $pending_delete_hash->{$key};
1485
1486     if (%$pending_delete_hash) {
1487         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1488     } else {
1489         delete $conf->{pending}->{delete};
1490     }
1491 }
1492
1493 sub vmconfig_register_unused_drive {
1494     my ($storecfg, $vmid, $conf, $drive) = @_;
1495
1496     if (!drive_is_cdrom($drive)) {
1497         my $volid = $drive->{file};
1498         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1499             add_unused_volume($conf, $volid, $vmid);
1500         }
1501     }
1502 }
1503
1504 sub vmconfig_cleanup_pending {
1505     my ($conf) = @_;
1506
1507     # remove pending changes when nothing changed
1508     my $changes;
1509     foreach my $opt (keys %{$conf->{pending}}) {
1510         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
1511             $changes = 1;
1512             delete $conf->{pending}->{$opt};
1513         }
1514     }
1515
1516     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
1517     my $pending_delete_hash = {};
1518     while (my ($opt, $force) = each %$current_delete_hash) {
1519         if (defined($conf->{$opt})) {
1520             $pending_delete_hash->{$opt} = $force;
1521         } else {
1522             $changes = 1;
1523         }
1524     }
1525
1526     if (%$pending_delete_hash) {
1527         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1528     } else {
1529         delete $conf->{pending}->{delete};
1530     }
1531
1532     return $changes;
1533 }
1534
1535 my $valid_smbios1_options = {
1536     manufacturer => '\S+',
1537     product => '\S+',
1538     version => '\S+',
1539     serial => '\S+',
1540     uuid => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1541     sku => '\S+',
1542     family => '\S+',
1543 };
1544
1545 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
1546 sub parse_smbios1 {
1547     my ($data) = @_;
1548
1549     my $res = {};
1550
1551     foreach my $kvp (split(/,/, $data)) {
1552         return undef if $kvp !~ m/^(\S+)=(.+)$/;
1553         my ($k, $v) = split(/=/, $kvp);
1554         return undef if !defined($k) || !defined($v);
1555         return undef if !$valid_smbios1_options->{$k};
1556         return undef if $v !~ m/^$valid_smbios1_options->{$k}$/;
1557         $res->{$k} = $v;
1558     }
1559
1560     return $res;
1561 }
1562
1563 sub print_smbios1 {
1564     my ($smbios1) = @_;
1565
1566     my $data = '';
1567     foreach my $k (keys %$smbios1) {
1568         next if !defined($smbios1->{$k});
1569         next if !$valid_smbios1_options->{$k};
1570         $data .= ',' if $data;
1571         $data .= "$k=$smbios1->{$k}";
1572     }
1573     return $data;
1574 }
1575
1576 PVE::JSONSchema::register_format('pve-qm-smbios1', \&verify_smbios1);
1577 sub verify_smbios1 {
1578     my ($value, $noerr) = @_;
1579
1580     return $value if parse_smbios1($value);
1581
1582     return undef if $noerr;
1583
1584     die "unable to parse smbios (type 1) options\n";
1585 }
1586
1587 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
1588 sub verify_bootdisk {
1589     my ($value, $noerr) = @_;
1590
1591     return $value if valid_drivename($value);
1592
1593     return undef if $noerr;
1594
1595     die "invalid boot disk '$value'\n";
1596 }
1597
1598 PVE::JSONSchema::register_format('pve-qm-numanode', \&verify_numa);
1599 sub verify_numa {
1600     my ($value, $noerr) = @_;
1601
1602     return $value if parse_numa($value);
1603
1604     return undef if $noerr;
1605
1606     die "unable to parse numa options\n";
1607 }
1608
1609 PVE::JSONSchema::register_format('pve-qm-net', \&verify_net);
1610 sub verify_net {
1611     my ($value, $noerr) = @_;
1612
1613     return $value if parse_net($value);
1614
1615     return undef if $noerr;
1616
1617     die "unable to parse network options\n";
1618 }
1619
1620 PVE::JSONSchema::register_format('pve-qm-drive', \&verify_drive);
1621 sub verify_drive {
1622     my ($value, $noerr) = @_;
1623
1624     return $value if parse_drive(undef, $value);
1625
1626     return undef if $noerr;
1627
1628     die "unable to parse drive options\n";
1629 }
1630
1631 PVE::JSONSchema::register_format('pve-qm-hostpci', \&verify_hostpci);
1632 sub verify_hostpci {
1633     my ($value, $noerr) = @_;
1634
1635     return $value if parse_hostpci($value);
1636
1637     return undef if $noerr;
1638
1639     die "unable to parse pci id\n";
1640 }
1641
1642 PVE::JSONSchema::register_format('pve-qm-watchdog', \&verify_watchdog);
1643 sub verify_watchdog {
1644     my ($value, $noerr) = @_;
1645
1646     return $value if parse_watchdog($value);
1647
1648     return undef if $noerr;
1649
1650     die "unable to parse watchdog options\n";
1651 }
1652
1653 sub parse_watchdog {
1654     my ($value) = @_;
1655
1656     return undef if !$value;
1657
1658     my $res = {};
1659
1660     foreach my $p (split(/,/, $value)) {
1661         next if $p =~ m/^\s*$/;
1662
1663         if ($p =~ m/^(model=)?(i6300esb|ib700)$/) {
1664             $res->{model} = $2;
1665         } elsif ($p =~ m/^(action=)?(reset|shutdown|poweroff|pause|debug|none)$/) {
1666             $res->{action} = $2;
1667         } else {
1668             return undef;
1669         }
1670     }
1671
1672     return $res;
1673 }
1674
1675 sub parse_usb_device {
1676     my ($value) = @_;
1677
1678     return undef if !$value;
1679
1680     my @dl = split(/,/, $value);
1681     my $found;
1682
1683     my $res = {};
1684     foreach my $v (@dl) {
1685         if ($v =~ m/^host=(0x)?([0-9A-Fa-f]{4}):(0x)?([0-9A-Fa-f]{4})$/) {
1686             $found = 1;
1687             $res->{vendorid} = $2;
1688             $res->{productid} = $4;
1689         } elsif ($v =~ m/^host=(\d+)\-(\d+(\.\d+)*)$/) {
1690             $found = 1;
1691             $res->{hostbus} = $1;
1692             $res->{hostport} = $2;
1693         } elsif ($v =~ m/^spice$/) {
1694             $found = 1;
1695             $res->{spice} = 1;
1696         } else {
1697             return undef;
1698         }
1699     }
1700     return undef if !$found;
1701
1702     return $res;
1703 }
1704
1705 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
1706 sub verify_usb_device {
1707     my ($value, $noerr) = @_;
1708
1709     return $value if parse_usb_device($value);
1710
1711     return undef if $noerr;
1712
1713     die "unable to parse usb device\n";
1714 }
1715
1716 # add JSON properties for create and set function
1717 sub json_config_properties {
1718     my $prop = shift;
1719
1720     foreach my $opt (keys %$confdesc) {
1721         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
1722         $prop->{$opt} = $confdesc->{$opt};
1723     }
1724
1725     return $prop;
1726 }
1727
1728 sub check_type {
1729     my ($key, $value) = @_;
1730
1731     die "unknown setting '$key'\n" if !$confdesc->{$key};
1732
1733     my $type = $confdesc->{$key}->{type};
1734
1735     if (!defined($value)) {
1736         die "got undefined value\n";
1737     }
1738
1739     if ($value =~ m/[\n\r]/) {
1740         die "property contains a line feed\n";
1741     }
1742
1743     if ($type eq 'boolean') {
1744         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
1745         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
1746         die "type check ('boolean') failed - got '$value'\n";
1747     } elsif ($type eq 'integer') {
1748         return int($1) if $value =~ m/^(\d+)$/;
1749         die "type check ('integer') failed - got '$value'\n";
1750     } elsif ($type eq 'number') {
1751         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
1752         die "type check ('number') failed - got '$value'\n";
1753     } elsif ($type eq 'string') {
1754         if (my $fmt = $confdesc->{$key}->{format}) {
1755             if ($fmt eq 'pve-qm-drive') {
1756                 # special case - we need to pass $key to parse_drive()
1757                 my $drive = parse_drive($key, $value);
1758                 return $value if $drive;
1759                 die "unable to parse drive options\n";
1760             }
1761             PVE::JSONSchema::check_format($fmt, $value);
1762             return $value;
1763         }
1764         $value =~ s/^\"(.*)\"$/$1/;
1765         return $value;
1766     } else {
1767         die "internal error"
1768     }
1769 }
1770
1771 sub lock_config_full {
1772     my ($vmid, $timeout, $code, @param) = @_;
1773
1774     my $filename = config_file_lock($vmid);
1775
1776     my $res = lock_file($filename, $timeout, $code, @param);
1777
1778     die $@ if $@;
1779
1780     return $res;
1781 }
1782
1783 sub lock_config_mode {
1784     my ($vmid, $timeout, $shared, $code, @param) = @_;
1785
1786     my $filename = config_file_lock($vmid);
1787
1788     my $res = lock_file_full($filename, $timeout, $shared, $code, @param);
1789
1790     die $@ if $@;
1791
1792     return $res;
1793 }
1794
1795 sub lock_config {
1796     my ($vmid, $code, @param) = @_;
1797
1798     return lock_config_full($vmid, 10, $code, @param);
1799 }
1800
1801 sub cfs_config_path {
1802     my ($vmid, $node) = @_;
1803
1804     $node = $nodename if !$node;
1805     return "nodes/$node/qemu-server/$vmid.conf";
1806 }
1807
1808 sub check_iommu_support{
1809     #fixme : need to check IOMMU support
1810     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
1811
1812     my $iommu=1;
1813     return $iommu;
1814
1815 }
1816
1817 sub config_file {
1818     my ($vmid, $node) = @_;
1819
1820     my $cfspath = cfs_config_path($vmid, $node);
1821     return "/etc/pve/$cfspath";
1822 }
1823
1824 sub config_file_lock {
1825     my ($vmid) = @_;
1826
1827     return "$lock_dir/lock-$vmid.conf";
1828 }
1829
1830 sub touch_config {
1831     my ($vmid) = @_;
1832
1833     my $conf = config_file($vmid);
1834     utime undef, undef, $conf;
1835 }
1836
1837 sub destroy_vm {
1838     my ($storecfg, $vmid, $keep_empty_config) = @_;
1839
1840     my $conffile = config_file($vmid);
1841
1842     my $conf = load_config($vmid);
1843
1844     check_lock($conf);
1845
1846     # only remove disks owned by this VM
1847     foreach_drive($conf, sub {
1848         my ($ds, $drive) = @_;
1849
1850         return if drive_is_cdrom($drive);
1851
1852         my $volid = $drive->{file};
1853
1854         return if !$volid || $volid =~ m|^/|;
1855
1856         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
1857         return if !$path || !$owner || ($owner != $vmid);
1858
1859         PVE::Storage::vdisk_free($storecfg, $volid);
1860     });
1861
1862     if ($keep_empty_config) {
1863         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
1864     } else {
1865         unlink $conffile;
1866     }
1867
1868     # also remove unused disk
1869     eval {
1870         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
1871
1872         eval {
1873             PVE::Storage::foreach_volid($dl, sub {
1874                 my ($volid, $sid, $volname, $d) = @_;
1875                 PVE::Storage::vdisk_free($storecfg, $volid);
1876             });
1877         };
1878         warn $@ if $@;
1879
1880     };
1881     warn $@ if $@;
1882 }
1883
1884 sub load_config {
1885     my ($vmid, $node) = @_;
1886
1887     my $cfspath = cfs_config_path($vmid, $node);
1888
1889     my $conf = PVE::Cluster::cfs_read_file($cfspath);
1890
1891     die "no such VM ('$vmid')\n" if !defined($conf);
1892
1893     return $conf;
1894 }
1895
1896 sub parse_vm_config {
1897     my ($filename, $raw) = @_;
1898
1899     return undef if !defined($raw);
1900
1901     my $res = {
1902         digest => Digest::SHA::sha1_hex($raw),
1903         snapshots => {},
1904         pending => {},
1905     };
1906
1907     $filename =~ m|/qemu-server/(\d+)\.conf$|
1908         || die "got strange filename '$filename'";
1909
1910     my $vmid = $1;
1911
1912     my $conf = $res;
1913     my $descr;
1914     my $section = '';
1915
1916     my @lines = split(/\n/, $raw);
1917     foreach my $line (@lines) {
1918         next if $line =~ m/^\s*$/;
1919
1920         if ($line =~ m/^\[PENDING\]\s*$/i) {
1921             $section = 'pending';
1922             if (defined($descr)) {
1923                 $descr =~ s/\s+$//;
1924                 $conf->{description} = $descr;
1925             }
1926             $descr = undef;
1927             $conf = $res->{$section} = {};
1928             next;
1929
1930         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
1931             $section = $1;
1932             if (defined($descr)) {
1933                 $descr =~ s/\s+$//;
1934                 $conf->{description} = $descr;
1935             }
1936             $descr = undef;
1937             $conf = $res->{snapshots}->{$section} = {};
1938             next;
1939         }
1940
1941         if ($line =~ m/^\#(.*)\s*$/) {
1942             $descr = '' if !defined($descr);
1943             $descr .= PVE::Tools::decode_text($1) . "\n";
1944             next;
1945         }
1946
1947         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
1948             $descr = '' if !defined($descr);
1949             $descr .= PVE::Tools::decode_text($2);
1950         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
1951             $conf->{snapstate} = $1;
1952         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
1953             my $key = $1;
1954             my $value = $2;
1955             $conf->{$key} = $value;
1956         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
1957             my $value = $1;
1958             if ($section eq 'pending') {
1959                 $conf->{delete} = $value; # we parse this later
1960             } else {
1961                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
1962             }
1963         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
1964             my $key = $1;
1965             my $value = $2;
1966             eval { $value = check_type($key, $value); };
1967             if ($@) {
1968                 warn "vm $vmid - unable to parse value of '$key' - $@";
1969             } else {
1970                 my $fmt = $confdesc->{$key}->{format};
1971                 if ($fmt && $fmt eq 'pve-qm-drive') {
1972                     my $v = parse_drive($key, $value);
1973                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
1974                         $v->{file} = $volid;
1975                         $value = print_drive($vmid, $v);
1976                     } else {
1977                         warn "vm $vmid - unable to parse value of '$key'\n";
1978                         next;
1979                     }
1980                 }
1981
1982                 if ($key eq 'cdrom') {
1983                     $conf->{ide2} = $value;
1984                 } else {
1985                     $conf->{$key} = $value;
1986                 }
1987             }
1988         }
1989     }
1990
1991     if (defined($descr)) {
1992         $descr =~ s/\s+$//;
1993         $conf->{description} = $descr;
1994     }
1995     delete $res->{snapstate}; # just to be sure
1996
1997     return $res;
1998 }
1999
2000 sub write_vm_config {
2001     my ($filename, $conf) = @_;
2002
2003     delete $conf->{snapstate}; # just to be sure
2004
2005     if ($conf->{cdrom}) {
2006         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2007         $conf->{ide2} = $conf->{cdrom};
2008         delete $conf->{cdrom};
2009     }
2010
2011     # we do not use 'smp' any longer
2012     if ($conf->{sockets}) {
2013         delete $conf->{smp};
2014     } elsif ($conf->{smp}) {
2015         $conf->{sockets} = $conf->{smp};
2016         delete $conf->{cores};
2017         delete $conf->{smp};
2018     }
2019
2020     my $used_volids = {};
2021
2022     my $cleanup_config = sub {
2023         my ($cref, $pending, $snapname) = @_;
2024
2025         foreach my $key (keys %$cref) {
2026             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2027                 $key eq 'snapstate' || $key eq 'pending';
2028             my $value = $cref->{$key};
2029             if ($key eq 'delete') {
2030                 die "propertry 'delete' is only allowed in [PENDING]\n"
2031                     if !$pending;
2032                 # fixme: check syntax?
2033                 next;
2034             }
2035             eval { $value = check_type($key, $value); };
2036             die "unable to parse value of '$key' - $@" if $@;
2037
2038             $cref->{$key} = $value;
2039
2040             if (!$snapname && valid_drivename($key)) {
2041                 my $drive = parse_drive($key, $value);
2042                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2043             }
2044         }
2045     };
2046
2047     &$cleanup_config($conf);
2048
2049     &$cleanup_config($conf->{pending}, 1);
2050
2051     foreach my $snapname (keys %{$conf->{snapshots}}) {
2052         die "internal error" if $snapname eq 'pending';
2053         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2054     }
2055
2056     # remove 'unusedX' settings if we re-add a volume
2057     foreach my $key (keys %$conf) {
2058         my $value = $conf->{$key};
2059         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2060             delete $conf->{$key};
2061         }
2062     }
2063
2064     my $generate_raw_config = sub {
2065         my ($conf, $pending) = @_;
2066
2067         my $raw = '';
2068
2069         # add description as comment to top of file
2070         if (defined(my $descr = $conf->{description})) {
2071             if ($descr) {
2072                 foreach my $cl (split(/\n/, $descr)) {
2073                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2074                 }
2075             } else {
2076                 $raw .= "#\n" if $pending;
2077             }
2078         }
2079
2080         foreach my $key (sort keys %$conf) {
2081             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2082             $raw .= "$key: $conf->{$key}\n";
2083         }
2084         return $raw;
2085     };
2086
2087     my $raw = &$generate_raw_config($conf);
2088
2089     if (scalar(keys %{$conf->{pending}})){
2090         $raw .= "\n[PENDING]\n";
2091         $raw .= &$generate_raw_config($conf->{pending}, 1);
2092     }
2093
2094     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2095         $raw .= "\n[$snapname]\n";
2096         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2097     }
2098
2099     return $raw;
2100 }
2101
2102 sub update_config_nolock {
2103     my ($vmid, $conf, $skiplock) = @_;
2104
2105     check_lock($conf) if !$skiplock;
2106
2107     my $cfspath = cfs_config_path($vmid);
2108
2109     PVE::Cluster::cfs_write_file($cfspath, $conf);
2110 }
2111
2112 sub update_config {
2113     my ($vmid, $conf, $skiplock) = @_;
2114
2115     lock_config($vmid, &update_config_nolock, $conf, $skiplock);
2116 }
2117
2118 sub load_defaults {
2119
2120     my $res = {};
2121
2122     # we use static defaults from our JSON schema configuration
2123     foreach my $key (keys %$confdesc) {
2124         if (defined(my $default = $confdesc->{$key}->{default})) {
2125             $res->{$key} = $default;
2126         }
2127     }
2128
2129     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2130     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2131
2132     return $res;
2133 }
2134
2135 sub config_list {
2136     my $vmlist = PVE::Cluster::get_vmlist();
2137     my $res = {};
2138     return $res if !$vmlist || !$vmlist->{ids};
2139     my $ids = $vmlist->{ids};
2140
2141     foreach my $vmid (keys %$ids) {
2142         my $d = $ids->{$vmid};
2143         next if !$d->{node} || $d->{node} ne $nodename;
2144         next if !$d->{type} || $d->{type} ne 'qemu';
2145         $res->{$vmid}->{exists} = 1;
2146     }
2147     return $res;
2148 }
2149
2150 # test if VM uses local resources (to prevent migration)
2151 sub check_local_resources {
2152     my ($conf, $noerr) = @_;
2153
2154     my $loc_res = 0;
2155
2156     $loc_res = 1 if $conf->{hostusb}; # old syntax
2157     $loc_res = 1 if $conf->{hostpci}; # old syntax
2158
2159     foreach my $k (keys %$conf) {
2160         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2161         # sockets are safe: they will recreated be on the target side post-migrate
2162         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2163         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2164     }
2165
2166     die "VM uses local resources\n" if $loc_res && !$noerr;
2167
2168     return $loc_res;
2169 }
2170
2171 # check if used storages are available on all nodes (use by migrate)
2172 sub check_storage_availability {
2173     my ($storecfg, $conf, $node) = @_;
2174
2175     foreach_drive($conf, sub {
2176         my ($ds, $drive) = @_;
2177
2178         my $volid = $drive->{file};
2179         return if !$volid;
2180
2181         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2182         return if !$sid;
2183
2184         # check if storage is available on both nodes
2185         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2186         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2187    });
2188 }
2189
2190 # list nodes where all VM images are available (used by has_feature API)
2191 sub shared_nodes {
2192     my ($conf, $storecfg) = @_;
2193
2194     my $nodelist = PVE::Cluster::get_nodelist();
2195     my $nodehash = { map { $_ => 1 } @$nodelist };
2196     my $nodename = PVE::INotify::nodename();
2197
2198     foreach_drive($conf, sub {
2199         my ($ds, $drive) = @_;
2200
2201         my $volid = $drive->{file};
2202         return if !$volid;
2203
2204         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2205         if ($storeid) {
2206             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2207             if ($scfg->{disable}) {
2208                 $nodehash = {};
2209             } elsif (my $avail = $scfg->{nodes}) {
2210                 foreach my $node (keys %$nodehash) {
2211                     delete $nodehash->{$node} if !$avail->{$node};
2212                 }
2213             } elsif (!$scfg->{shared}) {
2214                 foreach my $node (keys %$nodehash) {
2215                     delete $nodehash->{$node} if $node ne $nodename
2216                 }
2217             }
2218         }
2219     });
2220
2221     return $nodehash
2222 }
2223
2224 sub check_lock {
2225     my ($conf) = @_;
2226
2227     die "VM is locked ($conf->{lock})\n" if $conf->{lock};
2228 }
2229
2230 sub check_cmdline {
2231     my ($pidfile, $pid) = @_;
2232
2233     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2234     if (defined($fh)) {
2235         my $line = <$fh>;
2236         $fh->close;
2237         return undef if !$line;
2238         my @param = split(/\0/, $line);
2239
2240         my $cmd = $param[0];
2241         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2242
2243         for (my $i = 0; $i < scalar (@param); $i++) {
2244             my $p = $param[$i];
2245             next if !$p;
2246             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2247                 my $p = $param[$i+1];
2248                 return 1 if $p && ($p eq $pidfile);
2249                 return undef;
2250             }
2251         }
2252     }
2253     return undef;
2254 }
2255
2256 sub check_running {
2257     my ($vmid, $nocheck, $node) = @_;
2258
2259     my $filename = config_file($vmid, $node);
2260
2261     die "unable to find configuration file for VM $vmid - no such machine\n"
2262         if !$nocheck && ! -f $filename;
2263
2264     my $pidfile = pidfile_name($vmid);
2265
2266     if (my $fd = IO::File->new("<$pidfile")) {
2267         my $st = stat($fd);
2268         my $line = <$fd>;
2269         close($fd);
2270
2271         my $mtime = $st->mtime;
2272         if ($mtime > time()) {
2273             warn "file '$filename' modified in future\n";
2274         }
2275
2276         if ($line =~ m/^(\d+)$/) {
2277             my $pid = $1;
2278             if (check_cmdline($pidfile, $pid)) {
2279                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2280                     return $pid;
2281                 }
2282             }
2283         }
2284     }
2285
2286     return undef;
2287 }
2288
2289 sub vzlist {
2290
2291     my $vzlist = config_list();
2292
2293     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2294
2295     while (defined(my $de = $fd->read)) {
2296         next if $de !~ m/^(\d+)\.pid$/;
2297         my $vmid = $1;
2298         next if !defined($vzlist->{$vmid});
2299         if (my $pid = check_running($vmid)) {
2300             $vzlist->{$vmid}->{pid} = $pid;
2301         }
2302     }
2303
2304     return $vzlist;
2305 }
2306
2307 sub disksize {
2308     my ($storecfg, $conf) = @_;
2309
2310     my $bootdisk = $conf->{bootdisk};
2311     return undef if !$bootdisk;
2312     return undef if !valid_drivename($bootdisk);
2313
2314     return undef if !$conf->{$bootdisk};
2315
2316     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2317     return undef if !defined($drive);
2318
2319     return undef if drive_is_cdrom($drive);
2320
2321     my $volid = $drive->{file};
2322     return undef if !$volid;
2323
2324     return $drive->{size};
2325 }
2326
2327 my $last_proc_pid_stat;
2328
2329 # get VM status information
2330 # This must be fast and should not block ($full == false)
2331 # We only query KVM using QMP if $full == true (this can be slow)
2332 sub vmstatus {
2333     my ($opt_vmid, $full) = @_;
2334
2335     my $res = {};
2336
2337     my $storecfg = PVE::Storage::config();
2338
2339     my $list = vzlist();
2340     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2341
2342     my $cpucount = $cpuinfo->{cpus} || 1;
2343
2344     foreach my $vmid (keys %$list) {
2345         next if $opt_vmid && ($vmid ne $opt_vmid);
2346
2347         my $cfspath = cfs_config_path($vmid);
2348         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2349
2350         my $d = {};
2351         $d->{pid} = $list->{$vmid}->{pid};
2352
2353         # fixme: better status?
2354         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2355
2356         my $size = disksize($storecfg, $conf);
2357         if (defined($size)) {
2358             $d->{disk} = 0; # no info available
2359             $d->{maxdisk} = $size;
2360         } else {
2361             $d->{disk} = 0;
2362             $d->{maxdisk} = 0;
2363         }
2364
2365         $d->{cpus} = ($conf->{sockets} || 1) * ($conf->{cores} || 1);
2366         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2367         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2368
2369         $d->{name} = $conf->{name} || "VM $vmid";
2370         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024) : 0;
2371
2372         if ($conf->{balloon}) {
2373             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2374             $d->{shares} = defined($conf->{shares}) ? $conf->{shares} : 1000;
2375         }
2376
2377         $d->{uptime} = 0;
2378         $d->{cpu} = 0;
2379         $d->{mem} = 0;
2380
2381         $d->{netout} = 0;
2382         $d->{netin} = 0;
2383
2384         $d->{diskread} = 0;
2385         $d->{diskwrite} = 0;
2386
2387         $d->{template} = is_template($conf);
2388
2389         $res->{$vmid} = $d;
2390     }
2391
2392     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2393     foreach my $dev (keys %$netdev) {
2394         next if $dev !~ m/^tap([1-9]\d*)i/;
2395         my $vmid = $1;
2396         my $d = $res->{$vmid};
2397         next if !$d;
2398
2399         $d->{netout} += $netdev->{$dev}->{receive};
2400         $d->{netin} += $netdev->{$dev}->{transmit};
2401
2402         if ($full) {
2403             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2404             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2405         }
2406
2407     }
2408
2409     my $ctime = gettimeofday;
2410
2411     foreach my $vmid (keys %$list) {
2412
2413         my $d = $res->{$vmid};
2414         my $pid = $d->{pid};
2415         next if !$pid;
2416
2417         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2418         next if !$pstat; # not running
2419
2420         my $used = $pstat->{utime} + $pstat->{stime};
2421
2422         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2423
2424         if ($pstat->{vsize}) {
2425             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2426         }
2427
2428         my $old = $last_proc_pid_stat->{$pid};
2429         if (!$old) {
2430             $last_proc_pid_stat->{$pid} = {
2431                 time => $ctime,
2432                 used => $used,
2433                 cpu => 0,
2434             };
2435             next;
2436         }
2437
2438         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2439
2440         if ($dtime > 1000) {
2441             my $dutime = $used -  $old->{used};
2442
2443             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2444             $last_proc_pid_stat->{$pid} = {
2445                 time => $ctime,
2446                 used => $used,
2447                 cpu => $d->{cpu},
2448             };
2449         } else {
2450             $d->{cpu} = $old->{cpu};
2451         }
2452     }
2453
2454     return $res if !$full;
2455
2456     my $qmpclient = PVE::QMPClient->new();
2457
2458     my $ballooncb = sub {
2459         my ($vmid, $resp) = @_;
2460
2461         my $info = $resp->{'return'};
2462         return if !$info->{max_mem};
2463
2464         my $d = $res->{$vmid};
2465
2466         # use memory assigned to VM
2467         $d->{maxmem} = $info->{max_mem};
2468         $d->{balloon} = $info->{actual};
2469
2470         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2471             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2472             $d->{freemem} = $info->{free_mem};
2473         }
2474
2475         $d->{ballooninfo} = $info;
2476     };
2477
2478     my $blockstatscb = sub {
2479         my ($vmid, $resp) = @_;
2480         my $data = $resp->{'return'} || [];
2481         my $totalrdbytes = 0;
2482         my $totalwrbytes = 0;
2483
2484         for my $blockstat (@$data) {
2485             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2486             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2487
2488             $blockstat->{device} =~ s/drive-//;
2489             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2490         }
2491         $res->{$vmid}->{diskread} = $totalrdbytes;
2492         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2493     };
2494
2495     my $statuscb = sub {
2496         my ($vmid, $resp) = @_;
2497
2498         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2499         # this fails if ballon driver is not loaded, so this must be
2500         # the last commnand (following command are aborted if this fails).
2501         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2502
2503         my $status = 'unknown';
2504         if (!defined($status = $resp->{'return'}->{status})) {
2505             warn "unable to get VM status\n";
2506             return;
2507         }
2508
2509         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2510     };
2511
2512     foreach my $vmid (keys %$list) {
2513         next if $opt_vmid && ($vmid ne $opt_vmid);
2514         next if !$res->{$vmid}->{pid}; # not running
2515         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2516     }
2517
2518     $qmpclient->queue_execute(undef, 1);
2519
2520     foreach my $vmid (keys %$list) {
2521         next if $opt_vmid && ($vmid ne $opt_vmid);
2522         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2523     }
2524
2525     return $res;
2526 }
2527
2528 sub foreach_dimm {
2529     my ($conf, $vmid, $memory, $sockets, $func) = @_;
2530
2531     my $dimm_id = 0;
2532     my $current_size = 1024;
2533     my $dimm_size = 512;
2534     return if $current_size == $memory;
2535
2536     for (my $j = 0; $j < 8; $j++) {
2537         for (my $i = 0; $i < 32; $i++) {
2538             my $name = "dimm${dimm_id}";
2539             $dimm_id++;
2540             my $numanode = $i % $sockets;
2541             $current_size += $dimm_size;
2542             &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
2543             return  $current_size if $current_size >= $memory;
2544         }
2545         $dimm_size *= 2;
2546     }
2547 }
2548
2549 sub foreach_reverse_dimm {
2550     my ($conf, $vmid, $memory, $sockets, $func) = @_;
2551
2552     my $dimm_id = 253;
2553     my $current_size = 4177920;
2554     my $dimm_size = 65536;
2555     return if $current_size == $memory;
2556
2557     for (my $j = 0; $j < 8; $j++) {
2558         for (my $i = 0; $i < 32; $i++) {
2559             my $name = "dimm${dimm_id}";
2560             $dimm_id--;
2561             my $numanode = $i % $sockets;
2562             $current_size -= $dimm_size;
2563             &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
2564             return  $current_size if $current_size <= $memory;
2565         }
2566         $dimm_size /= 2;
2567     }
2568 }
2569
2570 sub foreach_drive {
2571     my ($conf, $func) = @_;
2572
2573     foreach my $ds (keys %$conf) {
2574         next if !valid_drivename($ds);
2575
2576         my $drive = parse_drive($ds, $conf->{$ds});
2577         next if !$drive;
2578
2579         &$func($ds, $drive);
2580     }
2581 }
2582
2583 sub foreach_volid {
2584     my ($conf, $func) = @_;
2585
2586     my $volhash = {};
2587
2588     my $test_volid = sub {
2589         my ($volid, $is_cdrom) = @_;
2590
2591         return if !$volid;
2592
2593         $volhash->{$volid} = $is_cdrom || 0;
2594     };
2595
2596     foreach_drive($conf, sub {
2597         my ($ds, $drive) = @_;
2598         &$test_volid($drive->{file}, drive_is_cdrom($drive));
2599     });
2600
2601     foreach my $snapname (keys %{$conf->{snapshots}}) {
2602         my $snap = $conf->{snapshots}->{$snapname};
2603         &$test_volid($snap->{vmstate}, 0);
2604         foreach_drive($snap, sub {
2605             my ($ds, $drive) = @_;
2606             &$test_volid($drive->{file}, drive_is_cdrom($drive));
2607         });
2608     }
2609
2610     foreach my $volid (keys %$volhash) {
2611         &$func($volid, $volhash->{$volid});
2612     }
2613 }
2614
2615 sub vga_conf_has_spice {
2616     my ($vga) = @_;
2617
2618     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
2619
2620     return $1 || 1;
2621 }
2622
2623 sub config_to_command {
2624     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2625
2626     my $cmd = [];
2627     my $globalFlags = [];
2628     my $machineFlags = [];
2629     my $rtcFlags = [];
2630     my $cpuFlags = [];
2631     my $devices = [];
2632     my $pciaddr = '';
2633     my $bridges = {};
2634     my $kvmver = kvm_user_version();
2635     my $vernum = 0; # unknown
2636     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
2637         $vernum = $1*1000000+$2*1000;
2638     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
2639         $vernum = $1*1000000+$2*1000+$3;
2640     }
2641
2642     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
2643
2644     my $have_ovz = -f '/proc/vz/vestat';
2645
2646     my $q35 = machine_type_is_q35($conf);
2647     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2648     my $machine_type = $forcemachine || $conf->{machine};
2649     my $use_old_bios_files = undef;
2650     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
2651
2652     my $cpuunits = defined($conf->{cpuunits}) ?
2653             $conf->{cpuunits} : $defaults->{cpuunits};
2654
2655     push @$cmd, '/usr/bin/systemd-run';
2656     push @$cmd, '--scope';
2657     push @$cmd, '--slice', "qemu";
2658     push @$cmd, '--unit', $vmid;
2659     # set KillMode=none, so that systemd don't kill those scopes
2660     # at shutdown (pve-manager service should stop the VMs instead)
2661     push @$cmd, '-p', "KillMode=none";
2662     push @$cmd, '-p', "CPUShares=$cpuunits";
2663     if ($conf->{cpulimit}) {
2664         my $cpulimit = int($conf->{cpulimit} * 100);
2665         push @$cmd, '-p', "CPUQuota=$cpulimit\%";
2666     }
2667
2668     push @$cmd, '/usr/bin/kvm';
2669
2670     push @$cmd, '-id', $vmid;
2671
2672     my $use_virtio = 0;
2673
2674     my $qmpsocket = qmp_socket($vmid);
2675     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2676     push @$cmd, '-mon', "chardev=qmp,mode=control";
2677
2678     my $socket = vnc_socket($vmid);
2679     push @$cmd,  '-vnc', "unix:$socket,x509,password";
2680
2681     push @$cmd, '-pidfile' , pidfile_name($vmid);
2682
2683     push @$cmd, '-daemonize';
2684
2685     if ($conf->{smbios1}) {
2686         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
2687     }
2688
2689     if ($q35) {
2690         # the q35 chipset support native usb2, so we enable usb controller
2691         # by default for this machine type
2692         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
2693     } else {
2694         $pciaddr = print_pci_addr("piix3", $bridges);
2695         push @$devices, '-device', "piix3-usb-uhci,id=uhci$pciaddr.0x2";
2696
2697         my $use_usb2 = 0;
2698         for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2699             next if !$conf->{"usb$i"};
2700             $use_usb2 = 1;
2701         }
2702         # include usb device config
2703         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-usb.cfg' if $use_usb2;
2704     }
2705
2706     my $vga = $conf->{vga};
2707
2708     my $qxlnum = vga_conf_has_spice($vga);
2709     $vga = 'qxl' if $qxlnum;
2710
2711     if (!$vga) {
2712         if ($conf->{ostype} && ($conf->{ostype} eq 'win8' ||
2713                                 $conf->{ostype} eq 'win7' ||
2714                                 $conf->{ostype} eq 'w2k8')) {
2715             $vga = 'std';
2716         } else {
2717             $vga = 'cirrus';
2718         }
2719     }
2720
2721     # enable absolute mouse coordinates (needed by vnc)
2722     my $tablet;
2723     if (defined($conf->{tablet})) {
2724         $tablet = $conf->{tablet};
2725     } else {
2726         $tablet = $defaults->{tablet};
2727         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
2728         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
2729     }
2730
2731     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
2732
2733     # host pci devices
2734     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
2735         my $d = parse_hostpci($conf->{"hostpci$i"});
2736         next if !$d;
2737
2738         my $pcie = $d->{pcie};
2739         if($pcie){
2740             die "q35 machine model is not enabled" if !$q35;
2741             $pciaddr = print_pcie_addr("hostpci$i");
2742         }else{
2743             $pciaddr = print_pci_addr("hostpci$i", $bridges);
2744         }
2745
2746         my $rombar = $d->{rombar} && $d->{rombar} eq 'off' ? ",rombar=0" : "";
2747         my $xvga = $d->{'x-vga'} && $d->{'x-vga'} eq 'on' ? ",x-vga=on" : "";
2748         if ($xvga && $xvga ne '') {
2749             push @$cpuFlags, 'kvm=off';
2750             $vga = 'none';
2751         }
2752         my $pcidevices = $d->{pciid};
2753         my $multifunction = 1 if @$pcidevices > 1;
2754
2755         my $j=0;
2756         foreach my $pcidevice (@$pcidevices) {
2757
2758             my $id = "hostpci$i";
2759             $id .= ".$j" if $multifunction;
2760             my $addr = $pciaddr;
2761             $addr .= ".$j" if $multifunction;
2762             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
2763
2764             if($j == 0){
2765                 $devicestr .= "$rombar$xvga";
2766                 $devicestr .= ",multifunction=on" if $multifunction;
2767             }
2768
2769             push @$devices, '-device', $devicestr;
2770             $j++;
2771         }
2772     }
2773
2774     # usb devices
2775     for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2776         my $d = parse_usb_device($conf->{"usb$i"});
2777         next if !$d;
2778         if ($d->{vendorid} && $d->{productid}) {
2779             push @$devices, '-device', "usb-host,vendorid=0x$d->{vendorid},productid=0x$d->{productid}";
2780         } elsif (defined($d->{hostbus}) && defined($d->{hostport})) {
2781             push @$devices, '-device', "usb-host,hostbus=$d->{hostbus},hostport=$d->{hostport}";
2782         } elsif ($d->{spice}) {
2783             # usb redir support for spice
2784             push @$devices, '-chardev', "spicevmc,id=usbredirchardev$i,name=usbredir";
2785             push @$devices, '-device', "usb-redir,chardev=usbredirchardev$i,id=usbredirdev$i,bus=ehci.0";
2786         }
2787     }
2788
2789     # serial devices
2790     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2791         if (my $path = $conf->{"serial$i"}) {
2792             if ($path eq 'socket') {
2793                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
2794                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
2795                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2796             } else {
2797                 die "no such serial device\n" if ! -c $path;
2798                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
2799                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2800             }
2801         }
2802     }
2803
2804     # parallel devices
2805     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
2806         if (my $path = $conf->{"parallel$i"}) {
2807             die "no such parallel device\n" if ! -c $path;
2808             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
2809             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
2810             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
2811         }
2812     }
2813
2814     my $vmname = $conf->{name} || "vm$vmid";
2815
2816     push @$cmd, '-name', $vmname;
2817
2818     my $sockets = 1;
2819     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
2820     $sockets = $conf->{sockets} if  $conf->{sockets};
2821
2822     my $cores = $conf->{cores} || 1;
2823
2824     my $maxcpus = $sockets * $cores;
2825
2826     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
2827
2828     my $allowed_vcpus = $cpuinfo->{cpus};
2829
2830     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
2831         if ($allowed_vcpus < $maxcpus);
2832
2833     push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
2834
2835     push @$cmd, '-nodefaults';
2836
2837     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2838
2839     my $bootindex_hash = {};
2840     my $i = 1;
2841     foreach my $o (split(//, $bootorder)) {
2842         $bootindex_hash->{$o} = $i*100;
2843         $i++;
2844     }
2845
2846     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000";
2847
2848     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
2849
2850     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
2851
2852     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
2853
2854     # time drift fix
2855     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
2856
2857     my $nokvm = defined($conf->{kvm}) && $conf->{kvm} == 0 ? 1 : 0;
2858     my $useLocaltime = $conf->{localtime};
2859
2860     if (my $ost = $conf->{ostype}) {
2861         # other, wxp, w2k, w2k3, w2k8, wvista, win7, win8, l24, l26, solaris
2862
2863         if ($ost =~ m/^w/) { # windows
2864             $useLocaltime = 1 if !defined($conf->{localtime});
2865
2866             # use time drift fix when acpi is enabled
2867             if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
2868                 $tdf = 1 if !defined($conf->{tdf});
2869             }
2870         }
2871
2872         if ($ost eq 'win7' || $ost eq 'win8' || $ost eq 'w2k8' ||
2873             $ost eq 'wvista') {
2874             push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
2875             push @$cmd, '-no-hpet';
2876             if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
2877                 push @$cpuFlags , 'hv_spinlocks=0x1fff' if !$nokvm;
2878                 push @$cpuFlags , 'hv_vapic' if !$nokvm;
2879                 push @$cpuFlags , 'hv_time' if !$nokvm;
2880
2881             } else {
2882                 push @$cpuFlags , 'hv_spinlocks=0xffff' if !$nokvm;
2883             }
2884         }
2885
2886         if ($ost eq 'win7' || $ost eq 'win8') {
2887             push @$cpuFlags , 'hv_relaxed' if !$nokvm;
2888         }
2889     }
2890
2891     push @$rtcFlags, 'driftfix=slew' if $tdf;
2892
2893     if ($nokvm) {
2894         push @$machineFlags, 'accel=tcg';
2895     } else {
2896         die "No accelerator found!\n" if !$cpuinfo->{hvm};
2897     }
2898
2899     if ($machine_type) {
2900         push @$machineFlags, "type=${machine_type}";
2901     }
2902
2903     if ($conf->{startdate}) {
2904         push @$rtcFlags, "base=$conf->{startdate}";
2905     } elsif ($useLocaltime) {
2906         push @$rtcFlags, 'base=localtime';
2907     }
2908
2909     my $cpu = $nokvm ? "qemu64" : "kvm64";
2910     $cpu = $conf->{cpu} if $conf->{cpu};
2911
2912     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
2913
2914     push @$cpuFlags , '-x2apic'
2915         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
2916
2917     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
2918
2919     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
2920
2921     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
2922
2923         push @$cpuFlags , '+kvm_pv_unhalt' if !$nokvm;
2924         push @$cpuFlags , '+kvm_pv_eoi' if !$nokvm;
2925     }
2926
2927     push @$cpuFlags, 'enforce' if $cpu ne 'host' && !$nokvm;
2928
2929     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
2930
2931     push @$cmd, '-cpu', $cpu;
2932
2933     my $memory = $conf->{memory} || $defaults->{memory};
2934     my $static_memory = 0;
2935     my $dimm_memory = 0;
2936
2937     if ($hotplug_features->{memory}) {
2938         die "Numa need to be enabled for memory hotplug\n" if !$conf->{numa};
2939         die "Total memory is bigger than ${MAX_MEM}MB\n" if $memory > $MAX_MEM;
2940         $static_memory = $STATICMEM;
2941         die "minimum memory must be ${static_memory}MB\n" if($memory < $static_memory);
2942         $dimm_memory = $memory - $static_memory;
2943         push @$cmd, '-m', "size=${static_memory},slots=255,maxmem=${MAX_MEM}M";
2944
2945     } else {
2946
2947         $static_memory = $memory;
2948         push @$cmd, '-m', $static_memory;
2949     }
2950
2951     if ($conf->{numa}) {
2952
2953         my $numa_totalmemory = undef;
2954         for (my $i = 0; $i < $MAX_NUMA; $i++) {
2955             next if !$conf->{"numa$i"};
2956             my $numa = parse_numa($conf->{"numa$i"});
2957             next if !$numa;
2958             # memory
2959             die "missing numa node$i memory value\n" if !$numa->{memory};
2960             my $numa_memory = $numa->{memory};
2961             $numa_totalmemory += $numa_memory;
2962             my $numa_object = "memory-backend-ram,id=ram-node$i,size=${numa_memory}M";
2963
2964             # cpus
2965             my $cpus_start = $numa->{cpus}->{start};
2966             die "missing numa node$i cpus\n" if !defined($cpus_start);
2967             my $cpus_end = $numa->{cpus}->{end} if defined($numa->{cpus}->{end});
2968             my $cpus = $cpus_start;
2969             if (defined($cpus_end)) {
2970                 $cpus .= "-$cpus_end";
2971                 die "numa node$i :  cpu range $cpus is incorrect\n" if $cpus_end <= $cpus_start;
2972             }
2973
2974             # hostnodes
2975             my $hostnodes_start = $numa->{hostnodes}->{start};
2976             if (defined($hostnodes_start)) {
2977                 my $hostnodes_end = $numa->{hostnodes}->{end} if defined($numa->{hostnodes}->{end});
2978                 my $hostnodes = $hostnodes_start;
2979                 if (defined($hostnodes_end)) {
2980                     $hostnodes .= "-$hostnodes_end";
2981                     die "host node $hostnodes range is incorrect\n" if $hostnodes_end <= $hostnodes_start;
2982                 }
2983
2984                 my $hostnodes_end_range = defined($hostnodes_end) ? $hostnodes_end : $hostnodes_start;
2985                 for (my $i = $hostnodes_start; $i <= $hostnodes_end_range; $i++ ) {
2986                     die "host numa node$i don't exist\n" if ! -d "/sys/devices/system/node/node$i/";
2987                 }
2988
2989                 # policy
2990                 my $policy = $numa->{policy};
2991                 die "you need to define a policy for hostnode $hostnodes\n" if !$policy;
2992                 $numa_object .= ",host-nodes=$hostnodes,policy=$policy";
2993             }
2994
2995             push @$cmd, '-object', $numa_object;
2996             push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
2997         }
2998
2999         die "total memory for NUMA nodes must be equal to vm static memory\n"
3000             if $numa_totalmemory && $numa_totalmemory != $static_memory;
3001
3002         #if no custom tology, we split memory and cores across numa nodes
3003         if(!$numa_totalmemory) {
3004
3005             my $numa_memory = ($static_memory / $sockets) . "M";
3006
3007             for (my $i = 0; $i < $sockets; $i++)  {
3008
3009                 my $cpustart = ($cores * $i);
3010                 my $cpuend = ($cpustart + $cores - 1) if $cores && $cores > 1;
3011                 my $cpus = $cpustart;
3012                 $cpus .= "-$cpuend" if $cpuend;
3013
3014                 push @$cmd, '-object', "memory-backend-ram,size=$numa_memory,id=ram-node$i";
3015                 push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
3016             }
3017         }
3018     }
3019
3020     if ($hotplug_features->{memory}) {
3021         foreach_dimm($conf, $vmid, $memory, $sockets, sub {
3022             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3023             push @$cmd, "-object" , "memory-backend-ram,id=mem-$name,size=${dimm_size}M";
3024             push @$cmd, "-device", "pc-dimm,id=$name,memdev=mem-$name,node=$numanode";
3025
3026             #if dimm_memory is not aligned to dimm map
3027             if($current_size > $memory) {
3028                  $conf->{memory} = $current_size;
3029                  update_config_nolock($vmid, $conf, 1);
3030             }
3031         });
3032     }
3033
3034     push @$cmd, '-S' if $conf->{freeze};
3035
3036     # set keyboard layout
3037     my $kb = $conf->{keyboard} || $defaults->{keyboard};
3038     push @$cmd, '-k', $kb if $kb;
3039
3040     # enable sound
3041     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3042     #push @$cmd, '-soundhw', 'es1370';
3043     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3044
3045     if($conf->{agent}) {
3046         my $qgasocket = qmp_socket($vmid, 1);
3047         my $pciaddr = print_pci_addr("qga0", $bridges);
3048         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3049         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3050         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3051     }
3052
3053     my $spice_port;
3054
3055     if ($qxlnum) {
3056         if ($qxlnum > 1) {
3057             if ($conf->{ostype} && $conf->{ostype} =~ m/^w/){
3058                 for(my $i = 1; $i < $qxlnum; $i++){
3059                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3060                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3061                 }
3062             } else {
3063                 # assume other OS works like Linux
3064                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3065                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3066             }
3067         }
3068
3069         my $pciaddr = print_pci_addr("spice", $bridges);
3070
3071         my $nodename = PVE::INotify::nodename();
3072         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3073         $spice_port = PVE::Tools::next_spice_port($pfamily);
3074
3075         push @$devices, '-spice', "tls-port=${spice_port},addr=localhost,tls-ciphers=DES-CBC3-SHA,seamless-migration=on";
3076
3077         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3078         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3079         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3080     }
3081
3082     # enable balloon by default, unless explicitly disabled
3083     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3084         $pciaddr = print_pci_addr("balloon0", $bridges);
3085         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3086     }
3087
3088     if ($conf->{watchdog}) {
3089         my $wdopts = parse_watchdog($conf->{watchdog});
3090         $pciaddr = print_pci_addr("watchdog", $bridges);
3091         my $watchdog = $wdopts->{model} || 'i6300esb';
3092         push @$devices, '-device', "$watchdog$pciaddr";
3093         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3094     }
3095
3096     my $vollist = [];
3097     my $scsicontroller = {};
3098     my $ahcicontroller = {};
3099     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3100
3101     # Add iscsi initiator name if available
3102     if (my $initiator = get_initiator_name()) {
3103         push @$devices, '-iscsi', "initiator-name=$initiator";
3104     }
3105
3106     foreach_drive($conf, sub {
3107         my ($ds, $drive) = @_;
3108
3109         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3110             push @$vollist, $drive->{file};
3111         }
3112
3113         $use_virtio = 1 if $ds =~ m/^virtio/;
3114
3115         if (drive_is_cdrom ($drive)) {
3116             if ($bootindex_hash->{d}) {
3117                 $drive->{bootindex} = $bootindex_hash->{d};
3118                 $bootindex_hash->{d} += 1;
3119             }
3120         } else {
3121             if ($bootindex_hash->{c}) {
3122                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3123                 $bootindex_hash->{c} += 1;
3124             }
3125         }
3126
3127         if($drive->{interface} eq 'virtio'){
3128            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3129         }
3130
3131         if ($drive->{interface} eq 'scsi') {
3132
3133             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3134
3135             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3136             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3137
3138             my $iothread = '';
3139             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3140                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3141                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3142             }
3143
3144             my $queues = '';
3145             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3146                 $queues = ",num_queues=$drive->{queues}";
3147             }
3148
3149             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3150             $scsicontroller->{$controller}=1;
3151         }
3152
3153         if ($drive->{interface} eq 'sata') {
3154            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3155            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3156            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3157            $ahcicontroller->{$controller}=1;
3158         }
3159
3160         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3161         push @$devices, '-drive',$drive_cmd;
3162         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3163     });
3164
3165     for (my $i = 0; $i < $MAX_NETS; $i++) {
3166          next if !$conf->{"net$i"};
3167          my $d = parse_net($conf->{"net$i"});
3168          next if !$d;
3169
3170          $use_virtio = 1 if $d->{model} eq 'virtio';
3171
3172          if ($bootindex_hash->{n}) {
3173             $d->{bootindex} = $bootindex_hash->{n};
3174             $bootindex_hash->{n} += 1;
3175          }
3176
3177          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3178          push @$devices, '-netdev', $netdevfull;
3179
3180          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3181          push @$devices, '-device', $netdevicefull;
3182     }
3183
3184     if (!$q35) {
3185         # add pci bridges
3186         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3187            $bridges->{1} = 1;
3188            $bridges->{2} = 1;
3189         }
3190
3191         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3192
3193         while (my ($k, $v) = each %$bridges) {
3194             $pciaddr = print_pci_addr("pci.$k");
3195             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3196         }
3197     }
3198
3199     # add custom args
3200     if ($conf->{args}) {
3201         my $aa = PVE::Tools::split_args($conf->{args});
3202         push @$cmd, @$aa;
3203     }
3204
3205     push @$cmd, @$devices;
3206     push @$cmd, '-rtc', join(',', @$rtcFlags)
3207         if scalar(@$rtcFlags);
3208     push @$cmd, '-machine', join(',', @$machineFlags)
3209         if scalar(@$machineFlags);
3210     push @$cmd, '-global', join(',', @$globalFlags)
3211         if scalar(@$globalFlags);
3212
3213     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3214 }
3215
3216 sub vnc_socket {
3217     my ($vmid) = @_;
3218     return "${var_run_tmpdir}/$vmid.vnc";
3219 }
3220
3221 sub spice_port {
3222     my ($vmid) = @_;
3223
3224     my $res = vm_mon_cmd($vmid, 'query-spice');
3225
3226     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3227 }
3228
3229 sub qmp_socket {
3230     my ($vmid, $qga) = @_;
3231     my $sockettype = $qga ? 'qga' : 'qmp';
3232     return "${var_run_tmpdir}/$vmid.$sockettype";
3233 }
3234
3235 sub pidfile_name {
3236     my ($vmid) = @_;
3237     return "${var_run_tmpdir}/$vmid.pid";
3238 }
3239
3240 sub vm_devices_list {
3241     my ($vmid) = @_;
3242
3243     my $res = vm_mon_cmd($vmid, 'query-pci');
3244     my $devices = {};
3245     foreach my $pcibus (@$res) {
3246         foreach my $device (@{$pcibus->{devices}}) {
3247             next if !$device->{'qdev_id'};
3248             if ($device->{'pci_bridge'}) {
3249                 $devices->{$device->{'qdev_id'}} = 1;
3250                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3251                     next if !$bridge_device->{'qdev_id'};
3252                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3253                     $devices->{$device->{'qdev_id'}}++;
3254                 }
3255             } else {
3256                 $devices->{$device->{'qdev_id'}} = 1;
3257             }
3258         }
3259     }
3260
3261     my $resblock = vm_mon_cmd($vmid, 'query-block');
3262     foreach my $block (@$resblock) {
3263         if($block->{device} =~ m/^drive-(\S+)/){
3264                 $devices->{$1} = 1;
3265         }
3266     }
3267
3268     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3269     foreach my $mice (@$resmice) {
3270         if ($mice->{name} eq 'QEMU HID Tablet') {
3271             $devices->{tablet} = 1;
3272             last;
3273         }
3274     }
3275
3276     return $devices;
3277 }
3278
3279 sub vm_deviceplug {
3280     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3281
3282     my $q35 = machine_type_is_q35($conf);
3283
3284     my $devices_list = vm_devices_list($vmid);
3285     return 1 if defined($devices_list->{$deviceid});
3286
3287     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3288
3289     if ($deviceid eq 'tablet') {
3290
3291         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3292
3293     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3294
3295         qemu_iothread_add($vmid, $deviceid, $device);
3296
3297         qemu_driveadd($storecfg, $vmid, $device);
3298         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3299
3300         qemu_deviceadd($vmid, $devicefull);
3301         eval { qemu_deviceaddverify($vmid, $deviceid); };
3302         if (my $err = $@) {
3303             eval { qemu_drivedel($vmid, $deviceid); };
3304             warn $@ if $@;
3305             die $err;
3306         }
3307
3308     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3309
3310
3311         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3312         my $pciaddr = print_pci_addr($deviceid);
3313         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3314
3315         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3316
3317         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3318             qemu_iothread_add($vmid, $deviceid, $device);
3319             $devicefull .= ",iothread=iothread-$deviceid";
3320         }
3321
3322         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3323             $devicefull .= ",num_queues=$device->{queues}";
3324         }
3325
3326         qemu_deviceadd($vmid, $devicefull);
3327         qemu_deviceaddverify($vmid, $deviceid);
3328
3329     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3330
3331         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3332         qemu_driveadd($storecfg, $vmid, $device);
3333
3334         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3335         eval { qemu_deviceadd($vmid, $devicefull); };
3336         if (my $err = $@) {
3337             eval { qemu_drivedel($vmid, $deviceid); };
3338             warn $@ if $@;
3339             die $err;
3340         }
3341
3342     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3343
3344         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3345
3346         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
3347         my $use_old_bios_files = undef;
3348         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3349
3350         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3351         qemu_deviceadd($vmid, $netdevicefull);
3352         eval { qemu_deviceaddverify($vmid, $deviceid); };
3353         if (my $err = $@) {
3354             eval { qemu_netdevdel($vmid, $deviceid); };
3355             warn $@ if $@;
3356             die $err;
3357         }
3358
3359     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3360
3361         my $bridgeid = $2;
3362         my $pciaddr = print_pci_addr($deviceid);
3363         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3364
3365         qemu_deviceadd($vmid, $devicefull);
3366         qemu_deviceaddverify($vmid, $deviceid);
3367
3368     } else {
3369         die "can't hotplug device '$deviceid'\n";
3370     }
3371
3372     return 1;
3373 }
3374
3375 # fixme: this should raise exceptions on error!
3376 sub vm_deviceunplug {
3377     my ($vmid, $conf, $deviceid) = @_;
3378
3379     my $devices_list = vm_devices_list($vmid);
3380     return 1 if !defined($devices_list->{$deviceid});
3381
3382     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3383
3384     if ($deviceid eq 'tablet') {
3385
3386         qemu_devicedel($vmid, $deviceid);
3387
3388     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3389
3390         qemu_devicedel($vmid, $deviceid);
3391         qemu_devicedelverify($vmid, $deviceid);
3392         qemu_drivedel($vmid, $deviceid);
3393         qemu_iothread_del($conf, $vmid, $deviceid);
3394
3395     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3396
3397         qemu_devicedel($vmid, $deviceid);
3398         qemu_devicedelverify($vmid, $deviceid);
3399         qemu_iothread_del($conf, $vmid, $deviceid);
3400
3401     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3402
3403         #qemu 2.3 segfault on drive_del with virtioscsi + iothread
3404         my $device = parse_drive($deviceid, $conf->{$deviceid});
3405         die "virtioscsi with iothread is not hot-unplugglable currently" if $device->{iothread};
3406
3407         qemu_devicedel($vmid, $deviceid);
3408         qemu_drivedel($vmid, $deviceid);
3409         qemu_deletescsihw($conf, $vmid, $deviceid);
3410
3411     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3412
3413         qemu_devicedel($vmid, $deviceid);
3414         qemu_devicedelverify($vmid, $deviceid);
3415         qemu_netdevdel($vmid, $deviceid);
3416
3417     } else {
3418         die "can't unplug device '$deviceid'\n";
3419     }
3420
3421     return 1;
3422 }
3423
3424 sub qemu_deviceadd {
3425     my ($vmid, $devicefull) = @_;
3426
3427     $devicefull = "driver=".$devicefull;
3428     my %options =  split(/[=,]/, $devicefull);
3429
3430     vm_mon_cmd($vmid, "device_add" , %options);
3431 }
3432
3433 sub qemu_devicedel {
3434     my ($vmid, $deviceid) = @_;
3435
3436     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3437 }
3438
3439 sub qemu_iothread_add {
3440     my($vmid, $deviceid, $device) = @_;
3441
3442     if ($device->{iothread}) {
3443         my $iothreads = vm_iothreads_list($vmid);
3444         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3445     }
3446 }
3447
3448 sub qemu_iothread_del {
3449     my($conf, $vmid, $deviceid) = @_;
3450
3451     my $device = parse_drive($deviceid, $conf->{$deviceid});
3452     if ($device->{iothread}) {
3453         my $iothreads = vm_iothreads_list($vmid);
3454         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3455     }
3456 }
3457
3458 sub qemu_objectadd {
3459     my($vmid, $objectid, $qomtype) = @_;
3460
3461     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3462
3463     return 1;
3464 }
3465
3466 sub qemu_objectdel {
3467     my($vmid, $objectid) = @_;
3468
3469     vm_mon_cmd($vmid, "object-del", id => $objectid);
3470
3471     return 1;
3472 }
3473
3474 sub qemu_driveadd {
3475     my ($storecfg, $vmid, $device) = @_;
3476
3477     my $drive = print_drive_full($storecfg, $vmid, $device);
3478     $drive =~ s/\\/\\\\/g;
3479     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3480
3481     # If the command succeeds qemu prints: "OK"
3482     return 1 if $ret =~ m/OK/s;
3483
3484     die "adding drive failed: $ret\n";
3485 }
3486
3487 sub qemu_drivedel {
3488     my($vmid, $deviceid) = @_;
3489
3490     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3491     $ret =~ s/^\s+//;
3492
3493     return 1 if $ret eq "";
3494
3495     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3496     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3497
3498     die "deleting drive $deviceid failed : $ret\n";
3499 }
3500
3501 sub qemu_deviceaddverify {
3502     my ($vmid, $deviceid) = @_;
3503
3504     for (my $i = 0; $i <= 5; $i++) {
3505          my $devices_list = vm_devices_list($vmid);
3506          return 1 if defined($devices_list->{$deviceid});
3507          sleep 1;
3508     }
3509
3510     die "error on hotplug device '$deviceid'\n";
3511 }
3512
3513
3514 sub qemu_devicedelverify {
3515     my ($vmid, $deviceid) = @_;
3516
3517     # need to verify that the device is correctly removed as device_del
3518     # is async and empty return is not reliable
3519
3520     for (my $i = 0; $i <= 5; $i++) {
3521          my $devices_list = vm_devices_list($vmid);
3522          return 1 if !defined($devices_list->{$deviceid});
3523          sleep 1;
3524     }
3525
3526     die "error on hot-unplugging device '$deviceid'\n";
3527 }
3528
3529 sub qemu_findorcreatescsihw {
3530     my ($storecfg, $conf, $vmid, $device) = @_;
3531
3532     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3533
3534     my $scsihwid="$controller_prefix$controller";
3535     my $devices_list = vm_devices_list($vmid);
3536
3537     if(!defined($devices_list->{$scsihwid})) {
3538         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3539     }
3540
3541     return 1;
3542 }
3543
3544 sub qemu_deletescsihw {
3545     my ($conf, $vmid, $opt) = @_;
3546
3547     my $device = parse_drive($opt, $conf->{$opt});
3548
3549     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3550         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3551         return 1;
3552     }
3553
3554     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3555
3556     my $devices_list = vm_devices_list($vmid);
3557     foreach my $opt (keys %{$devices_list}) {
3558         if (PVE::QemuServer::valid_drivename($opt)) {
3559             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3560             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3561                 return 1;
3562             }
3563         }
3564     }
3565
3566     my $scsihwid="scsihw$controller";
3567
3568     vm_deviceunplug($vmid, $conf, $scsihwid);
3569
3570     return 1;
3571 }
3572
3573 sub qemu_add_pci_bridge {
3574     my ($storecfg, $conf, $vmid, $device) = @_;
3575
3576     my $bridges = {};
3577
3578     my $bridgeid;
3579
3580     print_pci_addr($device, $bridges);
3581
3582     while (my ($k, $v) = each %$bridges) {
3583         $bridgeid = $k;
3584     }
3585     return 1 if !defined($bridgeid) || $bridgeid < 1;
3586
3587     my $bridge = "pci.$bridgeid";
3588     my $devices_list = vm_devices_list($vmid);
3589
3590     if (!defined($devices_list->{$bridge})) {
3591         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3592     }
3593
3594     return 1;
3595 }
3596
3597 sub qemu_set_link_status {
3598     my ($vmid, $device, $up) = @_;
3599
3600     vm_mon_cmd($vmid, "set_link", name => $device,
3601                up => $up ? JSON::true : JSON::false);
3602 }
3603
3604 sub qemu_netdevadd {
3605     my ($vmid, $conf, $device, $deviceid) = @_;
3606
3607     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
3608     my %options =  split(/[=,]/, $netdev);
3609
3610     vm_mon_cmd($vmid, "netdev_add",  %options);
3611     return 1;
3612 }
3613
3614 sub qemu_netdevdel {
3615     my ($vmid, $deviceid) = @_;
3616
3617     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3618 }
3619
3620 sub qemu_cpu_hotplug {
3621     my ($vmid, $conf, $vcpus) = @_;
3622
3623     my $sockets = 1;
3624     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3625     $sockets = $conf->{sockets} if  $conf->{sockets};
3626     my $cores = $conf->{cores} || 1;
3627     my $maxcpus = $sockets * $cores;
3628
3629     $vcpus = $maxcpus if !$vcpus;
3630
3631     die "you can't add more vcpus than maxcpus\n"
3632         if $vcpus > $maxcpus;
3633
3634     my $currentvcpus = $conf->{vcpus} || $maxcpus;
3635     die "online cpu unplug is not yet possible\n"
3636         if $vcpus < $currentvcpus;
3637
3638     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3639     die "vcpus in running vm is different than configuration\n"
3640         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
3641
3642     for (my $i = $currentvcpus; $i < $vcpus; $i++) {
3643         vm_mon_cmd($vmid, "cpu-add", id => int($i));
3644     }
3645 }
3646
3647 sub qemu_memory_hotplug {
3648     my ($vmid, $conf, $defaults, $opt, $value) = @_;
3649
3650     return $value if !check_running($vmid);
3651
3652     my $memory = $conf->{memory} || $defaults->{memory};
3653     $value = $defaults->{memory} if !$value;
3654     return $value if $value == $memory;
3655
3656     my $static_memory = $STATICMEM;
3657     my $dimm_memory = $memory - $static_memory;
3658
3659     die "memory can't be lower than $static_memory MB" if $value < $static_memory;
3660     die "you cannot add more memory than $MAX_MEM MB!\n" if $memory > $MAX_MEM;
3661
3662
3663     my $sockets = 1;
3664     $sockets = $conf->{sockets} if $conf->{sockets};
3665
3666     if($value > $memory) {
3667
3668         foreach_dimm($conf, $vmid, $value, $sockets, sub {
3669             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3670
3671                 return if $current_size <= $conf->{memory};
3672
3673                 eval { vm_mon_cmd($vmid, "object-add", 'qom-type' => "memory-backend-ram", id => "mem-$name", props => { size => int($dimm_size*1024*1024) } ) };
3674                 if (my $err = $@) {
3675                     eval { qemu_objectdel($vmid, "mem-$name"); };
3676                     die $err;
3677                 }
3678
3679                 eval { vm_mon_cmd($vmid, "device_add", driver => "pc-dimm", id => "$name", memdev => "mem-$name", node => $numanode) };
3680                 if (my $err = $@) {
3681                     eval { qemu_objectdel($vmid, "mem-$name"); };
3682                     die $err;
3683                 }
3684                 #update conf after each succesful module hotplug
3685                 $conf->{memory} = $current_size;
3686                 update_config_nolock($vmid, $conf, 1);
3687         });
3688
3689     } else {
3690
3691         foreach_reverse_dimm($conf, $vmid, $value, $sockets, sub {
3692             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3693
3694                 return if $current_size >= $conf->{memory};
3695                 print "try to unplug memory dimm $name\n";
3696
3697                 my $retry = 0;
3698                 while (1) {
3699                     eval { qemu_devicedel($vmid, $name) };
3700                     sleep 3;
3701                     my $dimm_list = qemu_dimm_list($vmid);
3702                     last if !$dimm_list->{$name};
3703                     raise_param_exc({ $name => "error unplug memory module" }) if $retry > 5;
3704                     $retry++;
3705                 }
3706
3707                 #update conf after each succesful module unplug
3708                 $conf->{memory} = $current_size;
3709
3710                 eval { qemu_objectdel($vmid, "mem-$name"); };
3711                 update_config_nolock($vmid, $conf, 1);
3712         });
3713     }
3714 }
3715
3716 sub qemu_dimm_list {
3717     my ($vmid) = @_;
3718
3719     my $dimmarray = vm_mon_cmd_nocheck($vmid, "query-memory-devices");
3720     my $dimms = {};
3721
3722     foreach my $dimm (@$dimmarray) {
3723
3724         $dimms->{$dimm->{data}->{id}}->{id} = $dimm->{data}->{id};
3725         $dimms->{$dimm->{data}->{id}}->{node} = $dimm->{data}->{node};
3726         $dimms->{$dimm->{data}->{id}}->{addr} = $dimm->{data}->{addr};
3727         $dimms->{$dimm->{data}->{id}}->{size} = $dimm->{data}->{size};
3728         $dimms->{$dimm->{data}->{id}}->{slot} = $dimm->{data}->{slot};
3729     }
3730     return $dimms;
3731 }
3732
3733 sub qemu_block_set_io_throttle {
3734     my ($vmid, $deviceid, $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr) = @_;
3735
3736     return if !check_running($vmid) ;
3737
3738     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid, bps => int($bps), bps_rd => int($bps_rd), bps_wr => int($bps_wr), iops => int($iops), iops_rd => int($iops_rd), iops_wr => int($iops_wr));
3739
3740 }
3741
3742 # old code, only used to shutdown old VM after update
3743 sub __read_avail {
3744     my ($fh, $timeout) = @_;
3745
3746     my $sel = new IO::Select;
3747     $sel->add($fh);
3748
3749     my $res = '';
3750     my $buf;
3751
3752     my @ready;
3753     while (scalar (@ready = $sel->can_read($timeout))) {
3754         my $count;
3755         if ($count = $fh->sysread($buf, 8192)) {
3756             if ($buf =~ /^(.*)\(qemu\) $/s) {
3757                 $res .= $1;
3758                 last;
3759             } else {
3760                 $res .= $buf;
3761             }
3762         } else {
3763             if (!defined($count)) {
3764                 die "$!\n";
3765             }
3766             last;
3767         }
3768     }
3769
3770     die "monitor read timeout\n" if !scalar(@ready);
3771
3772     return $res;
3773 }
3774
3775 # old code, only used to shutdown old VM after update
3776 sub vm_monitor_command {
3777     my ($vmid, $cmdstr, $nocheck) = @_;
3778
3779     my $res;
3780
3781     eval {
3782         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
3783
3784         my $sname = "${var_run_tmpdir}/$vmid.mon";
3785
3786         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
3787             die "unable to connect to VM $vmid socket - $!\n";
3788
3789         my $timeout = 3;
3790
3791         # hack: migrate sometime blocks the monitor (when migrate_downtime
3792         # is set)
3793         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3794             $timeout = 60*60; # 1 hour
3795         }
3796
3797         # read banner;
3798         my $data = __read_avail($sock, $timeout);
3799
3800         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
3801             die "got unexpected qemu monitor banner\n";
3802         }
3803
3804         my $sel = new IO::Select;
3805         $sel->add($sock);
3806
3807         if (!scalar(my @ready = $sel->can_write($timeout))) {
3808             die "monitor write error - timeout";
3809         }
3810
3811         my $fullcmd = "$cmdstr\r";
3812
3813         # syslog('info', "VM $vmid monitor command: $cmdstr");
3814
3815         my $b;
3816         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
3817             die "monitor write error - $!";
3818         }
3819
3820         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
3821
3822         $timeout = 20;
3823
3824         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3825             $timeout = 60*60; # 1 hour
3826         } elsif ($cmdstr =~ m/^(eject|change)/) {
3827             $timeout = 60; # note: cdrom mount command is slow
3828         }
3829         if ($res = __read_avail($sock, $timeout)) {
3830
3831             my @lines = split("\r?\n", $res);
3832
3833             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
3834
3835             $res = join("\n", @lines);
3836             $res .= "\n";
3837         }
3838     };
3839
3840     my $err = $@;
3841
3842     if ($err) {
3843         syslog("err", "VM $vmid monitor command failed - $err");
3844         die $err;
3845     }
3846
3847     return $res;
3848 }
3849
3850 sub qemu_block_resize {
3851     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
3852
3853     my $running = check_running($vmid);
3854
3855     return if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
3856
3857     return if !$running;
3858
3859     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
3860
3861 }
3862
3863 sub qemu_volume_snapshot {
3864     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3865
3866     my $running = check_running($vmid);
3867
3868     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
3869         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
3870     } else {
3871         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
3872     }
3873 }
3874
3875 sub qemu_volume_snapshot_delete {
3876     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3877
3878     my $running = check_running($vmid);
3879
3880     return if !PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
3881
3882     return if !$running;
3883
3884     vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
3885 }
3886
3887 sub set_migration_caps {
3888     my ($vmid) = @_;
3889
3890     my $cap_ref = [];
3891
3892     my $enabled_cap = {
3893         "auto-converge" => 1,
3894         "xbzrle" => 1,
3895         "x-rdma-pin-all" => 0,
3896         "zero-blocks" => 0,
3897         "compress" => 0
3898     };
3899
3900     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
3901
3902     for my $supported_capability (@$supported_capabilities) {
3903         push @$cap_ref, {
3904             capability => $supported_capability->{capability},
3905             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
3906         };
3907     }
3908
3909     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
3910 }
3911
3912 my $fast_plug_option = {
3913     'lock' => 1,
3914     'name' => 1,
3915     'onboot' => 1,
3916     'shares' => 1,
3917     'startup' => 1,
3918     'description' => 1,
3919 };
3920
3921 # hotplug changes in [PENDING]
3922 # $selection hash can be used to only apply specified options, for
3923 # example: { cores => 1 } (only apply changed 'cores')
3924 # $errors ref is used to return error messages
3925 sub vmconfig_hotplug_pending {
3926     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
3927
3928     my $defaults = load_defaults();
3929
3930     # commit values which do not have any impact on running VM first
3931     # Note: those option cannot raise errors, we we do not care about
3932     # $selection and always apply them.
3933
3934     my $add_error = sub {
3935         my ($opt, $msg) = @_;
3936         $errors->{$opt} = "hotplug problem - $msg";
3937     };
3938
3939     my $changes = 0;
3940     foreach my $opt (keys %{$conf->{pending}}) { # add/change
3941         if ($fast_plug_option->{$opt}) {
3942             $conf->{$opt} = $conf->{pending}->{$opt};
3943             delete $conf->{pending}->{$opt};
3944             $changes = 1;
3945         }
3946     }
3947
3948     if ($changes) {
3949         update_config_nolock($vmid, $conf, 1);
3950         $conf = load_config($vmid); # update/reload
3951     }
3952
3953     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3954
3955     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
3956     while (my ($opt, $force) = each %$pending_delete_hash) {
3957         next if $selection && !$selection->{$opt};
3958         eval {
3959             if ($opt eq 'hotplug') {
3960                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
3961             } elsif ($opt eq 'tablet') {
3962                 die "skip\n" if !$hotplug_features->{usb};
3963                 if ($defaults->{tablet}) {
3964                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
3965                 } else {
3966                     vm_deviceunplug($vmid, $conf, $opt);
3967                 }
3968             } elsif ($opt eq 'vcpus') {
3969                 die "skip\n" if !$hotplug_features->{cpu};
3970                 qemu_cpu_hotplug($vmid, $conf, undef);
3971             } elsif ($opt eq 'balloon') {
3972                 # enable balloon device is not hotpluggable
3973                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
3974             } elsif ($fast_plug_option->{$opt}) {
3975                 # do nothing
3976             } elsif ($opt =~ m/^net(\d+)$/) {
3977                 die "skip\n" if !$hotplug_features->{network};
3978                 vm_deviceunplug($vmid, $conf, $opt);
3979             } elsif (valid_drivename($opt)) {
3980                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
3981                 vm_deviceunplug($vmid, $conf, $opt);
3982                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
3983             } elsif ($opt =~ m/^memory$/) {
3984                 die "skip\n" if !$hotplug_features->{memory};
3985                 qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
3986             } elsif ($opt eq 'cpuunits') {
3987                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
3988             } elsif ($opt eq 'cpulimit') {
3989                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
3990             } else {
3991                 die "skip\n";
3992             }
3993         };
3994         if (my $err = $@) {
3995             &$add_error($opt, $err) if $err ne "skip\n";
3996         } else {
3997             # save new config if hotplug was successful
3998             delete $conf->{$opt};
3999             vmconfig_undelete_pending_option($conf, $opt);
4000             update_config_nolock($vmid, $conf, 1);
4001             $conf = load_config($vmid); # update/reload
4002         }
4003     }
4004
4005     foreach my $opt (keys %{$conf->{pending}}) {
4006         next if $selection && !$selection->{$opt};
4007         my $value = $conf->{pending}->{$opt};
4008         eval {
4009             if ($opt eq 'hotplug') {
4010                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4011             } elsif ($opt eq 'tablet') {
4012                 die "skip\n" if !$hotplug_features->{usb};
4013                 if ($value == 1) {
4014                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4015                 } elsif ($value == 0) {
4016                     vm_deviceunplug($vmid, $conf, $opt);
4017                 }
4018             } elsif ($opt eq 'vcpus') {
4019                 die "skip\n" if !$hotplug_features->{cpu};
4020                 qemu_cpu_hotplug($vmid, $conf, $value);
4021             } elsif ($opt eq 'balloon') {
4022                 # enable/disable balloning device is not hotpluggable
4023                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4024                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4025                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4026
4027                 # allow manual ballooning if shares is set to zero
4028                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4029                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4030                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4031                 }
4032             } elsif ($opt =~ m/^net(\d+)$/) {
4033                 # some changes can be done without hotplug
4034                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4035                                     $vmid, $opt, $value);
4036             } elsif (valid_drivename($opt)) {
4037                 # some changes can be done without hotplug
4038                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4039                                      $vmid, $opt, $value, 1);
4040             } elsif ($opt =~ m/^memory$/) { #dimms
4041                 die "skip\n" if !$hotplug_features->{memory};
4042                 $value = qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4043             } elsif ($opt eq 'cpuunits') {
4044                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4045             } elsif ($opt eq 'cpulimit') {
4046                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4047                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4048             } else {
4049                 die "skip\n";  # skip non-hot-pluggable options
4050             }
4051         };
4052         if (my $err = $@) {
4053             &$add_error($opt, $err) if $err ne "skip\n";
4054         } else {
4055             # save new config if hotplug was successful
4056             $conf->{$opt} = $value;
4057             delete $conf->{pending}->{$opt};
4058             update_config_nolock($vmid, $conf, 1);
4059             $conf = load_config($vmid); # update/reload
4060         }
4061     }
4062 }
4063
4064 sub try_deallocate_drive {
4065     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4066
4067     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4068         my $volid = $drive->{file};
4069         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4070             my $sid = PVE::Storage::parse_volume_id($volid);
4071             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4072
4073             # check if the disk is really unused
4074             my $used_paths = PVE::QemuServer::get_used_paths($vmid, $storecfg, $conf, 1, $key);
4075             my $path = PVE::Storage::path($storecfg, $volid);
4076             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4077                    if $used_paths->{$path};
4078             PVE::Storage::vdisk_free($storecfg, $volid);
4079             return 1;
4080         } else {
4081             # If vm is not owner of this disk remove from config
4082             return 1;
4083         }
4084     }
4085
4086     return undef;
4087 }
4088
4089 sub vmconfig_delete_or_detach_drive {
4090     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4091
4092     my $drive = parse_drive($opt, $conf->{$opt});
4093
4094     my $rpcenv = PVE::RPCEnvironment::get();
4095     my $authuser = $rpcenv->get_user();
4096
4097     if ($force) {
4098         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4099         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4100     } else {
4101         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4102     }
4103 }
4104
4105 sub vmconfig_apply_pending {
4106     my ($vmid, $conf, $storecfg) = @_;
4107
4108     # cold plug
4109
4110     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4111     while (my ($opt, $force) = each %$pending_delete_hash) {
4112         die "internal error" if $opt =~ m/^unused/;
4113         $conf = load_config($vmid); # update/reload
4114         if (!defined($conf->{$opt})) {
4115             vmconfig_undelete_pending_option($conf, $opt);
4116             update_config_nolock($vmid, $conf, 1);
4117         } elsif (valid_drivename($opt)) {
4118             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4119             vmconfig_undelete_pending_option($conf, $opt);
4120             delete $conf->{$opt};
4121             update_config_nolock($vmid, $conf, 1);
4122         } else {
4123             vmconfig_undelete_pending_option($conf, $opt);
4124             delete $conf->{$opt};
4125             update_config_nolock($vmid, $conf, 1);
4126         }
4127     }
4128
4129     $conf = load_config($vmid); # update/reload
4130
4131     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4132         $conf = load_config($vmid); # update/reload
4133
4134         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4135             # skip if nothing changed
4136         } elsif (valid_drivename($opt)) {
4137             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4138                 if defined($conf->{$opt});
4139             $conf->{$opt} = $conf->{pending}->{$opt};
4140         } else {
4141             $conf->{$opt} = $conf->{pending}->{$opt};
4142         }
4143
4144         delete $conf->{pending}->{$opt};
4145         update_config_nolock($vmid, $conf, 1);
4146     }
4147 }
4148
4149 my $safe_num_ne = sub {
4150     my ($a, $b) = @_;
4151
4152     return 0 if !defined($a) && !defined($b);
4153     return 1 if !defined($a);
4154     return 1 if !defined($b);
4155
4156     return $a != $b;
4157 };
4158
4159 my $safe_string_ne = sub {
4160     my ($a, $b) = @_;
4161
4162     return 0 if !defined($a) && !defined($b);
4163     return 1 if !defined($a);
4164     return 1 if !defined($b);
4165
4166     return $a ne $b;
4167 };
4168
4169 sub vmconfig_update_net {
4170     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4171
4172     my $newnet = parse_net($value);
4173
4174     if ($conf->{$opt}) {
4175         my $oldnet = parse_net($conf->{$opt});
4176
4177         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4178             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4179             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4180             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4181
4182             # for non online change, we try to hot-unplug
4183             die "skip\n" if !$hotplug;
4184             vm_deviceunplug($vmid, $conf, $opt);
4185         } else {
4186
4187             die "internal error" if $opt !~ m/net(\d+)/;
4188             my $iface = "tap${vmid}i$1";
4189
4190             if (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4191                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4192             }
4193
4194             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4195                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4196                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4197                 PVE::Network::tap_unplug($iface);
4198                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall});
4199             }
4200
4201             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4202                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4203             }
4204
4205             return 1;
4206         }
4207     }
4208
4209     if ($hotplug) {
4210         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4211     } else {
4212         die "skip\n";
4213     }
4214 }
4215
4216 sub vmconfig_update_disk {
4217     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4218
4219     # fixme: do we need force?
4220
4221     my $drive = parse_drive($opt, $value);
4222
4223     if ($conf->{$opt}) {
4224
4225         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4226
4227             my $media = $drive->{media} || 'disk';
4228             my $oldmedia = $old_drive->{media} || 'disk';
4229             die "unable to change media type\n" if $media ne $oldmedia;
4230
4231             if (!drive_is_cdrom($old_drive)) {
4232
4233                 if ($drive->{file} ne $old_drive->{file}) {
4234
4235                     die "skip\n" if !$hotplug;
4236
4237                     # unplug and register as unused
4238                     vm_deviceunplug($vmid, $conf, $opt);
4239                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4240
4241                 } else {
4242                     # update existing disk
4243
4244                     # skip non hotpluggable value
4245                     if (&$safe_num_ne($drive->{discard}, $old_drive->{discard}) ||
4246                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4247                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4248                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4249                         die "skip\n";
4250                     }
4251
4252                     # apply throttle
4253                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4254                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4255                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4256                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4257                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4258                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4259                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4260                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4261                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4262                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4263                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4264                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max})) {
4265
4266                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4267                                                    ($drive->{mbps} || 0)*1024*1024,
4268                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4269                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4270                                                    $drive->{iops} || 0,
4271                                                    $drive->{iops_rd} || 0,
4272                                                    $drive->{iops_wr} || 0,
4273                                                    ($drive->{mbps_max} || 0)*1024*1024,
4274                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4275                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4276                                                    $drive->{iops_max} || 0,
4277                                                    $drive->{iops_rd_max} || 0,
4278                                                    $drive->{iops_wr_max} || 0);
4279
4280                     }
4281
4282                     return 1;
4283                 }
4284
4285             } else { # cdrom
4286
4287                 if ($drive->{file} eq 'none') {
4288                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4289                 } else {
4290                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4291                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4292                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4293                 }
4294
4295                 return 1;
4296             }
4297         }
4298     }
4299
4300     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4301     # hotplug new disks
4302     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]);
4303     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4304 }
4305
4306 sub vm_start {
4307     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4308         $forcemachine, $spice_ticket) = @_;
4309
4310     lock_config($vmid, sub {
4311         my $conf = load_config($vmid, $migratedfrom);
4312
4313         die "you can't start a vm if it's a template\n" if is_template($conf);
4314
4315         check_lock($conf) if !$skiplock;
4316
4317         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4318
4319         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4320             vmconfig_apply_pending($vmid, $conf, $storecfg);
4321             $conf = load_config($vmid); # update/reload
4322         }
4323
4324         my $defaults = load_defaults();
4325
4326         # set environment variable useful inside network script
4327         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4328
4329         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4330
4331         my $migrate_port = 0;
4332         my $migrate_uri;
4333         if ($statefile) {
4334             if ($statefile eq 'tcp') {
4335                 my $localip = "localhost";
4336                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4337                 my $nodename = PVE::INotify::nodename();
4338                 if ($datacenterconf->{migration_unsecure}) {
4339                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4340                         $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4341                 }
4342                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4343                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4344                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4345                 push @$cmd, '-incoming', $migrate_uri;
4346                 push @$cmd, '-S';
4347             } else {
4348                 push @$cmd, '-loadstate', $statefile;
4349             }
4350         } elsif ($paused) {
4351             push @$cmd, '-S';
4352         }
4353
4354         # host pci devices
4355         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4356           my $d = parse_hostpci($conf->{"hostpci$i"});
4357           next if !$d;
4358           my $pcidevices = $d->{pciid};
4359           foreach my $pcidevice (@$pcidevices) {
4360                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4361
4362                 my $info = pci_device_info("0000:$pciid");
4363                 die "IOMMU not present\n" if !check_iommu_support();
4364                 die "no pci device info for device '$pciid'\n" if !$info;
4365                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4366                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4367           }
4368         }
4369
4370         PVE::Storage::activate_volumes($storecfg, $vollist);
4371
4372         eval  { run_command($cmd, timeout => $statefile ? undef : 30,
4373                     umask => 0077); };
4374         my $err = $@;
4375         die "start failed: $err" if $err;
4376
4377         print "migration listens on $migrate_uri\n" if $migrate_uri;
4378
4379         if ($statefile && $statefile ne 'tcp')  {
4380             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4381             warn $@ if $@;
4382         }
4383
4384         if ($migratedfrom) {
4385
4386             eval {
4387                 set_migration_caps($vmid);
4388             };
4389             warn $@ if $@;
4390
4391             if ($spice_port) {
4392                 print "spice listens on port $spice_port\n";
4393                 if ($spice_ticket) {
4394                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4395                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4396                 }
4397             }
4398
4399         } else {
4400
4401             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4402                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4403                     if $conf->{balloon};
4404             }
4405
4406             foreach my $opt (keys %$conf) {
4407                 next if $opt !~  m/^net\d+$/;
4408                 my $nicconf = parse_net($conf->{$opt});
4409                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4410             }
4411         }
4412
4413         vm_mon_cmd_nocheck($vmid, 'qom-set',
4414                     path => "machine/peripheral/balloon0",
4415                     property => "guest-stats-polling-interval",
4416                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4417
4418     });
4419 }
4420
4421 sub vm_mon_cmd {
4422     my ($vmid, $execute, %params) = @_;
4423
4424     my $cmd = { execute => $execute, arguments => \%params };
4425     vm_qmp_command($vmid, $cmd);
4426 }
4427
4428 sub vm_mon_cmd_nocheck {
4429     my ($vmid, $execute, %params) = @_;
4430
4431     my $cmd = { execute => $execute, arguments => \%params };
4432     vm_qmp_command($vmid, $cmd, 1);
4433 }
4434
4435 sub vm_qmp_command {
4436     my ($vmid, $cmd, $nocheck) = @_;
4437
4438     my $res;
4439
4440     my $timeout;
4441     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4442         $timeout = $cmd->{arguments}->{timeout};
4443         delete $cmd->{arguments}->{timeout};
4444     }
4445
4446     eval {
4447         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4448         my $sname = qmp_socket($vmid);
4449         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4450             my $qmpclient = PVE::QMPClient->new();
4451
4452             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4453         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
4454             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
4455                 if scalar(%{$cmd->{arguments}});
4456             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
4457         } else {
4458             die "unable to open monitor socket\n";
4459         }
4460     };
4461     if (my $err = $@) {
4462         syslog("err", "VM $vmid qmp command failed - $err");
4463         die $err;
4464     }
4465
4466     return $res;
4467 }
4468
4469 sub vm_human_monitor_command {
4470     my ($vmid, $cmdline) = @_;
4471
4472     my $res;
4473
4474     my $cmd = {
4475         execute => 'human-monitor-command',
4476         arguments => { 'command-line' => $cmdline},
4477     };
4478
4479     return vm_qmp_command($vmid, $cmd);
4480 }
4481
4482 sub vm_commandline {
4483     my ($storecfg, $vmid) = @_;
4484
4485     my $conf = load_config($vmid);
4486
4487     my $defaults = load_defaults();
4488
4489     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
4490
4491     return join(' ', @$cmd);
4492 }
4493
4494 sub vm_reset {
4495     my ($vmid, $skiplock) = @_;
4496
4497     lock_config($vmid, sub {
4498
4499         my $conf = load_config($vmid);
4500
4501         check_lock($conf) if !$skiplock;
4502
4503         vm_mon_cmd($vmid, "system_reset");
4504     });
4505 }
4506
4507 sub get_vm_volumes {
4508     my ($conf) = @_;
4509
4510     my $vollist = [];
4511     foreach_volid($conf, sub {
4512         my ($volid, $is_cdrom) = @_;
4513
4514         return if $volid =~ m|^/|;
4515
4516         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4517         return if !$sid;
4518
4519         push @$vollist, $volid;
4520     });
4521
4522     return $vollist;
4523 }
4524
4525 sub vm_stop_cleanup {
4526     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
4527
4528     eval {
4529
4530         if (!$keepActive) {
4531             my $vollist = get_vm_volumes($conf);
4532             PVE::Storage::deactivate_volumes($storecfg, $vollist);
4533         }
4534
4535         foreach my $ext (qw(mon qmp pid vnc qga)) {
4536             unlink "/var/run/qemu-server/${vmid}.$ext";
4537         }
4538
4539         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
4540     };
4541     warn $@ if $@; # avoid errors - just warn
4542 }
4543
4544 # Note: use $nockeck to skip tests if VM configuration file exists.
4545 # We need that when migration VMs to other nodes (files already moved)
4546 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
4547 sub vm_stop {
4548     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
4549
4550     $force = 1 if !defined($force) && !$shutdown;
4551
4552     if ($migratedfrom){
4553         my $pid = check_running($vmid, $nocheck, $migratedfrom);
4554         kill 15, $pid if $pid;
4555         my $conf = load_config($vmid, $migratedfrom);
4556         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
4557         return;
4558     }
4559
4560     lock_config($vmid, sub {
4561
4562         my $pid = check_running($vmid, $nocheck);
4563         return if !$pid;
4564
4565         my $conf;
4566         if (!$nocheck) {
4567             $conf = load_config($vmid);
4568             check_lock($conf) if !$skiplock;
4569             if (!defined($timeout) && $shutdown && $conf->{startup}) {
4570                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
4571                 $timeout = $opts->{down} if $opts->{down};
4572             }
4573         }
4574
4575         $timeout = 60 if !defined($timeout);
4576
4577         eval {
4578             if ($shutdown) {
4579                 if (defined($conf) && $conf->{agent}) {
4580                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
4581                 } else {
4582                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
4583                 }
4584             } else {
4585                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
4586             }
4587         };
4588         my $err = $@;
4589
4590         if (!$err) {
4591             my $count = 0;
4592             while (($count < $timeout) && check_running($vmid, $nocheck)) {
4593                 $count++;
4594                 sleep 1;
4595             }
4596
4597             if ($count >= $timeout) {
4598                 if ($force) {
4599                     warn "VM still running - terminating now with SIGTERM\n";
4600                     kill 15, $pid;
4601                 } else {
4602                     die "VM quit/powerdown failed - got timeout\n";
4603                 }
4604             } else {
4605                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4606                 return;
4607             }
4608         } else {
4609             if ($force) {
4610                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
4611                 kill 15, $pid;
4612             } else {
4613                 die "VM quit/powerdown failed\n";
4614             }
4615         }
4616
4617         # wait again
4618         $timeout = 10;
4619
4620         my $count = 0;
4621         while (($count < $timeout) && check_running($vmid, $nocheck)) {
4622             $count++;
4623             sleep 1;
4624         }
4625
4626         if ($count >= $timeout) {
4627             warn "VM still running - terminating now with SIGKILL\n";
4628             kill 9, $pid;
4629             sleep 1;
4630         }
4631
4632         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4633    });
4634 }
4635
4636 sub vm_suspend {
4637     my ($vmid, $skiplock) = @_;
4638
4639     lock_config($vmid, sub {
4640
4641         my $conf = load_config($vmid);
4642
4643         check_lock($conf) if !($skiplock || ($conf->{lock} && $conf->{lock} eq 'backup'));
4644
4645         vm_mon_cmd($vmid, "stop");
4646     });
4647 }
4648
4649 sub vm_resume {
4650     my ($vmid, $skiplock, $nocheck) = @_;
4651
4652     lock_config($vmid, sub {
4653
4654         if (!$nocheck) {
4655
4656             my $conf = load_config($vmid);
4657
4658             check_lock($conf) if !($skiplock || ($conf->{lock} && $conf->{lock} eq 'backup'));
4659
4660             vm_mon_cmd($vmid, "cont");
4661
4662         } else {
4663             vm_mon_cmd_nocheck($vmid, "cont");
4664         }
4665     });
4666 }
4667
4668 sub vm_sendkey {
4669     my ($vmid, $skiplock, $key) = @_;
4670
4671     lock_config($vmid, sub {
4672
4673         my $conf = load_config($vmid);
4674
4675         # there is no qmp command, so we use the human monitor command
4676         vm_human_monitor_command($vmid, "sendkey $key");
4677     });
4678 }
4679
4680 sub vm_destroy {
4681     my ($storecfg, $vmid, $skiplock) = @_;
4682
4683     lock_config($vmid, sub {
4684
4685         my $conf = load_config($vmid);
4686
4687         check_lock($conf) if !$skiplock;
4688
4689         if (!check_running($vmid)) {
4690             destroy_vm($storecfg, $vmid);
4691         } else {
4692             die "VM $vmid is running - destroy failed\n";
4693         }
4694     });
4695 }
4696
4697 # pci helpers
4698
4699 sub file_write {
4700     my ($filename, $buf) = @_;
4701
4702     my $fh = IO::File->new($filename, "w");
4703     return undef if !$fh;
4704
4705     my $res = print $fh $buf;
4706
4707     $fh->close();
4708
4709     return $res;
4710 }
4711
4712 sub pci_device_info {
4713     my ($name) = @_;
4714
4715     my $res;
4716
4717     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
4718     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
4719
4720     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
4721     return undef if !defined($irq) || $irq !~ m/^\d+$/;
4722
4723     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
4724     return undef if !defined($vendor) || $vendor !~ s/^0x//;
4725
4726     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
4727     return undef if !defined($product) || $product !~ s/^0x//;
4728
4729     $res = {
4730         name => $name,
4731         vendor => $vendor,
4732         product => $product,
4733         domain => $domain,
4734         bus => $bus,
4735         slot => $slot,
4736         func => $func,
4737         irq => $irq,
4738         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
4739     };
4740
4741     return $res;
4742 }
4743
4744 sub pci_dev_reset {
4745     my ($dev) = @_;
4746
4747     my $name = $dev->{name};
4748
4749     my $fn = "$pcisysfs/devices/$name/reset";
4750
4751     return file_write($fn, "1");
4752 }
4753
4754 sub pci_dev_bind_to_vfio {
4755     my ($dev) = @_;
4756
4757     my $name = $dev->{name};
4758
4759     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4760
4761     if (!-d $vfio_basedir) {
4762         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4763     }
4764     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4765
4766     my $testdir = "$vfio_basedir/$name";
4767     return 1 if -d $testdir;
4768
4769     my $data = "$dev->{vendor} $dev->{product}";
4770     return undef if !file_write("$vfio_basedir/new_id", $data);
4771
4772     my $fn = "$pcisysfs/devices/$name/driver/unbind";
4773     if (!file_write($fn, $name)) {
4774         return undef if -f $fn;
4775     }
4776
4777     $fn = "$vfio_basedir/bind";
4778     if (! -d $testdir) {
4779         return undef if !file_write($fn, $name);
4780     }
4781
4782     return -d $testdir;
4783 }
4784
4785 sub pci_dev_group_bind_to_vfio {
4786     my ($pciid) = @_;
4787
4788     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4789
4790     if (!-d $vfio_basedir) {
4791         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4792     }
4793     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4794
4795     # get IOMMU group devices
4796     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
4797       my @devs = grep /^0000:/, readdir($D);
4798     closedir($D);
4799
4800     foreach my $pciid (@devs) {
4801         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
4802
4803         # pci bridges, switches or root ports are not supported
4804         # they have a pci_bus subdirectory so skip them
4805         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
4806
4807         my $info = pci_device_info($1);
4808         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
4809     }
4810
4811     return 1;
4812 }
4813
4814 sub print_pci_addr {
4815     my ($id, $bridges) = @_;
4816
4817     my $res = '';
4818     my $devices = {
4819         piix3 => { bus => 0, addr => 1 },
4820         #addr2 : first videocard
4821         balloon0 => { bus => 0, addr => 3 },
4822         watchdog => { bus => 0, addr => 4 },
4823         scsihw0 => { bus => 0, addr => 5 },
4824         'pci.3' => { bus => 0, addr => 5 }, #can also be used for virtio-scsi-single bridge
4825         scsihw1 => { bus => 0, addr => 6 },
4826         ahci0 => { bus => 0, addr => 7 },
4827         qga0 => { bus => 0, addr => 8 },
4828         spice => { bus => 0, addr => 9 },
4829         virtio0 => { bus => 0, addr => 10 },
4830         virtio1 => { bus => 0, addr => 11 },
4831         virtio2 => { bus => 0, addr => 12 },
4832         virtio3 => { bus => 0, addr => 13 },
4833         virtio4 => { bus => 0, addr => 14 },
4834         virtio5 => { bus => 0, addr => 15 },
4835         hostpci0 => { bus => 0, addr => 16 },
4836         hostpci1 => { bus => 0, addr => 17 },
4837         net0 => { bus => 0, addr => 18 },
4838         net1 => { bus => 0, addr => 19 },
4839         net2 => { bus => 0, addr => 20 },
4840         net3 => { bus => 0, addr => 21 },
4841         net4 => { bus => 0, addr => 22 },
4842         net5 => { bus => 0, addr => 23 },
4843         vga1 => { bus => 0, addr => 24 },
4844         vga2 => { bus => 0, addr => 25 },
4845         vga3 => { bus => 0, addr => 26 },
4846         hostpci2 => { bus => 0, addr => 27 },
4847         hostpci3 => { bus => 0, addr => 28 },
4848         #addr29 : usb-host (pve-usb.cfg)
4849         'pci.1' => { bus => 0, addr => 30 },
4850         'pci.2' => { bus => 0, addr => 31 },
4851         'net6' => { bus => 1, addr => 1 },
4852         'net7' => { bus => 1, addr => 2 },
4853         'net8' => { bus => 1, addr => 3 },
4854         'net9' => { bus => 1, addr => 4 },
4855         'net10' => { bus => 1, addr => 5 },
4856         'net11' => { bus => 1, addr => 6 },
4857         'net12' => { bus => 1, addr => 7 },
4858         'net13' => { bus => 1, addr => 8 },
4859         'net14' => { bus => 1, addr => 9 },
4860         'net15' => { bus => 1, addr => 10 },
4861         'net16' => { bus => 1, addr => 11 },
4862         'net17' => { bus => 1, addr => 12 },
4863         'net18' => { bus => 1, addr => 13 },
4864         'net19' => { bus => 1, addr => 14 },
4865         'net20' => { bus => 1, addr => 15 },
4866         'net21' => { bus => 1, addr => 16 },
4867         'net22' => { bus => 1, addr => 17 },
4868         'net23' => { bus => 1, addr => 18 },
4869         'net24' => { bus => 1, addr => 19 },
4870         'net25' => { bus => 1, addr => 20 },
4871         'net26' => { bus => 1, addr => 21 },
4872         'net27' => { bus => 1, addr => 22 },
4873         'net28' => { bus => 1, addr => 23 },
4874         'net29' => { bus => 1, addr => 24 },
4875         'net30' => { bus => 1, addr => 25 },
4876         'net31' => { bus => 1, addr => 26 },
4877         'virtio6' => { bus => 2, addr => 1 },
4878         'virtio7' => { bus => 2, addr => 2 },
4879         'virtio8' => { bus => 2, addr => 3 },
4880         'virtio9' => { bus => 2, addr => 4 },
4881         'virtio10' => { bus => 2, addr => 5 },
4882         'virtio11' => { bus => 2, addr => 6 },
4883         'virtio12' => { bus => 2, addr => 7 },
4884         'virtio13' => { bus => 2, addr => 8 },
4885         'virtio14' => { bus => 2, addr => 9 },
4886         'virtio15' => { bus => 2, addr => 10 },
4887         'virtioscsi0' => { bus => 3, addr => 1 },
4888         'virtioscsi1' => { bus => 3, addr => 2 },
4889         'virtioscsi2' => { bus => 3, addr => 3 },
4890         'virtioscsi3' => { bus => 3, addr => 4 },
4891         'virtioscsi4' => { bus => 3, addr => 5 },
4892         'virtioscsi5' => { bus => 3, addr => 6 },
4893         'virtioscsi6' => { bus => 3, addr => 7 },
4894         'virtioscsi7' => { bus => 3, addr => 8 },
4895         'virtioscsi8' => { bus => 3, addr => 9 },
4896         'virtioscsi9' => { bus => 3, addr => 10 },
4897         'virtioscsi10' => { bus => 3, addr => 11 },
4898         'virtioscsi11' => { bus => 3, addr => 12 },
4899         'virtioscsi12' => { bus => 3, addr => 13 },
4900         'virtioscsi13' => { bus => 3, addr => 14 },
4901         'virtioscsi14' => { bus => 3, addr => 15 },
4902         'virtioscsi15' => { bus => 3, addr => 16 },
4903         'virtioscsi16' => { bus => 3, addr => 17 },
4904         'virtioscsi17' => { bus => 3, addr => 18 },
4905         'virtioscsi18' => { bus => 3, addr => 19 },
4906         'virtioscsi19' => { bus => 3, addr => 20 },
4907         'virtioscsi20' => { bus => 3, addr => 21 },
4908         'virtioscsi21' => { bus => 3, addr => 22 },
4909         'virtioscsi22' => { bus => 3, addr => 23 },
4910         'virtioscsi23' => { bus => 3, addr => 24 },
4911         'virtioscsi24' => { bus => 3, addr => 25 },
4912         'virtioscsi25' => { bus => 3, addr => 26 },
4913         'virtioscsi26' => { bus => 3, addr => 27 },
4914         'virtioscsi27' => { bus => 3, addr => 28 },
4915         'virtioscsi28' => { bus => 3, addr => 29 },
4916         'virtioscsi29' => { bus => 3, addr => 30 },
4917         'virtioscsi30' => { bus => 3, addr => 31 },
4918
4919     };
4920
4921     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
4922            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
4923            my $bus = $devices->{$id}->{bus};
4924            $res = ",bus=pci.$bus,addr=$addr";
4925            $bridges->{$bus} = 1 if $bridges;
4926     }
4927     return $res;
4928
4929 }
4930
4931 sub print_pcie_addr {
4932     my ($id) = @_;
4933
4934     my $res = '';
4935     my $devices = {
4936         hostpci0 => { bus => "ich9-pcie-port-1", addr => 0 },
4937         hostpci1 => { bus => "ich9-pcie-port-2", addr => 0 },
4938         hostpci2 => { bus => "ich9-pcie-port-3", addr => 0 },
4939         hostpci3 => { bus => "ich9-pcie-port-4", addr => 0 },
4940     };
4941
4942     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
4943            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
4944            my $bus = $devices->{$id}->{bus};
4945            $res = ",bus=$bus,addr=$addr";
4946     }
4947     return $res;
4948
4949 }
4950
4951 # vzdump restore implementaion
4952
4953 sub tar_archive_read_firstfile {
4954     my $archive = shift;
4955
4956     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
4957
4958     # try to detect archive type first
4959     my $pid = open (TMP, "tar tf '$archive'|") ||
4960         die "unable to open file '$archive'\n";
4961     my $firstfile = <TMP>;
4962     kill 15, $pid;
4963     close TMP;
4964
4965     die "ERROR: archive contaions no data\n" if !$firstfile;
4966     chomp $firstfile;
4967
4968     return $firstfile;
4969 }
4970
4971 sub tar_restore_cleanup {
4972     my ($storecfg, $statfile) = @_;
4973
4974     print STDERR "starting cleanup\n";
4975
4976     if (my $fd = IO::File->new($statfile, "r")) {
4977         while (defined(my $line = <$fd>)) {
4978             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
4979                 my $volid = $2;
4980                 eval {
4981                     if ($volid =~ m|^/|) {
4982                         unlink $volid || die 'unlink failed\n';
4983                     } else {
4984                         PVE::Storage::vdisk_free($storecfg, $volid);
4985                     }
4986                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
4987                 };
4988                 print STDERR "unable to cleanup '$volid' - $@" if $@;
4989             } else {
4990                 print STDERR "unable to parse line in statfile - $line";
4991             }
4992         }
4993         $fd->close();
4994     }
4995 }
4996
4997 sub restore_archive {
4998     my ($archive, $vmid, $user, $opts) = @_;
4999
5000     my $format = $opts->{format};
5001     my $comp;
5002
5003     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5004         $format = 'tar' if !$format;
5005         $comp = 'gzip';
5006     } elsif ($archive =~ m/\.tar$/) {
5007         $format = 'tar' if !$format;
5008     } elsif ($archive =~ m/.tar.lzo$/) {
5009         $format = 'tar' if !$format;
5010         $comp = 'lzop';
5011     } elsif ($archive =~ m/\.vma$/) {
5012         $format = 'vma' if !$format;
5013     } elsif ($archive =~ m/\.vma\.gz$/) {
5014         $format = 'vma' if !$format;
5015         $comp = 'gzip';
5016     } elsif ($archive =~ m/\.vma\.lzo$/) {
5017         $format = 'vma' if !$format;
5018         $comp = 'lzop';
5019     } else {
5020         $format = 'vma' if !$format; # default
5021     }
5022
5023     # try to detect archive format
5024     if ($format eq 'tar') {
5025         return restore_tar_archive($archive, $vmid, $user, $opts);
5026     } else {
5027         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5028     }
5029 }
5030
5031 sub restore_update_config_line {
5032     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5033
5034     return if $line =~ m/^\#qmdump\#/;
5035     return if $line =~ m/^\#vzdump\#/;
5036     return if $line =~ m/^lock:/;
5037     return if $line =~ m/^unused\d+:/;
5038     return if $line =~ m/^parent:/;
5039     return if $line =~ m/^template:/; # restored VM is never a template
5040
5041     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5042         # try to convert old 1.X settings
5043         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5044         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5045             my ($model, $macaddr) = split(/\=/, $devconfig);
5046             $macaddr = PVE::Tools::random_ether_addr() if !$macaddr || $unique;
5047             my $net = {
5048                 model => $model,
5049                 bridge => "vmbr$ind",
5050                 macaddr => $macaddr,
5051             };
5052             my $netstr = print_net($net);
5053
5054             print $outfd "net$cookie->{netcount}: $netstr\n";
5055             $cookie->{netcount}++;
5056         }
5057     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5058         my ($id, $netstr) = ($1, $2);
5059         my $net = parse_net($netstr);
5060         $net->{macaddr} = PVE::Tools::random_ether_addr() if $net->{macaddr};
5061         $netstr = print_net($net);
5062         print $outfd "$id: $netstr\n";
5063     } elsif ($line =~ m/^((ide|scsi|virtio|sata)\d+):\s*(\S+)\s*$/) {
5064         my $virtdev = $1;
5065         my $value = $3;
5066         if ($line =~ m/backup=no/) {
5067             print $outfd "#$line";
5068         } elsif ($virtdev && $map->{$virtdev}) {
5069             my $di = parse_drive($virtdev, $value);
5070             delete $di->{format}; # format can change on restore
5071             $di->{file} = $map->{$virtdev};
5072             $value = print_drive($vmid, $di);
5073             print $outfd "$virtdev: $value\n";
5074         } else {
5075             print $outfd $line;
5076         }
5077     } else {
5078         print $outfd $line;
5079     }
5080 }
5081
5082 sub scan_volids {
5083     my ($cfg, $vmid) = @_;
5084
5085     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5086
5087     my $volid_hash = {};
5088     foreach my $storeid (keys %$info) {
5089         foreach my $item (@{$info->{$storeid}}) {
5090             next if !($item->{volid} && $item->{size});
5091             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5092             $volid_hash->{$item->{volid}} = $item;
5093         }
5094     }
5095
5096     return $volid_hash;
5097 }
5098
5099 sub get_used_paths {
5100     my ($vmid, $storecfg, $conf, $scan_snapshots, $skip_drive) = @_;
5101
5102     my $used_path = {};
5103
5104     my $scan_config = sub {
5105         my ($cref, $snapname) = @_;
5106
5107         foreach my $key (keys %$cref) {
5108             my $value = $cref->{$key};
5109             if (valid_drivename($key)) {
5110                 next if $skip_drive && $key eq $skip_drive;
5111                 my $drive = parse_drive($key, $value);
5112                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5113                 if ($drive->{file} =~ m!^/!) {
5114                     $used_path->{$drive->{file}}++; # = 1;
5115                 } else {
5116                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5117                     next if !$storeid;
5118                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5119                     next if !$scfg;
5120                     my $path = PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5121                     $used_path->{$path}++; # = 1;
5122                 }
5123             }
5124         }
5125     };
5126
5127     &$scan_config($conf);
5128
5129     undef $skip_drive;
5130
5131     if ($scan_snapshots) {
5132         foreach my $snapname (keys %{$conf->{snapshots}}) {
5133             &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5134         }
5135     }
5136
5137     return $used_path;
5138 }
5139
5140 sub update_disksize {
5141     my ($vmid, $conf, $volid_hash) = @_;
5142
5143     my $changes;
5144
5145     my $used = {};
5146
5147     # Note: it is allowed to define multiple storages with same path (alias), so
5148     # we need to check both 'volid' and real 'path' (two different volid can point
5149     # to the same path).
5150
5151     my $usedpath = {};
5152
5153     # update size info
5154     foreach my $opt (keys %$conf) {
5155         if (valid_drivename($opt)) {
5156             my $drive = parse_drive($opt, $conf->{$opt});
5157             my $volid = $drive->{file};
5158             next if !$volid;
5159
5160             $used->{$volid} = 1;
5161             if ($volid_hash->{$volid} &&
5162                 (my $path = $volid_hash->{$volid}->{path})) {
5163                 $usedpath->{$path} = 1;
5164             }
5165
5166             next if drive_is_cdrom($drive);
5167             next if !$volid_hash->{$volid};
5168
5169             $drive->{size} = $volid_hash->{$volid}->{size};
5170             my $new = print_drive($vmid, $drive);
5171             if ($new ne $conf->{$opt}) {
5172                 $changes = 1;
5173                 $conf->{$opt} = $new;
5174             }
5175         }
5176     }
5177
5178     # remove 'unusedX' entry if volume is used
5179     foreach my $opt (keys %$conf) {
5180         next if $opt !~ m/^unused\d+$/;
5181         my $volid = $conf->{$opt};
5182         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5183         if ($used->{$volid} || ($path && $usedpath->{$path})) {
5184             $changes = 1;
5185             delete $conf->{$opt};
5186         }
5187     }
5188
5189     foreach my $volid (sort keys %$volid_hash) {
5190         next if $volid =~ m/vm-$vmid-state-/;
5191         next if $used->{$volid};
5192         my $path = $volid_hash->{$volid}->{path};
5193         next if !$path; # just to be sure
5194         next if $usedpath->{$path};
5195         $changes = 1;
5196         add_unused_volume($conf, $volid);
5197         $usedpath->{$path} = 1; # avoid to add more than once (aliases)
5198     }
5199
5200     return $changes;
5201 }
5202
5203 sub rescan {
5204     my ($vmid, $nolock) = @_;
5205
5206     my $cfg = PVE::Cluster::cfs_read_file("storage.cfg");
5207
5208     my $volid_hash = scan_volids($cfg, $vmid);
5209
5210     my $updatefn =  sub {
5211         my ($vmid) = @_;
5212
5213         my $conf = load_config($vmid);
5214
5215         check_lock($conf);
5216
5217         my $vm_volids = {};
5218         foreach my $volid (keys %$volid_hash) {
5219             my $info = $volid_hash->{$volid};
5220             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5221         }
5222
5223         my $changes = update_disksize($vmid, $conf, $vm_volids);
5224
5225         update_config_nolock($vmid, $conf, 1) if $changes;
5226     };
5227
5228     if (defined($vmid)) {
5229         if ($nolock) {
5230             &$updatefn($vmid);
5231         } else {
5232             lock_config($vmid, $updatefn, $vmid);
5233         }
5234     } else {
5235         my $vmlist = config_list();
5236         foreach my $vmid (keys %$vmlist) {
5237             if ($nolock) {
5238                 &$updatefn($vmid);
5239             } else {
5240                 lock_config($vmid, $updatefn, $vmid);
5241             }
5242         }
5243     }
5244 }
5245
5246 sub restore_vma_archive {
5247     my ($archive, $vmid, $user, $opts, $comp) = @_;
5248
5249     my $input = $archive eq '-' ? "<&STDIN" : undef;
5250     my $readfrom = $archive;
5251
5252     my $uncomp = '';
5253     if ($comp) {
5254         $readfrom = '-';
5255         my $qarchive = PVE::Tools::shellquote($archive);
5256         if ($comp eq 'gzip') {
5257             $uncomp = "zcat $qarchive|";
5258         } elsif ($comp eq 'lzop') {
5259             $uncomp = "lzop -d -c $qarchive|";
5260         } else {
5261             die "unknown compression method '$comp'\n";
5262         }
5263
5264     }
5265
5266     my $tmpdir = "/var/tmp/vzdumptmp$$";
5267     rmtree $tmpdir;
5268
5269     # disable interrupts (always do cleanups)
5270     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5271         warn "got interrupt - ignored\n";
5272     };
5273
5274     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5275     POSIX::mkfifo($mapfifo, 0600);
5276     my $fifofh;
5277
5278     my $openfifo = sub {
5279         open($fifofh, '>', $mapfifo) || die $!;
5280     };
5281
5282     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5283
5284     my $oldtimeout;
5285     my $timeout = 5;
5286
5287     my $devinfo = {};
5288
5289     my $rpcenv = PVE::RPCEnvironment::get();
5290
5291     my $conffile = config_file($vmid);
5292     my $tmpfn = "$conffile.$$.tmp";
5293
5294     # Note: $oldconf is undef if VM does not exists
5295     my $oldconf = PVE::Cluster::cfs_read_file(cfs_config_path($vmid));
5296
5297     my $print_devmap = sub {
5298         my $virtdev_hash = {};
5299
5300         my $cfgfn = "$tmpdir/qemu-server.conf";
5301
5302         # we can read the config - that is already extracted
5303         my $fh = IO::File->new($cfgfn, "r") ||
5304             "unable to read qemu-server.conf - $!\n";
5305
5306         my $fwcfgfn = "$tmpdir/qemu-server.fw";
5307         PVE::Tools::file_copy($fwcfgfn, "/etc/pve/firewall/$vmid.fw")
5308             if -f $fwcfgfn;
5309
5310         while (defined(my $line = <$fh>)) {
5311             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5312                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5313                 die "archive does not contain data for drive '$virtdev'\n"
5314                     if !$devinfo->{$devname};
5315                 if (defined($opts->{storage})) {
5316                     $storeid = $opts->{storage} || 'local';
5317                 } elsif (!$storeid) {
5318                     $storeid = 'local';
5319                 }
5320                 $format = 'raw' if !$format;
5321                 $devinfo->{$devname}->{devname} = $devname;
5322                 $devinfo->{$devname}->{virtdev} = $virtdev;
5323                 $devinfo->{$devname}->{format} = $format;
5324                 $devinfo->{$devname}->{storeid} = $storeid;
5325
5326                 # check permission on storage
5327                 my $pool = $opts->{pool}; # todo: do we need that?
5328                 if ($user ne 'root@pam') {
5329                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5330                 }
5331
5332                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5333             }
5334         }
5335
5336         foreach my $devname (keys %$devinfo) {
5337             die "found no device mapping information for device '$devname'\n"
5338                 if !$devinfo->{$devname}->{virtdev};
5339         }
5340
5341         my $cfg = cfs_read_file('storage.cfg');
5342
5343         # create empty/temp config
5344         if ($oldconf) {
5345             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5346             foreach_drive($oldconf, sub {
5347                 my ($ds, $drive) = @_;
5348
5349                 return if drive_is_cdrom($drive);
5350
5351                 my $volid = $drive->{file};
5352
5353                 return if !$volid || $volid =~ m|^/|;
5354
5355                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5356                 return if !$path || !$owner || ($owner != $vmid);
5357
5358                 # Note: only delete disk we want to restore
5359                 # other volumes will become unused
5360                 if ($virtdev_hash->{$ds}) {
5361                     PVE::Storage::vdisk_free($cfg, $volid);
5362                 }
5363             });
5364         }
5365
5366         my $map = {};
5367         foreach my $virtdev (sort keys %$virtdev_hash) {
5368             my $d = $virtdev_hash->{$virtdev};
5369             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5370             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5371
5372             # test if requested format is supported
5373             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5374             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5375             $d->{format} = $defFormat if !$supported;
5376
5377             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5378                                                   $d->{format}, undef, $alloc_size);
5379             print STDERR "new volume ID is '$volid'\n";
5380             $d->{volid} = $volid;
5381             my $path = PVE::Storage::path($cfg, $volid);
5382
5383             PVE::Storage::activate_volumes($cfg,[$volid]);
5384
5385             my $write_zeros = 1;
5386             # fixme: what other storages types initialize volumes with zero?
5387             if ($scfg->{type} eq 'dir' || $scfg->{type} eq 'nfs' || $scfg->{type} eq 'glusterfs' ||
5388                 $scfg->{type} eq 'sheepdog' || $scfg->{type} eq 'rbd') {
5389                 $write_zeros = 0;
5390             }
5391
5392             print $fifofh "${write_zeros}:$d->{devname}=$path\n";
5393
5394             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5395             $map->{$virtdev} = $volid;
5396         }
5397
5398         $fh->seek(0, 0) || die "seek failed - $!\n";
5399
5400         my $outfd = new IO::File ($tmpfn, "w") ||
5401             die "unable to write config for VM $vmid\n";
5402
5403         my $cookie = { netcount => 0 };
5404         while (defined(my $line = <$fh>)) {
5405             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5406         }
5407
5408         $fh->close();
5409         $outfd->close();
5410     };
5411
5412     eval {
5413         # enable interrupts
5414         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5415             die "interrupted by signal\n";
5416         };
5417         local $SIG{ALRM} = sub { die "got timeout\n"; };
5418
5419         $oldtimeout = alarm($timeout);
5420
5421         my $parser = sub {
5422             my $line = shift;
5423
5424             print "$line\n";
5425
5426             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5427                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5428                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5429             } elsif ($line =~ m/^CTIME: /) {
5430                 # we correctly received the vma config, so we can disable
5431                 # the timeout now for disk allocation (set to 10 minutes, so
5432                 # that we always timeout if something goes wrong)
5433                 alarm(600);
5434                 &$print_devmap();
5435                 print $fifofh "done\n";
5436                 my $tmp = $oldtimeout || 0;
5437                 $oldtimeout = undef;
5438                 alarm($tmp);
5439                 close($fifofh);
5440             }
5441         };
5442
5443         print "restore vma archive: $cmd\n";
5444         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5445     };
5446     my $err = $@;
5447
5448     alarm($oldtimeout) if $oldtimeout;
5449
5450     my $vollist = [];
5451     foreach my $devname (keys %$devinfo) {
5452         my $volid = $devinfo->{$devname}->{volid};
5453         push @$vollist, $volid if $volid;
5454     }
5455
5456     my $cfg = cfs_read_file('storage.cfg');
5457     PVE::Storage::deactivate_volumes($cfg, $vollist);
5458
5459     unlink $mapfifo;
5460
5461     if ($err) {
5462         rmtree $tmpdir;
5463         unlink $tmpfn;
5464
5465         foreach my $devname (keys %$devinfo) {
5466             my $volid = $devinfo->{$devname}->{volid};
5467             next if !$volid;
5468             eval {
5469                 if ($volid =~ m|^/|) {
5470                     unlink $volid || die 'unlink failed\n';
5471                 } else {
5472                     PVE::Storage::vdisk_free($cfg, $volid);
5473                 }
5474                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5475             };
5476             print STDERR "unable to cleanup '$volid' - $@" if $@;
5477         }
5478         die $err;
5479     }
5480
5481     rmtree $tmpdir;
5482
5483     rename($tmpfn, $conffile) ||
5484         die "unable to commit configuration file '$conffile'\n";
5485
5486     PVE::Cluster::cfs_update(); # make sure we read new file
5487
5488     eval { rescan($vmid, 1); };
5489     warn $@ if $@;
5490 }
5491
5492 sub restore_tar_archive {
5493     my ($archive, $vmid, $user, $opts) = @_;
5494
5495     if ($archive ne '-') {
5496         my $firstfile = tar_archive_read_firstfile($archive);
5497         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5498             if $firstfile ne 'qemu-server.conf';
5499     }
5500
5501     my $storecfg = cfs_read_file('storage.cfg');
5502
5503     # destroy existing data - keep empty config
5504     my $vmcfgfn = config_file($vmid);
5505     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5506
5507     my $tocmd = "/usr/lib/qemu-server/qmextract";
5508
5509     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5510     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5511     $tocmd .= ' --prealloc' if $opts->{prealloc};
5512     $tocmd .= ' --info' if $opts->{info};
5513
5514     # tar option "xf" does not autodetect compression when read from STDIN,
5515     # so we pipe to zcat
5516     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5517         PVE::Tools::shellquote("--to-command=$tocmd");
5518
5519     my $tmpdir = "/var/tmp/vzdumptmp$$";
5520     mkpath $tmpdir;
5521
5522     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5523     local $ENV{VZDUMP_VMID} = $vmid;
5524     local $ENV{VZDUMP_USER} = $user;
5525
5526     my $conffile = config_file($vmid);
5527     my $tmpfn = "$conffile.$$.tmp";
5528
5529     # disable interrupts (always do cleanups)
5530     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5531         print STDERR "got interrupt - ignored\n";
5532     };
5533
5534     eval {
5535         # enable interrupts
5536         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5537             die "interrupted by signal\n";
5538         };
5539
5540         if ($archive eq '-') {
5541             print "extracting archive from STDIN\n";
5542             run_command($cmd, input => "<&STDIN");
5543         } else {
5544             print "extracting archive '$archive'\n";
5545             run_command($cmd);
5546         }
5547
5548         return if $opts->{info};
5549
5550         # read new mapping
5551         my $map = {};
5552         my $statfile = "$tmpdir/qmrestore.stat";
5553         if (my $fd = IO::File->new($statfile, "r")) {
5554             while (defined (my $line = <$fd>)) {
5555                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5556                     $map->{$1} = $2 if $1;
5557                 } else {
5558                     print STDERR "unable to parse line in statfile - $line\n";
5559                 }
5560             }
5561             $fd->close();
5562         }
5563
5564         my $confsrc = "$tmpdir/qemu-server.conf";
5565
5566         my $srcfd = new IO::File($confsrc, "r") ||
5567             die "unable to open file '$confsrc'\n";
5568
5569         my $outfd = new IO::File ($tmpfn, "w") ||
5570             die "unable to write config for VM $vmid\n";
5571
5572         my $cookie = { netcount => 0 };
5573         while (defined (my $line = <$srcfd>)) {
5574             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5575         }
5576
5577         $srcfd->close();
5578         $outfd->close();
5579     };
5580     my $err = $@;
5581
5582     if ($err) {
5583
5584         unlink $tmpfn;
5585
5586         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
5587
5588         die $err;
5589     }
5590
5591     rmtree $tmpdir;
5592
5593     rename $tmpfn, $conffile ||
5594         die "unable to commit configuration file '$conffile'\n";
5595
5596     PVE::Cluster::cfs_update(); # make sure we read new file
5597
5598     eval { rescan($vmid, 1); };
5599     warn $@ if $@;
5600 };
5601
5602
5603 # Internal snapshots
5604
5605 # NOTE: Snapshot create/delete involves several non-atomic
5606 # action, and can take a long time.
5607 # So we try to avoid locking the file and use 'lock' variable
5608 # inside the config file instead.
5609
5610 my $snapshot_copy_config = sub {
5611     my ($source, $dest) = @_;
5612
5613     foreach my $k (keys %$source) {
5614         next if $k eq 'snapshots';
5615         next if $k eq 'snapstate';
5616         next if $k eq 'snaptime';
5617         next if $k eq 'vmstate';
5618         next if $k eq 'lock';
5619         next if $k eq 'digest';
5620         next if $k eq 'description';
5621         next if $k =~ m/^unused\d+$/;
5622
5623         $dest->{$k} = $source->{$k};
5624     }
5625 };
5626
5627 my $snapshot_apply_config = sub {
5628     my ($conf, $snap) = @_;
5629
5630     # copy snapshot list
5631     my $newconf = {
5632         snapshots => $conf->{snapshots},
5633     };
5634
5635     # keep description and list of unused disks
5636     foreach my $k (keys %$conf) {
5637         next if !($k =~ m/^unused\d+$/ || $k eq 'description');
5638         $newconf->{$k} = $conf->{$k};
5639     }
5640
5641     &$snapshot_copy_config($snap, $newconf);
5642
5643     return $newconf;
5644 };
5645
5646 sub foreach_writable_storage {
5647     my ($conf, $func) = @_;
5648
5649     my $sidhash = {};
5650
5651     foreach my $ds (keys %$conf) {
5652         next if !valid_drivename($ds);
5653
5654         my $drive = parse_drive($ds, $conf->{$ds});
5655         next if !$drive;
5656         next if drive_is_cdrom($drive);
5657
5658         my $volid = $drive->{file};
5659
5660         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5661         $sidhash->{$sid} = $sid if $sid;
5662     }
5663
5664     foreach my $sid (sort keys %$sidhash) {
5665         &$func($sid);
5666     }
5667 }
5668
5669 my $alloc_vmstate_volid = sub {
5670     my ($storecfg, $vmid, $conf, $snapname) = @_;
5671
5672     # Note: we try to be smart when selecting a $target storage
5673
5674     my $target;
5675
5676     # search shared storage first
5677     foreach_writable_storage($conf, sub {
5678         my ($sid) = @_;
5679         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
5680         return if !$scfg->{shared};
5681
5682         $target = $sid if !$target || $scfg->{path}; # prefer file based storage
5683     });
5684
5685     if (!$target) {
5686         # now search local storage
5687         foreach_writable_storage($conf, sub {
5688             my ($sid) = @_;
5689             my $scfg = PVE::Storage::storage_config($storecfg, $sid);
5690             return if $scfg->{shared};
5691
5692             $target = $sid if !$target || $scfg->{path}; # prefer file based storage;
5693         });
5694     }
5695
5696     $target = 'local' if !$target;
5697
5698     my $driver_state_size = 500; # assume 32MB is enough to safe all driver state;
5699     # we abort live save after $conf->{memory}, so we need at max twice that space
5700     my $size = $conf->{memory}*2 + $driver_state_size;
5701
5702     my $name = "vm-$vmid-state-$snapname";
5703     my $scfg = PVE::Storage::storage_config($storecfg, $target);
5704     $name .= ".raw" if $scfg->{path}; # add filename extension for file base storage
5705     my $volid = PVE::Storage::vdisk_alloc($storecfg, $target, $vmid, 'raw', $name, $size*1024);
5706
5707     return $volid;
5708 };
5709
5710 my $snapshot_prepare = sub {
5711     my ($vmid, $snapname, $save_vmstate, $comment) = @_;
5712
5713     my $snap;
5714
5715     my $updatefn =  sub {
5716
5717         my $conf = load_config($vmid);
5718
5719         die "you can't take a snapshot if it's a template\n"
5720             if is_template($conf);
5721
5722         check_lock($conf);
5723
5724         $conf->{lock} = 'snapshot';
5725
5726         die "snapshot name '$snapname' already used\n"
5727             if defined($conf->{snapshots}->{$snapname});
5728
5729         my $storecfg = PVE::Storage::config();
5730         die "snapshot feature is not available" if !has_feature('snapshot', $conf, $storecfg);
5731
5732         $snap = $conf->{snapshots}->{$snapname} = {};
5733
5734         if ($save_vmstate && check_running($vmid)) {
5735             $snap->{vmstate} = &$alloc_vmstate_volid($storecfg, $vmid, $conf, $snapname);
5736         }
5737
5738         &$snapshot_copy_config($conf, $snap);
5739
5740         $snap->{snapstate} = "prepare";
5741         $snap->{snaptime} = time();
5742         $snap->{description} = $comment if $comment;
5743
5744         # always overwrite machine if we save vmstate. This makes sure we
5745         # can restore it later using correct machine type
5746         $snap->{machine} = get_current_qemu_machine($vmid) if $snap->{vmstate};
5747
5748         update_config_nolock($vmid, $conf, 1);
5749     };
5750
5751     lock_config($vmid, $updatefn);
5752
5753     return $snap;
5754 };
5755
5756 my $snapshot_commit = sub {
5757     my ($vmid, $snapname) = @_;
5758
5759     my $updatefn = sub {
5760
5761         my $conf = load_config($vmid);
5762
5763         die "missing snapshot lock\n"
5764             if !($conf->{lock} && $conf->{lock} eq 'snapshot');
5765
5766         my $has_machine_config = defined($conf->{machine});
5767
5768         my $snap = $conf->{snapshots}->{$snapname};
5769
5770         die "snapshot '$snapname' does not exist\n" if !defined($snap);
5771
5772         die "wrong snapshot state\n"
5773             if !($snap->{snapstate} && $snap->{snapstate} eq "prepare");
5774
5775         delete $snap->{snapstate};
5776         delete $conf->{lock};
5777
5778         my $newconf = &$snapshot_apply_config($conf, $snap);
5779
5780         delete $newconf->{machine} if !$has_machine_config;
5781
5782         $newconf->{parent} = $snapname;
5783
5784         update_config_nolock($vmid, $newconf, 1);
5785     };
5786
5787     lock_config($vmid, $updatefn);
5788 };
5789
5790 sub snapshot_rollback {
5791     my ($vmid, $snapname) = @_;
5792
5793     my $prepare = 1;
5794
5795     my $storecfg = PVE::Storage::config();
5796
5797     my $conf = load_config($vmid);
5798
5799     my $get_snapshot_config = sub {
5800
5801         die "you can't rollback if vm is a template\n" if is_template($conf);
5802
5803         my $res = $conf->{snapshots}->{$snapname};
5804
5805         die "snapshot '$snapname' does not exist\n" if !defined($res);
5806
5807         return $res;
5808     };
5809
5810     my $snap = &$get_snapshot_config();
5811
5812     foreach_drive($snap, sub {
5813         my ($ds, $drive) = @_;
5814
5815         return if drive_is_cdrom($drive);
5816
5817         my $volid = $drive->{file};
5818
5819         PVE::Storage::volume_rollback_is_possible($storecfg, $volid, $snapname);
5820     });
5821
5822     my $updatefn = sub {
5823
5824         $conf = load_config($vmid);
5825
5826         $snap = &$get_snapshot_config();
5827
5828         die "unable to rollback to incomplete snapshot (snapstate = $snap->{snapstate})\n"
5829             if $snap->{snapstate};
5830
5831         if ($prepare) {
5832             check_lock($conf);
5833             vm_stop($storecfg, $vmid, undef, undef, 5, undef, undef);
5834         }
5835
5836         die "unable to rollback vm $vmid: vm is running\n"
5837             if check_running($vmid);
5838
5839         if ($prepare) {
5840             $conf->{lock} = 'rollback';
5841         } else {
5842             die "got wrong lock\n" if !($conf->{lock} && $conf->{lock} eq 'rollback');
5843             delete $conf->{lock};
5844         }
5845
5846         my $forcemachine;
5847
5848         if (!$prepare) {
5849             my $has_machine_config = defined($conf->{machine});
5850
5851             # copy snapshot config to current config
5852             $conf = &$snapshot_apply_config($conf, $snap);
5853             $conf->{parent} = $snapname;
5854
5855             # Note: old code did not store 'machine', so we try to be smart
5856             # and guess the snapshot was generated with kvm 1.4 (pc-i440fx-1.4).
5857             $forcemachine = $conf->{machine} || 'pc-i440fx-1.4';
5858             # we remove the 'machine' configuration if not explicitly specified
5859             # in the original config.
5860             delete $conf->{machine} if $snap->{vmstate} && !$has_machine_config;
5861         }
5862
5863         update_config_nolock($vmid, $conf, 1);
5864
5865         if (!$prepare && $snap->{vmstate}) {
5866             my $statefile = PVE::Storage::path($storecfg, $snap->{vmstate});
5867             vm_start($storecfg, $vmid, $statefile, undef, undef, undef, $forcemachine);
5868         }
5869     };
5870
5871     lock_config($vmid, $updatefn);
5872
5873     foreach_drive($snap, sub {
5874         my ($ds, $drive) = @_;
5875
5876         return if drive_is_cdrom($drive);
5877
5878         my $volid = $drive->{file};
5879         my $device = "drive-$ds";
5880
5881         PVE::Storage::volume_snapshot_rollback($storecfg, $volid, $snapname);
5882     });
5883
5884     $prepare = 0;
5885     lock_config($vmid, $updatefn);
5886 }
5887
5888 my $savevm_wait = sub {
5889     my ($vmid) = @_;
5890
5891     for(;;) {
5892         my $stat = vm_mon_cmd_nocheck($vmid, "query-savevm");
5893         if (!$stat->{status}) {
5894             die "savevm not active\n";
5895         } elsif ($stat->{status} eq 'active') {
5896             sleep(1);
5897             next;
5898         } elsif ($stat->{status} eq 'completed') {
5899             last;
5900         } else {
5901             die "query-savevm returned status '$stat->{status}'\n";
5902         }
5903     }
5904 };
5905
5906 sub do_snapshots_with_qemu {
5907     my ($storecfg, $volid) = @_;
5908
5909     my $storage_name = PVE::Storage::parse_volume_id($volid);
5910
5911     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
5912         && !$storecfg->{ids}->{$storage_name}->{krbd}){
5913         return 1;
5914     }
5915
5916     if ($volid =~ m/\.(qcow2|qed)$/){
5917         return 1;
5918     }
5919
5920     return undef;
5921 }
5922
5923 sub snapshot_create {
5924     my ($vmid, $snapname, $save_vmstate, $comment) = @_;
5925
5926     my $snap = &$snapshot_prepare($vmid, $snapname, $save_vmstate, $comment);
5927
5928     $save_vmstate = 0 if !$snap->{vmstate}; # vm is not running
5929
5930     my $config = load_config($vmid);
5931
5932     my $running = check_running($vmid);
5933
5934     my $freezefs = $running && $config->{agent};
5935     $freezefs = 0 if $snap->{vmstate}; # not needed if we save RAM
5936
5937     my $drivehash = {};
5938
5939     if ($freezefs) {
5940         eval { vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
5941         warn "guest-fsfreeze-freeze problems - $@" if $@;
5942     }
5943
5944     eval {
5945         # create internal snapshots of all drives
5946
5947         my $storecfg = PVE::Storage::config();
5948
5949         if ($running) {
5950             if ($snap->{vmstate}) {
5951                 my $path = PVE::Storage::path($storecfg, $snap->{vmstate});
5952                 vm_mon_cmd($vmid, "savevm-start", statefile => $path);
5953                 &$savevm_wait($vmid);
5954             } else {
5955                 vm_mon_cmd($vmid, "savevm-start");
5956             }
5957         };
5958
5959         foreach_drive($snap, sub {
5960             my ($ds, $drive) = @_;
5961
5962             return if drive_is_cdrom($drive);
5963
5964             my $volid = $drive->{file};
5965             my $device = "drive-$ds";
5966
5967             qemu_volume_snapshot($vmid, $device, $storecfg, $volid, $snapname);
5968             $drivehash->{$ds} = 1;
5969        });
5970     };
5971     my $err = $@;
5972
5973     if ($running) {
5974         eval { vm_mon_cmd($vmid, "savevm-end")  };
5975         warn $@ if $@;
5976
5977         if ($freezefs) {
5978             eval { vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
5979             warn "guest-fsfreeze-thaw problems - $@" if $@;
5980         }
5981
5982         # savevm-end is async, we need to wait
5983         for (;;) {
5984             my $stat = vm_mon_cmd_nocheck($vmid, "query-savevm");
5985             if (!$stat->{bytes}) {
5986                 last;
5987             } else {
5988                 print "savevm not yet finished\n";
5989                 sleep(1);
5990                 next;
5991             }
5992         }
5993     }
5994
5995     if ($err) {
5996         warn "snapshot create failed: starting cleanup\n";
5997         eval { snapshot_delete($vmid, $snapname, 0, $drivehash); };
5998         warn $@ if $@;
5999         die $err;
6000     }
6001
6002     &$snapshot_commit($vmid, $snapname);
6003 }
6004
6005 # Note: $drivehash is only set when called from snapshot_create.
6006 sub snapshot_delete {
6007     my ($vmid, $snapname, $force, $drivehash) = @_;
6008
6009     my $prepare = 1;
6010
6011     my $snap;
6012     my $unused = [];
6013
6014     my $unlink_parent = sub {
6015         my ($confref, $new_parent) = @_;
6016
6017         if ($confref->{parent} && $confref->{parent} eq $snapname) {
6018             if ($new_parent) {
6019                 $confref->{parent} = $new_parent;
6020             } else {
6021                 delete $confref->{parent};
6022             }
6023         }
6024     };
6025
6026     my $updatefn =  sub {
6027         my ($remove_drive) = @_;
6028
6029         my $conf = load_config($vmid);
6030
6031         if (!$drivehash) {
6032             check_lock($conf);
6033             die "you can't delete a snapshot if vm is a template\n"
6034                 if is_template($conf);
6035         }
6036
6037         $snap = $conf->{snapshots}->{$snapname};
6038
6039         die "snapshot '$snapname' does not exist\n" if !defined($snap);
6040
6041         # remove parent refs
6042         if (!$prepare) {
6043             &$unlink_parent($conf, $snap->{parent});
6044             foreach my $sn (keys %{$conf->{snapshots}}) {
6045                 next if $sn eq $snapname;
6046                 &$unlink_parent($conf->{snapshots}->{$sn}, $snap->{parent});
6047             }
6048         }
6049
6050         if ($remove_drive) {
6051             if ($remove_drive eq 'vmstate') {
6052                 delete $snap->{$remove_drive};
6053             } else {
6054                 my $drive = parse_drive($remove_drive, $snap->{$remove_drive});
6055                 my $volid = $drive->{file};
6056                 delete $snap->{$remove_drive};
6057                 add_unused_volume($conf, $volid);
6058             }
6059         }
6060
6061         if ($prepare) {
6062             $snap->{snapstate} = 'delete';
6063         } else {
6064             delete $conf->{snapshots}->{$snapname};
6065             delete $conf->{lock} if $drivehash;
6066             foreach my $volid (@$unused) {
6067                 add_unused_volume($conf, $volid);
6068             }
6069         }
6070
6071         update_config_nolock($vmid, $conf, 1);
6072     };
6073
6074     lock_config($vmid, $updatefn);
6075
6076     # now remove vmstate file
6077
6078     my $storecfg = PVE::Storage::config();
6079
6080     if ($snap->{vmstate}) {
6081         eval {  PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
6082         if (my $err = $@) {
6083             die $err if !$force;
6084             warn $err;
6085         }
6086         # save changes (remove vmstate from snapshot)
6087         lock_config($vmid, $updatefn, 'vmstate') if !$force;
6088     };
6089
6090     # now remove all internal snapshots
6091     foreach_drive($snap, sub {
6092         my ($ds, $drive) = @_;
6093
6094         return if drive_is_cdrom($drive);
6095
6096         my $volid = $drive->{file};
6097         my $device = "drive-$ds";
6098
6099         if (!$drivehash || $drivehash->{$ds}) {
6100             eval { qemu_volume_snapshot_delete($vmid, $device, $storecfg, $volid, $snapname); };
6101             if (my $err = $@) {
6102                 die $err if !$force;
6103                 warn $err;
6104             }
6105         }
6106
6107         # save changes (remove drive fron snapshot)
6108         lock_config($vmid, $updatefn, $ds) if !$force;
6109         push @$unused, $volid;
6110     });
6111
6112     # now cleanup config
6113     $prepare = 0;
6114     lock_config($vmid, $updatefn);
6115 }
6116
6117 sub has_feature {
6118     my ($feature, $conf, $storecfg, $snapname, $running) = @_;
6119
6120     my $err;
6121     foreach_drive($conf, sub {
6122         my ($ds, $drive) = @_;
6123
6124         return if drive_is_cdrom($drive);
6125         my $volid = $drive->{file};
6126         $err = 1 if !PVE::Storage::volume_has_feature($storecfg, $feature, $volid, $snapname, $running);
6127     });
6128
6129     return $err ? 0 : 1;
6130 }
6131
6132 sub template_create {
6133     my ($vmid, $conf, $disk) = @_;
6134
6135     my $storecfg = PVE::Storage::config();
6136
6137     foreach_drive($conf, sub {
6138         my ($ds, $drive) = @_;
6139
6140         return if drive_is_cdrom($drive);
6141         return if $disk && $ds ne $disk;
6142
6143         my $volid = $drive->{file};
6144         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6145
6146         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6147         $drive->{file} = $voliddst;
6148         $conf->{$ds} = print_drive($vmid, $drive);
6149         update_config_nolock($vmid, $conf, 1);
6150     });
6151 }
6152
6153 sub is_template {
6154     my ($conf) = @_;
6155
6156     return 1 if defined $conf->{template} && $conf->{template} == 1;
6157 }
6158
6159 sub qemu_img_convert {
6160     my ($src_volid, $dst_volid, $size, $snapname) = @_;
6161
6162     my $storecfg = PVE::Storage::config();
6163     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6164     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6165
6166     if ($src_storeid && $dst_storeid) {
6167
6168         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6169
6170         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6171         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6172
6173         my $src_format = qemu_img_format($src_scfg, $src_volname);
6174         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6175
6176         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6177         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6178
6179         my $cmd = [];
6180         push @$cmd, '/usr/bin/qemu-img', 'convert', '-t', 'writeback', '-p', '-n';
6181         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
6182         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path, $dst_path;
6183
6184         my $parser = sub {
6185             my $line = shift;
6186             if($line =~ m/\((\S+)\/100\%\)/){
6187                 my $percent = $1;
6188                 my $transferred = int($size * $percent / 100);
6189                 my $remaining = $size - $transferred;
6190
6191                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6192             }
6193
6194         };
6195
6196         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6197         my $err = $@;
6198         die "copy failed: $err" if $err;
6199     }
6200 }
6201
6202 sub qemu_img_format {
6203     my ($scfg, $volname) = @_;
6204
6205     if ($scfg->{path} && $volname =~ m/\.(raw|cow|qcow|qcow2|qed|vmdk|cloop)$/) {
6206         return $1;
6207     } else {
6208         return "raw";
6209     }
6210 }
6211
6212 sub qemu_drive_mirror {
6213     my ($vmid, $drive, $dst_volid, $vmiddst) = @_;
6214
6215     my $storecfg = PVE::Storage::config();
6216     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6217
6218     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6219
6220     my $format = qemu_img_format($dst_scfg, $dst_volname);
6221
6222     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6223
6224     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $dst_path };
6225     $opts->{format} = $format if $format;
6226
6227     print "drive mirror is starting (scanning bitmap) : this step can take some minutes/hours, depend of disk size and storage speed\n";
6228
6229     eval {
6230     vm_mon_cmd($vmid, "drive-mirror", %$opts);
6231         while (1) {
6232             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6233             my $stat = @$stats[0];
6234             die "mirroring job seem to have die. Maybe do you have bad sectors?" if !$stat;
6235             die "error job is not mirroring" if $stat->{type} ne "mirror";
6236
6237             my $busy = $stat->{busy};
6238             my $ready = $stat->{ready};
6239
6240             if (my $total = $stat->{len}) {
6241                 my $transferred = $stat->{offset} || 0;
6242                 my $remaining = $total - $transferred;
6243                 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6244
6245                 print "transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6246             }
6247
6248
6249             if ($stat->{ready} eq 'true') {
6250
6251                 last if $vmiddst != $vmid;
6252
6253                 # try to switch the disk if source and destination are on the same guest
6254                 eval { vm_mon_cmd($vmid, "block-job-complete", device => "drive-$drive") };
6255                 last if !$@;
6256                 die $@ if $@ !~ m/cannot be completed/;
6257             }
6258             sleep 1;
6259         }
6260
6261
6262     };
6263     my $err = $@;
6264
6265     my $cancel_job = sub {
6266         vm_mon_cmd($vmid, "block-job-cancel", device => "drive-$drive");
6267         while (1) {
6268             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6269             my $stat = @$stats[0];
6270             last if !$stat;
6271             sleep 1;
6272         }
6273     };
6274
6275     if ($err) {
6276         eval { &$cancel_job(); };
6277         die "mirroring error: $err";
6278     }
6279
6280     if ($vmiddst != $vmid) {
6281         # if we clone a disk for a new target vm, we don't switch the disk
6282         &$cancel_job(); # so we call block-job-cancel
6283     }
6284 }
6285
6286 sub clone_disk {
6287     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6288         $newvmid, $storage, $format, $full, $newvollist) = @_;
6289
6290     my $newvolid;
6291
6292     if (!$full) {
6293         print "create linked clone of drive $drivename ($drive->{file})\n";
6294         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6295         push @$newvollist, $newvolid;
6296     } else {
6297         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6298         $storeid = $storage if $storage;
6299
6300         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6301         if (!$format) {
6302             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6303             $format = qemu_img_format($scfg, $volname);
6304         }
6305
6306         # test if requested format is supported - else use default
6307         my $supported = grep { $_ eq $format } @$validFormats;
6308         $format = $defFormat if !$supported;
6309
6310         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6311
6312         print "create full clone of drive $drivename ($drive->{file})\n";
6313         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $format, undef, ($size/1024));
6314         push @$newvollist, $newvolid;
6315
6316         PVE::Storage::activate_volumes($storecfg, $newvollist);
6317
6318         if (!$running || $snapname) {
6319             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname);
6320         } else {
6321             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid);
6322         }
6323     }
6324
6325     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6326
6327     my $disk = $drive;
6328     $disk->{format} = undef;
6329     $disk->{file} = $newvolid;
6330     $disk->{size} = $size;
6331
6332     return $disk;
6333 }
6334
6335 # this only works if VM is running
6336 sub get_current_qemu_machine {
6337     my ($vmid) = @_;
6338
6339     my $cmd = { execute => 'query-machines', arguments => {} };
6340     my $res = vm_qmp_command($vmid, $cmd);
6341
6342     my ($current, $default);
6343     foreach my $e (@$res) {
6344         $default = $e->{name} if $e->{'is-default'};
6345         $current = $e->{name} if $e->{'is-current'};
6346     }
6347
6348     # fallback to the default machine if current is not supported by qemu
6349     return $current || $default || 'pc';
6350 }
6351
6352 sub qemu_machine_feature_enabled {
6353     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6354
6355     my $current_major;
6356     my $current_minor;
6357
6358     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6359
6360         $current_major = $3;
6361         $current_minor = $4;
6362
6363     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6364
6365         $current_major = $1;
6366         $current_minor = $2;
6367     }
6368
6369     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
6370
6371
6372 }
6373
6374 sub qemu_machine_pxe {
6375     my ($vmid, $conf, $machine) = @_;
6376
6377     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6378
6379     foreach my $opt (keys %$conf) {
6380         next if $opt !~ m/^net(\d+)$/;
6381         my $net = PVE::QemuServer::parse_net($conf->{$opt});
6382         next if !$net;
6383         my $romfile = PVE::QemuServer::vm_mon_cmd_nocheck($vmid, 'qom-get', path => $opt, property => 'romfile');
6384         return $machine.".pxe" if $romfile =~ m/pxe/;
6385         last;
6386     }
6387
6388 }
6389
6390 sub qemu_use_old_bios_files {
6391     my ($machine_type) = @_;
6392
6393     return if !$machine_type;
6394
6395     my $use_old_bios_files = undef;
6396
6397     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6398         $machine_type = $1;
6399         $use_old_bios_files = 1;
6400     } else {
6401         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6402         # load new efi bios files on migration. So this hack is required to allow
6403         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6404         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6405         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, undef, 2, 4);
6406     }
6407
6408     return ($use_old_bios_files, $machine_type);
6409 }
6410
6411 sub lspci {
6412
6413     my $devices = {};
6414
6415     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6416             my (undef, $id, $function) = @_;
6417             my $res = { id => $id, function => $function};
6418             push @{$devices->{$id}}, $res;
6419     });
6420
6421     return $devices;
6422 }
6423
6424 sub vm_iothreads_list {
6425     my ($vmid) = @_;
6426
6427     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6428
6429     my $iothreads = {};
6430     foreach my $iothread (@$res) {
6431         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6432     }
6433
6434     return $iothreads;
6435 }
6436
6437 sub scsihw_infos {
6438     my ($conf, $drive) = @_;
6439
6440     my $maxdev = 0;
6441
6442     if ($conf->{scsihw} && ($conf->{scsihw} =~ m/^lsi/)) {
6443         $maxdev = 7;
6444     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6445         $maxdev = 1;
6446     } else {
6447         $maxdev = 256;
6448     }
6449
6450     my $controller = int($drive->{index} / $maxdev);
6451     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6452
6453     return ($maxdev, $controller, $controller_prefix);
6454 }
6455
6456 # bash completion helper
6457
6458 sub complete_backup_archives {
6459     my ($cmdname, $pname, $cvalue) = @_;
6460
6461     my $cfg = PVE::Storage::config();
6462
6463     my $storeid;
6464
6465     if ($cvalue =~ m/^([^:]+):/) {
6466         $storeid = $1;
6467     }
6468
6469     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6470
6471     my $res = [];
6472     foreach my $id (keys %$data) {
6473         foreach my $item (@{$data->{$id}}) {
6474             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6475             push @$res, $item->{volid} if defined($item->{volid});
6476         }
6477     }
6478
6479     return $res;
6480 }
6481
6482 my $complete_vmid_full = sub {
6483     my ($running) = @_;
6484
6485     my $idlist = vmstatus();
6486
6487     my $res = [];
6488
6489     foreach my $id (keys %$idlist) {
6490         my $d = $idlist->{$id};
6491         if (defined($running)) {
6492             next if $d->{template};
6493             next if $running && $d->{status} ne 'running';
6494             next if !$running && $d->{status} eq 'running';
6495         }
6496         push @$res, $id;
6497
6498     }
6499     return $res;
6500 };
6501
6502 sub complete_vmid {
6503     return &$complete_vmid_full();
6504 }
6505
6506 sub complete_vmid_stopped {
6507     return &$complete_vmid_full(0);
6508 }
6509
6510 sub complete_vmid_running {
6511     return &$complete_vmid_full(1);
6512 }
6513
6514 sub complete_storage {
6515
6516     my $cfg = PVE::Storage::config();
6517     my $ids = $cfg->{ids};
6518
6519     my $res = [];
6520     foreach my $sid (keys %$ids) {
6521         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6522         next if !$ids->{$sid}->{content}->{images};
6523         push @$res, $sid;
6524     }
6525
6526     return $res;
6527 }
6528
6529 1;