PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QMPClient;
  31 use PVE::RPCEnvironment;
  32 use Time::HiRes qw(gettimeofday);
  33
  34 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  35
  36 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  37
  38 # Note about locking: we use flock on the config file protect
  39 # against concurent actions.
  40 # Aditionaly, we have a 'lock' setting in the config file. This
  41 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  42 # allowed when such lock is set. But you can ignore this kind of
  43 # lock with the --skiplock flag.
  44
  45 cfs_register_file('/qemu-server/',
  46                   \&parse_vm_config,
  47                   \&write_vm_config);
  48
  49 PVE::JSONSchema::register_standard_option('skiplock', {
  50     description => "Ignore locks - only root is allowed to use this option.",
  51     type => 'boolean',
  52     optional => 1,
  53 });
  54
  55 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  56     description => "Some command save/restore state from this location.",
  57     type => 'string',
  58     maxLength => 128,
  59     optional => 1,
  60 });
  61
  62 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  63     description => "The name of the snapshot.",
  64     type => 'string', format => 'pve-configid',
  65     maxLength => 40,
  66 });
  67
  68 #no warnings 'redefine';
  69
  70 sub cgroups_write {
  71    my ($controller, $vmid, $option, $value) = @_;
  72
  73    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  74    PVE::ProcFSTools::write_proc_entry($path, $value);
  75
  76 }
  77
  78 my $nodename = PVE::INotify::nodename();
  79
  80 mkdir "/etc/pve/nodes/$nodename";
  81 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  82 mkdir $confdir;
  83
  84 my $var_run_tmpdir = "/var/run/qemu-server";
  85 mkdir $var_run_tmpdir;
  86
  87 my $lock_dir = "/var/lock/qemu-server";
  88 mkdir $lock_dir;
  89
  90 my $pcisysfs = "/sys/bus/pci";
  91
  92 my $confdesc = {
  93     onboot => {
  94         optional => 1,
  95         type => 'boolean',
  96         description => "Specifies whether a VM will be started during system bootup.",
  97         default => 0,
  98     },
  99     autostart => {
 100         optional => 1,
 101         type => 'boolean',
 102         description => "Automatic restart after crash (currently ignored).",
 103         default => 0,
 104     },
 105     hotplug => {
 106         optional => 1,
 107         type => 'string', format => 'pve-hotplug-features',
 108         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 109         default => 'network,disk,usb',
 110     },
 111     reboot => {
 112         optional => 1,
 113         type => 'boolean',
 114         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 115         default => 1,
 116     },
 117     lock => {
 118         optional => 1,
 119         type => 'string',
 120         description => "Lock/unlock the VM.",
 121         enum => [qw(migrate backup snapshot rollback)],
 122     },
 123     cpulimit => {
 124         optional => 1,
 125         type => 'number',
 126         description => "Limit of CPU usage. Note if the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 127         minimum => 0,
 128         maximum => 128,
 129         default => 0,
 130     },
 131     cpuunits => {
 132         optional => 1,
 133         type => 'integer',
 134         description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 135         minimum => 0,
 136         maximum => 500000,
 137         default => 1000,
 138     },
 139     memory => {
 140         optional => 1,
 141         type => 'integer',
 142         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 143         minimum => 16,
 144         default => 512,
 145     },
 146     balloon => {
 147         optional => 1,
 148         type => 'integer',
 149         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 150         minimum => 0,
 151     },
 152     shares => {
 153         optional => 1,
 154         type => 'integer',
 155         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 156         minimum => 0,
 157         maximum => 50000,
 158         default => 1000,
 159     },
 160     keyboard => {
 161         optional => 1,
 162         type => 'string',
 163         description => "Keybord layout for vnc server. Default is read from the datacenter configuration file.",
 164         enum => PVE::Tools::kvmkeymaplist(),
 165         default => 'en-us',
 166     },
 167     name => {
 168         optional => 1,
 169         type => 'string', format => 'dns-name',
 170         description => "Set a name for the VM. Only used on the configuration web interface.",
 171     },
 172     scsihw => {
 173         optional => 1,
 174         type => 'string',
 175         description => "scsi controller model",
 176         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 177         default => 'lsi',
 178     },
 179     description => {
 180         optional => 1,
 181         type => 'string',
 182         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 183     },
 184     ostype => {
 185         optional => 1,
 186         type => 'string',
 187         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 l24 l26 solaris)],
 188         description => <<EODESC,
 189 Used to enable special optimization/features for specific
 190 operating systems:
 191
 192 other  => unspecified OS
 193 wxp    => Microsoft Windows XP
 194 w2k    => Microsoft Windows 2000
 195 w2k3   => Microsoft Windows 2003
 196 w2k8   => Microsoft Windows 2008
 197 wvista => Microsoft Windows Vista
 198 win7   => Microsoft Windows 7
 199 win8   => Microsoft Windows 8/2012
 200 l24    => Linux 2.4 Kernel
 201 l26    => Linux 2.6/3.X Kernel
 202 solaris => solaris/opensolaris/openindiania kernel
 203
 204 other|l24|l26|solaris                       ... no special behaviour
 205 wxp|w2k|w2k3|w2k8|wvista|win7|win8  ... use --localtime switch
 206 EODESC
 207     },
 208     boot => {
 209         optional => 1,
 210         type => 'string',
 211         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 212         pattern => '[acdn]{1,4}',
 213         default => 'cdn',
 214     },
 215     bootdisk => {
 216         optional => 1,
 217         type => 'string', format => 'pve-qm-bootdisk',
 218         description => "Enable booting from specified disk.",
 219         pattern => '(ide|sata|scsi|virtio)\d+',
 220     },
 221     smp => {
 222         optional => 1,
 223         type => 'integer',
 224         description => "The number of CPUs. Please use option -sockets instead.",
 225         minimum => 1,
 226         default => 1,
 227     },
 228     sockets => {
 229         optional => 1,
 230         type => 'integer',
 231         description => "The number of CPU sockets.",
 232         minimum => 1,
 233         default => 1,
 234     },
 235     cores => {
 236         optional => 1,
 237         type => 'integer',
 238         description => "The number of cores per socket.",
 239         minimum => 1,
 240         default => 1,
 241     },
 242     numa => {
 243         optional => 1,
 244         type => 'boolean',
 245         description => "Enable/disable Numa.",
 246         default => 0,
 247     },
 248     vcpus => {
 249         optional => 1,
 250         type => 'integer',
 251         description => "Number of hotplugged vcpus.",
 252         minimum => 1,
 253         default => 0,
 254     },
 255     acpi => {
 256         optional => 1,
 257         type => 'boolean',
 258         description => "Enable/disable ACPI.",
 259         default => 1,
 260     },
 261     agent => {
 262         optional => 1,
 263         type => 'boolean',
 264         description => "Enable/disable Qemu GuestAgent.",
 265         default => 0,
 266     },
 267     kvm => {
 268         optional => 1,
 269         type => 'boolean',
 270         description => "Enable/disable KVM hardware virtualization.",
 271         default => 1,
 272     },
 273     tdf => {
 274         optional => 1,
 275         type => 'boolean',
 276         description => "Enable/disable time drift fix.",
 277         default => 0,
 278     },
 279     localtime => {
 280         optional => 1,
 281         type => 'boolean',
 282         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 283     },
 284     freeze => {
 285         optional => 1,
 286         type => 'boolean',
 287         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 288     },
 289     vga => {
 290         optional => 1,
 291         type => 'string',
 292         description => "Select VGA type. If you want to use high resolution modes (>= 1280x1024x16) then you should use option 'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and 'cirrur' for other OS types. Option 'qxl' enables the SPICE display sever. You can also run without any graphic card using a serial devive as terminal.",
 293         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 294     },
 295     watchdog => {
 296         optional => 1,
 297         type => 'string', format => 'pve-qm-watchdog',
 298         typetext => '[[model=]i6300esb|ib700] [,[action=]reset|shutdown|poweroff|pause|debug|none]',
 299         description => "Create a virtual hardware watchdog device.  Once enabled (by a guest action), the watchdog must be periodically polled by an agent inside the guest or else the guest will be restarted (or execute the action specified)",
 300     },
 301     startdate => {
 302         optional => 1,
 303         type => 'string',
 304         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 305         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 306         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 307         default => 'now',
 308     },
 309     startup =>  get_standard_option('pve-startup-order'),
 310     template => {
 311         optional => 1,
 312         type => 'boolean',
 313         description => "Enable/disable Template.",
 314         default => 0,
 315     },
 316     args => {
 317         optional => 1,
 318         type => 'string',
 319         description => <<EODESCR,
 320 Note: this option is for experts only. It allows you to pass arbitrary arguments to kvm, for example:
 321
 322 args: -no-reboot -no-hpet
 323 EODESCR
 324     },
 325     tablet => {
 326         optional => 1,
 327         type => 'boolean',
 328         default => 1,
 329         description => "Enable/disable the usb tablet device. This device is usually needed to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with normal VNC clients. If you're running lots of console-only guests on one host, you may consider disabling this to save some context switches. This is turned of by default if you use spice (vga=qxl).",
 330     },
 331     migrate_speed => {
 332         optional => 1,
 333         type => 'integer',
 334         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 335         minimum => 0,
 336         default => 0,
 337     },
 338     migrate_downtime => {
 339         optional => 1,
 340         type => 'number',
 341         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 342         minimum => 0,
 343         default => 0.1,
 344     },
 345     cdrom => {
 346         optional => 1,
 347         type => 'string', format => 'pve-qm-drive',
 348         typetext => 'volume',
 349         description => "This is an alias for option -ide2",
 350     },
 351     cpu => {
 352         optional => 1,
 353         description => "Emulated CPU type.",
 354         type => 'string',
 355         enum => [ qw(486 athlon pentium pentium2 pentium3 coreduo core2duo kvm32 kvm64 qemu32 qemu64 phenom Conroe Penryn Nehalem Westmere SandyBridge IvyBridge Haswell Broadwell Opteron_G1 Opteron_G2 Opteron_G3 Opteron_G4 Opteron_G5 host) ],
 356         default => 'kvm64',
 357     },
 358     parent => get_standard_option('pve-snapshot-name', {
 359         optional => 1,
 360         description => "Parent snapshot name. This is used internally, and should not be modified.",
 361     }),
 362     snaptime => {
 363         optional => 1,
 364         description => "Timestamp for snapshots.",
 365         type => 'integer',
 366         minimum => 0,
 367     },
 368     vmstate => {
 369         optional => 1,
 370         type => 'string', format => 'pve-volume-id',
 371         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 372     },
 373     machine => {
 374         description => "Specific the Qemu machine type.",
 375         type => 'string',
 376         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
 377         maxLength => 40,
 378         optional => 1,
 379     },
 380     smbios1 => {
 381         description => "Specify SMBIOS type 1 fields.",
 382         type => 'string', format => 'pve-qm-smbios1',
 383         typetext => "[manufacturer=str][,product=str][,version=str][,serial=str] [,uuid=uuid][,sku=str][,family=str]",
 384         maxLength => 256,
 385         optional => 1,
 386     },
 387     protection => {
 388         optional => 1,
 389         type => 'boolean',
 390         description => "Sets the protection flag of the VM. This will prevent the remove operation.",
 391         default => 0,
 392     },
 393 };
 394
 395 # what about other qemu settings ?
 396 #cpu => 'string',
 397 #machine => 'string',
 398 #fda => 'file',
 399 #fdb => 'file',
 400 #mtdblock => 'file',
 401 #sd => 'file',
 402 #pflash => 'file',
 403 #snapshot => 'bool',
 404 #bootp => 'file',
 405 ##tftp => 'dir',
 406 ##smb => 'dir',
 407 #kernel => 'file',
 408 #append => 'string',
 409 #initrd => 'file',
 410 ##soundhw => 'string',
 411
 412 while (my ($k, $v) = each %$confdesc) {
 413     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 414 }
 415
 416 my $MAX_IDE_DISKS = 4;
 417 my $MAX_SCSI_DISKS = 14;
 418 my $MAX_VIRTIO_DISKS = 16;
 419 my $MAX_SATA_DISKS = 6;
 420 my $MAX_USB_DEVICES = 5;
 421 my $MAX_NETS = 32;
 422 my $MAX_UNUSED_DISKS = 8;
 423 my $MAX_HOSTPCI_DEVICES = 4;
 424 my $MAX_SERIAL_PORTS = 4;
 425 my $MAX_PARALLEL_PORTS = 3;
 426 my $MAX_NUMA = 8;
 427 my $MAX_MEM = 4194304;
 428 my $STATICMEM = 1024;
 429
 430 my $numadesc = {
 431     optional => 1,
 432     type => 'string', format => 'pve-qm-numanode',
 433     typetext => "cpus=<id[-id],memory=<mb>[[,hostnodes=<id[-id]>] [,policy=<preferred|bind|interleave>]]",
 434     description => "numa topology",
 435 };
 436 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 437
 438 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 439     $confdesc->{"numa$i"} = $numadesc;
 440 }
 441
 442 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 443                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 444                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 445 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 446
 447 my $netdesc = {
 448     optional => 1,
 449     type => 'string', format => 'pve-qm-net',
 450     typetext => "MODEL=XX:XX:XX:XX:XX:XX [,bridge=<dev>][,queues=<nbqueues>][,rate=<mbps>] [,tag=<vlanid>][,firewall=0|1],link_down=0|1]",
 451     description => <<EODESCR,
 452 Specify network devices.
 453
 454 MODEL is one of: $nic_model_list_txt
 455
 456 XX:XX:XX:XX:XX:XX should be an unique MAC address. This is
 457 automatically generated if not specified.
 458
 459 The bridge parameter can be used to automatically add the interface to a bridge device. The Proxmox VE standard bridge is called 'vmbr0'.
 460
 461 Option 'rate' is used to limit traffic bandwidth from and to this interface. It is specified as floating point number, unit is 'Megabytes per second'.
 462
 463 If you specify no bridge, we create a kvm 'user' (NATed) network device, which provides DHCP and DNS services. The following addresses are used:
 464
 465 10.0.2.2   Gateway
 466 10.0.2.3   DNS Server
 467 10.0.2.4   SMB Server
 468
 469 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 470
 471 EODESCR
 472 };
 473 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 474
 475 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 476     $confdesc->{"net$i"} = $netdesc;
 477 }
 478
 479 my $drivename_hash;
 480
 481 my $idedesc = {
 482     optional => 1,
 483     type => 'string', format => 'pve-qm-drive',
 484     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads] [,discard=ignore|on] [,serial=serial][,model=model]',
 485     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
 486 };
 487 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
 488
 489 my $scsidesc = {
 490     optional => 1,
 491     type => 'string', format => 'pve-qm-drive',
 492     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads] [,discard=ignore|on] [,iothread=on] [,queues=<nbqueues>] [,serial=serial]',
 493     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
 494 };
 495 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 496
 497 my $satadesc = {
 498     optional => 1,
 499     type => 'string', format => 'pve-qm-drive',
 500     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads]  [,discard=ignore|on] [,serial=serial]',
 501     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
 502 };
 503 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
 504
 505 my $virtiodesc = {
 506     optional => 1,
 507     type => 'string', format => 'pve-qm-drive',
 508     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads]  [,discard=ignore|on] [,iothread=on] [,serial=serial]',
 509     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
 510 };
 511 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
 512
 513 my $usbdesc = {
 514     optional => 1,
 515     type => 'string', format => 'pve-qm-usb-device',
 516     typetext => 'host=HOSTUSBDEVICE|spice',
 517     description => <<EODESCR,
 518 Configure an USB device (n is 0 to 4). This can be used to
 519 pass-through usb devices to the guest. HOSTUSBDEVICE syntax is:
 520
 521 'bus-port(.port)*' (decimal numbers) or
 522 'vendor_id:product_id' (hexadeciaml numbers)
 523
 524 You can use the 'lsusb -t' command to list existing usb devices.
 525
 526 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 527
 528 The value 'spice' can be used to add a usb redirection devices for spice.
 529
 530 EODESCR
 531 };
 532 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
 533
 534 my $hostpcidesc = {
 535         optional => 1,
 536         type => 'string', format => 'pve-qm-hostpci',
 537         typetext => "[host=]HOSTPCIDEVICE [,rombar=on|off] [,pcie=0|1] [,x-vga=on|off]",
 538         description => <<EODESCR,
 539 Map host pci devices. HOSTPCIDEVICE syntax is:
 540
 541 'bus:dev.func' (hexadecimal numbers)
 542
 543 You can us the 'lspci' command to list existing pci devices.
 544
 545 The 'rombar' option determines whether or not the device's ROM will be visible in the guest's memory map (default is 'on').
 546
 547 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 548
 549 Experimental: user reported problems with this option.
 550 EODESCR
 551 };
 552 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
 553
 554 my $serialdesc = {
 555         optional => 1,
 556         type => 'string',
 557         pattern => '(/dev/.+|socket)',
 558         description =>  <<EODESCR,
 559 Create a serial device inside the VM (n is 0 to 3), and pass through a host serial device (i.e. /dev/ttyS0), or create a unix socket on the host side (use 'qm terminal' to open a terminal connection).
 560
 561 Note: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
 562
 563 Experimental: user reported problems with this option.
 564 EODESCR
 565 };
 566
 567 my $paralleldesc= {
 568         optional => 1,
 569         type => 'string',
 570         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
 571         description =>  <<EODESCR,
 572 Map host parallel devices (n is 0 to 2).
 573
 574 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 575
 576 Experimental: user reported problems with this option.
 577 EODESCR
 578 };
 579
 580 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
 581     $confdesc->{"parallel$i"} = $paralleldesc;
 582 }
 583
 584 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
 585     $confdesc->{"serial$i"} = $serialdesc;
 586 }
 587
 588 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
 589     $confdesc->{"hostpci$i"} = $hostpcidesc;
 590 }
 591
 592 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
 593     $drivename_hash->{"ide$i"} = 1;
 594     $confdesc->{"ide$i"} = $idedesc;
 595 }
 596
 597 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
 598     $drivename_hash->{"sata$i"} = 1;
 599     $confdesc->{"sata$i"} = $satadesc;
 600 }
 601
 602 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
 603     $drivename_hash->{"scsi$i"} = 1;
 604     $confdesc->{"scsi$i"} = $scsidesc ;
 605 }
 606
 607 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
 608     $drivename_hash->{"virtio$i"} = 1;
 609     $confdesc->{"virtio$i"} = $virtiodesc;
 610 }
 611
 612 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
 613     $confdesc->{"usb$i"} = $usbdesc;
 614 }
 615
 616 my $unuseddesc = {
 617     optional => 1,
 618     type => 'string', format => 'pve-volume-id',
 619     description => "Reference to unused volumes.",
 620 };
 621
 622 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
 623     $confdesc->{"unused$i"} = $unuseddesc;
 624 }
 625
 626 my $kvm_api_version = 0;
 627
 628 sub kvm_version {
 629
 630     return $kvm_api_version if $kvm_api_version;
 631
 632     my $fh = IO::File->new("</dev/kvm") ||
 633         return 0;
 634
 635     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
 636         $kvm_api_version = $v;
 637     }
 638
 639     $fh->close();
 640
 641     return  $kvm_api_version;
 642 }
 643
 644 my $kvm_user_version;
 645
 646 sub kvm_user_version {
 647
 648     return $kvm_user_version if $kvm_user_version;
 649
 650     $kvm_user_version = 'unknown';
 651
 652     my $tmp = `kvm -help 2>/dev/null`;
 653
 654     if ($tmp =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
 655         $kvm_user_version = $2;
 656     }
 657
 658     return $kvm_user_version;
 659
 660 }
 661
 662 my $kernel_has_vhost_net = -c '/dev/vhost-net';
 663
 664 sub disknames {
 665     # order is important - used to autoselect boot disk
 666     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
 667             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
 668             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
 669             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))));
 670 }
 671
 672 sub valid_drivename {
 673     my $dev = shift;
 674
 675     return defined($drivename_hash->{$dev});
 676 }
 677
 678 sub option_exists {
 679     my $key = shift;
 680     return defined($confdesc->{$key});
 681 }
 682
 683 sub nic_models {
 684     return $nic_model_list;
 685 }
 686
 687 sub os_list_description {
 688
 689     return {
 690         other => 'Other',
 691         wxp => 'Windows XP',
 692         w2k => 'Windows 2000',
 693         w2k3 =>, 'Windows 2003',
 694         w2k8 => 'Windows 2008',
 695         wvista => 'Windows Vista',
 696         win7 => 'Windows 7',
 697         win8 => 'Windows 8/2012',
 698         l24 => 'Linux 2.4',
 699         l26 => 'Linux 2.6',
 700     };
 701 }
 702
 703 my $cdrom_path;
 704
 705 sub get_cdrom_path {
 706
 707     return  $cdrom_path if $cdrom_path;
 708
 709     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
 710     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
 711     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
 712 }
 713
 714 sub get_iso_path {
 715     my ($storecfg, $vmid, $cdrom) = @_;
 716
 717     if ($cdrom eq 'cdrom') {
 718         return get_cdrom_path();
 719     } elsif ($cdrom eq 'none') {
 720         return '';
 721     } elsif ($cdrom =~ m|^/|) {
 722         return $cdrom;
 723     } else {
 724         return PVE::Storage::path($storecfg, $cdrom);
 725     }
 726 }
 727
 728 # try to convert old style file names to volume IDs
 729 sub filename_to_volume_id {
 730     my ($vmid, $file, $media) = @_;
 731
 732     if (!($file eq 'none' || $file eq 'cdrom' ||
 733           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
 734
 735         return undef if $file =~ m|/|;
 736
 737         if ($media && $media eq 'cdrom') {
 738             $file = "local:iso/$file";
 739         } else {
 740             $file = "local:$vmid/$file";
 741         }
 742     }
 743
 744     return $file;
 745 }
 746
 747 sub verify_media_type {
 748     my ($opt, $vtype, $media) = @_;
 749
 750     return if !$media;
 751
 752     my $etype;
 753     if ($media eq 'disk') {
 754         $etype = 'images';
 755     } elsif ($media eq 'cdrom') {
 756         $etype = 'iso';
 757     } else {
 758         die "internal error";
 759     }
 760
 761     return if ($vtype eq $etype);
 762
 763     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
 764 }
 765
 766 sub cleanup_drive_path {
 767     my ($opt, $storecfg, $drive) = @_;
 768
 769     # try to convert filesystem paths to volume IDs
 770
 771     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
 772         ($drive->{file} !~ m|^/dev/.+|) &&
 773         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
 774         ($drive->{file} !~ m/^\d+$/)) {
 775         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
 776         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
 777         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
 778         verify_media_type($opt, $vtype, $drive->{media});
 779         $drive->{file} = $volid;
 780     }
 781
 782     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
 783 }
 784
 785 sub create_conf_nolock {
 786     my ($vmid, $settings) = @_;
 787
 788     my $filename = config_file($vmid);
 789
 790     die "configuration file '$filename' already exists\n" if -f $filename;
 791
 792     my $defaults = load_defaults();
 793
 794     $settings->{name} = "vm$vmid" if !$settings->{name};
 795     $settings->{memory} = $defaults->{memory} if !$settings->{memory};
 796
 797     my $data = '';
 798     foreach my $opt (keys %$settings) {
 799         next if !$confdesc->{$opt};
 800
 801         my $value = $settings->{$opt};
 802         next if !$value;
 803
 804         $data .= "$opt: $value\n";
 805     }
 806
 807     PVE::Tools::file_set_contents($filename, $data);
 808 }
 809
 810 sub parse_hotplug_features {
 811     my ($data) = @_;
 812
 813     my $res = {};
 814
 815     return $res if $data eq '0';
 816
 817     $data = $confdesc->{hotplug}->{default} if $data eq '1';
 818
 819     foreach my $feature (PVE::Tools::split_list($data)) {
 820         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
 821             $res->{$1} = 1;
 822         } else {
 823             warn "ignoring unknown hotplug feature '$feature'\n";
 824         }
 825     }
 826     return $res;
 827 }
 828
 829 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
 830 sub pve_verify_hotplug_features {
 831     my ($value, $noerr) = @_;
 832
 833     return $value if parse_hotplug_features($value);
 834
 835     return undef if $noerr;
 836
 837     die "unable to parse hotplug option\n";
 838 }
 839
 840 my $parse_size = sub {
 841     my ($value) = @_;
 842
 843     return undef if $value !~ m/^(\d+(\.\d+)?)([KMG])?$/;
 844     my ($size, $unit) = ($1, $3);
 845     if ($unit) {
 846         if ($unit eq 'K') {
 847             $size = $size * 1024;
 848         } elsif ($unit eq 'M') {
 849             $size = $size * 1024 * 1024;
 850         } elsif ($unit eq 'G') {
 851             $size = $size * 1024 * 1024 * 1024;
 852         }
 853     }
 854     return int($size);
 855 };
 856
 857 my $format_size = sub {
 858     my ($size) = @_;
 859
 860     $size = int($size);
 861
 862     my $kb = int($size/1024);
 863     return $size if $kb*1024 != $size;
 864
 865     my $mb = int($kb/1024);
 866     return "${kb}K" if $mb*1024 != $kb;
 867
 868     my $gb = int($mb/1024);
 869     return "${mb}M" if $gb*1024 != $mb;
 870
 871     return "${gb}G";
 872 };
 873
 874 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
 875 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
 876 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
 877 #        [,aio=native|threads][,discard=ignore|on][,iothread=on]
 878 #        [,serial=serial][,model=model]
 879
 880 sub parse_drive {
 881     my ($key, $data) = @_;
 882
 883     my $res = {};
 884
 885     # $key may be undefined - used to verify JSON parameters
 886     if (!defined($key)) {
 887         $res->{interface} = 'unknown'; # should not harm when used to verify parameters
 888         $res->{index} = 0;
 889     } elsif ($key =~ m/^([^\d]+)(\d+)$/) {
 890         $res->{interface} = $1;
 891         $res->{index} = $2;
 892     } else {
 893         return undef;
 894     }
 895
 896     foreach my $p (split (/,/, $data)) {
 897         next if $p =~ m/^\s*$/;
 898
 899         if ($p =~ m/^(file|volume|cyls|heads|secs|trans|media|snapshot|cache|format|rerror|werror|backup|aio|bps|mbps|mbps_max|bps_rd|mbps_rd|mbps_rd_max|bps_wr|mbps_wr|mbps_wr_max|iops|iops_max|iops_rd|iops_rd_max|iops_wr|iops_wr_max|size|discard|iothread|queues|serial|model)=(.+)$/) {
 900             my ($k, $v) = ($1, $2);
 901
 902             $k = 'file' if $k eq 'volume';
 903
 904             return undef if defined $res->{$k};
 905
 906             if ($k eq 'bps' || $k eq 'bps_rd' || $k eq 'bps_wr') {
 907                 return undef if !$v || $v !~ m/^\d+/;
 908                 $k = "m$k";
 909                 $v = sprintf("%.3f", $v / (1024*1024));
 910             }
 911             $res->{$k} = $v;
 912         } else {
 913             if (!$res->{file} && $p !~ m/=/) {
 914                 $res->{file} = $p;
 915             } else {
 916                 return undef;
 917             }
 918         }
 919     }
 920
 921     return undef if !$res->{file};
 922
 923     return undef if $res->{cache} &&
 924         $res->{cache} !~ m/^(off|none|writethrough|writeback|unsafe|directsync)$/;
 925     return undef if $res->{snapshot} && $res->{snapshot} !~ m/^(on|off)$/;
 926     return undef if $res->{cyls} && $res->{cyls} !~ m/^\d+$/;
 927     return undef if $res->{heads} && $res->{heads} !~ m/^\d+$/;
 928     return undef if $res->{secs} && $res->{secs} !~ m/^\d+$/;
 929     return undef if $res->{media} && $res->{media} !~ m/^(disk|cdrom)$/;
 930     return undef if $res->{trans} && $res->{trans} !~ m/^(none|lba|auto)$/;
 931     return undef if $res->{format} && $res->{format} !~ m/^(raw|cow|qcow|qed|qcow2|vmdk|cloop)$/;
 932     return undef if $res->{rerror} && $res->{rerror} !~ m/^(ignore|report|stop)$/;
 933     return undef if $res->{werror} && $res->{werror} !~ m/^(enospc|ignore|report|stop)$/;
 934     return undef if $res->{backup} && $res->{backup} !~ m/^(yes|no)$/;
 935     return undef if $res->{aio} && $res->{aio} !~ m/^(native|threads)$/;
 936     return undef if $res->{discard} && $res->{discard} !~ m/^(ignore|on)$/;
 937     return undef if $res->{iothread} && $res->{iothread} !~ m/^(on)$/;
 938     return undef if $res->{queues} && ($res->{queues} !~ m/^\d+$/ || $res->{queues} < 2);
 939
 940     return undef if $res->{mbps_rd} && $res->{mbps};
 941     return undef if $res->{mbps_wr} && $res->{mbps};
 942
 943     return undef if $res->{mbps} && $res->{mbps} !~ m/^\d+(\.\d+)?$/;
 944     return undef if $res->{mbps_max} && $res->{mbps_max} !~ m/^\d+(\.\d+)?$/;
 945     return undef if $res->{mbps_rd} && $res->{mbps_rd} !~ m/^\d+(\.\d+)?$/;
 946     return undef if $res->{mbps_rd_max} && $res->{mbps_rd_max} !~ m/^\d+(\.\d+)?$/;
 947     return undef if $res->{mbps_wr} && $res->{mbps_wr} !~ m/^\d+(\.\d+)?$/;
 948     return undef if $res->{mbps_wr_max} && $res->{mbps_wr_max} !~ m/^\d+(\.\d+)?$/;
 949
 950     return undef if $res->{iops_rd} && $res->{iops};
 951     return undef if $res->{iops_wr} && $res->{iops};
 952
 953
 954     return undef if $res->{iops} && $res->{iops} !~ m/^\d+$/;
 955     return undef if $res->{iops_max} && $res->{iops_max} !~ m/^\d+$/;
 956     return undef if $res->{iops_rd} && $res->{iops_rd} !~ m/^\d+$/;
 957     return undef if $res->{iops_rd_max} && $res->{iops_rd_max} !~ m/^\d+$/;
 958     return undef if $res->{iops_wr} && $res->{iops_wr} !~ m/^\d+$/;
 959     return undef if $res->{iops_wr_max} && $res->{iops_wr_max} !~ m/^\d+$/;
 960
 961     if ($res->{size}) {
 962         return undef if !defined($res->{size} = &$parse_size($res->{size}));
 963     }
 964
 965     if ($res->{media} && ($res->{media} eq 'cdrom')) {
 966         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
 967         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
 968         return undef if $res->{interface} eq 'virtio';
 969     }
 970
 971     # rerror does not work with scsi drives
 972     if ($res->{rerror}) {
 973         return undef if $res->{interface} eq 'scsi';
 974     }
 975
 976     return $res;
 977 }
 978
 979 my @qemu_drive_options = qw(heads secs cyls trans media format cache snapshot rerror werror aio discard iops iops_rd iops_wr iops_max iops_rd_max iops_wr_max serial);
 980
 981 sub print_drive {
 982     my ($vmid, $drive) = @_;
 983
 984     my $opts = '';
 985     foreach my $o (@qemu_drive_options, 'mbps', 'mbps_rd', 'mbps_wr', 'mbps_max', 'mbps_rd_max', 'mbps_wr_max', 'backup', 'iothread', 'queues') {
 986         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
 987     }
 988
 989     if ($drive->{size}) {
 990         $opts .= ",size=" . &$format_size($drive->{size});
 991     }
 992
 993     if (my $model = $drive->{model}) {
 994         $opts .= ",model=$model";
 995     }
 996
 997     return "$drive->{file}$opts";
 998 }
 999
1000 sub scsi_inquiry {
1001     my($fh, $noerr) = @_;
1002
1003     my $SG_IO = 0x2285;
1004     my $SG_GET_VERSION_NUM = 0x2282;
1005
1006     my $versionbuf = "\x00" x 8;
1007     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1008     if (!$ret) {
1009         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1010         return undef;
1011     }
1012     my $version = unpack("I", $versionbuf);
1013     if ($version < 30000) {
1014         die "scsi generic interface too old\n"  if !$noerr;
1015         return undef;
1016     }
1017
1018     my $buf = "\x00" x 36;
1019     my $sensebuf = "\x00" x 8;
1020     my $cmd = pack("C x3 C x1", 0x12, 36);
1021
1022     # see /usr/include/scsi/sg.h
1023     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1024
1025     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1026                       length($sensebuf), 0, length($buf), $buf,
1027                       $cmd, $sensebuf, 6000);
1028
1029     $ret = ioctl($fh, $SG_IO, $packet);
1030     if (!$ret) {
1031         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1032         return undef;
1033     }
1034
1035     my @res = unpack($sg_io_hdr_t, $packet);
1036     if ($res[17] || $res[18]) {
1037         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1038         return undef;
1039     }
1040
1041     my $res = {};
1042     (my $byte0, my $byte1, $res->{vendor},
1043      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1044
1045     $res->{removable} = $byte1 & 128 ? 1 : 0;
1046     $res->{type} = $byte0 & 31;
1047
1048     return $res;
1049 }
1050
1051 sub path_is_scsi {
1052     my ($path) = @_;
1053
1054     my $fh = IO::File->new("+<$path") || return undef;
1055     my $res = scsi_inquiry($fh, 1);
1056     close($fh);
1057
1058     return $res;
1059 }
1060
1061 sub machine_type_is_q35 {
1062     my ($conf) = @_;
1063
1064     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1065 }
1066
1067 sub print_tabletdevice_full {
1068     my ($conf) = @_;
1069
1070     my $q35 = machine_type_is_q35($conf);
1071
1072     # we use uhci for old VMs because tablet driver was buggy in older qemu
1073     my $usbbus = $q35 ? "ehci" : "uhci";
1074
1075     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1076 }
1077
1078 sub print_drivedevice_full {
1079     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1080
1081     my $device = '';
1082     my $maxdev = 0;
1083
1084     if ($drive->{interface} eq 'virtio') {
1085         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1086         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1087         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1088     } elsif ($drive->{interface} eq 'scsi') {
1089
1090         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1091         my $unit = $drive->{index} % $maxdev;
1092         my $devicetype = 'hd';
1093         my $path = '';
1094         if (drive_is_cdrom($drive)) {
1095               $devicetype = 'cd';
1096         } else {
1097               if ($drive->{file} =~ m|^/|) {
1098                   $path = $drive->{file};
1099               } else {
1100                   $path = PVE::Storage::path($storecfg, $drive->{file});
1101               }
1102
1103               if($path =~ m/^iscsi\:\/\//){
1104                   $devicetype = 'generic';
1105               } else {
1106                   if (my $info = path_is_scsi($path)) {
1107                       if ($info->{type} == 0) {
1108                           $devicetype = 'block';
1109                       } elsif ($info->{type} == 1) { # tape
1110                           $devicetype = 'generic';
1111                       }
1112                   }
1113               }
1114          }
1115
1116         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1117             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1118         } else {
1119             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1120         }
1121
1122     } elsif ($drive->{interface} eq 'ide'){
1123         $maxdev = 2;
1124         my $controller = int($drive->{index} / $maxdev);
1125         my $unit = $drive->{index} % $maxdev;
1126         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1127
1128         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1129         if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
1130             $device .= ",model=$model";
1131         }
1132     } elsif ($drive->{interface} eq 'sata'){
1133         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1134         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1135         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1136     } elsif ($drive->{interface} eq 'usb') {
1137         die "implement me";
1138         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1139     } else {
1140         die "unsupported interface type";
1141     }
1142
1143     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1144
1145     return $device;
1146 }
1147
1148 sub get_initiator_name {
1149     my $initiator;
1150
1151     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1152     while (defined(my $line = <$fh>)) {
1153         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1154         $initiator = $1;
1155         last;
1156     }
1157     $fh->close();
1158
1159     return $initiator;
1160 }
1161
1162 sub print_drive_full {
1163     my ($storecfg, $vmid, $drive) = @_;
1164
1165     my $path;
1166     my $volid = $drive->{file};
1167     my $format;
1168
1169     if (drive_is_cdrom($drive)) {
1170         $path = get_iso_path($storecfg, $vmid, $volid);
1171     } else {
1172         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1173         if ($storeid) {
1174             $path = PVE::Storage::path($storecfg, $volid);
1175             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1176             $format = qemu_img_format($scfg, $volname);
1177         } else {
1178             $path = $volid;
1179         }
1180    }
1181
1182     my $opts = '';
1183     foreach my $o (@qemu_drive_options) {
1184         next if $o eq 'bootindex';
1185         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1186     }
1187
1188     $opts .= ",format=$format" if $format && !$drive->{format};
1189
1190     foreach my $o (qw(bps bps_rd bps_wr)) {
1191         my $v = $drive->{"m$o"};
1192         $opts .= ",$o=" . int($v*1024*1024) if $v;
1193     }
1194
1195     my $cache_direct = 0;
1196
1197     if (my $cache = $drive->{cache}) {
1198         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1199     } elsif (!drive_is_cdrom($drive)) {
1200         $opts .= ",cache=none";
1201         $cache_direct = 1;
1202     }
1203
1204     # aio native works only with O_DIRECT
1205     if (!$drive->{aio}) {
1206         if($cache_direct) {
1207             $opts .= ",aio=native";
1208         } else {
1209             $opts .= ",aio=threads";
1210         }
1211     }
1212
1213     my $detectzeroes = $drive->{discard} ? "unmap" : "on";
1214     $opts .= ",detect-zeroes=$detectzeroes" if !drive_is_cdrom($drive);
1215
1216     my $pathinfo = $path ? "file=$path," : '';
1217
1218     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1219 }
1220
1221 sub print_netdevice_full {
1222     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1223
1224     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1225
1226     my $device = $net->{model};
1227     if ($net->{model} eq 'virtio') {
1228          $device = 'virtio-net-pci';
1229      };
1230
1231     my $pciaddr = print_pci_addr("$netid", $bridges);
1232     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1233     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1234         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1235         my $vectors = $net->{queues} * 2 + 2;
1236         $tmpstr .= ",vectors=$vectors,mq=on";
1237     }
1238     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1239
1240     if ($use_old_bios_files) {
1241         my $romfile;
1242         if ($device eq 'virtio-net-pci') {
1243             $romfile = 'pxe-virtio.rom';
1244         } elsif ($device eq 'e1000') {
1245             $romfile = 'pxe-e1000.rom';
1246         } elsif ($device eq 'ne2k') {
1247             $romfile = 'pxe-ne2k_pci.rom';
1248         } elsif ($device eq 'pcnet') {
1249             $romfile = 'pxe-pcnet.rom';
1250         } elsif ($device eq 'rtl8139') {
1251             $romfile = 'pxe-rtl8139.rom';
1252         }
1253         $tmpstr .= ",romfile=$romfile" if $romfile;
1254     }
1255
1256     return $tmpstr;
1257 }
1258
1259 sub print_netdev_full {
1260     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1261
1262     my $i = '';
1263     if ($netid =~ m/^net(\d+)$/) {
1264         $i = int($1);
1265     }
1266
1267     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1268
1269     my $ifname = "tap${vmid}i$i";
1270
1271     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1272     die "interface name '$ifname' is too long (max 15 character)\n"
1273         if length($ifname) >= 16;
1274
1275     my $vhostparam = '';
1276     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1277
1278     my $vmname = $conf->{name} || "vm$vmid";
1279
1280     my $netdev = "";
1281     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1282
1283     if ($net->{bridge}) {
1284         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1285     } else {
1286         $netdev = "type=user,id=$netid,hostname=$vmname";
1287     }
1288
1289     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1290
1291     return $netdev;
1292 }
1293
1294 sub drive_is_cdrom {
1295     my ($drive) = @_;
1296
1297     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1298
1299 }
1300
1301 sub parse_numa {
1302     my ($data) = @_;
1303
1304     my $res = {};
1305
1306     foreach my $kvp (split(/,/, $data)) {
1307
1308         if ($kvp =~ m/^memory=(\S+)$/) {
1309             $res->{memory} = $1;
1310         } elsif ($kvp =~ m/^policy=(preferred|bind|interleave)$/) {
1311             $res->{policy} = $1;
1312         } elsif ($kvp =~ m/^cpus=(\d+)(-(\d+))?$/) {
1313             $res->{cpus}->{start} = $1;
1314             $res->{cpus}->{end} = $3;
1315         } elsif ($kvp =~ m/^hostnodes=(\d+)(-(\d+))?$/) {
1316             $res->{hostnodes}->{start} = $1;
1317             $res->{hostnodes}->{end} = $3;
1318         } else {
1319             return undef;
1320         }
1321     }
1322
1323     return $res;
1324 }
1325
1326 sub parse_hostpci {
1327     my ($value) = @_;
1328
1329     return undef if !$value;
1330
1331
1332     my @list = split(/,/, $value);
1333     my $found;
1334
1335     my $res = {};
1336     foreach my $kv (@list) {
1337
1338         if ($kv =~ m/^(host=)?([a-f0-9]{2}:[a-f0-9]{2})(\.([a-f0-9]))?$/) {
1339             $found = 1;
1340             if(defined($4)){
1341                 push @{$res->{pciid}}, { id => $2 , function => $4};
1342
1343             }else{
1344                 my $pcidevices = lspci($2);
1345                 $res->{pciid} = $pcidevices->{$2};
1346             }
1347         } elsif ($kv =~ m/^rombar=(on|off)$/) {
1348             $res->{rombar} = $1;
1349         } elsif ($kv =~ m/^x-vga=(on|off)$/) {
1350             $res->{'x-vga'} = $1;
1351         } elsif ($kv =~ m/^pcie=(\d+)$/) {
1352             $res->{pcie} = 1 if $1 == 1;
1353         } else {
1354             warn "unknown hostpci setting '$kv'\n";
1355         }
1356     }
1357
1358     return undef if !$found;
1359
1360     return $res;
1361 }
1362
1363 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1364 sub parse_net {
1365     my ($data) = @_;
1366
1367     my $res = {};
1368
1369     foreach my $kvp (split(/,/, $data)) {
1370
1371         if ($kvp =~ m/^(ne2k_pci|e1000|e1000-82540em|e1000-82544gc|e1000-82545em|rtl8139|pcnet|virtio|ne2k_isa|i82551|i82557b|i82559er|vmxnet3)(=([0-9a-f]{2}(:[0-9a-f]{2}){5}))?$/i) {
1372             my $model = lc($1);
1373             my $mac = defined($3) ? uc($3) : PVE::Tools::random_ether_addr();
1374             $res->{model} = $model;
1375             $res->{macaddr} = $mac;
1376         } elsif ($kvp =~ m/^bridge=(\S+)$/) {
1377             $res->{bridge} = $1;
1378         } elsif ($kvp =~ m/^queues=(\d+)$/) {
1379             $res->{queues} = $1;
1380         } elsif ($kvp =~ m/^rate=(\d+(\.\d+)?)$/) {
1381             $res->{rate} = $1;
1382         } elsif ($kvp =~ m/^tag=(\d+)$/) {
1383             $res->{tag} = $1;
1384         } elsif ($kvp =~ m/^firewall=([01])$/) {
1385             $res->{firewall} = $1;
1386         } elsif ($kvp =~ m/^link_down=([01])$/) {
1387             $res->{link_down} = $1;
1388         } else {
1389             return undef;
1390         }
1391
1392     }
1393
1394     return undef if !$res->{model};
1395
1396     return $res;
1397 }
1398
1399 sub print_net {
1400     my $net = shift;
1401
1402     my $res = "$net->{model}";
1403     $res .= "=$net->{macaddr}" if $net->{macaddr};
1404     $res .= ",bridge=$net->{bridge}" if $net->{bridge};
1405     $res .= ",rate=$net->{rate}" if $net->{rate};
1406     $res .= ",tag=$net->{tag}" if $net->{tag};
1407     $res .= ",firewall=1" if $net->{firewall};
1408     $res .= ",link_down=1" if $net->{link_down};
1409     $res .= ",queues=$net->{queues}" if $net->{queues};
1410
1411     return $res;
1412 }
1413
1414 sub add_random_macs {
1415     my ($settings) = @_;
1416
1417     foreach my $opt (keys %$settings) {
1418         next if $opt !~ m/^net(\d+)$/;
1419         my $net = parse_net($settings->{$opt});
1420         next if !$net;
1421         $settings->{$opt} = print_net($net);
1422     }
1423 }
1424
1425 sub add_unused_volume {
1426     my ($config, $volid) = @_;
1427
1428     my $key;
1429     for (my $ind = $MAX_UNUSED_DISKS - 1; $ind >= 0; $ind--) {
1430         my $test = "unused$ind";
1431         if (my $vid = $config->{$test}) {
1432             return if $vid eq $volid; # do not add duplicates
1433         } else {
1434             $key = $test;
1435         }
1436     }
1437
1438     die "To many unused volume - please delete them first.\n" if !$key;
1439
1440     $config->{$key} = $volid;
1441
1442     return $key;
1443 }
1444
1445 sub vm_is_volid_owner {
1446     my ($storecfg, $vmid, $volid) = @_;
1447
1448     if ($volid !~  m|^/|) {
1449         my ($path, $owner);
1450         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1451         if ($owner && ($owner == $vmid)) {
1452             return 1;
1453         }
1454     }
1455
1456     return undef;
1457 }
1458
1459 sub split_flagged_list {
1460     my $text = shift || '';
1461     $text =~ s/[,;]/ /g;
1462     $text =~ s/^\s+//;
1463     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
1464 }
1465
1466 sub join_flagged_list {
1467     my ($how, $lst) = @_;
1468     join $how, map { $lst->{$_} . $_ } keys %$lst;
1469 }
1470
1471 sub vmconfig_delete_pending_option {
1472     my ($conf, $key, $force) = @_;
1473
1474     delete $conf->{pending}->{$key};
1475     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1476     $pending_delete_hash->{$key} = $force ? '!' : '';
1477     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1478 }
1479
1480 sub vmconfig_undelete_pending_option {
1481     my ($conf, $key) = @_;
1482
1483     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1484     delete $pending_delete_hash->{$key};
1485
1486     if (%$pending_delete_hash) {
1487         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1488     } else {
1489         delete $conf->{pending}->{delete};
1490     }
1491 }
1492
1493 sub vmconfig_register_unused_drive {
1494     my ($storecfg, $vmid, $conf, $drive) = @_;
1495
1496     if (!drive_is_cdrom($drive)) {
1497         my $volid = $drive->{file};
1498         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1499             add_unused_volume($conf, $volid, $vmid);
1500         }
1501     }
1502 }
1503
1504 sub vmconfig_cleanup_pending {
1505     my ($conf) = @_;
1506
1507     # remove pending changes when nothing changed
1508     my $changes;
1509     foreach my $opt (keys %{$conf->{pending}}) {
1510         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
1511             $changes = 1;
1512             delete $conf->{pending}->{$opt};
1513         }
1514     }
1515
1516     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
1517     my $pending_delete_hash = {};
1518     while (my ($opt, $force) = each %$current_delete_hash) {
1519         if (defined($conf->{$opt})) {
1520             $pending_delete_hash->{$opt} = $force;
1521         } else {
1522             $changes = 1;
1523         }
1524     }
1525
1526     if (%$pending_delete_hash) {
1527         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1528     } else {
1529         delete $conf->{pending}->{delete};
1530     }
1531
1532     return $changes;
1533 }
1534
1535 my $valid_smbios1_options = {
1536     manufacturer => '\S+',
1537     product => '\S+',
1538     version => '\S+',
1539     serial => '\S+',
1540     uuid => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1541     sku => '\S+',
1542     family => '\S+',
1543 };
1544
1545 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
1546 sub parse_smbios1 {
1547     my ($data) = @_;
1548
1549     my $res = {};
1550
1551     foreach my $kvp (split(/,/, $data)) {
1552         return undef if $kvp !~ m/^(\S+)=(.+)$/;
1553         my ($k, $v) = split(/=/, $kvp);
1554         return undef if !defined($k) || !defined($v);
1555         return undef if !$valid_smbios1_options->{$k};
1556         return undef if $v !~ m/^$valid_smbios1_options->{$k}$/;
1557         $res->{$k} = $v;
1558     }
1559
1560     return $res;
1561 }
1562
1563 sub print_smbios1 {
1564     my ($smbios1) = @_;
1565
1566     my $data = '';
1567     foreach my $k (keys %$smbios1) {
1568         next if !defined($smbios1->{$k});
1569         next if !$valid_smbios1_options->{$k};
1570         $data .= ',' if $data;
1571         $data .= "$k=$smbios1->{$k}";
1572     }
1573     return $data;
1574 }
1575
1576 PVE::JSONSchema::register_format('pve-qm-smbios1', \&verify_smbios1);
1577 sub verify_smbios1 {
1578     my ($value, $noerr) = @_;
1579
1580     return $value if parse_smbios1($value);
1581
1582     return undef if $noerr;
1583
1584     die "unable to parse smbios (type 1) options\n";
1585 }
1586
1587 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
1588 sub verify_bootdisk {
1589     my ($value, $noerr) = @_;
1590
1591     return $value if valid_drivename($value);
1592
1593     return undef if $noerr;
1594
1595     die "invalid boot disk '$value'\n";
1596 }
1597
1598 PVE::JSONSchema::register_format('pve-qm-numanode', \&verify_numa);
1599 sub verify_numa {
1600     my ($value, $noerr) = @_;
1601
1602     return $value if parse_numa($value);
1603
1604     return undef if $noerr;
1605
1606     die "unable to parse numa options\n";
1607 }
1608
1609 PVE::JSONSchema::register_format('pve-qm-net', \&verify_net);
1610 sub verify_net {
1611     my ($value, $noerr) = @_;
1612
1613     return $value if parse_net($value);
1614
1615     return undef if $noerr;
1616
1617     die "unable to parse network options\n";
1618 }
1619
1620 PVE::JSONSchema::register_format('pve-qm-drive', \&verify_drive);
1621 sub verify_drive {
1622     my ($value, $noerr) = @_;
1623
1624     return $value if parse_drive(undef, $value);
1625
1626     return undef if $noerr;
1627
1628     die "unable to parse drive options\n";
1629 }
1630
1631 PVE::JSONSchema::register_format('pve-qm-hostpci', \&verify_hostpci);
1632 sub verify_hostpci {
1633     my ($value, $noerr) = @_;
1634
1635     return $value if parse_hostpci($value);
1636
1637     return undef if $noerr;
1638
1639     die "unable to parse pci id\n";
1640 }
1641
1642 PVE::JSONSchema::register_format('pve-qm-watchdog', \&verify_watchdog);
1643 sub verify_watchdog {
1644     my ($value, $noerr) = @_;
1645
1646     return $value if parse_watchdog($value);
1647
1648     return undef if $noerr;
1649
1650     die "unable to parse watchdog options\n";
1651 }
1652
1653 sub parse_watchdog {
1654     my ($value) = @_;
1655
1656     return undef if !$value;
1657
1658     my $res = {};
1659
1660     foreach my $p (split(/,/, $value)) {
1661         next if $p =~ m/^\s*$/;
1662
1663         if ($p =~ m/^(model=)?(i6300esb|ib700)$/) {
1664             $res->{model} = $2;
1665         } elsif ($p =~ m/^(action=)?(reset|shutdown|poweroff|pause|debug|none)$/) {
1666             $res->{action} = $2;
1667         } else {
1668             return undef;
1669         }
1670     }
1671
1672     return $res;
1673 }
1674
1675 sub parse_usb_device {
1676     my ($value) = @_;
1677
1678     return undef if !$value;
1679
1680     my @dl = split(/,/, $value);
1681     my $found;
1682
1683     my $res = {};
1684     foreach my $v (@dl) {
1685         if ($v =~ m/^host=(0x)?([0-9A-Fa-f]{4}):(0x)?([0-9A-Fa-f]{4})$/) {
1686             $found = 1;
1687             $res->{vendorid} = $2;
1688             $res->{productid} = $4;
1689         } elsif ($v =~ m/^host=(\d+)\-(\d+(\.\d+)*)$/) {
1690             $found = 1;
1691             $res->{hostbus} = $1;
1692             $res->{hostport} = $2;
1693         } elsif ($v =~ m/^spice$/) {
1694             $found = 1;
1695             $res->{spice} = 1;
1696         } else {
1697             return undef;
1698         }
1699     }
1700     return undef if !$found;
1701
1702     return $res;
1703 }
1704
1705 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
1706 sub verify_usb_device {
1707     my ($value, $noerr) = @_;
1708
1709     return $value if parse_usb_device($value);
1710
1711     return undef if $noerr;
1712
1713     die "unable to parse usb device\n";
1714 }
1715
1716 # add JSON properties for create and set function
1717 sub json_config_properties {
1718     my $prop = shift;
1719
1720     foreach my $opt (keys %$confdesc) {
1721         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
1722         $prop->{$opt} = $confdesc->{$opt};
1723     }
1724
1725     return $prop;
1726 }
1727
1728 sub check_type {
1729     my ($key, $value) = @_;
1730
1731     die "unknown setting '$key'\n" if !$confdesc->{$key};
1732
1733     my $type = $confdesc->{$key}->{type};
1734
1735     if (!defined($value)) {
1736         die "got undefined value\n";
1737     }
1738
1739     if ($value =~ m/[\n\r]/) {
1740         die "property contains a line feed\n";
1741     }
1742
1743     if ($type eq 'boolean') {
1744         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
1745         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
1746         die "type check ('boolean') failed - got '$value'\n";
1747     } elsif ($type eq 'integer') {
1748         return int($1) if $value =~ m/^(\d+)$/;
1749         die "type check ('integer') failed - got '$value'\n";
1750     } elsif ($type eq 'number') {
1751         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
1752         die "type check ('number') failed - got '$value'\n";
1753     } elsif ($type eq 'string') {
1754         if (my $fmt = $confdesc->{$key}->{format}) {
1755             if ($fmt eq 'pve-qm-drive') {
1756                 # special case - we need to pass $key to parse_drive()
1757                 my $drive = parse_drive($key, $value);
1758                 return $value if $drive;
1759                 die "unable to parse drive options\n";
1760             }
1761             PVE::JSONSchema::check_format($fmt, $value);
1762             return $value;
1763         }
1764         $value =~ s/^\"(.*)\"$/$1/;
1765         return $value;
1766     } else {
1767         die "internal error"
1768     }
1769 }
1770
1771 sub lock_config_full {
1772     my ($vmid, $timeout, $code, @param) = @_;
1773
1774     my $filename = config_file_lock($vmid);
1775
1776     my $res = lock_file($filename, $timeout, $code, @param);
1777
1778     die $@ if $@;
1779
1780     return $res;
1781 }
1782
1783 sub lock_config_mode {
1784     my ($vmid, $timeout, $shared, $code, @param) = @_;
1785
1786     my $filename = config_file_lock($vmid);
1787
1788     my $res = lock_file_full($filename, $timeout, $shared, $code, @param);
1789
1790     die $@ if $@;
1791
1792     return $res;
1793 }
1794
1795 sub lock_config {
1796     my ($vmid, $code, @param) = @_;
1797
1798     return lock_config_full($vmid, 10, $code, @param);
1799 }
1800
1801 sub cfs_config_path {
1802     my ($vmid, $node) = @_;
1803
1804     $node = $nodename if !$node;
1805     return "nodes/$node/qemu-server/$vmid.conf";
1806 }
1807
1808 sub check_iommu_support{
1809     #fixme : need to check IOMMU support
1810     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
1811
1812     my $iommu=1;
1813     return $iommu;
1814
1815 }
1816
1817 sub config_file {
1818     my ($vmid, $node) = @_;
1819
1820     my $cfspath = cfs_config_path($vmid, $node);
1821     return "/etc/pve/$cfspath";
1822 }
1823
1824 sub config_file_lock {
1825     my ($vmid) = @_;
1826
1827     return "$lock_dir/lock-$vmid.conf";
1828 }
1829
1830 sub touch_config {
1831     my ($vmid) = @_;
1832
1833     my $conf = config_file($vmid);
1834     utime undef, undef, $conf;
1835 }
1836
1837 sub destroy_vm {
1838     my ($storecfg, $vmid, $keep_empty_config) = @_;
1839
1840     my $conffile = config_file($vmid);
1841
1842     my $conf = load_config($vmid);
1843
1844     check_lock($conf);
1845
1846     # only remove disks owned by this VM
1847     foreach_drive($conf, sub {
1848         my ($ds, $drive) = @_;
1849
1850         return if drive_is_cdrom($drive);
1851
1852         my $volid = $drive->{file};
1853
1854         return if !$volid || $volid =~ m|^/|;
1855
1856         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
1857         return if !$path || !$owner || ($owner != $vmid);
1858
1859         PVE::Storage::vdisk_free($storecfg, $volid);
1860     });
1861
1862     if ($keep_empty_config) {
1863         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
1864     } else {
1865         unlink $conffile;
1866     }
1867
1868     # also remove unused disk
1869     eval {
1870         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
1871
1872         eval {
1873             PVE::Storage::foreach_volid($dl, sub {
1874                 my ($volid, $sid, $volname, $d) = @_;
1875                 PVE::Storage::vdisk_free($storecfg, $volid);
1876             });
1877         };
1878         warn $@ if $@;
1879
1880     };
1881     warn $@ if $@;
1882 }
1883
1884 sub load_config {
1885     my ($vmid, $node) = @_;
1886
1887     my $cfspath = cfs_config_path($vmid, $node);
1888
1889     my $conf = PVE::Cluster::cfs_read_file($cfspath);
1890
1891     die "no such VM ('$vmid')\n" if !defined($conf);
1892
1893     return $conf;
1894 }
1895
1896 sub parse_vm_config {
1897     my ($filename, $raw) = @_;
1898
1899     return undef if !defined($raw);
1900
1901     my $res = {
1902         digest => Digest::SHA::sha1_hex($raw),
1903         snapshots => {},
1904         pending => {},
1905     };
1906
1907     $filename =~ m|/qemu-server/(\d+)\.conf$|
1908         || die "got strange filename '$filename'";
1909
1910     my $vmid = $1;
1911
1912     my $conf = $res;
1913     my $descr;
1914     my $section = '';
1915
1916     my @lines = split(/\n/, $raw);
1917     foreach my $line (@lines) {
1918         next if $line =~ m/^\s*$/;
1919
1920         if ($line =~ m/^\[PENDING\]\s*$/i) {
1921             $section = 'pending';
1922             if (defined($descr)) {
1923                 $descr =~ s/\s+$//;
1924                 $conf->{description} = $descr;
1925             }
1926             $descr = undef;
1927             $conf = $res->{$section} = {};
1928             next;
1929
1930         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
1931             $section = $1;
1932             if (defined($descr)) {
1933                 $descr =~ s/\s+$//;
1934                 $conf->{description} = $descr;
1935             }
1936             $descr = undef;
1937             $conf = $res->{snapshots}->{$section} = {};
1938             next;
1939         }
1940
1941         if ($line =~ m/^\#(.*)\s*$/) {
1942             $descr = '' if !defined($descr);
1943             $descr .= PVE::Tools::decode_text($1) . "\n";
1944             next;
1945         }
1946
1947         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
1948             $descr = '' if !defined($descr);
1949             $descr .= PVE::Tools::decode_text($2);
1950         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
1951             $conf->{snapstate} = $1;
1952         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
1953             my $key = $1;
1954             my $value = $2;
1955             $conf->{$key} = $value;
1956         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
1957             my $value = $1;
1958             if ($section eq 'pending') {
1959                 $conf->{delete} = $value; # we parse this later
1960             } else {
1961                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
1962             }
1963         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
1964             my $key = $1;
1965             my $value = $2;
1966             eval { $value = check_type($key, $value); };
1967             if ($@) {
1968                 warn "vm $vmid - unable to parse value of '$key' - $@";
1969             } else {
1970                 my $fmt = $confdesc->{$key}->{format};
1971                 if ($fmt && $fmt eq 'pve-qm-drive') {
1972                     my $v = parse_drive($key, $value);
1973                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
1974                         $v->{file} = $volid;
1975                         $value = print_drive($vmid, $v);
1976                     } else {
1977                         warn "vm $vmid - unable to parse value of '$key'\n";
1978                         next;
1979                     }
1980                 }
1981
1982                 if ($key eq 'cdrom') {
1983                     $conf->{ide2} = $value;
1984                 } else {
1985                     $conf->{$key} = $value;
1986                 }
1987             }
1988         }
1989     }
1990
1991     if (defined($descr)) {
1992         $descr =~ s/\s+$//;
1993         $conf->{description} = $descr;
1994     }
1995     delete $res->{snapstate}; # just to be sure
1996
1997     return $res;
1998 }
1999
2000 sub write_vm_config {
2001     my ($filename, $conf) = @_;
2002
2003     delete $conf->{snapstate}; # just to be sure
2004
2005     if ($conf->{cdrom}) {
2006         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2007         $conf->{ide2} = $conf->{cdrom};
2008         delete $conf->{cdrom};
2009     }
2010
2011     # we do not use 'smp' any longer
2012     if ($conf->{sockets}) {
2013         delete $conf->{smp};
2014     } elsif ($conf->{smp}) {
2015         $conf->{sockets} = $conf->{smp};
2016         delete $conf->{cores};
2017         delete $conf->{smp};
2018     }
2019
2020     my $used_volids = {};
2021
2022     my $cleanup_config = sub {
2023         my ($cref, $pending, $snapname) = @_;
2024
2025         foreach my $key (keys %$cref) {
2026             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2027                 $key eq 'snapstate' || $key eq 'pending';
2028             my $value = $cref->{$key};
2029             if ($key eq 'delete') {
2030                 die "propertry 'delete' is only allowed in [PENDING]\n"
2031                     if !$pending;
2032                 # fixme: check syntax?
2033                 next;
2034             }
2035             eval { $value = check_type($key, $value); };
2036             die "unable to parse value of '$key' - $@" if $@;
2037
2038             $cref->{$key} = $value;
2039
2040             if (!$snapname && valid_drivename($key)) {
2041                 my $drive = parse_drive($key, $value);
2042                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2043             }
2044         }
2045     };
2046
2047     &$cleanup_config($conf);
2048
2049     &$cleanup_config($conf->{pending}, 1);
2050
2051     foreach my $snapname (keys %{$conf->{snapshots}}) {
2052         die "internal error" if $snapname eq 'pending';
2053         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2054     }
2055
2056     # remove 'unusedX' settings if we re-add a volume
2057     foreach my $key (keys %$conf) {
2058         my $value = $conf->{$key};
2059         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2060             delete $conf->{$key};
2061         }
2062     }
2063
2064     my $generate_raw_config = sub {
2065         my ($conf, $pending) = @_;
2066
2067         my $raw = '';
2068
2069         # add description as comment to top of file
2070         if (defined(my $descr = $conf->{description})) {
2071             if ($descr) {
2072                 foreach my $cl (split(/\n/, $descr)) {
2073                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2074                 }
2075             } else {
2076                 $raw .= "#\n" if $pending;
2077             }
2078         }
2079
2080         foreach my $key (sort keys %$conf) {
2081             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2082             $raw .= "$key: $conf->{$key}\n";
2083         }
2084         return $raw;
2085     };
2086
2087     my $raw = &$generate_raw_config($conf);
2088
2089     if (scalar(keys %{$conf->{pending}})){
2090         $raw .= "\n[PENDING]\n";
2091         $raw .= &$generate_raw_config($conf->{pending}, 1);
2092     }
2093
2094     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2095         $raw .= "\n[$snapname]\n";
2096         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2097     }
2098
2099     return $raw;
2100 }
2101
2102 sub update_config_nolock {
2103     my ($vmid, $conf, $skiplock) = @_;
2104
2105     check_lock($conf) if !$skiplock;
2106
2107     my $cfspath = cfs_config_path($vmid);
2108
2109     PVE::Cluster::cfs_write_file($cfspath, $conf);
2110 }
2111
2112 sub update_config {
2113     my ($vmid, $conf, $skiplock) = @_;
2114
2115     lock_config($vmid, &update_config_nolock, $conf, $skiplock);
2116 }
2117
2118 sub load_defaults {
2119
2120     my $res = {};
2121
2122     # we use static defaults from our JSON schema configuration
2123     foreach my $key (keys %$confdesc) {
2124         if (defined(my $default = $confdesc->{$key}->{default})) {
2125             $res->{$key} = $default;
2126         }
2127     }
2128
2129     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2130     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2131
2132     return $res;
2133 }
2134
2135 sub config_list {
2136     my $vmlist = PVE::Cluster::get_vmlist();
2137     my $res = {};
2138     return $res if !$vmlist || !$vmlist->{ids};
2139     my $ids = $vmlist->{ids};
2140
2141     foreach my $vmid (keys %$ids) {
2142         my $d = $ids->{$vmid};
2143         next if !$d->{node} || $d->{node} ne $nodename;
2144         next if !$d->{type} || $d->{type} ne 'qemu';
2145         $res->{$vmid}->{exists} = 1;
2146     }
2147     return $res;
2148 }
2149
2150 # test if VM uses local resources (to prevent migration)
2151 sub check_local_resources {
2152     my ($conf, $noerr) = @_;
2153
2154     my $loc_res = 0;
2155
2156     $loc_res = 1 if $conf->{hostusb}; # old syntax
2157     $loc_res = 1 if $conf->{hostpci}; # old syntax
2158
2159     foreach my $k (keys %$conf) {
2160         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2161         # sockets are safe: they will recreated be on the target side post-migrate
2162         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2163         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2164     }
2165
2166     die "VM uses local resources\n" if $loc_res && !$noerr;
2167
2168     return $loc_res;
2169 }
2170
2171 # check if used storages are available on all nodes (use by migrate)
2172 sub check_storage_availability {
2173     my ($storecfg, $conf, $node) = @_;
2174
2175     foreach_drive($conf, sub {
2176         my ($ds, $drive) = @_;
2177
2178         my $volid = $drive->{file};
2179         return if !$volid;
2180
2181         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2182         return if !$sid;
2183
2184         # check if storage is available on both nodes
2185         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2186         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2187    });
2188 }
2189
2190 # list nodes where all VM images are available (used by has_feature API)
2191 sub shared_nodes {
2192     my ($conf, $storecfg) = @_;
2193
2194     my $nodelist = PVE::Cluster::get_nodelist();
2195     my $nodehash = { map { $_ => 1 } @$nodelist };
2196     my $nodename = PVE::INotify::nodename();
2197
2198     foreach_drive($conf, sub {
2199         my ($ds, $drive) = @_;
2200
2201         my $volid = $drive->{file};
2202         return if !$volid;
2203
2204         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2205         if ($storeid) {
2206             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2207             if ($scfg->{disable}) {
2208                 $nodehash = {};
2209             } elsif (my $avail = $scfg->{nodes}) {
2210                 foreach my $node (keys %$nodehash) {
2211                     delete $nodehash->{$node} if !$avail->{$node};
2212                 }
2213             } elsif (!$scfg->{shared}) {
2214                 foreach my $node (keys %$nodehash) {
2215                     delete $nodehash->{$node} if $node ne $nodename
2216                 }
2217             }
2218         }
2219     });
2220
2221     return $nodehash
2222 }
2223
2224 sub check_lock {
2225     my ($conf) = @_;
2226
2227     die "VM is locked ($conf->{lock})\n" if $conf->{lock};
2228 }
2229
2230 sub check_cmdline {
2231     my ($pidfile, $pid) = @_;
2232
2233     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2234     if (defined($fh)) {
2235         my $line = <$fh>;
2236         $fh->close;
2237         return undef if !$line;
2238         my @param = split(/\0/, $line);
2239
2240         my $cmd = $param[0];
2241         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2242
2243         for (my $i = 0; $i < scalar (@param); $i++) {
2244             my $p = $param[$i];
2245             next if !$p;
2246             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2247                 my $p = $param[$i+1];
2248                 return 1 if $p && ($p eq $pidfile);
2249                 return undef;
2250             }
2251         }
2252     }
2253     return undef;
2254 }
2255
2256 sub check_running {
2257     my ($vmid, $nocheck, $node) = @_;
2258
2259     my $filename = config_file($vmid, $node);
2260
2261     die "unable to find configuration file for VM $vmid - no such machine\n"
2262         if !$nocheck && ! -f $filename;
2263
2264     my $pidfile = pidfile_name($vmid);
2265
2266     if (my $fd = IO::File->new("<$pidfile")) {
2267         my $st = stat($fd);
2268         my $line = <$fd>;
2269         close($fd);
2270
2271         my $mtime = $st->mtime;
2272         if ($mtime > time()) {
2273             warn "file '$filename' modified in future\n";
2274         }
2275
2276         if ($line =~ m/^(\d+)$/) {
2277             my $pid = $1;
2278             if (check_cmdline($pidfile, $pid)) {
2279                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2280                     return $pid;
2281                 }
2282             }
2283         }
2284     }
2285
2286     return undef;
2287 }
2288
2289 sub vzlist {
2290
2291     my $vzlist = config_list();
2292
2293     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2294
2295     while (defined(my $de = $fd->read)) {
2296         next if $de !~ m/^(\d+)\.pid$/;
2297         my $vmid = $1;
2298         next if !defined($vzlist->{$vmid});
2299         if (my $pid = check_running($vmid)) {
2300             $vzlist->{$vmid}->{pid} = $pid;
2301         }
2302     }
2303
2304     return $vzlist;
2305 }
2306
2307 sub disksize {
2308     my ($storecfg, $conf) = @_;
2309
2310     my $bootdisk = $conf->{bootdisk};
2311     return undef if !$bootdisk;
2312     return undef if !valid_drivename($bootdisk);
2313
2314     return undef if !$conf->{$bootdisk};
2315
2316     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2317     return undef if !defined($drive);
2318
2319     return undef if drive_is_cdrom($drive);
2320
2321     my $volid = $drive->{file};
2322     return undef if !$volid;
2323
2324     return $drive->{size};
2325 }
2326
2327 my $last_proc_pid_stat;
2328
2329 # get VM status information
2330 # This must be fast and should not block ($full == false)
2331 # We only query KVM using QMP if $full == true (this can be slow)
2332 sub vmstatus {
2333     my ($opt_vmid, $full) = @_;
2334
2335     my $res = {};
2336
2337     my $storecfg = PVE::Storage::config();
2338
2339     my $list = vzlist();
2340     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2341
2342     my $cpucount = $cpuinfo->{cpus} || 1;
2343
2344     foreach my $vmid (keys %$list) {
2345         next if $opt_vmid && ($vmid ne $opt_vmid);
2346
2347         my $cfspath = cfs_config_path($vmid);
2348         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2349
2350         my $d = {};
2351         $d->{pid} = $list->{$vmid}->{pid};
2352
2353         # fixme: better status?
2354         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2355
2356         my $size = disksize($storecfg, $conf);
2357         if (defined($size)) {
2358             $d->{disk} = 0; # no info available
2359             $d->{maxdisk} = $size;
2360         } else {
2361             $d->{disk} = 0;
2362             $d->{maxdisk} = 0;
2363         }
2364
2365         $d->{cpus} = ($conf->{sockets} || 1) * ($conf->{cores} || 1);
2366         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2367         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2368
2369         $d->{name} = $conf->{name} || "VM $vmid";
2370         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024) : 0;
2371
2372         if ($conf->{balloon}) {
2373             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2374             $d->{shares} = defined($conf->{shares}) ? $conf->{shares} : 1000;
2375         }
2376
2377         $d->{uptime} = 0;
2378         $d->{cpu} = 0;
2379         $d->{mem} = 0;
2380
2381         $d->{netout} = 0;
2382         $d->{netin} = 0;
2383
2384         $d->{diskread} = 0;
2385         $d->{diskwrite} = 0;
2386
2387         $d->{template} = is_template($conf);
2388
2389         $res->{$vmid} = $d;
2390     }
2391
2392     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2393     foreach my $dev (keys %$netdev) {
2394         next if $dev !~ m/^tap([1-9]\d*)i/;
2395         my $vmid = $1;
2396         my $d = $res->{$vmid};
2397         next if !$d;
2398
2399         $d->{netout} += $netdev->{$dev}->{receive};
2400         $d->{netin} += $netdev->{$dev}->{transmit};
2401
2402         if ($full) {
2403             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2404             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2405         }
2406
2407     }
2408
2409     my $ctime = gettimeofday;
2410
2411     foreach my $vmid (keys %$list) {
2412
2413         my $d = $res->{$vmid};
2414         my $pid = $d->{pid};
2415         next if !$pid;
2416
2417         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2418         next if !$pstat; # not running
2419
2420         my $used = $pstat->{utime} + $pstat->{stime};
2421
2422         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2423
2424         if ($pstat->{vsize}) {
2425             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2426         }
2427
2428         my $old = $last_proc_pid_stat->{$pid};
2429         if (!$old) {
2430             $last_proc_pid_stat->{$pid} = {
2431                 time => $ctime,
2432                 used => $used,
2433                 cpu => 0,
2434             };
2435             next;
2436         }
2437
2438         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2439
2440         if ($dtime > 1000) {
2441             my $dutime = $used -  $old->{used};
2442
2443             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2444             $last_proc_pid_stat->{$pid} = {
2445                 time => $ctime,
2446                 used => $used,
2447                 cpu => $d->{cpu},
2448             };
2449         } else {
2450             $d->{cpu} = $old->{cpu};
2451         }
2452     }
2453
2454     return $res if !$full;
2455
2456     my $qmpclient = PVE::QMPClient->new();
2457
2458     my $ballooncb = sub {
2459         my ($vmid, $resp) = @_;
2460
2461         my $info = $resp->{'return'};
2462         return if !$info->{max_mem};
2463
2464         my $d = $res->{$vmid};
2465
2466         # use memory assigned to VM
2467         $d->{maxmem} = $info->{max_mem};
2468         $d->{balloon} = $info->{actual};
2469
2470         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2471             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2472             $d->{freemem} = $info->{free_mem};
2473         }
2474
2475         $d->{ballooninfo} = $info;
2476     };
2477
2478     my $blockstatscb = sub {
2479         my ($vmid, $resp) = @_;
2480         my $data = $resp->{'return'} || [];
2481         my $totalrdbytes = 0;
2482         my $totalwrbytes = 0;
2483
2484         for my $blockstat (@$data) {
2485             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2486             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2487
2488             $blockstat->{device} =~ s/drive-//;
2489             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2490         }
2491         $res->{$vmid}->{diskread} = $totalrdbytes;
2492         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2493     };
2494
2495     my $statuscb = sub {
2496         my ($vmid, $resp) = @_;
2497
2498         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2499         # this fails if ballon driver is not loaded, so this must be
2500         # the last commnand (following command are aborted if this fails).
2501         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2502
2503         my $status = 'unknown';
2504         if (!defined($status = $resp->{'return'}->{status})) {
2505             warn "unable to get VM status\n";
2506             return;
2507         }
2508
2509         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2510     };
2511
2512     foreach my $vmid (keys %$list) {
2513         next if $opt_vmid && ($vmid ne $opt_vmid);
2514         next if !$res->{$vmid}->{pid}; # not running
2515         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2516     }
2517
2518     $qmpclient->queue_execute(undef, 1);
2519
2520     foreach my $vmid (keys %$list) {
2521         next if $opt_vmid && ($vmid ne $opt_vmid);
2522         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2523     }
2524
2525     return $res;
2526 }
2527
2528 sub foreach_dimm {
2529     my ($conf, $vmid, $memory, $sockets, $func) = @_;
2530
2531     my $dimm_id = 0;
2532     my $current_size = 1024;
2533     my $dimm_size = 512;
2534     return if $current_size == $memory;
2535
2536     for (my $j = 0; $j < 8; $j++) {
2537         for (my $i = 0; $i < 32; $i++) {
2538             my $name = "dimm${dimm_id}";
2539             $dimm_id++;
2540             my $numanode = $i % $sockets;
2541             $current_size += $dimm_size;
2542             &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
2543             return  $current_size if $current_size >= $memory;
2544         }
2545         $dimm_size *= 2;
2546     }
2547 }
2548
2549 sub foreach_reverse_dimm {
2550     my ($conf, $vmid, $memory, $sockets, $func) = @_;
2551
2552     my $dimm_id = 253;
2553     my $current_size = 4177920;
2554     my $dimm_size = 65536;
2555     return if $current_size == $memory;
2556
2557     for (my $j = 0; $j < 8; $j++) {
2558         for (my $i = 0; $i < 32; $i++) {
2559             my $name = "dimm${dimm_id}";
2560             $dimm_id--;
2561             my $numanode = $i % $sockets;
2562             $current_size -= $dimm_size;
2563             &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
2564             return  $current_size if $current_size <= $memory;
2565         }
2566         $dimm_size /= 2;
2567     }
2568 }
2569
2570 sub foreach_drive {
2571     my ($conf, $func) = @_;
2572
2573     foreach my $ds (keys %$conf) {
2574         next if !valid_drivename($ds);
2575
2576         my $drive = parse_drive($ds, $conf->{$ds});
2577         next if !$drive;
2578
2579         &$func($ds, $drive);
2580     }
2581 }
2582
2583 sub foreach_volid {
2584     my ($conf, $func) = @_;
2585
2586     my $volhash = {};
2587
2588     my $test_volid = sub {
2589         my ($volid, $is_cdrom) = @_;
2590
2591         return if !$volid;
2592
2593         $volhash->{$volid} = $is_cdrom || 0;
2594     };
2595
2596     foreach_drive($conf, sub {
2597         my ($ds, $drive) = @_;
2598         &$test_volid($drive->{file}, drive_is_cdrom($drive));
2599     });
2600
2601     foreach my $snapname (keys %{$conf->{snapshots}}) {
2602         my $snap = $conf->{snapshots}->{$snapname};
2603         &$test_volid($snap->{vmstate}, 0);
2604         foreach_drive($snap, sub {
2605             my ($ds, $drive) = @_;
2606             &$test_volid($drive->{file}, drive_is_cdrom($drive));
2607         });
2608     }
2609
2610     foreach my $volid (keys %$volhash) {
2611         &$func($volid, $volhash->{$volid});
2612     }
2613 }
2614
2615 sub vga_conf_has_spice {
2616     my ($vga) = @_;
2617
2618     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
2619
2620     return $1 || 1;
2621 }
2622
2623 sub config_to_command {
2624     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2625
2626     my $cmd = [];
2627     my $globalFlags = [];
2628     my $machineFlags = [];
2629     my $rtcFlags = [];
2630     my $cpuFlags = [];
2631     my $devices = [];
2632     my $pciaddr = '';
2633     my $bridges = {};
2634     my $kvmver = kvm_user_version();
2635     my $vernum = 0; # unknown
2636     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
2637         $vernum = $1*1000000+$2*1000;
2638     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
2639         $vernum = $1*1000000+$2*1000+$3;
2640     }
2641
2642     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
2643
2644     my $have_ovz = -f '/proc/vz/vestat';
2645
2646     my $q35 = machine_type_is_q35($conf);
2647     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2648     my $machine_type = $forcemachine || $conf->{machine};
2649     my $use_old_bios_files = undef;
2650     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
2651
2652     my $cpuunits = defined($conf->{cpuunits}) ?
2653             $conf->{cpuunits} : $defaults->{cpuunits};
2654
2655     push @$cmd, '/usr/bin/systemd-run';
2656     push @$cmd, '--scope';
2657     push @$cmd, '--slice', "qemu";
2658     push @$cmd, '--unit', $vmid;
2659     # set KillMode=none, so that systemd don't kill those scopes
2660     # at shutdown (pve-manager service should stop the VMs instead)
2661     push @$cmd, '-p', "KillMode=none";
2662     push @$cmd, '-p', "CPUShares=$cpuunits";
2663     if ($conf->{cpulimit}) {
2664         my $cpulimit = int($conf->{cpulimit} * 100);
2665         push @$cmd, '-p', "CPUQuota=$cpulimit\%";
2666     }
2667
2668     push @$cmd, '/usr/bin/kvm';
2669
2670     push @$cmd, '-id', $vmid;
2671
2672     my $use_virtio = 0;
2673
2674     my $qmpsocket = qmp_socket($vmid);
2675     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2676     push @$cmd, '-mon', "chardev=qmp,mode=control";
2677
2678     my $socket = vnc_socket($vmid);
2679     push @$cmd,  '-vnc', "unix:$socket,x509,password";
2680
2681     push @$cmd, '-pidfile' , pidfile_name($vmid);
2682
2683     push @$cmd, '-daemonize';
2684
2685     if ($conf->{smbios1}) {
2686         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
2687     }
2688
2689     if ($q35) {
2690         # the q35 chipset support native usb2, so we enable usb controller
2691         # by default for this machine type
2692         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
2693     } else {
2694         $pciaddr = print_pci_addr("piix3", $bridges);
2695         push @$devices, '-device', "piix3-usb-uhci,id=uhci$pciaddr.0x2";
2696
2697         my $use_usb2 = 0;
2698         for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2699             next if !$conf->{"usb$i"};
2700             $use_usb2 = 1;
2701         }
2702         # include usb device config
2703         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-usb.cfg' if $use_usb2;
2704     }
2705
2706     my $vga = $conf->{vga};
2707
2708     my $qxlnum = vga_conf_has_spice($vga);
2709     $vga = 'qxl' if $qxlnum;
2710
2711     if (!$vga) {
2712         if ($conf->{ostype} && ($conf->{ostype} eq 'win8' ||
2713                                 $conf->{ostype} eq 'win7' ||
2714                                 $conf->{ostype} eq 'w2k8')) {
2715             $vga = 'std';
2716         } else {
2717             $vga = 'cirrus';
2718         }
2719     }
2720
2721     # enable absolute mouse coordinates (needed by vnc)
2722     my $tablet;
2723     if (defined($conf->{tablet})) {
2724         $tablet = $conf->{tablet};
2725     } else {
2726         $tablet = $defaults->{tablet};
2727         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
2728         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
2729     }
2730
2731     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
2732
2733     # host pci devices
2734     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
2735         my $d = parse_hostpci($conf->{"hostpci$i"});
2736         next if !$d;
2737
2738         my $pcie = $d->{pcie};
2739         if($pcie){
2740             die "q35 machine model is not enabled" if !$q35;
2741             $pciaddr = print_pcie_addr("hostpci$i");
2742         }else{
2743             $pciaddr = print_pci_addr("hostpci$i", $bridges);
2744         }
2745
2746         my $rombar = $d->{rombar} && $d->{rombar} eq 'off' ? ",rombar=0" : "";
2747         my $xvga = $d->{'x-vga'} && $d->{'x-vga'} eq 'on' ? ",x-vga=on" : "";
2748         if ($xvga && $xvga ne '') {
2749             push @$cpuFlags, 'kvm=off';
2750             $vga = 'none';
2751         }
2752         my $pcidevices = $d->{pciid};
2753         my $multifunction = 1 if @$pcidevices > 1;
2754
2755         my $j=0;
2756         foreach my $pcidevice (@$pcidevices) {
2757
2758             my $id = "hostpci$i";
2759             $id .= ".$j" if $multifunction;
2760             my $addr = $pciaddr;
2761             $addr .= ".$j" if $multifunction;
2762             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
2763
2764             if($j == 0){
2765                 $devicestr .= "$rombar$xvga";
2766                 $devicestr .= ",multifunction=on" if $multifunction;
2767             }
2768
2769             push @$devices, '-device', $devicestr;
2770             $j++;
2771         }
2772     }
2773
2774     # usb devices
2775     for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2776         my $d = parse_usb_device($conf->{"usb$i"});
2777         next if !$d;
2778         if ($d->{vendorid} && $d->{productid}) {
2779             push @$devices, '-device', "usb-host,vendorid=0x$d->{vendorid},productid=0x$d->{productid}";
2780         } elsif (defined($d->{hostbus}) && defined($d->{hostport})) {
2781             push @$devices, '-device', "usb-host,hostbus=$d->{hostbus},hostport=$d->{hostport}";
2782         } elsif ($d->{spice}) {
2783             # usb redir support for spice
2784             push @$devices, '-chardev', "spicevmc,id=usbredirchardev$i,name=usbredir";
2785             push @$devices, '-device', "usb-redir,chardev=usbredirchardev$i,id=usbredirdev$i,bus=ehci.0";
2786         }
2787     }
2788
2789     # serial devices
2790     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2791         if (my $path = $conf->{"serial$i"}) {
2792             if ($path eq 'socket') {
2793                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
2794                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
2795                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2796             } else {
2797                 die "no such serial device\n" if ! -c $path;
2798                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
2799                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2800             }
2801         }
2802     }
2803
2804     # parallel devices
2805     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
2806         if (my $path = $conf->{"parallel$i"}) {
2807             die "no such parallel device\n" if ! -c $path;
2808             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
2809             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
2810             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
2811         }
2812     }
2813
2814     my $vmname = $conf->{name} || "vm$vmid";
2815
2816     push @$cmd, '-name', $vmname;
2817
2818     my $sockets = 1;
2819     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
2820     $sockets = $conf->{sockets} if  $conf->{sockets};
2821
2822     my $cores = $conf->{cores} || 1;
2823
2824     my $maxcpus = $sockets * $cores;
2825
2826     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
2827
2828     my $allowed_vcpus = $cpuinfo->{cpus};
2829
2830     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
2831         if ($allowed_vcpus < $maxcpus);
2832
2833     push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
2834
2835     push @$cmd, '-nodefaults';
2836
2837     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2838
2839     my $bootindex_hash = {};
2840     my $i = 1;
2841     foreach my $o (split(//, $bootorder)) {
2842         $bootindex_hash->{$o} = $i*100;
2843         $i++;
2844     }
2845
2846     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000";
2847
2848     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
2849
2850     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
2851
2852     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
2853
2854     # time drift fix
2855     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
2856
2857     my $nokvm = defined($conf->{kvm}) && $conf->{kvm} == 0 ? 1 : 0;
2858     my $useLocaltime = $conf->{localtime};
2859
2860     if (my $ost = $conf->{ostype}) {
2861         # other, wxp, w2k, w2k3, w2k8, wvista, win7, win8, l24, l26, solaris
2862
2863         if ($ost =~ m/^w/) { # windows
2864             $useLocaltime = 1 if !defined($conf->{localtime});
2865
2866             # use time drift fix when acpi is enabled
2867             if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
2868                 $tdf = 1 if !defined($conf->{tdf});
2869             }
2870         }
2871
2872         if ($ost eq 'win7' || $ost eq 'win8' || $ost eq 'w2k8' ||
2873             $ost eq 'wvista') {
2874             push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
2875             push @$cmd, '-no-hpet';
2876             if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
2877                 push @$cpuFlags , 'hv_spinlocks=0x1fff' if !$nokvm;
2878                 push @$cpuFlags , 'hv_vapic' if !$nokvm;
2879                 push @$cpuFlags , 'hv_time' if !$nokvm;
2880
2881             } else {
2882                 push @$cpuFlags , 'hv_spinlocks=0xffff' if !$nokvm;
2883             }
2884         }
2885
2886         if ($ost eq 'win7' || $ost eq 'win8') {
2887             push @$cpuFlags , 'hv_relaxed' if !$nokvm;
2888         }
2889     }
2890
2891     push @$rtcFlags, 'driftfix=slew' if $tdf;
2892
2893     if ($nokvm) {
2894         push @$machineFlags, 'accel=tcg';
2895     } else {
2896         die "No accelerator found!\n" if !$cpuinfo->{hvm};
2897     }
2898
2899     if ($machine_type) {
2900         push @$machineFlags, "type=${machine_type}";
2901     }
2902
2903     if ($conf->{startdate}) {
2904         push @$rtcFlags, "base=$conf->{startdate}";
2905     } elsif ($useLocaltime) {
2906         push @$rtcFlags, 'base=localtime';
2907     }
2908
2909     my $cpu = $nokvm ? "qemu64" : "kvm64";
2910     $cpu = $conf->{cpu} if $conf->{cpu};
2911
2912     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
2913
2914     push @$cpuFlags , '-x2apic'
2915         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
2916
2917     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
2918
2919     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
2920
2921     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
2922
2923         push @$cpuFlags , '+kvm_pv_unhalt' if !$nokvm;
2924         push @$cpuFlags , '+kvm_pv_eoi' if !$nokvm;
2925         push @$cpuFlags , '-kvm_steal_time' if !$nokvm;
2926     }
2927
2928     push @$cpuFlags, 'enforce' if $cpu ne 'host' && !$nokvm;
2929
2930     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
2931
2932     push @$cmd, '-cpu', $cpu;
2933
2934     my $memory = $conf->{memory} || $defaults->{memory};
2935     my $static_memory = 0;
2936     my $dimm_memory = 0;
2937
2938     if ($hotplug_features->{memory}) {
2939         die "Numa need to be enabled for memory hotplug\n" if !$conf->{numa};
2940         die "Total memory is bigger than ${MAX_MEM}MB\n" if $memory > $MAX_MEM;
2941         $static_memory = $STATICMEM;
2942         die "minimum memory must be ${static_memory}MB\n" if($memory < $static_memory);
2943         $dimm_memory = $memory - $static_memory;
2944         push @$cmd, '-m', "size=${static_memory},slots=255,maxmem=${MAX_MEM}M";
2945
2946     } else {
2947
2948         $static_memory = $memory;
2949         push @$cmd, '-m', $static_memory;
2950     }
2951
2952     if ($conf->{numa}) {
2953
2954         my $numa_totalmemory = undef;
2955         for (my $i = 0; $i < $MAX_NUMA; $i++) {
2956             next if !$conf->{"numa$i"};
2957             my $numa = parse_numa($conf->{"numa$i"});
2958             next if !$numa;
2959             # memory
2960             die "missing numa node$i memory value\n" if !$numa->{memory};
2961             my $numa_memory = $numa->{memory};
2962             $numa_totalmemory += $numa_memory;
2963             my $numa_object = "memory-backend-ram,id=ram-node$i,size=${numa_memory}M";
2964
2965             # cpus
2966             my $cpus_start = $numa->{cpus}->{start};
2967             die "missing numa node$i cpus\n" if !defined($cpus_start);
2968             my $cpus_end = $numa->{cpus}->{end} if defined($numa->{cpus}->{end});
2969             my $cpus = $cpus_start;
2970             if (defined($cpus_end)) {
2971                 $cpus .= "-$cpus_end";
2972                 die "numa node$i :  cpu range $cpus is incorrect\n" if $cpus_end <= $cpus_start;
2973             }
2974
2975             # hostnodes
2976             my $hostnodes_start = $numa->{hostnodes}->{start};
2977             if (defined($hostnodes_start)) {
2978                 my $hostnodes_end = $numa->{hostnodes}->{end} if defined($numa->{hostnodes}->{end});
2979                 my $hostnodes = $hostnodes_start;
2980                 if (defined($hostnodes_end)) {
2981                     $hostnodes .= "-$hostnodes_end";
2982                     die "host node $hostnodes range is incorrect\n" if $hostnodes_end <= $hostnodes_start;
2983                 }
2984
2985                 my $hostnodes_end_range = defined($hostnodes_end) ? $hostnodes_end : $hostnodes_start;
2986                 for (my $i = $hostnodes_start; $i <= $hostnodes_end_range; $i++ ) {
2987                     die "host numa node$i don't exist\n" if ! -d "/sys/devices/system/node/node$i/";
2988                 }
2989
2990                 # policy
2991                 my $policy = $numa->{policy};
2992                 die "you need to define a policy for hostnode $hostnodes\n" if !$policy;
2993                 $numa_object .= ",host-nodes=$hostnodes,policy=$policy";
2994             }
2995
2996             push @$cmd, '-object', $numa_object;
2997             push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
2998         }
2999
3000         die "total memory for NUMA nodes must be equal to vm static memory\n"
3001             if $numa_totalmemory && $numa_totalmemory != $static_memory;
3002
3003         #if no custom tology, we split memory and cores across numa nodes
3004         if(!$numa_totalmemory) {
3005
3006             my $numa_memory = ($static_memory / $sockets) . "M";
3007
3008             for (my $i = 0; $i < $sockets; $i++)  {
3009
3010                 my $cpustart = ($cores * $i);
3011                 my $cpuend = ($cpustart + $cores - 1) if $cores && $cores > 1;
3012                 my $cpus = $cpustart;
3013                 $cpus .= "-$cpuend" if $cpuend;
3014
3015                 push @$cmd, '-object', "memory-backend-ram,size=$numa_memory,id=ram-node$i";
3016                 push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
3017             }
3018         }
3019     }
3020
3021     if ($hotplug_features->{memory}) {
3022         foreach_dimm($conf, $vmid, $memory, $sockets, sub {
3023             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3024             push @$cmd, "-object" , "memory-backend-ram,id=mem-$name,size=${dimm_size}M";
3025             push @$cmd, "-device", "pc-dimm,id=$name,memdev=mem-$name,node=$numanode";
3026
3027             #if dimm_memory is not aligned to dimm map
3028             if($current_size > $memory) {
3029                  $conf->{memory} = $current_size;
3030                  update_config_nolock($vmid, $conf, 1);
3031             }
3032         });
3033     }
3034
3035     push @$cmd, '-S' if $conf->{freeze};
3036
3037     # set keyboard layout
3038     my $kb = $conf->{keyboard} || $defaults->{keyboard};
3039     push @$cmd, '-k', $kb if $kb;
3040
3041     # enable sound
3042     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3043     #push @$cmd, '-soundhw', 'es1370';
3044     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3045
3046     if($conf->{agent}) {
3047         my $qgasocket = qmp_socket($vmid, 1);
3048         my $pciaddr = print_pci_addr("qga0", $bridges);
3049         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3050         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3051         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3052     }
3053
3054     my $spice_port;
3055
3056     if ($qxlnum) {
3057         if ($qxlnum > 1) {
3058             if ($conf->{ostype} && $conf->{ostype} =~ m/^w/){
3059                 for(my $i = 1; $i < $qxlnum; $i++){
3060                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3061                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3062                 }
3063             } else {
3064                 # assume other OS works like Linux
3065                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3066                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3067             }
3068         }
3069
3070         my $pciaddr = print_pci_addr("spice", $bridges);
3071
3072         my $nodename = PVE::INotify::nodename();
3073         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3074         $spice_port = PVE::Tools::next_spice_port($pfamily);
3075
3076         push @$devices, '-spice', "tls-port=${spice_port},addr=localhost,tls-ciphers=DES-CBC3-SHA,seamless-migration=on";
3077
3078         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3079         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3080         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3081     }
3082
3083     # enable balloon by default, unless explicitly disabled
3084     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3085         $pciaddr = print_pci_addr("balloon0", $bridges);
3086         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3087     }
3088
3089     if ($conf->{watchdog}) {
3090         my $wdopts = parse_watchdog($conf->{watchdog});
3091         $pciaddr = print_pci_addr("watchdog", $bridges);
3092         my $watchdog = $wdopts->{model} || 'i6300esb';
3093         push @$devices, '-device', "$watchdog$pciaddr";
3094         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3095     }
3096
3097     my $vollist = [];
3098     my $scsicontroller = {};
3099     my $ahcicontroller = {};
3100     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3101
3102     # Add iscsi initiator name if available
3103     if (my $initiator = get_initiator_name()) {
3104         push @$devices, '-iscsi', "initiator-name=$initiator";
3105     }
3106
3107     foreach_drive($conf, sub {
3108         my ($ds, $drive) = @_;
3109
3110         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3111             push @$vollist, $drive->{file};
3112         }
3113
3114         $use_virtio = 1 if $ds =~ m/^virtio/;
3115
3116         if (drive_is_cdrom ($drive)) {
3117             if ($bootindex_hash->{d}) {
3118                 $drive->{bootindex} = $bootindex_hash->{d};
3119                 $bootindex_hash->{d} += 1;
3120             }
3121         } else {
3122             if ($bootindex_hash->{c}) {
3123                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3124                 $bootindex_hash->{c} += 1;
3125             }
3126         }
3127
3128         if($drive->{interface} eq 'virtio'){
3129            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3130         }
3131
3132         if ($drive->{interface} eq 'scsi') {
3133
3134             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3135
3136             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3137             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3138
3139             my $iothread = '';
3140             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3141                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3142                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3143             }
3144
3145             my $queues = '';
3146             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3147                 $queues = ",num_queues=$drive->{queues}";
3148             }
3149
3150             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3151             $scsicontroller->{$controller}=1;
3152         }
3153
3154         if ($drive->{interface} eq 'sata') {
3155            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3156            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3157            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3158            $ahcicontroller->{$controller}=1;
3159         }
3160
3161         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3162         push @$devices, '-drive',$drive_cmd;
3163         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3164     });
3165
3166     for (my $i = 0; $i < $MAX_NETS; $i++) {
3167          next if !$conf->{"net$i"};
3168          my $d = parse_net($conf->{"net$i"});
3169          next if !$d;
3170
3171          $use_virtio = 1 if $d->{model} eq 'virtio';
3172
3173          if ($bootindex_hash->{n}) {
3174             $d->{bootindex} = $bootindex_hash->{n};
3175             $bootindex_hash->{n} += 1;
3176          }
3177
3178          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3179          push @$devices, '-netdev', $netdevfull;
3180
3181          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3182          push @$devices, '-device', $netdevicefull;
3183     }
3184
3185     if (!$q35) {
3186         # add pci bridges
3187         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3188            $bridges->{1} = 1;
3189            $bridges->{2} = 1;
3190         }
3191
3192         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3193
3194         while (my ($k, $v) = each %$bridges) {
3195             $pciaddr = print_pci_addr("pci.$k");
3196             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3197         }
3198     }
3199
3200     # add custom args
3201     if ($conf->{args}) {
3202         my $aa = PVE::Tools::split_args($conf->{args});
3203         push @$cmd, @$aa;
3204     }
3205
3206     push @$cmd, @$devices;
3207     push @$cmd, '-rtc', join(',', @$rtcFlags)
3208         if scalar(@$rtcFlags);
3209     push @$cmd, '-machine', join(',', @$machineFlags)
3210         if scalar(@$machineFlags);
3211     push @$cmd, '-global', join(',', @$globalFlags)
3212         if scalar(@$globalFlags);
3213
3214     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3215 }
3216
3217 sub vnc_socket {
3218     my ($vmid) = @_;
3219     return "${var_run_tmpdir}/$vmid.vnc";
3220 }
3221
3222 sub spice_port {
3223     my ($vmid) = @_;
3224
3225     my $res = vm_mon_cmd($vmid, 'query-spice');
3226
3227     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3228 }
3229
3230 sub qmp_socket {
3231     my ($vmid, $qga) = @_;
3232     my $sockettype = $qga ? 'qga' : 'qmp';
3233     return "${var_run_tmpdir}/$vmid.$sockettype";
3234 }
3235
3236 sub pidfile_name {
3237     my ($vmid) = @_;
3238     return "${var_run_tmpdir}/$vmid.pid";
3239 }
3240
3241 sub vm_devices_list {
3242     my ($vmid) = @_;
3243
3244     my $res = vm_mon_cmd($vmid, 'query-pci');
3245     my $devices = {};
3246     foreach my $pcibus (@$res) {
3247         foreach my $device (@{$pcibus->{devices}}) {
3248             next if !$device->{'qdev_id'};
3249             if ($device->{'pci_bridge'}) {
3250                 $devices->{$device->{'qdev_id'}} = 1;
3251                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3252                     next if !$bridge_device->{'qdev_id'};
3253                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3254                     $devices->{$device->{'qdev_id'}}++;
3255                 }
3256             } else {
3257                 $devices->{$device->{'qdev_id'}} = 1;
3258             }
3259         }
3260     }
3261
3262     my $resblock = vm_mon_cmd($vmid, 'query-block');
3263     foreach my $block (@$resblock) {
3264         if($block->{device} =~ m/^drive-(\S+)/){
3265                 $devices->{$1} = 1;
3266         }
3267     }
3268
3269     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3270     foreach my $mice (@$resmice) {
3271         if ($mice->{name} eq 'QEMU HID Tablet') {
3272             $devices->{tablet} = 1;
3273             last;
3274         }
3275     }
3276
3277     return $devices;
3278 }
3279
3280 sub vm_deviceplug {
3281     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3282
3283     my $q35 = machine_type_is_q35($conf);
3284
3285     my $devices_list = vm_devices_list($vmid);
3286     return 1 if defined($devices_list->{$deviceid});
3287
3288     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3289
3290     if ($deviceid eq 'tablet') {
3291
3292         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3293
3294     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3295
3296         qemu_iothread_add($vmid, $deviceid, $device);
3297
3298         qemu_driveadd($storecfg, $vmid, $device);
3299         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3300
3301         qemu_deviceadd($vmid, $devicefull);
3302         eval { qemu_deviceaddverify($vmid, $deviceid); };
3303         if (my $err = $@) {
3304             eval { qemu_drivedel($vmid, $deviceid); };
3305             warn $@ if $@;
3306             die $err;
3307         }
3308
3309     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3310
3311
3312         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3313         my $pciaddr = print_pci_addr($deviceid);
3314         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3315
3316         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3317
3318         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3319             qemu_iothread_add($vmid, $deviceid, $device);
3320             $devicefull .= ",iothread=iothread-$deviceid";
3321         }
3322
3323         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3324             $devicefull .= ",num_queues=$device->{queues}";
3325         }
3326
3327         qemu_deviceadd($vmid, $devicefull);
3328         qemu_deviceaddverify($vmid, $deviceid);
3329
3330     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3331
3332         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3333         qemu_driveadd($storecfg, $vmid, $device);
3334
3335         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3336         eval { qemu_deviceadd($vmid, $devicefull); };
3337         if (my $err = $@) {
3338             eval { qemu_drivedel($vmid, $deviceid); };
3339             warn $@ if $@;
3340             die $err;
3341         }
3342
3343     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3344
3345         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3346
3347         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
3348         my $use_old_bios_files = undef;
3349         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3350
3351         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3352         qemu_deviceadd($vmid, $netdevicefull);
3353         eval { qemu_deviceaddverify($vmid, $deviceid); };
3354         if (my $err = $@) {
3355             eval { qemu_netdevdel($vmid, $deviceid); };
3356             warn $@ if $@;
3357             die $err;
3358         }
3359
3360     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3361
3362         my $bridgeid = $2;
3363         my $pciaddr = print_pci_addr($deviceid);
3364         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3365
3366         qemu_deviceadd($vmid, $devicefull);
3367         qemu_deviceaddverify($vmid, $deviceid);
3368
3369     } else {
3370         die "can't hotplug device '$deviceid'\n";
3371     }
3372
3373     return 1;
3374 }
3375
3376 # fixme: this should raise exceptions on error!
3377 sub vm_deviceunplug {
3378     my ($vmid, $conf, $deviceid) = @_;
3379
3380     my $devices_list = vm_devices_list($vmid);
3381     return 1 if !defined($devices_list->{$deviceid});
3382
3383     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3384
3385     if ($deviceid eq 'tablet') {
3386
3387         qemu_devicedel($vmid, $deviceid);
3388
3389     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3390
3391         qemu_devicedel($vmid, $deviceid);
3392         qemu_devicedelverify($vmid, $deviceid);
3393         qemu_drivedel($vmid, $deviceid);
3394         qemu_iothread_del($conf, $vmid, $deviceid);
3395
3396     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3397
3398         qemu_devicedel($vmid, $deviceid);
3399         qemu_devicedelverify($vmid, $deviceid);
3400         qemu_iothread_del($conf, $vmid, $deviceid);
3401
3402     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3403
3404         #qemu 2.3 segfault on drive_del with virtioscsi + iothread
3405         my $device = parse_drive($deviceid, $conf->{$deviceid});
3406         die "virtioscsi with iothread is not hot-unplugglable currently" if $device->{iothread};
3407
3408         qemu_devicedel($vmid, $deviceid);
3409         qemu_drivedel($vmid, $deviceid);
3410         qemu_deletescsihw($conf, $vmid, $deviceid);
3411
3412     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3413
3414         qemu_devicedel($vmid, $deviceid);
3415         qemu_devicedelverify($vmid, $deviceid);
3416         qemu_netdevdel($vmid, $deviceid);
3417
3418     } else {
3419         die "can't unplug device '$deviceid'\n";
3420     }
3421
3422     return 1;
3423 }
3424
3425 sub qemu_deviceadd {
3426     my ($vmid, $devicefull) = @_;
3427
3428     $devicefull = "driver=".$devicefull;
3429     my %options =  split(/[=,]/, $devicefull);
3430
3431     vm_mon_cmd($vmid, "device_add" , %options);
3432 }
3433
3434 sub qemu_devicedel {
3435     my ($vmid, $deviceid) = @_;
3436
3437     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3438 }
3439
3440 sub qemu_iothread_add {
3441     my($vmid, $deviceid, $device) = @_;
3442
3443     if ($device->{iothread}) {
3444         my $iothreads = vm_iothreads_list($vmid);
3445         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3446     }
3447 }
3448
3449 sub qemu_iothread_del {
3450     my($conf, $vmid, $deviceid) = @_;
3451
3452     my $device = parse_drive($deviceid, $conf->{$deviceid});
3453     if ($device->{iothread}) {
3454         my $iothreads = vm_iothreads_list($vmid);
3455         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3456     }
3457 }
3458
3459 sub qemu_objectadd {
3460     my($vmid, $objectid, $qomtype) = @_;
3461
3462     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3463
3464     return 1;
3465 }
3466
3467 sub qemu_objectdel {
3468     my($vmid, $objectid) = @_;
3469
3470     vm_mon_cmd($vmid, "object-del", id => $objectid);
3471
3472     return 1;
3473 }
3474
3475 sub qemu_driveadd {
3476     my ($storecfg, $vmid, $device) = @_;
3477
3478     my $drive = print_drive_full($storecfg, $vmid, $device);
3479     $drive =~ s/\\/\\\\/g;
3480     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3481
3482     # If the command succeeds qemu prints: "OK"
3483     return 1 if $ret =~ m/OK/s;
3484
3485     die "adding drive failed: $ret\n";
3486 }
3487
3488 sub qemu_drivedel {
3489     my($vmid, $deviceid) = @_;
3490
3491     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3492     $ret =~ s/^\s+//;
3493
3494     return 1 if $ret eq "";
3495
3496     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3497     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3498
3499     die "deleting drive $deviceid failed : $ret\n";
3500 }
3501
3502 sub qemu_deviceaddverify {
3503     my ($vmid, $deviceid) = @_;
3504
3505     for (my $i = 0; $i <= 5; $i++) {
3506          my $devices_list = vm_devices_list($vmid);
3507          return 1 if defined($devices_list->{$deviceid});
3508          sleep 1;
3509     }
3510
3511     die "error on hotplug device '$deviceid'\n";
3512 }
3513
3514
3515 sub qemu_devicedelverify {
3516     my ($vmid, $deviceid) = @_;
3517
3518     # need to verify that the device is correctly removed as device_del
3519     # is async and empty return is not reliable
3520
3521     for (my $i = 0; $i <= 5; $i++) {
3522          my $devices_list = vm_devices_list($vmid);
3523          return 1 if !defined($devices_list->{$deviceid});
3524          sleep 1;
3525     }
3526
3527     die "error on hot-unplugging device '$deviceid'\n";
3528 }
3529
3530 sub qemu_findorcreatescsihw {
3531     my ($storecfg, $conf, $vmid, $device) = @_;
3532
3533     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3534
3535     my $scsihwid="$controller_prefix$controller";
3536     my $devices_list = vm_devices_list($vmid);
3537
3538     if(!defined($devices_list->{$scsihwid})) {
3539         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3540     }
3541
3542     return 1;
3543 }
3544
3545 sub qemu_deletescsihw {
3546     my ($conf, $vmid, $opt) = @_;
3547
3548     my $device = parse_drive($opt, $conf->{$opt});
3549
3550     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3551         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3552         return 1;
3553     }
3554
3555     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3556
3557     my $devices_list = vm_devices_list($vmid);
3558     foreach my $opt (keys %{$devices_list}) {
3559         if (PVE::QemuServer::valid_drivename($opt)) {
3560             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3561             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3562                 return 1;
3563             }
3564         }
3565     }
3566
3567     my $scsihwid="scsihw$controller";
3568
3569     vm_deviceunplug($vmid, $conf, $scsihwid);
3570
3571     return 1;
3572 }
3573
3574 sub qemu_add_pci_bridge {
3575     my ($storecfg, $conf, $vmid, $device) = @_;
3576
3577     my $bridges = {};
3578
3579     my $bridgeid;
3580
3581     print_pci_addr($device, $bridges);
3582
3583     while (my ($k, $v) = each %$bridges) {
3584         $bridgeid = $k;
3585     }
3586     return 1 if !defined($bridgeid) || $bridgeid < 1;
3587
3588     my $bridge = "pci.$bridgeid";
3589     my $devices_list = vm_devices_list($vmid);
3590
3591     if (!defined($devices_list->{$bridge})) {
3592         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3593     }
3594
3595     return 1;
3596 }
3597
3598 sub qemu_set_link_status {
3599     my ($vmid, $device, $up) = @_;
3600
3601     vm_mon_cmd($vmid, "set_link", name => $device,
3602                up => $up ? JSON::true : JSON::false);
3603 }
3604
3605 sub qemu_netdevadd {
3606     my ($vmid, $conf, $device, $deviceid) = @_;
3607
3608     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
3609     my %options =  split(/[=,]/, $netdev);
3610
3611     vm_mon_cmd($vmid, "netdev_add",  %options);
3612     return 1;
3613 }
3614
3615 sub qemu_netdevdel {
3616     my ($vmid, $deviceid) = @_;
3617
3618     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3619 }
3620
3621 sub qemu_cpu_hotplug {
3622     my ($vmid, $conf, $vcpus) = @_;
3623
3624     my $sockets = 1;
3625     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3626     $sockets = $conf->{sockets} if  $conf->{sockets};
3627     my $cores = $conf->{cores} || 1;
3628     my $maxcpus = $sockets * $cores;
3629
3630     $vcpus = $maxcpus if !$vcpus;
3631
3632     die "you can't add more vcpus than maxcpus\n"
3633         if $vcpus > $maxcpus;
3634
3635     my $currentvcpus = $conf->{vcpus} || $maxcpus;
3636     die "online cpu unplug is not yet possible\n"
3637         if $vcpus < $currentvcpus;
3638
3639     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3640     die "vcpus in running vm is different than configuration\n"
3641         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
3642
3643     for (my $i = $currentvcpus; $i < $vcpus; $i++) {
3644         vm_mon_cmd($vmid, "cpu-add", id => int($i));
3645     }
3646 }
3647
3648 sub qemu_memory_hotplug {
3649     my ($vmid, $conf, $defaults, $opt, $value) = @_;
3650
3651     return $value if !check_running($vmid);
3652
3653     my $memory = $conf->{memory} || $defaults->{memory};
3654     $value = $defaults->{memory} if !$value;
3655     return $value if $value == $memory;
3656
3657     my $static_memory = $STATICMEM;
3658     my $dimm_memory = $memory - $static_memory;
3659
3660     die "memory can't be lower than $static_memory MB" if $value < $static_memory;
3661     die "you cannot add more memory than $MAX_MEM MB!\n" if $memory > $MAX_MEM;
3662
3663
3664     my $sockets = 1;
3665     $sockets = $conf->{sockets} if $conf->{sockets};
3666
3667     if($value > $memory) {
3668
3669         foreach_dimm($conf, $vmid, $value, $sockets, sub {
3670             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3671
3672                 return if $current_size <= $conf->{memory};
3673
3674                 eval { vm_mon_cmd($vmid, "object-add", 'qom-type' => "memory-backend-ram", id => "mem-$name", props => { size => int($dimm_size*1024*1024) } ) };
3675                 if (my $err = $@) {
3676                     eval { qemu_objectdel($vmid, "mem-$name"); };
3677                     die $err;
3678                 }
3679
3680                 eval { vm_mon_cmd($vmid, "device_add", driver => "pc-dimm", id => "$name", memdev => "mem-$name", node => $numanode) };
3681                 if (my $err = $@) {
3682                     eval { qemu_objectdel($vmid, "mem-$name"); };
3683                     die $err;
3684                 }
3685                 #update conf after each succesful module hotplug
3686                 $conf->{memory} = $current_size;
3687                 update_config_nolock($vmid, $conf, 1);
3688         });
3689
3690     } else {
3691
3692         foreach_reverse_dimm($conf, $vmid, $value, $sockets, sub {
3693             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3694
3695                 return if $current_size >= $conf->{memory};
3696                 print "try to unplug memory dimm $name\n";
3697
3698                 my $retry = 0;
3699                 while (1) {
3700                     eval { qemu_devicedel($vmid, $name) };
3701                     sleep 3;
3702                     my $dimm_list = qemu_dimm_list($vmid);
3703                     last if !$dimm_list->{$name};
3704                     raise_param_exc({ $name => "error unplug memory module" }) if $retry > 5;
3705                     $retry++;
3706                 }
3707
3708                 #update conf after each succesful module unplug
3709                 $conf->{memory} = $current_size;
3710
3711                 eval { qemu_objectdel($vmid, "mem-$name"); };
3712                 update_config_nolock($vmid, $conf, 1);
3713         });
3714     }
3715 }
3716
3717 sub qemu_dimm_list {
3718     my ($vmid) = @_;
3719
3720     my $dimmarray = vm_mon_cmd_nocheck($vmid, "query-memory-devices");
3721     my $dimms = {};
3722
3723     foreach my $dimm (@$dimmarray) {
3724
3725         $dimms->{$dimm->{data}->{id}}->{id} = $dimm->{data}->{id};
3726         $dimms->{$dimm->{data}->{id}}->{node} = $dimm->{data}->{node};
3727         $dimms->{$dimm->{data}->{id}}->{addr} = $dimm->{data}->{addr};
3728         $dimms->{$dimm->{data}->{id}}->{size} = $dimm->{data}->{size};
3729         $dimms->{$dimm->{data}->{id}}->{slot} = $dimm->{data}->{slot};
3730     }
3731     return $dimms;
3732 }
3733
3734 sub qemu_block_set_io_throttle {
3735     my ($vmid, $deviceid, $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr) = @_;
3736
3737     return if !check_running($vmid) ;
3738
3739     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid, bps => int($bps), bps_rd => int($bps_rd), bps_wr => int($bps_wr), iops => int($iops), iops_rd => int($iops_rd), iops_wr => int($iops_wr));
3740
3741 }
3742
3743 # old code, only used to shutdown old VM after update
3744 sub __read_avail {
3745     my ($fh, $timeout) = @_;
3746
3747     my $sel = new IO::Select;
3748     $sel->add($fh);
3749
3750     my $res = '';
3751     my $buf;
3752
3753     my @ready;
3754     while (scalar (@ready = $sel->can_read($timeout))) {
3755         my $count;
3756         if ($count = $fh->sysread($buf, 8192)) {
3757             if ($buf =~ /^(.*)\(qemu\) $/s) {
3758                 $res .= $1;
3759                 last;
3760             } else {
3761                 $res .= $buf;
3762             }
3763         } else {
3764             if (!defined($count)) {
3765                 die "$!\n";
3766             }
3767             last;
3768         }
3769     }
3770
3771     die "monitor read timeout\n" if !scalar(@ready);
3772
3773     return $res;
3774 }
3775
3776 # old code, only used to shutdown old VM after update
3777 sub vm_monitor_command {
3778     my ($vmid, $cmdstr, $nocheck) = @_;
3779
3780     my $res;
3781
3782     eval {
3783         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
3784
3785         my $sname = "${var_run_tmpdir}/$vmid.mon";
3786
3787         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
3788             die "unable to connect to VM $vmid socket - $!\n";
3789
3790         my $timeout = 3;
3791
3792         # hack: migrate sometime blocks the monitor (when migrate_downtime
3793         # is set)
3794         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3795             $timeout = 60*60; # 1 hour
3796         }
3797
3798         # read banner;
3799         my $data = __read_avail($sock, $timeout);
3800
3801         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
3802             die "got unexpected qemu monitor banner\n";
3803         }
3804
3805         my $sel = new IO::Select;
3806         $sel->add($sock);
3807
3808         if (!scalar(my @ready = $sel->can_write($timeout))) {
3809             die "monitor write error - timeout";
3810         }
3811
3812         my $fullcmd = "$cmdstr\r";
3813
3814         # syslog('info', "VM $vmid monitor command: $cmdstr");
3815
3816         my $b;
3817         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
3818             die "monitor write error - $!";
3819         }
3820
3821         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
3822
3823         $timeout = 20;
3824
3825         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3826             $timeout = 60*60; # 1 hour
3827         } elsif ($cmdstr =~ m/^(eject|change)/) {
3828             $timeout = 60; # note: cdrom mount command is slow
3829         }
3830         if ($res = __read_avail($sock, $timeout)) {
3831
3832             my @lines = split("\r?\n", $res);
3833
3834             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
3835
3836             $res = join("\n", @lines);
3837             $res .= "\n";
3838         }
3839     };
3840
3841     my $err = $@;
3842
3843     if ($err) {
3844         syslog("err", "VM $vmid monitor command failed - $err");
3845         die $err;
3846     }
3847
3848     return $res;
3849 }
3850
3851 sub qemu_block_resize {
3852     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
3853
3854     my $running = check_running($vmid);
3855
3856     return if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
3857
3858     return if !$running;
3859
3860     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
3861
3862 }
3863
3864 sub qemu_volume_snapshot {
3865     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3866
3867     my $running = check_running($vmid);
3868
3869     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
3870         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
3871     } else {
3872         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
3873     }
3874 }
3875
3876 sub qemu_volume_snapshot_delete {
3877     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3878
3879     my $running = check_running($vmid);
3880
3881     return if !PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
3882
3883     return if !$running;
3884
3885     vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
3886 }
3887
3888 sub set_migration_caps {
3889     my ($vmid) = @_;
3890
3891     my $cap_ref = [];
3892
3893     my $enabled_cap = {
3894         "auto-converge" => 1,
3895         "xbzrle" => 1,
3896         "x-rdma-pin-all" => 0,
3897         "zero-blocks" => 0,
3898         "compress" => 0
3899     };
3900
3901     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
3902
3903     for my $supported_capability (@$supported_capabilities) {
3904         push @$cap_ref, {
3905             capability => $supported_capability->{capability},
3906             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
3907         };
3908     }
3909
3910     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
3911 }
3912
3913 my $fast_plug_option = {
3914     'lock' => 1,
3915     'name' => 1,
3916     'onboot' => 1,
3917     'shares' => 1,
3918     'startup' => 1,
3919     'description' => 1,
3920 };
3921
3922 # hotplug changes in [PENDING]
3923 # $selection hash can be used to only apply specified options, for
3924 # example: { cores => 1 } (only apply changed 'cores')
3925 # $errors ref is used to return error messages
3926 sub vmconfig_hotplug_pending {
3927     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
3928
3929     my $defaults = load_defaults();
3930
3931     # commit values which do not have any impact on running VM first
3932     # Note: those option cannot raise errors, we we do not care about
3933     # $selection and always apply them.
3934
3935     my $add_error = sub {
3936         my ($opt, $msg) = @_;
3937         $errors->{$opt} = "hotplug problem - $msg";
3938     };
3939
3940     my $changes = 0;
3941     foreach my $opt (keys %{$conf->{pending}}) { # add/change
3942         if ($fast_plug_option->{$opt}) {
3943             $conf->{$opt} = $conf->{pending}->{$opt};
3944             delete $conf->{pending}->{$opt};
3945             $changes = 1;
3946         }
3947     }
3948
3949     if ($changes) {
3950         update_config_nolock($vmid, $conf, 1);
3951         $conf = load_config($vmid); # update/reload
3952     }
3953
3954     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3955
3956     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
3957     while (my ($opt, $force) = each %$pending_delete_hash) {
3958         next if $selection && !$selection->{$opt};
3959         eval {
3960             if ($opt eq 'hotplug') {
3961                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
3962             } elsif ($opt eq 'tablet') {
3963                 die "skip\n" if !$hotplug_features->{usb};
3964                 if ($defaults->{tablet}) {
3965                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
3966                 } else {
3967                     vm_deviceunplug($vmid, $conf, $opt);
3968                 }
3969             } elsif ($opt eq 'vcpus') {
3970                 die "skip\n" if !$hotplug_features->{cpu};
3971                 qemu_cpu_hotplug($vmid, $conf, undef);
3972             } elsif ($opt eq 'balloon') {
3973                 # enable balloon device is not hotpluggable
3974                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
3975             } elsif ($fast_plug_option->{$opt}) {
3976                 # do nothing
3977             } elsif ($opt =~ m/^net(\d+)$/) {
3978                 die "skip\n" if !$hotplug_features->{network};
3979                 vm_deviceunplug($vmid, $conf, $opt);
3980             } elsif (valid_drivename($opt)) {
3981                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
3982                 vm_deviceunplug($vmid, $conf, $opt);
3983                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
3984             } elsif ($opt =~ m/^memory$/) {
3985                 die "skip\n" if !$hotplug_features->{memory};
3986                 qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
3987             } elsif ($opt eq 'cpuunits') {
3988                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
3989             } elsif ($opt eq 'cpulimit') {
3990                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
3991             } else {
3992                 die "skip\n";
3993             }
3994         };
3995         if (my $err = $@) {
3996             &$add_error($opt, $err) if $err ne "skip\n";
3997         } else {
3998             # save new config if hotplug was successful
3999             delete $conf->{$opt};
4000             vmconfig_undelete_pending_option($conf, $opt);
4001             update_config_nolock($vmid, $conf, 1);
4002             $conf = load_config($vmid); # update/reload
4003         }
4004     }
4005
4006     foreach my $opt (keys %{$conf->{pending}}) {
4007         next if $selection && !$selection->{$opt};
4008         my $value = $conf->{pending}->{$opt};
4009         eval {
4010             if ($opt eq 'hotplug') {
4011                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4012             } elsif ($opt eq 'tablet') {
4013                 die "skip\n" if !$hotplug_features->{usb};
4014                 if ($value == 1) {
4015                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4016                 } elsif ($value == 0) {
4017                     vm_deviceunplug($vmid, $conf, $opt);
4018                 }
4019             } elsif ($opt eq 'vcpus') {
4020                 die "skip\n" if !$hotplug_features->{cpu};
4021                 qemu_cpu_hotplug($vmid, $conf, $value);
4022             } elsif ($opt eq 'balloon') {
4023                 # enable/disable balloning device is not hotpluggable
4024                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4025                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4026                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4027
4028                 # allow manual ballooning if shares is set to zero
4029                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4030                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4031                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4032                 }
4033             } elsif ($opt =~ m/^net(\d+)$/) {
4034                 # some changes can be done without hotplug
4035                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4036                                     $vmid, $opt, $value);
4037             } elsif (valid_drivename($opt)) {
4038                 # some changes can be done without hotplug
4039                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4040                                      $vmid, $opt, $value, 1);
4041             } elsif ($opt =~ m/^memory$/) { #dimms
4042                 die "skip\n" if !$hotplug_features->{memory};
4043                 $value = qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4044             } elsif ($opt eq 'cpuunits') {
4045                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4046             } elsif ($opt eq 'cpulimit') {
4047                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4048                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4049             } else {
4050                 die "skip\n";  # skip non-hot-pluggable options
4051             }
4052         };
4053         if (my $err = $@) {
4054             &$add_error($opt, $err) if $err ne "skip\n";
4055         } else {
4056             # save new config if hotplug was successful
4057             $conf->{$opt} = $value;
4058             delete $conf->{pending}->{$opt};
4059             update_config_nolock($vmid, $conf, 1);
4060             $conf = load_config($vmid); # update/reload
4061         }
4062     }
4063 }
4064
4065 sub try_deallocate_drive {
4066     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4067
4068     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4069         my $volid = $drive->{file};
4070         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4071             my $sid = PVE::Storage::parse_volume_id($volid);
4072             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4073
4074             # check if the disk is really unused
4075             my $used_paths = PVE::QemuServer::get_used_paths($vmid, $storecfg, $conf, 1, $key);
4076             my $path = PVE::Storage::path($storecfg, $volid);
4077             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4078                    if $used_paths->{$path};
4079             PVE::Storage::vdisk_free($storecfg, $volid);
4080             return 1;
4081         } else {
4082             # If vm is not owner of this disk remove from config
4083             return 1;
4084         }
4085     }
4086
4087     return undef;
4088 }
4089
4090 sub vmconfig_delete_or_detach_drive {
4091     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4092
4093     my $drive = parse_drive($opt, $conf->{$opt});
4094
4095     my $rpcenv = PVE::RPCEnvironment::get();
4096     my $authuser = $rpcenv->get_user();
4097
4098     if ($force) {
4099         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4100         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4101     } else {
4102         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4103     }
4104 }
4105
4106 sub vmconfig_apply_pending {
4107     my ($vmid, $conf, $storecfg) = @_;
4108
4109     # cold plug
4110
4111     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4112     while (my ($opt, $force) = each %$pending_delete_hash) {
4113         die "internal error" if $opt =~ m/^unused/;
4114         $conf = load_config($vmid); # update/reload
4115         if (!defined($conf->{$opt})) {
4116             vmconfig_undelete_pending_option($conf, $opt);
4117             update_config_nolock($vmid, $conf, 1);
4118         } elsif (valid_drivename($opt)) {
4119             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4120             vmconfig_undelete_pending_option($conf, $opt);
4121             delete $conf->{$opt};
4122             update_config_nolock($vmid, $conf, 1);
4123         } else {
4124             vmconfig_undelete_pending_option($conf, $opt);
4125             delete $conf->{$opt};
4126             update_config_nolock($vmid, $conf, 1);
4127         }
4128     }
4129
4130     $conf = load_config($vmid); # update/reload
4131
4132     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4133         $conf = load_config($vmid); # update/reload
4134
4135         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4136             # skip if nothing changed
4137         } elsif (valid_drivename($opt)) {
4138             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4139                 if defined($conf->{$opt});
4140             $conf->{$opt} = $conf->{pending}->{$opt};
4141         } else {
4142             $conf->{$opt} = $conf->{pending}->{$opt};
4143         }
4144
4145         delete $conf->{pending}->{$opt};
4146         update_config_nolock($vmid, $conf, 1);
4147     }
4148 }
4149
4150 my $safe_num_ne = sub {
4151     my ($a, $b) = @_;
4152
4153     return 0 if !defined($a) && !defined($b);
4154     return 1 if !defined($a);
4155     return 1 if !defined($b);
4156
4157     return $a != $b;
4158 };
4159
4160 my $safe_string_ne = sub {
4161     my ($a, $b) = @_;
4162
4163     return 0 if !defined($a) && !defined($b);
4164     return 1 if !defined($a);
4165     return 1 if !defined($b);
4166
4167     return $a ne $b;
4168 };
4169
4170 sub vmconfig_update_net {
4171     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4172
4173     my $newnet = parse_net($value);
4174
4175     if ($conf->{$opt}) {
4176         my $oldnet = parse_net($conf->{$opt});
4177
4178         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4179             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4180             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4181             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4182
4183             # for non online change, we try to hot-unplug
4184             die "skip\n" if !$hotplug;
4185             vm_deviceunplug($vmid, $conf, $opt);
4186         } else {
4187
4188             die "internal error" if $opt !~ m/net(\d+)/;
4189             my $iface = "tap${vmid}i$1";
4190
4191             if (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4192                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4193             }
4194
4195             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4196                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4197                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4198                 PVE::Network::tap_unplug($iface);
4199                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall});
4200             }
4201
4202             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4203                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4204             }
4205
4206             return 1;
4207         }
4208     }
4209
4210     if ($hotplug) {
4211         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4212     } else {
4213         die "skip\n";
4214     }
4215 }
4216
4217 sub vmconfig_update_disk {
4218     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4219
4220     # fixme: do we need force?
4221
4222     my $drive = parse_drive($opt, $value);
4223
4224     if ($conf->{$opt}) {
4225
4226         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4227
4228             my $media = $drive->{media} || 'disk';
4229             my $oldmedia = $old_drive->{media} || 'disk';
4230             die "unable to change media type\n" if $media ne $oldmedia;
4231
4232             if (!drive_is_cdrom($old_drive)) {
4233
4234                 if ($drive->{file} ne $old_drive->{file}) {
4235
4236                     die "skip\n" if !$hotplug;
4237
4238                     # unplug and register as unused
4239                     vm_deviceunplug($vmid, $conf, $opt);
4240                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4241
4242                 } else {
4243                     # update existing disk
4244
4245                     # skip non hotpluggable value
4246                     if (&$safe_num_ne($drive->{discard}, $old_drive->{discard}) ||
4247                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4248                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4249                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4250                         die "skip\n";
4251                     }
4252
4253                     # apply throttle
4254                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4255                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4256                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4257                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4258                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4259                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4260                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4261                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4262                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4263                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4264                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4265                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max})) {
4266
4267                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4268                                                    ($drive->{mbps} || 0)*1024*1024,
4269                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4270                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4271                                                    $drive->{iops} || 0,
4272                                                    $drive->{iops_rd} || 0,
4273                                                    $drive->{iops_wr} || 0,
4274                                                    ($drive->{mbps_max} || 0)*1024*1024,
4275                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4276                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4277                                                    $drive->{iops_max} || 0,
4278                                                    $drive->{iops_rd_max} || 0,
4279                                                    $drive->{iops_wr_max} || 0);
4280
4281                     }
4282
4283                     return 1;
4284                 }
4285
4286             } else { # cdrom
4287
4288                 if ($drive->{file} eq 'none') {
4289                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4290                 } else {
4291                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4292                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4293                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4294                 }
4295
4296                 return 1;
4297             }
4298         }
4299     }
4300
4301     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4302     # hotplug new disks
4303     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4304 }
4305
4306 sub vm_start {
4307     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4308         $forcemachine, $spice_ticket) = @_;
4309
4310     lock_config($vmid, sub {
4311         my $conf = load_config($vmid, $migratedfrom);
4312
4313         die "you can't start a vm if it's a template\n" if is_template($conf);
4314
4315         check_lock($conf) if !$skiplock;
4316
4317         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4318
4319         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4320             vmconfig_apply_pending($vmid, $conf, $storecfg);
4321             $conf = load_config($vmid); # update/reload
4322         }
4323
4324         my $defaults = load_defaults();
4325
4326         # set environment variable useful inside network script
4327         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4328
4329         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4330
4331         my $migrate_port = 0;
4332         my $migrate_uri;
4333         if ($statefile) {
4334             if ($statefile eq 'tcp') {
4335                 my $localip = "localhost";
4336                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4337                 my $nodename = PVE::INotify::nodename();
4338                 if ($datacenterconf->{migration_unsecure}) {
4339                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4340                         $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4341                 }
4342                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4343                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4344                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4345                 push @$cmd, '-incoming', $migrate_uri;
4346                 push @$cmd, '-S';
4347             } else {
4348                 push @$cmd, '-loadstate', $statefile;
4349             }
4350         } elsif ($paused) {
4351             push @$cmd, '-S';
4352         }
4353
4354         # host pci devices
4355         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4356           my $d = parse_hostpci($conf->{"hostpci$i"});
4357           next if !$d;
4358           my $pcidevices = $d->{pciid};
4359           foreach my $pcidevice (@$pcidevices) {
4360                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4361
4362                 my $info = pci_device_info("0000:$pciid");
4363                 die "IOMMU not present\n" if !check_iommu_support();
4364                 die "no pci device info for device '$pciid'\n" if !$info;
4365                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4366                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4367           }
4368         }
4369
4370         PVE::Storage::activate_volumes($storecfg, $vollist);
4371
4372         eval  { run_command($cmd, timeout => $statefile ? undef : 30,
4373                     umask => 0077); };
4374         my $err = $@;
4375         die "start failed: $err" if $err;
4376
4377         print "migration listens on $migrate_uri\n" if $migrate_uri;
4378
4379         if ($statefile && $statefile ne 'tcp')  {
4380             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4381             warn $@ if $@;
4382         }
4383
4384         if ($migratedfrom) {
4385
4386             eval {
4387                 set_migration_caps($vmid);
4388             };
4389             warn $@ if $@;
4390
4391             if ($spice_port) {
4392                 print "spice listens on port $spice_port\n";
4393                 if ($spice_ticket) {
4394                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4395                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4396                 }
4397             }
4398
4399         } else {
4400
4401             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4402                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4403                     if $conf->{balloon};
4404             }
4405
4406             foreach my $opt (keys %$conf) {
4407                 next if $opt !~  m/^net\d+$/;
4408                 my $nicconf = parse_net($conf->{$opt});
4409                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4410             }
4411         }
4412
4413         vm_mon_cmd_nocheck($vmid, 'qom-set',
4414                     path => "machine/peripheral/balloon0",
4415                     property => "guest-stats-polling-interval",
4416                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4417
4418     });
4419 }
4420
4421 sub vm_mon_cmd {
4422     my ($vmid, $execute, %params) = @_;
4423
4424     my $cmd = { execute => $execute, arguments => \%params };
4425     vm_qmp_command($vmid, $cmd);
4426 }
4427
4428 sub vm_mon_cmd_nocheck {
4429     my ($vmid, $execute, %params) = @_;
4430
4431     my $cmd = { execute => $execute, arguments => \%params };
4432     vm_qmp_command($vmid, $cmd, 1);
4433 }
4434
4435 sub vm_qmp_command {
4436     my ($vmid, $cmd, $nocheck) = @_;
4437
4438     my $res;
4439
4440     my $timeout;
4441     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4442         $timeout = $cmd->{arguments}->{timeout};
4443         delete $cmd->{arguments}->{timeout};
4444     }
4445
4446     eval {
4447         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4448         my $sname = qmp_socket($vmid);
4449         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4450             my $qmpclient = PVE::QMPClient->new();
4451
4452             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4453         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
4454             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
4455                 if scalar(%{$cmd->{arguments}});
4456             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
4457         } else {
4458             die "unable to open monitor socket\n";
4459         }
4460     };
4461     if (my $err = $@) {
4462         syslog("err", "VM $vmid qmp command failed - $err");
4463         die $err;
4464     }
4465
4466     return $res;
4467 }
4468
4469 sub vm_human_monitor_command {
4470     my ($vmid, $cmdline) = @_;
4471
4472     my $res;
4473
4474     my $cmd = {
4475         execute => 'human-monitor-command',
4476         arguments => { 'command-line' => $cmdline},
4477     };
4478
4479     return vm_qmp_command($vmid, $cmd);
4480 }
4481
4482 sub vm_commandline {
4483     my ($storecfg, $vmid) = @_;
4484
4485     my $conf = load_config($vmid);
4486
4487     my $defaults = load_defaults();
4488
4489     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
4490
4491     return join(' ', @$cmd);
4492 }
4493
4494 sub vm_reset {
4495     my ($vmid, $skiplock) = @_;
4496
4497     lock_config($vmid, sub {
4498
4499         my $conf = load_config($vmid);
4500
4501         check_lock($conf) if !$skiplock;
4502
4503         vm_mon_cmd($vmid, "system_reset");
4504     });
4505 }
4506
4507 sub get_vm_volumes {
4508     my ($conf) = @_;
4509
4510     my $vollist = [];
4511     foreach_volid($conf, sub {
4512         my ($volid, $is_cdrom) = @_;
4513
4514         return if $volid =~ m|^/|;
4515
4516         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4517         return if !$sid;
4518
4519         push @$vollist, $volid;
4520     });
4521
4522     return $vollist;
4523 }
4524
4525 sub vm_stop_cleanup {
4526     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
4527
4528     eval {
4529
4530         if (!$keepActive) {
4531             my $vollist = get_vm_volumes($conf);
4532             PVE::Storage::deactivate_volumes($storecfg, $vollist);
4533         }
4534
4535         foreach my $ext (qw(mon qmp pid vnc qga)) {
4536             unlink "/var/run/qemu-server/${vmid}.$ext";
4537         }
4538
4539         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
4540     };
4541     warn $@ if $@; # avoid errors - just warn
4542 }
4543
4544 # Note: use $nockeck to skip tests if VM configuration file exists.
4545 # We need that when migration VMs to other nodes (files already moved)
4546 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
4547 sub vm_stop {
4548     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
4549
4550     $force = 1 if !defined($force) && !$shutdown;
4551
4552     if ($migratedfrom){
4553         my $pid = check_running($vmid, $nocheck, $migratedfrom);
4554         kill 15, $pid if $pid;
4555         my $conf = load_config($vmid, $migratedfrom);
4556         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
4557         return;
4558     }
4559
4560     lock_config($vmid, sub {
4561
4562         my $pid = check_running($vmid, $nocheck);
4563         return if !$pid;
4564
4565         my $conf;
4566         if (!$nocheck) {
4567             $conf = load_config($vmid);
4568             check_lock($conf) if !$skiplock;
4569             if (!defined($timeout) && $shutdown && $conf->{startup}) {
4570                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
4571                 $timeout = $opts->{down} if $opts->{down};
4572             }
4573         }
4574
4575         $timeout = 60 if !defined($timeout);
4576
4577         eval {
4578             if ($shutdown) {
4579                 if (defined($conf) && $conf->{agent}) {
4580                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
4581                 } else {
4582                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
4583                 }
4584             } else {
4585                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
4586             }
4587         };
4588         my $err = $@;
4589
4590         if (!$err) {
4591             my $count = 0;
4592             while (($count < $timeout) && check_running($vmid, $nocheck)) {
4593                 $count++;
4594                 sleep 1;
4595             }
4596
4597             if ($count >= $timeout) {
4598                 if ($force) {
4599                     warn "VM still running - terminating now with SIGTERM\n";
4600                     kill 15, $pid;
4601                 } else {
4602                     die "VM quit/powerdown failed - got timeout\n";
4603                 }
4604             } else {
4605                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4606                 return;
4607             }
4608         } else {
4609             if ($force) {
4610                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
4611                 kill 15, $pid;
4612             } else {
4613                 die "VM quit/powerdown failed\n";
4614             }
4615         }
4616
4617         # wait again
4618         $timeout = 10;
4619
4620         my $count = 0;
4621         while (($count < $timeout) && check_running($vmid, $nocheck)) {
4622             $count++;
4623             sleep 1;
4624         }
4625
4626         if ($count >= $timeout) {
4627             warn "VM still running - terminating now with SIGKILL\n";
4628             kill 9, $pid;
4629             sleep 1;
4630         }
4631
4632         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4633    });
4634 }
4635
4636 sub vm_suspend {
4637     my ($vmid, $skiplock) = @_;
4638
4639     lock_config($vmid, sub {
4640
4641         my $conf = load_config($vmid);
4642
4643         check_lock($conf) if !($skiplock || ($conf->{lock} && $conf->{lock} eq 'backup'));
4644
4645         vm_mon_cmd($vmid, "stop");
4646     });
4647 }
4648
4649 sub vm_resume {
4650     my ($vmid, $skiplock, $nocheck) = @_;
4651
4652     lock_config($vmid, sub {
4653
4654         if (!$nocheck) {
4655
4656             my $conf = load_config($vmid);
4657
4658             check_lock($conf) if !($skiplock || ($conf->{lock} && $conf->{lock} eq 'backup'));
4659
4660             vm_mon_cmd($vmid, "cont");
4661
4662         } else {
4663             vm_mon_cmd_nocheck($vmid, "cont");
4664         }
4665     });
4666 }
4667
4668 sub vm_sendkey {
4669     my ($vmid, $skiplock, $key) = @_;
4670
4671     lock_config($vmid, sub {
4672
4673         my $conf = load_config($vmid);
4674
4675         # there is no qmp command, so we use the human monitor command
4676         vm_human_monitor_command($vmid, "sendkey $key");
4677     });
4678 }
4679
4680 sub vm_destroy {
4681     my ($storecfg, $vmid, $skiplock) = @_;
4682
4683     lock_config($vmid, sub {
4684
4685         my $conf = load_config($vmid);
4686
4687         check_lock($conf) if !$skiplock;
4688
4689         if (!check_running($vmid)) {
4690             destroy_vm($storecfg, $vmid);
4691         } else {
4692             die "VM $vmid is running - destroy failed\n";
4693         }
4694     });
4695 }
4696
4697 # pci helpers
4698
4699 sub file_write {
4700     my ($filename, $buf) = @_;
4701
4702     my $fh = IO::File->new($filename, "w");
4703     return undef if !$fh;
4704
4705     my $res = print $fh $buf;
4706
4707     $fh->close();
4708
4709     return $res;
4710 }
4711
4712 sub pci_device_info {
4713     my ($name) = @_;
4714
4715     my $res;
4716
4717     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
4718     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
4719
4720     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
4721     return undef if !defined($irq) || $irq !~ m/^\d+$/;
4722
4723     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
4724     return undef if !defined($vendor) || $vendor !~ s/^0x//;
4725
4726     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
4727     return undef if !defined($product) || $product !~ s/^0x//;
4728
4729     $res = {
4730         name => $name,
4731         vendor => $vendor,
4732         product => $product,
4733         domain => $domain,
4734         bus => $bus,
4735         slot => $slot,
4736         func => $func,
4737         irq => $irq,
4738         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
4739     };
4740
4741     return $res;
4742 }
4743
4744 sub pci_dev_reset {
4745     my ($dev) = @_;
4746
4747     my $name = $dev->{name};
4748
4749     my $fn = "$pcisysfs/devices/$name/reset";
4750
4751     return file_write($fn, "1");
4752 }
4753
4754 sub pci_dev_bind_to_vfio {
4755     my ($dev) = @_;
4756
4757     my $name = $dev->{name};
4758
4759     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4760
4761     if (!-d $vfio_basedir) {
4762         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4763     }
4764     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4765
4766     my $testdir = "$vfio_basedir/$name";
4767     return 1 if -d $testdir;
4768
4769     my $data = "$dev->{vendor} $dev->{product}";
4770     return undef if !file_write("$vfio_basedir/new_id", $data);
4771
4772     my $fn = "$pcisysfs/devices/$name/driver/unbind";
4773     if (!file_write($fn, $name)) {
4774         return undef if -f $fn;
4775     }
4776
4777     $fn = "$vfio_basedir/bind";
4778     if (! -d $testdir) {
4779         return undef if !file_write($fn, $name);
4780     }
4781
4782     return -d $testdir;
4783 }
4784
4785 sub pci_dev_group_bind_to_vfio {
4786     my ($pciid) = @_;
4787
4788     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4789
4790     if (!-d $vfio_basedir) {
4791         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4792     }
4793     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4794
4795     # get IOMMU group devices
4796     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
4797       my @devs = grep /^0000:/, readdir($D);
4798     closedir($D);
4799
4800     foreach my $pciid (@devs) {
4801         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
4802
4803         # pci bridges, switches or root ports are not supported
4804         # they have a pci_bus subdirectory so skip them
4805         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
4806
4807         my $info = pci_device_info($1);
4808         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
4809     }
4810
4811     return 1;
4812 }
4813
4814 sub print_pci_addr {
4815     my ($id, $bridges) = @_;
4816
4817     my $res = '';
4818     my $devices = {
4819         piix3 => { bus => 0, addr => 1 },
4820         #addr2 : first videocard
4821         balloon0 => { bus => 0, addr => 3 },
4822         watchdog => { bus => 0, addr => 4 },
4823         scsihw0 => { bus => 0, addr => 5 },
4824         'pci.3' => { bus => 0, addr => 5 }, #can also be used for virtio-scsi-single bridge
4825         scsihw1 => { bus => 0, addr => 6 },
4826         ahci0 => { bus => 0, addr => 7 },
4827         qga0 => { bus => 0, addr => 8 },
4828         spice => { bus => 0, addr => 9 },
4829         virtio0 => { bus => 0, addr => 10 },
4830         virtio1 => { bus => 0, addr => 11 },
4831         virtio2 => { bus => 0, addr => 12 },
4832         virtio3 => { bus => 0, addr => 13 },
4833         virtio4 => { bus => 0, addr => 14 },
4834         virtio5 => { bus => 0, addr => 15 },
4835         hostpci0 => { bus => 0, addr => 16 },
4836         hostpci1 => { bus => 0, addr => 17 },
4837         net0 => { bus => 0, addr => 18 },
4838         net1 => { bus => 0, addr => 19 },
4839         net2 => { bus => 0, addr => 20 },
4840         net3 => { bus => 0, addr => 21 },
4841         net4 => { bus => 0, addr => 22 },
4842         net5 => { bus => 0, addr => 23 },
4843         vga1 => { bus => 0, addr => 24 },
4844         vga2 => { bus => 0, addr => 25 },
4845         vga3 => { bus => 0, addr => 26 },
4846         hostpci2 => { bus => 0, addr => 27 },
4847         hostpci3 => { bus => 0, addr => 28 },
4848         #addr29 : usb-host (pve-usb.cfg)
4849         'pci.1' => { bus => 0, addr => 30 },
4850         'pci.2' => { bus => 0, addr => 31 },
4851         'net6' => { bus => 1, addr => 1 },
4852         'net7' => { bus => 1, addr => 2 },
4853         'net8' => { bus => 1, addr => 3 },
4854         'net9' => { bus => 1, addr => 4 },
4855         'net10' => { bus => 1, addr => 5 },
4856         'net11' => { bus => 1, addr => 6 },
4857         'net12' => { bus => 1, addr => 7 },
4858         'net13' => { bus => 1, addr => 8 },
4859         'net14' => { bus => 1, addr => 9 },
4860         'net15' => { bus => 1, addr => 10 },
4861         'net16' => { bus => 1, addr => 11 },
4862         'net17' => { bus => 1, addr => 12 },
4863         'net18' => { bus => 1, addr => 13 },
4864         'net19' => { bus => 1, addr => 14 },
4865         'net20' => { bus => 1, addr => 15 },
4866         'net21' => { bus => 1, addr => 16 },
4867         'net22' => { bus => 1, addr => 17 },
4868         'net23' => { bus => 1, addr => 18 },
4869         'net24' => { bus => 1, addr => 19 },
4870         'net25' => { bus => 1, addr => 20 },
4871         'net26' => { bus => 1, addr => 21 },
4872         'net27' => { bus => 1, addr => 22 },
4873         'net28' => { bus => 1, addr => 23 },
4874         'net29' => { bus => 1, addr => 24 },
4875         'net30' => { bus => 1, addr => 25 },
4876         'net31' => { bus => 1, addr => 26 },
4877         'virtio6' => { bus => 2, addr => 1 },
4878         'virtio7' => { bus => 2, addr => 2 },
4879         'virtio8' => { bus => 2, addr => 3 },
4880         'virtio9' => { bus => 2, addr => 4 },
4881         'virtio10' => { bus => 2, addr => 5 },
4882         'virtio11' => { bus => 2, addr => 6 },
4883         'virtio12' => { bus => 2, addr => 7 },
4884         'virtio13' => { bus => 2, addr => 8 },
4885         'virtio14' => { bus => 2, addr => 9 },
4886         'virtio15' => { bus => 2, addr => 10 },
4887         'virtioscsi0' => { bus => 3, addr => 1 },
4888         'virtioscsi1' => { bus => 3, addr => 2 },
4889         'virtioscsi2' => { bus => 3, addr => 3 },
4890         'virtioscsi3' => { bus => 3, addr => 4 },
4891         'virtioscsi4' => { bus => 3, addr => 5 },
4892         'virtioscsi5' => { bus => 3, addr => 6 },
4893         'virtioscsi6' => { bus => 3, addr => 7 },
4894         'virtioscsi7' => { bus => 3, addr => 8 },
4895         'virtioscsi8' => { bus => 3, addr => 9 },
4896         'virtioscsi9' => { bus => 3, addr => 10 },
4897         'virtioscsi10' => { bus => 3, addr => 11 },
4898         'virtioscsi11' => { bus => 3, addr => 12 },
4899         'virtioscsi12' => { bus => 3, addr => 13 },
4900         'virtioscsi13' => { bus => 3, addr => 14 },
4901         'virtioscsi14' => { bus => 3, addr => 15 },
4902         'virtioscsi15' => { bus => 3, addr => 16 },
4903         'virtioscsi16' => { bus => 3, addr => 17 },
4904         'virtioscsi17' => { bus => 3, addr => 18 },
4905         'virtioscsi18' => { bus => 3, addr => 19 },
4906         'virtioscsi19' => { bus => 3, addr => 20 },
4907         'virtioscsi20' => { bus => 3, addr => 21 },
4908         'virtioscsi21' => { bus => 3, addr => 22 },
4909         'virtioscsi22' => { bus => 3, addr => 23 },
4910         'virtioscsi23' => { bus => 3, addr => 24 },
4911         'virtioscsi24' => { bus => 3, addr => 25 },
4912         'virtioscsi25' => { bus => 3, addr => 26 },
4913         'virtioscsi26' => { bus => 3, addr => 27 },
4914         'virtioscsi27' => { bus => 3, addr => 28 },
4915         'virtioscsi28' => { bus => 3, addr => 29 },
4916         'virtioscsi29' => { bus => 3, addr => 30 },
4917         'virtioscsi30' => { bus => 3, addr => 31 },
4918
4919     };
4920
4921     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
4922            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
4923            my $bus = $devices->{$id}->{bus};
4924            $res = ",bus=pci.$bus,addr=$addr";
4925            $bridges->{$bus} = 1 if $bridges;
4926     }
4927     return $res;
4928
4929 }
4930
4931 sub print_pcie_addr {
4932     my ($id) = @_;
4933
4934     my $res = '';
4935     my $devices = {
4936         hostpci0 => { bus => "ich9-pcie-port-1", addr => 0 },
4937         hostpci1 => { bus => "ich9-pcie-port-2", addr => 0 },
4938         hostpci2 => { bus => "ich9-pcie-port-3", addr => 0 },
4939         hostpci3 => { bus => "ich9-pcie-port-4", addr => 0 },
4940     };
4941
4942     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
4943            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
4944            my $bus = $devices->{$id}->{bus};
4945            $res = ",bus=$bus,addr=$addr";
4946     }
4947     return $res;
4948
4949 }
4950
4951 # vzdump restore implementaion
4952
4953 sub tar_archive_read_firstfile {
4954     my $archive = shift;
4955
4956     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
4957
4958     # try to detect archive type first
4959     my $pid = open (TMP, "tar tf '$archive'|") ||
4960         die "unable to open file '$archive'\n";
4961     my $firstfile = <TMP>;
4962     kill 15, $pid;
4963     close TMP;
4964
4965     die "ERROR: archive contaions no data\n" if !$firstfile;
4966     chomp $firstfile;
4967
4968     return $firstfile;
4969 }
4970
4971 sub tar_restore_cleanup {
4972     my ($storecfg, $statfile) = @_;
4973
4974     print STDERR "starting cleanup\n";
4975
4976     if (my $fd = IO::File->new($statfile, "r")) {
4977         while (defined(my $line = <$fd>)) {
4978             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
4979                 my $volid = $2;
4980                 eval {
4981                     if ($volid =~ m|^/|) {
4982                         unlink $volid || die 'unlink failed\n';
4983                     } else {
4984                         PVE::Storage::vdisk_free($storecfg, $volid);
4985                     }
4986                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
4987                 };
4988                 print STDERR "unable to cleanup '$volid' - $@" if $@;
4989             } else {
4990                 print STDERR "unable to parse line in statfile - $line";
4991             }
4992         }
4993         $fd->close();
4994     }
4995 }
4996
4997 sub restore_archive {
4998     my ($archive, $vmid, $user, $opts) = @_;
4999
5000     my $format = $opts->{format};
5001     my $comp;
5002
5003     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5004         $format = 'tar' if !$format;
5005         $comp = 'gzip';
5006     } elsif ($archive =~ m/\.tar$/) {
5007         $format = 'tar' if !$format;
5008     } elsif ($archive =~ m/.tar.lzo$/) {
5009         $format = 'tar' if !$format;
5010         $comp = 'lzop';
5011     } elsif ($archive =~ m/\.vma$/) {
5012         $format = 'vma' if !$format;
5013     } elsif ($archive =~ m/\.vma\.gz$/) {
5014         $format = 'vma' if !$format;
5015         $comp = 'gzip';
5016     } elsif ($archive =~ m/\.vma\.lzo$/) {
5017         $format = 'vma' if !$format;
5018         $comp = 'lzop';
5019     } else {
5020         $format = 'vma' if !$format; # default
5021     }
5022
5023     # try to detect archive format
5024     if ($format eq 'tar') {
5025         return restore_tar_archive($archive, $vmid, $user, $opts);
5026     } else {
5027         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5028     }
5029 }
5030
5031 sub restore_update_config_line {
5032     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5033
5034     return if $line =~ m/^\#qmdump\#/;
5035     return if $line =~ m/^\#vzdump\#/;
5036     return if $line =~ m/^lock:/;
5037     return if $line =~ m/^unused\d+:/;
5038     return if $line =~ m/^parent:/;
5039     return if $line =~ m/^template:/; # restored VM is never a template
5040
5041     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5042         # try to convert old 1.X settings
5043         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5044         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5045             my ($model, $macaddr) = split(/\=/, $devconfig);
5046             $macaddr = PVE::Tools::random_ether_addr() if !$macaddr || $unique;
5047             my $net = {
5048                 model => $model,
5049                 bridge => "vmbr$ind",
5050                 macaddr => $macaddr,
5051             };
5052             my $netstr = print_net($net);
5053
5054             print $outfd "net$cookie->{netcount}: $netstr\n";
5055             $cookie->{netcount}++;
5056         }
5057     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5058         my ($id, $netstr) = ($1, $2);
5059         my $net = parse_net($netstr);
5060         $net->{macaddr} = PVE::Tools::random_ether_addr() if $net->{macaddr};
5061         $netstr = print_net($net);
5062         print $outfd "$id: $netstr\n";
5063     } elsif ($line =~ m/^((ide|scsi|virtio|sata)\d+):\s*(\S+)\s*$/) {
5064         my $virtdev = $1;
5065         my $value = $3;
5066         if ($line =~ m/backup=no/) {
5067             print $outfd "#$line";
5068         } elsif ($virtdev && $map->{$virtdev}) {
5069             my $di = parse_drive($virtdev, $value);
5070             delete $di->{format}; # format can change on restore
5071             $di->{file} = $map->{$virtdev};
5072             $value = print_drive($vmid, $di);
5073             print $outfd "$virtdev: $value\n";
5074         } else {
5075             print $outfd $line;
5076         }
5077     } else {
5078         print $outfd $line;
5079     }
5080 }
5081
5082 sub scan_volids {
5083     my ($cfg, $vmid) = @_;
5084
5085     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5086
5087     my $volid_hash = {};
5088     foreach my $storeid (keys %$info) {
5089         foreach my $item (@{$info->{$storeid}}) {
5090             next if !($item->{volid} && $item->{size});
5091             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5092             $volid_hash->{$item->{volid}} = $item;
5093         }
5094     }
5095
5096     return $volid_hash;
5097 }
5098
5099 sub get_used_paths {
5100     my ($vmid, $storecfg, $conf, $scan_snapshots, $skip_drive) = @_;
5101
5102     my $used_path = {};
5103
5104     my $scan_config = sub {
5105         my ($cref, $snapname) = @_;
5106
5107         foreach my $key (keys %$cref) {
5108             my $value = $cref->{$key};
5109             if (valid_drivename($key)) {
5110                 next if $skip_drive && $key eq $skip_drive;
5111                 my $drive = parse_drive($key, $value);
5112                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5113                 if ($drive->{file} =~ m!^/!) {
5114                     $used_path->{$drive->{file}}++; # = 1;
5115                 } else {
5116                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5117                     next if !$storeid;
5118                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5119                     next if !$scfg;
5120                     my $path = PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5121                     $used_path->{$path}++; # = 1;
5122                 }
5123             }
5124         }
5125     };
5126
5127     &$scan_config($conf);
5128
5129     undef $skip_drive;
5130
5131     if ($scan_snapshots) {
5132         foreach my $snapname (keys %{$conf->{snapshots}}) {
5133             &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5134         }
5135     }
5136
5137     return $used_path;
5138 }
5139
5140 sub update_disksize {
5141     my ($vmid, $conf, $volid_hash) = @_;
5142
5143     my $changes;
5144
5145     my $used = {};
5146
5147     # Note: it is allowed to define multiple storages with same path (alias), so
5148     # we need to check both 'volid' and real 'path' (two different volid can point
5149     # to the same path).
5150
5151     my $usedpath = {};
5152
5153     # update size info
5154     foreach my $opt (keys %$conf) {
5155         if (valid_drivename($opt)) {
5156             my $drive = parse_drive($opt, $conf->{$opt});
5157             my $volid = $drive->{file};
5158             next if !$volid;
5159
5160             $used->{$volid} = 1;
5161             if ($volid_hash->{$volid} &&
5162                 (my $path = $volid_hash->{$volid}->{path})) {
5163                 $usedpath->{$path} = 1;
5164             }
5165
5166             next if drive_is_cdrom($drive);
5167             next if !$volid_hash->{$volid};
5168
5169             $drive->{size} = $volid_hash->{$volid}->{size};
5170             my $new = print_drive($vmid, $drive);
5171             if ($new ne $conf->{$opt}) {
5172                 $changes = 1;
5173                 $conf->{$opt} = $new;
5174             }
5175         }
5176     }
5177
5178     # remove 'unusedX' entry if volume is used
5179     foreach my $opt (keys %$conf) {
5180         next if $opt !~ m/^unused\d+$/;
5181         my $volid = $conf->{$opt};
5182         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5183         if ($used->{$volid} || ($path && $usedpath->{$path})) {
5184             $changes = 1;
5185             delete $conf->{$opt};
5186         }
5187     }
5188
5189     foreach my $volid (sort keys %$volid_hash) {
5190         next if $volid =~ m/vm-$vmid-state-/;
5191         next if $used->{$volid};
5192         my $path = $volid_hash->{$volid}->{path};
5193         next if !$path; # just to be sure
5194         next if $usedpath->{$path};
5195         $changes = 1;
5196         add_unused_volume($conf, $volid);
5197         $usedpath->{$path} = 1; # avoid to add more than once (aliases)
5198     }
5199
5200     return $changes;
5201 }
5202
5203 sub rescan {
5204     my ($vmid, $nolock) = @_;
5205
5206     my $cfg = PVE::Cluster::cfs_read_file("storage.cfg");
5207
5208     my $volid_hash = scan_volids($cfg, $vmid);
5209
5210     my $updatefn =  sub {
5211         my ($vmid) = @_;
5212
5213         my $conf = load_config($vmid);
5214
5215         check_lock($conf);
5216
5217         my $vm_volids = {};
5218         foreach my $volid (keys %$volid_hash) {
5219             my $info = $volid_hash->{$volid};
5220             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5221         }
5222
5223         my $changes = update_disksize($vmid, $conf, $vm_volids);
5224
5225         update_config_nolock($vmid, $conf, 1) if $changes;
5226     };
5227
5228     if (defined($vmid)) {
5229         if ($nolock) {
5230             &$updatefn($vmid);
5231         } else {
5232             lock_config($vmid, $updatefn, $vmid);
5233         }
5234     } else {
5235         my $vmlist = config_list();
5236         foreach my $vmid (keys %$vmlist) {
5237             if ($nolock) {
5238                 &$updatefn($vmid);
5239             } else {
5240                 lock_config($vmid, $updatefn, $vmid);
5241             }
5242         }
5243     }
5244 }
5245
5246 sub restore_vma_archive {
5247     my ($archive, $vmid, $user, $opts, $comp) = @_;
5248
5249     my $input = $archive eq '-' ? "<&STDIN" : undef;
5250     my $readfrom = $archive;
5251
5252     my $uncomp = '';
5253     if ($comp) {
5254         $readfrom = '-';
5255         my $qarchive = PVE::Tools::shellquote($archive);
5256         if ($comp eq 'gzip') {
5257             $uncomp = "zcat $qarchive|";
5258         } elsif ($comp eq 'lzop') {
5259             $uncomp = "lzop -d -c $qarchive|";
5260         } else {
5261             die "unknown compression method '$comp'\n";
5262         }
5263
5264     }
5265
5266     my $tmpdir = "/var/tmp/vzdumptmp$$";
5267     rmtree $tmpdir;
5268
5269     # disable interrupts (always do cleanups)
5270     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5271         warn "got interrupt - ignored\n";
5272     };
5273
5274     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5275     POSIX::mkfifo($mapfifo, 0600);
5276     my $fifofh;
5277
5278     my $openfifo = sub {
5279         open($fifofh, '>', $mapfifo) || die $!;
5280     };
5281
5282     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5283
5284     my $oldtimeout;
5285     my $timeout = 5;
5286
5287     my $devinfo = {};
5288
5289     my $rpcenv = PVE::RPCEnvironment::get();
5290
5291     my $conffile = config_file($vmid);
5292     my $tmpfn = "$conffile.$$.tmp";
5293
5294     # Note: $oldconf is undef if VM does not exists
5295     my $oldconf = PVE::Cluster::cfs_read_file(cfs_config_path($vmid));
5296
5297     my $print_devmap = sub {
5298         my $virtdev_hash = {};
5299
5300         my $cfgfn = "$tmpdir/qemu-server.conf";
5301
5302         # we can read the config - that is already extracted
5303         my $fh = IO::File->new($cfgfn, "r") ||
5304             "unable to read qemu-server.conf - $!\n";
5305
5306         while (defined(my $line = <$fh>)) {
5307             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5308                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5309                 die "archive does not contain data for drive '$virtdev'\n"
5310                     if !$devinfo->{$devname};
5311                 if (defined($opts->{storage})) {
5312                     $storeid = $opts->{storage} || 'local';
5313                 } elsif (!$storeid) {
5314                     $storeid = 'local';
5315                 }
5316                 $format = 'raw' if !$format;
5317                 $devinfo->{$devname}->{devname} = $devname;
5318                 $devinfo->{$devname}->{virtdev} = $virtdev;
5319                 $devinfo->{$devname}->{format} = $format;
5320                 $devinfo->{$devname}->{storeid} = $storeid;
5321
5322                 # check permission on storage
5323                 my $pool = $opts->{pool}; # todo: do we need that?
5324                 if ($user ne 'root@pam') {
5325                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5326                 }
5327
5328                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5329             }
5330         }
5331
5332         foreach my $devname (keys %$devinfo) {
5333             die "found no device mapping information for device '$devname'\n"
5334                 if !$devinfo->{$devname}->{virtdev};
5335         }
5336
5337         my $cfg = cfs_read_file('storage.cfg');
5338
5339         # create empty/temp config
5340         if ($oldconf) {
5341             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5342             foreach_drive($oldconf, sub {
5343                 my ($ds, $drive) = @_;
5344
5345                 return if drive_is_cdrom($drive);
5346
5347                 my $volid = $drive->{file};
5348
5349                 return if !$volid || $volid =~ m|^/|;
5350
5351                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5352                 return if !$path || !$owner || ($owner != $vmid);
5353
5354                 # Note: only delete disk we want to restore
5355                 # other volumes will become unused
5356                 if ($virtdev_hash->{$ds}) {
5357                     PVE::Storage::vdisk_free($cfg, $volid);
5358                 }
5359             });
5360         }
5361
5362         my $map = {};
5363         foreach my $virtdev (sort keys %$virtdev_hash) {
5364             my $d = $virtdev_hash->{$virtdev};
5365             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5366             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5367
5368             # test if requested format is supported
5369             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5370             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5371             $d->{format} = $defFormat if !$supported;
5372
5373             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5374                                                   $d->{format}, undef, $alloc_size);
5375             print STDERR "new volume ID is '$volid'\n";
5376             $d->{volid} = $volid;
5377             my $path = PVE::Storage::path($cfg, $volid);
5378
5379             PVE::Storage::activate_volumes($cfg,[$volid]);
5380
5381             my $write_zeros = 1;
5382             # fixme: what other storages types initialize volumes with zero?
5383             if ($scfg->{type} eq 'dir' || $scfg->{type} eq 'nfs' || $scfg->{type} eq 'glusterfs' ||
5384                 $scfg->{type} eq 'sheepdog' || $scfg->{type} eq 'rbd') {
5385                 $write_zeros = 0;
5386             }
5387
5388             print $fifofh "${write_zeros}:$d->{devname}=$path\n";
5389
5390             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5391             $map->{$virtdev} = $volid;
5392         }
5393
5394         $fh->seek(0, 0) || die "seek failed - $!\n";
5395
5396         my $outfd = new IO::File ($tmpfn, "w") ||
5397             die "unable to write config for VM $vmid\n";
5398
5399         my $cookie = { netcount => 0 };
5400         while (defined(my $line = <$fh>)) {
5401             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5402         }
5403
5404         $fh->close();
5405         $outfd->close();
5406     };
5407
5408     eval {
5409         # enable interrupts
5410         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5411             die "interrupted by signal\n";
5412         };
5413         local $SIG{ALRM} = sub { die "got timeout\n"; };
5414
5415         $oldtimeout = alarm($timeout);
5416
5417         my $parser = sub {
5418             my $line = shift;
5419
5420             print "$line\n";
5421
5422             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5423                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5424                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5425             } elsif ($line =~ m/^CTIME: /) {
5426                 # we correctly received the vma config, so we can disable
5427                 # the timeout now for disk allocation (set to 10 minutes, so
5428                 # that we always timeout if something goes wrong)
5429                 alarm(600);
5430                 &$print_devmap();
5431                 print $fifofh "done\n";
5432                 my $tmp = $oldtimeout || 0;
5433                 $oldtimeout = undef;
5434                 alarm($tmp);
5435                 close($fifofh);
5436             }
5437         };
5438
5439         print "restore vma archive: $cmd\n";
5440         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5441     };
5442     my $err = $@;
5443
5444     alarm($oldtimeout) if $oldtimeout;
5445
5446     my $vollist = [];
5447     foreach my $devname (keys %$devinfo) {
5448         my $volid = $devinfo->{$devname}->{volid};
5449         push @$vollist, $volid if $volid;
5450     }
5451
5452     my $cfg = cfs_read_file('storage.cfg');
5453     PVE::Storage::deactivate_volumes($cfg, $vollist);
5454
5455     unlink $mapfifo;
5456
5457     if ($err) {
5458         rmtree $tmpdir;
5459         unlink $tmpfn;
5460
5461         foreach my $devname (keys %$devinfo) {
5462             my $volid = $devinfo->{$devname}->{volid};
5463             next if !$volid;
5464             eval {
5465                 if ($volid =~ m|^/|) {
5466                     unlink $volid || die 'unlink failed\n';
5467                 } else {
5468                     PVE::Storage::vdisk_free($cfg, $volid);
5469                 }
5470                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5471             };
5472             print STDERR "unable to cleanup '$volid' - $@" if $@;
5473         }
5474         die $err;
5475     }
5476
5477     rmtree $tmpdir;
5478
5479     rename($tmpfn, $conffile) ||
5480         die "unable to commit configuration file '$conffile'\n";
5481
5482     PVE::Cluster::cfs_update(); # make sure we read new file
5483
5484     eval { rescan($vmid, 1); };
5485     warn $@ if $@;
5486 }
5487
5488 sub restore_tar_archive {
5489     my ($archive, $vmid, $user, $opts) = @_;
5490
5491     if ($archive ne '-') {
5492         my $firstfile = tar_archive_read_firstfile($archive);
5493         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5494             if $firstfile ne 'qemu-server.conf';
5495     }
5496
5497     my $storecfg = cfs_read_file('storage.cfg');
5498
5499     # destroy existing data - keep empty config
5500     my $vmcfgfn = config_file($vmid);
5501     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5502
5503     my $tocmd = "/usr/lib/qemu-server/qmextract";
5504
5505     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5506     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5507     $tocmd .= ' --prealloc' if $opts->{prealloc};
5508     $tocmd .= ' --info' if $opts->{info};
5509
5510     # tar option "xf" does not autodetect compression when read from STDIN,
5511     # so we pipe to zcat
5512     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5513         PVE::Tools::shellquote("--to-command=$tocmd");
5514
5515     my $tmpdir = "/var/tmp/vzdumptmp$$";
5516     mkpath $tmpdir;
5517
5518     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5519     local $ENV{VZDUMP_VMID} = $vmid;
5520     local $ENV{VZDUMP_USER} = $user;
5521
5522     my $conffile = config_file($vmid);
5523     my $tmpfn = "$conffile.$$.tmp";
5524
5525     # disable interrupts (always do cleanups)
5526     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5527         print STDERR "got interrupt - ignored\n";
5528     };
5529
5530     eval {
5531         # enable interrupts
5532         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5533             die "interrupted by signal\n";
5534         };
5535
5536         if ($archive eq '-') {
5537             print "extracting archive from STDIN\n";
5538             run_command($cmd, input => "<&STDIN");
5539         } else {
5540             print "extracting archive '$archive'\n";
5541             run_command($cmd);
5542         }
5543
5544         return if $opts->{info};
5545
5546         # read new mapping
5547         my $map = {};
5548         my $statfile = "$tmpdir/qmrestore.stat";
5549         if (my $fd = IO::File->new($statfile, "r")) {
5550             while (defined (my $line = <$fd>)) {
5551                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5552                     $map->{$1} = $2 if $1;
5553                 } else {
5554                     print STDERR "unable to parse line in statfile - $line\n";
5555                 }
5556             }
5557             $fd->close();
5558         }
5559
5560         my $confsrc = "$tmpdir/qemu-server.conf";
5561
5562         my $srcfd = new IO::File($confsrc, "r") ||
5563             die "unable to open file '$confsrc'\n";
5564
5565         my $outfd = new IO::File ($tmpfn, "w") ||
5566             die "unable to write config for VM $vmid\n";
5567
5568         my $cookie = { netcount => 0 };
5569         while (defined (my $line = <$srcfd>)) {
5570             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5571         }
5572
5573         $srcfd->close();
5574         $outfd->close();
5575     };
5576     my $err = $@;
5577
5578     if ($err) {
5579
5580         unlink $tmpfn;
5581
5582         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
5583
5584         die $err;
5585     }
5586
5587     rmtree $tmpdir;
5588
5589     rename $tmpfn, $conffile ||
5590         die "unable to commit configuration file '$conffile'\n";
5591
5592     PVE::Cluster::cfs_update(); # make sure we read new file
5593
5594     eval { rescan($vmid, 1); };
5595     warn $@ if $@;
5596 };
5597
5598
5599 # Internal snapshots
5600
5601 # NOTE: Snapshot create/delete involves several non-atomic
5602 # action, and can take a long time.
5603 # So we try to avoid locking the file and use 'lock' variable
5604 # inside the config file instead.
5605
5606 my $snapshot_copy_config = sub {
5607     my ($source, $dest) = @_;
5608
5609     foreach my $k (keys %$source) {
5610         next if $k eq 'snapshots';
5611         next if $k eq 'snapstate';
5612         next if $k eq 'snaptime';
5613         next if $k eq 'vmstate';
5614         next if $k eq 'lock';
5615         next if $k eq 'digest';
5616         next if $k eq 'description';
5617         next if $k =~ m/^unused\d+$/;
5618
5619         $dest->{$k} = $source->{$k};
5620     }
5621 };
5622
5623 my $snapshot_apply_config = sub {
5624     my ($conf, $snap) = @_;
5625
5626     # copy snapshot list
5627     my $newconf = {
5628         snapshots => $conf->{snapshots},
5629     };
5630
5631     # keep description and list of unused disks
5632     foreach my $k (keys %$conf) {
5633         next if !($k =~ m/^unused\d+$/ || $k eq 'description');
5634         $newconf->{$k} = $conf->{$k};
5635     }
5636
5637     &$snapshot_copy_config($snap, $newconf);
5638
5639     return $newconf;
5640 };
5641
5642 sub foreach_writable_storage {
5643     my ($conf, $func) = @_;
5644
5645     my $sidhash = {};
5646
5647     foreach my $ds (keys %$conf) {
5648         next if !valid_drivename($ds);
5649
5650         my $drive = parse_drive($ds, $conf->{$ds});
5651         next if !$drive;
5652         next if drive_is_cdrom($drive);
5653
5654         my $volid = $drive->{file};
5655
5656         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5657         $sidhash->{$sid} = $sid if $sid;
5658     }
5659
5660     foreach my $sid (sort keys %$sidhash) {
5661         &$func($sid);
5662     }
5663 }
5664
5665 my $alloc_vmstate_volid = sub {
5666     my ($storecfg, $vmid, $conf, $snapname) = @_;
5667
5668     # Note: we try to be smart when selecting a $target storage
5669
5670     my $target;
5671
5672     # search shared storage first
5673     foreach_writable_storage($conf, sub {
5674         my ($sid) = @_;
5675         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
5676         return if !$scfg->{shared};
5677
5678         $target = $sid if !$target || $scfg->{path}; # prefer file based storage
5679     });
5680
5681     if (!$target) {
5682         # now search local storage
5683         foreach_writable_storage($conf, sub {
5684             my ($sid) = @_;
5685             my $scfg = PVE::Storage::storage_config($storecfg, $sid);
5686             return if $scfg->{shared};
5687
5688             $target = $sid if !$target || $scfg->{path}; # prefer file based storage;
5689         });
5690     }
5691
5692     $target = 'local' if !$target;
5693
5694     my $driver_state_size = 500; # assume 32MB is enough to safe all driver state;
5695     # we abort live save after $conf->{memory}, so we need at max twice that space
5696     my $size = $conf->{memory}*2 + $driver_state_size;
5697
5698     my $name = "vm-$vmid-state-$snapname";
5699     my $scfg = PVE::Storage::storage_config($storecfg, $target);
5700     $name .= ".raw" if $scfg->{path}; # add filename extension for file base storage
5701     my $volid = PVE::Storage::vdisk_alloc($storecfg, $target, $vmid, 'raw', $name, $size*1024);
5702
5703     return $volid;
5704 };
5705
5706 my $snapshot_prepare = sub {
5707     my ($vmid, $snapname, $save_vmstate, $comment) = @_;
5708
5709     my $snap;
5710
5711     my $updatefn =  sub {
5712
5713         my $conf = load_config($vmid);
5714
5715         die "you can't take a snapshot if it's a template\n"
5716             if is_template($conf);
5717
5718         check_lock($conf);
5719
5720         $conf->{lock} = 'snapshot';
5721
5722         die "snapshot name '$snapname' already used\n"
5723             if defined($conf->{snapshots}->{$snapname});
5724
5725         my $storecfg = PVE::Storage::config();
5726         die "snapshot feature is not available" if !has_feature('snapshot', $conf, $storecfg);
5727
5728         $snap = $conf->{snapshots}->{$snapname} = {};
5729
5730         if ($save_vmstate && check_running($vmid)) {
5731             $snap->{vmstate} = &$alloc_vmstate_volid($storecfg, $vmid, $conf, $snapname);
5732         }
5733
5734         &$snapshot_copy_config($conf, $snap);
5735
5736         $snap->{snapstate} = "prepare";
5737         $snap->{snaptime} = time();
5738         $snap->{description} = $comment if $comment;
5739
5740         # always overwrite machine if we save vmstate. This makes sure we
5741         # can restore it later using correct machine type
5742         $snap->{machine} = get_current_qemu_machine($vmid) if $snap->{vmstate};
5743
5744         update_config_nolock($vmid, $conf, 1);
5745     };
5746
5747     lock_config($vmid, $updatefn);
5748
5749     return $snap;
5750 };
5751
5752 my $snapshot_commit = sub {
5753     my ($vmid, $snapname) = @_;
5754
5755     my $updatefn = sub {
5756
5757         my $conf = load_config($vmid);
5758
5759         die "missing snapshot lock\n"
5760             if !($conf->{lock} && $conf->{lock} eq 'snapshot');
5761
5762         my $has_machine_config = defined($conf->{machine});
5763
5764         my $snap = $conf->{snapshots}->{$snapname};
5765
5766         die "snapshot '$snapname' does not exist\n" if !defined($snap);
5767
5768         die "wrong snapshot state\n"
5769             if !($snap->{snapstate} && $snap->{snapstate} eq "prepare");
5770
5771         delete $snap->{snapstate};
5772         delete $conf->{lock};
5773
5774         my $newconf = &$snapshot_apply_config($conf, $snap);
5775
5776         delete $newconf->{machine} if !$has_machine_config;
5777
5778         $newconf->{parent} = $snapname;
5779
5780         update_config_nolock($vmid, $newconf, 1);
5781     };
5782
5783     lock_config($vmid, $updatefn);
5784 };
5785
5786 sub snapshot_rollback {
5787     my ($vmid, $snapname) = @_;
5788
5789     my $prepare = 1;
5790
5791     my $storecfg = PVE::Storage::config();
5792
5793     my $conf = load_config($vmid);
5794
5795     my $get_snapshot_config = sub {
5796
5797         die "you can't rollback if vm is a template\n" if is_template($conf);
5798
5799         my $res = $conf->{snapshots}->{$snapname};
5800
5801         die "snapshot '$snapname' does not exist\n" if !defined($res);
5802
5803         return $res;
5804     };
5805
5806     my $snap = &$get_snapshot_config();
5807
5808     foreach_drive($snap, sub {
5809         my ($ds, $drive) = @_;
5810
5811         return if drive_is_cdrom($drive);
5812
5813         my $volid = $drive->{file};
5814
5815         PVE::Storage::volume_rollback_is_possible($storecfg, $volid, $snapname);
5816     });
5817
5818     my $updatefn = sub {
5819
5820         $conf = load_config($vmid);
5821
5822         $snap = &$get_snapshot_config();
5823
5824         die "unable to rollback to incomplete snapshot (snapstate = $snap->{snapstate})\n"
5825             if $snap->{snapstate};
5826
5827         if ($prepare) {
5828             check_lock($conf);
5829             vm_stop($storecfg, $vmid, undef, undef, 5, undef, undef);
5830         }
5831
5832         die "unable to rollback vm $vmid: vm is running\n"
5833             if check_running($vmid);
5834
5835         if ($prepare) {
5836             $conf->{lock} = 'rollback';
5837         } else {
5838             die "got wrong lock\n" if !($conf->{lock} && $conf->{lock} eq 'rollback');
5839             delete $conf->{lock};
5840         }
5841
5842         my $forcemachine;
5843
5844         if (!$prepare) {
5845             my $has_machine_config = defined($conf->{machine});
5846
5847             # copy snapshot config to current config
5848             $conf = &$snapshot_apply_config($conf, $snap);
5849             $conf->{parent} = $snapname;
5850
5851             # Note: old code did not store 'machine', so we try to be smart
5852             # and guess the snapshot was generated with kvm 1.4 (pc-i440fx-1.4).
5853             $forcemachine = $conf->{machine} || 'pc-i440fx-1.4';
5854             # we remove the 'machine' configuration if not explicitly specified
5855             # in the original config.
5856             delete $conf->{machine} if $snap->{vmstate} && !$has_machine_config;
5857         }
5858
5859         update_config_nolock($vmid, $conf, 1);
5860
5861         if (!$prepare && $snap->{vmstate}) {
5862             my $statefile = PVE::Storage::path($storecfg, $snap->{vmstate});
5863             vm_start($storecfg, $vmid, $statefile, undef, undef, undef, $forcemachine);
5864         }
5865     };
5866
5867     lock_config($vmid, $updatefn);
5868
5869     foreach_drive($snap, sub {
5870         my ($ds, $drive) = @_;
5871
5872         return if drive_is_cdrom($drive);
5873
5874         my $volid = $drive->{file};
5875         my $device = "drive-$ds";
5876
5877         PVE::Storage::volume_snapshot_rollback($storecfg, $volid, $snapname);
5878     });
5879
5880     $prepare = 0;
5881     lock_config($vmid, $updatefn);
5882 }
5883
5884 my $savevm_wait = sub {
5885     my ($vmid) = @_;
5886
5887     for(;;) {
5888         my $stat = vm_mon_cmd_nocheck($vmid, "query-savevm");
5889         if (!$stat->{status}) {
5890             die "savevm not active\n";
5891         } elsif ($stat->{status} eq 'active') {
5892             sleep(1);
5893             next;
5894         } elsif ($stat->{status} eq 'completed') {
5895             last;
5896         } else {
5897             die "query-savevm returned status '$stat->{status}'\n";
5898         }
5899     }
5900 };
5901
5902 sub do_snapshots_with_qemu {
5903     my ($storecfg, $volid) = @_;
5904
5905     my $storage_name = PVE::Storage::parse_volume_id($volid);
5906
5907     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
5908         && !$storecfg->{ids}->{$storage_name}->{krbd}){
5909         return 1;
5910     }
5911
5912     if ($volid =~ m/\.(qcow2|qed)$/){
5913         return 1;
5914     }
5915
5916     return undef;
5917 }
5918
5919 sub snapshot_create {
5920     my ($vmid, $snapname, $save_vmstate, $comment) = @_;
5921
5922     my $snap = &$snapshot_prepare($vmid, $snapname, $save_vmstate, $comment);
5923
5924     $save_vmstate = 0 if !$snap->{vmstate}; # vm is not running
5925
5926     my $config = load_config($vmid);
5927
5928     my $running = check_running($vmid);
5929
5930     my $freezefs = $running && $config->{agent};
5931     $freezefs = 0 if $snap->{vmstate}; # not needed if we save RAM
5932
5933     my $drivehash = {};
5934
5935     if ($freezefs) {
5936         eval { vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
5937         warn "guest-fsfreeze-freeze problems - $@" if $@;
5938     }
5939
5940     eval {
5941         # create internal snapshots of all drives
5942
5943         my $storecfg = PVE::Storage::config();
5944
5945         if ($running) {
5946             if ($snap->{vmstate}) {
5947                 my $path = PVE::Storage::path($storecfg, $snap->{vmstate});
5948                 vm_mon_cmd($vmid, "savevm-start", statefile => $path);
5949                 &$savevm_wait($vmid);
5950             } else {
5951                 vm_mon_cmd($vmid, "savevm-start");
5952             }
5953         };
5954
5955         foreach_drive($snap, sub {
5956             my ($ds, $drive) = @_;
5957
5958             return if drive_is_cdrom($drive);
5959
5960             my $volid = $drive->{file};
5961             my $device = "drive-$ds";
5962
5963             qemu_volume_snapshot($vmid, $device, $storecfg, $volid, $snapname);
5964             $drivehash->{$ds} = 1;
5965        });
5966     };
5967     my $err = $@;
5968
5969     if ($running) {
5970         eval { vm_mon_cmd($vmid, "savevm-end")  };
5971         warn $@ if $@;
5972
5973         if ($freezefs) {
5974             eval { vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
5975             warn "guest-fsfreeze-thaw problems - $@" if $@;
5976         }
5977
5978         # savevm-end is async, we need to wait
5979         for (;;) {
5980             my $stat = vm_mon_cmd_nocheck($vmid, "query-savevm");
5981             if (!$stat->{bytes}) {
5982                 last;
5983             } else {
5984                 print "savevm not yet finished\n";
5985                 sleep(1);
5986                 next;
5987             }
5988         }
5989     }
5990
5991     if ($err) {
5992         warn "snapshot create failed: starting cleanup\n";
5993         eval { snapshot_delete($vmid, $snapname, 0, $drivehash); };
5994         warn $@ if $@;
5995         die $err;
5996     }
5997
5998     &$snapshot_commit($vmid, $snapname);
5999 }
6000
6001 # Note: $drivehash is only set when called from snapshot_create.
6002 sub snapshot_delete {
6003     my ($vmid, $snapname, $force, $drivehash) = @_;
6004
6005     my $prepare = 1;
6006
6007     my $snap;
6008     my $unused = [];
6009
6010     my $unlink_parent = sub {
6011         my ($confref, $new_parent) = @_;
6012
6013         if ($confref->{parent} && $confref->{parent} eq $snapname) {
6014             if ($new_parent) {
6015                 $confref->{parent} = $new_parent;
6016             } else {
6017                 delete $confref->{parent};
6018             }
6019         }
6020     };
6021
6022     my $updatefn =  sub {
6023         my ($remove_drive) = @_;
6024
6025         my $conf = load_config($vmid);
6026
6027         if (!$drivehash) {
6028             check_lock($conf);
6029             die "you can't delete a snapshot if vm is a template\n"
6030                 if is_template($conf);
6031         }
6032
6033         $snap = $conf->{snapshots}->{$snapname};
6034
6035         die "snapshot '$snapname' does not exist\n" if !defined($snap);
6036
6037         # remove parent refs
6038         if (!$prepare) {
6039             &$unlink_parent($conf, $snap->{parent});
6040             foreach my $sn (keys %{$conf->{snapshots}}) {
6041                 next if $sn eq $snapname;
6042                 &$unlink_parent($conf->{snapshots}->{$sn}, $snap->{parent});
6043             }
6044         }
6045
6046         if ($remove_drive) {
6047             if ($remove_drive eq 'vmstate') {
6048                 delete $snap->{$remove_drive};
6049             } else {
6050                 my $drive = parse_drive($remove_drive, $snap->{$remove_drive});
6051                 my $volid = $drive->{file};
6052                 delete $snap->{$remove_drive};
6053                 add_unused_volume($conf, $volid);
6054             }
6055         }
6056
6057         if ($prepare) {
6058             $snap->{snapstate} = 'delete';
6059         } else {
6060             delete $conf->{snapshots}->{$snapname};
6061             delete $conf->{lock} if $drivehash;
6062             foreach my $volid (@$unused) {
6063                 add_unused_volume($conf, $volid);
6064             }
6065         }
6066
6067         update_config_nolock($vmid, $conf, 1);
6068     };
6069
6070     lock_config($vmid, $updatefn);
6071
6072     # now remove vmstate file
6073
6074     my $storecfg = PVE::Storage::config();
6075
6076     if ($snap->{vmstate}) {
6077         eval {  PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
6078         if (my $err = $@) {
6079             die $err if !$force;
6080             warn $err;
6081         }
6082         # save changes (remove vmstate from snapshot)
6083         lock_config($vmid, $updatefn, 'vmstate') if !$force;
6084     };
6085
6086     # now remove all internal snapshots
6087     foreach_drive($snap, sub {
6088         my ($ds, $drive) = @_;
6089
6090         return if drive_is_cdrom($drive);
6091
6092         my $volid = $drive->{file};
6093         my $device = "drive-$ds";
6094
6095         if (!$drivehash || $drivehash->{$ds}) {
6096             eval { qemu_volume_snapshot_delete($vmid, $device, $storecfg, $volid, $snapname); };
6097             if (my $err = $@) {
6098                 die $err if !$force;
6099                 warn $err;
6100             }
6101         }
6102
6103         # save changes (remove drive fron snapshot)
6104         lock_config($vmid, $updatefn, $ds) if !$force;
6105         push @$unused, $volid;
6106     });
6107
6108     # now cleanup config
6109     $prepare = 0;
6110     lock_config($vmid, $updatefn);
6111 }
6112
6113 sub has_feature {
6114     my ($feature, $conf, $storecfg, $snapname, $running) = @_;
6115
6116     my $err;
6117     foreach_drive($conf, sub {
6118         my ($ds, $drive) = @_;
6119
6120         return if drive_is_cdrom($drive);
6121         my $volid = $drive->{file};
6122         $err = 1 if !PVE::Storage::volume_has_feature($storecfg, $feature, $volid, $snapname, $running);
6123     });
6124
6125     return $err ? 0 : 1;
6126 }
6127
6128 sub template_create {
6129     my ($vmid, $conf, $disk) = @_;
6130
6131     my $storecfg = PVE::Storage::config();
6132
6133     foreach_drive($conf, sub {
6134         my ($ds, $drive) = @_;
6135
6136         return if drive_is_cdrom($drive);
6137         return if $disk && $ds ne $disk;
6138
6139         my $volid = $drive->{file};
6140         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6141
6142         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6143         $drive->{file} = $voliddst;
6144         $conf->{$ds} = print_drive($vmid, $drive);
6145         update_config_nolock($vmid, $conf, 1);
6146     });
6147 }
6148
6149 sub is_template {
6150     my ($conf) = @_;
6151
6152     return 1 if defined $conf->{template} && $conf->{template} == 1;
6153 }
6154
6155 sub qemu_img_convert {
6156     my ($src_volid, $dst_volid, $size, $snapname) = @_;
6157
6158     my $storecfg = PVE::Storage::config();
6159     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6160     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6161
6162     if ($src_storeid && $dst_storeid) {
6163
6164         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6165
6166         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6167         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6168
6169         my $src_format = qemu_img_format($src_scfg, $src_volname);
6170         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6171
6172         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6173         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6174
6175         my $cmd = [];
6176         push @$cmd, '/usr/bin/qemu-img', 'convert', '-t', 'writeback', '-p', '-n';
6177         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
6178         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path, $dst_path;
6179
6180         my $parser = sub {
6181             my $line = shift;
6182             if($line =~ m/\((\S+)\/100\%\)/){
6183                 my $percent = $1;
6184                 my $transferred = int($size * $percent / 100);
6185                 my $remaining = $size - $transferred;
6186
6187                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6188             }
6189
6190         };
6191
6192         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6193         my $err = $@;
6194         die "copy failed: $err" if $err;
6195     }
6196 }
6197
6198 sub qemu_img_format {
6199     my ($scfg, $volname) = @_;
6200
6201     if ($scfg->{path} && $volname =~ m/\.(raw|cow|qcow|qcow2|qed|vmdk|cloop)$/) {
6202         return $1;
6203     } else {
6204         return "raw";
6205     }
6206 }
6207
6208 sub qemu_drive_mirror {
6209     my ($vmid, $drive, $dst_volid, $vmiddst) = @_;
6210
6211     my $storecfg = PVE::Storage::config();
6212     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6213
6214     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6215
6216     my $format = qemu_img_format($dst_scfg, $dst_volname);
6217
6218     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6219
6220     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $dst_path };
6221     $opts->{format} = $format if $format;
6222
6223     print "drive mirror is starting (scanning bitmap) : this step can take some minutes/hours, depend of disk size and storage speed\n";
6224
6225     eval {
6226     vm_mon_cmd($vmid, "drive-mirror", %$opts);
6227         while (1) {
6228             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6229             my $stat = @$stats[0];
6230             die "mirroring job seem to have die. Maybe do you have bad sectors?" if !$stat;
6231             die "error job is not mirroring" if $stat->{type} ne "mirror";
6232
6233             my $busy = $stat->{busy};
6234             my $ready = $stat->{ready};
6235
6236             if (my $total = $stat->{len}) {
6237                 my $transferred = $stat->{offset} || 0;
6238                 my $remaining = $total - $transferred;
6239                 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6240
6241                 print "transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6242             }
6243
6244
6245             if ($stat->{ready} eq 'true') {
6246
6247                 last if $vmiddst != $vmid;
6248
6249                 # try to switch the disk if source and destination are on the same guest
6250                 eval { vm_mon_cmd($vmid, "block-job-complete", device => "drive-$drive") };
6251                 last if !$@;
6252                 die $@ if $@ !~ m/cannot be completed/;
6253             }
6254             sleep 1;
6255         }
6256
6257
6258     };
6259     my $err = $@;
6260
6261     my $cancel_job = sub {
6262         vm_mon_cmd($vmid, "block-job-cancel", device => "drive-$drive");
6263         while (1) {
6264             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6265             my $stat = @$stats[0];
6266             last if !$stat;
6267             sleep 1;
6268         }
6269     };
6270
6271     if ($err) {
6272         eval { &$cancel_job(); };
6273         die "mirroring error: $err";
6274     }
6275
6276     if ($vmiddst != $vmid) {
6277         # if we clone a disk for a new target vm, we don't switch the disk
6278         &$cancel_job(); # so we call block-job-cancel
6279     }
6280 }
6281
6282 sub clone_disk {
6283     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6284         $newvmid, $storage, $format, $full, $newvollist) = @_;
6285
6286     my $newvolid;
6287
6288     if (!$full) {
6289         print "create linked clone of drive $drivename ($drive->{file})\n";
6290         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6291         push @$newvollist, $newvolid;
6292     } else {
6293         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6294         $storeid = $storage if $storage;
6295
6296         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6297         if (!$format) {
6298             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6299             $format = qemu_img_format($scfg, $volname);
6300         }
6301
6302         # test if requested format is supported - else use default
6303         my $supported = grep { $_ eq $format } @$validFormats;
6304         $format = $defFormat if !$supported;
6305
6306         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6307
6308         print "create full clone of drive $drivename ($drive->{file})\n";
6309         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $format, undef, ($size/1024));
6310         push @$newvollist, $newvolid;
6311
6312         PVE::Storage::activate_volumes($storecfg, $newvollist);
6313
6314         if (!$running || $snapname) {
6315             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname);
6316         } else {
6317             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid);
6318         }
6319     }
6320
6321     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6322
6323     my $disk = $drive;
6324     $disk->{format} = undef;
6325     $disk->{file} = $newvolid;
6326     $disk->{size} = $size;
6327
6328     return $disk;
6329 }
6330
6331 # this only works if VM is running
6332 sub get_current_qemu_machine {
6333     my ($vmid) = @_;
6334
6335     my $cmd = { execute => 'query-machines', arguments => {} };
6336     my $res = vm_qmp_command($vmid, $cmd);
6337
6338     my ($current, $default);
6339     foreach my $e (@$res) {
6340         $default = $e->{name} if $e->{'is-default'};
6341         $current = $e->{name} if $e->{'is-current'};
6342     }
6343
6344     # fallback to the default machine if current is not supported by qemu
6345     return $current || $default || 'pc';
6346 }
6347
6348 sub qemu_machine_feature_enabled {
6349     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6350
6351     my $current_major;
6352     my $current_minor;
6353
6354     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6355
6356         $current_major = $3;
6357         $current_minor = $4;
6358
6359     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6360
6361         $current_major = $1;
6362         $current_minor = $2;
6363     }
6364
6365     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
6366
6367
6368 }
6369
6370 sub qemu_machine_pxe {
6371     my ($vmid, $conf, $machine) = @_;
6372
6373     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6374
6375     foreach my $opt (keys %$conf) {
6376         next if $opt !~ m/^net(\d+)$/;
6377         my $net = PVE::QemuServer::parse_net($conf->{$opt});
6378         next if !$net;
6379         my $romfile = PVE::QemuServer::vm_mon_cmd_nocheck($vmid, 'qom-get', path => $opt, property => 'romfile');
6380         return $machine.".pxe" if $romfile =~ m/pxe/;
6381         last;
6382     }
6383
6384 }
6385
6386 sub qemu_use_old_bios_files {
6387     my ($machine_type) = @_;
6388
6389     return if !$machine_type;
6390
6391     my $use_old_bios_files = undef;
6392
6393     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6394         $machine_type = $1;
6395         $use_old_bios_files = 1;
6396     } else {
6397         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6398         # load new efi bios files on migration. So this hack is required to allow
6399         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6400         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6401         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, undef, 2, 4);
6402     }
6403
6404     return ($use_old_bios_files, $machine_type);
6405 }
6406
6407 sub lspci {
6408
6409     my $devices = {};
6410
6411     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6412             my (undef, $id, $function) = @_;
6413             my $res = { id => $id, function => $function};
6414             push @{$devices->{$id}}, $res;
6415     });
6416
6417     return $devices;
6418 }
6419
6420 sub vm_iothreads_list {
6421     my ($vmid) = @_;
6422
6423     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6424
6425     my $iothreads = {};
6426     foreach my $iothread (@$res) {
6427         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6428     }
6429
6430     return $iothreads;
6431 }
6432
6433 sub scsihw_infos {
6434     my ($conf, $drive) = @_;
6435
6436     my $maxdev = 0;
6437
6438     if ($conf->{scsihw} && ($conf->{scsihw} =~ m/^lsi/)) {
6439         $maxdev = 7;
6440     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6441         $maxdev = 1;
6442     } else {
6443         $maxdev = 256;
6444     }
6445
6446     my $controller = int($drive->{index} / $maxdev);
6447     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6448
6449     return ($maxdev, $controller, $controller_prefix);
6450 }
6451
6452 # bash completion helper
6453
6454 sub complete_backup_archives {
6455     my ($cmdname, $pname, $cvalue) = @_;
6456
6457     my $cfg = PVE::Storage::config();
6458
6459     my $storeid;
6460
6461     if ($cvalue =~ m/^([^:]+):/) {
6462         $storeid = $1;
6463     }
6464
6465     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6466
6467     my $res = [];
6468     foreach my $id (keys %$data) {
6469         foreach my $item (@{$data->{$id}}) {
6470             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6471             push @$res, $item->{volid} if defined($item->{volid});
6472         }
6473     }
6474
6475     return $res;
6476 }
6477
6478 my $complete_vmid_full = sub {
6479     my ($running) = @_;
6480
6481     my $idlist = vmstatus();
6482
6483     my $res = [];
6484
6485     foreach my $id (keys %$idlist) {
6486         my $d = $idlist->{$id};
6487         if (defined($running)) {
6488             next if $d->{template};
6489             next if $running && $d->{status} ne 'running';
6490             next if !$running && $d->{status} eq 'running';
6491         }
6492         push @$res, $id;
6493
6494     }
6495     return $res;
6496 };
6497
6498 sub complete_vmid {
6499     return &$complete_vmid_full();
6500 }
6501
6502 sub complete_vmid_stopped {
6503     return &$complete_vmid_full(0);
6504 }
6505
6506 sub complete_vmid_running {
6507     return &$complete_vmid_full(1);
6508 }
6509
6510 sub complete_storage {
6511
6512     my $cfg = PVE::Storage::config();
6513     my $ids = $cfg->{ids};
6514
6515     my $res = [];
6516     foreach my $sid (keys %$ids) {
6517         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6518         next if !$ids->{$sid}->{content}->{images};
6519         push @$res, $sid;
6520     }
6521
6522     return $res;
6523 }
6524
6525 1;