PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QMPClient;
  31 use PVE::RPCEnvironment;
  32 use Time::HiRes qw(gettimeofday);
  33
  34 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  35
  36 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  37
  38 # Note about locking: we use flock on the config file protect
  39 # against concurent actions.
  40 # Aditionaly, we have a 'lock' setting in the config file. This
  41 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  42 # allowed when such lock is set. But you can ignore this kind of
  43 # lock with the --skiplock flag.
  44
  45 cfs_register_file('/qemu-server/',
  46                   \&parse_vm_config,
  47                   \&write_vm_config);
  48
  49 PVE::JSONSchema::register_standard_option('skiplock', {
  50     description => "Ignore locks - only root is allowed to use this option.",
  51     type => 'boolean',
  52     optional => 1,
  53 });
  54
  55 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  56     description => "Some command save/restore state from this location.",
  57     type => 'string',
  58     maxLength => 128,
  59     optional => 1,
  60 });
  61
  62 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  63     description => "The name of the snapshot.",
  64     type => 'string', format => 'pve-configid',
  65     maxLength => 40,
  66 });
  67
  68 #no warnings 'redefine';
  69
  70 sub cgroups_write {
  71    my ($controller, $vmid, $option, $value) = @_;
  72
  73    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  74    PVE::ProcFSTools::write_proc_entry($path, $value);
  75
  76 }
  77
  78 my $nodename = PVE::INotify::nodename();
  79
  80 mkdir "/etc/pve/nodes/$nodename";
  81 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  82 mkdir $confdir;
  83
  84 my $var_run_tmpdir = "/var/run/qemu-server";
  85 mkdir $var_run_tmpdir;
  86
  87 my $lock_dir = "/var/lock/qemu-server";
  88 mkdir $lock_dir;
  89
  90 my $pcisysfs = "/sys/bus/pci";
  91
  92 my $confdesc = {
  93     onboot => {
  94         optional => 1,
  95         type => 'boolean',
  96         description => "Specifies whether a VM will be started during system bootup.",
  97         default => 0,
  98     },
  99     autostart => {
 100         optional => 1,
 101         type => 'boolean',
 102         description => "Automatic restart after crash (currently ignored).",
 103         default => 0,
 104     },
 105     hotplug => {
 106         optional => 1,
 107         type => 'string', format => 'pve-hotplug-features',
 108         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 109         default => 'network,disk,usb',
 110     },
 111     reboot => {
 112         optional => 1,
 113         type => 'boolean',
 114         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 115         default => 1,
 116     },
 117     lock => {
 118         optional => 1,
 119         type => 'string',
 120         description => "Lock/unlock the VM.",
 121         enum => [qw(migrate backup snapshot rollback)],
 122     },
 123     cpulimit => {
 124         optional => 1,
 125         type => 'number',
 126         description => "Limit of CPU usage. Note if the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 127         minimum => 0,
 128         maximum => 128,
 129         default => 0,
 130     },
 131     cpuunits => {
 132         optional => 1,
 133         type => 'integer',
 134         description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 135         minimum => 0,
 136         maximum => 500000,
 137         default => 1000,
 138     },
 139     memory => {
 140         optional => 1,
 141         type => 'integer',
 142         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 143         minimum => 16,
 144         default => 512,
 145     },
 146     balloon => {
 147         optional => 1,
 148         type => 'integer',
 149         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 150         minimum => 0,
 151     },
 152     shares => {
 153         optional => 1,
 154         type => 'integer',
 155         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 156         minimum => 0,
 157         maximum => 50000,
 158         default => 1000,
 159     },
 160     keyboard => {
 161         optional => 1,
 162         type => 'string',
 163         description => "Keybord layout for vnc server. Default is read from the datacenter configuration file.",
 164         enum => PVE::Tools::kvmkeymaplist(),
 165         default => 'en-us',
 166     },
 167     name => {
 168         optional => 1,
 169         type => 'string', format => 'dns-name',
 170         description => "Set a name for the VM. Only used on the configuration web interface.",
 171     },
 172     scsihw => {
 173         optional => 1,
 174         type => 'string',
 175         description => "scsi controller model",
 176         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 177         default => 'lsi',
 178     },
 179     description => {
 180         optional => 1,
 181         type => 'string',
 182         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 183     },
 184     ostype => {
 185         optional => 1,
 186         type => 'string',
 187         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 l24 l26 solaris)],
 188         description => <<EODESC,
 189 Used to enable special optimization/features for specific
 190 operating systems:
 191
 192 other  => unspecified OS
 193 wxp    => Microsoft Windows XP
 194 w2k    => Microsoft Windows 2000
 195 w2k3   => Microsoft Windows 2003
 196 w2k8   => Microsoft Windows 2008
 197 wvista => Microsoft Windows Vista
 198 win7   => Microsoft Windows 7
 199 win8   => Microsoft Windows 8/2012
 200 l24    => Linux 2.4 Kernel
 201 l26    => Linux 2.6/3.X Kernel
 202 solaris => solaris/opensolaris/openindiania kernel
 203
 204 other|l24|l26|solaris                       ... no special behaviour
 205 wxp|w2k|w2k3|w2k8|wvista|win7|win8  ... use --localtime switch
 206 EODESC
 207     },
 208     boot => {
 209         optional => 1,
 210         type => 'string',
 211         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 212         pattern => '[acdn]{1,4}',
 213         default => 'cdn',
 214     },
 215     bootdisk => {
 216         optional => 1,
 217         type => 'string', format => 'pve-qm-bootdisk',
 218         description => "Enable booting from specified disk.",
 219         pattern => '(ide|sata|scsi|virtio)\d+',
 220     },
 221     smp => {
 222         optional => 1,
 223         type => 'integer',
 224         description => "The number of CPUs. Please use option -sockets instead.",
 225         minimum => 1,
 226         default => 1,
 227     },
 228     sockets => {
 229         optional => 1,
 230         type => 'integer',
 231         description => "The number of CPU sockets.",
 232         minimum => 1,
 233         default => 1,
 234     },
 235     cores => {
 236         optional => 1,
 237         type => 'integer',
 238         description => "The number of cores per socket.",
 239         minimum => 1,
 240         default => 1,
 241     },
 242     numa => {
 243         optional => 1,
 244         type => 'boolean',
 245         description => "Enable/disable Numa.",
 246         default => 0,
 247     },
 248     vcpus => {
 249         optional => 1,
 250         type => 'integer',
 251         description => "Number of hotplugged vcpus.",
 252         minimum => 1,
 253         default => 0,
 254     },
 255     acpi => {
 256         optional => 1,
 257         type => 'boolean',
 258         description => "Enable/disable ACPI.",
 259         default => 1,
 260     },
 261     agent => {
 262         optional => 1,
 263         type => 'boolean',
 264         description => "Enable/disable Qemu GuestAgent.",
 265         default => 0,
 266     },
 267     kvm => {
 268         optional => 1,
 269         type => 'boolean',
 270         description => "Enable/disable KVM hardware virtualization.",
 271         default => 1,
 272     },
 273     tdf => {
 274         optional => 1,
 275         type => 'boolean',
 276         description => "Enable/disable time drift fix.",
 277         default => 0,
 278     },
 279     localtime => {
 280         optional => 1,
 281         type => 'boolean',
 282         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 283     },
 284     freeze => {
 285         optional => 1,
 286         type => 'boolean',
 287         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 288     },
 289     vga => {
 290         optional => 1,
 291         type => 'string',
 292         description => "Select VGA type. If you want to use high resolution modes (>= 1280x1024x16) then you should use option 'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and 'cirrur' for other OS types. Option 'qxl' enables the SPICE display sever. You can also run without any graphic card using a serial devive as terminal.",
 293         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 294     },
 295     watchdog => {
 296         optional => 1,
 297         type => 'string', format => 'pve-qm-watchdog',
 298         typetext => '[[model=]i6300esb|ib700] [,[action=]reset|shutdown|poweroff|pause|debug|none]',
 299         description => "Create a virtual hardware watchdog device.  Once enabled (by a guest action), the watchdog must be periodically polled by an agent inside the guest or else the guest will be restarted (or execute the action specified)",
 300     },
 301     startdate => {
 302         optional => 1,
 303         type => 'string',
 304         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 305         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 306         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 307         default => 'now',
 308     },
 309     startup =>  get_standard_option('pve-startup-order'),
 310     template => {
 311         optional => 1,
 312         type => 'boolean',
 313         description => "Enable/disable Template.",
 314         default => 0,
 315     },
 316     args => {
 317         optional => 1,
 318         type => 'string',
 319         description => <<EODESCR,
 320 Note: this option is for experts only. It allows you to pass arbitrary arguments to kvm, for example:
 321
 322 args: -no-reboot -no-hpet
 323 EODESCR
 324     },
 325     tablet => {
 326         optional => 1,
 327         type => 'boolean',
 328         default => 1,
 329         description => "Enable/disable the usb tablet device. This device is usually needed to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with normal VNC clients. If you're running lots of console-only guests on one host, you may consider disabling this to save some context switches. This is turned of by default if you use spice (vga=qxl).",
 330     },
 331     migrate_speed => {
 332         optional => 1,
 333         type => 'integer',
 334         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 335         minimum => 0,
 336         default => 0,
 337     },
 338     migrate_downtime => {
 339         optional => 1,
 340         type => 'number',
 341         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 342         minimum => 0,
 343         default => 0.1,
 344     },
 345     cdrom => {
 346         optional => 1,
 347         type => 'string', format => 'pve-qm-drive',
 348         typetext => 'volume',
 349         description => "This is an alias for option -ide2",
 350     },
 351     cpu => {
 352         optional => 1,
 353         description => "Emulated CPU type.",
 354         type => 'string',
 355         enum => [ qw(486 athlon pentium pentium2 pentium3 coreduo core2duo kvm32 kvm64 qemu32 qemu64 phenom Conroe Penryn Nehalem Westmere SandyBridge IvyBridge Haswell Broadwell Opteron_G1 Opteron_G2 Opteron_G3 Opteron_G4 Opteron_G5 host) ],
 356         default => 'kvm64',
 357     },
 358     parent => get_standard_option('pve-snapshot-name', {
 359         optional => 1,
 360         description => "Parent snapshot name. This is used internally, and should not be modified.",
 361     }),
 362     snaptime => {
 363         optional => 1,
 364         description => "Timestamp for snapshots.",
 365         type => 'integer',
 366         minimum => 0,
 367     },
 368     vmstate => {
 369         optional => 1,
 370         type => 'string', format => 'pve-volume-id',
 371         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 372     },
 373     machine => {
 374         description => "Specific the Qemu machine type.",
 375         type => 'string',
 376         pattern => '(pc|pc(-i440fx)?-\d+\.\d+|q35|pc-q35-\d+\.\d+)',
 377         maxLength => 40,
 378         optional => 1,
 379     },
 380     smbios1 => {
 381         description => "Specify SMBIOS type 1 fields.",
 382         type => 'string', format => 'pve-qm-smbios1',
 383         typetext => "[manufacturer=str][,product=str][,version=str][,serial=str] [,uuid=uuid][,sku=str][,family=str]",
 384         maxLength => 256,
 385         optional => 1,
 386     },
 387     protection => {
 388         optional => 1,
 389         type => 'boolean',
 390         description => "Sets the protection flag of the VM. This will prevent the remove operation.",
 391         default => 0,
 392     },
 393 };
 394
 395 # what about other qemu settings ?
 396 #cpu => 'string',
 397 #machine => 'string',
 398 #fda => 'file',
 399 #fdb => 'file',
 400 #mtdblock => 'file',
 401 #sd => 'file',
 402 #pflash => 'file',
 403 #snapshot => 'bool',
 404 #bootp => 'file',
 405 ##tftp => 'dir',
 406 ##smb => 'dir',
 407 #kernel => 'file',
 408 #append => 'string',
 409 #initrd => 'file',
 410 ##soundhw => 'string',
 411
 412 while (my ($k, $v) = each %$confdesc) {
 413     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 414 }
 415
 416 my $MAX_IDE_DISKS = 4;
 417 my $MAX_SCSI_DISKS = 14;
 418 my $MAX_VIRTIO_DISKS = 16;
 419 my $MAX_SATA_DISKS = 6;
 420 my $MAX_USB_DEVICES = 5;
 421 my $MAX_NETS = 32;
 422 my $MAX_UNUSED_DISKS = 8;
 423 my $MAX_HOSTPCI_DEVICES = 4;
 424 my $MAX_SERIAL_PORTS = 4;
 425 my $MAX_PARALLEL_PORTS = 3;
 426 my $MAX_NUMA = 8;
 427 my $MAX_MEM = 4194304;
 428 my $STATICMEM = 1024;
 429
 430 my $numadesc = {
 431     optional => 1,
 432     type => 'string', format => 'pve-qm-numanode',
 433     typetext => "cpus=<id[-id],memory=<mb>[[,hostnodes=<id[-id]>] [,policy=<preferred|bind|interleave>]]",
 434     description => "numa topology",
 435 };
 436 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 437
 438 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 439     $confdesc->{"numa$i"} = $numadesc;
 440 }
 441
 442 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 443                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 444                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 445 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 446
 447 my $netdesc = {
 448     optional => 1,
 449     type => 'string', format => 'pve-qm-net',
 450     typetext => "MODEL=XX:XX:XX:XX:XX:XX [,bridge=<dev>][,queues=<nbqueues>][,rate=<mbps>] [,tag=<vlanid>][,firewall=0|1],link_down=0|1]",
 451     description => <<EODESCR,
 452 Specify network devices.
 453
 454 MODEL is one of: $nic_model_list_txt
 455
 456 XX:XX:XX:XX:XX:XX should be an unique MAC address. This is
 457 automatically generated if not specified.
 458
 459 The bridge parameter can be used to automatically add the interface to a bridge device. The Proxmox VE standard bridge is called 'vmbr0'.
 460
 461 Option 'rate' is used to limit traffic bandwidth from and to this interface. It is specified as floating point number, unit is 'Megabytes per second'.
 462
 463 If you specify no bridge, we create a kvm 'user' (NATed) network device, which provides DHCP and DNS services. The following addresses are used:
 464
 465 10.0.2.2   Gateway
 466 10.0.2.3   DNS Server
 467 10.0.2.4   SMB Server
 468
 469 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 470
 471 EODESCR
 472 };
 473 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 474
 475 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 476     $confdesc->{"net$i"} = $netdesc;
 477 }
 478
 479 my $drivename_hash;
 480
 481 my $idedesc = {
 482     optional => 1,
 483     type => 'string', format => 'pve-qm-drive',
 484     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads] [,discard=ignore|on] [,serial=serial][,model=model]',
 485     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
 486 };
 487 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
 488
 489 my $scsidesc = {
 490     optional => 1,
 491     type => 'string', format => 'pve-qm-drive',
 492     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads] [,discard=ignore|on] [,iothread=on] [,queues=<nbqueues>] [,serial=serial]',
 493     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
 494 };
 495 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 496
 497 my $satadesc = {
 498     optional => 1,
 499     type => 'string', format => 'pve-qm-drive',
 500     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads]  [,discard=ignore|on] [,serial=serial]',
 501     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
 502 };
 503 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
 504
 505 my $virtiodesc = {
 506     optional => 1,
 507     type => 'string', format => 'pve-qm-drive',
 508     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads]  [,discard=ignore|on] [,iothread=on] [,serial=serial]',
 509     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
 510 };
 511 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
 512
 513 my $usbdesc = {
 514     optional => 1,
 515     type => 'string', format => 'pve-qm-usb-device',
 516     typetext => 'host=HOSTUSBDEVICE|spice',
 517     description => <<EODESCR,
 518 Configure an USB device (n is 0 to 4). This can be used to
 519 pass-through usb devices to the guest. HOSTUSBDEVICE syntax is:
 520
 521 'bus-port(.port)*' (decimal numbers) or
 522 'vendor_id:product_id' (hexadeciaml numbers)
 523
 524 You can use the 'lsusb -t' command to list existing usb devices.
 525
 526 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 527
 528 The value 'spice' can be used to add a usb redirection devices for spice.
 529
 530 EODESCR
 531 };
 532 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
 533
 534 my $hostpcidesc = {
 535         optional => 1,
 536         type => 'string', format => 'pve-qm-hostpci',
 537         typetext => "[host=]HOSTPCIDEVICE [,rombar=on|off] [,pcie=0|1] [,x-vga=on|off]",
 538         description => <<EODESCR,
 539 Map host pci devices. HOSTPCIDEVICE syntax is:
 540
 541 'bus:dev.func' (hexadecimal numbers)
 542
 543 You can us the 'lspci' command to list existing pci devices.
 544
 545 The 'rombar' option determines whether or not the device's ROM will be visible in the guest's memory map (default is 'on').
 546
 547 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 548
 549 Experimental: user reported problems with this option.
 550 EODESCR
 551 };
 552 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
 553
 554 my $serialdesc = {
 555         optional => 1,
 556         type => 'string',
 557         pattern => '(/dev/.+|socket)',
 558         description =>  <<EODESCR,
 559 Create a serial device inside the VM (n is 0 to 3), and pass through a host serial device (i.e. /dev/ttyS0), or create a unix socket on the host side (use 'qm terminal' to open a terminal connection).
 560
 561 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 562
 563 Experimental: user reported problems with this option.
 564 EODESCR
 565 };
 566
 567 my $paralleldesc= {
 568         optional => 1,
 569         type => 'string',
 570         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
 571         description =>  <<EODESCR,
 572 Map host parallel devices (n is 0 to 2).
 573
 574 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 575
 576 Experimental: user reported problems with this option.
 577 EODESCR
 578 };
 579
 580 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
 581     $confdesc->{"parallel$i"} = $paralleldesc;
 582 }
 583
 584 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
 585     $confdesc->{"serial$i"} = $serialdesc;
 586 }
 587
 588 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
 589     $confdesc->{"hostpci$i"} = $hostpcidesc;
 590 }
 591
 592 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
 593     $drivename_hash->{"ide$i"} = 1;
 594     $confdesc->{"ide$i"} = $idedesc;
 595 }
 596
 597 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
 598     $drivename_hash->{"sata$i"} = 1;
 599     $confdesc->{"sata$i"} = $satadesc;
 600 }
 601
 602 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
 603     $drivename_hash->{"scsi$i"} = 1;
 604     $confdesc->{"scsi$i"} = $scsidesc ;
 605 }
 606
 607 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
 608     $drivename_hash->{"virtio$i"} = 1;
 609     $confdesc->{"virtio$i"} = $virtiodesc;
 610 }
 611
 612 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
 613     $confdesc->{"usb$i"} = $usbdesc;
 614 }
 615
 616 my $unuseddesc = {
 617     optional => 1,
 618     type => 'string', format => 'pve-volume-id',
 619     description => "Reference to unused volumes.",
 620 };
 621
 622 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
 623     $confdesc->{"unused$i"} = $unuseddesc;
 624 }
 625
 626 my $kvm_api_version = 0;
 627
 628 sub kvm_version {
 629
 630     return $kvm_api_version if $kvm_api_version;
 631
 632     my $fh = IO::File->new("</dev/kvm") ||
 633         return 0;
 634
 635     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
 636         $kvm_api_version = $v;
 637     }
 638
 639     $fh->close();
 640
 641     return  $kvm_api_version;
 642 }
 643
 644 my $kvm_user_version;
 645
 646 sub kvm_user_version {
 647
 648     return $kvm_user_version if $kvm_user_version;
 649
 650     $kvm_user_version = 'unknown';
 651
 652     my $tmp = `kvm -help 2>/dev/null`;
 653
 654     if ($tmp =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
 655         $kvm_user_version = $2;
 656     }
 657
 658     return $kvm_user_version;
 659
 660 }
 661
 662 my $kernel_has_vhost_net = -c '/dev/vhost-net';
 663
 664 sub disknames {
 665     # order is important - used to autoselect boot disk
 666     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
 667             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
 668             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
 669             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))));
 670 }
 671
 672 sub valid_drivename {
 673     my $dev = shift;
 674
 675     return defined($drivename_hash->{$dev});
 676 }
 677
 678 sub option_exists {
 679     my $key = shift;
 680     return defined($confdesc->{$key});
 681 }
 682
 683 sub nic_models {
 684     return $nic_model_list;
 685 }
 686
 687 sub os_list_description {
 688
 689     return {
 690         other => 'Other',
 691         wxp => 'Windows XP',
 692         w2k => 'Windows 2000',
 693         w2k3 =>, 'Windows 2003',
 694         w2k8 => 'Windows 2008',
 695         wvista => 'Windows Vista',
 696         win7 => 'Windows 7',
 697         win8 => 'Windows 8/2012',
 698         l24 => 'Linux 2.4',
 699         l26 => 'Linux 2.6',
 700     };
 701 }
 702
 703 my $cdrom_path;
 704
 705 sub get_cdrom_path {
 706
 707     return  $cdrom_path if $cdrom_path;
 708
 709     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
 710     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
 711     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
 712 }
 713
 714 sub get_iso_path {
 715     my ($storecfg, $vmid, $cdrom) = @_;
 716
 717     if ($cdrom eq 'cdrom') {
 718         return get_cdrom_path();
 719     } elsif ($cdrom eq 'none') {
 720         return '';
 721     } elsif ($cdrom =~ m|^/|) {
 722         return $cdrom;
 723     } else {
 724         return PVE::Storage::path($storecfg, $cdrom);
 725     }
 726 }
 727
 728 # try to convert old style file names to volume IDs
 729 sub filename_to_volume_id {
 730     my ($vmid, $file, $media) = @_;
 731
 732     if (!($file eq 'none' || $file eq 'cdrom' ||
 733           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
 734
 735         return undef if $file =~ m|/|;
 736
 737         if ($media && $media eq 'cdrom') {
 738             $file = "local:iso/$file";
 739         } else {
 740             $file = "local:$vmid/$file";
 741         }
 742     }
 743
 744     return $file;
 745 }
 746
 747 sub verify_media_type {
 748     my ($opt, $vtype, $media) = @_;
 749
 750     return if !$media;
 751
 752     my $etype;
 753     if ($media eq 'disk') {
 754         $etype = 'images';
 755     } elsif ($media eq 'cdrom') {
 756         $etype = 'iso';
 757     } else {
 758         die "internal error";
 759     }
 760
 761     return if ($vtype eq $etype);
 762
 763     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
 764 }
 765
 766 sub cleanup_drive_path {
 767     my ($opt, $storecfg, $drive) = @_;
 768
 769     # try to convert filesystem paths to volume IDs
 770
 771     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
 772         ($drive->{file} !~ m|^/dev/.+|) &&
 773         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
 774         ($drive->{file} !~ m/^\d+$/)) {
 775         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
 776         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
 777         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
 778         verify_media_type($opt, $vtype, $drive->{media});
 779         $drive->{file} = $volid;
 780     }
 781
 782     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
 783 }
 784
 785 sub create_conf_nolock {
 786     my ($vmid, $settings) = @_;
 787
 788     my $filename = config_file($vmid);
 789
 790     die "configuration file '$filename' already exists\n" if -f $filename;
 791
 792     my $defaults = load_defaults();
 793
 794     $settings->{name} = "vm$vmid" if !$settings->{name};
 795     $settings->{memory} = $defaults->{memory} if !$settings->{memory};
 796
 797     my $data = '';
 798     foreach my $opt (keys %$settings) {
 799         next if !$confdesc->{$opt};
 800
 801         my $value = $settings->{$opt};
 802         next if !$value;
 803
 804         $data .= "$opt: $value\n";
 805     }
 806
 807     PVE::Tools::file_set_contents($filename, $data);
 808 }
 809
 810 sub parse_hotplug_features {
 811     my ($data) = @_;
 812
 813     my $res = {};
 814
 815     return $res if $data eq '0';
 816
 817     $data = $confdesc->{hotplug}->{default} if $data eq '1';
 818
 819     foreach my $feature (PVE::Tools::split_list($data)) {
 820         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
 821             $res->{$1} = 1;
 822         } else {
 823             warn "ignoring unknown hotplug feature '$feature'\n";
 824         }
 825     }
 826     return $res;
 827 }
 828
 829 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
 830 sub pve_verify_hotplug_features {
 831     my ($value, $noerr) = @_;
 832
 833     return $value if parse_hotplug_features($value);
 834
 835     return undef if $noerr;
 836
 837     die "unable to parse hotplug option\n";
 838 }
 839
 840 my $parse_size = sub {
 841     my ($value) = @_;
 842
 843     return undef if $value !~ m/^(\d+(\.\d+)?)([KMG])?$/;
 844     my ($size, $unit) = ($1, $3);
 845     if ($unit) {
 846         if ($unit eq 'K') {
 847             $size = $size * 1024;
 848         } elsif ($unit eq 'M') {
 849             $size = $size * 1024 * 1024;
 850         } elsif ($unit eq 'G') {
 851             $size = $size * 1024 * 1024 * 1024;
 852         }
 853     }
 854     return int($size);
 855 };
 856
 857 my $format_size = sub {
 858     my ($size) = @_;
 859
 860     $size = int($size);
 861
 862     my $kb = int($size/1024);
 863     return $size if $kb*1024 != $size;
 864
 865     my $mb = int($kb/1024);
 866     return "${kb}K" if $mb*1024 != $kb;
 867
 868     my $gb = int($mb/1024);
 869     return "${mb}M" if $gb*1024 != $mb;
 870
 871     return "${gb}G";
 872 };
 873
 874 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
 875 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
 876 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
 877 #        [,aio=native|threads][,discard=ignore|on][,iothread=on]
 878 #        [,serial=serial][,model=model]
 879
 880 sub parse_drive {
 881     my ($key, $data) = @_;
 882
 883     my $res = {};
 884
 885     # $key may be undefined - used to verify JSON parameters
 886     if (!defined($key)) {
 887         $res->{interface} = 'unknown'; # should not harm when used to verify parameters
 888         $res->{index} = 0;
 889     } elsif ($key =~ m/^([^\d]+)(\d+)$/) {
 890         $res->{interface} = $1;
 891         $res->{index} = $2;
 892     } else {
 893         return undef;
 894     }
 895
 896     foreach my $p (split (/,/, $data)) {
 897         next if $p =~ m/^\s*$/;
 898
 899         if ($p =~ m/^(file|volume|cyls|heads|secs|trans|media|snapshot|cache|format|rerror|werror|backup|aio|bps|mbps|mbps_max|bps_rd|mbps_rd|mbps_rd_max|bps_wr|mbps_wr|mbps_wr_max|iops|iops_max|iops_rd|iops_rd_max|iops_wr|iops_wr_max|size|discard|iothread|queues|serial|model)=(.+)$/) {
 900             my ($k, $v) = ($1, $2);
 901
 902             $k = 'file' if $k eq 'volume';
 903
 904             return undef if defined $res->{$k};
 905
 906             if ($k eq 'bps' || $k eq 'bps_rd' || $k eq 'bps_wr') {
 907                 return undef if !$v || $v !~ m/^\d+/;
 908                 $k = "m$k";
 909                 $v = sprintf("%.3f", $v / (1024*1024));
 910             }
 911             $res->{$k} = $v;
 912         } else {
 913             if (!$res->{file} && $p !~ m/=/) {
 914                 $res->{file} = $p;
 915             } else {
 916                 return undef;
 917             }
 918         }
 919     }
 920
 921     return undef if !$res->{file};
 922
 923     return undef if $res->{cache} &&
 924         $res->{cache} !~ m/^(off|none|writethrough|writeback|unsafe|directsync)$/;
 925     return undef if $res->{snapshot} && $res->{snapshot} !~ m/^(on|off)$/;
 926     return undef if $res->{cyls} && $res->{cyls} !~ m/^\d+$/;
 927     return undef if $res->{heads} && $res->{heads} !~ m/^\d+$/;
 928     return undef if $res->{secs} && $res->{secs} !~ m/^\d+$/;
 929     return undef if $res->{media} && $res->{media} !~ m/^(disk|cdrom)$/;
 930     return undef if $res->{trans} && $res->{trans} !~ m/^(none|lba|auto)$/;
 931     return undef if $res->{format} && $res->{format} !~ m/^(raw|cow|qcow|qed|qcow2|vmdk|cloop)$/;
 932     return undef if $res->{rerror} && $res->{rerror} !~ m/^(ignore|report|stop)$/;
 933     return undef if $res->{werror} && $res->{werror} !~ m/^(enospc|ignore|report|stop)$/;
 934     return undef if $res->{backup} && $res->{backup} !~ m/^(yes|no)$/;
 935     return undef if $res->{aio} && $res->{aio} !~ m/^(native|threads)$/;
 936     return undef if $res->{discard} && $res->{discard} !~ m/^(ignore|on)$/;
 937     return undef if $res->{iothread} && $res->{iothread} !~ m/^(on)$/;
 938     return undef if $res->{queues} && ($res->{queues} !~ m/^\d+$/ || $res->{queues} < 2);
 939
 940     return undef if $res->{mbps_rd} && $res->{mbps};
 941     return undef if $res->{mbps_wr} && $res->{mbps};
 942
 943     return undef if $res->{mbps} && $res->{mbps} !~ m/^\d+(\.\d+)?$/;
 944     return undef if $res->{mbps_max} && $res->{mbps_max} !~ m/^\d+(\.\d+)?$/;
 945     return undef if $res->{mbps_rd} && $res->{mbps_rd} !~ m/^\d+(\.\d+)?$/;
 946     return undef if $res->{mbps_rd_max} && $res->{mbps_rd_max} !~ m/^\d+(\.\d+)?$/;
 947     return undef if $res->{mbps_wr} && $res->{mbps_wr} !~ m/^\d+(\.\d+)?$/;
 948     return undef if $res->{mbps_wr_max} && $res->{mbps_wr_max} !~ m/^\d+(\.\d+)?$/;
 949
 950     return undef if $res->{iops_rd} && $res->{iops};
 951     return undef if $res->{iops_wr} && $res->{iops};
 952
 953
 954     return undef if $res->{iops} && $res->{iops} !~ m/^\d+$/;
 955     return undef if $res->{iops_max} && $res->{iops_max} !~ m/^\d+$/;
 956     return undef if $res->{iops_rd} && $res->{iops_rd} !~ m/^\d+$/;
 957     return undef if $res->{iops_rd_max} && $res->{iops_rd_max} !~ m/^\d+$/;
 958     return undef if $res->{iops_wr} && $res->{iops_wr} !~ m/^\d+$/;
 959     return undef if $res->{iops_wr_max} && $res->{iops_wr_max} !~ m/^\d+$/;
 960
 961     if ($res->{size}) {
 962         return undef if !defined($res->{size} = &$parse_size($res->{size}));
 963     }
 964
 965     if ($res->{media} && ($res->{media} eq 'cdrom')) {
 966         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
 967         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
 968         return undef if $res->{interface} eq 'virtio';
 969     }
 970
 971     # rerror does not work with scsi drives
 972     if ($res->{rerror}) {
 973         return undef if $res->{interface} eq 'scsi';
 974     }
 975
 976     return $res;
 977 }
 978
 979 my @qemu_drive_options = qw(heads secs cyls trans media format cache snapshot rerror werror aio discard iops iops_rd iops_wr iops_max iops_rd_max iops_wr_max serial);
 980
 981 sub print_drive {
 982     my ($vmid, $drive) = @_;
 983
 984     my $opts = '';
 985     foreach my $o (@qemu_drive_options, 'mbps', 'mbps_rd', 'mbps_wr', 'mbps_max', 'mbps_rd_max', 'mbps_wr_max', 'backup', 'iothread', 'queues') {
 986         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
 987     }
 988
 989     if ($drive->{size}) {
 990         $opts .= ",size=" . &$format_size($drive->{size});
 991     }
 992
 993     if (my $model = $drive->{model}) {
 994         $opts .= ",model=$model";
 995     }
 996
 997     return "$drive->{file}$opts";
 998 }
 999
1000 sub scsi_inquiry {
1001     my($fh, $noerr) = @_;
1002
1003     my $SG_IO = 0x2285;
1004     my $SG_GET_VERSION_NUM = 0x2282;
1005
1006     my $versionbuf = "\x00" x 8;
1007     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1008     if (!$ret) {
1009         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1010         return undef;
1011     }
1012     my $version = unpack("I", $versionbuf);
1013     if ($version < 30000) {
1014         die "scsi generic interface too old\n"  if !$noerr;
1015         return undef;
1016     }
1017
1018     my $buf = "\x00" x 36;
1019     my $sensebuf = "\x00" x 8;
1020     my $cmd = pack("C x3 C x1", 0x12, 36);
1021
1022     # see /usr/include/scsi/sg.h
1023     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1024
1025     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1026                       length($sensebuf), 0, length($buf), $buf,
1027                       $cmd, $sensebuf, 6000);
1028
1029     $ret = ioctl($fh, $SG_IO, $packet);
1030     if (!$ret) {
1031         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1032         return undef;
1033     }
1034
1035     my @res = unpack($sg_io_hdr_t, $packet);
1036     if ($res[17] || $res[18]) {
1037         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1038         return undef;
1039     }
1040
1041     my $res = {};
1042     (my $byte0, my $byte1, $res->{vendor},
1043      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1044
1045     $res->{removable} = $byte1 & 128 ? 1 : 0;
1046     $res->{type} = $byte0 & 31;
1047
1048     return $res;
1049 }
1050
1051 sub path_is_scsi {
1052     my ($path) = @_;
1053
1054     my $fh = IO::File->new("+<$path") || return undef;
1055     my $res = scsi_inquiry($fh, 1);
1056     close($fh);
1057
1058     return $res;
1059 }
1060
1061 sub machine_type_is_q35 {
1062     my ($conf) = @_;
1063
1064     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1065 }
1066
1067 sub print_tabletdevice_full {
1068     my ($conf) = @_;
1069
1070     my $q35 = machine_type_is_q35($conf);
1071
1072     # we use uhci for old VMs because tablet driver was buggy in older qemu
1073     my $usbbus = $q35 ? "ehci" : "uhci";
1074
1075     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1076 }
1077
1078 sub print_drivedevice_full {
1079     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1080
1081     my $device = '';
1082     my $maxdev = 0;
1083
1084     if ($drive->{interface} eq 'virtio') {
1085         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1086         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1087         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1088     } elsif ($drive->{interface} eq 'scsi') {
1089
1090         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1091         my $unit = $drive->{index} % $maxdev;
1092         my $devicetype = 'hd';
1093         my $path = '';
1094         if (drive_is_cdrom($drive)) {
1095               $devicetype = 'cd';
1096         } else {
1097               if ($drive->{file} =~ m|^/|) {
1098                   $path = $drive->{file};
1099               } else {
1100                   $path = PVE::Storage::path($storecfg, $drive->{file});
1101               }
1102
1103               if($path =~ m/^iscsi\:\/\//){
1104                   $devicetype = 'generic';
1105               } else {
1106                   if (my $info = path_is_scsi($path)) {
1107                       if ($info->{type} == 0) {
1108                           $devicetype = 'block';
1109                       } elsif ($info->{type} == 1) { # tape
1110                           $devicetype = 'generic';
1111                       }
1112                   }
1113               }
1114          }
1115
1116         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1117             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1118         } else {
1119             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1120         }
1121
1122     } elsif ($drive->{interface} eq 'ide'){
1123         $maxdev = 2;
1124         my $controller = int($drive->{index} / $maxdev);
1125         my $unit = $drive->{index} % $maxdev;
1126         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1127
1128         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1129         if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
1130             $device .= ",model=$model";
1131         }
1132     } elsif ($drive->{interface} eq 'sata'){
1133         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1134         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1135         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1136     } elsif ($drive->{interface} eq 'usb') {
1137         die "implement me";
1138         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1139     } else {
1140         die "unsupported interface type";
1141     }
1142
1143     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1144
1145     return $device;
1146 }
1147
1148 sub get_initiator_name {
1149     my $initiator;
1150
1151     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1152     while (defined(my $line = <$fh>)) {
1153         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1154         $initiator = $1;
1155         last;
1156     }
1157     $fh->close();
1158
1159     return $initiator;
1160 }
1161
1162 sub print_drive_full {
1163     my ($storecfg, $vmid, $drive) = @_;
1164
1165     my $path;
1166     my $volid = $drive->{file};
1167     my $format;
1168
1169     if (drive_is_cdrom($drive)) {
1170         $path = get_iso_path($storecfg, $vmid, $volid);
1171     } else {
1172         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1173         if ($storeid) {
1174             $path = PVE::Storage::path($storecfg, $volid);
1175             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1176             $format = qemu_img_format($scfg, $volname);
1177         } else {
1178             $path = $volid;
1179         }
1180    }
1181
1182     my $opts = '';
1183     foreach my $o (@qemu_drive_options) {
1184         next if $o eq 'bootindex';
1185         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1186     }
1187
1188     $opts .= ",format=$format" if $format && !$drive->{format};
1189
1190     foreach my $o (qw(bps bps_rd bps_wr)) {
1191         my $v = $drive->{"m$o"};
1192         $opts .= ",$o=" . int($v*1024*1024) if $v;
1193     }
1194
1195     my $cache_direct = 0;
1196
1197     if (my $cache = $drive->{cache}) {
1198         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1199     } elsif (!drive_is_cdrom($drive)) {
1200         $opts .= ",cache=none";
1201         $cache_direct = 1;
1202     }
1203
1204     # aio native works only with O_DIRECT
1205     if (!$drive->{aio}) {
1206         if($cache_direct) {
1207             $opts .= ",aio=native";
1208         } else {
1209             $opts .= ",aio=threads";
1210         }
1211     }
1212
1213     my $detectzeroes = $drive->{discard} ? "unmap" : "on";
1214     $opts .= ",detect-zeroes=$detectzeroes" if !drive_is_cdrom($drive);
1215
1216     my $pathinfo = $path ? "file=$path," : '';
1217
1218     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1219 }
1220
1221 sub print_netdevice_full {
1222     my ($vmid, $conf, $net, $netid, $bridges) = @_;
1223
1224     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1225
1226     my $device = $net->{model};
1227     if ($net->{model} eq 'virtio') {
1228          $device = 'virtio-net-pci';
1229      };
1230
1231     my $pciaddr = print_pci_addr("$netid", $bridges);
1232     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1233     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1234         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1235         my $vectors = $net->{queues} * 2 + 2;
1236         $tmpstr .= ",vectors=$vectors,mq=on";
1237     }
1238     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1239     return $tmpstr;
1240 }
1241
1242 sub print_netdev_full {
1243     my ($vmid, $conf, $net, $netid) = @_;
1244
1245     my $i = '';
1246     if ($netid =~ m/^net(\d+)$/) {
1247         $i = int($1);
1248     }
1249
1250     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1251
1252     my $ifname = "tap${vmid}i$i";
1253
1254     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1255     die "interface name '$ifname' is too long (max 15 character)\n"
1256         if length($ifname) >= 16;
1257
1258     my $vhostparam = '';
1259     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1260
1261     my $vmname = $conf->{name} || "vm$vmid";
1262
1263     my $netdev = "";
1264
1265     if ($net->{bridge}) {
1266         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/pve-bridge,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1267     } else {
1268         $netdev = "type=user,id=$netid,hostname=$vmname";
1269     }
1270
1271     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1272
1273     return $netdev;
1274 }
1275
1276 sub drive_is_cdrom {
1277     my ($drive) = @_;
1278
1279     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1280
1281 }
1282
1283 sub parse_numa {
1284     my ($data) = @_;
1285
1286     my $res = {};
1287
1288     foreach my $kvp (split(/,/, $data)) {
1289
1290         if ($kvp =~ m/^memory=(\S+)$/) {
1291             $res->{memory} = $1;
1292         } elsif ($kvp =~ m/^policy=(preferred|bind|interleave)$/) {
1293             $res->{policy} = $1;
1294         } elsif ($kvp =~ m/^cpus=(\d+)(-(\d+))?$/) {
1295             $res->{cpus}->{start} = $1;
1296             $res->{cpus}->{end} = $3;
1297         } elsif ($kvp =~ m/^hostnodes=(\d+)(-(\d+))?$/) {
1298             $res->{hostnodes}->{start} = $1;
1299             $res->{hostnodes}->{end} = $3;
1300         } else {
1301             return undef;
1302         }
1303     }
1304
1305     return $res;
1306 }
1307
1308 sub parse_hostpci {
1309     my ($value) = @_;
1310
1311     return undef if !$value;
1312
1313
1314     my @list = split(/,/, $value);
1315     my $found;
1316
1317     my $res = {};
1318     foreach my $kv (@list) {
1319
1320         if ($kv =~ m/^(host=)?([a-f0-9]{2}:[a-f0-9]{2})(\.([a-f0-9]))?$/) {
1321             $found = 1;
1322             if(defined($4)){
1323                 push @{$res->{pciid}}, { id => $2 , function => $4};
1324
1325             }else{
1326                 my $pcidevices = lspci($2);
1327                 $res->{pciid} = $pcidevices->{$2};
1328             }
1329         } elsif ($kv =~ m/^rombar=(on|off)$/) {
1330             $res->{rombar} = $1;
1331         } elsif ($kv =~ m/^x-vga=(on|off)$/) {
1332             $res->{'x-vga'} = $1;
1333         } elsif ($kv =~ m/^pcie=(\d+)$/) {
1334             $res->{pcie} = 1 if $1 == 1;
1335         } else {
1336             warn "unknown hostpci setting '$kv'\n";
1337         }
1338     }
1339
1340     return undef if !$found;
1341
1342     return $res;
1343 }
1344
1345 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1346 sub parse_net {
1347     my ($data) = @_;
1348
1349     my $res = {};
1350
1351     foreach my $kvp (split(/,/, $data)) {
1352
1353         if ($kvp =~ m/^(ne2k_pci|e1000|e1000-82540em|e1000-82544gc|e1000-82545em|rtl8139|pcnet|virtio|ne2k_isa|i82551|i82557b|i82559er|vmxnet3)(=([0-9a-f]{2}(:[0-9a-f]{2}){5}))?$/i) {
1354             my $model = lc($1);
1355             my $mac = defined($3) ? uc($3) : PVE::Tools::random_ether_addr();
1356             $res->{model} = $model;
1357             $res->{macaddr} = $mac;
1358         } elsif ($kvp =~ m/^bridge=(\S+)$/) {
1359             $res->{bridge} = $1;
1360         } elsif ($kvp =~ m/^queues=(\d+)$/) {
1361             $res->{queues} = $1;
1362         } elsif ($kvp =~ m/^rate=(\d+(\.\d+)?)$/) {
1363             $res->{rate} = $1;
1364         } elsif ($kvp =~ m/^tag=(\d+)$/) {
1365             $res->{tag} = $1;
1366         } elsif ($kvp =~ m/^firewall=([01])$/) {
1367             $res->{firewall} = $1;
1368         } elsif ($kvp =~ m/^link_down=([01])$/) {
1369             $res->{link_down} = $1;
1370         } else {
1371             return undef;
1372         }
1373
1374     }
1375
1376     return undef if !$res->{model};
1377
1378     return $res;
1379 }
1380
1381 sub print_net {
1382     my $net = shift;
1383
1384     my $res = "$net->{model}";
1385     $res .= "=$net->{macaddr}" if $net->{macaddr};
1386     $res .= ",bridge=$net->{bridge}" if $net->{bridge};
1387     $res .= ",rate=$net->{rate}" if $net->{rate};
1388     $res .= ",tag=$net->{tag}" if $net->{tag};
1389     $res .= ",firewall=1" if $net->{firewall};
1390     $res .= ",link_down=1" if $net->{link_down};
1391     $res .= ",queues=$net->{queues}" if $net->{queues};
1392
1393     return $res;
1394 }
1395
1396 sub add_random_macs {
1397     my ($settings) = @_;
1398
1399     foreach my $opt (keys %$settings) {
1400         next if $opt !~ m/^net(\d+)$/;
1401         my $net = parse_net($settings->{$opt});
1402         next if !$net;
1403         $settings->{$opt} = print_net($net);
1404     }
1405 }
1406
1407 sub add_unused_volume {
1408     my ($config, $volid) = @_;
1409
1410     my $key;
1411     for (my $ind = $MAX_UNUSED_DISKS - 1; $ind >= 0; $ind--) {
1412         my $test = "unused$ind";
1413         if (my $vid = $config->{$test}) {
1414             return if $vid eq $volid; # do not add duplicates
1415         } else {
1416             $key = $test;
1417         }
1418     }
1419
1420     die "To many unused volume - please delete them first.\n" if !$key;
1421
1422     $config->{$key} = $volid;
1423
1424     return $key;
1425 }
1426
1427 sub vm_is_volid_owner {
1428     my ($storecfg, $vmid, $volid) = @_;
1429
1430     if ($volid !~  m|^/|) {
1431         my ($path, $owner);
1432         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1433         if ($owner && ($owner == $vmid)) {
1434             return 1;
1435         }
1436     }
1437
1438     return undef;
1439 }
1440
1441 sub split_flagged_list {
1442     my $text = shift || '';
1443     $text =~ s/[,;]/ /g;
1444     $text =~ s/^\s+//;
1445     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
1446 }
1447
1448 sub join_flagged_list {
1449     my ($how, $lst) = @_;
1450     join $how, map { $lst->{$_} . $_ } keys %$lst;
1451 }
1452
1453 sub vmconfig_delete_pending_option {
1454     my ($conf, $key, $force) = @_;
1455
1456     delete $conf->{pending}->{$key};
1457     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1458     $pending_delete_hash->{$key} = $force ? '!' : '';
1459     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1460 }
1461
1462 sub vmconfig_undelete_pending_option {
1463     my ($conf, $key) = @_;
1464
1465     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1466     delete $pending_delete_hash->{$key};
1467
1468     if (%$pending_delete_hash) {
1469         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1470     } else {
1471         delete $conf->{pending}->{delete};
1472     }
1473 }
1474
1475 sub vmconfig_register_unused_drive {
1476     my ($storecfg, $vmid, $conf, $drive) = @_;
1477
1478     if (!drive_is_cdrom($drive)) {
1479         my $volid = $drive->{file};
1480         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1481             add_unused_volume($conf, $volid, $vmid);
1482         }
1483     }
1484 }
1485
1486 sub vmconfig_cleanup_pending {
1487     my ($conf) = @_;
1488
1489     # remove pending changes when nothing changed
1490     my $changes;
1491     foreach my $opt (keys %{$conf->{pending}}) {
1492         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
1493             $changes = 1;
1494             delete $conf->{pending}->{$opt};
1495         }
1496     }
1497
1498     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
1499     my $pending_delete_hash = {};
1500     while (my ($opt, $force) = each %$current_delete_hash) {
1501         if (defined($conf->{$opt})) {
1502             $pending_delete_hash->{$opt} = $force;
1503         } else {
1504             $changes = 1;
1505         }
1506     }
1507
1508     if (%$pending_delete_hash) {
1509         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1510     } else {
1511         delete $conf->{pending}->{delete};
1512     }
1513
1514     return $changes;
1515 }
1516
1517 my $valid_smbios1_options = {
1518     manufacturer => '\S+',
1519     product => '\S+',
1520     version => '\S+',
1521     serial => '\S+',
1522     uuid => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1523     sku => '\S+',
1524     family => '\S+',
1525 };
1526
1527 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
1528 sub parse_smbios1 {
1529     my ($data) = @_;
1530
1531     my $res = {};
1532
1533     foreach my $kvp (split(/,/, $data)) {
1534         return undef if $kvp !~ m/^(\S+)=(.+)$/;
1535         my ($k, $v) = split(/=/, $kvp);
1536         return undef if !defined($k) || !defined($v);
1537         return undef if !$valid_smbios1_options->{$k};
1538         return undef if $v !~ m/^$valid_smbios1_options->{$k}$/;
1539         $res->{$k} = $v;
1540     }
1541
1542     return $res;
1543 }
1544
1545 sub print_smbios1 {
1546     my ($smbios1) = @_;
1547
1548     my $data = '';
1549     foreach my $k (keys %$smbios1) {
1550         next if !defined($smbios1->{$k});
1551         next if !$valid_smbios1_options->{$k};
1552         $data .= ',' if $data;
1553         $data .= "$k=$smbios1->{$k}";
1554     }
1555     return $data;
1556 }
1557
1558 PVE::JSONSchema::register_format('pve-qm-smbios1', \&verify_smbios1);
1559 sub verify_smbios1 {
1560     my ($value, $noerr) = @_;
1561
1562     return $value if parse_smbios1($value);
1563
1564     return undef if $noerr;
1565
1566     die "unable to parse smbios (type 1) options\n";
1567 }
1568
1569 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
1570 sub verify_bootdisk {
1571     my ($value, $noerr) = @_;
1572
1573     return $value if valid_drivename($value);
1574
1575     return undef if $noerr;
1576
1577     die "invalid boot disk '$value'\n";
1578 }
1579
1580 PVE::JSONSchema::register_format('pve-qm-numanode', \&verify_numa);
1581 sub verify_numa {
1582     my ($value, $noerr) = @_;
1583
1584     return $value if parse_numa($value);
1585
1586     return undef if $noerr;
1587
1588     die "unable to parse numa options\n";
1589 }
1590
1591 PVE::JSONSchema::register_format('pve-qm-net', \&verify_net);
1592 sub verify_net {
1593     my ($value, $noerr) = @_;
1594
1595     return $value if parse_net($value);
1596
1597     return undef if $noerr;
1598
1599     die "unable to parse network options\n";
1600 }
1601
1602 PVE::JSONSchema::register_format('pve-qm-drive', \&verify_drive);
1603 sub verify_drive {
1604     my ($value, $noerr) = @_;
1605
1606     return $value if parse_drive(undef, $value);
1607
1608     return undef if $noerr;
1609
1610     die "unable to parse drive options\n";
1611 }
1612
1613 PVE::JSONSchema::register_format('pve-qm-hostpci', \&verify_hostpci);
1614 sub verify_hostpci {
1615     my ($value, $noerr) = @_;
1616
1617     return $value if parse_hostpci($value);
1618
1619     return undef if $noerr;
1620
1621     die "unable to parse pci id\n";
1622 }
1623
1624 PVE::JSONSchema::register_format('pve-qm-watchdog', \&verify_watchdog);
1625 sub verify_watchdog {
1626     my ($value, $noerr) = @_;
1627
1628     return $value if parse_watchdog($value);
1629
1630     return undef if $noerr;
1631
1632     die "unable to parse watchdog options\n";
1633 }
1634
1635 sub parse_watchdog {
1636     my ($value) = @_;
1637
1638     return undef if !$value;
1639
1640     my $res = {};
1641
1642     foreach my $p (split(/,/, $value)) {
1643         next if $p =~ m/^\s*$/;
1644
1645         if ($p =~ m/^(model=)?(i6300esb|ib700)$/) {
1646             $res->{model} = $2;
1647         } elsif ($p =~ m/^(action=)?(reset|shutdown|poweroff|pause|debug|none)$/) {
1648             $res->{action} = $2;
1649         } else {
1650             return undef;
1651         }
1652     }
1653
1654     return $res;
1655 }
1656
1657 sub parse_usb_device {
1658     my ($value) = @_;
1659
1660     return undef if !$value;
1661
1662     my @dl = split(/,/, $value);
1663     my $found;
1664
1665     my $res = {};
1666     foreach my $v (@dl) {
1667         if ($v =~ m/^host=(0x)?([0-9A-Fa-f]{4}):(0x)?([0-9A-Fa-f]{4})$/) {
1668             $found = 1;
1669             $res->{vendorid} = $2;
1670             $res->{productid} = $4;
1671         } elsif ($v =~ m/^host=(\d+)\-(\d+(\.\d+)*)$/) {
1672             $found = 1;
1673             $res->{hostbus} = $1;
1674             $res->{hostport} = $2;
1675         } elsif ($v =~ m/^spice$/) {
1676             $found = 1;
1677             $res->{spice} = 1;
1678         } else {
1679             return undef;
1680         }
1681     }
1682     return undef if !$found;
1683
1684     return $res;
1685 }
1686
1687 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
1688 sub verify_usb_device {
1689     my ($value, $noerr) = @_;
1690
1691     return $value if parse_usb_device($value);
1692
1693     return undef if $noerr;
1694
1695     die "unable to parse usb device\n";
1696 }
1697
1698 # add JSON properties for create and set function
1699 sub json_config_properties {
1700     my $prop = shift;
1701
1702     foreach my $opt (keys %$confdesc) {
1703         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
1704         $prop->{$opt} = $confdesc->{$opt};
1705     }
1706
1707     return $prop;
1708 }
1709
1710 sub check_type {
1711     my ($key, $value) = @_;
1712
1713     die "unknown setting '$key'\n" if !$confdesc->{$key};
1714
1715     my $type = $confdesc->{$key}->{type};
1716
1717     if (!defined($value)) {
1718         die "got undefined value\n";
1719     }
1720
1721     if ($value =~ m/[\n\r]/) {
1722         die "property contains a line feed\n";
1723     }
1724
1725     if ($type eq 'boolean') {
1726         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
1727         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
1728         die "type check ('boolean') failed - got '$value'\n";
1729     } elsif ($type eq 'integer') {
1730         return int($1) if $value =~ m/^(\d+)$/;
1731         die "type check ('integer') failed - got '$value'\n";
1732     } elsif ($type eq 'number') {
1733         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
1734         die "type check ('number') failed - got '$value'\n";
1735     } elsif ($type eq 'string') {
1736         if (my $fmt = $confdesc->{$key}->{format}) {
1737             if ($fmt eq 'pve-qm-drive') {
1738                 # special case - we need to pass $key to parse_drive()
1739                 my $drive = parse_drive($key, $value);
1740                 return $value if $drive;
1741                 die "unable to parse drive options\n";
1742             }
1743             PVE::JSONSchema::check_format($fmt, $value);
1744             return $value;
1745         }
1746         $value =~ s/^\"(.*)\"$/$1/;
1747         return $value;
1748     } else {
1749         die "internal error"
1750     }
1751 }
1752
1753 sub lock_config_full {
1754     my ($vmid, $timeout, $code, @param) = @_;
1755
1756     my $filename = config_file_lock($vmid);
1757
1758     my $res = lock_file($filename, $timeout, $code, @param);
1759
1760     die $@ if $@;
1761
1762     return $res;
1763 }
1764
1765 sub lock_config_mode {
1766     my ($vmid, $timeout, $shared, $code, @param) = @_;
1767
1768     my $filename = config_file_lock($vmid);
1769
1770     my $res = lock_file_full($filename, $timeout, $shared, $code, @param);
1771
1772     die $@ if $@;
1773
1774     return $res;
1775 }
1776
1777 sub lock_config {
1778     my ($vmid, $code, @param) = @_;
1779
1780     return lock_config_full($vmid, 10, $code, @param);
1781 }
1782
1783 sub cfs_config_path {
1784     my ($vmid, $node) = @_;
1785
1786     $node = $nodename if !$node;
1787     return "nodes/$node/qemu-server/$vmid.conf";
1788 }
1789
1790 sub check_iommu_support{
1791     #fixme : need to check IOMMU support
1792     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
1793
1794     my $iommu=1;
1795     return $iommu;
1796
1797 }
1798
1799 sub config_file {
1800     my ($vmid, $node) = @_;
1801
1802     my $cfspath = cfs_config_path($vmid, $node);
1803     return "/etc/pve/$cfspath";
1804 }
1805
1806 sub config_file_lock {
1807     my ($vmid) = @_;
1808
1809     return "$lock_dir/lock-$vmid.conf";
1810 }
1811
1812 sub touch_config {
1813     my ($vmid) = @_;
1814
1815     my $conf = config_file($vmid);
1816     utime undef, undef, $conf;
1817 }
1818
1819 sub destroy_vm {
1820     my ($storecfg, $vmid, $keep_empty_config) = @_;
1821
1822     my $conffile = config_file($vmid);
1823
1824     my $conf = load_config($vmid);
1825
1826     check_lock($conf);
1827
1828     # only remove disks owned by this VM
1829     foreach_drive($conf, sub {
1830         my ($ds, $drive) = @_;
1831
1832         return if drive_is_cdrom($drive);
1833
1834         my $volid = $drive->{file};
1835
1836         return if !$volid || $volid =~ m|^/|;
1837
1838         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
1839         return if !$path || !$owner || ($owner != $vmid);
1840
1841         PVE::Storage::vdisk_free($storecfg, $volid);
1842     });
1843
1844     if ($keep_empty_config) {
1845         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
1846     } else {
1847         unlink $conffile;
1848     }
1849
1850     # also remove unused disk
1851     eval {
1852         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
1853
1854         eval {
1855             PVE::Storage::foreach_volid($dl, sub {
1856                 my ($volid, $sid, $volname, $d) = @_;
1857                 PVE::Storage::vdisk_free($storecfg, $volid);
1858             });
1859         };
1860         warn $@ if $@;
1861
1862     };
1863     warn $@ if $@;
1864 }
1865
1866 sub load_config {
1867     my ($vmid, $node) = @_;
1868
1869     my $cfspath = cfs_config_path($vmid, $node);
1870
1871     my $conf = PVE::Cluster::cfs_read_file($cfspath);
1872
1873     die "no such VM ('$vmid')\n" if !defined($conf);
1874
1875     return $conf;
1876 }
1877
1878 sub parse_vm_config {
1879     my ($filename, $raw) = @_;
1880
1881     return undef if !defined($raw);
1882
1883     my $res = {
1884         digest => Digest::SHA::sha1_hex($raw),
1885         snapshots => {},
1886         pending => {},
1887     };
1888
1889     $filename =~ m|/qemu-server/(\d+)\.conf$|
1890         || die "got strange filename '$filename'";
1891
1892     my $vmid = $1;
1893
1894     my $conf = $res;
1895     my $descr;
1896     my $section = '';
1897
1898     my @lines = split(/\n/, $raw);
1899     foreach my $line (@lines) {
1900         next if $line =~ m/^\s*$/;
1901
1902         if ($line =~ m/^\[PENDING\]\s*$/i) {
1903             $section = 'pending';
1904             if (defined($descr)) {
1905                 $descr =~ s/\s+$//;
1906                 $conf->{description} = $descr;
1907             }
1908             $descr = undef;
1909             $conf = $res->{$section} = {};
1910             next;
1911
1912         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
1913             $section = $1;
1914             if (defined($descr)) {
1915                 $descr =~ s/\s+$//;
1916                 $conf->{description} = $descr;
1917             }
1918             $descr = undef;
1919             $conf = $res->{snapshots}->{$section} = {};
1920             next;
1921         }
1922
1923         if ($line =~ m/^\#(.*)\s*$/) {
1924             $descr = '' if !defined($descr);
1925             $descr .= PVE::Tools::decode_text($1) . "\n";
1926             next;
1927         }
1928
1929         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
1930             $descr = '' if !defined($descr);
1931             $descr .= PVE::Tools::decode_text($2);
1932         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
1933             $conf->{snapstate} = $1;
1934         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
1935             my $key = $1;
1936             my $value = $2;
1937             $conf->{$key} = $value;
1938         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
1939             my $value = $1;
1940             if ($section eq 'pending') {
1941                 $conf->{delete} = $value; # we parse this later
1942             } else {
1943                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
1944             }
1945         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
1946             my $key = $1;
1947             my $value = $2;
1948             eval { $value = check_type($key, $value); };
1949             if ($@) {
1950                 warn "vm $vmid - unable to parse value of '$key' - $@";
1951             } else {
1952                 my $fmt = $confdesc->{$key}->{format};
1953                 if ($fmt && $fmt eq 'pve-qm-drive') {
1954                     my $v = parse_drive($key, $value);
1955                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
1956                         $v->{file} = $volid;
1957                         $value = print_drive($vmid, $v);
1958                     } else {
1959                         warn "vm $vmid - unable to parse value of '$key'\n";
1960                         next;
1961                     }
1962                 }
1963
1964                 if ($key eq 'cdrom') {
1965                     $conf->{ide2} = $value;
1966                 } else {
1967                     $conf->{$key} = $value;
1968                 }
1969             }
1970         }
1971     }
1972
1973     if (defined($descr)) {
1974         $descr =~ s/\s+$//;
1975         $conf->{description} = $descr;
1976     }
1977     delete $res->{snapstate}; # just to be sure
1978
1979     return $res;
1980 }
1981
1982 sub write_vm_config {
1983     my ($filename, $conf) = @_;
1984
1985     delete $conf->{snapstate}; # just to be sure
1986
1987     if ($conf->{cdrom}) {
1988         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
1989         $conf->{ide2} = $conf->{cdrom};
1990         delete $conf->{cdrom};
1991     }
1992
1993     # we do not use 'smp' any longer
1994     if ($conf->{sockets}) {
1995         delete $conf->{smp};
1996     } elsif ($conf->{smp}) {
1997         $conf->{sockets} = $conf->{smp};
1998         delete $conf->{cores};
1999         delete $conf->{smp};
2000     }
2001
2002     my $used_volids = {};
2003
2004     my $cleanup_config = sub {
2005         my ($cref, $pending, $snapname) = @_;
2006
2007         foreach my $key (keys %$cref) {
2008             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2009                 $key eq 'snapstate' || $key eq 'pending';
2010             my $value = $cref->{$key};
2011             if ($key eq 'delete') {
2012                 die "propertry 'delete' is only allowed in [PENDING]\n"
2013                     if !$pending;
2014                 # fixme: check syntax?
2015                 next;
2016             }
2017             eval { $value = check_type($key, $value); };
2018             die "unable to parse value of '$key' - $@" if $@;
2019
2020             $cref->{$key} = $value;
2021
2022             if (!$snapname && valid_drivename($key)) {
2023                 my $drive = parse_drive($key, $value);
2024                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2025             }
2026         }
2027     };
2028
2029     &$cleanup_config($conf);
2030
2031     &$cleanup_config($conf->{pending}, 1);
2032
2033     foreach my $snapname (keys %{$conf->{snapshots}}) {
2034         die "internal error" if $snapname eq 'pending';
2035         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2036     }
2037
2038     # remove 'unusedX' settings if we re-add a volume
2039     foreach my $key (keys %$conf) {
2040         my $value = $conf->{$key};
2041         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2042             delete $conf->{$key};
2043         }
2044     }
2045
2046     my $generate_raw_config = sub {
2047         my ($conf, $pending) = @_;
2048
2049         my $raw = '';
2050
2051         # add description as comment to top of file
2052         if (defined(my $descr = $conf->{description})) {
2053             if ($descr) {
2054                 foreach my $cl (split(/\n/, $descr)) {
2055                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2056                 }
2057             } else {
2058                 $raw .= "#\n" if $pending;
2059             }
2060         }
2061
2062         foreach my $key (sort keys %$conf) {
2063             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2064             $raw .= "$key: $conf->{$key}\n";
2065         }
2066         return $raw;
2067     };
2068
2069     my $raw = &$generate_raw_config($conf);
2070
2071     if (scalar(keys %{$conf->{pending}})){
2072         $raw .= "\n[PENDING]\n";
2073         $raw .= &$generate_raw_config($conf->{pending}, 1);
2074     }
2075
2076     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2077         $raw .= "\n[$snapname]\n";
2078         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2079     }
2080
2081     return $raw;
2082 }
2083
2084 sub update_config_nolock {
2085     my ($vmid, $conf, $skiplock) = @_;
2086
2087     check_lock($conf) if !$skiplock;
2088
2089     my $cfspath = cfs_config_path($vmid);
2090
2091     PVE::Cluster::cfs_write_file($cfspath, $conf);
2092 }
2093
2094 sub update_config {
2095     my ($vmid, $conf, $skiplock) = @_;
2096
2097     lock_config($vmid, &update_config_nolock, $conf, $skiplock);
2098 }
2099
2100 sub load_defaults {
2101
2102     my $res = {};
2103
2104     # we use static defaults from our JSON schema configuration
2105     foreach my $key (keys %$confdesc) {
2106         if (defined(my $default = $confdesc->{$key}->{default})) {
2107             $res->{$key} = $default;
2108         }
2109     }
2110
2111     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2112     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2113
2114     return $res;
2115 }
2116
2117 sub config_list {
2118     my $vmlist = PVE::Cluster::get_vmlist();
2119     my $res = {};
2120     return $res if !$vmlist || !$vmlist->{ids};
2121     my $ids = $vmlist->{ids};
2122
2123     foreach my $vmid (keys %$ids) {
2124         my $d = $ids->{$vmid};
2125         next if !$d->{node} || $d->{node} ne $nodename;
2126         next if !$d->{type} || $d->{type} ne 'qemu';
2127         $res->{$vmid}->{exists} = 1;
2128     }
2129     return $res;
2130 }
2131
2132 # test if VM uses local resources (to prevent migration)
2133 sub check_local_resources {
2134     my ($conf, $noerr) = @_;
2135
2136     my $loc_res = 0;
2137
2138     $loc_res = 1 if $conf->{hostusb}; # old syntax
2139     $loc_res = 1 if $conf->{hostpci}; # old syntax
2140
2141     foreach my $k (keys %$conf) {
2142         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2143         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2144     }
2145
2146     die "VM uses local resources\n" if $loc_res && !$noerr;
2147
2148     return $loc_res;
2149 }
2150
2151 # check if used storages are available on all nodes (use by migrate)
2152 sub check_storage_availability {
2153     my ($storecfg, $conf, $node) = @_;
2154
2155     foreach_drive($conf, sub {
2156         my ($ds, $drive) = @_;
2157
2158         my $volid = $drive->{file};
2159         return if !$volid;
2160
2161         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2162         return if !$sid;
2163
2164         # check if storage is available on both nodes
2165         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2166         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2167    });
2168 }
2169
2170 # list nodes where all VM images are available (used by has_feature API)
2171 sub shared_nodes {
2172     my ($conf, $storecfg) = @_;
2173
2174     my $nodelist = PVE::Cluster::get_nodelist();
2175     my $nodehash = { map { $_ => 1 } @$nodelist };
2176     my $nodename = PVE::INotify::nodename();
2177
2178     foreach_drive($conf, sub {
2179         my ($ds, $drive) = @_;
2180
2181         my $volid = $drive->{file};
2182         return if !$volid;
2183
2184         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2185         if ($storeid) {
2186             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2187             if ($scfg->{disable}) {
2188                 $nodehash = {};
2189             } elsif (my $avail = $scfg->{nodes}) {
2190                 foreach my $node (keys %$nodehash) {
2191                     delete $nodehash->{$node} if !$avail->{$node};
2192                 }
2193             } elsif (!$scfg->{shared}) {
2194                 foreach my $node (keys %$nodehash) {
2195                     delete $nodehash->{$node} if $node ne $nodename
2196                 }
2197             }
2198         }
2199     });
2200
2201     return $nodehash
2202 }
2203
2204 sub check_lock {
2205     my ($conf) = @_;
2206
2207     die "VM is locked ($conf->{lock})\n" if $conf->{lock};
2208 }
2209
2210 sub check_cmdline {
2211     my ($pidfile, $pid) = @_;
2212
2213     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2214     if (defined($fh)) {
2215         my $line = <$fh>;
2216         $fh->close;
2217         return undef if !$line;
2218         my @param = split(/\0/, $line);
2219
2220         my $cmd = $param[0];
2221         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2222
2223         for (my $i = 0; $i < scalar (@param); $i++) {
2224             my $p = $param[$i];
2225             next if !$p;
2226             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2227                 my $p = $param[$i+1];
2228                 return 1 if $p && ($p eq $pidfile);
2229                 return undef;
2230             }
2231         }
2232     }
2233     return undef;
2234 }
2235
2236 sub check_running {
2237     my ($vmid, $nocheck, $node) = @_;
2238
2239     my $filename = config_file($vmid, $node);
2240
2241     die "unable to find configuration file for VM $vmid - no such machine\n"
2242         if !$nocheck && ! -f $filename;
2243
2244     my $pidfile = pidfile_name($vmid);
2245
2246     if (my $fd = IO::File->new("<$pidfile")) {
2247         my $st = stat($fd);
2248         my $line = <$fd>;
2249         close($fd);
2250
2251         my $mtime = $st->mtime;
2252         if ($mtime > time()) {
2253             warn "file '$filename' modified in future\n";
2254         }
2255
2256         if ($line =~ m/^(\d+)$/) {
2257             my $pid = $1;
2258             if (check_cmdline($pidfile, $pid)) {
2259                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2260                     return $pid;
2261                 }
2262             }
2263         }
2264     }
2265
2266     return undef;
2267 }
2268
2269 sub vzlist {
2270
2271     my $vzlist = config_list();
2272
2273     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2274
2275     while (defined(my $de = $fd->read)) {
2276         next if $de !~ m/^(\d+)\.pid$/;
2277         my $vmid = $1;
2278         next if !defined($vzlist->{$vmid});
2279         if (my $pid = check_running($vmid)) {
2280             $vzlist->{$vmid}->{pid} = $pid;
2281         }
2282     }
2283
2284     return $vzlist;
2285 }
2286
2287 sub disksize {
2288     my ($storecfg, $conf) = @_;
2289
2290     my $bootdisk = $conf->{bootdisk};
2291     return undef if !$bootdisk;
2292     return undef if !valid_drivename($bootdisk);
2293
2294     return undef if !$conf->{$bootdisk};
2295
2296     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2297     return undef if !defined($drive);
2298
2299     return undef if drive_is_cdrom($drive);
2300
2301     my $volid = $drive->{file};
2302     return undef if !$volid;
2303
2304     return $drive->{size};
2305 }
2306
2307 my $last_proc_pid_stat;
2308
2309 # get VM status information
2310 # This must be fast and should not block ($full == false)
2311 # We only query KVM using QMP if $full == true (this can be slow)
2312 sub vmstatus {
2313     my ($opt_vmid, $full) = @_;
2314
2315     my $res = {};
2316
2317     my $storecfg = PVE::Storage::config();
2318
2319     my $list = vzlist();
2320     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2321
2322     my $cpucount = $cpuinfo->{cpus} || 1;
2323
2324     foreach my $vmid (keys %$list) {
2325         next if $opt_vmid && ($vmid ne $opt_vmid);
2326
2327         my $cfspath = cfs_config_path($vmid);
2328         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2329
2330         my $d = {};
2331         $d->{pid} = $list->{$vmid}->{pid};
2332
2333         # fixme: better status?
2334         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2335
2336         my $size = disksize($storecfg, $conf);
2337         if (defined($size)) {
2338             $d->{disk} = 0; # no info available
2339             $d->{maxdisk} = $size;
2340         } else {
2341             $d->{disk} = 0;
2342             $d->{maxdisk} = 0;
2343         }
2344
2345         $d->{cpus} = ($conf->{sockets} || 1) * ($conf->{cores} || 1);
2346         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2347         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2348
2349         $d->{name} = $conf->{name} || "VM $vmid";
2350         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024) : 0;
2351
2352         if ($conf->{balloon}) {
2353             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2354             $d->{shares} = defined($conf->{shares}) ? $conf->{shares} : 1000;
2355         }
2356
2357         $d->{uptime} = 0;
2358         $d->{cpu} = 0;
2359         $d->{mem} = 0;
2360
2361         $d->{netout} = 0;
2362         $d->{netin} = 0;
2363
2364         $d->{diskread} = 0;
2365         $d->{diskwrite} = 0;
2366
2367         $d->{template} = is_template($conf);
2368
2369         $res->{$vmid} = $d;
2370     }
2371
2372     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2373     foreach my $dev (keys %$netdev) {
2374         next if $dev !~ m/^tap([1-9]\d*)i/;
2375         my $vmid = $1;
2376         my $d = $res->{$vmid};
2377         next if !$d;
2378
2379         $d->{netout} += $netdev->{$dev}->{receive};
2380         $d->{netin} += $netdev->{$dev}->{transmit};
2381
2382         if ($full) {
2383             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2384             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2385         }
2386
2387     }
2388
2389     my $ctime = gettimeofday;
2390
2391     foreach my $vmid (keys %$list) {
2392
2393         my $d = $res->{$vmid};
2394         my $pid = $d->{pid};
2395         next if !$pid;
2396
2397         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2398         next if !$pstat; # not running
2399
2400         my $used = $pstat->{utime} + $pstat->{stime};
2401
2402         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2403
2404         if ($pstat->{vsize}) {
2405             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2406         }
2407
2408         my $old = $last_proc_pid_stat->{$pid};
2409         if (!$old) {
2410             $last_proc_pid_stat->{$pid} = {
2411                 time => $ctime,
2412                 used => $used,
2413                 cpu => 0,
2414             };
2415             next;
2416         }
2417
2418         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2419
2420         if ($dtime > 1000) {
2421             my $dutime = $used -  $old->{used};
2422
2423             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2424             $last_proc_pid_stat->{$pid} = {
2425                 time => $ctime,
2426                 used => $used,
2427                 cpu => $d->{cpu},
2428             };
2429         } else {
2430             $d->{cpu} = $old->{cpu};
2431         }
2432     }
2433
2434     return $res if !$full;
2435
2436     my $qmpclient = PVE::QMPClient->new();
2437
2438     my $ballooncb = sub {
2439         my ($vmid, $resp) = @_;
2440
2441         my $info = $resp->{'return'};
2442         return if !$info->{max_mem};
2443
2444         my $d = $res->{$vmid};
2445
2446         # use memory assigned to VM
2447         $d->{maxmem} = $info->{max_mem};
2448         $d->{balloon} = $info->{actual};
2449
2450         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2451             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2452             $d->{freemem} = $info->{free_mem};
2453         }
2454
2455         $d->{ballooninfo} = $info;
2456     };
2457
2458     my $blockstatscb = sub {
2459         my ($vmid, $resp) = @_;
2460         my $data = $resp->{'return'} || [];
2461         my $totalrdbytes = 0;
2462         my $totalwrbytes = 0;
2463
2464         for my $blockstat (@$data) {
2465             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2466             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2467
2468             $blockstat->{device} =~ s/drive-//;
2469             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2470         }
2471         $res->{$vmid}->{diskread} = $totalrdbytes;
2472         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2473     };
2474
2475     my $statuscb = sub {
2476         my ($vmid, $resp) = @_;
2477
2478         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2479         # this fails if ballon driver is not loaded, so this must be
2480         # the last commnand (following command are aborted if this fails).
2481         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2482
2483         my $status = 'unknown';
2484         if (!defined($status = $resp->{'return'}->{status})) {
2485             warn "unable to get VM status\n";
2486             return;
2487         }
2488
2489         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2490     };
2491
2492     foreach my $vmid (keys %$list) {
2493         next if $opt_vmid && ($vmid ne $opt_vmid);
2494         next if !$res->{$vmid}->{pid}; # not running
2495         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2496     }
2497
2498     $qmpclient->queue_execute(undef, 1);
2499
2500     foreach my $vmid (keys %$list) {
2501         next if $opt_vmid && ($vmid ne $opt_vmid);
2502         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2503     }
2504
2505     return $res;
2506 }
2507
2508 sub foreach_dimm {
2509     my ($conf, $vmid, $memory, $sockets, $func) = @_;
2510
2511     my $dimm_id = 0;
2512     my $current_size = 1024;
2513     my $dimm_size = 512;
2514     return if $current_size == $memory;
2515
2516     for (my $j = 0; $j < 8; $j++) {
2517         for (my $i = 0; $i < 32; $i++) {
2518             my $name = "dimm${dimm_id}";
2519             $dimm_id++;
2520             my $numanode = $i % $sockets;
2521             $current_size += $dimm_size;
2522             &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
2523             return  $current_size if $current_size >= $memory;
2524         }
2525         $dimm_size *= 2;
2526     }
2527 }
2528
2529 sub foreach_reverse_dimm {
2530     my ($conf, $vmid, $memory, $sockets, $func) = @_;
2531
2532     my $dimm_id = 253;
2533     my $current_size = 4177920;
2534     my $dimm_size = 65536;
2535     return if $current_size == $memory;
2536
2537     for (my $j = 0; $j < 8; $j++) {
2538         for (my $i = 0; $i < 32; $i++) {
2539             my $name = "dimm${dimm_id}";
2540             $dimm_id--;
2541             my $numanode = $i % $sockets;
2542             $current_size -= $dimm_size;
2543             &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
2544             return  $current_size if $current_size <= $memory;
2545         }
2546         $dimm_size /= 2;
2547     }
2548 }
2549
2550 sub foreach_drive {
2551     my ($conf, $func) = @_;
2552
2553     foreach my $ds (keys %$conf) {
2554         next if !valid_drivename($ds);
2555
2556         my $drive = parse_drive($ds, $conf->{$ds});
2557         next if !$drive;
2558
2559         &$func($ds, $drive);
2560     }
2561 }
2562
2563 sub foreach_volid {
2564     my ($conf, $func) = @_;
2565
2566     my $volhash = {};
2567
2568     my $test_volid = sub {
2569         my ($volid, $is_cdrom) = @_;
2570
2571         return if !$volid;
2572
2573         $volhash->{$volid} = $is_cdrom || 0;
2574     };
2575
2576     foreach_drive($conf, sub {
2577         my ($ds, $drive) = @_;
2578         &$test_volid($drive->{file}, drive_is_cdrom($drive));
2579     });
2580
2581     foreach my $snapname (keys %{$conf->{snapshots}}) {
2582         my $snap = $conf->{snapshots}->{$snapname};
2583         &$test_volid($snap->{vmstate}, 0);
2584         foreach_drive($snap, sub {
2585             my ($ds, $drive) = @_;
2586             &$test_volid($drive->{file}, drive_is_cdrom($drive));
2587         });
2588     }
2589
2590     foreach my $volid (keys %$volhash) {
2591         &$func($volid, $volhash->{$volid});
2592     }
2593 }
2594
2595 sub vga_conf_has_spice {
2596     my ($vga) = @_;
2597
2598     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
2599
2600     return $1 || 1;
2601 }
2602
2603 sub config_to_command {
2604     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2605
2606     my $cmd = [];
2607     my $globalFlags = [];
2608     my $machineFlags = [];
2609     my $rtcFlags = [];
2610     my $cpuFlags = [];
2611     my $devices = [];
2612     my $pciaddr = '';
2613     my $bridges = {};
2614     my $kvmver = kvm_user_version();
2615     my $vernum = 0; # unknown
2616     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
2617         $vernum = $1*1000000+$2*1000;
2618     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
2619         $vernum = $1*1000000+$2*1000+$3;
2620     }
2621
2622     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
2623
2624     my $have_ovz = -f '/proc/vz/vestat';
2625
2626     my $q35 = machine_type_is_q35($conf);
2627     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2628     my $machine_type = $forcemachine || $conf->{machine};
2629
2630     my $cpuunits = defined($conf->{cpuunits}) ?
2631             $conf->{cpuunits} : $defaults->{cpuunits};
2632
2633     push @$cmd, '/usr/bin/systemd-run';
2634     push @$cmd, '--scope';
2635     push @$cmd, '--slice', "qemu";
2636     push @$cmd, '--unit', $vmid;
2637     push @$cmd, '-p', "CPUShares=$cpuunits";
2638     if ($conf->{cpulimit}) {
2639         my $cpulimit = int($conf->{cpulimit} * 100);
2640         push @$cmd, '-p', "CPUQuota=$cpulimit\%";
2641     }
2642
2643     push @$cmd, '/usr/bin/kvm';
2644
2645     push @$cmd, '-id', $vmid;
2646
2647     my $use_virtio = 0;
2648
2649     my $qmpsocket = qmp_socket($vmid);
2650     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2651     push @$cmd, '-mon', "chardev=qmp,mode=control";
2652
2653     my $socket = vnc_socket($vmid);
2654     push @$cmd,  '-vnc', "unix:$socket,x509,password";
2655
2656     push @$cmd, '-pidfile' , pidfile_name($vmid);
2657
2658     push @$cmd, '-daemonize';
2659
2660     if ($conf->{smbios1}) {
2661         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
2662     }
2663
2664     if ($q35) {
2665         # the q35 chipset support native usb2, so we enable usb controller
2666         # by default for this machine type
2667         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
2668     } else {
2669         $pciaddr = print_pci_addr("piix3", $bridges);
2670         push @$devices, '-device', "piix3-usb-uhci,id=uhci$pciaddr.0x2";
2671
2672         my $use_usb2 = 0;
2673         for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2674             next if !$conf->{"usb$i"};
2675             $use_usb2 = 1;
2676         }
2677         # include usb device config
2678         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-usb.cfg' if $use_usb2;
2679     }
2680
2681     my $vga = $conf->{vga};
2682
2683     my $qxlnum = vga_conf_has_spice($vga);
2684     $vga = 'qxl' if $qxlnum;
2685
2686     if (!$vga) {
2687         if ($conf->{ostype} && ($conf->{ostype} eq 'win8' ||
2688                                 $conf->{ostype} eq 'win7' ||
2689                                 $conf->{ostype} eq 'w2k8')) {
2690             $vga = 'std';
2691         } else {
2692             $vga = 'cirrus';
2693         }
2694     }
2695
2696     # enable absolute mouse coordinates (needed by vnc)
2697     my $tablet;
2698     if (defined($conf->{tablet})) {
2699         $tablet = $conf->{tablet};
2700     } else {
2701         $tablet = $defaults->{tablet};
2702         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
2703         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
2704     }
2705
2706     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
2707
2708     # host pci devices
2709     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
2710         my $d = parse_hostpci($conf->{"hostpci$i"});
2711         next if !$d;
2712
2713         my $pcie = $d->{pcie};
2714         if($pcie){
2715             die "q35 machine model is not enabled" if !$q35;
2716             $pciaddr = print_pcie_addr("hostpci$i");
2717         }else{
2718             $pciaddr = print_pci_addr("hostpci$i", $bridges);
2719         }
2720
2721         my $rombar = $d->{rombar} && $d->{rombar} eq 'off' ? ",rombar=0" : "";
2722         my $xvga = $d->{'x-vga'} && $d->{'x-vga'} eq 'on' ? ",x-vga=on" : "";
2723         if ($xvga && $xvga ne '') {
2724             push @$cpuFlags, 'kvm=off';
2725             $vga = 'none';
2726         }
2727         my $pcidevices = $d->{pciid};
2728         my $multifunction = 1 if @$pcidevices > 1;
2729
2730         my $j=0;
2731         foreach my $pcidevice (@$pcidevices) {
2732
2733             my $id = "hostpci$i";
2734             $id .= ".$j" if $multifunction;
2735             my $addr = $pciaddr;
2736             $addr .= ".$j" if $multifunction;
2737             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
2738
2739             if($j == 0){
2740                 $devicestr .= "$rombar$xvga";
2741                 $devicestr .= ",multifunction=on" if $multifunction;
2742             }
2743
2744             push @$devices, '-device', $devicestr;
2745             $j++;
2746         }
2747     }
2748
2749     # usb devices
2750     for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2751         my $d = parse_usb_device($conf->{"usb$i"});
2752         next if !$d;
2753         if ($d->{vendorid} && $d->{productid}) {
2754             push @$devices, '-device', "usb-host,vendorid=0x$d->{vendorid},productid=0x$d->{productid}";
2755         } elsif (defined($d->{hostbus}) && defined($d->{hostport})) {
2756             push @$devices, '-device', "usb-host,hostbus=$d->{hostbus},hostport=$d->{hostport}";
2757         } elsif ($d->{spice}) {
2758             # usb redir support for spice
2759             push @$devices, '-chardev', "spicevmc,id=usbredirchardev$i,name=usbredir";
2760             push @$devices, '-device', "usb-redir,chardev=usbredirchardev$i,id=usbredirdev$i,bus=ehci.0";
2761         }
2762     }
2763
2764     # serial devices
2765     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2766         if (my $path = $conf->{"serial$i"}) {
2767             if ($path eq 'socket') {
2768                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
2769                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
2770                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2771             } else {
2772                 die "no such serial device\n" if ! -c $path;
2773                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
2774                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2775             }
2776         }
2777     }
2778
2779     # parallel devices
2780     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
2781         if (my $path = $conf->{"parallel$i"}) {
2782             die "no such parallel device\n" if ! -c $path;
2783             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
2784             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
2785             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
2786         }
2787     }
2788
2789     my $vmname = $conf->{name} || "vm$vmid";
2790
2791     push @$cmd, '-name', $vmname;
2792
2793     my $sockets = 1;
2794     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
2795     $sockets = $conf->{sockets} if  $conf->{sockets};
2796
2797     my $cores = $conf->{cores} || 1;
2798
2799     my $maxcpus = $sockets * $cores;
2800
2801     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
2802
2803     my $allowed_vcpus = $cpuinfo->{cpus};
2804
2805     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
2806         if ($allowed_vcpus < $maxcpus);
2807
2808     push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
2809
2810     push @$cmd, '-nodefaults';
2811
2812     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2813
2814     my $bootindex_hash = {};
2815     my $i = 1;
2816     foreach my $o (split(//, $bootorder)) {
2817         $bootindex_hash->{$o} = $i*100;
2818         $i++;
2819     }
2820
2821     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000";
2822
2823     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
2824
2825     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
2826
2827     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
2828
2829     # time drift fix
2830     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
2831
2832     my $nokvm = defined($conf->{kvm}) && $conf->{kvm} == 0 ? 1 : 0;
2833     my $useLocaltime = $conf->{localtime};
2834
2835     if (my $ost = $conf->{ostype}) {
2836         # other, wxp, w2k, w2k3, w2k8, wvista, win7, win8, l24, l26, solaris
2837
2838         if ($ost =~ m/^w/) { # windows
2839             $useLocaltime = 1 if !defined($conf->{localtime});
2840
2841             # use time drift fix when acpi is enabled
2842             if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
2843                 $tdf = 1 if !defined($conf->{tdf});
2844             }
2845         }
2846
2847         if ($ost eq 'win7' || $ost eq 'win8' || $ost eq 'w2k8' ||
2848             $ost eq 'wvista') {
2849             push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
2850             push @$cmd, '-no-hpet';
2851             if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
2852                 push @$cpuFlags , 'hv_spinlocks=0x1fff' if !$nokvm;
2853                 push @$cpuFlags , 'hv_vapic' if !$nokvm;
2854                 push @$cpuFlags , 'hv_time' if !$nokvm;
2855
2856             } else {
2857                 push @$cpuFlags , 'hv_spinlocks=0xffff' if !$nokvm;
2858             }
2859         }
2860
2861         if ($ost eq 'win7' || $ost eq 'win8') {
2862             push @$cpuFlags , 'hv_relaxed' if !$nokvm;
2863         }
2864     }
2865
2866     push @$rtcFlags, 'driftfix=slew' if $tdf;
2867
2868     if ($nokvm) {
2869         push @$machineFlags, 'accel=tcg';
2870     } else {
2871         die "No accelerator found!\n" if !$cpuinfo->{hvm};
2872     }
2873
2874     if ($machine_type) {
2875         push @$machineFlags, "type=${machine_type}";
2876     }
2877
2878     if ($conf->{startdate}) {
2879         push @$rtcFlags, "base=$conf->{startdate}";
2880     } elsif ($useLocaltime) {
2881         push @$rtcFlags, 'base=localtime';
2882     }
2883
2884     my $cpu = $nokvm ? "qemu64" : "kvm64";
2885     $cpu = $conf->{cpu} if $conf->{cpu};
2886
2887     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
2888
2889     push @$cpuFlags , '-x2apic'
2890         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
2891
2892     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
2893
2894     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
2895
2896     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
2897
2898         push @$cpuFlags , '+kvm_pv_unhalt' if !$nokvm;
2899         push @$cpuFlags , '+kvm_pv_eoi' if !$nokvm;
2900         push @$cpuFlags , '-kvm_steal_time' if !$nokvm;
2901     }
2902
2903     push @$cpuFlags, 'enforce' if $cpu ne 'host' && !$nokvm;
2904
2905     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
2906
2907     push @$cmd, '-cpu', $cpu;
2908
2909     my $memory = $conf->{memory} || $defaults->{memory};
2910     my $static_memory = 0;
2911     my $dimm_memory = 0;
2912
2913     if ($hotplug_features->{memory}) {
2914         die "Numa need to be enabled for memory hotplug\n" if !$conf->{numa};
2915         die "Total memory is bigger than ${MAX_MEM}MB\n" if $memory > $MAX_MEM;
2916         $static_memory = $STATICMEM;
2917         die "minimum memory must be ${static_memory}MB\n" if($memory < $static_memory);
2918         $dimm_memory = $memory - $static_memory;
2919         push @$cmd, '-m', "size=${static_memory},slots=255,maxmem=${MAX_MEM}M";
2920
2921     } else {
2922
2923         $static_memory = $memory;
2924         push @$cmd, '-m', $static_memory;
2925     }
2926
2927     if ($conf->{numa}) {
2928
2929         my $numa_totalmemory = undef;
2930         for (my $i = 0; $i < $MAX_NUMA; $i++) {
2931             next if !$conf->{"numa$i"};
2932             my $numa = parse_numa($conf->{"numa$i"});
2933             next if !$numa;
2934             # memory
2935             die "missing numa node$i memory value\n" if !$numa->{memory};
2936             my $numa_memory = $numa->{memory};
2937             $numa_totalmemory += $numa_memory;
2938             my $numa_object = "memory-backend-ram,id=ram-node$i,size=${numa_memory}M";
2939
2940             # cpus
2941             my $cpus_start = $numa->{cpus}->{start};
2942             die "missing numa node$i cpus\n" if !defined($cpus_start);
2943             my $cpus_end = $numa->{cpus}->{end} if defined($numa->{cpus}->{end});
2944             my $cpus = $cpus_start;
2945             if (defined($cpus_end)) {
2946                 $cpus .= "-$cpus_end";
2947                 die "numa node$i :  cpu range $cpus is incorrect\n" if $cpus_end <= $cpus_start;
2948             }
2949
2950             # hostnodes
2951             my $hostnodes_start = $numa->{hostnodes}->{start};
2952             if (defined($hostnodes_start)) {
2953                 my $hostnodes_end = $numa->{hostnodes}->{end} if defined($numa->{hostnodes}->{end});
2954                 my $hostnodes = $hostnodes_start;
2955                 if (defined($hostnodes_end)) {
2956                     $hostnodes .= "-$hostnodes_end";
2957                     die "host node $hostnodes range is incorrect\n" if $hostnodes_end <= $hostnodes_start;
2958                 }
2959
2960                 my $hostnodes_end_range = defined($hostnodes_end) ? $hostnodes_end : $hostnodes_start;
2961                 for (my $i = $hostnodes_start; $i <= $hostnodes_end_range; $i++ ) {
2962                     die "host numa node$i don't exist\n" if ! -d "/sys/devices/system/node/node$i/";
2963                 }
2964
2965                 # policy
2966                 my $policy = $numa->{policy};
2967                 die "you need to define a policy for hostnode $hostnodes\n" if !$policy;
2968                 $numa_object .= ",host-nodes=$hostnodes,policy=$policy";
2969             }
2970
2971             push @$cmd, '-object', $numa_object;
2972             push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
2973         }
2974
2975         die "total memory for NUMA nodes must be equal to vm static memory\n"
2976             if $numa_totalmemory && $numa_totalmemory != $static_memory;
2977
2978         #if no custom tology, we split memory and cores across numa nodes
2979         if(!$numa_totalmemory) {
2980
2981             my $numa_memory = ($static_memory / $sockets) . "M";
2982
2983             for (my $i = 0; $i < $sockets; $i++)  {
2984
2985                 my $cpustart = ($cores * $i);
2986                 my $cpuend = ($cpustart + $cores - 1) if $cores && $cores > 1;
2987                 my $cpus = $cpustart;
2988                 $cpus .= "-$cpuend" if $cpuend;
2989
2990                 push @$cmd, '-object', "memory-backend-ram,size=$numa_memory,id=ram-node$i";
2991                 push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
2992             }
2993         }
2994     }
2995
2996     if ($hotplug_features->{memory}) {
2997         foreach_dimm($conf, $vmid, $memory, $sockets, sub {
2998             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
2999             push @$cmd, "-object" , "memory-backend-ram,id=mem-$name,size=${dimm_size}M";
3000             push @$cmd, "-device", "pc-dimm,id=$name,memdev=mem-$name,node=$numanode";
3001
3002             #if dimm_memory is not aligned to dimm map
3003             if($current_size > $memory) {
3004                  $conf->{memory} = $current_size;
3005                  update_config_nolock($vmid, $conf, 1);
3006             }
3007         });
3008     }
3009
3010     push @$cmd, '-S' if $conf->{freeze};
3011
3012     # set keyboard layout
3013     my $kb = $conf->{keyboard} || $defaults->{keyboard};
3014     push @$cmd, '-k', $kb if $kb;
3015
3016     # enable sound
3017     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3018     #push @$cmd, '-soundhw', 'es1370';
3019     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3020
3021     if($conf->{agent}) {
3022         my $qgasocket = qmp_socket($vmid, 1);
3023         my $pciaddr = print_pci_addr("qga0", $bridges);
3024         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3025         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3026         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3027     }
3028
3029     my $spice_port;
3030
3031     if ($qxlnum) {
3032         if ($qxlnum > 1) {
3033             if ($conf->{ostype} && $conf->{ostype} =~ m/^w/){
3034                 for(my $i = 1; $i < $qxlnum; $i++){
3035                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3036                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3037                 }
3038             } else {
3039                 # assume other OS works like Linux
3040                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3041                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3042             }
3043         }
3044
3045         my $pciaddr = print_pci_addr("spice", $bridges);
3046
3047         my $nodename = PVE::INotify::nodename();
3048         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3049         $spice_port = PVE::Tools::next_spice_port($pfamily);
3050
3051         push @$devices, '-spice', "tls-port=${spice_port},addr=localhost,tls-ciphers=DES-CBC3-SHA,seamless-migration=on";
3052
3053         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3054         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3055         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3056     }
3057
3058     # enable balloon by default, unless explicitly disabled
3059     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3060         $pciaddr = print_pci_addr("balloon0", $bridges);
3061         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3062     }
3063
3064     if ($conf->{watchdog}) {
3065         my $wdopts = parse_watchdog($conf->{watchdog});
3066         $pciaddr = print_pci_addr("watchdog", $bridges);
3067         my $watchdog = $wdopts->{model} || 'i6300esb';
3068         push @$devices, '-device', "$watchdog$pciaddr";
3069         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3070     }
3071
3072     my $vollist = [];
3073     my $scsicontroller = {};
3074     my $ahcicontroller = {};
3075     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3076
3077     # Add iscsi initiator name if available
3078     if (my $initiator = get_initiator_name()) {
3079         push @$devices, '-iscsi', "initiator-name=$initiator";
3080     }
3081
3082     foreach_drive($conf, sub {
3083         my ($ds, $drive) = @_;
3084
3085         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3086             push @$vollist, $drive->{file};
3087         }
3088
3089         $use_virtio = 1 if $ds =~ m/^virtio/;
3090
3091         if (drive_is_cdrom ($drive)) {
3092             if ($bootindex_hash->{d}) {
3093                 $drive->{bootindex} = $bootindex_hash->{d};
3094                 $bootindex_hash->{d} += 1;
3095             }
3096         } else {
3097             if ($bootindex_hash->{c}) {
3098                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3099                 $bootindex_hash->{c} += 1;
3100             }
3101         }
3102
3103         if($drive->{interface} eq 'virtio'){
3104            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3105         }
3106
3107         if ($drive->{interface} eq 'scsi') {
3108
3109             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3110
3111             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3112             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3113
3114             my $iothread = '';
3115             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3116                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3117                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3118             }
3119
3120             my $queues = '';
3121             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3122                 $queues = ",num_queues=$drive->{queues}";
3123             }
3124
3125             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3126             $scsicontroller->{$controller}=1;
3127         }
3128
3129         if ($drive->{interface} eq 'sata') {
3130            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3131            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3132            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3133            $ahcicontroller->{$controller}=1;
3134         }
3135
3136         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3137         push @$devices, '-drive',$drive_cmd;
3138         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3139     });
3140
3141     for (my $i = 0; $i < $MAX_NETS; $i++) {
3142          next if !$conf->{"net$i"};
3143          my $d = parse_net($conf->{"net$i"});
3144          next if !$d;
3145
3146          $use_virtio = 1 if $d->{model} eq 'virtio';
3147
3148          if ($bootindex_hash->{n}) {
3149             $d->{bootindex} = $bootindex_hash->{n};
3150             $bootindex_hash->{n} += 1;
3151          }
3152
3153          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3154          push @$devices, '-netdev', $netdevfull;
3155
3156          my $netdevicefull = print_netdevice_full($vmid,$conf,$d,"net$i",$bridges);
3157          push @$devices, '-device', $netdevicefull;
3158     }
3159
3160     if (!$q35) {
3161         # add pci bridges
3162         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3163            $bridges->{1} = 1;
3164            $bridges->{2} = 1;
3165         }
3166
3167         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3168
3169         while (my ($k, $v) = each %$bridges) {
3170             $pciaddr = print_pci_addr("pci.$k");
3171             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3172         }
3173     }
3174
3175     # add custom args
3176     if ($conf->{args}) {
3177         my $aa = PVE::Tools::split_args($conf->{args});
3178         push @$cmd, @$aa;
3179     }
3180
3181     push @$cmd, @$devices;
3182     push @$cmd, '-rtc', join(',', @$rtcFlags)
3183         if scalar(@$rtcFlags);
3184     push @$cmd, '-machine', join(',', @$machineFlags)
3185         if scalar(@$machineFlags);
3186     push @$cmd, '-global', join(',', @$globalFlags)
3187         if scalar(@$globalFlags);
3188
3189     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3190 }
3191
3192 sub vnc_socket {
3193     my ($vmid) = @_;
3194     return "${var_run_tmpdir}/$vmid.vnc";
3195 }
3196
3197 sub spice_port {
3198     my ($vmid) = @_;
3199
3200     my $res = vm_mon_cmd($vmid, 'query-spice');
3201
3202     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3203 }
3204
3205 sub qmp_socket {
3206     my ($vmid, $qga) = @_;
3207     my $sockettype = $qga ? 'qga' : 'qmp';
3208     return "${var_run_tmpdir}/$vmid.$sockettype";
3209 }
3210
3211 sub pidfile_name {
3212     my ($vmid) = @_;
3213     return "${var_run_tmpdir}/$vmid.pid";
3214 }
3215
3216 sub vm_devices_list {
3217     my ($vmid) = @_;
3218
3219     my $res = vm_mon_cmd($vmid, 'query-pci');
3220     my $devices = {};
3221     foreach my $pcibus (@$res) {
3222         foreach my $device (@{$pcibus->{devices}}) {
3223             next if !$device->{'qdev_id'};
3224             if ($device->{'pci_bridge'}) {
3225                 $devices->{$device->{'qdev_id'}} = 1;
3226                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3227                     next if !$bridge_device->{'qdev_id'};
3228                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3229                     $devices->{$device->{'qdev_id'}}++;
3230                 }
3231             } else {
3232                 $devices->{$device->{'qdev_id'}} = 1;
3233             }
3234         }
3235     }
3236
3237     my $resblock = vm_mon_cmd($vmid, 'query-block');
3238     foreach my $block (@$resblock) {
3239         if($block->{device} =~ m/^drive-(\S+)/){
3240                 $devices->{$1} = 1;
3241         }
3242     }
3243
3244     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3245     foreach my $mice (@$resmice) {
3246         if ($mice->{name} eq 'QEMU HID Tablet') {
3247             $devices->{tablet} = 1;
3248             last;
3249         }
3250     }
3251
3252     return $devices;
3253 }
3254
3255 sub vm_deviceplug {
3256     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3257
3258     my $q35 = machine_type_is_q35($conf);
3259
3260     my $devices_list = vm_devices_list($vmid);
3261     return 1 if defined($devices_list->{$deviceid});
3262
3263     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3264
3265     if ($deviceid eq 'tablet') {
3266
3267         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3268
3269     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3270
3271         qemu_iothread_add($vmid, $deviceid, $device);
3272
3273         qemu_driveadd($storecfg, $vmid, $device);
3274         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3275
3276         qemu_deviceadd($vmid, $devicefull);
3277         eval { qemu_deviceaddverify($vmid, $deviceid); };
3278         if (my $err = $@) {
3279             eval { qemu_drivedel($vmid, $deviceid); };
3280             warn $@ if $@;
3281             die $err;
3282         }
3283
3284     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3285
3286
3287         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3288         my $pciaddr = print_pci_addr($deviceid);
3289         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3290
3291         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3292
3293         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3294             qemu_iothread_add($vmid, $deviceid, $device);
3295             $devicefull .= ",iothread=iothread-$deviceid";
3296         }
3297
3298         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3299             $devicefull .= ",num_queues=$device->{queues}";
3300         }
3301
3302         qemu_deviceadd($vmid, $devicefull);
3303         qemu_deviceaddverify($vmid, $deviceid);
3304
3305     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3306
3307         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3308         qemu_driveadd($storecfg, $vmid, $device);
3309
3310         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3311         eval { qemu_deviceadd($vmid, $devicefull); };
3312         if (my $err = $@) {
3313             eval { qemu_drivedel($vmid, $deviceid); };
3314             warn $@ if $@;
3315             die $err;
3316         }
3317
3318     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3319
3320         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3321         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid);
3322         qemu_deviceadd($vmid, $netdevicefull);
3323         eval { qemu_deviceaddverify($vmid, $deviceid); };
3324         if (my $err = $@) {
3325             eval { qemu_netdevdel($vmid, $deviceid); };
3326             warn $@ if $@;
3327             die $err;
3328         }
3329
3330     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3331
3332         my $bridgeid = $2;
3333         my $pciaddr = print_pci_addr($deviceid);
3334         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3335
3336         qemu_deviceadd($vmid, $devicefull);
3337         qemu_deviceaddverify($vmid, $deviceid);
3338
3339     } else {
3340         die "can't hotplug device '$deviceid'\n";
3341     }
3342
3343     return 1;
3344 }
3345
3346 # fixme: this should raise exceptions on error!
3347 sub vm_deviceunplug {
3348     my ($vmid, $conf, $deviceid) = @_;
3349
3350     my $devices_list = vm_devices_list($vmid);
3351     return 1 if !defined($devices_list->{$deviceid});
3352
3353     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3354
3355     if ($deviceid eq 'tablet') {
3356
3357         qemu_devicedel($vmid, $deviceid);
3358
3359     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3360
3361         qemu_devicedel($vmid, $deviceid);
3362         qemu_devicedelverify($vmid, $deviceid);
3363         qemu_drivedel($vmid, $deviceid);
3364         qemu_iothread_del($conf, $vmid, $deviceid);
3365
3366     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3367
3368         qemu_devicedel($vmid, $deviceid);
3369         qemu_devicedelverify($vmid, $deviceid);
3370         qemu_iothread_del($conf, $vmid, $deviceid);
3371
3372     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3373
3374         #qemu 2.3 segfault on drive_del with virtioscsi + iothread
3375         my $device = parse_drive($deviceid, $conf->{$deviceid});
3376         die "virtioscsi with iothread is not hot-unplugglable currently" if $device->{iothread};
3377
3378         qemu_devicedel($vmid, $deviceid);
3379         qemu_drivedel($vmid, $deviceid);
3380         qemu_deletescsihw($conf, $vmid, $deviceid);
3381
3382     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3383
3384         qemu_devicedel($vmid, $deviceid);
3385         qemu_devicedelverify($vmid, $deviceid);
3386         qemu_netdevdel($vmid, $deviceid);
3387
3388     } else {
3389         die "can't unplug device '$deviceid'\n";
3390     }
3391
3392     return 1;
3393 }
3394
3395 sub qemu_deviceadd {
3396     my ($vmid, $devicefull) = @_;
3397
3398     $devicefull = "driver=".$devicefull;
3399     my %options =  split(/[=,]/, $devicefull);
3400
3401     vm_mon_cmd($vmid, "device_add" , %options);
3402 }
3403
3404 sub qemu_devicedel {
3405     my ($vmid, $deviceid) = @_;
3406
3407     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3408 }
3409
3410 sub qemu_iothread_add {
3411     my($vmid, $deviceid, $device) = @_;
3412
3413     if ($device->{iothread}) {
3414         my $iothreads = vm_iothreads_list($vmid);
3415         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3416     }
3417 }
3418
3419 sub qemu_iothread_del {
3420     my($conf, $vmid, $deviceid) = @_;
3421
3422     my $device = parse_drive($deviceid, $conf->{$deviceid});
3423     if ($device->{iothread}) {
3424         my $iothreads = vm_iothreads_list($vmid);
3425         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3426     }
3427 }
3428
3429 sub qemu_objectadd {
3430     my($vmid, $objectid, $qomtype) = @_;
3431
3432     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3433
3434     return 1;
3435 }
3436
3437 sub qemu_objectdel {
3438     my($vmid, $objectid) = @_;
3439
3440     vm_mon_cmd($vmid, "object-del", id => $objectid);
3441
3442     return 1;
3443 }
3444
3445 sub qemu_driveadd {
3446     my ($storecfg, $vmid, $device) = @_;
3447
3448     my $drive = print_drive_full($storecfg, $vmid, $device);
3449     $drive =~ s/\\/\\\\/g;
3450     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3451
3452     # If the command succeeds qemu prints: "OK"
3453     return 1 if $ret =~ m/OK/s;
3454
3455     die "adding drive failed: $ret\n";
3456 }
3457
3458 sub qemu_drivedel {
3459     my($vmid, $deviceid) = @_;
3460
3461     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3462     $ret =~ s/^\s+//;
3463
3464     return 1 if $ret eq "";
3465
3466     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3467     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3468
3469     die "deleting drive $deviceid failed : $ret\n";
3470 }
3471
3472 sub qemu_deviceaddverify {
3473     my ($vmid, $deviceid) = @_;
3474
3475     for (my $i = 0; $i <= 5; $i++) {
3476          my $devices_list = vm_devices_list($vmid);
3477          return 1 if defined($devices_list->{$deviceid});
3478          sleep 1;
3479     }
3480
3481     die "error on hotplug device '$deviceid'\n";
3482 }
3483
3484
3485 sub qemu_devicedelverify {
3486     my ($vmid, $deviceid) = @_;
3487
3488     # need to verify that the device is correctly removed as device_del
3489     # is async and empty return is not reliable
3490
3491     for (my $i = 0; $i <= 5; $i++) {
3492          my $devices_list = vm_devices_list($vmid);
3493          return 1 if !defined($devices_list->{$deviceid});
3494          sleep 1;
3495     }
3496
3497     die "error on hot-unplugging device '$deviceid'\n";
3498 }
3499
3500 sub qemu_findorcreatescsihw {
3501     my ($storecfg, $conf, $vmid, $device) = @_;
3502
3503     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3504
3505     my $scsihwid="$controller_prefix$controller";
3506     my $devices_list = vm_devices_list($vmid);
3507
3508     if(!defined($devices_list->{$scsihwid})) {
3509         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3510     }
3511
3512     return 1;
3513 }
3514
3515 sub qemu_deletescsihw {
3516     my ($conf, $vmid, $opt) = @_;
3517
3518     my $device = parse_drive($opt, $conf->{$opt});
3519
3520     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3521         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3522         return 1;
3523     }
3524
3525     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3526
3527     my $devices_list = vm_devices_list($vmid);
3528     foreach my $opt (keys %{$devices_list}) {
3529         if (PVE::QemuServer::valid_drivename($opt)) {
3530             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3531             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3532                 return 1;
3533             }
3534         }
3535     }
3536
3537     my $scsihwid="scsihw$controller";
3538
3539     vm_deviceunplug($vmid, $conf, $scsihwid);
3540
3541     return 1;
3542 }
3543
3544 sub qemu_add_pci_bridge {
3545     my ($storecfg, $conf, $vmid, $device) = @_;
3546
3547     my $bridges = {};
3548
3549     my $bridgeid;
3550
3551     print_pci_addr($device, $bridges);
3552
3553     while (my ($k, $v) = each %$bridges) {
3554         $bridgeid = $k;
3555     }
3556     return 1 if !defined($bridgeid) || $bridgeid < 1;
3557
3558     my $bridge = "pci.$bridgeid";
3559     my $devices_list = vm_devices_list($vmid);
3560
3561     if (!defined($devices_list->{$bridge})) {
3562         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3563     }
3564
3565     return 1;
3566 }
3567
3568 sub qemu_set_link_status {
3569     my ($vmid, $device, $up) = @_;
3570
3571     vm_mon_cmd($vmid, "set_link", name => $device,
3572                up => $up ? JSON::true : JSON::false);
3573 }
3574
3575 sub qemu_netdevadd {
3576     my ($vmid, $conf, $device, $deviceid) = @_;
3577
3578     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid);
3579     my %options =  split(/[=,]/, $netdev);
3580
3581     vm_mon_cmd($vmid, "netdev_add",  %options);
3582     return 1;
3583 }
3584
3585 sub qemu_netdevdel {
3586     my ($vmid, $deviceid) = @_;
3587
3588     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3589 }
3590
3591 sub qemu_cpu_hotplug {
3592     my ($vmid, $conf, $vcpus) = @_;
3593
3594     my $sockets = 1;
3595     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3596     $sockets = $conf->{sockets} if  $conf->{sockets};
3597     my $cores = $conf->{cores} || 1;
3598     my $maxcpus = $sockets * $cores;
3599
3600     $vcpus = $maxcpus if !$vcpus;
3601
3602     die "you can't add more vcpus than maxcpus\n"
3603         if $vcpus > $maxcpus;
3604
3605     my $currentvcpus = $conf->{vcpus} || $maxcpus;
3606     die "online cpu unplug is not yet possible\n"
3607         if $vcpus < $currentvcpus;
3608
3609     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3610     die "vcpus in running vm is different than configuration\n"
3611         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
3612
3613     for (my $i = $currentvcpus; $i < $vcpus; $i++) {
3614         vm_mon_cmd($vmid, "cpu-add", id => int($i));
3615     }
3616 }
3617
3618 sub qemu_memory_hotplug {
3619     my ($vmid, $conf, $defaults, $opt, $value) = @_;
3620
3621     return $value if !check_running($vmid);
3622
3623     my $memory = $conf->{memory} || $defaults->{memory};
3624     $value = $defaults->{memory} if !$value;
3625     return $value if $value == $memory;
3626
3627     my $static_memory = $STATICMEM;
3628     my $dimm_memory = $memory - $static_memory;
3629
3630     die "memory can't be lower than $static_memory MB" if $value < $static_memory;
3631     die "you cannot add more memory than $MAX_MEM MB!\n" if $memory > $MAX_MEM;
3632
3633
3634     my $sockets = 1;
3635     $sockets = $conf->{sockets} if $conf->{sockets};
3636
3637     if($value > $memory) {
3638
3639         foreach_dimm($conf, $vmid, $value, $sockets, sub {
3640             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3641
3642                 return if $current_size <= $conf->{memory};
3643
3644                 eval { vm_mon_cmd($vmid, "object-add", 'qom-type' => "memory-backend-ram", id => "mem-$name", props => { size => int($dimm_size*1024*1024) } ) };
3645                 if (my $err = $@) {
3646                     eval { qemu_objectdel($vmid, "mem-$name"); };
3647                     die $err;
3648                 }
3649
3650                 eval { vm_mon_cmd($vmid, "device_add", driver => "pc-dimm", id => "$name", memdev => "mem-$name", node => $numanode) };
3651                 if (my $err = $@) {
3652                     eval { qemu_objectdel($vmid, "mem-$name"); };
3653                     die $err;
3654                 }
3655                 #update conf after each succesful module hotplug
3656                 $conf->{memory} = $current_size;
3657                 update_config_nolock($vmid, $conf, 1);
3658         });
3659
3660     } else {
3661
3662         foreach_reverse_dimm($conf, $vmid, $value, $sockets, sub {
3663             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3664
3665                 return if $current_size >= $conf->{memory};
3666                 print "try to unplug memory dimm $name\n";
3667
3668                 my $retry = 0;
3669                 while (1) {
3670                     eval { qemu_devicedel($vmid, $name) };
3671                     sleep 3;
3672                     my $dimm_list = qemu_dimm_list($vmid);
3673                     last if !$dimm_list->{$name};
3674                     raise_param_exc({ $name => "error unplug memory module" }) if $retry > 5;
3675                     $retry++;
3676                 }
3677
3678                 #update conf after each succesful module unplug
3679                 $conf->{memory} = $current_size;
3680
3681                 eval { qemu_objectdel($vmid, "mem-$name"); };
3682                 update_config_nolock($vmid, $conf, 1);
3683         });
3684     }
3685 }
3686
3687 sub qemu_dimm_list {
3688     my ($vmid) = @_;
3689
3690     my $dimmarray = vm_mon_cmd_nocheck($vmid, "query-memory-devices");
3691     my $dimms = {};
3692
3693     foreach my $dimm (@$dimmarray) {
3694
3695         $dimms->{$dimm->{data}->{id}}->{id} = $dimm->{data}->{id};
3696         $dimms->{$dimm->{data}->{id}}->{node} = $dimm->{data}->{node};
3697         $dimms->{$dimm->{data}->{id}}->{addr} = $dimm->{data}->{addr};
3698         $dimms->{$dimm->{data}->{id}}->{size} = $dimm->{data}->{size};
3699         $dimms->{$dimm->{data}->{id}}->{slot} = $dimm->{data}->{slot};
3700     }
3701     return $dimms;
3702 }
3703
3704 sub qemu_block_set_io_throttle {
3705     my ($vmid, $deviceid, $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr) = @_;
3706
3707     return if !check_running($vmid) ;
3708
3709     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid, bps => int($bps), bps_rd => int($bps_rd), bps_wr => int($bps_wr), iops => int($iops), iops_rd => int($iops_rd), iops_wr => int($iops_wr));
3710
3711 }
3712
3713 # old code, only used to shutdown old VM after update
3714 sub __read_avail {
3715     my ($fh, $timeout) = @_;
3716
3717     my $sel = new IO::Select;
3718     $sel->add($fh);
3719
3720     my $res = '';
3721     my $buf;
3722
3723     my @ready;
3724     while (scalar (@ready = $sel->can_read($timeout))) {
3725         my $count;
3726         if ($count = $fh->sysread($buf, 8192)) {
3727             if ($buf =~ /^(.*)\(qemu\) $/s) {
3728                 $res .= $1;
3729                 last;
3730             } else {
3731                 $res .= $buf;
3732             }
3733         } else {
3734             if (!defined($count)) {
3735                 die "$!\n";
3736             }
3737             last;
3738         }
3739     }
3740
3741     die "monitor read timeout\n" if !scalar(@ready);
3742
3743     return $res;
3744 }
3745
3746 # old code, only used to shutdown old VM after update
3747 sub vm_monitor_command {
3748     my ($vmid, $cmdstr, $nocheck) = @_;
3749
3750     my $res;
3751
3752     eval {
3753         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
3754
3755         my $sname = "${var_run_tmpdir}/$vmid.mon";
3756
3757         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
3758             die "unable to connect to VM $vmid socket - $!\n";
3759
3760         my $timeout = 3;
3761
3762         # hack: migrate sometime blocks the monitor (when migrate_downtime
3763         # is set)
3764         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3765             $timeout = 60*60; # 1 hour
3766         }
3767
3768         # read banner;
3769         my $data = __read_avail($sock, $timeout);
3770
3771         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
3772             die "got unexpected qemu monitor banner\n";
3773         }
3774
3775         my $sel = new IO::Select;
3776         $sel->add($sock);
3777
3778         if (!scalar(my @ready = $sel->can_write($timeout))) {
3779             die "monitor write error - timeout";
3780         }
3781
3782         my $fullcmd = "$cmdstr\r";
3783
3784         # syslog('info', "VM $vmid monitor command: $cmdstr");
3785
3786         my $b;
3787         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
3788             die "monitor write error - $!";
3789         }
3790
3791         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
3792
3793         $timeout = 20;
3794
3795         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3796             $timeout = 60*60; # 1 hour
3797         } elsif ($cmdstr =~ m/^(eject|change)/) {
3798             $timeout = 60; # note: cdrom mount command is slow
3799         }
3800         if ($res = __read_avail($sock, $timeout)) {
3801
3802             my @lines = split("\r?\n", $res);
3803
3804             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
3805
3806             $res = join("\n", @lines);
3807             $res .= "\n";
3808         }
3809     };
3810
3811     my $err = $@;
3812
3813     if ($err) {
3814         syslog("err", "VM $vmid monitor command failed - $err");
3815         die $err;
3816     }
3817
3818     return $res;
3819 }
3820
3821 sub qemu_block_resize {
3822     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
3823
3824     my $running = check_running($vmid);
3825
3826     return if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
3827
3828     return if !$running;
3829
3830     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
3831
3832 }
3833
3834 sub qemu_volume_snapshot {
3835     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3836
3837     my $running = check_running($vmid);
3838
3839     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
3840         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
3841     } else {
3842         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
3843     }
3844 }
3845
3846 sub qemu_volume_snapshot_delete {
3847     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3848
3849     my $running = check_running($vmid);
3850
3851     return if !PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
3852
3853     return if !$running;
3854
3855     vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
3856 }
3857
3858 sub set_migration_caps {
3859     my ($vmid) = @_;
3860
3861     my $cap_ref = [];
3862
3863     my $enabled_cap = {
3864         "auto-converge" => 1,
3865         "xbzrle" => 1,
3866         "x-rdma-pin-all" => 0,
3867         "zero-blocks" => 0,
3868         "compress" => 0
3869     };
3870
3871     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
3872
3873     for my $supported_capability (@$supported_capabilities) {
3874         push @$cap_ref, {
3875             capability => $supported_capability->{capability},
3876             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
3877         };
3878     }
3879
3880     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
3881 }
3882
3883 my $fast_plug_option = {
3884     'lock' => 1,
3885     'name' => 1,
3886     'onboot' => 1,
3887     'shares' => 1,
3888     'startup' => 1,
3889     'description' => 1,
3890 };
3891
3892 # hotplug changes in [PENDING]
3893 # $selection hash can be used to only apply specified options, for
3894 # example: { cores => 1 } (only apply changed 'cores')
3895 # $errors ref is used to return error messages
3896 sub vmconfig_hotplug_pending {
3897     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
3898
3899     my $defaults = load_defaults();
3900
3901     # commit values which do not have any impact on running VM first
3902     # Note: those option cannot raise errors, we we do not care about
3903     # $selection and always apply them.
3904
3905     my $add_error = sub {
3906         my ($opt, $msg) = @_;
3907         $errors->{$opt} = "hotplug problem - $msg";
3908     };
3909
3910     my $changes = 0;
3911     foreach my $opt (keys %{$conf->{pending}}) { # add/change
3912         if ($fast_plug_option->{$opt}) {
3913             $conf->{$opt} = $conf->{pending}->{$opt};
3914             delete $conf->{pending}->{$opt};
3915             $changes = 1;
3916         }
3917     }
3918
3919     if ($changes) {
3920         update_config_nolock($vmid, $conf, 1);
3921         $conf = load_config($vmid); # update/reload
3922     }
3923
3924     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3925
3926     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
3927     while (my ($opt, $force) = each %$pending_delete_hash) {
3928         next if $selection && !$selection->{$opt};
3929         eval {
3930             if ($opt eq 'hotplug') {
3931                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
3932             } elsif ($opt eq 'tablet') {
3933                 die "skip\n" if !$hotplug_features->{usb};
3934                 if ($defaults->{tablet}) {
3935                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
3936                 } else {
3937                     vm_deviceunplug($vmid, $conf, $opt);
3938                 }
3939             } elsif ($opt eq 'vcpus') {
3940                 die "skip\n" if !$hotplug_features->{cpu};
3941                 qemu_cpu_hotplug($vmid, $conf, undef);
3942             } elsif ($opt eq 'balloon') {
3943                 # enable balloon device is not hotpluggable
3944                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
3945             } elsif ($fast_plug_option->{$opt}) {
3946                 # do nothing
3947             } elsif ($opt =~ m/^net(\d+)$/) {
3948                 die "skip\n" if !$hotplug_features->{network};
3949                 vm_deviceunplug($vmid, $conf, $opt);
3950             } elsif (valid_drivename($opt)) {
3951                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
3952                 vm_deviceunplug($vmid, $conf, $opt);
3953                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
3954             } elsif ($opt =~ m/^memory$/) {
3955                 die "skip\n" if !$hotplug_features->{memory};
3956                 qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
3957             } elsif ($opt eq 'cpuunits') {
3958                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
3959             } elsif ($opt eq 'cpulimit') {
3960                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
3961             } else {
3962                 die "skip\n";
3963             }
3964         };
3965         if (my $err = $@) {
3966             &$add_error($opt, $err) if $err ne "skip\n";
3967         } else {
3968             # save new config if hotplug was successful
3969             delete $conf->{$opt};
3970             vmconfig_undelete_pending_option($conf, $opt);
3971             update_config_nolock($vmid, $conf, 1);
3972             $conf = load_config($vmid); # update/reload
3973         }
3974     }
3975
3976     foreach my $opt (keys %{$conf->{pending}}) {
3977         next if $selection && !$selection->{$opt};
3978         my $value = $conf->{pending}->{$opt};
3979         eval {
3980             if ($opt eq 'hotplug') {
3981                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
3982             } elsif ($opt eq 'tablet') {
3983                 die "skip\n" if !$hotplug_features->{usb};
3984                 if ($value == 1) {
3985                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
3986                 } elsif ($value == 0) {
3987                     vm_deviceunplug($vmid, $conf, $opt);
3988                 }
3989             } elsif ($opt eq 'vcpus') {
3990                 die "skip\n" if !$hotplug_features->{cpu};
3991                 qemu_cpu_hotplug($vmid, $conf, $value);
3992             } elsif ($opt eq 'balloon') {
3993                 # enable/disable balloning device is not hotpluggable
3994                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
3995                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
3996                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
3997
3998                 # allow manual ballooning if shares is set to zero
3999                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4000                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4001                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4002                 }
4003             } elsif ($opt =~ m/^net(\d+)$/) {
4004                 # some changes can be done without hotplug
4005                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4006                                     $vmid, $opt, $value);
4007             } elsif (valid_drivename($opt)) {
4008                 # some changes can be done without hotplug
4009                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4010                                      $vmid, $opt, $value, 1);
4011             } elsif ($opt =~ m/^memory$/) { #dimms
4012                 die "skip\n" if !$hotplug_features->{memory};
4013                 $value = qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4014             } elsif ($opt eq 'cpuunits') {
4015                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4016             } elsif ($opt eq 'cpulimit') {
4017                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4018                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4019             } else {
4020                 die "skip\n";  # skip non-hot-pluggable options
4021             }
4022         };
4023         if (my $err = $@) {
4024             &$add_error($opt, $err) if $err ne "skip\n";
4025         } else {
4026             # save new config if hotplug was successful
4027             $conf->{$opt} = $value;
4028             delete $conf->{pending}->{$opt};
4029             update_config_nolock($vmid, $conf, 1);
4030             $conf = load_config($vmid); # update/reload
4031         }
4032     }
4033 }
4034
4035 sub try_deallocate_drive {
4036     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4037
4038     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4039         my $volid = $drive->{file};
4040         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4041             my $sid = PVE::Storage::parse_volume_id($volid);
4042             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4043
4044             # check if the disk is really unused
4045             my $used_paths = PVE::QemuServer::get_used_paths($vmid, $storecfg, $conf, 1, $key);
4046             my $path = PVE::Storage::path($storecfg, $volid);
4047             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4048                    if $used_paths->{$path};
4049             PVE::Storage::vdisk_free($storecfg, $volid);
4050             return 1;
4051         } else {
4052             # If vm is not owner of this disk remove from config
4053             return 1;
4054         }
4055     }
4056
4057     return undef;
4058 }
4059
4060 sub vmconfig_delete_or_detach_drive {
4061     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4062
4063     my $drive = parse_drive($opt, $conf->{$opt});
4064
4065     my $rpcenv = PVE::RPCEnvironment::get();
4066     my $authuser = $rpcenv->get_user();
4067
4068     if ($force) {
4069         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4070         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4071     } else {
4072         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4073     }
4074 }
4075
4076 sub vmconfig_apply_pending {
4077     my ($vmid, $conf, $storecfg) = @_;
4078
4079     # cold plug
4080
4081     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4082     while (my ($opt, $force) = each %$pending_delete_hash) {
4083         die "internal error" if $opt =~ m/^unused/;
4084         $conf = load_config($vmid); # update/reload
4085         if (!defined($conf->{$opt})) {
4086             vmconfig_undelete_pending_option($conf, $opt);
4087             update_config_nolock($vmid, $conf, 1);
4088         } elsif (valid_drivename($opt)) {
4089             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4090             vmconfig_undelete_pending_option($conf, $opt);
4091             delete $conf->{$opt};
4092             update_config_nolock($vmid, $conf, 1);
4093         } else {
4094             vmconfig_undelete_pending_option($conf, $opt);
4095             delete $conf->{$opt};
4096             update_config_nolock($vmid, $conf, 1);
4097         }
4098     }
4099
4100     $conf = load_config($vmid); # update/reload
4101
4102     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4103         $conf = load_config($vmid); # update/reload
4104
4105         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4106             # skip if nothing changed
4107         } elsif (valid_drivename($opt)) {
4108             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4109                 if defined($conf->{$opt});
4110             $conf->{$opt} = $conf->{pending}->{$opt};
4111         } else {
4112             $conf->{$opt} = $conf->{pending}->{$opt};
4113         }
4114
4115         delete $conf->{pending}->{$opt};
4116         update_config_nolock($vmid, $conf, 1);
4117     }
4118 }
4119
4120 my $safe_num_ne = sub {
4121     my ($a, $b) = @_;
4122
4123     return 0 if !defined($a) && !defined($b);
4124     return 1 if !defined($a);
4125     return 1 if !defined($b);
4126
4127     return $a != $b;
4128 };
4129
4130 my $safe_string_ne = sub {
4131     my ($a, $b) = @_;
4132
4133     return 0 if !defined($a) && !defined($b);
4134     return 1 if !defined($a);
4135     return 1 if !defined($b);
4136
4137     return $a ne $b;
4138 };
4139
4140 sub vmconfig_update_net {
4141     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4142
4143     my $newnet = parse_net($value);
4144
4145     if ($conf->{$opt}) {
4146         my $oldnet = parse_net($conf->{$opt});
4147
4148         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4149             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4150             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4151             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4152
4153             # for non online change, we try to hot-unplug
4154             die "skip\n" if !$hotplug;
4155             vm_deviceunplug($vmid, $conf, $opt);
4156         } else {
4157
4158             die "internal error" if $opt !~ m/net(\d+)/;
4159             my $iface = "tap${vmid}i$1";
4160
4161             if (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4162                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4163             }
4164
4165             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4166                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4167                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4168                 PVE::Network::tap_unplug($iface);
4169                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall});
4170             }
4171
4172             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4173                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4174             }
4175
4176             return 1;
4177         }
4178     }
4179
4180     if ($hotplug) {
4181         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4182     } else {
4183         die "skip\n";
4184     }
4185 }
4186
4187 sub vmconfig_update_disk {
4188     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4189
4190     # fixme: do we need force?
4191
4192     my $drive = parse_drive($opt, $value);
4193
4194     if ($conf->{$opt}) {
4195
4196         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4197
4198             my $media = $drive->{media} || 'disk';
4199             my $oldmedia = $old_drive->{media} || 'disk';
4200             die "unable to change media type\n" if $media ne $oldmedia;
4201
4202             if (!drive_is_cdrom($old_drive)) {
4203
4204                 if ($drive->{file} ne $old_drive->{file}) {
4205
4206                     die "skip\n" if !$hotplug;
4207
4208                     # unplug and register as unused
4209                     vm_deviceunplug($vmid, $conf, $opt);
4210                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4211
4212                 } else {
4213                     # update existing disk
4214
4215                     # skip non hotpluggable value
4216                     if (&$safe_num_ne($drive->{discard}, $old_drive->{discard}) ||
4217                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4218                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4219                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4220                         die "skip\n";
4221                     }
4222
4223                     # apply throttle
4224                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4225                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4226                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4227                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4228                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4229                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4230                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4231                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4232                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4233                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4234                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4235                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max})) {
4236
4237                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4238                                                    ($drive->{mbps} || 0)*1024*1024,
4239                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4240                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4241                                                    $drive->{iops} || 0,
4242                                                    $drive->{iops_rd} || 0,
4243                                                    $drive->{iops_wr} || 0,
4244                                                    ($drive->{mbps_max} || 0)*1024*1024,
4245                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4246                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4247                                                    $drive->{iops_max} || 0,
4248                                                    $drive->{iops_rd_max} || 0,
4249                                                    $drive->{iops_wr_max} || 0);
4250
4251                     }
4252
4253                     return 1;
4254                 }
4255
4256             } else { # cdrom
4257
4258                 if ($drive->{file} eq 'none') {
4259                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4260                 } else {
4261                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4262                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4263                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4264                 }
4265
4266                 return 1;
4267             }
4268         }
4269     }
4270
4271     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4272     # hotplug new disks
4273     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4274 }
4275
4276 sub vm_start {
4277     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused, $forcemachine, $spice_ticket) = @_;
4278
4279     lock_config($vmid, sub {
4280         my $conf = load_config($vmid, $migratedfrom);
4281
4282         die "you can't start a vm if it's a template\n" if is_template($conf);
4283
4284         check_lock($conf) if !$skiplock;
4285
4286         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4287
4288         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4289             vmconfig_apply_pending($vmid, $conf, $storecfg);
4290             $conf = load_config($vmid); # update/reload
4291         }
4292
4293         my $defaults = load_defaults();
4294
4295         # set environment variable useful inside network script
4296         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4297
4298         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4299
4300         my $migrate_port = 0;
4301         my $migrate_uri;
4302         if ($statefile) {
4303             if ($statefile eq 'tcp') {
4304                 my $localip = "localhost";
4305                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4306                 my $nodename = PVE::INotify::nodename();
4307                 if ($datacenterconf->{migration_unsecure}) {
4308                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4309                 }
4310                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4311                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4312                 $migrate_uri = "tcp:[${localip}]:${migrate_port}";
4313                 push @$cmd, '-incoming', $migrate_uri;
4314                 push @$cmd, '-S';
4315             } else {
4316                 push @$cmd, '-loadstate', $statefile;
4317             }
4318         } elsif ($paused) {
4319             push @$cmd, '-S';
4320         }
4321
4322         # host pci devices
4323         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4324           my $d = parse_hostpci($conf->{"hostpci$i"});
4325           next if !$d;
4326           my $pcidevices = $d->{pciid};
4327           foreach my $pcidevice (@$pcidevices) {
4328                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4329
4330                 my $info = pci_device_info("0000:$pciid");
4331                 die "IOMMU not present\n" if !check_iommu_support();
4332                 die "no pci device info for device '$pciid'\n" if !$info;
4333                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4334                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4335           }
4336         }
4337
4338         PVE::Storage::activate_volumes($storecfg, $vollist);
4339
4340         eval  { run_command($cmd, timeout => $statefile ? undef : 30,
4341                     umask => 0077); };
4342         my $err = $@;
4343         die "start failed: $err" if $err;
4344
4345         print "migration listens on $migrate_uri\n" if $migrate_uri;
4346
4347         if ($statefile && $statefile ne 'tcp')  {
4348             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4349             warn $@ if $@;
4350         }
4351
4352         if ($migratedfrom) {
4353
4354             eval {
4355                 set_migration_caps($vmid);
4356             };
4357             warn $@ if $@;
4358
4359             if ($spice_port) {
4360                 print "spice listens on port $spice_port\n";
4361                 if ($spice_ticket) {
4362                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4363                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4364                 }
4365             }
4366
4367         } else {
4368
4369             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4370                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4371                     if $conf->{balloon};
4372             }
4373
4374             foreach my $opt (keys %$conf) {
4375                 next if $opt !~  m/^net\d+$/;
4376                 my $nicconf = parse_net($conf->{$opt});
4377                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4378             }
4379         }
4380
4381         vm_mon_cmd_nocheck($vmid, 'qom-set',
4382                     path => "machine/peripheral/balloon0",
4383                     property => "guest-stats-polling-interval",
4384                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4385
4386     });
4387 }
4388
4389 sub vm_mon_cmd {
4390     my ($vmid, $execute, %params) = @_;
4391
4392     my $cmd = { execute => $execute, arguments => \%params };
4393     vm_qmp_command($vmid, $cmd);
4394 }
4395
4396 sub vm_mon_cmd_nocheck {
4397     my ($vmid, $execute, %params) = @_;
4398
4399     my $cmd = { execute => $execute, arguments => \%params };
4400     vm_qmp_command($vmid, $cmd, 1);
4401 }
4402
4403 sub vm_qmp_command {
4404     my ($vmid, $cmd, $nocheck) = @_;
4405
4406     my $res;
4407
4408     my $timeout;
4409     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4410         $timeout = $cmd->{arguments}->{timeout};
4411         delete $cmd->{arguments}->{timeout};
4412     }
4413
4414     eval {
4415         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4416         my $sname = qmp_socket($vmid);
4417         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4418             my $qmpclient = PVE::QMPClient->new();
4419
4420             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4421         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
4422             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
4423                 if scalar(%{$cmd->{arguments}});
4424             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
4425         } else {
4426             die "unable to open monitor socket\n";
4427         }
4428     };
4429     if (my $err = $@) {
4430         syslog("err", "VM $vmid qmp command failed - $err");
4431         die $err;
4432     }
4433
4434     return $res;
4435 }
4436
4437 sub vm_human_monitor_command {
4438     my ($vmid, $cmdline) = @_;
4439
4440     my $res;
4441
4442     my $cmd = {
4443         execute => 'human-monitor-command',
4444         arguments => { 'command-line' => $cmdline},
4445     };
4446
4447     return vm_qmp_command($vmid, $cmd);
4448 }
4449
4450 sub vm_commandline {
4451     my ($storecfg, $vmid) = @_;
4452
4453     my $conf = load_config($vmid);
4454
4455     my $defaults = load_defaults();
4456
4457     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
4458
4459     return join(' ', @$cmd);
4460 }
4461
4462 sub vm_reset {
4463     my ($vmid, $skiplock) = @_;
4464
4465     lock_config($vmid, sub {
4466
4467         my $conf = load_config($vmid);
4468
4469         check_lock($conf) if !$skiplock;
4470
4471         vm_mon_cmd($vmid, "system_reset");
4472     });
4473 }
4474
4475 sub get_vm_volumes {
4476     my ($conf) = @_;
4477
4478     my $vollist = [];
4479     foreach_volid($conf, sub {
4480         my ($volid, $is_cdrom) = @_;
4481
4482         return if $volid =~ m|^/|;
4483
4484         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4485         return if !$sid;
4486
4487         push @$vollist, $volid;
4488     });
4489
4490     return $vollist;
4491 }
4492
4493 sub vm_stop_cleanup {
4494     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
4495
4496     eval {
4497
4498         if (!$keepActive) {
4499             my $vollist = get_vm_volumes($conf);
4500             PVE::Storage::deactivate_volumes($storecfg, $vollist);
4501         }
4502
4503         foreach my $ext (qw(mon qmp pid vnc qga)) {
4504             unlink "/var/run/qemu-server/${vmid}.$ext";
4505         }
4506
4507         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
4508     };
4509     warn $@ if $@; # avoid errors - just warn
4510 }
4511
4512 # Note: use $nockeck to skip tests if VM configuration file exists.
4513 # We need that when migration VMs to other nodes (files already moved)
4514 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
4515 sub vm_stop {
4516     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
4517
4518     $force = 1 if !defined($force) && !$shutdown;
4519
4520     if ($migratedfrom){
4521         my $pid = check_running($vmid, $nocheck, $migratedfrom);
4522         kill 15, $pid if $pid;
4523         my $conf = load_config($vmid, $migratedfrom);
4524         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
4525         return;
4526     }
4527
4528     lock_config($vmid, sub {
4529
4530         my $pid = check_running($vmid, $nocheck);
4531         return if !$pid;
4532
4533         my $conf;
4534         if (!$nocheck) {
4535             $conf = load_config($vmid);
4536             check_lock($conf) if !$skiplock;
4537             if (!defined($timeout) && $shutdown && $conf->{startup}) {
4538                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
4539                 $timeout = $opts->{down} if $opts->{down};
4540             }
4541         }
4542
4543         $timeout = 60 if !defined($timeout);
4544
4545         eval {
4546             if ($shutdown) {
4547                 if (defined($conf) && $conf->{agent}) {
4548                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
4549                 } else {
4550                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
4551                 }
4552             } else {
4553                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
4554             }
4555         };
4556         my $err = $@;
4557
4558         if (!$err) {
4559             my $count = 0;
4560             while (($count < $timeout) && check_running($vmid, $nocheck)) {
4561                 $count++;
4562                 sleep 1;
4563             }
4564
4565             if ($count >= $timeout) {
4566                 if ($force) {
4567                     warn "VM still running - terminating now with SIGTERM\n";
4568                     kill 15, $pid;
4569                 } else {
4570                     die "VM quit/powerdown failed - got timeout\n";
4571                 }
4572             } else {
4573                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4574                 return;
4575             }
4576         } else {
4577             if ($force) {
4578                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
4579                 kill 15, $pid;
4580             } else {
4581                 die "VM quit/powerdown failed\n";
4582             }
4583         }
4584
4585         # wait again
4586         $timeout = 10;
4587
4588         my $count = 0;
4589         while (($count < $timeout) && check_running($vmid, $nocheck)) {
4590             $count++;
4591             sleep 1;
4592         }
4593
4594         if ($count >= $timeout) {
4595             warn "VM still running - terminating now with SIGKILL\n";
4596             kill 9, $pid;
4597             sleep 1;
4598         }
4599
4600         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4601    });
4602 }
4603
4604 sub vm_suspend {
4605     my ($vmid, $skiplock) = @_;
4606
4607     lock_config($vmid, sub {
4608
4609         my $conf = load_config($vmid);
4610
4611         check_lock($conf) if !($skiplock || ($conf->{lock} && $conf->{lock} eq 'backup'));
4612
4613         vm_mon_cmd($vmid, "stop");
4614     });
4615 }
4616
4617 sub vm_resume {
4618     my ($vmid, $skiplock, $nocheck) = @_;
4619
4620     lock_config($vmid, sub {
4621
4622         if (!$nocheck) {
4623
4624             my $conf = load_config($vmid);
4625
4626             check_lock($conf) if !($skiplock || ($conf->{lock} && $conf->{lock} eq 'backup'));
4627
4628             vm_mon_cmd($vmid, "cont");
4629
4630         } else {
4631             vm_mon_cmd_nocheck($vmid, "cont");
4632         }
4633     });
4634 }
4635
4636 sub vm_sendkey {
4637     my ($vmid, $skiplock, $key) = @_;
4638
4639     lock_config($vmid, sub {
4640
4641         my $conf = load_config($vmid);
4642
4643         # there is no qmp command, so we use the human monitor command
4644         vm_human_monitor_command($vmid, "sendkey $key");
4645     });
4646 }
4647
4648 sub vm_destroy {
4649     my ($storecfg, $vmid, $skiplock) = @_;
4650
4651     lock_config($vmid, sub {
4652
4653         my $conf = load_config($vmid);
4654
4655         check_lock($conf) if !$skiplock;
4656
4657         if (!check_running($vmid)) {
4658             destroy_vm($storecfg, $vmid);
4659         } else {
4660             die "VM $vmid is running - destroy failed\n";
4661         }
4662     });
4663 }
4664
4665 # pci helpers
4666
4667 sub file_write {
4668     my ($filename, $buf) = @_;
4669
4670     my $fh = IO::File->new($filename, "w");
4671     return undef if !$fh;
4672
4673     my $res = print $fh $buf;
4674
4675     $fh->close();
4676
4677     return $res;
4678 }
4679
4680 sub pci_device_info {
4681     my ($name) = @_;
4682
4683     my $res;
4684
4685     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
4686     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
4687
4688     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
4689     return undef if !defined($irq) || $irq !~ m/^\d+$/;
4690
4691     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
4692     return undef if !defined($vendor) || $vendor !~ s/^0x//;
4693
4694     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
4695     return undef if !defined($product) || $product !~ s/^0x//;
4696
4697     $res = {
4698         name => $name,
4699         vendor => $vendor,
4700         product => $product,
4701         domain => $domain,
4702         bus => $bus,
4703         slot => $slot,
4704         func => $func,
4705         irq => $irq,
4706         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
4707     };
4708
4709     return $res;
4710 }
4711
4712 sub pci_dev_reset {
4713     my ($dev) = @_;
4714
4715     my $name = $dev->{name};
4716
4717     my $fn = "$pcisysfs/devices/$name/reset";
4718
4719     return file_write($fn, "1");
4720 }
4721
4722 sub pci_dev_bind_to_vfio {
4723     my ($dev) = @_;
4724
4725     my $name = $dev->{name};
4726
4727     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4728
4729     if (!-d $vfio_basedir) {
4730         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4731     }
4732     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4733
4734     my $testdir = "$vfio_basedir/$name";
4735     return 1 if -d $testdir;
4736
4737     my $data = "$dev->{vendor} $dev->{product}";
4738     return undef if !file_write("$vfio_basedir/new_id", $data);
4739
4740     my $fn = "$pcisysfs/devices/$name/driver/unbind";
4741     if (!file_write($fn, $name)) {
4742         return undef if -f $fn;
4743     }
4744
4745     $fn = "$vfio_basedir/bind";
4746     if (! -d $testdir) {
4747         return undef if !file_write($fn, $name);
4748     }
4749
4750     return -d $testdir;
4751 }
4752
4753 sub pci_dev_group_bind_to_vfio {
4754     my ($pciid) = @_;
4755
4756     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4757
4758     if (!-d $vfio_basedir) {
4759         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4760     }
4761     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4762
4763     # get IOMMU group devices
4764     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
4765       my @devs = grep /^0000:/, readdir($D);
4766     closedir($D);
4767
4768     foreach my $pciid (@devs) {
4769         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
4770
4771         # pci bridges, switches or root ports are not supported
4772         # they have a pci_bus subdirectory so skip them
4773         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
4774
4775         my $info = pci_device_info($1);
4776         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
4777     }
4778
4779     return 1;
4780 }
4781
4782 sub print_pci_addr {
4783     my ($id, $bridges) = @_;
4784
4785     my $res = '';
4786     my $devices = {
4787         piix3 => { bus => 0, addr => 1 },
4788         #addr2 : first videocard
4789         balloon0 => { bus => 0, addr => 3 },
4790         watchdog => { bus => 0, addr => 4 },
4791         scsihw0 => { bus => 0, addr => 5 },
4792         'pci.3' => { bus => 0, addr => 5 }, #can also be used for virtio-scsi-single bridge
4793         scsihw1 => { bus => 0, addr => 6 },
4794         ahci0 => { bus => 0, addr => 7 },
4795         qga0 => { bus => 0, addr => 8 },
4796         spice => { bus => 0, addr => 9 },
4797         virtio0 => { bus => 0, addr => 10 },
4798         virtio1 => { bus => 0, addr => 11 },
4799         virtio2 => { bus => 0, addr => 12 },
4800         virtio3 => { bus => 0, addr => 13 },
4801         virtio4 => { bus => 0, addr => 14 },
4802         virtio5 => { bus => 0, addr => 15 },
4803         hostpci0 => { bus => 0, addr => 16 },
4804         hostpci1 => { bus => 0, addr => 17 },
4805         net0 => { bus => 0, addr => 18 },
4806         net1 => { bus => 0, addr => 19 },
4807         net2 => { bus => 0, addr => 20 },
4808         net3 => { bus => 0, addr => 21 },
4809         net4 => { bus => 0, addr => 22 },
4810         net5 => { bus => 0, addr => 23 },
4811         vga1 => { bus => 0, addr => 24 },
4812         vga2 => { bus => 0, addr => 25 },
4813         vga3 => { bus => 0, addr => 26 },
4814         hostpci2 => { bus => 0, addr => 27 },
4815         hostpci3 => { bus => 0, addr => 28 },
4816         #addr29 : usb-host (pve-usb.cfg)
4817         'pci.1' => { bus => 0, addr => 30 },
4818         'pci.2' => { bus => 0, addr => 31 },
4819         'net6' => { bus => 1, addr => 1 },
4820         'net7' => { bus => 1, addr => 2 },
4821         'net8' => { bus => 1, addr => 3 },
4822         'net9' => { bus => 1, addr => 4 },
4823         'net10' => { bus => 1, addr => 5 },
4824         'net11' => { bus => 1, addr => 6 },
4825         'net12' => { bus => 1, addr => 7 },
4826         'net13' => { bus => 1, addr => 8 },
4827         'net14' => { bus => 1, addr => 9 },
4828         'net15' => { bus => 1, addr => 10 },
4829         'net16' => { bus => 1, addr => 11 },
4830         'net17' => { bus => 1, addr => 12 },
4831         'net18' => { bus => 1, addr => 13 },
4832         'net19' => { bus => 1, addr => 14 },
4833         'net20' => { bus => 1, addr => 15 },
4834         'net21' => { bus => 1, addr => 16 },
4835         'net22' => { bus => 1, addr => 17 },
4836         'net23' => { bus => 1, addr => 18 },
4837         'net24' => { bus => 1, addr => 19 },
4838         'net25' => { bus => 1, addr => 20 },
4839         'net26' => { bus => 1, addr => 21 },
4840         'net27' => { bus => 1, addr => 22 },
4841         'net28' => { bus => 1, addr => 23 },
4842         'net29' => { bus => 1, addr => 24 },
4843         'net30' => { bus => 1, addr => 25 },
4844         'net31' => { bus => 1, addr => 26 },
4845         'virtio6' => { bus => 2, addr => 1 },
4846         'virtio7' => { bus => 2, addr => 2 },
4847         'virtio8' => { bus => 2, addr => 3 },
4848         'virtio9' => { bus => 2, addr => 4 },
4849         'virtio10' => { bus => 2, addr => 5 },
4850         'virtio11' => { bus => 2, addr => 6 },
4851         'virtio12' => { bus => 2, addr => 7 },
4852         'virtio13' => { bus => 2, addr => 8 },
4853         'virtio14' => { bus => 2, addr => 9 },
4854         'virtio15' => { bus => 2, addr => 10 },
4855         'virtioscsi0' => { bus => 3, addr => 1 },
4856         'virtioscsi1' => { bus => 3, addr => 2 },
4857         'virtioscsi2' => { bus => 3, addr => 3 },
4858         'virtioscsi3' => { bus => 3, addr => 4 },
4859         'virtioscsi4' => { bus => 3, addr => 5 },
4860         'virtioscsi5' => { bus => 3, addr => 6 },
4861         'virtioscsi6' => { bus => 3, addr => 7 },
4862         'virtioscsi7' => { bus => 3, addr => 8 },
4863         'virtioscsi8' => { bus => 3, addr => 9 },
4864         'virtioscsi9' => { bus => 3, addr => 10 },
4865         'virtioscsi10' => { bus => 3, addr => 11 },
4866         'virtioscsi11' => { bus => 3, addr => 12 },
4867         'virtioscsi12' => { bus => 3, addr => 13 },
4868         'virtioscsi13' => { bus => 3, addr => 14 },
4869         'virtioscsi14' => { bus => 3, addr => 15 },
4870         'virtioscsi15' => { bus => 3, addr => 16 },
4871         'virtioscsi16' => { bus => 3, addr => 17 },
4872         'virtioscsi17' => { bus => 3, addr => 18 },
4873         'virtioscsi18' => { bus => 3, addr => 19 },
4874         'virtioscsi19' => { bus => 3, addr => 20 },
4875         'virtioscsi20' => { bus => 3, addr => 21 },
4876         'virtioscsi21' => { bus => 3, addr => 22 },
4877         'virtioscsi22' => { bus => 3, addr => 23 },
4878         'virtioscsi23' => { bus => 3, addr => 24 },
4879         'virtioscsi24' => { bus => 3, addr => 25 },
4880         'virtioscsi25' => { bus => 3, addr => 26 },
4881         'virtioscsi26' => { bus => 3, addr => 27 },
4882         'virtioscsi27' => { bus => 3, addr => 28 },
4883         'virtioscsi28' => { bus => 3, addr => 29 },
4884         'virtioscsi29' => { bus => 3, addr => 30 },
4885         'virtioscsi30' => { bus => 3, addr => 31 },
4886
4887     };
4888
4889     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
4890            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
4891            my $bus = $devices->{$id}->{bus};
4892            $res = ",bus=pci.$bus,addr=$addr";
4893            $bridges->{$bus} = 1 if $bridges;
4894     }
4895     return $res;
4896
4897 }
4898
4899 sub print_pcie_addr {
4900     my ($id) = @_;
4901
4902     my $res = '';
4903     my $devices = {
4904         hostpci0 => { bus => "ich9-pcie-port-1", addr => 0 },
4905         hostpci1 => { bus => "ich9-pcie-port-2", addr => 0 },
4906         hostpci2 => { bus => "ich9-pcie-port-3", addr => 0 },
4907         hostpci3 => { bus => "ich9-pcie-port-4", addr => 0 },
4908     };
4909
4910     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
4911            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
4912            my $bus = $devices->{$id}->{bus};
4913            $res = ",bus=$bus,addr=$addr";
4914     }
4915     return $res;
4916
4917 }
4918
4919 # vzdump restore implementaion
4920
4921 sub tar_archive_read_firstfile {
4922     my $archive = shift;
4923
4924     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
4925
4926     # try to detect archive type first
4927     my $pid = open (TMP, "tar tf '$archive'|") ||
4928         die "unable to open file '$archive'\n";
4929     my $firstfile = <TMP>;
4930     kill 15, $pid;
4931     close TMP;
4932
4933     die "ERROR: archive contaions no data\n" if !$firstfile;
4934     chomp $firstfile;
4935
4936     return $firstfile;
4937 }
4938
4939 sub tar_restore_cleanup {
4940     my ($storecfg, $statfile) = @_;
4941
4942     print STDERR "starting cleanup\n";
4943
4944     if (my $fd = IO::File->new($statfile, "r")) {
4945         while (defined(my $line = <$fd>)) {
4946             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
4947                 my $volid = $2;
4948                 eval {
4949                     if ($volid =~ m|^/|) {
4950                         unlink $volid || die 'unlink failed\n';
4951                     } else {
4952                         PVE::Storage::vdisk_free($storecfg, $volid);
4953                     }
4954                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
4955                 };
4956                 print STDERR "unable to cleanup '$volid' - $@" if $@;
4957             } else {
4958                 print STDERR "unable to parse line in statfile - $line";
4959             }
4960         }
4961         $fd->close();
4962     }
4963 }
4964
4965 sub restore_archive {
4966     my ($archive, $vmid, $user, $opts) = @_;
4967
4968     my $format = $opts->{format};
4969     my $comp;
4970
4971     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
4972         $format = 'tar' if !$format;
4973         $comp = 'gzip';
4974     } elsif ($archive =~ m/\.tar$/) {
4975         $format = 'tar' if !$format;
4976     } elsif ($archive =~ m/.tar.lzo$/) {
4977         $format = 'tar' if !$format;
4978         $comp = 'lzop';
4979     } elsif ($archive =~ m/\.vma$/) {
4980         $format = 'vma' if !$format;
4981     } elsif ($archive =~ m/\.vma\.gz$/) {
4982         $format = 'vma' if !$format;
4983         $comp = 'gzip';
4984     } elsif ($archive =~ m/\.vma\.lzo$/) {
4985         $format = 'vma' if !$format;
4986         $comp = 'lzop';
4987     } else {
4988         $format = 'vma' if !$format; # default
4989     }
4990
4991     # try to detect archive format
4992     if ($format eq 'tar') {
4993         return restore_tar_archive($archive, $vmid, $user, $opts);
4994     } else {
4995         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
4996     }
4997 }
4998
4999 sub restore_update_config_line {
5000     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5001
5002     return if $line =~ m/^\#qmdump\#/;
5003     return if $line =~ m/^\#vzdump\#/;
5004     return if $line =~ m/^lock:/;
5005     return if $line =~ m/^unused\d+:/;
5006     return if $line =~ m/^parent:/;
5007     return if $line =~ m/^template:/; # restored VM is never a template
5008
5009     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5010         # try to convert old 1.X settings
5011         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5012         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5013             my ($model, $macaddr) = split(/\=/, $devconfig);
5014             $macaddr = PVE::Tools::random_ether_addr() if !$macaddr || $unique;
5015             my $net = {
5016                 model => $model,
5017                 bridge => "vmbr$ind",
5018                 macaddr => $macaddr,
5019             };
5020             my $netstr = print_net($net);
5021
5022             print $outfd "net$cookie->{netcount}: $netstr\n";
5023             $cookie->{netcount}++;
5024         }
5025     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5026         my ($id, $netstr) = ($1, $2);
5027         my $net = parse_net($netstr);
5028         $net->{macaddr} = PVE::Tools::random_ether_addr() if $net->{macaddr};
5029         $netstr = print_net($net);
5030         print $outfd "$id: $netstr\n";
5031     } elsif ($line =~ m/^((ide|scsi|virtio|sata)\d+):\s*(\S+)\s*$/) {
5032         my $virtdev = $1;
5033         my $value = $3;
5034         if ($line =~ m/backup=no/) {
5035             print $outfd "#$line";
5036         } elsif ($virtdev && $map->{$virtdev}) {
5037             my $di = parse_drive($virtdev, $value);
5038             delete $di->{format}; # format can change on restore
5039             $di->{file} = $map->{$virtdev};
5040             $value = print_drive($vmid, $di);
5041             print $outfd "$virtdev: $value\n";
5042         } else {
5043             print $outfd $line;
5044         }
5045     } else {
5046         print $outfd $line;
5047     }
5048 }
5049
5050 sub scan_volids {
5051     my ($cfg, $vmid) = @_;
5052
5053     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5054
5055     my $volid_hash = {};
5056     foreach my $storeid (keys %$info) {
5057         foreach my $item (@{$info->{$storeid}}) {
5058             next if !($item->{volid} && $item->{size});
5059             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5060             $volid_hash->{$item->{volid}} = $item;
5061         }
5062     }
5063
5064     return $volid_hash;
5065 }
5066
5067 sub get_used_paths {
5068     my ($vmid, $storecfg, $conf, $scan_snapshots, $skip_drive) = @_;
5069
5070     my $used_path = {};
5071
5072     my $scan_config = sub {
5073         my ($cref, $snapname) = @_;
5074
5075         foreach my $key (keys %$cref) {
5076             my $value = $cref->{$key};
5077             if (valid_drivename($key)) {
5078                 next if $skip_drive && $key eq $skip_drive;
5079                 my $drive = parse_drive($key, $value);
5080                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5081                 if ($drive->{file} =~ m!^/!) {
5082                     $used_path->{$drive->{file}}++; # = 1;
5083                 } else {
5084                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5085                     next if !$storeid;
5086                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5087                     next if !$scfg;
5088                     my $path = PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5089                     $used_path->{$path}++; # = 1;
5090                 }
5091             }
5092         }
5093     };
5094
5095     &$scan_config($conf);
5096
5097     undef $skip_drive;
5098
5099     if ($scan_snapshots) {
5100         foreach my $snapname (keys %{$conf->{snapshots}}) {
5101             &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5102         }
5103     }
5104
5105     return $used_path;
5106 }
5107
5108 sub update_disksize {
5109     my ($vmid, $conf, $volid_hash) = @_;
5110
5111     my $changes;
5112
5113     my $used = {};
5114
5115     # Note: it is allowed to define multiple storages with same path (alias), so
5116     # we need to check both 'volid' and real 'path' (two different volid can point
5117     # to the same path).
5118
5119     my $usedpath = {};
5120
5121     # update size info
5122     foreach my $opt (keys %$conf) {
5123         if (valid_drivename($opt)) {
5124             my $drive = parse_drive($opt, $conf->{$opt});
5125             my $volid = $drive->{file};
5126             next if !$volid;
5127
5128             $used->{$volid} = 1;
5129             if ($volid_hash->{$volid} &&
5130                 (my $path = $volid_hash->{$volid}->{path})) {
5131                 $usedpath->{$path} = 1;
5132             }
5133
5134             next if drive_is_cdrom($drive);
5135             next if !$volid_hash->{$volid};
5136
5137             $drive->{size} = $volid_hash->{$volid}->{size};
5138             my $new = print_drive($vmid, $drive);
5139             if ($new ne $conf->{$opt}) {
5140                 $changes = 1;
5141                 $conf->{$opt} = $new;
5142             }
5143         }
5144     }
5145
5146     # remove 'unusedX' entry if volume is used
5147     foreach my $opt (keys %$conf) {
5148         next if $opt !~ m/^unused\d+$/;
5149         my $volid = $conf->{$opt};
5150         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5151         if ($used->{$volid} || ($path && $usedpath->{$path})) {
5152             $changes = 1;
5153             delete $conf->{$opt};
5154         }
5155     }
5156
5157     foreach my $volid (sort keys %$volid_hash) {
5158         next if $volid =~ m/vm-$vmid-state-/;
5159         next if $used->{$volid};
5160         my $path = $volid_hash->{$volid}->{path};
5161         next if !$path; # just to be sure
5162         next if $usedpath->{$path};
5163         $changes = 1;
5164         add_unused_volume($conf, $volid);
5165         $usedpath->{$path} = 1; # avoid to add more than once (aliases)
5166     }
5167
5168     return $changes;
5169 }
5170
5171 sub rescan {
5172     my ($vmid, $nolock) = @_;
5173
5174     my $cfg = PVE::Cluster::cfs_read_file("storage.cfg");
5175
5176     my $volid_hash = scan_volids($cfg, $vmid);
5177
5178     my $updatefn =  sub {
5179         my ($vmid) = @_;
5180
5181         my $conf = load_config($vmid);
5182
5183         check_lock($conf);
5184
5185         my $vm_volids = {};
5186         foreach my $volid (keys %$volid_hash) {
5187             my $info = $volid_hash->{$volid};
5188             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5189         }
5190
5191         my $changes = update_disksize($vmid, $conf, $vm_volids);
5192
5193         update_config_nolock($vmid, $conf, 1) if $changes;
5194     };
5195
5196     if (defined($vmid)) {
5197         if ($nolock) {
5198             &$updatefn($vmid);
5199         } else {
5200             lock_config($vmid, $updatefn, $vmid);
5201         }
5202     } else {
5203         my $vmlist = config_list();
5204         foreach my $vmid (keys %$vmlist) {
5205             if ($nolock) {
5206                 &$updatefn($vmid);
5207             } else {
5208                 lock_config($vmid, $updatefn, $vmid);
5209             }
5210         }
5211     }
5212 }
5213
5214 sub restore_vma_archive {
5215     my ($archive, $vmid, $user, $opts, $comp) = @_;
5216
5217     my $input = $archive eq '-' ? "<&STDIN" : undef;
5218     my $readfrom = $archive;
5219
5220     my $uncomp = '';
5221     if ($comp) {
5222         $readfrom = '-';
5223         my $qarchive = PVE::Tools::shellquote($archive);
5224         if ($comp eq 'gzip') {
5225             $uncomp = "zcat $qarchive|";
5226         } elsif ($comp eq 'lzop') {
5227             $uncomp = "lzop -d -c $qarchive|";
5228         } else {
5229             die "unknown compression method '$comp'\n";
5230         }
5231
5232     }
5233
5234     my $tmpdir = "/var/tmp/vzdumptmp$$";
5235     rmtree $tmpdir;
5236
5237     # disable interrupts (always do cleanups)
5238     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5239         warn "got interrupt - ignored\n";
5240     };
5241
5242     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5243     POSIX::mkfifo($mapfifo, 0600);
5244     my $fifofh;
5245
5246     my $openfifo = sub {
5247         open($fifofh, '>', $mapfifo) || die $!;
5248     };
5249
5250     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5251
5252     my $oldtimeout;
5253     my $timeout = 5;
5254
5255     my $devinfo = {};
5256
5257     my $rpcenv = PVE::RPCEnvironment::get();
5258
5259     my $conffile = config_file($vmid);
5260     my $tmpfn = "$conffile.$$.tmp";
5261
5262     # Note: $oldconf is undef if VM does not exists
5263     my $oldconf = PVE::Cluster::cfs_read_file(cfs_config_path($vmid));
5264
5265     my $print_devmap = sub {
5266         my $virtdev_hash = {};
5267
5268         my $cfgfn = "$tmpdir/qemu-server.conf";
5269
5270         # we can read the config - that is already extracted
5271         my $fh = IO::File->new($cfgfn, "r") ||
5272             "unable to read qemu-server.conf - $!\n";
5273
5274         while (defined(my $line = <$fh>)) {
5275             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5276                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5277                 die "archive does not contain data for drive '$virtdev'\n"
5278                     if !$devinfo->{$devname};
5279                 if (defined($opts->{storage})) {
5280                     $storeid = $opts->{storage} || 'local';
5281                 } elsif (!$storeid) {
5282                     $storeid = 'local';
5283                 }
5284                 $format = 'raw' if !$format;
5285                 $devinfo->{$devname}->{devname} = $devname;
5286                 $devinfo->{$devname}->{virtdev} = $virtdev;
5287                 $devinfo->{$devname}->{format} = $format;
5288                 $devinfo->{$devname}->{storeid} = $storeid;
5289
5290                 # check permission on storage
5291                 my $pool = $opts->{pool}; # todo: do we need that?
5292                 if ($user ne 'root@pam') {
5293                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5294                 }
5295
5296                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5297             }
5298         }
5299
5300         foreach my $devname (keys %$devinfo) {
5301             die "found no device mapping information for device '$devname'\n"
5302                 if !$devinfo->{$devname}->{virtdev};
5303         }
5304
5305         my $cfg = cfs_read_file('storage.cfg');
5306
5307         # create empty/temp config
5308         if ($oldconf) {
5309             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5310             foreach_drive($oldconf, sub {
5311                 my ($ds, $drive) = @_;
5312
5313                 return if drive_is_cdrom($drive);
5314
5315                 my $volid = $drive->{file};
5316
5317                 return if !$volid || $volid =~ m|^/|;
5318
5319                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5320                 return if !$path || !$owner || ($owner != $vmid);
5321
5322                 # Note: only delete disk we want to restore
5323                 # other volumes will become unused
5324                 if ($virtdev_hash->{$ds}) {
5325                     PVE::Storage::vdisk_free($cfg, $volid);
5326                 }
5327             });
5328         }
5329
5330         my $map = {};
5331         foreach my $virtdev (sort keys %$virtdev_hash) {
5332             my $d = $virtdev_hash->{$virtdev};
5333             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5334             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5335
5336             # test if requested format is supported
5337             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5338             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5339             $d->{format} = $defFormat if !$supported;
5340
5341             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5342                                                   $d->{format}, undef, $alloc_size);
5343             print STDERR "new volume ID is '$volid'\n";
5344             $d->{volid} = $volid;
5345             my $path = PVE::Storage::path($cfg, $volid);
5346
5347             PVE::Storage::activate_volumes($cfg,[$volid]);
5348
5349             my $write_zeros = 1;
5350             # fixme: what other storages types initialize volumes with zero?
5351             if ($scfg->{type} eq 'dir' || $scfg->{type} eq 'nfs' || $scfg->{type} eq 'glusterfs' ||
5352                 $scfg->{type} eq 'sheepdog' || $scfg->{type} eq 'rbd') {
5353                 $write_zeros = 0;
5354             }
5355
5356             print $fifofh "${write_zeros}:$d->{devname}=$path\n";
5357
5358             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5359             $map->{$virtdev} = $volid;
5360         }
5361
5362         $fh->seek(0, 0) || die "seek failed - $!\n";
5363
5364         my $outfd = new IO::File ($tmpfn, "w") ||
5365             die "unable to write config for VM $vmid\n";
5366
5367         my $cookie = { netcount => 0 };
5368         while (defined(my $line = <$fh>)) {
5369             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5370         }
5371
5372         $fh->close();
5373         $outfd->close();
5374     };
5375
5376     eval {
5377         # enable interrupts
5378         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5379             die "interrupted by signal\n";
5380         };
5381         local $SIG{ALRM} = sub { die "got timeout\n"; };
5382
5383         $oldtimeout = alarm($timeout);
5384
5385         my $parser = sub {
5386             my $line = shift;
5387
5388             print "$line\n";
5389
5390             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5391                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5392                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5393             } elsif ($line =~ m/^CTIME: /) {
5394                 # we correctly received the vma config, so we can disable
5395                 # the timeout now for disk allocation (set to 10 minutes, so
5396                 # that we always timeout if something goes wrong)
5397                 alarm(600);
5398                 &$print_devmap();
5399                 print $fifofh "done\n";
5400                 my $tmp = $oldtimeout || 0;
5401                 $oldtimeout = undef;
5402                 alarm($tmp);
5403                 close($fifofh);
5404             }
5405         };
5406
5407         print "restore vma archive: $cmd\n";
5408         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5409     };
5410     my $err = $@;
5411
5412     alarm($oldtimeout) if $oldtimeout;
5413
5414     my $vollist = [];
5415     foreach my $devname (keys %$devinfo) {
5416         my $volid = $devinfo->{$devname}->{volid};
5417         push @$vollist, $volid if $volid;
5418     }
5419
5420     my $cfg = cfs_read_file('storage.cfg');
5421     PVE::Storage::deactivate_volumes($cfg, $vollist);
5422
5423     unlink $mapfifo;
5424
5425     if ($err) {
5426         rmtree $tmpdir;
5427         unlink $tmpfn;
5428
5429         foreach my $devname (keys %$devinfo) {
5430             my $volid = $devinfo->{$devname}->{volid};
5431             next if !$volid;
5432             eval {
5433                 if ($volid =~ m|^/|) {
5434                     unlink $volid || die 'unlink failed\n';
5435                 } else {
5436                     PVE::Storage::vdisk_free($cfg, $volid);
5437                 }
5438                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5439             };
5440             print STDERR "unable to cleanup '$volid' - $@" if $@;
5441         }
5442         die $err;
5443     }
5444
5445     rmtree $tmpdir;
5446
5447     rename($tmpfn, $conffile) ||
5448         die "unable to commit configuration file '$conffile'\n";
5449
5450     PVE::Cluster::cfs_update(); # make sure we read new file
5451
5452     eval { rescan($vmid, 1); };
5453     warn $@ if $@;
5454 }
5455
5456 sub restore_tar_archive {
5457     my ($archive, $vmid, $user, $opts) = @_;
5458
5459     if ($archive ne '-') {
5460         my $firstfile = tar_archive_read_firstfile($archive);
5461         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5462             if $firstfile ne 'qemu-server.conf';
5463     }
5464
5465     my $storecfg = cfs_read_file('storage.cfg');
5466
5467     # destroy existing data - keep empty config
5468     my $vmcfgfn = config_file($vmid);
5469     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5470
5471     my $tocmd = "/usr/lib/qemu-server/qmextract";
5472
5473     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5474     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5475     $tocmd .= ' --prealloc' if $opts->{prealloc};
5476     $tocmd .= ' --info' if $opts->{info};
5477
5478     # tar option "xf" does not autodetect compression when read from STDIN,
5479     # so we pipe to zcat
5480     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5481         PVE::Tools::shellquote("--to-command=$tocmd");
5482
5483     my $tmpdir = "/var/tmp/vzdumptmp$$";
5484     mkpath $tmpdir;
5485
5486     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5487     local $ENV{VZDUMP_VMID} = $vmid;
5488     local $ENV{VZDUMP_USER} = $user;
5489
5490     my $conffile = config_file($vmid);
5491     my $tmpfn = "$conffile.$$.tmp";
5492
5493     # disable interrupts (always do cleanups)
5494     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5495         print STDERR "got interrupt - ignored\n";
5496     };
5497
5498     eval {
5499         # enable interrupts
5500         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5501             die "interrupted by signal\n";
5502         };
5503
5504         if ($archive eq '-') {
5505             print "extracting archive from STDIN\n";
5506             run_command($cmd, input => "<&STDIN");
5507         } else {
5508             print "extracting archive '$archive'\n";
5509             run_command($cmd);
5510         }
5511
5512         return if $opts->{info};
5513
5514         # read new mapping
5515         my $map = {};
5516         my $statfile = "$tmpdir/qmrestore.stat";
5517         if (my $fd = IO::File->new($statfile, "r")) {
5518             while (defined (my $line = <$fd>)) {
5519                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5520                     $map->{$1} = $2 if $1;
5521                 } else {
5522                     print STDERR "unable to parse line in statfile - $line\n";
5523                 }
5524             }
5525             $fd->close();
5526         }
5527
5528         my $confsrc = "$tmpdir/qemu-server.conf";
5529
5530         my $srcfd = new IO::File($confsrc, "r") ||
5531             die "unable to open file '$confsrc'\n";
5532
5533         my $outfd = new IO::File ($tmpfn, "w") ||
5534             die "unable to write config for VM $vmid\n";
5535
5536         my $cookie = { netcount => 0 };
5537         while (defined (my $line = <$srcfd>)) {
5538             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5539         }
5540
5541         $srcfd->close();
5542         $outfd->close();
5543     };
5544     my $err = $@;
5545
5546     if ($err) {
5547
5548         unlink $tmpfn;
5549
5550         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
5551
5552         die $err;
5553     }
5554
5555     rmtree $tmpdir;
5556
5557     rename $tmpfn, $conffile ||
5558         die "unable to commit configuration file '$conffile'\n";
5559
5560     PVE::Cluster::cfs_update(); # make sure we read new file
5561
5562     eval { rescan($vmid, 1); };
5563     warn $@ if $@;
5564 };
5565
5566
5567 # Internal snapshots
5568
5569 # NOTE: Snapshot create/delete involves several non-atomic
5570 # action, and can take a long time.
5571 # So we try to avoid locking the file and use 'lock' variable
5572 # inside the config file instead.
5573
5574 my $snapshot_copy_config = sub {
5575     my ($source, $dest) = @_;
5576
5577     foreach my $k (keys %$source) {
5578         next if $k eq 'snapshots';
5579         next if $k eq 'snapstate';
5580         next if $k eq 'snaptime';
5581         next if $k eq 'vmstate';
5582         next if $k eq 'lock';
5583         next if $k eq 'digest';
5584         next if $k eq 'description';
5585         next if $k =~ m/^unused\d+$/;
5586
5587         $dest->{$k} = $source->{$k};
5588     }
5589 };
5590
5591 my $snapshot_apply_config = sub {
5592     my ($conf, $snap) = @_;
5593
5594     # copy snapshot list
5595     my $newconf = {
5596         snapshots => $conf->{snapshots},
5597     };
5598
5599     # keep description and list of unused disks
5600     foreach my $k (keys %$conf) {
5601         next if !($k =~ m/^unused\d+$/ || $k eq 'description');
5602         $newconf->{$k} = $conf->{$k};
5603     }
5604
5605     &$snapshot_copy_config($snap, $newconf);
5606
5607     return $newconf;
5608 };
5609
5610 sub foreach_writable_storage {
5611     my ($conf, $func) = @_;
5612
5613     my $sidhash = {};
5614
5615     foreach my $ds (keys %$conf) {
5616         next if !valid_drivename($ds);
5617
5618         my $drive = parse_drive($ds, $conf->{$ds});
5619         next if !$drive;
5620         next if drive_is_cdrom($drive);
5621
5622         my $volid = $drive->{file};
5623
5624         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5625         $sidhash->{$sid} = $sid if $sid;
5626     }
5627
5628     foreach my $sid (sort keys %$sidhash) {
5629         &$func($sid);
5630     }
5631 }
5632
5633 my $alloc_vmstate_volid = sub {
5634     my ($storecfg, $vmid, $conf, $snapname) = @_;
5635
5636     # Note: we try to be smart when selecting a $target storage
5637
5638     my $target;
5639
5640     # search shared storage first
5641     foreach_writable_storage($conf, sub {
5642         my ($sid) = @_;
5643         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
5644         return if !$scfg->{shared};
5645
5646         $target = $sid if !$target || $scfg->{path}; # prefer file based storage
5647     });
5648
5649     if (!$target) {
5650         # now search local storage
5651         foreach_writable_storage($conf, sub {
5652             my ($sid) = @_;
5653             my $scfg = PVE::Storage::storage_config($storecfg, $sid);
5654             return if $scfg->{shared};
5655
5656             $target = $sid if !$target || $scfg->{path}; # prefer file based storage;
5657         });
5658     }
5659
5660     $target = 'local' if !$target;
5661
5662     my $driver_state_size = 500; # assume 32MB is enough to safe all driver state;
5663     # we abort live save after $conf->{memory}, so we need at max twice that space
5664     my $size = $conf->{memory}*2 + $driver_state_size;
5665
5666     my $name = "vm-$vmid-state-$snapname";
5667     my $scfg = PVE::Storage::storage_config($storecfg, $target);
5668     $name .= ".raw" if $scfg->{path}; # add filename extension for file base storage
5669     my $volid = PVE::Storage::vdisk_alloc($storecfg, $target, $vmid, 'raw', $name, $size*1024);
5670
5671     return $volid;
5672 };
5673
5674 my $snapshot_prepare = sub {
5675     my ($vmid, $snapname, $save_vmstate, $comment) = @_;
5676
5677     my $snap;
5678
5679     my $updatefn =  sub {
5680
5681         my $conf = load_config($vmid);
5682
5683         die "you can't take a snapshot if it's a template\n"
5684             if is_template($conf);
5685
5686         check_lock($conf);
5687
5688         $conf->{lock} = 'snapshot';
5689
5690         die "snapshot name '$snapname' already used\n"
5691             if defined($conf->{snapshots}->{$snapname});
5692
5693         my $storecfg = PVE::Storage::config();
5694         die "snapshot feature is not available" if !has_feature('snapshot', $conf, $storecfg);
5695
5696         $snap = $conf->{snapshots}->{$snapname} = {};
5697
5698         if ($save_vmstate && check_running($vmid)) {
5699             $snap->{vmstate} = &$alloc_vmstate_volid($storecfg, $vmid, $conf, $snapname);
5700         }
5701
5702         &$snapshot_copy_config($conf, $snap);
5703
5704         $snap->{snapstate} = "prepare";
5705         $snap->{snaptime} = time();
5706         $snap->{description} = $comment if $comment;
5707
5708         # always overwrite machine if we save vmstate. This makes sure we
5709         # can restore it later using correct machine type
5710         $snap->{machine} = get_current_qemu_machine($vmid) if $snap->{vmstate};
5711
5712         update_config_nolock($vmid, $conf, 1);
5713     };
5714
5715     lock_config($vmid, $updatefn);
5716
5717     return $snap;
5718 };
5719
5720 my $snapshot_commit = sub {
5721     my ($vmid, $snapname) = @_;
5722
5723     my $updatefn = sub {
5724
5725         my $conf = load_config($vmid);
5726
5727         die "missing snapshot lock\n"
5728             if !($conf->{lock} && $conf->{lock} eq 'snapshot');
5729
5730         my $has_machine_config = defined($conf->{machine});
5731
5732         my $snap = $conf->{snapshots}->{$snapname};
5733
5734         die "snapshot '$snapname' does not exist\n" if !defined($snap);
5735
5736         die "wrong snapshot state\n"
5737             if !($snap->{snapstate} && $snap->{snapstate} eq "prepare");
5738
5739         delete $snap->{snapstate};
5740         delete $conf->{lock};
5741
5742         my $newconf = &$snapshot_apply_config($conf, $snap);
5743
5744         delete $newconf->{machine} if !$has_machine_config;
5745
5746         $newconf->{parent} = $snapname;
5747
5748         update_config_nolock($vmid, $newconf, 1);
5749     };
5750
5751     lock_config($vmid, $updatefn);
5752 };
5753
5754 sub snapshot_rollback {
5755     my ($vmid, $snapname) = @_;
5756
5757     my $prepare = 1;
5758
5759     my $storecfg = PVE::Storage::config();
5760
5761     my $conf = load_config($vmid);
5762
5763     my $get_snapshot_config = sub {
5764
5765         die "you can't rollback if vm is a template\n" if is_template($conf);
5766
5767         my $res = $conf->{snapshots}->{$snapname};
5768
5769         die "snapshot '$snapname' does not exist\n" if !defined($res);
5770
5771         return $res;
5772     };
5773
5774     my $snap = &$get_snapshot_config();
5775
5776     foreach_drive($snap, sub {
5777         my ($ds, $drive) = @_;
5778
5779         return if drive_is_cdrom($drive);
5780
5781         my $volid = $drive->{file};
5782
5783         PVE::Storage::volume_rollback_is_possible($storecfg, $volid, $snapname);
5784     });
5785
5786     my $updatefn = sub {
5787
5788         $conf = load_config($vmid);
5789
5790         $snap = &$get_snapshot_config();
5791
5792         die "unable to rollback to incomplete snapshot (snapstate = $snap->{snapstate})\n"
5793             if $snap->{snapstate};
5794
5795         if ($prepare) {
5796             check_lock($conf);
5797             vm_stop($storecfg, $vmid, undef, undef, 5, undef, undef);
5798         }
5799
5800         die "unable to rollback vm $vmid: vm is running\n"
5801             if check_running($vmid);
5802
5803         if ($prepare) {
5804             $conf->{lock} = 'rollback';
5805         } else {
5806             die "got wrong lock\n" if !($conf->{lock} && $conf->{lock} eq 'rollback');
5807             delete $conf->{lock};
5808         }
5809
5810         my $forcemachine;
5811
5812         if (!$prepare) {
5813             my $has_machine_config = defined($conf->{machine});
5814
5815             # copy snapshot config to current config
5816             $conf = &$snapshot_apply_config($conf, $snap);
5817             $conf->{parent} = $snapname;
5818
5819             # Note: old code did not store 'machine', so we try to be smart
5820             # and guess the snapshot was generated with kvm 1.4 (pc-i440fx-1.4).
5821             $forcemachine = $conf->{machine} || 'pc-i440fx-1.4';
5822             # we remove the 'machine' configuration if not explicitly specified
5823             # in the original config.
5824             delete $conf->{machine} if $snap->{vmstate} && !$has_machine_config;
5825         }
5826
5827         update_config_nolock($vmid, $conf, 1);
5828
5829         if (!$prepare && $snap->{vmstate}) {
5830             my $statefile = PVE::Storage::path($storecfg, $snap->{vmstate});
5831             vm_start($storecfg, $vmid, $statefile, undef, undef, undef, $forcemachine);
5832         }
5833     };
5834
5835     lock_config($vmid, $updatefn);
5836
5837     foreach_drive($snap, sub {
5838         my ($ds, $drive) = @_;
5839
5840         return if drive_is_cdrom($drive);
5841
5842         my $volid = $drive->{file};
5843         my $device = "drive-$ds";
5844
5845         PVE::Storage::volume_snapshot_rollback($storecfg, $volid, $snapname);
5846     });
5847
5848     $prepare = 0;
5849     lock_config($vmid, $updatefn);
5850 }
5851
5852 my $savevm_wait = sub {
5853     my ($vmid) = @_;
5854
5855     for(;;) {
5856         my $stat = vm_mon_cmd_nocheck($vmid, "query-savevm");
5857         if (!$stat->{status}) {
5858             die "savevm not active\n";
5859         } elsif ($stat->{status} eq 'active') {
5860             sleep(1);
5861             next;
5862         } elsif ($stat->{status} eq 'completed') {
5863             last;
5864         } else {
5865             die "query-savevm returned status '$stat->{status}'\n";
5866         }
5867     }
5868 };
5869
5870 sub do_snapshots_with_qemu {
5871     my ($storecfg, $volid) = @_;
5872
5873     my $storage_name = PVE::Storage::parse_volume_id($volid);
5874
5875     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
5876         && !$storecfg->{ids}->{$storage_name}->{krbd}){
5877         return 1;
5878     }
5879
5880     if ($volid =~ m/\.(qcow2|qed)$/){
5881         return 1;
5882     }
5883
5884     return undef;
5885 }
5886
5887 sub snapshot_create {
5888     my ($vmid, $snapname, $save_vmstate, $comment) = @_;
5889
5890     my $snap = &$snapshot_prepare($vmid, $snapname, $save_vmstate, $comment);
5891
5892     $save_vmstate = 0 if !$snap->{vmstate}; # vm is not running
5893
5894     my $config = load_config($vmid);
5895
5896     my $running = check_running($vmid);
5897
5898     my $freezefs = $running && $config->{agent};
5899     $freezefs = 0 if $snap->{vmstate}; # not needed if we save RAM
5900
5901     my $drivehash = {};
5902
5903     if ($freezefs) {
5904         eval { vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
5905         warn "guest-fsfreeze-freeze problems - $@" if $@;
5906     }
5907
5908     eval {
5909         # create internal snapshots of all drives
5910
5911         my $storecfg = PVE::Storage::config();
5912
5913         if ($running) {
5914             if ($snap->{vmstate}) {
5915                 my $path = PVE::Storage::path($storecfg, $snap->{vmstate});
5916                 vm_mon_cmd($vmid, "savevm-start", statefile => $path);
5917                 &$savevm_wait($vmid);
5918             } else {
5919                 vm_mon_cmd($vmid, "savevm-start");
5920             }
5921         };
5922
5923         foreach_drive($snap, sub {
5924             my ($ds, $drive) = @_;
5925
5926             return if drive_is_cdrom($drive);
5927
5928             my $volid = $drive->{file};
5929             my $device = "drive-$ds";
5930
5931             qemu_volume_snapshot($vmid, $device, $storecfg, $volid, $snapname);
5932             $drivehash->{$ds} = 1;
5933        });
5934     };
5935     my $err = $@;
5936
5937     if ($running) {
5938         eval { vm_mon_cmd($vmid, "savevm-end")  };
5939         warn $@ if $@;
5940
5941         if ($freezefs) {
5942             eval { vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
5943             warn "guest-fsfreeze-thaw problems - $@" if $@;
5944         }
5945
5946         # savevm-end is async, we need to wait
5947         for (;;) {
5948             my $stat = vm_mon_cmd_nocheck($vmid, "query-savevm");
5949             if (!$stat->{bytes}) {
5950                 last;
5951             } else {
5952                 print "savevm not yet finished\n";
5953                 sleep(1);
5954                 next;
5955             }
5956         }
5957     }
5958
5959     if ($err) {
5960         warn "snapshot create failed: starting cleanup\n";
5961         eval { snapshot_delete($vmid, $snapname, 0, $drivehash); };
5962         warn $@ if $@;
5963         die $err;
5964     }
5965
5966     &$snapshot_commit($vmid, $snapname);
5967 }
5968
5969 # Note: $drivehash is only set when called from snapshot_create.
5970 sub snapshot_delete {
5971     my ($vmid, $snapname, $force, $drivehash) = @_;
5972
5973     my $prepare = 1;
5974
5975     my $snap;
5976     my $unused = [];
5977
5978     my $unlink_parent = sub {
5979         my ($confref, $new_parent) = @_;
5980
5981         if ($confref->{parent} && $confref->{parent} eq $snapname) {
5982             if ($new_parent) {
5983                 $confref->{parent} = $new_parent;
5984             } else {
5985                 delete $confref->{parent};
5986             }
5987         }
5988     };
5989
5990     my $updatefn =  sub {
5991         my ($remove_drive) = @_;
5992
5993         my $conf = load_config($vmid);
5994
5995         if (!$drivehash) {
5996             check_lock($conf);
5997             die "you can't delete a snapshot if vm is a template\n"
5998                 if is_template($conf);
5999         }
6000
6001         $snap = $conf->{snapshots}->{$snapname};
6002
6003         die "snapshot '$snapname' does not exist\n" if !defined($snap);
6004
6005         # remove parent refs
6006         if (!$prepare) {
6007             &$unlink_parent($conf, $snap->{parent});
6008             foreach my $sn (keys %{$conf->{snapshots}}) {
6009                 next if $sn eq $snapname;
6010                 &$unlink_parent($conf->{snapshots}->{$sn}, $snap->{parent});
6011             }
6012         }
6013
6014         if ($remove_drive) {
6015             if ($remove_drive eq 'vmstate') {
6016                 delete $snap->{$remove_drive};
6017             } else {
6018                 my $drive = parse_drive($remove_drive, $snap->{$remove_drive});
6019                 my $volid = $drive->{file};
6020                 delete $snap->{$remove_drive};
6021                 add_unused_volume($conf, $volid);
6022             }
6023         }
6024
6025         if ($prepare) {
6026             $snap->{snapstate} = 'delete';
6027         } else {
6028             delete $conf->{snapshots}->{$snapname};
6029             delete $conf->{lock} if $drivehash;
6030             foreach my $volid (@$unused) {
6031                 add_unused_volume($conf, $volid);
6032             }
6033         }
6034
6035         update_config_nolock($vmid, $conf, 1);
6036     };
6037
6038     lock_config($vmid, $updatefn);
6039
6040     # now remove vmstate file
6041
6042     my $storecfg = PVE::Storage::config();
6043
6044     if ($snap->{vmstate}) {
6045         eval {  PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
6046         if (my $err = $@) {
6047             die $err if !$force;
6048             warn $err;
6049         }
6050         # save changes (remove vmstate from snapshot)
6051         lock_config($vmid, $updatefn, 'vmstate') if !$force;
6052     };
6053
6054     # now remove all internal snapshots
6055     foreach_drive($snap, sub {
6056         my ($ds, $drive) = @_;
6057
6058         return if drive_is_cdrom($drive);
6059
6060         my $volid = $drive->{file};
6061         my $device = "drive-$ds";
6062
6063         if (!$drivehash || $drivehash->{$ds}) {
6064             eval { qemu_volume_snapshot_delete($vmid, $device, $storecfg, $volid, $snapname); };
6065             if (my $err = $@) {
6066                 die $err if !$force;
6067                 warn $err;
6068             }
6069         }
6070
6071         # save changes (remove drive fron snapshot)
6072         lock_config($vmid, $updatefn, $ds) if !$force;
6073         push @$unused, $volid;
6074     });
6075
6076     # now cleanup config
6077     $prepare = 0;
6078     lock_config($vmid, $updatefn);
6079 }
6080
6081 sub has_feature {
6082     my ($feature, $conf, $storecfg, $snapname, $running) = @_;
6083
6084     my $err;
6085     foreach_drive($conf, sub {
6086         my ($ds, $drive) = @_;
6087
6088         return if drive_is_cdrom($drive);
6089         my $volid = $drive->{file};
6090         $err = 1 if !PVE::Storage::volume_has_feature($storecfg, $feature, $volid, $snapname, $running);
6091     });
6092
6093     return $err ? 0 : 1;
6094 }
6095
6096 sub template_create {
6097     my ($vmid, $conf, $disk) = @_;
6098
6099     my $storecfg = PVE::Storage::config();
6100
6101     foreach_drive($conf, sub {
6102         my ($ds, $drive) = @_;
6103
6104         return if drive_is_cdrom($drive);
6105         return if $disk && $ds ne $disk;
6106
6107         my $volid = $drive->{file};
6108         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6109
6110         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6111         $drive->{file} = $voliddst;
6112         $conf->{$ds} = print_drive($vmid, $drive);
6113         update_config_nolock($vmid, $conf, 1);
6114     });
6115 }
6116
6117 sub is_template {
6118     my ($conf) = @_;
6119
6120     return 1 if defined $conf->{template} && $conf->{template} == 1;
6121 }
6122
6123 sub qemu_img_convert {
6124     my ($src_volid, $dst_volid, $size, $snapname) = @_;
6125
6126     my $storecfg = PVE::Storage::config();
6127     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6128     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6129
6130     if ($src_storeid && $dst_storeid) {
6131         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6132         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6133
6134         my $src_format = qemu_img_format($src_scfg, $src_volname);
6135         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6136
6137         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6138         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6139
6140         my $cmd = [];
6141         push @$cmd, '/usr/bin/qemu-img', 'convert', '-t', 'writeback', '-p', '-n';
6142         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
6143         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path, $dst_path;
6144
6145         my $parser = sub {
6146             my $line = shift;
6147             if($line =~ m/\((\S+)\/100\%\)/){
6148                 my $percent = $1;
6149                 my $transferred = int($size * $percent / 100);
6150                 my $remaining = $size - $transferred;
6151
6152                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6153             }
6154
6155         };
6156
6157         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6158         my $err = $@;
6159         die "copy failed: $err" if $err;
6160     }
6161 }
6162
6163 sub qemu_img_format {
6164     my ($scfg, $volname) = @_;
6165
6166     if ($scfg->{path} && $volname =~ m/\.(raw|cow|qcow|qcow2|qed|vmdk|cloop)$/) {
6167         return $1;
6168     } else {
6169         return "raw";
6170     }
6171 }
6172
6173 sub qemu_drive_mirror {
6174     my ($vmid, $drive, $dst_volid, $vmiddst) = @_;
6175
6176     my $storecfg = PVE::Storage::config();
6177     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6178
6179     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6180
6181     my $format = qemu_img_format($dst_scfg, $dst_volname);
6182
6183     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6184
6185     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $dst_path };
6186     $opts->{format} = $format if $format;
6187
6188     print "drive mirror is starting (scanning bitmap) : this step can take some minutes/hours, depend of disk size and storage speed\n";
6189
6190     eval {
6191     vm_mon_cmd($vmid, "drive-mirror", %$opts);
6192         while (1) {
6193             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6194             my $stat = @$stats[0];
6195             die "mirroring job seem to have die. Maybe do you have bad sectors?" if !$stat;
6196             die "error job is not mirroring" if $stat->{type} ne "mirror";
6197
6198             my $busy = $stat->{busy};
6199             my $ready = $stat->{ready};
6200
6201             if (my $total = $stat->{len}) {
6202                 my $transferred = $stat->{offset} || 0;
6203                 my $remaining = $total - $transferred;
6204                 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6205
6206                 print "transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6207             }
6208
6209
6210             if ($stat->{ready} eq 'true') {
6211
6212                 last if $vmiddst != $vmid;
6213
6214                 # try to switch the disk if source and destination are on the same guest
6215                 eval { vm_mon_cmd($vmid, "block-job-complete", device => "drive-$drive") };
6216                 last if !$@;
6217                 die $@ if $@ !~ m/cannot be completed/;
6218             }
6219             sleep 1;
6220         }
6221
6222
6223     };
6224     my $err = $@;
6225
6226     my $cancel_job = sub {
6227         vm_mon_cmd($vmid, "block-job-cancel", device => "drive-$drive");
6228         while (1) {
6229             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6230             my $stat = @$stats[0];
6231             last if !$stat;
6232             sleep 1;
6233         }
6234     };
6235
6236     if ($err) {
6237         eval { &$cancel_job(); };
6238         die "mirroring error: $err";
6239     }
6240
6241     if ($vmiddst != $vmid) {
6242         # if we clone a disk for a new target vm, we don't switch the disk
6243         &$cancel_job(); # so we call block-job-cancel
6244     }
6245 }
6246
6247 sub clone_disk {
6248     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6249         $newvmid, $storage, $format, $full, $newvollist) = @_;
6250
6251     my $newvolid;
6252
6253     if (!$full) {
6254         print "create linked clone of drive $drivename ($drive->{file})\n";
6255         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6256         push @$newvollist, $newvolid;
6257     } else {
6258         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6259         $storeid = $storage if $storage;
6260
6261         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6262         if (!$format) {
6263             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6264             $format = qemu_img_format($scfg, $volname);
6265         }
6266
6267         # test if requested format is supported - else use default
6268         my $supported = grep { $_ eq $format } @$validFormats;
6269         $format = $defFormat if !$supported;
6270
6271         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6272
6273         print "create full clone of drive $drivename ($drive->{file})\n";
6274         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $format, undef, ($size/1024));
6275         push @$newvollist, $newvolid;
6276
6277         PVE::Storage::activate_volumes($storecfg, $newvollist);
6278
6279         if (!$running || $snapname) {
6280             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname);
6281         } else {
6282             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid);
6283         }
6284     }
6285
6286     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6287
6288     my $disk = $drive;
6289     $disk->{format} = undef;
6290     $disk->{file} = $newvolid;
6291     $disk->{size} = $size;
6292
6293     return $disk;
6294 }
6295
6296 # this only works if VM is running
6297 sub get_current_qemu_machine {
6298     my ($vmid) = @_;
6299
6300     my $cmd = { execute => 'query-machines', arguments => {} };
6301     my $res = vm_qmp_command($vmid, $cmd);
6302
6303     my ($current, $default);
6304     foreach my $e (@$res) {
6305         $default = $e->{name} if $e->{'is-default'};
6306         $current = $e->{name} if $e->{'is-current'};
6307     }
6308
6309     # fallback to the default machine if current is not supported by qemu
6310     return $current || $default || 'pc';
6311 }
6312
6313 sub qemu_machine_feature_enabled {
6314     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6315
6316     my $current_major;
6317     my $current_minor;
6318
6319     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6320
6321         $current_major = $3;
6322         $current_minor = $4;
6323
6324     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6325
6326         $current_major = $1;
6327         $current_minor = $2;
6328     }
6329
6330     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
6331
6332
6333 }
6334
6335 sub lspci {
6336
6337     my $devices = {};
6338
6339     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6340             my (undef, $id, $function) = @_;
6341             my $res = { id => $id, function => $function};
6342             push @{$devices->{$id}}, $res;
6343     });
6344
6345     return $devices;
6346 }
6347
6348 sub vm_iothreads_list {
6349     my ($vmid) = @_;
6350
6351     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6352
6353     my $iothreads = {};
6354     foreach my $iothread (@$res) {
6355         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6356     }
6357
6358     return $iothreads;
6359 }
6360
6361 sub scsihw_infos {
6362     my ($conf, $drive) = @_;
6363
6364     my $maxdev = 0;
6365
6366     if ($conf->{scsihw} && ($conf->{scsihw} =~ m/^lsi/)) {
6367         $maxdev = 7;
6368     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6369         $maxdev = 1;
6370     } else {
6371         $maxdev = 256;
6372     }
6373
6374     my $controller = int($drive->{index} / $maxdev);
6375     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6376
6377     return ($maxdev, $controller, $controller_prefix);
6378 }
6379
6380 # bash completion helper
6381
6382 sub complete_backup_archives {
6383     my ($cmdname, $pname, $cvalue) = @_;
6384
6385     my $cfg = PVE::Storage::config();
6386
6387     my $storeid;
6388
6389     if ($cvalue =~ m/^([^:]+):/) {
6390         $storeid = $1;
6391     }
6392
6393     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6394
6395     my $res = [];
6396     foreach my $id (keys %$data) {
6397         foreach my $item (@{$data->{$id}}) {
6398             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6399             push @$res, $item->{volid} if defined($item->{volid});
6400         }
6401     }
6402
6403     return $res;
6404 }
6405
6406 my $complete_vmid_full = sub {
6407     my ($running) = @_;
6408
6409     my $idlist = vmstatus();
6410
6411     my $res = [];
6412
6413     foreach my $id (keys %$idlist) {
6414         my $d = $idlist->{$id};
6415         if (defined($running)) {
6416             next if $d->{template};
6417             next if $running && $d->{status} ne 'running';
6418             next if !$running && $d->{status} eq 'running';
6419         }
6420         push @$res, $id;
6421
6422     }
6423     return $res;
6424 };
6425
6426 sub complete_vmid {
6427     return &$complete_vmid_full();
6428 }
6429
6430 sub complete_vmid_stopped {
6431     return &$complete_vmid_full(0);
6432 }
6433
6434 sub complete_vmid_running {
6435     return &$complete_vmid_full(1);
6436 }
6437
6438 sub complete_storage {
6439
6440     my $cfg = PVE::Storage::config();
6441     my $ids = $cfg->{ids};
6442
6443     my $res = [];
6444     foreach my $sid (keys %$ids) {
6445         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6446         push @$res, $sid;
6447     }
6448
6449     return $res;
6450 }
6451
6452 1;