PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QMPClient;
  31 use PVE::RPCEnvironment;
  32 use Time::HiRes qw(gettimeofday);
  33
  34 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  35
  36 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  37
  38 # Note about locking: we use flock on the config file protect
  39 # against concurent actions.
  40 # Aditionaly, we have a 'lock' setting in the config file. This
  41 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  42 # allowed when such lock is set. But you can ignore this kind of
  43 # lock with the --skiplock flag.
  44
  45 cfs_register_file('/qemu-server/',
  46                   \&parse_vm_config,
  47                   \&write_vm_config);
  48
  49 PVE::JSONSchema::register_standard_option('skiplock', {
  50     description => "Ignore locks - only root is allowed to use this option.",
  51     type => 'boolean',
  52     optional => 1,
  53 });
  54
  55 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  56     description => "Some command save/restore state from this location.",
  57     type => 'string',
  58     maxLength => 128,
  59     optional => 1,
  60 });
  61
  62 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  63     description => "The name of the snapshot.",
  64     type => 'string', format => 'pve-configid',
  65     maxLength => 40,
  66 });
  67
  68 #no warnings 'redefine';
  69
  70 sub cgroups_write {
  71    my ($controller, $vmid, $option, $value) = @_;
  72
  73    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  74    PVE::ProcFSTools::write_proc_entry($path, $value);
  75
  76 }
  77
  78 my $nodename = PVE::INotify::nodename();
  79
  80 mkdir "/etc/pve/nodes/$nodename";
  81 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  82 mkdir $confdir;
  83
  84 my $var_run_tmpdir = "/var/run/qemu-server";
  85 mkdir $var_run_tmpdir;
  86
  87 my $lock_dir = "/var/lock/qemu-server";
  88 mkdir $lock_dir;
  89
  90 my $pcisysfs = "/sys/bus/pci";
  91
  92 my $confdesc = {
  93     onboot => {
  94         optional => 1,
  95         type => 'boolean',
  96         description => "Specifies whether a VM will be started during system bootup.",
  97         default => 0,
  98     },
  99     autostart => {
 100         optional => 1,
 101         type => 'boolean',
 102         description => "Automatic restart after crash (currently ignored).",
 103         default => 0,
 104     },
 105     hotplug => {
 106         optional => 1,
 107         type => 'string', format => 'pve-hotplug-features',
 108         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 109         default => 'network,disk,usb',
 110     },
 111     reboot => {
 112         optional => 1,
 113         type => 'boolean',
 114         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 115         default => 1,
 116     },
 117     lock => {
 118         optional => 1,
 119         type => 'string',
 120         description => "Lock/unlock the VM.",
 121         enum => [qw(migrate backup snapshot rollback)],
 122     },
 123     cpulimit => {
 124         optional => 1,
 125         type => 'number',
 126         description => "Limit of CPU usage. Note if the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 127         minimum => 0,
 128         maximum => 128,
 129         default => 0,
 130     },
 131     cpuunits => {
 132         optional => 1,
 133         type => 'integer',
 134         description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 135         minimum => 0,
 136         maximum => 500000,
 137         default => 1000,
 138     },
 139     memory => {
 140         optional => 1,
 141         type => 'integer',
 142         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 143         minimum => 16,
 144         default => 512,
 145     },
 146     balloon => {
 147         optional => 1,
 148         type => 'integer',
 149         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 150         minimum => 0,
 151     },
 152     shares => {
 153         optional => 1,
 154         type => 'integer',
 155         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 156         minimum => 0,
 157         maximum => 50000,
 158         default => 1000,
 159     },
 160     keyboard => {
 161         optional => 1,
 162         type => 'string',
 163         description => "Keybord layout for vnc server. Default is read from the datacenter configuration file.",
 164         enum => PVE::Tools::kvmkeymaplist(),
 165         default => 'en-us',
 166     },
 167     name => {
 168         optional => 1,
 169         type => 'string', format => 'dns-name',
 170         description => "Set a name for the VM. Only used on the configuration web interface.",
 171     },
 172     scsihw => {
 173         optional => 1,
 174         type => 'string',
 175         description => "scsi controller model",
 176         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 177         default => 'lsi',
 178     },
 179     description => {
 180         optional => 1,
 181         type => 'string',
 182         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 183     },
 184     ostype => {
 185         optional => 1,
 186         type => 'string',
 187         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 l24 l26 solaris)],
 188         description => <<EODESC,
 189 Used to enable special optimization/features for specific
 190 operating systems:
 191
 192 other  => unspecified OS
 193 wxp    => Microsoft Windows XP
 194 w2k    => Microsoft Windows 2000
 195 w2k3   => Microsoft Windows 2003
 196 w2k8   => Microsoft Windows 2008
 197 wvista => Microsoft Windows Vista
 198 win7   => Microsoft Windows 7
 199 win8   => Microsoft Windows 8/2012
 200 l24    => Linux 2.4 Kernel
 201 l26    => Linux 2.6/3.X Kernel
 202 solaris => solaris/opensolaris/openindiania kernel
 203
 204 other|l24|l26|solaris                       ... no special behaviour
 205 wxp|w2k|w2k3|w2k8|wvista|win7|win8  ... use --localtime switch
 206 EODESC
 207     },
 208     boot => {
 209         optional => 1,
 210         type => 'string',
 211         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 212         pattern => '[acdn]{1,4}',
 213         default => 'cdn',
 214     },
 215     bootdisk => {
 216         optional => 1,
 217         type => 'string', format => 'pve-qm-bootdisk',
 218         description => "Enable booting from specified disk.",
 219         pattern => '(ide|sata|scsi|virtio)\d+',
 220     },
 221     smp => {
 222         optional => 1,
 223         type => 'integer',
 224         description => "The number of CPUs. Please use option -sockets instead.",
 225         minimum => 1,
 226         default => 1,
 227     },
 228     sockets => {
 229         optional => 1,
 230         type => 'integer',
 231         description => "The number of CPU sockets.",
 232         minimum => 1,
 233         default => 1,
 234     },
 235     cores => {
 236         optional => 1,
 237         type => 'integer',
 238         description => "The number of cores per socket.",
 239         minimum => 1,
 240         default => 1,
 241     },
 242     numa => {
 243         optional => 1,
 244         type => 'boolean',
 245         description => "Enable/disable Numa.",
 246         default => 0,
 247     },
 248     vcpus => {
 249         optional => 1,
 250         type => 'integer',
 251         description => "Number of hotplugged vcpus.",
 252         minimum => 1,
 253         default => 0,
 254     },
 255     acpi => {
 256         optional => 1,
 257         type => 'boolean',
 258         description => "Enable/disable ACPI.",
 259         default => 1,
 260     },
 261     agent => {
 262         optional => 1,
 263         type => 'boolean',
 264         description => "Enable/disable Qemu GuestAgent.",
 265         default => 0,
 266     },
 267     kvm => {
 268         optional => 1,
 269         type => 'boolean',
 270         description => "Enable/disable KVM hardware virtualization.",
 271         default => 1,
 272     },
 273     tdf => {
 274         optional => 1,
 275         type => 'boolean',
 276         description => "Enable/disable time drift fix.",
 277         default => 0,
 278     },
 279     localtime => {
 280         optional => 1,
 281         type => 'boolean',
 282         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 283     },
 284     freeze => {
 285         optional => 1,
 286         type => 'boolean',
 287         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 288     },
 289     vga => {
 290         optional => 1,
 291         type => 'string',
 292         description => "Select VGA type. If you want to use high resolution modes (>= 1280x1024x16) then you should use option 'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and 'cirrur' for other OS types. Option 'qxl' enables the SPICE display sever. You can also run without any graphic card using a serial devive as terminal.",
 293         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 294     },
 295     watchdog => {
 296         optional => 1,
 297         type => 'string', format => 'pve-qm-watchdog',
 298         typetext => '[[model=]i6300esb|ib700] [,[action=]reset|shutdown|poweroff|pause|debug|none]',
 299         description => "Create a virtual hardware watchdog device.  Once enabled (by a guest action), the watchdog must be periodically polled by an agent inside the guest or else the guest will be restarted (or execute the action specified)",
 300     },
 301     startdate => {
 302         optional => 1,
 303         type => 'string',
 304         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 305         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 306         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 307         default => 'now',
 308     },
 309     startup =>  get_standard_option('pve-startup-order'),
 310     template => {
 311         optional => 1,
 312         type => 'boolean',
 313         description => "Enable/disable Template.",
 314         default => 0,
 315     },
 316     args => {
 317         optional => 1,
 318         type => 'string',
 319         description => <<EODESCR,
 320 Note: this option is for experts only. It allows you to pass arbitrary arguments to kvm, for example:
 321
 322 args: -no-reboot -no-hpet
 323 EODESCR
 324     },
 325     tablet => {
 326         optional => 1,
 327         type => 'boolean',
 328         default => 1,
 329         description => "Enable/disable the usb tablet device. This device is usually needed to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with normal VNC clients. If you're running lots of console-only guests on one host, you may consider disabling this to save some context switches. This is turned of by default if you use spice (vga=qxl).",
 330     },
 331     migrate_speed => {
 332         optional => 1,
 333         type => 'integer',
 334         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 335         minimum => 0,
 336         default => 0,
 337     },
 338     migrate_downtime => {
 339         optional => 1,
 340         type => 'number',
 341         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 342         minimum => 0,
 343         default => 0.1,
 344     },
 345     cdrom => {
 346         optional => 1,
 347         type => 'string', format => 'pve-qm-drive',
 348         typetext => 'volume',
 349         description => "This is an alias for option -ide2",
 350     },
 351     cpu => {
 352         optional => 1,
 353         description => "Emulated CPU type.",
 354         type => 'string',
 355         enum => [ qw(486 athlon pentium pentium2 pentium3 coreduo core2duo kvm32 kvm64 qemu32 qemu64 phenom Conroe Penryn Nehalem Westmere SandyBridge IvyBridge Haswell Broadwell Opteron_G1 Opteron_G2 Opteron_G3 Opteron_G4 Opteron_G5 host) ],
 356         default => 'kvm64',
 357     },
 358     parent => get_standard_option('pve-snapshot-name', {
 359         optional => 1,
 360         description => "Parent snapshot name. This is used internally, and should not be modified.",
 361     }),
 362     snaptime => {
 363         optional => 1,
 364         description => "Timestamp for snapshots.",
 365         type => 'integer',
 366         minimum => 0,
 367     },
 368     vmstate => {
 369         optional => 1,
 370         type => 'string', format => 'pve-volume-id',
 371         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 372     },
 373     machine => {
 374         description => "Specific the Qemu machine type.",
 375         type => 'string',
 376         pattern => '(pc|pc(-i440fx)?-\d+\.\d+|q35|pc-q35-\d+\.\d+)',
 377         maxLength => 40,
 378         optional => 1,
 379     },
 380     smbios1 => {
 381         description => "Specify SMBIOS type 1 fields.",
 382         type => 'string', format => 'pve-qm-smbios1',
 383         typetext => "[manufacturer=str][,product=str][,version=str][,serial=str] [,uuid=uuid][,sku=str][,family=str]",
 384         maxLength => 256,
 385         optional => 1,
 386     },
 387     protection => {
 388         optional => 1,
 389         type => 'boolean',
 390         description => "Sets the protection flag of the VM. This will prevent the remove operation.",
 391         default => 0,
 392     },
 393 };
 394
 395 # what about other qemu settings ?
 396 #cpu => 'string',
 397 #machine => 'string',
 398 #fda => 'file',
 399 #fdb => 'file',
 400 #mtdblock => 'file',
 401 #sd => 'file',
 402 #pflash => 'file',
 403 #snapshot => 'bool',
 404 #bootp => 'file',
 405 ##tftp => 'dir',
 406 ##smb => 'dir',
 407 #kernel => 'file',
 408 #append => 'string',
 409 #initrd => 'file',
 410 ##soundhw => 'string',
 411
 412 while (my ($k, $v) = each %$confdesc) {
 413     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 414 }
 415
 416 my $MAX_IDE_DISKS = 4;
 417 my $MAX_SCSI_DISKS = 14;
 418 my $MAX_VIRTIO_DISKS = 16;
 419 my $MAX_SATA_DISKS = 6;
 420 my $MAX_USB_DEVICES = 5;
 421 my $MAX_NETS = 32;
 422 my $MAX_UNUSED_DISKS = 8;
 423 my $MAX_HOSTPCI_DEVICES = 4;
 424 my $MAX_SERIAL_PORTS = 4;
 425 my $MAX_PARALLEL_PORTS = 3;
 426 my $MAX_NUMA = 8;
 427 my $MAX_MEM = 4194304;
 428 my $STATICMEM = 1024;
 429
 430 my $numadesc = {
 431     optional => 1,
 432     type => 'string', format => 'pve-qm-numanode',
 433     typetext => "cpus=<id[-id],memory=<mb>[[,hostnodes=<id[-id]>] [,policy=<preferred|bind|interleave>]]",
 434     description => "numa topology",
 435 };
 436 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 437
 438 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 439     $confdesc->{"numa$i"} = $numadesc;
 440 }
 441
 442 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 443                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 444                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 445 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 446
 447 my $netdesc = {
 448     optional => 1,
 449     type => 'string', format => 'pve-qm-net',
 450     typetext => "MODEL=XX:XX:XX:XX:XX:XX [,bridge=<dev>][,queues=<nbqueues>][,rate=<mbps>] [,tag=<vlanid>][,firewall=0|1],link_down=0|1]",
 451     description => <<EODESCR,
 452 Specify network devices.
 453
 454 MODEL is one of: $nic_model_list_txt
 455
 456 XX:XX:XX:XX:XX:XX should be an unique MAC address. This is
 457 automatically generated if not specified.
 458
 459 The bridge parameter can be used to automatically add the interface to a bridge device. The Proxmox VE standard bridge is called 'vmbr0'.
 460
 461 Option 'rate' is used to limit traffic bandwidth from and to this interface. It is specified as floating point number, unit is 'Megabytes per second'.
 462
 463 If you specify no bridge, we create a kvm 'user' (NATed) network device, which provides DHCP and DNS services. The following addresses are used:
 464
 465 10.0.2.2   Gateway
 466 10.0.2.3   DNS Server
 467 10.0.2.4   SMB Server
 468
 469 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 470
 471 EODESCR
 472 };
 473 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 474
 475 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 476     $confdesc->{"net$i"} = $netdesc;
 477 }
 478
 479 my $drivename_hash;
 480
 481 my $idedesc = {
 482     optional => 1,
 483     type => 'string', format => 'pve-qm-drive',
 484     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads] [,discard=ignore|on] [,serial=serial][,model=model]',
 485     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
 486 };
 487 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
 488
 489 my $scsidesc = {
 490     optional => 1,
 491     type => 'string', format => 'pve-qm-drive',
 492     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads] [,discard=ignore|on] [,iothread=on] [,queues=<nbqueues>] [,serial=serial]',
 493     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
 494 };
 495 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 496
 497 my $satadesc = {
 498     optional => 1,
 499     type => 'string', format => 'pve-qm-drive',
 500     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads]  [,discard=ignore|on] [,serial=serial]',
 501     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
 502 };
 503 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
 504
 505 my $virtiodesc = {
 506     optional => 1,
 507     type => 'string', format => 'pve-qm-drive',
 508     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads]  [,discard=ignore|on] [,iothread=on] [,serial=serial]',
 509     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
 510 };
 511 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
 512
 513 my $usbdesc = {
 514     optional => 1,
 515     type => 'string', format => 'pve-qm-usb-device',
 516     typetext => 'host=HOSTUSBDEVICE|spice',
 517     description => <<EODESCR,
 518 Configure an USB device (n is 0 to 4). This can be used to
 519 pass-through usb devices to the guest. HOSTUSBDEVICE syntax is:
 520
 521 'bus-port(.port)*' (decimal numbers) or
 522 'vendor_id:product_id' (hexadeciaml numbers)
 523
 524 You can use the 'lsusb -t' command to list existing usb devices.
 525
 526 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 527
 528 The value 'spice' can be used to add a usb redirection devices for spice.
 529
 530 EODESCR
 531 };
 532 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
 533
 534 my $hostpcidesc = {
 535         optional => 1,
 536         type => 'string', format => 'pve-qm-hostpci',
 537         typetext => "[host=]HOSTPCIDEVICE [,rombar=on|off] [,pcie=0|1] [,x-vga=on|off]",
 538         description => <<EODESCR,
 539 Map host pci devices. HOSTPCIDEVICE syntax is:
 540
 541 'bus:dev.func' (hexadecimal numbers)
 542
 543 You can us the 'lspci' command to list existing pci devices.
 544
 545 The 'rombar' option determines whether or not the device's ROM will be visible in the guest's memory map (default is 'on').
 546
 547 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 548
 549 Experimental: user reported problems with this option.
 550 EODESCR
 551 };
 552 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
 553
 554 my $serialdesc = {
 555         optional => 1,
 556         type => 'string',
 557         pattern => '(/dev/.+|socket)',
 558         description =>  <<EODESCR,
 559 Create a serial device inside the VM (n is 0 to 3), and pass through a host serial device (i.e. /dev/ttyS0), or create a unix socket on the host side (use 'qm terminal' to open a terminal connection).
 560
 561 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 562
 563 Experimental: user reported problems with this option.
 564 EODESCR
 565 };
 566
 567 my $paralleldesc= {
 568         optional => 1,
 569         type => 'string',
 570         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
 571         description =>  <<EODESCR,
 572 Map host parallel devices (n is 0 to 2).
 573
 574 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 575
 576 Experimental: user reported problems with this option.
 577 EODESCR
 578 };
 579
 580 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
 581     $confdesc->{"parallel$i"} = $paralleldesc;
 582 }
 583
 584 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
 585     $confdesc->{"serial$i"} = $serialdesc;
 586 }
 587
 588 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
 589     $confdesc->{"hostpci$i"} = $hostpcidesc;
 590 }
 591
 592 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
 593     $drivename_hash->{"ide$i"} = 1;
 594     $confdesc->{"ide$i"} = $idedesc;
 595 }
 596
 597 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
 598     $drivename_hash->{"sata$i"} = 1;
 599     $confdesc->{"sata$i"} = $satadesc;
 600 }
 601
 602 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
 603     $drivename_hash->{"scsi$i"} = 1;
 604     $confdesc->{"scsi$i"} = $scsidesc ;
 605 }
 606
 607 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
 608     $drivename_hash->{"virtio$i"} = 1;
 609     $confdesc->{"virtio$i"} = $virtiodesc;
 610 }
 611
 612 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
 613     $confdesc->{"usb$i"} = $usbdesc;
 614 }
 615
 616 my $unuseddesc = {
 617     optional => 1,
 618     type => 'string', format => 'pve-volume-id',
 619     description => "Reference to unused volumes.",
 620 };
 621
 622 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
 623     $confdesc->{"unused$i"} = $unuseddesc;
 624 }
 625
 626 my $kvm_api_version = 0;
 627
 628 sub kvm_version {
 629
 630     return $kvm_api_version if $kvm_api_version;
 631
 632     my $fh = IO::File->new("</dev/kvm") ||
 633         return 0;
 634
 635     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
 636         $kvm_api_version = $v;
 637     }
 638
 639     $fh->close();
 640
 641     return  $kvm_api_version;
 642 }
 643
 644 my $kvm_user_version;
 645
 646 sub kvm_user_version {
 647
 648     return $kvm_user_version if $kvm_user_version;
 649
 650     $kvm_user_version = 'unknown';
 651
 652     my $tmp = `kvm -help 2>/dev/null`;
 653
 654     if ($tmp =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
 655         $kvm_user_version = $2;
 656     }
 657
 658     return $kvm_user_version;
 659
 660 }
 661
 662 my $kernel_has_vhost_net = -c '/dev/vhost-net';
 663
 664 sub disknames {
 665     # order is important - used to autoselect boot disk
 666     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
 667             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
 668             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
 669             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))));
 670 }
 671
 672 sub valid_drivename {
 673     my $dev = shift;
 674
 675     return defined($drivename_hash->{$dev});
 676 }
 677
 678 sub option_exists {
 679     my $key = shift;
 680     return defined($confdesc->{$key});
 681 }
 682
 683 sub nic_models {
 684     return $nic_model_list;
 685 }
 686
 687 sub os_list_description {
 688
 689     return {
 690         other => 'Other',
 691         wxp => 'Windows XP',
 692         w2k => 'Windows 2000',
 693         w2k3 =>, 'Windows 2003',
 694         w2k8 => 'Windows 2008',
 695         wvista => 'Windows Vista',
 696         win7 => 'Windows 7',
 697         win8 => 'Windows 8/2012',
 698         l24 => 'Linux 2.4',
 699         l26 => 'Linux 2.6',
 700     };
 701 }
 702
 703 my $cdrom_path;
 704
 705 sub get_cdrom_path {
 706
 707     return  $cdrom_path if $cdrom_path;
 708
 709     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
 710     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
 711     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
 712 }
 713
 714 sub get_iso_path {
 715     my ($storecfg, $vmid, $cdrom) = @_;
 716
 717     if ($cdrom eq 'cdrom') {
 718         return get_cdrom_path();
 719     } elsif ($cdrom eq 'none') {
 720         return '';
 721     } elsif ($cdrom =~ m|^/|) {
 722         return $cdrom;
 723     } else {
 724         return PVE::Storage::path($storecfg, $cdrom);
 725     }
 726 }
 727
 728 # try to convert old style file names to volume IDs
 729 sub filename_to_volume_id {
 730     my ($vmid, $file, $media) = @_;
 731
 732     if (!($file eq 'none' || $file eq 'cdrom' ||
 733           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
 734
 735         return undef if $file =~ m|/|;
 736
 737         if ($media && $media eq 'cdrom') {
 738             $file = "local:iso/$file";
 739         } else {
 740             $file = "local:$vmid/$file";
 741         }
 742     }
 743
 744     return $file;
 745 }
 746
 747 sub verify_media_type {
 748     my ($opt, $vtype, $media) = @_;
 749
 750     return if !$media;
 751
 752     my $etype;
 753     if ($media eq 'disk') {
 754         $etype = 'images';
 755     } elsif ($media eq 'cdrom') {
 756         $etype = 'iso';
 757     } else {
 758         die "internal error";
 759     }
 760
 761     return if ($vtype eq $etype);
 762
 763     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
 764 }
 765
 766 sub cleanup_drive_path {
 767     my ($opt, $storecfg, $drive) = @_;
 768
 769     # try to convert filesystem paths to volume IDs
 770
 771     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
 772         ($drive->{file} !~ m|^/dev/.+|) &&
 773         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
 774         ($drive->{file} !~ m/^\d+$/)) {
 775         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
 776         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
 777         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
 778         verify_media_type($opt, $vtype, $drive->{media});
 779         $drive->{file} = $volid;
 780     }
 781
 782     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
 783 }
 784
 785 sub create_conf_nolock {
 786     my ($vmid, $settings) = @_;
 787
 788     my $filename = config_file($vmid);
 789
 790     die "configuration file '$filename' already exists\n" if -f $filename;
 791
 792     my $defaults = load_defaults();
 793
 794     $settings->{name} = "vm$vmid" if !$settings->{name};
 795     $settings->{memory} = $defaults->{memory} if !$settings->{memory};
 796
 797     my $data = '';
 798     foreach my $opt (keys %$settings) {
 799         next if !$confdesc->{$opt};
 800
 801         my $value = $settings->{$opt};
 802         next if !$value;
 803
 804         $data .= "$opt: $value\n";
 805     }
 806
 807     PVE::Tools::file_set_contents($filename, $data);
 808 }
 809
 810 sub parse_hotplug_features {
 811     my ($data) = @_;
 812
 813     my $res = {};
 814
 815     return $res if $data eq '0';
 816
 817     $data = $confdesc->{hotplug}->{default} if $data eq '1';
 818
 819     foreach my $feature (PVE::Tools::split_list($data)) {
 820         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
 821             $res->{$1} = 1;
 822         } else {
 823             warn "ignoring unknown hotplug feature '$feature'\n";
 824         }
 825     }
 826     return $res;
 827 }
 828
 829 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
 830 sub pve_verify_hotplug_features {
 831     my ($value, $noerr) = @_;
 832
 833     return $value if parse_hotplug_features($value);
 834
 835     return undef if $noerr;
 836
 837     die "unable to parse hotplug option\n";
 838 }
 839
 840 my $parse_size = sub {
 841     my ($value) = @_;
 842
 843     return undef if $value !~ m/^(\d+(\.\d+)?)([KMG])?$/;
 844     my ($size, $unit) = ($1, $3);
 845     if ($unit) {
 846         if ($unit eq 'K') {
 847             $size = $size * 1024;
 848         } elsif ($unit eq 'M') {
 849             $size = $size * 1024 * 1024;
 850         } elsif ($unit eq 'G') {
 851             $size = $size * 1024 * 1024 * 1024;
 852         }
 853     }
 854     return int($size);
 855 };
 856
 857 my $format_size = sub {
 858     my ($size) = @_;
 859
 860     $size = int($size);
 861
 862     my $kb = int($size/1024);
 863     return $size if $kb*1024 != $size;
 864
 865     my $mb = int($kb/1024);
 866     return "${kb}K" if $mb*1024 != $kb;
 867
 868     my $gb = int($mb/1024);
 869     return "${mb}M" if $gb*1024 != $mb;
 870
 871     return "${gb}G";
 872 };
 873
 874 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
 875 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
 876 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
 877 #        [,aio=native|threads][,discard=ignore|on][,iothread=on]
 878 #        [,serial=serial][,model=model]
 879
 880 sub parse_drive {
 881     my ($key, $data) = @_;
 882
 883     my $res = {};
 884
 885     # $key may be undefined - used to verify JSON parameters
 886     if (!defined($key)) {
 887         $res->{interface} = 'unknown'; # should not harm when used to verify parameters
 888         $res->{index} = 0;
 889     } elsif ($key =~ m/^([^\d]+)(\d+)$/) {
 890         $res->{interface} = $1;
 891         $res->{index} = $2;
 892     } else {
 893         return undef;
 894     }
 895
 896     foreach my $p (split (/,/, $data)) {
 897         next if $p =~ m/^\s*$/;
 898
 899         if ($p =~ m/^(file|volume|cyls|heads|secs|trans|media|snapshot|cache|format|rerror|werror|backup|aio|bps|mbps|mbps_max|bps_rd|mbps_rd|mbps_rd_max|bps_wr|mbps_wr|mbps_wr_max|iops|iops_max|iops_rd|iops_rd_max|iops_wr|iops_wr_max|size|discard|iothread|queues|serial|model)=(.+)$/) {
 900             my ($k, $v) = ($1, $2);
 901
 902             $k = 'file' if $k eq 'volume';
 903
 904             return undef if defined $res->{$k};
 905
 906             if ($k eq 'bps' || $k eq 'bps_rd' || $k eq 'bps_wr') {
 907                 return undef if !$v || $v !~ m/^\d+/;
 908                 $k = "m$k";
 909                 $v = sprintf("%.3f", $v / (1024*1024));
 910             }
 911             $res->{$k} = $v;
 912         } else {
 913             if (!$res->{file} && $p !~ m/=/) {
 914                 $res->{file} = $p;
 915             } else {
 916                 return undef;
 917             }
 918         }
 919     }
 920
 921     return undef if !$res->{file};
 922
 923     return undef if $res->{cache} &&
 924         $res->{cache} !~ m/^(off|none|writethrough|writeback|unsafe|directsync)$/;
 925     return undef if $res->{snapshot} && $res->{snapshot} !~ m/^(on|off)$/;
 926     return undef if $res->{cyls} && $res->{cyls} !~ m/^\d+$/;
 927     return undef if $res->{heads} && $res->{heads} !~ m/^\d+$/;
 928     return undef if $res->{secs} && $res->{secs} !~ m/^\d+$/;
 929     return undef if $res->{media} && $res->{media} !~ m/^(disk|cdrom)$/;
 930     return undef if $res->{trans} && $res->{trans} !~ m/^(none|lba|auto)$/;
 931     return undef if $res->{format} && $res->{format} !~ m/^(raw|cow|qcow|qed|qcow2|vmdk|cloop)$/;
 932     return undef if $res->{rerror} && $res->{rerror} !~ m/^(ignore|report|stop)$/;
 933     return undef if $res->{werror} && $res->{werror} !~ m/^(enospc|ignore|report|stop)$/;
 934     return undef if $res->{backup} && $res->{backup} !~ m/^(yes|no)$/;
 935     return undef if $res->{aio} && $res->{aio} !~ m/^(native|threads)$/;
 936     return undef if $res->{discard} && $res->{discard} !~ m/^(ignore|on)$/;
 937     return undef if $res->{iothread} && $res->{iothread} !~ m/^(on)$/;
 938     return undef if $res->{queues} && ($res->{queues} !~ m/^\d+$/ || $res->{queues} < 2);
 939
 940     return undef if $res->{mbps_rd} && $res->{mbps};
 941     return undef if $res->{mbps_wr} && $res->{mbps};
 942
 943     return undef if $res->{mbps} && $res->{mbps} !~ m/^\d+(\.\d+)?$/;
 944     return undef if $res->{mbps_max} && $res->{mbps_max} !~ m/^\d+(\.\d+)?$/;
 945     return undef if $res->{mbps_rd} && $res->{mbps_rd} !~ m/^\d+(\.\d+)?$/;
 946     return undef if $res->{mbps_rd_max} && $res->{mbps_rd_max} !~ m/^\d+(\.\d+)?$/;
 947     return undef if $res->{mbps_wr} && $res->{mbps_wr} !~ m/^\d+(\.\d+)?$/;
 948     return undef if $res->{mbps_wr_max} && $res->{mbps_wr_max} !~ m/^\d+(\.\d+)?$/;
 949
 950     return undef if $res->{iops_rd} && $res->{iops};
 951     return undef if $res->{iops_wr} && $res->{iops};
 952
 953
 954     return undef if $res->{iops} && $res->{iops} !~ m/^\d+$/;
 955     return undef if $res->{iops_max} && $res->{iops_max} !~ m/^\d+$/;
 956     return undef if $res->{iops_rd} && $res->{iops_rd} !~ m/^\d+$/;
 957     return undef if $res->{iops_rd_max} && $res->{iops_rd_max} !~ m/^\d+$/;
 958     return undef if $res->{iops_wr} && $res->{iops_wr} !~ m/^\d+$/;
 959     return undef if $res->{iops_wr_max} && $res->{iops_wr_max} !~ m/^\d+$/;
 960
 961     if ($res->{size}) {
 962         return undef if !defined($res->{size} = &$parse_size($res->{size}));
 963     }
 964
 965     if ($res->{media} && ($res->{media} eq 'cdrom')) {
 966         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
 967         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
 968         return undef if $res->{interface} eq 'virtio';
 969     }
 970
 971     # rerror does not work with scsi drives
 972     if ($res->{rerror}) {
 973         return undef if $res->{interface} eq 'scsi';
 974     }
 975
 976     return $res;
 977 }
 978
 979 my @qemu_drive_options = qw(heads secs cyls trans media format cache snapshot rerror werror aio discard iops iops_rd iops_wr iops_max iops_rd_max iops_wr_max serial);
 980
 981 sub print_drive {
 982     my ($vmid, $drive) = @_;
 983
 984     my $opts = '';
 985     foreach my $o (@qemu_drive_options, 'mbps', 'mbps_rd', 'mbps_wr', 'mbps_max', 'mbps_rd_max', 'mbps_wr_max', 'backup', 'iothread', 'queues') {
 986         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
 987     }
 988
 989     if ($drive->{size}) {
 990         $opts .= ",size=" . &$format_size($drive->{size});
 991     }
 992
 993     if (my $model = $drive->{model}) {
 994         $opts .= ",model=$model";
 995     }
 996
 997     return "$drive->{file}$opts";
 998 }
 999
1000 sub scsi_inquiry {
1001     my($fh, $noerr) = @_;
1002
1003     my $SG_IO = 0x2285;
1004     my $SG_GET_VERSION_NUM = 0x2282;
1005
1006     my $versionbuf = "\x00" x 8;
1007     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1008     if (!$ret) {
1009         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1010         return undef;
1011     }
1012     my $version = unpack("I", $versionbuf);
1013     if ($version < 30000) {
1014         die "scsi generic interface too old\n"  if !$noerr;
1015         return undef;
1016     }
1017
1018     my $buf = "\x00" x 36;
1019     my $sensebuf = "\x00" x 8;
1020     my $cmd = pack("C x3 C x1", 0x12, 36);
1021
1022     # see /usr/include/scsi/sg.h
1023     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1024
1025     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1026                       length($sensebuf), 0, length($buf), $buf,
1027                       $cmd, $sensebuf, 6000);
1028
1029     $ret = ioctl($fh, $SG_IO, $packet);
1030     if (!$ret) {
1031         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1032         return undef;
1033     }
1034
1035     my @res = unpack($sg_io_hdr_t, $packet);
1036     if ($res[17] || $res[18]) {
1037         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1038         return undef;
1039     }
1040
1041     my $res = {};
1042     (my $byte0, my $byte1, $res->{vendor},
1043      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1044
1045     $res->{removable} = $byte1 & 128 ? 1 : 0;
1046     $res->{type} = $byte0 & 31;
1047
1048     return $res;
1049 }
1050
1051 sub path_is_scsi {
1052     my ($path) = @_;
1053
1054     my $fh = IO::File->new("+<$path") || return undef;
1055     my $res = scsi_inquiry($fh, 1);
1056     close($fh);
1057
1058     return $res;
1059 }
1060
1061 sub machine_type_is_q35 {
1062     my ($conf) = @_;
1063
1064     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1065 }
1066
1067 sub print_tabletdevice_full {
1068     my ($conf) = @_;
1069
1070     my $q35 = machine_type_is_q35($conf);
1071
1072     # we use uhci for old VMs because tablet driver was buggy in older qemu
1073     my $usbbus = $q35 ? "ehci" : "uhci";
1074
1075     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1076 }
1077
1078 sub print_drivedevice_full {
1079     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1080
1081     my $device = '';
1082     my $maxdev = 0;
1083
1084     if ($drive->{interface} eq 'virtio') {
1085         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1086         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1087         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1088     } elsif ($drive->{interface} eq 'scsi') {
1089
1090         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1091         my $unit = $drive->{index} % $maxdev;
1092         my $devicetype = 'hd';
1093         my $path = '';
1094         if (drive_is_cdrom($drive)) {
1095               $devicetype = 'cd';
1096         } else {
1097               if ($drive->{file} =~ m|^/|) {
1098                   $path = $drive->{file};
1099               } else {
1100                   $path = PVE::Storage::path($storecfg, $drive->{file});
1101               }
1102
1103               if($path =~ m/^iscsi\:\/\//){
1104                   $devicetype = 'generic';
1105               } else {
1106                   if (my $info = path_is_scsi($path)) {
1107                       if ($info->{type} == 0) {
1108                           $devicetype = 'block';
1109                       } elsif ($info->{type} == 1) { # tape
1110                           $devicetype = 'generic';
1111                       }
1112                   }
1113               }
1114          }
1115
1116         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1117             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1118         } else {
1119             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1120         }
1121
1122     } elsif ($drive->{interface} eq 'ide'){
1123         $maxdev = 2;
1124         my $controller = int($drive->{index} / $maxdev);
1125         my $unit = $drive->{index} % $maxdev;
1126         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1127
1128         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1129         if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
1130             $device .= ",model=$model";
1131         }
1132     } elsif ($drive->{interface} eq 'sata'){
1133         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1134         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1135         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1136     } elsif ($drive->{interface} eq 'usb') {
1137         die "implement me";
1138         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1139     } else {
1140         die "unsupported interface type";
1141     }
1142
1143     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1144
1145     return $device;
1146 }
1147
1148 sub get_initiator_name {
1149     my $initiator;
1150
1151     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1152     while (defined(my $line = <$fh>)) {
1153         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1154         $initiator = $1;
1155         last;
1156     }
1157     $fh->close();
1158
1159     return $initiator;
1160 }
1161
1162 sub print_drive_full {
1163     my ($storecfg, $vmid, $drive) = @_;
1164
1165     my $path;
1166     my $volid = $drive->{file};
1167     my $format;
1168
1169     if (drive_is_cdrom($drive)) {
1170         $path = get_iso_path($storecfg, $vmid, $volid);
1171     } else {
1172         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1173         if ($storeid) {
1174             $path = PVE::Storage::path($storecfg, $volid);
1175             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1176             $format = qemu_img_format($scfg, $volname);
1177         } else {
1178             $path = $volid;
1179         }
1180    }
1181
1182     my $opts = '';
1183     foreach my $o (@qemu_drive_options) {
1184         next if $o eq 'bootindex';
1185         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1186     }
1187
1188     $opts .= ",format=$format" if $format && !$drive->{format};
1189
1190     foreach my $o (qw(bps bps_rd bps_wr)) {
1191         my $v = $drive->{"m$o"};
1192         $opts .= ",$o=" . int($v*1024*1024) if $v;
1193     }
1194
1195     my $cache_direct = 0;
1196
1197     if (my $cache = $drive->{cache}) {
1198         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1199     } elsif (!drive_is_cdrom($drive)) {
1200         $opts .= ",cache=none";
1201         $cache_direct = 1;
1202     }
1203
1204     # aio native works only with O_DIRECT
1205     if (!$drive->{aio}) {
1206         if($cache_direct) {
1207             $opts .= ",aio=native";
1208         } else {
1209             $opts .= ",aio=threads";
1210         }
1211     }
1212
1213     my $detectzeroes = $drive->{discard} ? "unmap" : "on";
1214     $opts .= ",detect-zeroes=$detectzeroes" if !drive_is_cdrom($drive);
1215
1216     my $pathinfo = $path ? "file=$path," : '';
1217
1218     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1219 }
1220
1221 sub print_netdevice_full {
1222     my ($vmid, $conf, $net, $netid, $bridges) = @_;
1223
1224     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1225
1226     my $device = $net->{model};
1227     if ($net->{model} eq 'virtio') {
1228          $device = 'virtio-net-pci';
1229      };
1230
1231     my $pciaddr = print_pci_addr("$netid", $bridges);
1232     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1233     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1234         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1235         my $vectors = $net->{queues} * 2 + 2;
1236         $tmpstr .= ",vectors=$vectors,mq=on";
1237     }
1238     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1239     return $tmpstr;
1240 }
1241
1242 sub print_netdev_full {
1243     my ($vmid, $conf, $net, $netid) = @_;
1244
1245     my $i = '';
1246     if ($netid =~ m/^net(\d+)$/) {
1247         $i = int($1);
1248     }
1249
1250     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1251
1252     my $ifname = "tap${vmid}i$i";
1253
1254     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1255     die "interface name '$ifname' is too long (max 15 character)\n"
1256         if length($ifname) >= 16;
1257
1258     my $vhostparam = '';
1259     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1260
1261     my $vmname = $conf->{name} || "vm$vmid";
1262
1263     my $netdev = "";
1264
1265     if ($net->{bridge}) {
1266         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/pve-bridge,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1267     } else {
1268         $netdev = "type=user,id=$netid,hostname=$vmname";
1269     }
1270
1271     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1272
1273     return $netdev;
1274 }
1275
1276 sub drive_is_cdrom {
1277     my ($drive) = @_;
1278
1279     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1280
1281 }
1282
1283 sub parse_numa {
1284     my ($data) = @_;
1285
1286     my $res = {};
1287
1288     foreach my $kvp (split(/,/, $data)) {
1289
1290         if ($kvp =~ m/^memory=(\S+)$/) {
1291             $res->{memory} = $1;
1292         } elsif ($kvp =~ m/^policy=(preferred|bind|interleave)$/) {
1293             $res->{policy} = $1;
1294         } elsif ($kvp =~ m/^cpus=(\d+)(-(\d+))?$/) {
1295             $res->{cpus}->{start} = $1;
1296             $res->{cpus}->{end} = $3;
1297         } elsif ($kvp =~ m/^hostnodes=(\d+)(-(\d+))?$/) {
1298             $res->{hostnodes}->{start} = $1;
1299             $res->{hostnodes}->{end} = $3;
1300         } else {
1301             return undef;
1302         }
1303     }
1304
1305     return $res;
1306 }
1307
1308 sub parse_hostpci {
1309     my ($value) = @_;
1310
1311     return undef if !$value;
1312
1313
1314     my @list = split(/,/, $value);
1315     my $found;
1316
1317     my $res = {};
1318     foreach my $kv (@list) {
1319
1320         if ($kv =~ m/^(host=)?([a-f0-9]{2}:[a-f0-9]{2})(\.([a-f0-9]))?$/) {
1321             $found = 1;
1322             if(defined($4)){
1323                 push @{$res->{pciid}}, { id => $2 , function => $4};
1324
1325             }else{
1326                 my $pcidevices = lspci($2);
1327                 $res->{pciid} = $pcidevices->{$2};
1328             }
1329         } elsif ($kv =~ m/^rombar=(on|off)$/) {
1330             $res->{rombar} = $1;
1331         } elsif ($kv =~ m/^x-vga=(on|off)$/) {
1332             $res->{'x-vga'} = $1;
1333         } elsif ($kv =~ m/^pcie=(\d+)$/) {
1334             $res->{pcie} = 1 if $1 == 1;
1335         } else {
1336             warn "unknown hostpci setting '$kv'\n";
1337         }
1338     }
1339
1340     return undef if !$found;
1341
1342     return $res;
1343 }
1344
1345 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1346 sub parse_net {
1347     my ($data) = @_;
1348
1349     my $res = {};
1350
1351     foreach my $kvp (split(/,/, $data)) {
1352
1353         if ($kvp =~ m/^(ne2k_pci|e1000|e1000-82540em|e1000-82544gc|e1000-82545em|rtl8139|pcnet|virtio|ne2k_isa|i82551|i82557b|i82559er|vmxnet3)(=([0-9a-f]{2}(:[0-9a-f]{2}){5}))?$/i) {
1354             my $model = lc($1);
1355             my $mac = defined($3) ? uc($3) : PVE::Tools::random_ether_addr();
1356             $res->{model} = $model;
1357             $res->{macaddr} = $mac;
1358         } elsif ($kvp =~ m/^bridge=(\S+)$/) {
1359             $res->{bridge} = $1;
1360         } elsif ($kvp =~ m/^queues=(\d+)$/) {
1361             $res->{queues} = $1;
1362         } elsif ($kvp =~ m/^rate=(\d+(\.\d+)?)$/) {
1363             $res->{rate} = $1;
1364         } elsif ($kvp =~ m/^tag=(\d+)$/) {
1365             $res->{tag} = $1;
1366         } elsif ($kvp =~ m/^firewall=([01])$/) {
1367             $res->{firewall} = $1;
1368         } elsif ($kvp =~ m/^link_down=([01])$/) {
1369             $res->{link_down} = $1;
1370         } else {
1371             return undef;
1372         }
1373
1374     }
1375
1376     return undef if !$res->{model};
1377
1378     return $res;
1379 }
1380
1381 sub print_net {
1382     my $net = shift;
1383
1384     my $res = "$net->{model}";
1385     $res .= "=$net->{macaddr}" if $net->{macaddr};
1386     $res .= ",bridge=$net->{bridge}" if $net->{bridge};
1387     $res .= ",rate=$net->{rate}" if $net->{rate};
1388     $res .= ",tag=$net->{tag}" if $net->{tag};
1389     $res .= ",firewall=1" if $net->{firewall};
1390     $res .= ",link_down=1" if $net->{link_down};
1391     $res .= ",queues=$net->{queues}" if $net->{queues};
1392
1393     return $res;
1394 }
1395
1396 sub add_random_macs {
1397     my ($settings) = @_;
1398
1399     foreach my $opt (keys %$settings) {
1400         next if $opt !~ m/^net(\d+)$/;
1401         my $net = parse_net($settings->{$opt});
1402         next if !$net;
1403         $settings->{$opt} = print_net($net);
1404     }
1405 }
1406
1407 sub add_unused_volume {
1408     my ($config, $volid) = @_;
1409
1410     my $key;
1411     for (my $ind = $MAX_UNUSED_DISKS - 1; $ind >= 0; $ind--) {
1412         my $test = "unused$ind";
1413         if (my $vid = $config->{$test}) {
1414             return if $vid eq $volid; # do not add duplicates
1415         } else {
1416             $key = $test;
1417         }
1418     }
1419
1420     die "To many unused volume - please delete them first.\n" if !$key;
1421
1422     $config->{$key} = $volid;
1423
1424     return $key;
1425 }
1426
1427 sub vm_is_volid_owner {
1428     my ($storecfg, $vmid, $volid) = @_;
1429
1430     if ($volid !~  m|^/|) {
1431         my ($path, $owner);
1432         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1433         if ($owner && ($owner == $vmid)) {
1434             return 1;
1435         }
1436     }
1437
1438     return undef;
1439 }
1440
1441 sub split_flagged_list {
1442     my $text = shift || '';
1443     $text =~ s/[,;]/ /g;
1444     $text =~ s/^\s+//;
1445     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
1446 }
1447
1448 sub join_flagged_list {
1449     my ($how, $lst) = @_;
1450     join $how, map { $lst->{$_} . $_ } keys %$lst;
1451 }
1452
1453 sub vmconfig_delete_pending_option {
1454     my ($conf, $key, $force) = @_;
1455
1456     delete $conf->{pending}->{$key};
1457     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1458     $pending_delete_hash->{$key} = $force ? '!' : '';
1459     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1460 }
1461
1462 sub vmconfig_undelete_pending_option {
1463     my ($conf, $key) = @_;
1464
1465     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1466     delete $pending_delete_hash->{$key};
1467
1468     if (%$pending_delete_hash) {
1469         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1470     } else {
1471         delete $conf->{pending}->{delete};
1472     }
1473 }
1474
1475 sub vmconfig_register_unused_drive {
1476     my ($storecfg, $vmid, $conf, $drive) = @_;
1477
1478     if (!drive_is_cdrom($drive)) {
1479         my $volid = $drive->{file};
1480         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1481             add_unused_volume($conf, $volid, $vmid);
1482         }
1483     }
1484 }
1485
1486 sub vmconfig_cleanup_pending {
1487     my ($conf) = @_;
1488
1489     # remove pending changes when nothing changed
1490     my $changes;
1491     foreach my $opt (keys %{$conf->{pending}}) {
1492         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
1493             $changes = 1;
1494             delete $conf->{pending}->{$opt};
1495         }
1496     }
1497
1498     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
1499     my $pending_delete_hash = {};
1500     while (my ($opt, $force) = each %$current_delete_hash) {
1501         if (defined($conf->{$opt})) {
1502             $pending_delete_hash->{$opt} = $force;
1503         } else {
1504             $changes = 1;
1505         }
1506     }
1507
1508     if (%$pending_delete_hash) {
1509         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1510     } else {
1511         delete $conf->{pending}->{delete};
1512     }
1513
1514     return $changes;
1515 }
1516
1517 my $valid_smbios1_options = {
1518     manufacturer => '\S+',
1519     product => '\S+',
1520     version => '\S+',
1521     serial => '\S+',
1522     uuid => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1523     sku => '\S+',
1524     family => '\S+',
1525 };
1526
1527 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
1528 sub parse_smbios1 {
1529     my ($data) = @_;
1530
1531     my $res = {};
1532
1533     foreach my $kvp (split(/,/, $data)) {
1534         return undef if $kvp !~ m/^(\S+)=(.+)$/;
1535         my ($k, $v) = split(/=/, $kvp);
1536         return undef if !defined($k) || !defined($v);
1537         return undef if !$valid_smbios1_options->{$k};
1538         return undef if $v !~ m/^$valid_smbios1_options->{$k}$/;
1539         $res->{$k} = $v;
1540     }
1541
1542     return $res;
1543 }
1544
1545 sub print_smbios1 {
1546     my ($smbios1) = @_;
1547
1548     my $data = '';
1549     foreach my $k (keys %$smbios1) {
1550         next if !defined($smbios1->{$k});
1551         next if !$valid_smbios1_options->{$k};
1552         $data .= ',' if $data;
1553         $data .= "$k=$smbios1->{$k}";
1554     }
1555     return $data;
1556 }
1557
1558 PVE::JSONSchema::register_format('pve-qm-smbios1', \&verify_smbios1);
1559 sub verify_smbios1 {
1560     my ($value, $noerr) = @_;
1561
1562     return $value if parse_smbios1($value);
1563
1564     return undef if $noerr;
1565
1566     die "unable to parse smbios (type 1) options\n";
1567 }
1568
1569 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
1570 sub verify_bootdisk {
1571     my ($value, $noerr) = @_;
1572
1573     return $value if valid_drivename($value);
1574
1575     return undef if $noerr;
1576
1577     die "invalid boot disk '$value'\n";
1578 }
1579
1580 PVE::JSONSchema::register_format('pve-qm-numanode', \&verify_numa);
1581 sub verify_numa {
1582     my ($value, $noerr) = @_;
1583
1584     return $value if parse_numa($value);
1585
1586     return undef if $noerr;
1587
1588     die "unable to parse numa options\n";
1589 }
1590
1591 PVE::JSONSchema::register_format('pve-qm-net', \&verify_net);
1592 sub verify_net {
1593     my ($value, $noerr) = @_;
1594
1595     return $value if parse_net($value);
1596
1597     return undef if $noerr;
1598
1599     die "unable to parse network options\n";
1600 }
1601
1602 PVE::JSONSchema::register_format('pve-qm-drive', \&verify_drive);
1603 sub verify_drive {
1604     my ($value, $noerr) = @_;
1605
1606     return $value if parse_drive(undef, $value);
1607
1608     return undef if $noerr;
1609
1610     die "unable to parse drive options\n";
1611 }
1612
1613 PVE::JSONSchema::register_format('pve-qm-hostpci', \&verify_hostpci);
1614 sub verify_hostpci {
1615     my ($value, $noerr) = @_;
1616
1617     return $value if parse_hostpci($value);
1618
1619     return undef if $noerr;
1620
1621     die "unable to parse pci id\n";
1622 }
1623
1624 PVE::JSONSchema::register_format('pve-qm-watchdog', \&verify_watchdog);
1625 sub verify_watchdog {
1626     my ($value, $noerr) = @_;
1627
1628     return $value if parse_watchdog($value);
1629
1630     return undef if $noerr;
1631
1632     die "unable to parse watchdog options\n";
1633 }
1634
1635 sub parse_watchdog {
1636     my ($value) = @_;
1637
1638     return undef if !$value;
1639
1640     my $res = {};
1641
1642     foreach my $p (split(/,/, $value)) {
1643         next if $p =~ m/^\s*$/;
1644
1645         if ($p =~ m/^(model=)?(i6300esb|ib700)$/) {
1646             $res->{model} = $2;
1647         } elsif ($p =~ m/^(action=)?(reset|shutdown|poweroff|pause|debug|none)$/) {
1648             $res->{action} = $2;
1649         } else {
1650             return undef;
1651         }
1652     }
1653
1654     return $res;
1655 }
1656
1657 sub parse_usb_device {
1658     my ($value) = @_;
1659
1660     return undef if !$value;
1661
1662     my @dl = split(/,/, $value);
1663     my $found;
1664
1665     my $res = {};
1666     foreach my $v (@dl) {
1667         if ($v =~ m/^host=(0x)?([0-9A-Fa-f]{4}):(0x)?([0-9A-Fa-f]{4})$/) {
1668             $found = 1;
1669             $res->{vendorid} = $2;
1670             $res->{productid} = $4;
1671         } elsif ($v =~ m/^host=(\d+)\-(\d+(\.\d+)*)$/) {
1672             $found = 1;
1673             $res->{hostbus} = $1;
1674             $res->{hostport} = $2;
1675         } elsif ($v =~ m/^spice$/) {
1676             $found = 1;
1677             $res->{spice} = 1;
1678         } else {
1679             return undef;
1680         }
1681     }
1682     return undef if !$found;
1683
1684     return $res;
1685 }
1686
1687 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
1688 sub verify_usb_device {
1689     my ($value, $noerr) = @_;
1690
1691     return $value if parse_usb_device($value);
1692
1693     return undef if $noerr;
1694
1695     die "unable to parse usb device\n";
1696 }
1697
1698 # add JSON properties for create and set function
1699 sub json_config_properties {
1700     my $prop = shift;
1701
1702     foreach my $opt (keys %$confdesc) {
1703         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
1704         $prop->{$opt} = $confdesc->{$opt};
1705     }
1706
1707     return $prop;
1708 }
1709
1710 sub check_type {
1711     my ($key, $value) = @_;
1712
1713     die "unknown setting '$key'\n" if !$confdesc->{$key};
1714
1715     my $type = $confdesc->{$key}->{type};
1716
1717     if (!defined($value)) {
1718         die "got undefined value\n";
1719     }
1720
1721     if ($value =~ m/[\n\r]/) {
1722         die "property contains a line feed\n";
1723     }
1724
1725     if ($type eq 'boolean') {
1726         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
1727         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
1728         die "type check ('boolean') failed - got '$value'\n";
1729     } elsif ($type eq 'integer') {
1730         return int($1) if $value =~ m/^(\d+)$/;
1731         die "type check ('integer') failed - got '$value'\n";
1732     } elsif ($type eq 'number') {
1733         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
1734         die "type check ('number') failed - got '$value'\n";
1735     } elsif ($type eq 'string') {
1736         if (my $fmt = $confdesc->{$key}->{format}) {
1737             if ($fmt eq 'pve-qm-drive') {
1738                 # special case - we need to pass $key to parse_drive()
1739                 my $drive = parse_drive($key, $value);
1740                 return $value if $drive;
1741                 die "unable to parse drive options\n";
1742             }
1743             PVE::JSONSchema::check_format($fmt, $value);
1744             return $value;
1745         }
1746         $value =~ s/^\"(.*)\"$/$1/;
1747         return $value;
1748     } else {
1749         die "internal error"
1750     }
1751 }
1752
1753 sub lock_config_full {
1754     my ($vmid, $timeout, $code, @param) = @_;
1755
1756     my $filename = config_file_lock($vmid);
1757
1758     my $res = lock_file($filename, $timeout, $code, @param);
1759
1760     die $@ if $@;
1761
1762     return $res;
1763 }
1764
1765 sub lock_config_mode {
1766     my ($vmid, $timeout, $shared, $code, @param) = @_;
1767
1768     my $filename = config_file_lock($vmid);
1769
1770     my $res = lock_file_full($filename, $timeout, $shared, $code, @param);
1771
1772     die $@ if $@;
1773
1774     return $res;
1775 }
1776
1777 sub lock_config {
1778     my ($vmid, $code, @param) = @_;
1779
1780     return lock_config_full($vmid, 10, $code, @param);
1781 }
1782
1783 sub cfs_config_path {
1784     my ($vmid, $node) = @_;
1785
1786     $node = $nodename if !$node;
1787     return "nodes/$node/qemu-server/$vmid.conf";
1788 }
1789
1790 sub check_iommu_support{
1791     #fixme : need to check IOMMU support
1792     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
1793
1794     my $iommu=1;
1795     return $iommu;
1796
1797 }
1798
1799 sub config_file {
1800     my ($vmid, $node) = @_;
1801
1802     my $cfspath = cfs_config_path($vmid, $node);
1803     return "/etc/pve/$cfspath";
1804 }
1805
1806 sub config_file_lock {
1807     my ($vmid) = @_;
1808
1809     return "$lock_dir/lock-$vmid.conf";
1810 }
1811
1812 sub touch_config {
1813     my ($vmid) = @_;
1814
1815     my $conf = config_file($vmid);
1816     utime undef, undef, $conf;
1817 }
1818
1819 sub destroy_vm {
1820     my ($storecfg, $vmid, $keep_empty_config) = @_;
1821
1822     my $conffile = config_file($vmid);
1823
1824     my $conf = load_config($vmid);
1825
1826     check_lock($conf);
1827
1828     # only remove disks owned by this VM
1829     foreach_drive($conf, sub {
1830         my ($ds, $drive) = @_;
1831
1832         return if drive_is_cdrom($drive);
1833
1834         my $volid = $drive->{file};
1835
1836         return if !$volid || $volid =~ m|^/|;
1837
1838         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
1839         return if !$path || !$owner || ($owner != $vmid);
1840
1841         PVE::Storage::vdisk_free($storecfg, $volid);
1842     });
1843
1844     if ($keep_empty_config) {
1845         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
1846     } else {
1847         unlink $conffile;
1848     }
1849
1850     # also remove unused disk
1851     eval {
1852         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
1853
1854         eval {
1855             PVE::Storage::foreach_volid($dl, sub {
1856                 my ($volid, $sid, $volname, $d) = @_;
1857                 PVE::Storage::vdisk_free($storecfg, $volid);
1858             });
1859         };
1860         warn $@ if $@;
1861
1862     };
1863     warn $@ if $@;
1864 }
1865
1866 sub load_config {
1867     my ($vmid, $node) = @_;
1868
1869     my $cfspath = cfs_config_path($vmid, $node);
1870
1871     my $conf = PVE::Cluster::cfs_read_file($cfspath);
1872
1873     die "no such VM ('$vmid')\n" if !defined($conf);
1874
1875     return $conf;
1876 }
1877
1878 sub parse_vm_config {
1879     my ($filename, $raw) = @_;
1880
1881     return undef if !defined($raw);
1882
1883     my $res = {
1884         digest => Digest::SHA::sha1_hex($raw),
1885         snapshots => {},
1886         pending => {},
1887     };
1888
1889     $filename =~ m|/qemu-server/(\d+)\.conf$|
1890         || die "got strange filename '$filename'";
1891
1892     my $vmid = $1;
1893
1894     my $conf = $res;
1895     my $descr;
1896     my $section = '';
1897
1898     my @lines = split(/\n/, $raw);
1899     foreach my $line (@lines) {
1900         next if $line =~ m/^\s*$/;
1901
1902         if ($line =~ m/^\[PENDING\]\s*$/i) {
1903             $section = 'pending';
1904             if (defined($descr)) {
1905                 $descr =~ s/\s+$//;
1906                 $conf->{description} = $descr;
1907             }
1908             $descr = undef;
1909             $conf = $res->{$section} = {};
1910             next;
1911
1912         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
1913             $section = $1;
1914             if (defined($descr)) {
1915                 $descr =~ s/\s+$//;
1916                 $conf->{description} = $descr;
1917             }
1918             $descr = undef;
1919             $conf = $res->{snapshots}->{$section} = {};
1920             next;
1921         }
1922
1923         if ($line =~ m/^\#(.*)\s*$/) {
1924             $descr = '' if !defined($descr);
1925             $descr .= PVE::Tools::decode_text($1) . "\n";
1926             next;
1927         }
1928
1929         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
1930             $descr = '' if !defined($descr);
1931             $descr .= PVE::Tools::decode_text($2);
1932         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
1933             $conf->{snapstate} = $1;
1934         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
1935             my $key = $1;
1936             my $value = $2;
1937             $conf->{$key} = $value;
1938         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
1939             my $value = $1;
1940             if ($section eq 'pending') {
1941                 $conf->{delete} = $value; # we parse this later
1942             } else {
1943                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
1944             }
1945         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
1946             my $key = $1;
1947             my $value = $2;
1948             eval { $value = check_type($key, $value); };
1949             if ($@) {
1950                 warn "vm $vmid - unable to parse value of '$key' - $@";
1951             } else {
1952                 my $fmt = $confdesc->{$key}->{format};
1953                 if ($fmt && $fmt eq 'pve-qm-drive') {
1954                     my $v = parse_drive($key, $value);
1955                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
1956                         $v->{file} = $volid;
1957                         $value = print_drive($vmid, $v);
1958                     } else {
1959                         warn "vm $vmid - unable to parse value of '$key'\n";
1960                         next;
1961                     }
1962                 }
1963
1964                 if ($key eq 'cdrom') {
1965                     $conf->{ide2} = $value;
1966                 } else {
1967                     $conf->{$key} = $value;
1968                 }
1969             }
1970         }
1971     }
1972
1973     if (defined($descr)) {
1974         $descr =~ s/\s+$//;
1975         $conf->{description} = $descr;
1976     }
1977     delete $res->{snapstate}; # just to be sure
1978
1979     return $res;
1980 }
1981
1982 sub write_vm_config {
1983     my ($filename, $conf) = @_;
1984
1985     delete $conf->{snapstate}; # just to be sure
1986
1987     if ($conf->{cdrom}) {
1988         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
1989         $conf->{ide2} = $conf->{cdrom};
1990         delete $conf->{cdrom};
1991     }
1992
1993     # we do not use 'smp' any longer
1994     if ($conf->{sockets}) {
1995         delete $conf->{smp};
1996     } elsif ($conf->{smp}) {
1997         $conf->{sockets} = $conf->{smp};
1998         delete $conf->{cores};
1999         delete $conf->{smp};
2000     }
2001
2002     my $used_volids = {};
2003
2004     my $cleanup_config = sub {
2005         my ($cref, $pending, $snapname) = @_;
2006
2007         foreach my $key (keys %$cref) {
2008             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2009                 $key eq 'snapstate' || $key eq 'pending';
2010             my $value = $cref->{$key};
2011             if ($key eq 'delete') {
2012                 die "propertry 'delete' is only allowed in [PENDING]\n"
2013                     if !$pending;
2014                 # fixme: check syntax?
2015                 next;
2016             }
2017             eval { $value = check_type($key, $value); };
2018             die "unable to parse value of '$key' - $@" if $@;
2019
2020             $cref->{$key} = $value;
2021
2022             if (!$snapname && valid_drivename($key)) {
2023                 my $drive = parse_drive($key, $value);
2024                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2025             }
2026         }
2027     };
2028
2029     &$cleanup_config($conf);
2030
2031     &$cleanup_config($conf->{pending}, 1);
2032
2033     foreach my $snapname (keys %{$conf->{snapshots}}) {
2034         die "internal error" if $snapname eq 'pending';
2035         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2036     }
2037
2038     # remove 'unusedX' settings if we re-add a volume
2039     foreach my $key (keys %$conf) {
2040         my $value = $conf->{$key};
2041         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2042             delete $conf->{$key};
2043         }
2044     }
2045
2046     my $generate_raw_config = sub {
2047         my ($conf, $pending) = @_;
2048
2049         my $raw = '';
2050
2051         # add description as comment to top of file
2052         if (defined(my $descr = $conf->{description})) {
2053             if ($descr) {
2054                 foreach my $cl (split(/\n/, $descr)) {
2055                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2056                 }
2057             } else {
2058                 $raw .= "#\n" if $pending;
2059             }
2060         }
2061
2062         foreach my $key (sort keys %$conf) {
2063             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2064             $raw .= "$key: $conf->{$key}\n";
2065         }
2066         return $raw;
2067     };
2068
2069     my $raw = &$generate_raw_config($conf);
2070
2071     if (scalar(keys %{$conf->{pending}})){
2072         $raw .= "\n[PENDING]\n";
2073         $raw .= &$generate_raw_config($conf->{pending}, 1);
2074     }
2075
2076     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2077         $raw .= "\n[$snapname]\n";
2078         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2079     }
2080
2081     return $raw;
2082 }
2083
2084 sub update_config_nolock {
2085     my ($vmid, $conf, $skiplock) = @_;
2086
2087     check_lock($conf) if !$skiplock;
2088
2089     my $cfspath = cfs_config_path($vmid);
2090
2091     PVE::Cluster::cfs_write_file($cfspath, $conf);
2092 }
2093
2094 sub update_config {
2095     my ($vmid, $conf, $skiplock) = @_;
2096
2097     lock_config($vmid, &update_config_nolock, $conf, $skiplock);
2098 }
2099
2100 sub load_defaults {
2101
2102     my $res = {};
2103
2104     # we use static defaults from our JSON schema configuration
2105     foreach my $key (keys %$confdesc) {
2106         if (defined(my $default = $confdesc->{$key}->{default})) {
2107             $res->{$key} = $default;
2108         }
2109     }
2110
2111     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2112     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2113
2114     return $res;
2115 }
2116
2117 sub config_list {
2118     my $vmlist = PVE::Cluster::get_vmlist();
2119     my $res = {};
2120     return $res if !$vmlist || !$vmlist->{ids};
2121     my $ids = $vmlist->{ids};
2122
2123     foreach my $vmid (keys %$ids) {
2124         my $d = $ids->{$vmid};
2125         next if !$d->{node} || $d->{node} ne $nodename;
2126         next if !$d->{type} || $d->{type} ne 'qemu';
2127         $res->{$vmid}->{exists} = 1;
2128     }
2129     return $res;
2130 }
2131
2132 # test if VM uses local resources (to prevent migration)
2133 sub check_local_resources {
2134     my ($conf, $noerr) = @_;
2135
2136     my $loc_res = 0;
2137
2138     $loc_res = 1 if $conf->{hostusb}; # old syntax
2139     $loc_res = 1 if $conf->{hostpci}; # old syntax
2140
2141     foreach my $k (keys %$conf) {
2142         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2143         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2144     }
2145
2146     die "VM uses local resources\n" if $loc_res && !$noerr;
2147
2148     return $loc_res;
2149 }
2150
2151 # check if used storages are available on all nodes (use by migrate)
2152 sub check_storage_availability {
2153     my ($storecfg, $conf, $node) = @_;
2154
2155     foreach_drive($conf, sub {
2156         my ($ds, $drive) = @_;
2157
2158         my $volid = $drive->{file};
2159         return if !$volid;
2160
2161         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2162         return if !$sid;
2163
2164         # check if storage is available on both nodes
2165         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2166         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2167    });
2168 }
2169
2170 # list nodes where all VM images are available (used by has_feature API)
2171 sub shared_nodes {
2172     my ($conf, $storecfg) = @_;
2173
2174     my $nodelist = PVE::Cluster::get_nodelist();
2175     my $nodehash = { map { $_ => 1 } @$nodelist };
2176     my $nodename = PVE::INotify::nodename();
2177
2178     foreach_drive($conf, sub {
2179         my ($ds, $drive) = @_;
2180
2181         my $volid = $drive->{file};
2182         return if !$volid;
2183
2184         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2185         if ($storeid) {
2186             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2187             if ($scfg->{disable}) {
2188                 $nodehash = {};
2189             } elsif (my $avail = $scfg->{nodes}) {
2190                 foreach my $node (keys %$nodehash) {
2191                     delete $nodehash->{$node} if !$avail->{$node};
2192                 }
2193             } elsif (!$scfg->{shared}) {
2194                 foreach my $node (keys %$nodehash) {
2195                     delete $nodehash->{$node} if $node ne $nodename
2196                 }
2197             }
2198         }
2199     });
2200
2201     return $nodehash
2202 }
2203
2204 sub check_lock {
2205     my ($conf) = @_;
2206
2207     die "VM is locked ($conf->{lock})\n" if $conf->{lock};
2208 }
2209
2210 sub check_cmdline {
2211     my ($pidfile, $pid) = @_;
2212
2213     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2214     if (defined($fh)) {
2215         my $line = <$fh>;
2216         $fh->close;
2217         return undef if !$line;
2218         my @param = split(/\0/, $line);
2219
2220         my $cmd = $param[0];
2221         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2222
2223         for (my $i = 0; $i < scalar (@param); $i++) {
2224             my $p = $param[$i];
2225             next if !$p;
2226             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2227                 my $p = $param[$i+1];
2228                 return 1 if $p && ($p eq $pidfile);
2229                 return undef;
2230             }
2231         }
2232     }
2233     return undef;
2234 }
2235
2236 sub check_running {
2237     my ($vmid, $nocheck, $node) = @_;
2238
2239     my $filename = config_file($vmid, $node);
2240
2241     die "unable to find configuration file for VM $vmid - no such machine\n"
2242         if !$nocheck && ! -f $filename;
2243
2244     my $pidfile = pidfile_name($vmid);
2245
2246     if (my $fd = IO::File->new("<$pidfile")) {
2247         my $st = stat($fd);
2248         my $line = <$fd>;
2249         close($fd);
2250
2251         my $mtime = $st->mtime;
2252         if ($mtime > time()) {
2253             warn "file '$filename' modified in future\n";
2254         }
2255
2256         if ($line =~ m/^(\d+)$/) {
2257             my $pid = $1;
2258             if (check_cmdline($pidfile, $pid)) {
2259                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2260                     return $pid;
2261                 }
2262             }
2263         }
2264     }
2265
2266     return undef;
2267 }
2268
2269 sub vzlist {
2270
2271     my $vzlist = config_list();
2272
2273     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2274
2275     while (defined(my $de = $fd->read)) {
2276         next if $de !~ m/^(\d+)\.pid$/;
2277         my $vmid = $1;
2278         next if !defined($vzlist->{$vmid});
2279         if (my $pid = check_running($vmid)) {
2280             $vzlist->{$vmid}->{pid} = $pid;
2281         }
2282     }
2283
2284     return $vzlist;
2285 }
2286
2287 sub disksize {
2288     my ($storecfg, $conf) = @_;
2289
2290     my $bootdisk = $conf->{bootdisk};
2291     return undef if !$bootdisk;
2292     return undef if !valid_drivename($bootdisk);
2293
2294     return undef if !$conf->{$bootdisk};
2295
2296     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2297     return undef if !defined($drive);
2298
2299     return undef if drive_is_cdrom($drive);
2300
2301     my $volid = $drive->{file};
2302     return undef if !$volid;
2303
2304     return $drive->{size};
2305 }
2306
2307 my $last_proc_pid_stat;
2308
2309 # get VM status information
2310 # This must be fast and should not block ($full == false)
2311 # We only query KVM using QMP if $full == true (this can be slow)
2312 sub vmstatus {
2313     my ($opt_vmid, $full) = @_;
2314
2315     my $res = {};
2316
2317     my $storecfg = PVE::Storage::config();
2318
2319     my $list = vzlist();
2320     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2321
2322     my $cpucount = $cpuinfo->{cpus} || 1;
2323
2324     foreach my $vmid (keys %$list) {
2325         next if $opt_vmid && ($vmid ne $opt_vmid);
2326
2327         my $cfspath = cfs_config_path($vmid);
2328         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2329
2330         my $d = {};
2331         $d->{pid} = $list->{$vmid}->{pid};
2332
2333         # fixme: better status?
2334         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2335
2336         my $size = disksize($storecfg, $conf);
2337         if (defined($size)) {
2338             $d->{disk} = 0; # no info available
2339             $d->{maxdisk} = $size;
2340         } else {
2341             $d->{disk} = 0;
2342             $d->{maxdisk} = 0;
2343         }
2344
2345         $d->{cpus} = ($conf->{sockets} || 1) * ($conf->{cores} || 1);
2346         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2347         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2348
2349         $d->{name} = $conf->{name} || "VM $vmid";
2350         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024) : 0;
2351
2352         if ($conf->{balloon}) {
2353             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2354             $d->{shares} = defined($conf->{shares}) ? $conf->{shares} : 1000;
2355         }
2356
2357         $d->{uptime} = 0;
2358         $d->{cpu} = 0;
2359         $d->{mem} = 0;
2360
2361         $d->{netout} = 0;
2362         $d->{netin} = 0;
2363
2364         $d->{diskread} = 0;
2365         $d->{diskwrite} = 0;
2366
2367         $d->{template} = is_template($conf);
2368
2369         $res->{$vmid} = $d;
2370     }
2371
2372     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2373     foreach my $dev (keys %$netdev) {
2374         next if $dev !~ m/^tap([1-9]\d*)i/;
2375         my $vmid = $1;
2376         my $d = $res->{$vmid};
2377         next if !$d;
2378
2379         $d->{netout} += $netdev->{$dev}->{receive};
2380         $d->{netin} += $netdev->{$dev}->{transmit};
2381
2382         if ($full) {
2383             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2384             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2385         }
2386
2387     }
2388
2389     my $ctime = gettimeofday;
2390
2391     foreach my $vmid (keys %$list) {
2392
2393         my $d = $res->{$vmid};
2394         my $pid = $d->{pid};
2395         next if !$pid;
2396
2397         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2398         next if !$pstat; # not running
2399
2400         my $used = $pstat->{utime} + $pstat->{stime};
2401
2402         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2403
2404         if ($pstat->{vsize}) {
2405             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2406         }
2407
2408         my $old = $last_proc_pid_stat->{$pid};
2409         if (!$old) {
2410             $last_proc_pid_stat->{$pid} = {
2411                 time => $ctime,
2412                 used => $used,
2413                 cpu => 0,
2414             };
2415             next;
2416         }
2417
2418         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2419
2420         if ($dtime > 1000) {
2421             my $dutime = $used -  $old->{used};
2422
2423             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2424             $last_proc_pid_stat->{$pid} = {
2425                 time => $ctime,
2426                 used => $used,
2427                 cpu => $d->{cpu},
2428             };
2429         } else {
2430             $d->{cpu} = $old->{cpu};
2431         }
2432     }
2433
2434     return $res if !$full;
2435
2436     my $qmpclient = PVE::QMPClient->new();
2437
2438     my $ballooncb = sub {
2439         my ($vmid, $resp) = @_;
2440
2441         my $info = $resp->{'return'};
2442         return if !$info->{max_mem};
2443
2444         my $d = $res->{$vmid};
2445
2446         # use memory assigned to VM
2447         $d->{maxmem} = $info->{max_mem};
2448         $d->{balloon} = $info->{actual};
2449
2450         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2451             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2452             $d->{freemem} = $info->{free_mem};
2453         }
2454
2455         $d->{ballooninfo} = $info;
2456     };
2457
2458     my $blockstatscb = sub {
2459         my ($vmid, $resp) = @_;
2460         my $data = $resp->{'return'} || [];
2461         my $totalrdbytes = 0;
2462         my $totalwrbytes = 0;
2463
2464         for my $blockstat (@$data) {
2465             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2466             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2467
2468             $blockstat->{device} =~ s/drive-//;
2469             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2470         }
2471         $res->{$vmid}->{diskread} = $totalrdbytes;
2472         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2473     };
2474
2475     my $statuscb = sub {
2476         my ($vmid, $resp) = @_;
2477
2478         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2479         # this fails if ballon driver is not loaded, so this must be
2480         # the last commnand (following command are aborted if this fails).
2481         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2482
2483         my $status = 'unknown';
2484         if (!defined($status = $resp->{'return'}->{status})) {
2485             warn "unable to get VM status\n";
2486             return;
2487         }
2488
2489         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2490     };
2491
2492     foreach my $vmid (keys %$list) {
2493         next if $opt_vmid && ($vmid ne $opt_vmid);
2494         next if !$res->{$vmid}->{pid}; # not running
2495         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2496     }
2497
2498     $qmpclient->queue_execute(undef, 1);
2499
2500     foreach my $vmid (keys %$list) {
2501         next if $opt_vmid && ($vmid ne $opt_vmid);
2502         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2503     }
2504
2505     return $res;
2506 }
2507
2508 sub foreach_dimm {
2509     my ($conf, $vmid, $memory, $sockets, $func) = @_;
2510
2511     my $dimm_id = 0;
2512     my $current_size = 1024;
2513     my $dimm_size = 512;
2514     return if $current_size == $memory;
2515
2516     for (my $j = 0; $j < 8; $j++) {
2517         for (my $i = 0; $i < 32; $i++) {
2518             my $name = "dimm${dimm_id}";
2519             $dimm_id++;
2520             my $numanode = $i % $sockets;
2521             $current_size += $dimm_size;
2522             &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
2523             return  $current_size if $current_size >= $memory;
2524         }
2525         $dimm_size *= 2;
2526     }
2527 }
2528
2529 sub foreach_reverse_dimm {
2530     my ($conf, $vmid, $memory, $sockets, $func) = @_;
2531
2532     my $dimm_id = 253;
2533     my $current_size = 4177920;
2534     my $dimm_size = 65536;
2535     return if $current_size == $memory;
2536
2537     for (my $j = 0; $j < 8; $j++) {
2538         for (my $i = 0; $i < 32; $i++) {
2539             my $name = "dimm${dimm_id}";
2540             $dimm_id--;
2541             my $numanode = $i % $sockets;
2542             $current_size -= $dimm_size;
2543             &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
2544             return  $current_size if $current_size <= $memory;
2545         }
2546         $dimm_size /= 2;
2547     }
2548 }
2549
2550 sub foreach_drive {
2551     my ($conf, $func) = @_;
2552
2553     foreach my $ds (keys %$conf) {
2554         next if !valid_drivename($ds);
2555
2556         my $drive = parse_drive($ds, $conf->{$ds});
2557         next if !$drive;
2558
2559         &$func($ds, $drive);
2560     }
2561 }
2562
2563 sub foreach_volid {
2564     my ($conf, $func) = @_;
2565
2566     my $volhash = {};
2567
2568     my $test_volid = sub {
2569         my ($volid, $is_cdrom) = @_;
2570
2571         return if !$volid;
2572
2573         $volhash->{$volid} = $is_cdrom || 0;
2574     };
2575
2576     foreach_drive($conf, sub {
2577         my ($ds, $drive) = @_;
2578         &$test_volid($drive->{file}, drive_is_cdrom($drive));
2579     });
2580
2581     foreach my $snapname (keys %{$conf->{snapshots}}) {
2582         my $snap = $conf->{snapshots}->{$snapname};
2583         &$test_volid($snap->{vmstate}, 0);
2584         foreach_drive($snap, sub {
2585             my ($ds, $drive) = @_;
2586             &$test_volid($drive->{file}, drive_is_cdrom($drive));
2587         });
2588     }
2589
2590     foreach my $volid (keys %$volhash) {
2591         &$func($volid, $volhash->{$volid});
2592     }
2593 }
2594
2595 sub vga_conf_has_spice {
2596     my ($vga) = @_;
2597
2598     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
2599
2600     return $1 || 1;
2601 }
2602
2603 sub config_to_command {
2604     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2605
2606     my $cmd = [];
2607     my $globalFlags = [];
2608     my $machineFlags = [];
2609     my $rtcFlags = [];
2610     my $cpuFlags = [];
2611     my $devices = [];
2612     my $pciaddr = '';
2613     my $bridges = {};
2614     my $kvmver = kvm_user_version();
2615     my $vernum = 0; # unknown
2616     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
2617         $vernum = $1*1000000+$2*1000;
2618     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
2619         $vernum = $1*1000000+$2*1000+$3;
2620     }
2621
2622     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
2623
2624     my $have_ovz = -f '/proc/vz/vestat';
2625
2626     my $q35 = machine_type_is_q35($conf);
2627     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2628     my $machine_type = $forcemachine || $conf->{machine};
2629
2630     my $cpuunits = defined($conf->{cpuunits}) ?
2631             $conf->{cpuunits} : $defaults->{cpuunits};
2632
2633     push @$cmd, '/usr/bin/systemd-run';
2634     push @$cmd, '--scope';
2635     push @$cmd, '--slice', "qemu";
2636     push @$cmd, '--unit', $vmid;
2637     push @$cmd, '-p', "CPUShares=$cpuunits";
2638     if ($conf->{cpulimit}) {
2639         my $cpulimit = int($conf->{cpulimit} * 100);
2640         push @$cmd, '-p', "CPUQuota=$cpulimit\%";
2641     }
2642
2643     push @$cmd, '/usr/bin/kvm';
2644
2645     push @$cmd, '-id', $vmid;
2646
2647     my $use_virtio = 0;
2648
2649     my $qmpsocket = qmp_socket($vmid);
2650     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2651     push @$cmd, '-mon', "chardev=qmp,mode=control";
2652
2653     my $socket = vnc_socket($vmid);
2654     push @$cmd,  '-vnc', "unix:$socket,x509,password";
2655
2656     push @$cmd, '-pidfile' , pidfile_name($vmid);
2657
2658     push @$cmd, '-daemonize';
2659
2660     if ($conf->{smbios1}) {
2661         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
2662     }
2663
2664     if ($q35) {
2665         # the q35 chipset support native usb2, so we enable usb controller
2666         # by default for this machine type
2667         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
2668     } else {
2669         $pciaddr = print_pci_addr("piix3", $bridges);
2670         push @$devices, '-device', "piix3-usb-uhci,id=uhci$pciaddr.0x2";
2671
2672         my $use_usb2 = 0;
2673         for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2674             next if !$conf->{"usb$i"};
2675             $use_usb2 = 1;
2676         }
2677         # include usb device config
2678         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-usb.cfg' if $use_usb2;
2679     }
2680
2681     my $vga = $conf->{vga};
2682
2683     my $qxlnum = vga_conf_has_spice($vga);
2684     $vga = 'qxl' if $qxlnum;
2685
2686     if (!$vga) {
2687         if ($conf->{ostype} && ($conf->{ostype} eq 'win8' ||
2688                                 $conf->{ostype} eq 'win7' ||
2689                                 $conf->{ostype} eq 'w2k8')) {
2690             $vga = 'std';
2691         } else {
2692             $vga = 'cirrus';
2693         }
2694     }
2695
2696     # enable absolute mouse coordinates (needed by vnc)
2697     my $tablet;
2698     if (defined($conf->{tablet})) {
2699         $tablet = $conf->{tablet};
2700     } else {
2701         $tablet = $defaults->{tablet};
2702         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
2703         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
2704     }
2705
2706     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
2707
2708     # host pci devices
2709     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
2710         my $d = parse_hostpci($conf->{"hostpci$i"});
2711         next if !$d;
2712
2713         my $pcie = $d->{pcie};
2714         if($pcie){
2715             die "q35 machine model is not enabled" if !$q35;
2716             $pciaddr = print_pcie_addr("hostpci$i");
2717         }else{
2718             $pciaddr = print_pci_addr("hostpci$i", $bridges);
2719         }
2720
2721         my $rombar = $d->{rombar} && $d->{rombar} eq 'off' ? ",rombar=0" : "";
2722         my $xvga = $d->{'x-vga'} && $d->{'x-vga'} eq 'on' ? ",x-vga=on" : "";
2723         if ($xvga && $xvga ne '') {
2724             push @$cpuFlags, 'kvm=off';
2725             $vga = 'none';
2726         }
2727         my $pcidevices = $d->{pciid};
2728         my $multifunction = 1 if @$pcidevices > 1;
2729
2730         my $j=0;
2731         foreach my $pcidevice (@$pcidevices) {
2732
2733             my $id = "hostpci$i";
2734             $id .= ".$j" if $multifunction;
2735             my $addr = $pciaddr;
2736             $addr .= ".$j" if $multifunction;
2737             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
2738
2739             if($j == 0){
2740                 $devicestr .= "$rombar$xvga";
2741                 $devicestr .= ",multifunction=on" if $multifunction;
2742             }
2743
2744             push @$devices, '-device', $devicestr;
2745             $j++;
2746         }
2747     }
2748
2749     # usb devices
2750     for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2751         my $d = parse_usb_device($conf->{"usb$i"});
2752         next if !$d;
2753         if ($d->{vendorid} && $d->{productid}) {
2754             push @$devices, '-device', "usb-host,vendorid=0x$d->{vendorid},productid=0x$d->{productid}";
2755         } elsif (defined($d->{hostbus}) && defined($d->{hostport})) {
2756             push @$devices, '-device', "usb-host,hostbus=$d->{hostbus},hostport=$d->{hostport}";
2757         } elsif ($d->{spice}) {
2758             # usb redir support for spice
2759             push @$devices, '-chardev', "spicevmc,id=usbredirchardev$i,name=usbredir";
2760             push @$devices, '-device', "usb-redir,chardev=usbredirchardev$i,id=usbredirdev$i,bus=ehci.0";
2761         }
2762     }
2763
2764     # serial devices
2765     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2766         if (my $path = $conf->{"serial$i"}) {
2767             if ($path eq 'socket') {
2768                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
2769                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
2770                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2771             } else {
2772                 die "no such serial device\n" if ! -c $path;
2773                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
2774                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2775             }
2776         }
2777     }
2778
2779     # parallel devices
2780     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
2781         if (my $path = $conf->{"parallel$i"}) {
2782             die "no such parallel device\n" if ! -c $path;
2783             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
2784             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
2785             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
2786         }
2787     }
2788
2789     my $vmname = $conf->{name} || "vm$vmid";
2790
2791     push @$cmd, '-name', $vmname;
2792
2793     my $sockets = 1;
2794     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
2795     $sockets = $conf->{sockets} if  $conf->{sockets};
2796
2797     my $cores = $conf->{cores} || 1;
2798
2799     my $maxcpus = $sockets * $cores;
2800
2801     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
2802
2803     my $allowed_vcpus = $cpuinfo->{cpus};
2804
2805     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
2806         if ($allowed_vcpus < $maxcpus);
2807
2808     push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
2809
2810     push @$cmd, '-nodefaults';
2811
2812     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2813
2814     my $bootindex_hash = {};
2815     my $i = 1;
2816     foreach my $o (split(//, $bootorder)) {
2817         $bootindex_hash->{$o} = $i*100;
2818         $i++;
2819     }
2820
2821     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000";
2822
2823     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
2824
2825     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
2826
2827     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
2828
2829     # time drift fix
2830     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
2831
2832     my $nokvm = defined($conf->{kvm}) && $conf->{kvm} == 0 ? 1 : 0;
2833     my $useLocaltime = $conf->{localtime};
2834
2835     if (my $ost = $conf->{ostype}) {
2836         # other, wxp, w2k, w2k3, w2k8, wvista, win7, win8, l24, l26, solaris
2837
2838         if ($ost =~ m/^w/) { # windows
2839             $useLocaltime = 1 if !defined($conf->{localtime});
2840
2841             # use time drift fix when acpi is enabled
2842             if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
2843                 $tdf = 1 if !defined($conf->{tdf});
2844             }
2845         }
2846
2847         if ($ost eq 'win7' || $ost eq 'win8' || $ost eq 'w2k8' ||
2848             $ost eq 'wvista') {
2849             push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
2850             push @$cmd, '-no-hpet';
2851             if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
2852                 push @$cpuFlags , 'hv_spinlocks=0x1fff' if !$nokvm;
2853                 push @$cpuFlags , 'hv_vapic' if !$nokvm;
2854                 push @$cpuFlags , 'hv_time' if !$nokvm;
2855
2856             } else {
2857                 push @$cpuFlags , 'hv_spinlocks=0xffff' if !$nokvm;
2858             }
2859         }
2860
2861         if ($ost eq 'win7' || $ost eq 'win8') {
2862             push @$cpuFlags , 'hv_relaxed' if !$nokvm;
2863         }
2864     }
2865
2866     push @$rtcFlags, 'driftfix=slew' if $tdf;
2867
2868     if ($nokvm) {
2869         push @$machineFlags, 'accel=tcg';
2870     } else {
2871         die "No accelerator found!\n" if !$cpuinfo->{hvm};
2872     }
2873
2874     if ($machine_type) {
2875         push @$machineFlags, "type=${machine_type}";
2876     }
2877
2878     if ($conf->{startdate}) {
2879         push @$rtcFlags, "base=$conf->{startdate}";
2880     } elsif ($useLocaltime) {
2881         push @$rtcFlags, 'base=localtime';
2882     }
2883
2884     my $cpu = $nokvm ? "qemu64" : "kvm64";
2885     $cpu = $conf->{cpu} if $conf->{cpu};
2886
2887     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
2888
2889     push @$cpuFlags , '-x2apic'
2890         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
2891
2892     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
2893
2894     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
2895
2896     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
2897
2898         push @$cpuFlags , '+kvm_pv_unhalt' if !$nokvm;
2899         push @$cpuFlags , '+kvm_pv_eoi' if !$nokvm;
2900         push @$cpuFlags , '-kvm_steal_time' if !$nokvm;
2901     }
2902
2903     push @$cpuFlags, 'enforce' if $cpu ne 'host' && !$nokvm;
2904
2905     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
2906
2907     push @$cmd, '-cpu', $cpu;
2908
2909     my $memory = $conf->{memory} || $defaults->{memory};
2910     my $static_memory = 0;
2911     my $dimm_memory = 0;
2912
2913     if ($hotplug_features->{memory}) {
2914         die "Numa need to be enabled for memory hotplug\n" if !$conf->{numa};
2915         die "Total memory is bigger than ${MAX_MEM}MB\n" if $memory > $MAX_MEM;
2916         $static_memory = $STATICMEM;
2917         die "minimum memory must be ${static_memory}MB\n" if($memory < $static_memory);
2918         $dimm_memory = $memory - $static_memory;
2919         push @$cmd, '-m', "size=${static_memory},slots=255,maxmem=${MAX_MEM}M";
2920
2921     } else {
2922
2923         $static_memory = $memory;
2924         push @$cmd, '-m', $static_memory;
2925     }
2926
2927     if ($conf->{numa}) {
2928
2929         my $numa_totalmemory = undef;
2930         for (my $i = 0; $i < $MAX_NUMA; $i++) {
2931             next if !$conf->{"numa$i"};
2932             my $numa = parse_numa($conf->{"numa$i"});
2933             next if !$numa;
2934             # memory
2935             die "missing numa node$i memory value\n" if !$numa->{memory};
2936             my $numa_memory = $numa->{memory};
2937             $numa_totalmemory += $numa_memory;
2938             my $numa_object = "memory-backend-ram,id=ram-node$i,size=${numa_memory}M";
2939
2940             # cpus
2941             my $cpus_start = $numa->{cpus}->{start};
2942             die "missing numa node$i cpus\n" if !defined($cpus_start);
2943             my $cpus_end = $numa->{cpus}->{end} if defined($numa->{cpus}->{end});
2944             my $cpus = $cpus_start;
2945             if (defined($cpus_end)) {
2946                 $cpus .= "-$cpus_end";
2947                 die "numa node$i :  cpu range $cpus is incorrect\n" if $cpus_end <= $cpus_start;
2948             }
2949
2950             # hostnodes
2951             my $hostnodes_start = $numa->{hostnodes}->{start};
2952             if (defined($hostnodes_start)) {
2953                 my $hostnodes_end = $numa->{hostnodes}->{end} if defined($numa->{hostnodes}->{end});
2954                 my $hostnodes = $hostnodes_start;
2955                 if (defined($hostnodes_end)) {
2956                     $hostnodes .= "-$hostnodes_end";
2957                     die "host node $hostnodes range is incorrect\n" if $hostnodes_end <= $hostnodes_start;
2958                 }
2959
2960                 my $hostnodes_end_range = defined($hostnodes_end) ? $hostnodes_end : $hostnodes_start;
2961                 for (my $i = $hostnodes_start; $i <= $hostnodes_end_range; $i++ ) {
2962                     die "host numa node$i don't exist\n" if ! -d "/sys/devices/system/node/node$i/";
2963                 }
2964
2965                 # policy
2966                 my $policy = $numa->{policy};
2967                 die "you need to define a policy for hostnode $hostnodes\n" if !$policy;
2968                 $numa_object .= ",host-nodes=$hostnodes,policy=$policy";
2969             }
2970
2971             push @$cmd, '-object', $numa_object;
2972             push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
2973         }
2974
2975         die "total memory for NUMA nodes must be equal to vm static memory\n"
2976             if $numa_totalmemory && $numa_totalmemory != $static_memory;
2977
2978         #if no custom tology, we split memory and cores across numa nodes
2979         if(!$numa_totalmemory) {
2980
2981             my $numa_memory = ($static_memory / $sockets) . "M";
2982
2983             for (my $i = 0; $i < $sockets; $i++)  {
2984
2985                 my $cpustart = ($cores * $i);
2986                 my $cpuend = ($cpustart + $cores - 1) if $cores && $cores > 1;
2987                 my $cpus = $cpustart;
2988                 $cpus .= "-$cpuend" if $cpuend;
2989
2990                 push @$cmd, '-object', "memory-backend-ram,size=$numa_memory,id=ram-node$i";
2991                 push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
2992             }
2993         }
2994     }
2995
2996     if ($hotplug_features->{memory}) {
2997         foreach_dimm($conf, $vmid, $memory, $sockets, sub {
2998             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
2999             push @$cmd, "-object" , "memory-backend-ram,id=mem-$name,size=${dimm_size}M";
3000             push @$cmd, "-device", "pc-dimm,id=$name,memdev=mem-$name,node=$numanode";
3001
3002             #if dimm_memory is not aligned to dimm map
3003             if($current_size > $memory) {
3004                  $conf->{memory} = $current_size;
3005                  update_config_nolock($vmid, $conf, 1);
3006             }
3007         });
3008     }
3009
3010     push @$cmd, '-S' if $conf->{freeze};
3011
3012     # set keyboard layout
3013     my $kb = $conf->{keyboard} || $defaults->{keyboard};
3014     push @$cmd, '-k', $kb if $kb;
3015
3016     # enable sound
3017     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3018     #push @$cmd, '-soundhw', 'es1370';
3019     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3020
3021     if($conf->{agent}) {
3022         my $qgasocket = qmp_socket($vmid, 1);
3023         my $pciaddr = print_pci_addr("qga0", $bridges);
3024         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3025         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3026         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3027     }
3028
3029     my $spice_port;
3030
3031     if ($qxlnum) {
3032         if ($qxlnum > 1) {
3033             if ($conf->{ostype} && $conf->{ostype} =~ m/^w/){
3034                 for(my $i = 1; $i < $qxlnum; $i++){
3035                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3036                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3037                 }
3038             } else {
3039                 # assume other OS works like Linux
3040                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3041                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3042             }
3043         }
3044
3045         my $pciaddr = print_pci_addr("spice", $bridges);
3046
3047         my $nodename = PVE::INotify::nodename();
3048         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3049         $spice_port = PVE::Tools::next_spice_port($pfamily);
3050
3051         push @$devices, '-spice', "tls-port=${spice_port},addr=localhost,tls-ciphers=DES-CBC3-SHA,seamless-migration=on";
3052
3053         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3054         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3055         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3056     }
3057
3058     # enable balloon by default, unless explicitly disabled
3059     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3060         $pciaddr = print_pci_addr("balloon0", $bridges);
3061         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3062     }
3063
3064     if ($conf->{watchdog}) {
3065         my $wdopts = parse_watchdog($conf->{watchdog});
3066         $pciaddr = print_pci_addr("watchdog", $bridges);
3067         my $watchdog = $wdopts->{model} || 'i6300esb';
3068         push @$devices, '-device', "$watchdog$pciaddr";
3069         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3070     }
3071
3072     my $vollist = [];
3073     my $scsicontroller = {};
3074     my $ahcicontroller = {};
3075     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3076
3077     # Add iscsi initiator name if available
3078     if (my $initiator = get_initiator_name()) {
3079         push @$devices, '-iscsi', "initiator-name=$initiator";
3080     }
3081
3082     foreach_drive($conf, sub {
3083         my ($ds, $drive) = @_;
3084
3085         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3086             push @$vollist, $drive->{file};
3087         }
3088
3089         $use_virtio = 1 if $ds =~ m/^virtio/;
3090
3091         if (drive_is_cdrom ($drive)) {
3092             if ($bootindex_hash->{d}) {
3093                 $drive->{bootindex} = $bootindex_hash->{d};
3094                 $bootindex_hash->{d} += 1;
3095             }
3096         } else {
3097             if ($bootindex_hash->{c}) {
3098                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3099                 $bootindex_hash->{c} += 1;
3100             }
3101         }
3102
3103         if($drive->{interface} eq 'virtio'){
3104            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3105         }
3106
3107         if ($drive->{interface} eq 'scsi') {
3108
3109             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3110
3111             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3112             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3113
3114             my $iothread = '';
3115             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3116                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3117                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3118             }
3119
3120             my $queues = '';
3121             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3122                 $queues = ",num_queues=$drive->{queues}";
3123             }
3124
3125             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3126             $scsicontroller->{$controller}=1;
3127         }
3128
3129         if ($drive->{interface} eq 'sata') {
3130            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3131            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3132            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3133            $ahcicontroller->{$controller}=1;
3134         }
3135
3136         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3137         push @$devices, '-drive',$drive_cmd;
3138         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3139     });
3140
3141     for (my $i = 0; $i < $MAX_NETS; $i++) {
3142          next if !$conf->{"net$i"};
3143          my $d = parse_net($conf->{"net$i"});
3144          next if !$d;
3145
3146          $use_virtio = 1 if $d->{model} eq 'virtio';
3147
3148          if ($bootindex_hash->{n}) {
3149             $d->{bootindex} = $bootindex_hash->{n};
3150             $bootindex_hash->{n} += 1;
3151          }
3152
3153          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3154          push @$devices, '-netdev', $netdevfull;
3155
3156          my $netdevicefull = print_netdevice_full($vmid,$conf,$d,"net$i",$bridges);
3157          push @$devices, '-device', $netdevicefull;
3158     }
3159
3160     if (!$q35) {
3161         # add pci bridges
3162         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3163            $bridges->{1} = 1;
3164            $bridges->{2} = 1;
3165         }
3166
3167         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3168
3169         while (my ($k, $v) = each %$bridges) {
3170             $pciaddr = print_pci_addr("pci.$k");
3171             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3172         }
3173     }
3174
3175     # add custom args
3176     if ($conf->{args}) {
3177         my $aa = PVE::Tools::split_args($conf->{args});
3178         push @$cmd, @$aa;
3179     }
3180
3181     push @$cmd, @$devices;
3182     push @$cmd, '-rtc', join(',', @$rtcFlags)
3183         if scalar(@$rtcFlags);
3184     push @$cmd, '-machine', join(',', @$machineFlags)
3185         if scalar(@$machineFlags);
3186     push @$cmd, '-global', join(',', @$globalFlags)
3187         if scalar(@$globalFlags);
3188
3189     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3190 }
3191
3192 sub vnc_socket {
3193     my ($vmid) = @_;
3194     return "${var_run_tmpdir}/$vmid.vnc";
3195 }
3196
3197 sub spice_port {
3198     my ($vmid) = @_;
3199
3200     my $res = vm_mon_cmd($vmid, 'query-spice');
3201
3202     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3203 }
3204
3205 sub qmp_socket {
3206     my ($vmid, $qga) = @_;
3207     my $sockettype = $qga ? 'qga' : 'qmp';
3208     return "${var_run_tmpdir}/$vmid.$sockettype";
3209 }
3210
3211 sub pidfile_name {
3212     my ($vmid) = @_;
3213     return "${var_run_tmpdir}/$vmid.pid";
3214 }
3215
3216 sub vm_devices_list {
3217     my ($vmid) = @_;
3218
3219     my $res = vm_mon_cmd($vmid, 'query-pci');
3220     my $devices = {};
3221     foreach my $pcibus (@$res) {
3222         foreach my $device (@{$pcibus->{devices}}) {
3223             next if !$device->{'qdev_id'};
3224             if ($device->{'pci_bridge'}) {
3225                 $devices->{$device->{'qdev_id'}} = 1;
3226                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3227                     next if !$bridge_device->{'qdev_id'};
3228                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3229                     $devices->{$device->{'qdev_id'}}++;
3230                 }
3231             } else {
3232                 $devices->{$device->{'qdev_id'}} = 1;
3233             }
3234         }
3235     }
3236
3237     my $resblock = vm_mon_cmd($vmid, 'query-block');
3238     foreach my $block (@$resblock) {
3239         if($block->{device} =~ m/^drive-(\S+)/){
3240                 $devices->{$1} = 1;
3241         }
3242     }
3243
3244     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3245     foreach my $mice (@$resmice) {
3246         if ($mice->{name} eq 'QEMU HID Tablet') {
3247             $devices->{tablet} = 1;
3248             last;
3249         }
3250     }
3251
3252     return $devices;
3253 }
3254
3255 sub vm_deviceplug {
3256     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3257
3258     my $q35 = machine_type_is_q35($conf);
3259
3260     my $devices_list = vm_devices_list($vmid);
3261     return 1 if defined($devices_list->{$deviceid});
3262
3263     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3264
3265     if ($deviceid eq 'tablet') {
3266
3267         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3268
3269     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3270
3271         qemu_iothread_add($vmid, $deviceid, $device);
3272
3273         qemu_driveadd($storecfg, $vmid, $device);
3274         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3275
3276         qemu_deviceadd($vmid, $devicefull);
3277         eval { qemu_deviceaddverify($vmid, $deviceid); };
3278         if (my $err = $@) {
3279             eval { qemu_drivedel($vmid, $deviceid); };
3280             warn $@ if $@;
3281             die $err;
3282         }
3283
3284     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3285
3286
3287         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3288         my $pciaddr = print_pci_addr($deviceid);
3289         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3290
3291         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3292
3293         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3294             qemu_iothread_add($vmid, $deviceid, $device);
3295             $devicefull .= ",iothread=iothread-$deviceid";
3296         }
3297
3298         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3299             $devicefull .= ",num_queues=$device->{queues}";
3300         }
3301
3302         qemu_deviceadd($vmid, $devicefull);
3303         qemu_deviceaddverify($vmid, $deviceid);
3304
3305     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3306
3307         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3308         qemu_driveadd($storecfg, $vmid, $device);
3309
3310         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3311         eval { qemu_deviceadd($vmid, $devicefull); };
3312         if (my $err = $@) {
3313             eval { qemu_drivedel($vmid, $deviceid); };
3314             warn $@ if $@;
3315             die $err;
3316         }
3317
3318     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3319
3320         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3321         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid);
3322         qemu_deviceadd($vmid, $netdevicefull);
3323         eval { qemu_deviceaddverify($vmid, $deviceid); };
3324         if (my $err = $@) {
3325             eval { qemu_netdevdel($vmid, $deviceid); };
3326             warn $@ if $@;
3327             die $err;
3328         }
3329
3330     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3331
3332         my $bridgeid = $2;
3333         my $pciaddr = print_pci_addr($deviceid);
3334         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3335
3336         qemu_deviceadd($vmid, $devicefull);
3337         qemu_deviceaddverify($vmid, $deviceid);
3338
3339     } else {
3340         die "can't hotplug device '$deviceid'\n";
3341     }
3342
3343     return 1;
3344 }
3345
3346 # fixme: this should raise exceptions on error!
3347 sub vm_deviceunplug {
3348     my ($vmid, $conf, $deviceid) = @_;
3349
3350     my $devices_list = vm_devices_list($vmid);
3351     return 1 if !defined($devices_list->{$deviceid});
3352
3353     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3354
3355     if ($deviceid eq 'tablet') {
3356
3357         qemu_devicedel($vmid, $deviceid);
3358
3359     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3360
3361         qemu_devicedel($vmid, $deviceid);
3362         qemu_devicedelverify($vmid, $deviceid);
3363         qemu_drivedel($vmid, $deviceid);
3364         qemu_iothread_del($conf, $vmid, $deviceid);
3365
3366     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3367
3368         qemu_devicedel($vmid, $deviceid);
3369         qemu_devicedelverify($vmid, $deviceid);
3370         qemu_iothread_del($conf, $vmid, $deviceid);
3371
3372     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3373
3374         #qemu 2.3 segfault on drive_del with virtioscsi + iothread
3375         my $device = parse_drive($deviceid, $conf->{$deviceid});
3376         die "virtioscsi with iothread is not hot-unplugglable currently" if $device->{iothread};
3377
3378         qemu_devicedel($vmid, $deviceid);
3379         qemu_drivedel($vmid, $deviceid);
3380         qemu_deletescsihw($conf, $vmid, $deviceid);
3381
3382     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3383
3384         qemu_devicedel($vmid, $deviceid);
3385         qemu_devicedelverify($vmid, $deviceid);
3386         qemu_netdevdel($vmid, $deviceid);
3387
3388     } else {
3389         die "can't unplug device '$deviceid'\n";
3390     }
3391
3392     return 1;
3393 }
3394
3395 sub qemu_deviceadd {
3396     my ($vmid, $devicefull) = @_;
3397
3398     $devicefull = "driver=".$devicefull;
3399     my %options =  split(/[=,]/, $devicefull);
3400
3401     vm_mon_cmd($vmid, "device_add" , %options);
3402 }
3403
3404 sub qemu_devicedel {
3405     my ($vmid, $deviceid) = @_;
3406
3407     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3408 }
3409
3410 sub qemu_iothread_add {
3411     my($vmid, $deviceid, $device) = @_;
3412
3413     if ($device->{iothread}) {
3414         my $iothreads = vm_iothreads_list($vmid);
3415         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3416     }
3417 }
3418
3419 sub qemu_iothread_del {
3420     my($conf, $vmid, $deviceid) = @_;
3421
3422     my $device = parse_drive($deviceid, $conf->{$deviceid});
3423     if ($device->{iothread}) {
3424         my $iothreads = vm_iothreads_list($vmid);
3425         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3426     }
3427 }
3428
3429 sub qemu_objectadd {
3430     my($vmid, $objectid, $qomtype) = @_;
3431
3432     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3433
3434     return 1;
3435 }
3436
3437 sub qemu_objectdel {
3438     my($vmid, $objectid) = @_;
3439
3440     vm_mon_cmd($vmid, "object-del", id => $objectid);
3441
3442     return 1;
3443 }
3444
3445 sub qemu_driveadd {
3446     my ($storecfg, $vmid, $device) = @_;
3447
3448     my $drive = print_drive_full($storecfg, $vmid, $device);
3449     $drive =~ s/\\/\\\\/g;
3450     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3451
3452     # If the command succeeds qemu prints: "OK"
3453     return 1 if $ret =~ m/OK/s;
3454
3455     die "adding drive failed: $ret\n";
3456 }
3457
3458 sub qemu_drivedel {
3459     my($vmid, $deviceid) = @_;
3460
3461     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3462     $ret =~ s/^\s+//;
3463
3464     return 1 if $ret eq "";
3465
3466     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3467     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3468
3469     die "deleting drive $deviceid failed : $ret\n";
3470 }
3471
3472 sub qemu_deviceaddverify {
3473     my ($vmid, $deviceid) = @_;
3474
3475     for (my $i = 0; $i <= 5; $i++) {
3476          my $devices_list = vm_devices_list($vmid);
3477          return 1 if defined($devices_list->{$deviceid});
3478          sleep 1;
3479     }
3480
3481     die "error on hotplug device '$deviceid'\n";
3482 }
3483
3484
3485 sub qemu_devicedelverify {
3486     my ($vmid, $deviceid) = @_;
3487
3488     # need to verify that the device is correctly removed as device_del
3489     # is async and empty return is not reliable
3490
3491     for (my $i = 0; $i <= 5; $i++) {
3492          my $devices_list = vm_devices_list($vmid);
3493          return 1 if !defined($devices_list->{$deviceid});
3494          sleep 1;
3495     }
3496
3497     die "error on hot-unplugging device '$deviceid'\n";
3498 }
3499
3500 sub qemu_findorcreatescsihw {
3501     my ($storecfg, $conf, $vmid, $device) = @_;
3502
3503     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3504
3505     my $scsihwid="$controller_prefix$controller";
3506     my $devices_list = vm_devices_list($vmid);
3507
3508     if(!defined($devices_list->{$scsihwid})) {
3509         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3510     }
3511
3512     return 1;
3513 }
3514
3515 sub qemu_deletescsihw {
3516     my ($conf, $vmid, $opt) = @_;
3517
3518     my $device = parse_drive($opt, $conf->{$opt});
3519
3520     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3521         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3522         return 1;
3523     }
3524
3525     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3526
3527     my $devices_list = vm_devices_list($vmid);
3528     foreach my $opt (keys %{$devices_list}) {
3529         if (PVE::QemuServer::valid_drivename($opt)) {
3530             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3531             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3532                 return 1;
3533             }
3534         }
3535     }
3536
3537     my $scsihwid="scsihw$controller";
3538
3539     vm_deviceunplug($vmid, $conf, $scsihwid);
3540
3541     return 1;
3542 }
3543
3544 sub qemu_add_pci_bridge {
3545     my ($storecfg, $conf, $vmid, $device) = @_;
3546
3547     my $bridges = {};
3548
3549     my $bridgeid;
3550
3551     print_pci_addr($device, $bridges);
3552
3553     while (my ($k, $v) = each %$bridges) {
3554         $bridgeid = $k;
3555     }
3556     return 1 if !defined($bridgeid) || $bridgeid < 1;
3557
3558     my $bridge = "pci.$bridgeid";
3559     my $devices_list = vm_devices_list($vmid);
3560
3561     if (!defined($devices_list->{$bridge})) {
3562         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3563     }
3564
3565     return 1;
3566 }
3567
3568 sub qemu_set_link_status {
3569     my ($vmid, $device, $up) = @_;
3570
3571     vm_mon_cmd($vmid, "set_link", name => $device,
3572                up => $up ? JSON::true : JSON::false);
3573 }
3574
3575 sub qemu_netdevadd {
3576     my ($vmid, $conf, $device, $deviceid) = @_;
3577
3578     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid);
3579     my %options =  split(/[=,]/, $netdev);
3580
3581     vm_mon_cmd($vmid, "netdev_add",  %options);
3582     return 1;
3583 }
3584
3585 sub qemu_netdevdel {
3586     my ($vmid, $deviceid) = @_;
3587
3588     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3589 }
3590
3591 sub qemu_cpu_hotplug {
3592     my ($vmid, $conf, $vcpus) = @_;
3593
3594     my $sockets = 1;
3595     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3596     $sockets = $conf->{sockets} if  $conf->{sockets};
3597     my $cores = $conf->{cores} || 1;
3598     my $maxcpus = $sockets * $cores;
3599
3600     $vcpus = $maxcpus if !$vcpus;
3601
3602     die "you can't add more vcpus than maxcpus\n"
3603         if $vcpus > $maxcpus;
3604
3605     my $currentvcpus = $conf->{vcpus} || $maxcpus;
3606     die "online cpu unplug is not yet possible\n"
3607         if $vcpus < $currentvcpus;
3608
3609     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3610     die "vcpus in running vm is different than configuration\n"
3611         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
3612
3613     for (my $i = $currentvcpus; $i < $vcpus; $i++) {
3614         vm_mon_cmd($vmid, "cpu-add", id => int($i));
3615     }
3616 }
3617
3618 sub qemu_memory_hotplug {
3619     my ($vmid, $conf, $defaults, $opt, $value) = @_;
3620
3621     return $value if !check_running($vmid);
3622
3623     my $memory = $conf->{memory} || $defaults->{memory};
3624     $value = $defaults->{memory} if !$value;
3625     return $value if $value == $memory;
3626
3627     my $static_memory = $STATICMEM;
3628     my $dimm_memory = $memory - $static_memory;
3629
3630     die "memory can't be lower than $static_memory MB" if $value < $static_memory;
3631     die "you cannot add more memory than $MAX_MEM MB!\n" if $memory > $MAX_MEM;
3632
3633
3634     my $sockets = 1;
3635     $sockets = $conf->{sockets} if $conf->{sockets};
3636
3637     if($value > $memory) {
3638
3639         foreach_dimm($conf, $vmid, $value, $sockets, sub {
3640             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3641
3642                 return if $current_size <= $conf->{memory};
3643
3644                 eval { vm_mon_cmd($vmid, "object-add", 'qom-type' => "memory-backend-ram", id => "mem-$name", props => { size => int($dimm_size*1024*1024) } ) };
3645                 if (my $err = $@) {
3646                     eval { qemu_objectdel($vmid, "mem-$name"); };
3647                     die $err;
3648                 }
3649
3650                 eval { vm_mon_cmd($vmid, "device_add", driver => "pc-dimm", id => "$name", memdev => "mem-$name", node => $numanode) };
3651                 if (my $err = $@) {
3652                     eval { qemu_objectdel($vmid, "mem-$name"); };
3653                     die $err;
3654                 }
3655                 #update conf after each succesful module hotplug
3656                 $conf->{memory} = $current_size;
3657                 update_config_nolock($vmid, $conf, 1);
3658         });
3659
3660     } else {
3661
3662         foreach_reverse_dimm($conf, $vmid, $value, $sockets, sub {
3663             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3664
3665                 return if $current_size >= $conf->{memory};
3666                 print "try to unplug memory dimm $name\n";
3667
3668                 my $retry = 0;
3669                 while (1) {
3670                     eval { qemu_devicedel($vmid, $name) };
3671                     sleep 3;
3672                     my $dimm_list = qemu_dimm_list($vmid);
3673                     last if !$dimm_list->{$name};
3674                     raise_param_exc({ $name => "error unplug memory module" }) if $retry > 5;
3675                     $retry++;
3676                 }
3677
3678                 #update conf after each succesful module unplug
3679                 $conf->{memory} = $current_size;
3680
3681                 eval { qemu_objectdel($vmid, "mem-$name"); };
3682                 update_config_nolock($vmid, $conf, 1);
3683         });
3684     }
3685 }
3686
3687 sub qemu_dimm_list {
3688     my ($vmid) = @_;
3689
3690     my $dimmarray = vm_mon_cmd_nocheck($vmid, "query-memory-devices");
3691     my $dimms = {};
3692
3693     foreach my $dimm (@$dimmarray) {
3694
3695         $dimms->{$dimm->{data}->{id}}->{id} = $dimm->{data}->{id};
3696         $dimms->{$dimm->{data}->{id}}->{node} = $dimm->{data}->{node};
3697         $dimms->{$dimm->{data}->{id}}->{addr} = $dimm->{data}->{addr};
3698         $dimms->{$dimm->{data}->{id}}->{size} = $dimm->{data}->{size};
3699         $dimms->{$dimm->{data}->{id}}->{slot} = $dimm->{data}->{slot};
3700     }
3701     return $dimms;
3702 }
3703
3704 sub qemu_block_set_io_throttle {
3705     my ($vmid, $deviceid, $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr) = @_;
3706
3707     return if !check_running($vmid) ;
3708
3709     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid, bps => int($bps), bps_rd => int($bps_rd), bps_wr => int($bps_wr), iops => int($iops), iops_rd => int($iops_rd), iops_wr => int($iops_wr));
3710
3711 }
3712
3713 # old code, only used to shutdown old VM after update
3714 sub __read_avail {
3715     my ($fh, $timeout) = @_;
3716
3717     my $sel = new IO::Select;
3718     $sel->add($fh);
3719
3720     my $res = '';
3721     my $buf;
3722
3723     my @ready;
3724     while (scalar (@ready = $sel->can_read($timeout))) {
3725         my $count;
3726         if ($count = $fh->sysread($buf, 8192)) {
3727             if ($buf =~ /^(.*)\(qemu\) $/s) {
3728                 $res .= $1;
3729                 last;
3730             } else {
3731                 $res .= $buf;
3732             }
3733         } else {
3734             if (!defined($count)) {
3735                 die "$!\n";
3736             }
3737             last;
3738         }
3739     }
3740
3741     die "monitor read timeout\n" if !scalar(@ready);
3742
3743     return $res;
3744 }
3745
3746 # old code, only used to shutdown old VM after update
3747 sub vm_monitor_command {
3748     my ($vmid, $cmdstr, $nocheck) = @_;
3749
3750     my $res;
3751
3752     eval {
3753         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
3754
3755         my $sname = "${var_run_tmpdir}/$vmid.mon";
3756
3757         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
3758             die "unable to connect to VM $vmid socket - $!\n";
3759
3760         my $timeout = 3;
3761
3762         # hack: migrate sometime blocks the monitor (when migrate_downtime
3763         # is set)
3764         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3765             $timeout = 60*60; # 1 hour
3766         }
3767
3768         # read banner;
3769         my $data = __read_avail($sock, $timeout);
3770
3771         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
3772             die "got unexpected qemu monitor banner\n";
3773         }
3774
3775         my $sel = new IO::Select;
3776         $sel->add($sock);
3777
3778         if (!scalar(my @ready = $sel->can_write($timeout))) {
3779             die "monitor write error - timeout";
3780         }
3781
3782         my $fullcmd = "$cmdstr\r";
3783
3784         # syslog('info', "VM $vmid monitor command: $cmdstr");
3785
3786         my $b;
3787         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
3788             die "monitor write error - $!";
3789         }
3790
3791         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
3792
3793         $timeout = 20;
3794
3795         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3796             $timeout = 60*60; # 1 hour
3797         } elsif ($cmdstr =~ m/^(eject|change)/) {
3798             $timeout = 60; # note: cdrom mount command is slow
3799         }
3800         if ($res = __read_avail($sock, $timeout)) {
3801
3802             my @lines = split("\r?\n", $res);
3803
3804             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
3805
3806             $res = join("\n", @lines);
3807             $res .= "\n";
3808         }
3809     };
3810
3811     my $err = $@;
3812
3813     if ($err) {
3814         syslog("err", "VM $vmid monitor command failed - $err");
3815         die $err;
3816     }
3817
3818     return $res;
3819 }
3820
3821 sub qemu_block_resize {
3822     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
3823
3824     my $running = check_running($vmid);
3825
3826     return if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
3827
3828     return if !$running;
3829
3830     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
3831
3832 }
3833
3834 sub qemu_volume_snapshot {
3835     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3836
3837     my $running = check_running($vmid);
3838
3839     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
3840         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
3841     } else {
3842         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
3843     }
3844 }
3845
3846 sub qemu_volume_snapshot_delete {
3847     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3848
3849     my $running = check_running($vmid);
3850
3851     return if !PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
3852
3853     return if !$running;
3854
3855     vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
3856 }
3857
3858 sub set_migration_caps {
3859     my ($vmid) = @_;
3860
3861     my $cap_ref = [];
3862
3863     my $enabled_cap = {
3864         "auto-converge" => 1,
3865         "xbzrle" => 1,
3866         "x-rdma-pin-all" => 0,
3867         "zero-blocks" => 0,
3868         "compress" => 0
3869     };
3870
3871     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
3872
3873     for my $supported_capability (@$supported_capabilities) {
3874         push @$cap_ref, {
3875             capability => $supported_capability->{capability},
3876             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
3877         };
3878     }
3879
3880     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
3881 }
3882
3883 my $fast_plug_option = {
3884     'lock' => 1,
3885     'name' => 1,
3886     'onboot' => 1,
3887     'shares' => 1,
3888     'startup' => 1,
3889     'description' => 1,
3890 };
3891
3892 # hotplug changes in [PENDING]
3893 # $selection hash can be used to only apply specified options, for
3894 # example: { cores => 1 } (only apply changed 'cores')
3895 # $errors ref is used to return error messages
3896 sub vmconfig_hotplug_pending {
3897     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
3898
3899     my $defaults = load_defaults();
3900
3901     # commit values which do not have any impact on running VM first
3902     # Note: those option cannot raise errors, we we do not care about
3903     # $selection and always apply them.
3904
3905     my $add_error = sub {
3906         my ($opt, $msg) = @_;
3907         $errors->{$opt} = "hotplug problem - $msg";
3908     };
3909
3910     my $changes = 0;
3911     foreach my $opt (keys %{$conf->{pending}}) { # add/change
3912         if ($fast_plug_option->{$opt}) {
3913             $conf->{$opt} = $conf->{pending}->{$opt};
3914             delete $conf->{pending}->{$opt};
3915             $changes = 1;
3916         }
3917     }
3918
3919     if ($changes) {
3920         update_config_nolock($vmid, $conf, 1);
3921         $conf = load_config($vmid); # update/reload
3922     }
3923
3924     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3925
3926     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
3927     while (my ($opt, $force) = each %$pending_delete_hash) {
3928         next if $selection && !$selection->{$opt};
3929         eval {
3930             if ($opt eq 'hotplug') {
3931                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
3932             } elsif ($opt eq 'tablet') {
3933                 die "skip\n" if !$hotplug_features->{usb};
3934                 if ($defaults->{tablet}) {
3935                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
3936                 } else {
3937                     vm_deviceunplug($vmid, $conf, $opt);
3938                 }
3939             } elsif ($opt eq 'vcpus') {
3940                 die "skip\n" if !$hotplug_features->{cpu};
3941                 qemu_cpu_hotplug($vmid, $conf, undef);
3942             } elsif ($opt eq 'balloon') {
3943                 # enable balloon device is not hotpluggable
3944                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
3945             } elsif ($fast_plug_option->{$opt}) {
3946                 # do nothing
3947             } elsif ($opt =~ m/^net(\d+)$/) {
3948                 die "skip\n" if !$hotplug_features->{network};
3949                 vm_deviceunplug($vmid, $conf, $opt);
3950             } elsif (valid_drivename($opt)) {
3951                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
3952                 vm_deviceunplug($vmid, $conf, $opt);
3953                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
3954             } elsif ($opt =~ m/^memory$/) {
3955                 die "skip\n" if !$hotplug_features->{memory};
3956                 qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
3957             } elsif ($opt eq 'cpuunits') {
3958                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
3959             } elsif ($opt eq 'cpulimit') {
3960                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
3961             } else {
3962                 die "skip\n";
3963             }
3964         };
3965         if (my $err = $@) {
3966             &$add_error($opt, $err) if $err ne "skip\n";
3967         } else {
3968             # save new config if hotplug was successful
3969             delete $conf->{$opt};
3970             vmconfig_undelete_pending_option($conf, $opt);
3971             update_config_nolock($vmid, $conf, 1);
3972             $conf = load_config($vmid); # update/reload
3973         }
3974     }
3975
3976     foreach my $opt (keys %{$conf->{pending}}) {
3977         next if $selection && !$selection->{$opt};
3978         my $value = $conf->{pending}->{$opt};
3979         eval {
3980             if ($opt eq 'hotplug') {
3981                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
3982             } elsif ($opt eq 'tablet') {
3983                 die "skip\n" if !$hotplug_features->{usb};
3984                 if ($value == 1) {
3985                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
3986                 } elsif ($value == 0) {
3987                     vm_deviceunplug($vmid, $conf, $opt);
3988                 }
3989             } elsif ($opt eq 'vcpus') {
3990                 die "skip\n" if !$hotplug_features->{cpu};
3991                 qemu_cpu_hotplug($vmid, $conf, $value);
3992             } elsif ($opt eq 'balloon') {
3993                 # enable/disable balloning device is not hotpluggable
3994                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
3995                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
3996                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
3997
3998                 # allow manual ballooning if shares is set to zero
3999                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4000                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4001                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4002                 }
4003             } elsif ($opt =~ m/^net(\d+)$/) {
4004                 # some changes can be done without hotplug
4005                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4006                                     $vmid, $opt, $value);
4007             } elsif (valid_drivename($opt)) {
4008                 # some changes can be done without hotplug
4009                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4010                                      $vmid, $opt, $value, 1);
4011             } elsif ($opt =~ m/^memory$/) { #dimms
4012                 die "skip\n" if !$hotplug_features->{memory};
4013                 $value = qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4014             } elsif ($opt eq 'cpuunits') {
4015                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4016             } elsif ($opt eq 'cpulimit') {
4017                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4018                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4019             } else {
4020                 die "skip\n";  # skip non-hot-pluggable options
4021             }
4022         };
4023         if (my $err = $@) {
4024             &$add_error($opt, $err) if $err ne "skip\n";
4025         } else {
4026             # save new config if hotplug was successful
4027             $conf->{$opt} = $value;
4028             delete $conf->{pending}->{$opt};
4029             update_config_nolock($vmid, $conf, 1);
4030             $conf = load_config($vmid); # update/reload
4031         }
4032     }
4033 }
4034
4035 sub try_deallocate_drive {
4036     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4037
4038     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4039         my $volid = $drive->{file};
4040         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4041             my $sid = PVE::Storage::parse_volume_id($volid);
4042             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4043
4044             # check if the disk is really unused
4045             my $used_paths = PVE::QemuServer::get_used_paths($vmid, $storecfg, $conf, 1, $key);
4046             my $path = PVE::Storage::path($storecfg, $volid);
4047             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4048                    if $used_paths->{$path};
4049             PVE::Storage::vdisk_free($storecfg, $volid);
4050             return 1;
4051         } else {
4052             # If vm is not owner of this disk remove from config
4053             return 1;
4054         }
4055     }
4056
4057     return undef;
4058 }
4059
4060 sub vmconfig_delete_or_detach_drive {
4061     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4062
4063     my $drive = parse_drive($opt, $conf->{$opt});
4064
4065     my $rpcenv = PVE::RPCEnvironment::get();
4066     my $authuser = $rpcenv->get_user();
4067
4068     if ($force) {
4069         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4070         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4071     } else {
4072         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4073     }
4074 }
4075
4076 sub vmconfig_apply_pending {
4077     my ($vmid, $conf, $storecfg) = @_;
4078
4079     # cold plug
4080
4081     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4082     while (my ($opt, $force) = each %$pending_delete_hash) {
4083         die "internal error" if $opt =~ m/^unused/;
4084         $conf = load_config($vmid); # update/reload
4085         if (!defined($conf->{$opt})) {
4086             vmconfig_undelete_pending_option($conf, $opt);
4087             update_config_nolock($vmid, $conf, 1);
4088         } elsif (valid_drivename($opt)) {
4089             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4090             vmconfig_undelete_pending_option($conf, $opt);
4091             delete $conf->{$opt};
4092             update_config_nolock($vmid, $conf, 1);
4093         } else {
4094             vmconfig_undelete_pending_option($conf, $opt);
4095             delete $conf->{$opt};
4096             update_config_nolock($vmid, $conf, 1);
4097         }
4098     }
4099
4100     $conf = load_config($vmid); # update/reload
4101
4102     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4103         $conf = load_config($vmid); # update/reload
4104
4105         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4106             # skip if nothing changed
4107         } elsif (valid_drivename($opt)) {
4108             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4109                 if defined($conf->{$opt});
4110             $conf->{$opt} = $conf->{pending}->{$opt};
4111         } else {
4112             $conf->{$opt} = $conf->{pending}->{$opt};
4113         }
4114
4115         delete $conf->{pending}->{$opt};
4116         update_config_nolock($vmid, $conf, 1);
4117     }
4118 }
4119
4120 my $safe_num_ne = sub {
4121     my ($a, $b) = @_;
4122
4123     return 0 if !defined($a) && !defined($b);
4124     return 1 if !defined($a);
4125     return 1 if !defined($b);
4126
4127     return $a != $b;
4128 };
4129
4130 my $safe_string_ne = sub {
4131     my ($a, $b) = @_;
4132
4133     return 0 if !defined($a) && !defined($b);
4134     return 1 if !defined($a);
4135     return 1 if !defined($b);
4136
4137     return $a ne $b;
4138 };
4139
4140 sub vmconfig_update_net {
4141     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4142
4143     my $newnet = parse_net($value);
4144
4145     if ($conf->{$opt}) {
4146         my $oldnet = parse_net($conf->{$opt});
4147
4148         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4149             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4150             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4151             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4152
4153             # for non online change, we try to hot-unplug
4154             die "skip\n" if !$hotplug;
4155             vm_deviceunplug($vmid, $conf, $opt);
4156         } else {
4157
4158             die "internal error" if $opt !~ m/net(\d+)/;
4159             my $iface = "tap${vmid}i$1";
4160
4161             if (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4162                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4163             }
4164
4165             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4166                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4167                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4168                 PVE::Network::tap_unplug($iface);
4169                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall});
4170             }
4171
4172             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4173                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4174             }
4175
4176             return 1;
4177         }
4178     }
4179
4180     if ($hotplug) {
4181         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4182     } else {
4183         die "skip\n";
4184     }
4185 }
4186
4187 sub vmconfig_update_disk {
4188     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4189
4190     # fixme: do we need force?
4191
4192     my $drive = parse_drive($opt, $value);
4193
4194     if ($conf->{$opt}) {
4195
4196         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4197
4198             my $media = $drive->{media} || 'disk';
4199             my $oldmedia = $old_drive->{media} || 'disk';
4200             die "unable to change media type\n" if $media ne $oldmedia;
4201
4202             if (!drive_is_cdrom($old_drive)) {
4203
4204                 if ($drive->{file} ne $old_drive->{file}) {
4205
4206                     die "skip\n" if !$hotplug;
4207
4208                     # unplug and register as unused
4209                     vm_deviceunplug($vmid, $conf, $opt);
4210                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4211
4212                 } else {
4213                     # update existing disk
4214
4215                     # skip non hotpluggable value
4216                     if (&$safe_num_ne($drive->{discard}, $old_drive->{discard}) ||
4217                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4218                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4219                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4220                         die "skip\n";
4221                     }
4222
4223                     # apply throttle
4224                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4225                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4226                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4227                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4228                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4229                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4230                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4231                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4232                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4233                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4234                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4235                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max})) {
4236
4237                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4238                                                    ($drive->{mbps} || 0)*1024*1024,
4239                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4240                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4241                                                    $drive->{iops} || 0,
4242                                                    $drive->{iops_rd} || 0,
4243                                                    $drive->{iops_wr} || 0,
4244                                                    ($drive->{mbps_max} || 0)*1024*1024,
4245                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4246                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4247                                                    $drive->{iops_max} || 0,
4248                                                    $drive->{iops_rd_max} || 0,
4249                                                    $drive->{iops_wr_max} || 0);
4250
4251                     }
4252
4253                     return 1;
4254                 }
4255
4256             } else { # cdrom
4257
4258                 if ($drive->{file} eq 'none') {
4259                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4260                 } else {
4261                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4262                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4263                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4264                 }
4265
4266                 return 1;
4267             }
4268         }
4269     }
4270
4271     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4272     # hotplug new disks
4273     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4274 }
4275
4276 sub vm_start {
4277     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused, $forcemachine, $spice_ticket) = @_;
4278
4279     lock_config($vmid, sub {
4280         my $conf = load_config($vmid, $migratedfrom);
4281
4282         die "you can't start a vm if it's a template\n" if is_template($conf);
4283
4284         check_lock($conf) if !$skiplock;
4285
4286         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4287
4288         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4289             vmconfig_apply_pending($vmid, $conf, $storecfg);
4290             $conf = load_config($vmid); # update/reload
4291         }
4292
4293         my $defaults = load_defaults();
4294
4295         # set environment variable useful inside network script
4296         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4297
4298         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4299
4300         my $migrate_port = 0;
4301         my $migrate_uri;
4302         if ($statefile) {
4303             if ($statefile eq 'tcp') {
4304                 my $localip = "localhost";
4305                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4306                 my $nodename = PVE::INotify::nodename();
4307                 if ($datacenterconf->{migration_unsecure}) {
4308                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4309                 }
4310                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4311                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4312                 $migrate_uri = "tcp:[${localip}]:${migrate_port}";
4313                 push @$cmd, '-incoming', $migrate_uri;
4314                 push @$cmd, '-S';
4315             } else {
4316                 push @$cmd, '-loadstate', $statefile;
4317             }
4318         } elsif ($paused) {
4319             push @$cmd, '-S';
4320         }
4321
4322         # host pci devices
4323         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4324           my $d = parse_hostpci($conf->{"hostpci$i"});
4325           next if !$d;
4326           my $pcidevices = $d->{pciid};
4327           foreach my $pcidevice (@$pcidevices) {
4328                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4329
4330                 my $info = pci_device_info("0000:$pciid");
4331                 die "IOMMU not present\n" if !check_iommu_support();
4332                 die "no pci device info for device '$pciid'\n" if !$info;
4333                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4334                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4335           }
4336         }
4337
4338         PVE::Storage::activate_volumes($storecfg, $vollist);
4339
4340         eval  { run_command($cmd, timeout => $statefile ? undef : 30,
4341                     umask => 0077); };
4342         my $err = $@;
4343         die "start failed: $err" if $err;
4344
4345         print "migration listens on $migrate_uri\n" if $migrate_uri;
4346
4347         if ($statefile && $statefile ne 'tcp')  {
4348             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4349             warn $@ if $@;
4350         }
4351
4352         if ($migratedfrom) {
4353
4354             eval {
4355                 set_migration_caps($vmid);
4356             };
4357             warn $@ if $@;
4358
4359             if ($spice_port) {
4360                 print "spice listens on port $spice_port\n";
4361                 if ($spice_ticket) {
4362                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4363                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4364                 }
4365             }
4366
4367         } else {
4368
4369             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4370                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4371                     if $conf->{balloon};
4372             }
4373
4374             foreach my $opt (keys %$conf) {
4375                 next if $opt !~  m/^net\d+$/;
4376                 my $nicconf = parse_net($conf->{$opt});
4377                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4378             }
4379         }
4380
4381         vm_mon_cmd_nocheck($vmid, 'qom-set',
4382                     path => "machine/peripheral/balloon0",
4383                     property => "guest-stats-polling-interval",
4384                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4385
4386     });
4387 }
4388
4389 sub vm_mon_cmd {
4390     my ($vmid, $execute, %params) = @_;
4391
4392     my $cmd = { execute => $execute, arguments => \%params };
4393     vm_qmp_command($vmid, $cmd);
4394 }
4395
4396 sub vm_mon_cmd_nocheck {
4397     my ($vmid, $execute, %params) = @_;
4398
4399     my $cmd = { execute => $execute, arguments => \%params };
4400     vm_qmp_command($vmid, $cmd, 1);
4401 }
4402
4403 sub vm_qmp_command {
4404     my ($vmid, $cmd, $nocheck) = @_;
4405
4406     my $res;
4407
4408     my $timeout;
4409     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4410         $timeout = $cmd->{arguments}->{timeout};
4411         delete $cmd->{arguments}->{timeout};
4412     }
4413
4414     eval {
4415         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4416         my $sname = qmp_socket($vmid);
4417         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4418             my $qmpclient = PVE::QMPClient->new();
4419
4420             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4421         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
4422             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
4423                 if scalar(%{$cmd->{arguments}});
4424             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
4425         } else {
4426             die "unable to open monitor socket\n";
4427         }
4428     };
4429     if (my $err = $@) {
4430         syslog("err", "VM $vmid qmp command failed - $err");
4431         die $err;
4432     }
4433
4434     return $res;
4435 }
4436
4437 sub vm_human_monitor_command {
4438     my ($vmid, $cmdline) = @_;
4439
4440     my $res;
4441
4442     my $cmd = {
4443         execute => 'human-monitor-command',
4444         arguments => { 'command-line' => $cmdline},
4445     };
4446
4447     return vm_qmp_command($vmid, $cmd);
4448 }
4449
4450 sub vm_commandline {
4451     my ($storecfg, $vmid) = @_;
4452
4453     my $conf = load_config($vmid);
4454
4455     my $defaults = load_defaults();
4456
4457     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
4458
4459     return join(' ', @$cmd);
4460 }
4461
4462 sub vm_reset {
4463     my ($vmid, $skiplock) = @_;
4464
4465     lock_config($vmid, sub {
4466
4467         my $conf = load_config($vmid);
4468
4469         check_lock($conf) if !$skiplock;
4470
4471         vm_mon_cmd($vmid, "system_reset");
4472     });
4473 }
4474
4475 sub get_vm_volumes {
4476     my ($conf) = @_;
4477
4478     my $vollist = [];
4479     foreach_volid($conf, sub {
4480         my ($volid, $is_cdrom) = @_;
4481
4482         return if $volid =~ m|^/|;
4483
4484         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4485         return if !$sid;
4486
4487         push @$vollist, $volid;
4488     });
4489
4490     return $vollist;
4491 }
4492
4493 sub vm_stop_cleanup {
4494     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
4495
4496     eval {
4497
4498         if (!$keepActive) {
4499             my $vollist = get_vm_volumes($conf);
4500             PVE::Storage::deactivate_volumes($storecfg, $vollist);
4501         }
4502
4503         foreach my $ext (qw(mon qmp pid vnc qga)) {
4504             unlink "/var/run/qemu-server/${vmid}.$ext";
4505         }
4506
4507         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
4508     };
4509     warn $@ if $@; # avoid errors - just warn
4510 }
4511
4512 # Note: use $nockeck to skip tests if VM configuration file exists.
4513 # We need that when migration VMs to other nodes (files already moved)
4514 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
4515 sub vm_stop {
4516     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
4517
4518     $force = 1 if !defined($force) && !$shutdown;
4519
4520     if ($migratedfrom){
4521         my $pid = check_running($vmid, $nocheck, $migratedfrom);
4522         kill 15, $pid if $pid;
4523         my $conf = load_config($vmid, $migratedfrom);
4524         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
4525         return;
4526     }
4527
4528     lock_config($vmid, sub {
4529
4530         my $pid = check_running($vmid, $nocheck);
4531         return if !$pid;
4532
4533         my $conf;
4534         if (!$nocheck) {
4535             $conf = load_config($vmid);
4536             check_lock($conf) if !$skiplock;
4537             if (!defined($timeout) && $shutdown && $conf->{startup}) {
4538                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
4539                 $timeout = $opts->{down} if $opts->{down};
4540             }
4541         }
4542
4543         $timeout = 60 if !defined($timeout);
4544
4545         eval {
4546             if ($shutdown) {
4547                 if (defined($conf) && $conf->{agent}) {
4548                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
4549                 } else {
4550                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
4551                 }
4552             } else {
4553                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
4554             }
4555         };
4556         my $err = $@;
4557
4558         if (!$err) {
4559             my $count = 0;
4560             while (($count < $timeout) && check_running($vmid, $nocheck)) {
4561                 $count++;
4562                 sleep 1;
4563             }
4564
4565             if ($count >= $timeout) {
4566                 if ($force) {
4567                     warn "VM still running - terminating now with SIGTERM\n";
4568                     kill 15, $pid;
4569                 } else {
4570                     die "VM quit/powerdown failed - got timeout\n";
4571                 }
4572             } else {
4573                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4574                 return;
4575             }
4576         } else {
4577             if ($force) {
4578                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
4579                 kill 15, $pid;
4580             } else {
4581                 die "VM quit/powerdown failed\n";
4582             }
4583         }
4584
4585         # wait again
4586         $timeout = 10;
4587
4588         my $count = 0;
4589         while (($count < $timeout) && check_running($vmid, $nocheck)) {
4590             $count++;
4591             sleep 1;
4592         }
4593
4594         if ($count >= $timeout) {
4595             warn "VM still running - terminating now with SIGKILL\n";
4596             kill 9, $pid;
4597             sleep 1;
4598         }
4599
4600         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4601    });
4602 }
4603
4604 sub vm_suspend {
4605     my ($vmid, $skiplock) = @_;
4606
4607     lock_config($vmid, sub {
4608
4609         my $conf = load_config($vmid);
4610
4611         check_lock($conf) if !($skiplock || ($conf->{lock} && $conf->{lock} eq 'backup'));
4612
4613         vm_mon_cmd($vmid, "stop");
4614     });
4615 }
4616
4617 sub vm_resume {
4618     my ($vmid, $skiplock) = @_;
4619
4620     lock_config($vmid, sub {
4621
4622         my $conf = load_config($vmid);
4623
4624         check_lock($conf) if !($skiplock || ($conf->{lock} && $conf->{lock} eq 'backup'));
4625
4626         vm_mon_cmd($vmid, "cont");
4627     });
4628 }
4629
4630 sub vm_sendkey {
4631     my ($vmid, $skiplock, $key) = @_;
4632
4633     lock_config($vmid, sub {
4634
4635         my $conf = load_config($vmid);
4636
4637         # there is no qmp command, so we use the human monitor command
4638         vm_human_monitor_command($vmid, "sendkey $key");
4639     });
4640 }
4641
4642 sub vm_destroy {
4643     my ($storecfg, $vmid, $skiplock) = @_;
4644
4645     lock_config($vmid, sub {
4646
4647         my $conf = load_config($vmid);
4648
4649         check_lock($conf) if !$skiplock;
4650
4651         if (!check_running($vmid)) {
4652             destroy_vm($storecfg, $vmid);
4653         } else {
4654             die "VM $vmid is running - destroy failed\n";
4655         }
4656     });
4657 }
4658
4659 # pci helpers
4660
4661 sub file_write {
4662     my ($filename, $buf) = @_;
4663
4664     my $fh = IO::File->new($filename, "w");
4665     return undef if !$fh;
4666
4667     my $res = print $fh $buf;
4668
4669     $fh->close();
4670
4671     return $res;
4672 }
4673
4674 sub pci_device_info {
4675     my ($name) = @_;
4676
4677     my $res;
4678
4679     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
4680     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
4681
4682     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
4683     return undef if !defined($irq) || $irq !~ m/^\d+$/;
4684
4685     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
4686     return undef if !defined($vendor) || $vendor !~ s/^0x//;
4687
4688     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
4689     return undef if !defined($product) || $product !~ s/^0x//;
4690
4691     $res = {
4692         name => $name,
4693         vendor => $vendor,
4694         product => $product,
4695         domain => $domain,
4696         bus => $bus,
4697         slot => $slot,
4698         func => $func,
4699         irq => $irq,
4700         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
4701     };
4702
4703     return $res;
4704 }
4705
4706 sub pci_dev_reset {
4707     my ($dev) = @_;
4708
4709     my $name = $dev->{name};
4710
4711     my $fn = "$pcisysfs/devices/$name/reset";
4712
4713     return file_write($fn, "1");
4714 }
4715
4716 sub pci_dev_bind_to_vfio {
4717     my ($dev) = @_;
4718
4719     my $name = $dev->{name};
4720
4721     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4722
4723     if (!-d $vfio_basedir) {
4724         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4725     }
4726     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4727
4728     my $testdir = "$vfio_basedir/$name";
4729     return 1 if -d $testdir;
4730
4731     my $data = "$dev->{vendor} $dev->{product}";
4732     return undef if !file_write("$vfio_basedir/new_id", $data);
4733
4734     my $fn = "$pcisysfs/devices/$name/driver/unbind";
4735     if (!file_write($fn, $name)) {
4736         return undef if -f $fn;
4737     }
4738
4739     $fn = "$vfio_basedir/bind";
4740     if (! -d $testdir) {
4741         return undef if !file_write($fn, $name);
4742     }
4743
4744     return -d $testdir;
4745 }
4746
4747 sub pci_dev_group_bind_to_vfio {
4748     my ($pciid) = @_;
4749
4750     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4751
4752     if (!-d $vfio_basedir) {
4753         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4754     }
4755     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4756
4757     # get IOMMU group devices
4758     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
4759       my @devs = grep /^0000:/, readdir($D);
4760     closedir($D);
4761
4762     foreach my $pciid (@devs) {
4763         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
4764
4765         # pci bridges, switches or root ports are not supported
4766         # they have a pci_bus subdirectory so skip them
4767         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
4768
4769         my $info = pci_device_info($1);
4770         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
4771     }
4772
4773     return 1;
4774 }
4775
4776 sub print_pci_addr {
4777     my ($id, $bridges) = @_;
4778
4779     my $res = '';
4780     my $devices = {
4781         piix3 => { bus => 0, addr => 1 },
4782         #addr2 : first videocard
4783         balloon0 => { bus => 0, addr => 3 },
4784         watchdog => { bus => 0, addr => 4 },
4785         scsihw0 => { bus => 0, addr => 5 },
4786         'pci.3' => { bus => 0, addr => 5 }, #can also be used for virtio-scsi-single bridge
4787         scsihw1 => { bus => 0, addr => 6 },
4788         ahci0 => { bus => 0, addr => 7 },
4789         qga0 => { bus => 0, addr => 8 },
4790         spice => { bus => 0, addr => 9 },
4791         virtio0 => { bus => 0, addr => 10 },
4792         virtio1 => { bus => 0, addr => 11 },
4793         virtio2 => { bus => 0, addr => 12 },
4794         virtio3 => { bus => 0, addr => 13 },
4795         virtio4 => { bus => 0, addr => 14 },
4796         virtio5 => { bus => 0, addr => 15 },
4797         hostpci0 => { bus => 0, addr => 16 },
4798         hostpci1 => { bus => 0, addr => 17 },
4799         net0 => { bus => 0, addr => 18 },
4800         net1 => { bus => 0, addr => 19 },
4801         net2 => { bus => 0, addr => 20 },
4802         net3 => { bus => 0, addr => 21 },
4803         net4 => { bus => 0, addr => 22 },
4804         net5 => { bus => 0, addr => 23 },
4805         vga1 => { bus => 0, addr => 24 },
4806         vga2 => { bus => 0, addr => 25 },
4807         vga3 => { bus => 0, addr => 26 },
4808         hostpci2 => { bus => 0, addr => 27 },
4809         hostpci3 => { bus => 0, addr => 28 },
4810         #addr29 : usb-host (pve-usb.cfg)
4811         'pci.1' => { bus => 0, addr => 30 },
4812         'pci.2' => { bus => 0, addr => 31 },
4813         'net6' => { bus => 1, addr => 1 },
4814         'net7' => { bus => 1, addr => 2 },
4815         'net8' => { bus => 1, addr => 3 },
4816         'net9' => { bus => 1, addr => 4 },
4817         'net10' => { bus => 1, addr => 5 },
4818         'net11' => { bus => 1, addr => 6 },
4819         'net12' => { bus => 1, addr => 7 },
4820         'net13' => { bus => 1, addr => 8 },
4821         'net14' => { bus => 1, addr => 9 },
4822         'net15' => { bus => 1, addr => 10 },
4823         'net16' => { bus => 1, addr => 11 },
4824         'net17' => { bus => 1, addr => 12 },
4825         'net18' => { bus => 1, addr => 13 },
4826         'net19' => { bus => 1, addr => 14 },
4827         'net20' => { bus => 1, addr => 15 },
4828         'net21' => { bus => 1, addr => 16 },
4829         'net22' => { bus => 1, addr => 17 },
4830         'net23' => { bus => 1, addr => 18 },
4831         'net24' => { bus => 1, addr => 19 },
4832         'net25' => { bus => 1, addr => 20 },
4833         'net26' => { bus => 1, addr => 21 },
4834         'net27' => { bus => 1, addr => 22 },
4835         'net28' => { bus => 1, addr => 23 },
4836         'net29' => { bus => 1, addr => 24 },
4837         'net30' => { bus => 1, addr => 25 },
4838         'net31' => { bus => 1, addr => 26 },
4839         'virtio6' => { bus => 2, addr => 1 },
4840         'virtio7' => { bus => 2, addr => 2 },
4841         'virtio8' => { bus => 2, addr => 3 },
4842         'virtio9' => { bus => 2, addr => 4 },
4843         'virtio10' => { bus => 2, addr => 5 },
4844         'virtio11' => { bus => 2, addr => 6 },
4845         'virtio12' => { bus => 2, addr => 7 },
4846         'virtio13' => { bus => 2, addr => 8 },
4847         'virtio14' => { bus => 2, addr => 9 },
4848         'virtio15' => { bus => 2, addr => 10 },
4849         'virtioscsi0' => { bus => 3, addr => 1 },
4850         'virtioscsi1' => { bus => 3, addr => 2 },
4851         'virtioscsi2' => { bus => 3, addr => 3 },
4852         'virtioscsi3' => { bus => 3, addr => 4 },
4853         'virtioscsi4' => { bus => 3, addr => 5 },
4854         'virtioscsi5' => { bus => 3, addr => 6 },
4855         'virtioscsi6' => { bus => 3, addr => 7 },
4856         'virtioscsi7' => { bus => 3, addr => 8 },
4857         'virtioscsi8' => { bus => 3, addr => 9 },
4858         'virtioscsi9' => { bus => 3, addr => 10 },
4859         'virtioscsi10' => { bus => 3, addr => 11 },
4860         'virtioscsi11' => { bus => 3, addr => 12 },
4861         'virtioscsi12' => { bus => 3, addr => 13 },
4862         'virtioscsi13' => { bus => 3, addr => 14 },
4863         'virtioscsi14' => { bus => 3, addr => 15 },
4864         'virtioscsi15' => { bus => 3, addr => 16 },
4865         'virtioscsi16' => { bus => 3, addr => 17 },
4866         'virtioscsi17' => { bus => 3, addr => 18 },
4867         'virtioscsi18' => { bus => 3, addr => 19 },
4868         'virtioscsi19' => { bus => 3, addr => 20 },
4869         'virtioscsi20' => { bus => 3, addr => 21 },
4870         'virtioscsi21' => { bus => 3, addr => 22 },
4871         'virtioscsi22' => { bus => 3, addr => 23 },
4872         'virtioscsi23' => { bus => 3, addr => 24 },
4873         'virtioscsi24' => { bus => 3, addr => 25 },
4874         'virtioscsi25' => { bus => 3, addr => 26 },
4875         'virtioscsi26' => { bus => 3, addr => 27 },
4876         'virtioscsi27' => { bus => 3, addr => 28 },
4877         'virtioscsi28' => { bus => 3, addr => 29 },
4878         'virtioscsi29' => { bus => 3, addr => 30 },
4879         'virtioscsi30' => { bus => 3, addr => 31 },
4880
4881     };
4882
4883     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
4884            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
4885            my $bus = $devices->{$id}->{bus};
4886            $res = ",bus=pci.$bus,addr=$addr";
4887            $bridges->{$bus} = 1 if $bridges;
4888     }
4889     return $res;
4890
4891 }
4892
4893 sub print_pcie_addr {
4894     my ($id) = @_;
4895
4896     my $res = '';
4897     my $devices = {
4898         hostpci0 => { bus => "ich9-pcie-port-1", addr => 0 },
4899         hostpci1 => { bus => "ich9-pcie-port-2", addr => 0 },
4900         hostpci2 => { bus => "ich9-pcie-port-3", addr => 0 },
4901         hostpci3 => { bus => "ich9-pcie-port-4", addr => 0 },
4902     };
4903
4904     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
4905            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
4906            my $bus = $devices->{$id}->{bus};
4907            $res = ",bus=$bus,addr=$addr";
4908     }
4909     return $res;
4910
4911 }
4912
4913 # vzdump restore implementaion
4914
4915 sub tar_archive_read_firstfile {
4916     my $archive = shift;
4917
4918     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
4919
4920     # try to detect archive type first
4921     my $pid = open (TMP, "tar tf '$archive'|") ||
4922         die "unable to open file '$archive'\n";
4923     my $firstfile = <TMP>;
4924     kill 15, $pid;
4925     close TMP;
4926
4927     die "ERROR: archive contaions no data\n" if !$firstfile;
4928     chomp $firstfile;
4929
4930     return $firstfile;
4931 }
4932
4933 sub tar_restore_cleanup {
4934     my ($storecfg, $statfile) = @_;
4935
4936     print STDERR "starting cleanup\n";
4937
4938     if (my $fd = IO::File->new($statfile, "r")) {
4939         while (defined(my $line = <$fd>)) {
4940             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
4941                 my $volid = $2;
4942                 eval {
4943                     if ($volid =~ m|^/|) {
4944                         unlink $volid || die 'unlink failed\n';
4945                     } else {
4946                         PVE::Storage::vdisk_free($storecfg, $volid);
4947                     }
4948                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
4949                 };
4950                 print STDERR "unable to cleanup '$volid' - $@" if $@;
4951             } else {
4952                 print STDERR "unable to parse line in statfile - $line";
4953             }
4954         }
4955         $fd->close();
4956     }
4957 }
4958
4959 sub restore_archive {
4960     my ($archive, $vmid, $user, $opts) = @_;
4961
4962     my $format = $opts->{format};
4963     my $comp;
4964
4965     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
4966         $format = 'tar' if !$format;
4967         $comp = 'gzip';
4968     } elsif ($archive =~ m/\.tar$/) {
4969         $format = 'tar' if !$format;
4970     } elsif ($archive =~ m/.tar.lzo$/) {
4971         $format = 'tar' if !$format;
4972         $comp = 'lzop';
4973     } elsif ($archive =~ m/\.vma$/) {
4974         $format = 'vma' if !$format;
4975     } elsif ($archive =~ m/\.vma\.gz$/) {
4976         $format = 'vma' if !$format;
4977         $comp = 'gzip';
4978     } elsif ($archive =~ m/\.vma\.lzo$/) {
4979         $format = 'vma' if !$format;
4980         $comp = 'lzop';
4981     } else {
4982         $format = 'vma' if !$format; # default
4983     }
4984
4985     # try to detect archive format
4986     if ($format eq 'tar') {
4987         return restore_tar_archive($archive, $vmid, $user, $opts);
4988     } else {
4989         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
4990     }
4991 }
4992
4993 sub restore_update_config_line {
4994     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
4995
4996     return if $line =~ m/^\#qmdump\#/;
4997     return if $line =~ m/^\#vzdump\#/;
4998     return if $line =~ m/^lock:/;
4999     return if $line =~ m/^unused\d+:/;
5000     return if $line =~ m/^parent:/;
5001     return if $line =~ m/^template:/; # restored VM is never a template
5002
5003     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5004         # try to convert old 1.X settings
5005         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5006         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5007             my ($model, $macaddr) = split(/\=/, $devconfig);
5008             $macaddr = PVE::Tools::random_ether_addr() if !$macaddr || $unique;
5009             my $net = {
5010                 model => $model,
5011                 bridge => "vmbr$ind",
5012                 macaddr => $macaddr,
5013             };
5014             my $netstr = print_net($net);
5015
5016             print $outfd "net$cookie->{netcount}: $netstr\n";
5017             $cookie->{netcount}++;
5018         }
5019     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5020         my ($id, $netstr) = ($1, $2);
5021         my $net = parse_net($netstr);
5022         $net->{macaddr} = PVE::Tools::random_ether_addr() if $net->{macaddr};
5023         $netstr = print_net($net);
5024         print $outfd "$id: $netstr\n";
5025     } elsif ($line =~ m/^((ide|scsi|virtio|sata)\d+):\s*(\S+)\s*$/) {
5026         my $virtdev = $1;
5027         my $value = $3;
5028         if ($line =~ m/backup=no/) {
5029             print $outfd "#$line";
5030         } elsif ($virtdev && $map->{$virtdev}) {
5031             my $di = parse_drive($virtdev, $value);
5032             delete $di->{format}; # format can change on restore
5033             $di->{file} = $map->{$virtdev};
5034             $value = print_drive($vmid, $di);
5035             print $outfd "$virtdev: $value\n";
5036         } else {
5037             print $outfd $line;
5038         }
5039     } else {
5040         print $outfd $line;
5041     }
5042 }
5043
5044 sub scan_volids {
5045     my ($cfg, $vmid) = @_;
5046
5047     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5048
5049     my $volid_hash = {};
5050     foreach my $storeid (keys %$info) {
5051         foreach my $item (@{$info->{$storeid}}) {
5052             next if !($item->{volid} && $item->{size});
5053             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5054             $volid_hash->{$item->{volid}} = $item;
5055         }
5056     }
5057
5058     return $volid_hash;
5059 }
5060
5061 sub get_used_paths {
5062     my ($vmid, $storecfg, $conf, $scan_snapshots, $skip_drive) = @_;
5063
5064     my $used_path = {};
5065
5066     my $scan_config = sub {
5067         my ($cref, $snapname) = @_;
5068
5069         foreach my $key (keys %$cref) {
5070             my $value = $cref->{$key};
5071             if (valid_drivename($key)) {
5072                 next if $skip_drive && $key eq $skip_drive;
5073                 my $drive = parse_drive($key, $value);
5074                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5075                 if ($drive->{file} =~ m!^/!) {
5076                     $used_path->{$drive->{file}}++; # = 1;
5077                 } else {
5078                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5079                     next if !$storeid;
5080                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5081                     next if !$scfg;
5082                     my $path = PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5083                     $used_path->{$path}++; # = 1;
5084                 }
5085             }
5086         }
5087     };
5088
5089     &$scan_config($conf);
5090
5091     undef $skip_drive;
5092
5093     if ($scan_snapshots) {
5094         foreach my $snapname (keys %{$conf->{snapshots}}) {
5095             &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5096         }
5097     }
5098
5099     return $used_path;
5100 }
5101
5102 sub update_disksize {
5103     my ($vmid, $conf, $volid_hash) = @_;
5104
5105     my $changes;
5106
5107     my $used = {};
5108
5109     # Note: it is allowed to define multiple storages with same path (alias), so
5110     # we need to check both 'volid' and real 'path' (two different volid can point
5111     # to the same path).
5112
5113     my $usedpath = {};
5114
5115     # update size info
5116     foreach my $opt (keys %$conf) {
5117         if (valid_drivename($opt)) {
5118             my $drive = parse_drive($opt, $conf->{$opt});
5119             my $volid = $drive->{file};
5120             next if !$volid;
5121
5122             $used->{$volid} = 1;
5123             if ($volid_hash->{$volid} &&
5124                 (my $path = $volid_hash->{$volid}->{path})) {
5125                 $usedpath->{$path} = 1;
5126             }
5127
5128             next if drive_is_cdrom($drive);
5129             next if !$volid_hash->{$volid};
5130
5131             $drive->{size} = $volid_hash->{$volid}->{size};
5132             my $new = print_drive($vmid, $drive);
5133             if ($new ne $conf->{$opt}) {
5134                 $changes = 1;
5135                 $conf->{$opt} = $new;
5136             }
5137         }
5138     }
5139
5140     # remove 'unusedX' entry if volume is used
5141     foreach my $opt (keys %$conf) {
5142         next if $opt !~ m/^unused\d+$/;
5143         my $volid = $conf->{$opt};
5144         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5145         if ($used->{$volid} || ($path && $usedpath->{$path})) {
5146             $changes = 1;
5147             delete $conf->{$opt};
5148         }
5149     }
5150
5151     foreach my $volid (sort keys %$volid_hash) {
5152         next if $volid =~ m/vm-$vmid-state-/;
5153         next if $used->{$volid};
5154         my $path = $volid_hash->{$volid}->{path};
5155         next if !$path; # just to be sure
5156         next if $usedpath->{$path};
5157         $changes = 1;
5158         add_unused_volume($conf, $volid);
5159         $usedpath->{$path} = 1; # avoid to add more than once (aliases)
5160     }
5161
5162     return $changes;
5163 }
5164
5165 sub rescan {
5166     my ($vmid, $nolock) = @_;
5167
5168     my $cfg = PVE::Cluster::cfs_read_file("storage.cfg");
5169
5170     my $volid_hash = scan_volids($cfg, $vmid);
5171
5172     my $updatefn =  sub {
5173         my ($vmid) = @_;
5174
5175         my $conf = load_config($vmid);
5176
5177         check_lock($conf);
5178
5179         my $vm_volids = {};
5180         foreach my $volid (keys %$volid_hash) {
5181             my $info = $volid_hash->{$volid};
5182             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5183         }
5184
5185         my $changes = update_disksize($vmid, $conf, $vm_volids);
5186
5187         update_config_nolock($vmid, $conf, 1) if $changes;
5188     };
5189
5190     if (defined($vmid)) {
5191         if ($nolock) {
5192             &$updatefn($vmid);
5193         } else {
5194             lock_config($vmid, $updatefn, $vmid);
5195         }
5196     } else {
5197         my $vmlist = config_list();
5198         foreach my $vmid (keys %$vmlist) {
5199             if ($nolock) {
5200                 &$updatefn($vmid);
5201             } else {
5202                 lock_config($vmid, $updatefn, $vmid);
5203             }
5204         }
5205     }
5206 }
5207
5208 sub restore_vma_archive {
5209     my ($archive, $vmid, $user, $opts, $comp) = @_;
5210
5211     my $input = $archive eq '-' ? "<&STDIN" : undef;
5212     my $readfrom = $archive;
5213
5214     my $uncomp = '';
5215     if ($comp) {
5216         $readfrom = '-';
5217         my $qarchive = PVE::Tools::shellquote($archive);
5218         if ($comp eq 'gzip') {
5219             $uncomp = "zcat $qarchive|";
5220         } elsif ($comp eq 'lzop') {
5221             $uncomp = "lzop -d -c $qarchive|";
5222         } else {
5223             die "unknown compression method '$comp'\n";
5224         }
5225
5226     }
5227
5228     my $tmpdir = "/var/tmp/vzdumptmp$$";
5229     rmtree $tmpdir;
5230
5231     # disable interrupts (always do cleanups)
5232     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5233         warn "got interrupt - ignored\n";
5234     };
5235
5236     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5237     POSIX::mkfifo($mapfifo, 0600);
5238     my $fifofh;
5239
5240     my $openfifo = sub {
5241         open($fifofh, '>', $mapfifo) || die $!;
5242     };
5243
5244     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5245
5246     my $oldtimeout;
5247     my $timeout = 5;
5248
5249     my $devinfo = {};
5250
5251     my $rpcenv = PVE::RPCEnvironment::get();
5252
5253     my $conffile = config_file($vmid);
5254     my $tmpfn = "$conffile.$$.tmp";
5255
5256     # Note: $oldconf is undef if VM does not exists
5257     my $oldconf = PVE::Cluster::cfs_read_file(cfs_config_path($vmid));
5258
5259     my $print_devmap = sub {
5260         my $virtdev_hash = {};
5261
5262         my $cfgfn = "$tmpdir/qemu-server.conf";
5263
5264         # we can read the config - that is already extracted
5265         my $fh = IO::File->new($cfgfn, "r") ||
5266             "unable to read qemu-server.conf - $!\n";
5267
5268         while (defined(my $line = <$fh>)) {
5269             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5270                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5271                 die "archive does not contain data for drive '$virtdev'\n"
5272                     if !$devinfo->{$devname};
5273                 if (defined($opts->{storage})) {
5274                     $storeid = $opts->{storage} || 'local';
5275                 } elsif (!$storeid) {
5276                     $storeid = 'local';
5277                 }
5278                 $format = 'raw' if !$format;
5279                 $devinfo->{$devname}->{devname} = $devname;
5280                 $devinfo->{$devname}->{virtdev} = $virtdev;
5281                 $devinfo->{$devname}->{format} = $format;
5282                 $devinfo->{$devname}->{storeid} = $storeid;
5283
5284                 # check permission on storage
5285                 my $pool = $opts->{pool}; # todo: do we need that?
5286                 if ($user ne 'root@pam') {
5287                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5288                 }
5289
5290                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5291             }
5292         }
5293
5294         foreach my $devname (keys %$devinfo) {
5295             die "found no device mapping information for device '$devname'\n"
5296                 if !$devinfo->{$devname}->{virtdev};
5297         }
5298
5299         my $cfg = cfs_read_file('storage.cfg');
5300
5301         # create empty/temp config
5302         if ($oldconf) {
5303             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5304             foreach_drive($oldconf, sub {
5305                 my ($ds, $drive) = @_;
5306
5307                 return if drive_is_cdrom($drive);
5308
5309                 my $volid = $drive->{file};
5310
5311                 return if !$volid || $volid =~ m|^/|;
5312
5313                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5314                 return if !$path || !$owner || ($owner != $vmid);
5315
5316                 # Note: only delete disk we want to restore
5317                 # other volumes will become unused
5318                 if ($virtdev_hash->{$ds}) {
5319                     PVE::Storage::vdisk_free($cfg, $volid);
5320                 }
5321             });
5322         }
5323
5324         my $map = {};
5325         foreach my $virtdev (sort keys %$virtdev_hash) {
5326             my $d = $virtdev_hash->{$virtdev};
5327             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5328             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5329
5330             # test if requested format is supported
5331             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5332             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5333             $d->{format} = $defFormat if !$supported;
5334
5335             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5336                                                   $d->{format}, undef, $alloc_size);
5337             print STDERR "new volume ID is '$volid'\n";
5338             $d->{volid} = $volid;
5339             my $path = PVE::Storage::path($cfg, $volid);
5340
5341             PVE::Storage::activate_volumes($cfg,[$volid]);
5342
5343             my $write_zeros = 1;
5344             # fixme: what other storages types initialize volumes with zero?
5345             if ($scfg->{type} eq 'dir' || $scfg->{type} eq 'nfs' || $scfg->{type} eq 'glusterfs' ||
5346                 $scfg->{type} eq 'sheepdog' || $scfg->{type} eq 'rbd') {
5347                 $write_zeros = 0;
5348             }
5349
5350             print $fifofh "${write_zeros}:$d->{devname}=$path\n";
5351
5352             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5353             $map->{$virtdev} = $volid;
5354         }
5355
5356         $fh->seek(0, 0) || die "seek failed - $!\n";
5357
5358         my $outfd = new IO::File ($tmpfn, "w") ||
5359             die "unable to write config for VM $vmid\n";
5360
5361         my $cookie = { netcount => 0 };
5362         while (defined(my $line = <$fh>)) {
5363             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5364         }
5365
5366         $fh->close();
5367         $outfd->close();
5368     };
5369
5370     eval {
5371         # enable interrupts
5372         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5373             die "interrupted by signal\n";
5374         };
5375         local $SIG{ALRM} = sub { die "got timeout\n"; };
5376
5377         $oldtimeout = alarm($timeout);
5378
5379         my $parser = sub {
5380             my $line = shift;
5381
5382             print "$line\n";
5383
5384             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5385                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5386                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5387             } elsif ($line =~ m/^CTIME: /) {
5388                 # we correctly received the vma config, so we can disable
5389                 # the timeout now for disk allocation (set to 10 minutes, so
5390                 # that we always timeout if something goes wrong)
5391                 alarm(600);
5392                 &$print_devmap();
5393                 print $fifofh "done\n";
5394                 my $tmp = $oldtimeout || 0;
5395                 $oldtimeout = undef;
5396                 alarm($tmp);
5397                 close($fifofh);
5398             }
5399         };
5400
5401         print "restore vma archive: $cmd\n";
5402         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5403     };
5404     my $err = $@;
5405
5406     alarm($oldtimeout) if $oldtimeout;
5407
5408     my $vollist = [];
5409     foreach my $devname (keys %$devinfo) {
5410         my $volid = $devinfo->{$devname}->{volid};
5411         push @$vollist, $volid if $volid;
5412     }
5413
5414     my $cfg = cfs_read_file('storage.cfg');
5415     PVE::Storage::deactivate_volumes($cfg, $vollist);
5416
5417     unlink $mapfifo;
5418
5419     if ($err) {
5420         rmtree $tmpdir;
5421         unlink $tmpfn;
5422
5423         foreach my $devname (keys %$devinfo) {
5424             my $volid = $devinfo->{$devname}->{volid};
5425             next if !$volid;
5426             eval {
5427                 if ($volid =~ m|^/|) {
5428                     unlink $volid || die 'unlink failed\n';
5429                 } else {
5430                     PVE::Storage::vdisk_free($cfg, $volid);
5431                 }
5432                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5433             };
5434             print STDERR "unable to cleanup '$volid' - $@" if $@;
5435         }
5436         die $err;
5437     }
5438
5439     rmtree $tmpdir;
5440
5441     rename($tmpfn, $conffile) ||
5442         die "unable to commit configuration file '$conffile'\n";
5443
5444     PVE::Cluster::cfs_update(); # make sure we read new file
5445
5446     eval { rescan($vmid, 1); };
5447     warn $@ if $@;
5448 }
5449
5450 sub restore_tar_archive {
5451     my ($archive, $vmid, $user, $opts) = @_;
5452
5453     if ($archive ne '-') {
5454         my $firstfile = tar_archive_read_firstfile($archive);
5455         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5456             if $firstfile ne 'qemu-server.conf';
5457     }
5458
5459     my $storecfg = cfs_read_file('storage.cfg');
5460
5461     # destroy existing data - keep empty config
5462     my $vmcfgfn = config_file($vmid);
5463     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5464
5465     my $tocmd = "/usr/lib/qemu-server/qmextract";
5466
5467     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5468     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5469     $tocmd .= ' --prealloc' if $opts->{prealloc};
5470     $tocmd .= ' --info' if $opts->{info};
5471
5472     # tar option "xf" does not autodetect compression when read from STDIN,
5473     # so we pipe to zcat
5474     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5475         PVE::Tools::shellquote("--to-command=$tocmd");
5476
5477     my $tmpdir = "/var/tmp/vzdumptmp$$";
5478     mkpath $tmpdir;
5479
5480     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5481     local $ENV{VZDUMP_VMID} = $vmid;
5482     local $ENV{VZDUMP_USER} = $user;
5483
5484     my $conffile = config_file($vmid);
5485     my $tmpfn = "$conffile.$$.tmp";
5486
5487     # disable interrupts (always do cleanups)
5488     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5489         print STDERR "got interrupt - ignored\n";
5490     };
5491
5492     eval {
5493         # enable interrupts
5494         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5495             die "interrupted by signal\n";
5496         };
5497
5498         if ($archive eq '-') {
5499             print "extracting archive from STDIN\n";
5500             run_command($cmd, input => "<&STDIN");
5501         } else {
5502             print "extracting archive '$archive'\n";
5503             run_command($cmd);
5504         }
5505
5506         return if $opts->{info};
5507
5508         # read new mapping
5509         my $map = {};
5510         my $statfile = "$tmpdir/qmrestore.stat";
5511         if (my $fd = IO::File->new($statfile, "r")) {
5512             while (defined (my $line = <$fd>)) {
5513                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5514                     $map->{$1} = $2 if $1;
5515                 } else {
5516                     print STDERR "unable to parse line in statfile - $line\n";
5517                 }
5518             }
5519             $fd->close();
5520         }
5521
5522         my $confsrc = "$tmpdir/qemu-server.conf";
5523
5524         my $srcfd = new IO::File($confsrc, "r") ||
5525             die "unable to open file '$confsrc'\n";
5526
5527         my $outfd = new IO::File ($tmpfn, "w") ||
5528             die "unable to write config for VM $vmid\n";
5529
5530         my $cookie = { netcount => 0 };
5531         while (defined (my $line = <$srcfd>)) {
5532             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5533         }
5534
5535         $srcfd->close();
5536         $outfd->close();
5537     };
5538     my $err = $@;
5539
5540     if ($err) {
5541
5542         unlink $tmpfn;
5543
5544         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
5545
5546         die $err;
5547     }
5548
5549     rmtree $tmpdir;
5550
5551     rename $tmpfn, $conffile ||
5552         die "unable to commit configuration file '$conffile'\n";
5553
5554     PVE::Cluster::cfs_update(); # make sure we read new file
5555
5556     eval { rescan($vmid, 1); };
5557     warn $@ if $@;
5558 };
5559
5560
5561 # Internal snapshots
5562
5563 # NOTE: Snapshot create/delete involves several non-atomic
5564 # action, and can take a long time.
5565 # So we try to avoid locking the file and use 'lock' variable
5566 # inside the config file instead.
5567
5568 my $snapshot_copy_config = sub {
5569     my ($source, $dest) = @_;
5570
5571     foreach my $k (keys %$source) {
5572         next if $k eq 'snapshots';
5573         next if $k eq 'snapstate';
5574         next if $k eq 'snaptime';
5575         next if $k eq 'vmstate';
5576         next if $k eq 'lock';
5577         next if $k eq 'digest';
5578         next if $k eq 'description';
5579         next if $k =~ m/^unused\d+$/;
5580
5581         $dest->{$k} = $source->{$k};
5582     }
5583 };
5584
5585 my $snapshot_apply_config = sub {
5586     my ($conf, $snap) = @_;
5587
5588     # copy snapshot list
5589     my $newconf = {
5590         snapshots => $conf->{snapshots},
5591     };
5592
5593     # keep description and list of unused disks
5594     foreach my $k (keys %$conf) {
5595         next if !($k =~ m/^unused\d+$/ || $k eq 'description');
5596         $newconf->{$k} = $conf->{$k};
5597     }
5598
5599     &$snapshot_copy_config($snap, $newconf);
5600
5601     return $newconf;
5602 };
5603
5604 sub foreach_writable_storage {
5605     my ($conf, $func) = @_;
5606
5607     my $sidhash = {};
5608
5609     foreach my $ds (keys %$conf) {
5610         next if !valid_drivename($ds);
5611
5612         my $drive = parse_drive($ds, $conf->{$ds});
5613         next if !$drive;
5614         next if drive_is_cdrom($drive);
5615
5616         my $volid = $drive->{file};
5617
5618         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5619         $sidhash->{$sid} = $sid if $sid;
5620     }
5621
5622     foreach my $sid (sort keys %$sidhash) {
5623         &$func($sid);
5624     }
5625 }
5626
5627 my $alloc_vmstate_volid = sub {
5628     my ($storecfg, $vmid, $conf, $snapname) = @_;
5629
5630     # Note: we try to be smart when selecting a $target storage
5631
5632     my $target;
5633
5634     # search shared storage first
5635     foreach_writable_storage($conf, sub {
5636         my ($sid) = @_;
5637         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
5638         return if !$scfg->{shared};
5639
5640         $target = $sid if !$target || $scfg->{path}; # prefer file based storage
5641     });
5642
5643     if (!$target) {
5644         # now search local storage
5645         foreach_writable_storage($conf, sub {
5646             my ($sid) = @_;
5647             my $scfg = PVE::Storage::storage_config($storecfg, $sid);
5648             return if $scfg->{shared};
5649
5650             $target = $sid if !$target || $scfg->{path}; # prefer file based storage;
5651         });
5652     }
5653
5654     $target = 'local' if !$target;
5655
5656     my $driver_state_size = 500; # assume 32MB is enough to safe all driver state;
5657     # we abort live save after $conf->{memory}, so we need at max twice that space
5658     my $size = $conf->{memory}*2 + $driver_state_size;
5659
5660     my $name = "vm-$vmid-state-$snapname";
5661     my $scfg = PVE::Storage::storage_config($storecfg, $target);
5662     $name .= ".raw" if $scfg->{path}; # add filename extension for file base storage
5663     my $volid = PVE::Storage::vdisk_alloc($storecfg, $target, $vmid, 'raw', $name, $size*1024);
5664
5665     return $volid;
5666 };
5667
5668 my $snapshot_prepare = sub {
5669     my ($vmid, $snapname, $save_vmstate, $comment) = @_;
5670
5671     my $snap;
5672
5673     my $updatefn =  sub {
5674
5675         my $conf = load_config($vmid);
5676
5677         die "you can't take a snapshot if it's a template\n"
5678             if is_template($conf);
5679
5680         check_lock($conf);
5681
5682         $conf->{lock} = 'snapshot';
5683
5684         die "snapshot name '$snapname' already used\n"
5685             if defined($conf->{snapshots}->{$snapname});
5686
5687         my $storecfg = PVE::Storage::config();
5688         die "snapshot feature is not available" if !has_feature('snapshot', $conf, $storecfg);
5689
5690         $snap = $conf->{snapshots}->{$snapname} = {};
5691
5692         if ($save_vmstate && check_running($vmid)) {
5693             $snap->{vmstate} = &$alloc_vmstate_volid($storecfg, $vmid, $conf, $snapname);
5694         }
5695
5696         &$snapshot_copy_config($conf, $snap);
5697
5698         $snap->{snapstate} = "prepare";
5699         $snap->{snaptime} = time();
5700         $snap->{description} = $comment if $comment;
5701
5702         # always overwrite machine if we save vmstate. This makes sure we
5703         # can restore it later using correct machine type
5704         $snap->{machine} = get_current_qemu_machine($vmid) if $snap->{vmstate};
5705
5706         update_config_nolock($vmid, $conf, 1);
5707     };
5708
5709     lock_config($vmid, $updatefn);
5710
5711     return $snap;
5712 };
5713
5714 my $snapshot_commit = sub {
5715     my ($vmid, $snapname) = @_;
5716
5717     my $updatefn = sub {
5718
5719         my $conf = load_config($vmid);
5720
5721         die "missing snapshot lock\n"
5722             if !($conf->{lock} && $conf->{lock} eq 'snapshot');
5723
5724         my $has_machine_config = defined($conf->{machine});
5725
5726         my $snap = $conf->{snapshots}->{$snapname};
5727
5728         die "snapshot '$snapname' does not exist\n" if !defined($snap);
5729
5730         die "wrong snapshot state\n"
5731             if !($snap->{snapstate} && $snap->{snapstate} eq "prepare");
5732
5733         delete $snap->{snapstate};
5734         delete $conf->{lock};
5735
5736         my $newconf = &$snapshot_apply_config($conf, $snap);
5737
5738         delete $newconf->{machine} if !$has_machine_config;
5739
5740         $newconf->{parent} = $snapname;
5741
5742         update_config_nolock($vmid, $newconf, 1);
5743     };
5744
5745     lock_config($vmid, $updatefn);
5746 };
5747
5748 sub snapshot_rollback {
5749     my ($vmid, $snapname) = @_;
5750
5751     my $prepare = 1;
5752
5753     my $storecfg = PVE::Storage::config();
5754
5755     my $conf = load_config($vmid);
5756
5757     my $get_snapshot_config = sub {
5758
5759         die "you can't rollback if vm is a template\n" if is_template($conf);
5760
5761         my $res = $conf->{snapshots}->{$snapname};
5762
5763         die "snapshot '$snapname' does not exist\n" if !defined($res);
5764
5765         return $res;
5766     };
5767
5768     my $snap = &$get_snapshot_config();
5769
5770     foreach_drive($snap, sub {
5771         my ($ds, $drive) = @_;
5772
5773         return if drive_is_cdrom($drive);
5774
5775         my $volid = $drive->{file};
5776
5777         PVE::Storage::volume_rollback_is_possible($storecfg, $volid, $snapname);
5778     });
5779
5780     my $updatefn = sub {
5781
5782         $conf = load_config($vmid);
5783
5784         $snap = &$get_snapshot_config();
5785
5786         die "unable to rollback to incomplete snapshot (snapstate = $snap->{snapstate})\n"
5787             if $snap->{snapstate};
5788
5789         if ($prepare) {
5790             check_lock($conf);
5791             vm_stop($storecfg, $vmid, undef, undef, 5, undef, undef);
5792         }
5793
5794         die "unable to rollback vm $vmid: vm is running\n"
5795             if check_running($vmid);
5796
5797         if ($prepare) {
5798             $conf->{lock} = 'rollback';
5799         } else {
5800             die "got wrong lock\n" if !($conf->{lock} && $conf->{lock} eq 'rollback');
5801             delete $conf->{lock};
5802         }
5803
5804         my $forcemachine;
5805
5806         if (!$prepare) {
5807             my $has_machine_config = defined($conf->{machine});
5808
5809             # copy snapshot config to current config
5810             $conf = &$snapshot_apply_config($conf, $snap);
5811             $conf->{parent} = $snapname;
5812
5813             # Note: old code did not store 'machine', so we try to be smart
5814             # and guess the snapshot was generated with kvm 1.4 (pc-i440fx-1.4).
5815             $forcemachine = $conf->{machine} || 'pc-i440fx-1.4';
5816             # we remove the 'machine' configuration if not explicitly specified
5817             # in the original config.
5818             delete $conf->{machine} if $snap->{vmstate} && !$has_machine_config;
5819         }
5820
5821         update_config_nolock($vmid, $conf, 1);
5822
5823         if (!$prepare && $snap->{vmstate}) {
5824             my $statefile = PVE::Storage::path($storecfg, $snap->{vmstate});
5825             vm_start($storecfg, $vmid, $statefile, undef, undef, undef, $forcemachine);
5826         }
5827     };
5828
5829     lock_config($vmid, $updatefn);
5830
5831     foreach_drive($snap, sub {
5832         my ($ds, $drive) = @_;
5833
5834         return if drive_is_cdrom($drive);
5835
5836         my $volid = $drive->{file};
5837         my $device = "drive-$ds";
5838
5839         PVE::Storage::volume_snapshot_rollback($storecfg, $volid, $snapname);
5840     });
5841
5842     $prepare = 0;
5843     lock_config($vmid, $updatefn);
5844 }
5845
5846 my $savevm_wait = sub {
5847     my ($vmid) = @_;
5848
5849     for(;;) {
5850         my $stat = vm_mon_cmd_nocheck($vmid, "query-savevm");
5851         if (!$stat->{status}) {
5852             die "savevm not active\n";
5853         } elsif ($stat->{status} eq 'active') {
5854             sleep(1);
5855             next;
5856         } elsif ($stat->{status} eq 'completed') {
5857             last;
5858         } else {
5859             die "query-savevm returned status '$stat->{status}'\n";
5860         }
5861     }
5862 };
5863
5864 sub do_snapshots_with_qemu {
5865     my ($storecfg, $volid) = @_;
5866
5867     my $storage_name = PVE::Storage::parse_volume_id($volid);
5868
5869     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
5870         && !$storecfg->{ids}->{$storage_name}->{krbd}){
5871         return 1;
5872     }
5873
5874     if ($volid =~ m/\.(qcow2|qed)$/){
5875         return 1;
5876     }
5877
5878     return undef;
5879 }
5880
5881 sub snapshot_create {
5882     my ($vmid, $snapname, $save_vmstate, $comment) = @_;
5883
5884     my $snap = &$snapshot_prepare($vmid, $snapname, $save_vmstate, $comment);
5885
5886     $save_vmstate = 0 if !$snap->{vmstate}; # vm is not running
5887
5888     my $config = load_config($vmid);
5889
5890     my $running = check_running($vmid);
5891
5892     my $freezefs = $running && $config->{agent};
5893     $freezefs = 0 if $snap->{vmstate}; # not needed if we save RAM
5894
5895     my $drivehash = {};
5896
5897     if ($freezefs) {
5898         eval { vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
5899         warn "guest-fsfreeze-freeze problems - $@" if $@;
5900     }
5901
5902     eval {
5903         # create internal snapshots of all drives
5904
5905         my $storecfg = PVE::Storage::config();
5906
5907         if ($running) {
5908             if ($snap->{vmstate}) {
5909                 my $path = PVE::Storage::path($storecfg, $snap->{vmstate});
5910                 vm_mon_cmd($vmid, "savevm-start", statefile => $path);
5911                 &$savevm_wait($vmid);
5912             } else {
5913                 vm_mon_cmd($vmid, "savevm-start");
5914             }
5915         };
5916
5917         foreach_drive($snap, sub {
5918             my ($ds, $drive) = @_;
5919
5920             return if drive_is_cdrom($drive);
5921
5922             my $volid = $drive->{file};
5923             my $device = "drive-$ds";
5924
5925             qemu_volume_snapshot($vmid, $device, $storecfg, $volid, $snapname);
5926             $drivehash->{$ds} = 1;
5927        });
5928     };
5929     my $err = $@;
5930
5931     if ($running) {
5932         eval { vm_mon_cmd($vmid, "savevm-end")  };
5933         warn $@ if $@;
5934
5935         if ($freezefs) {
5936             eval { vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
5937             warn "guest-fsfreeze-thaw problems - $@" if $@;
5938         }
5939
5940         # savevm-end is async, we need to wait
5941         for (;;) {
5942             my $stat = vm_mon_cmd_nocheck($vmid, "query-savevm");
5943             if (!$stat->{bytes}) {
5944                 last;
5945             } else {
5946                 print "savevm not yet finished\n";
5947                 sleep(1);
5948                 next;
5949             }
5950         }
5951     }
5952
5953     if ($err) {
5954         warn "snapshot create failed: starting cleanup\n";
5955         eval { snapshot_delete($vmid, $snapname, 0, $drivehash); };
5956         warn $@ if $@;
5957         die $err;
5958     }
5959
5960     &$snapshot_commit($vmid, $snapname);
5961 }
5962
5963 # Note: $drivehash is only set when called from snapshot_create.
5964 sub snapshot_delete {
5965     my ($vmid, $snapname, $force, $drivehash) = @_;
5966
5967     my $prepare = 1;
5968
5969     my $snap;
5970     my $unused = [];
5971
5972     my $unlink_parent = sub {
5973         my ($confref, $new_parent) = @_;
5974
5975         if ($confref->{parent} && $confref->{parent} eq $snapname) {
5976             if ($new_parent) {
5977                 $confref->{parent} = $new_parent;
5978             } else {
5979                 delete $confref->{parent};
5980             }
5981         }
5982     };
5983
5984     my $updatefn =  sub {
5985         my ($remove_drive) = @_;
5986
5987         my $conf = load_config($vmid);
5988
5989         if (!$drivehash) {
5990             check_lock($conf);
5991             die "you can't delete a snapshot if vm is a template\n"
5992                 if is_template($conf);
5993         }
5994
5995         $snap = $conf->{snapshots}->{$snapname};
5996
5997         die "snapshot '$snapname' does not exist\n" if !defined($snap);
5998
5999         # remove parent refs
6000         if (!$prepare) {
6001             &$unlink_parent($conf, $snap->{parent});
6002             foreach my $sn (keys %{$conf->{snapshots}}) {
6003                 next if $sn eq $snapname;
6004                 &$unlink_parent($conf->{snapshots}->{$sn}, $snap->{parent});
6005             }
6006         }
6007
6008         if ($remove_drive) {
6009             if ($remove_drive eq 'vmstate') {
6010                 delete $snap->{$remove_drive};
6011             } else {
6012                 my $drive = parse_drive($remove_drive, $snap->{$remove_drive});
6013                 my $volid = $drive->{file};
6014                 delete $snap->{$remove_drive};
6015                 add_unused_volume($conf, $volid);
6016             }
6017         }
6018
6019         if ($prepare) {
6020             $snap->{snapstate} = 'delete';
6021         } else {
6022             delete $conf->{snapshots}->{$snapname};
6023             delete $conf->{lock} if $drivehash;
6024             foreach my $volid (@$unused) {
6025                 add_unused_volume($conf, $volid);
6026             }
6027         }
6028
6029         update_config_nolock($vmid, $conf, 1);
6030     };
6031
6032     lock_config($vmid, $updatefn);
6033
6034     # now remove vmstate file
6035
6036     my $storecfg = PVE::Storage::config();
6037
6038     if ($snap->{vmstate}) {
6039         eval {  PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
6040         if (my $err = $@) {
6041             die $err if !$force;
6042             warn $err;
6043         }
6044         # save changes (remove vmstate from snapshot)
6045         lock_config($vmid, $updatefn, 'vmstate') if !$force;
6046     };
6047
6048     # now remove all internal snapshots
6049     foreach_drive($snap, sub {
6050         my ($ds, $drive) = @_;
6051
6052         return if drive_is_cdrom($drive);
6053
6054         my $volid = $drive->{file};
6055         my $device = "drive-$ds";
6056
6057         if (!$drivehash || $drivehash->{$ds}) {
6058             eval { qemu_volume_snapshot_delete($vmid, $device, $storecfg, $volid, $snapname); };
6059             if (my $err = $@) {
6060                 die $err if !$force;
6061                 warn $err;
6062             }
6063         }
6064
6065         # save changes (remove drive fron snapshot)
6066         lock_config($vmid, $updatefn, $ds) if !$force;
6067         push @$unused, $volid;
6068     });
6069
6070     # now cleanup config
6071     $prepare = 0;
6072     lock_config($vmid, $updatefn);
6073 }
6074
6075 sub has_feature {
6076     my ($feature, $conf, $storecfg, $snapname, $running) = @_;
6077
6078     my $err;
6079     foreach_drive($conf, sub {
6080         my ($ds, $drive) = @_;
6081
6082         return if drive_is_cdrom($drive);
6083         my $volid = $drive->{file};
6084         $err = 1 if !PVE::Storage::volume_has_feature($storecfg, $feature, $volid, $snapname, $running);
6085     });
6086
6087     return $err ? 0 : 1;
6088 }
6089
6090 sub template_create {
6091     my ($vmid, $conf, $disk) = @_;
6092
6093     my $storecfg = PVE::Storage::config();
6094
6095     foreach_drive($conf, sub {
6096         my ($ds, $drive) = @_;
6097
6098         return if drive_is_cdrom($drive);
6099         return if $disk && $ds ne $disk;
6100
6101         my $volid = $drive->{file};
6102         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6103
6104         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6105         $drive->{file} = $voliddst;
6106         $conf->{$ds} = print_drive($vmid, $drive);
6107         update_config_nolock($vmid, $conf, 1);
6108     });
6109 }
6110
6111 sub is_template {
6112     my ($conf) = @_;
6113
6114     return 1 if defined $conf->{template} && $conf->{template} == 1;
6115 }
6116
6117 sub qemu_img_convert {
6118     my ($src_volid, $dst_volid, $size, $snapname) = @_;
6119
6120     my $storecfg = PVE::Storage::config();
6121     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6122     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6123
6124     if ($src_storeid && $dst_storeid) {
6125         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6126         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6127
6128         my $src_format = qemu_img_format($src_scfg, $src_volname);
6129         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6130
6131         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6132         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6133
6134         my $cmd = [];
6135         push @$cmd, '/usr/bin/qemu-img', 'convert', '-t', 'writeback', '-p', '-n';
6136         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
6137         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path, $dst_path;
6138
6139         my $parser = sub {
6140             my $line = shift;
6141             if($line =~ m/\((\S+)\/100\%\)/){
6142                 my $percent = $1;
6143                 my $transferred = int($size * $percent / 100);
6144                 my $remaining = $size - $transferred;
6145
6146                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6147             }
6148
6149         };
6150
6151         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6152         my $err = $@;
6153         die "copy failed: $err" if $err;
6154     }
6155 }
6156
6157 sub qemu_img_format {
6158     my ($scfg, $volname) = @_;
6159
6160     if ($scfg->{path} && $volname =~ m/\.(raw|cow|qcow|qcow2|qed|vmdk|cloop)$/) {
6161         return $1;
6162     } else {
6163         return "raw";
6164     }
6165 }
6166
6167 sub qemu_drive_mirror {
6168     my ($vmid, $drive, $dst_volid, $vmiddst) = @_;
6169
6170     my $storecfg = PVE::Storage::config();
6171     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6172
6173     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6174
6175     my $format = qemu_img_format($dst_scfg, $dst_volname);
6176
6177     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6178
6179     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $dst_path };
6180     $opts->{format} = $format if $format;
6181
6182     print "drive mirror is starting (scanning bitmap) : this step can take some minutes/hours, depend of disk size and storage speed\n";
6183
6184     eval {
6185     vm_mon_cmd($vmid, "drive-mirror", %$opts);
6186         while (1) {
6187             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6188             my $stat = @$stats[0];
6189             die "mirroring job seem to have die. Maybe do you have bad sectors?" if !$stat;
6190             die "error job is not mirroring" if $stat->{type} ne "mirror";
6191
6192             my $busy = $stat->{busy};
6193             my $ready = $stat->{ready};
6194
6195             if (my $total = $stat->{len}) {
6196                 my $transferred = $stat->{offset} || 0;
6197                 my $remaining = $total - $transferred;
6198                 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6199
6200                 print "transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6201             }
6202
6203
6204             if ($stat->{ready} eq 'true') {
6205
6206                 last if $vmiddst != $vmid;
6207
6208                 # try to switch the disk if source and destination are on the same guest
6209                 eval { vm_mon_cmd($vmid, "block-job-complete", device => "drive-$drive") };
6210                 last if !$@;
6211                 die $@ if $@ !~ m/cannot be completed/;
6212             }
6213             sleep 1;
6214         }
6215
6216
6217     };
6218     my $err = $@;
6219
6220     my $cancel_job = sub {
6221         vm_mon_cmd($vmid, "block-job-cancel", device => "drive-$drive");
6222         while (1) {
6223             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6224             my $stat = @$stats[0];
6225             last if !$stat;
6226             sleep 1;
6227         }
6228     };
6229
6230     if ($err) {
6231         eval { &$cancel_job(); };
6232         die "mirroring error: $err";
6233     }
6234
6235     if ($vmiddst != $vmid) {
6236         # if we clone a disk for a new target vm, we don't switch the disk
6237         &$cancel_job(); # so we call block-job-cancel
6238     }
6239 }
6240
6241 sub clone_disk {
6242     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6243         $newvmid, $storage, $format, $full, $newvollist) = @_;
6244
6245     my $newvolid;
6246
6247     if (!$full) {
6248         print "create linked clone of drive $drivename ($drive->{file})\n";
6249         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6250         push @$newvollist, $newvolid;
6251     } else {
6252         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6253         $storeid = $storage if $storage;
6254
6255         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6256         if (!$format) {
6257             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6258             $format = qemu_img_format($scfg, $volname);
6259         }
6260
6261         # test if requested format is supported - else use default
6262         my $supported = grep { $_ eq $format } @$validFormats;
6263         $format = $defFormat if !$supported;
6264
6265         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6266
6267         print "create full clone of drive $drivename ($drive->{file})\n";
6268         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $format, undef, ($size/1024));
6269         push @$newvollist, $newvolid;
6270
6271         PVE::Storage::activate_volumes($storecfg, $newvollist);
6272
6273         if (!$running || $snapname) {
6274             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname);
6275         } else {
6276             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid);
6277         }
6278     }
6279
6280     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6281
6282     my $disk = $drive;
6283     $disk->{format} = undef;
6284     $disk->{file} = $newvolid;
6285     $disk->{size} = $size;
6286
6287     return $disk;
6288 }
6289
6290 # this only works if VM is running
6291 sub get_current_qemu_machine {
6292     my ($vmid) = @_;
6293
6294     my $cmd = { execute => 'query-machines', arguments => {} };
6295     my $res = vm_qmp_command($vmid, $cmd);
6296
6297     my ($current, $default);
6298     foreach my $e (@$res) {
6299         $default = $e->{name} if $e->{'is-default'};
6300         $current = $e->{name} if $e->{'is-current'};
6301     }
6302
6303     # fallback to the default machine if current is not supported by qemu
6304     return $current || $default || 'pc';
6305 }
6306
6307 sub qemu_machine_feature_enabled {
6308     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6309
6310     my $current_major;
6311     my $current_minor;
6312
6313     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6314
6315         $current_major = $3;
6316         $current_minor = $4;
6317
6318     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6319
6320         $current_major = $1;
6321         $current_minor = $2;
6322     }
6323
6324     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
6325
6326
6327 }
6328
6329 sub lspci {
6330
6331     my $devices = {};
6332
6333     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6334             my (undef, $id, $function) = @_;
6335             my $res = { id => $id, function => $function};
6336             push @{$devices->{$id}}, $res;
6337     });
6338
6339     return $devices;
6340 }
6341
6342 sub vm_iothreads_list {
6343     my ($vmid) = @_;
6344
6345     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6346
6347     my $iothreads = {};
6348     foreach my $iothread (@$res) {
6349         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6350     }
6351
6352     return $iothreads;
6353 }
6354
6355 sub scsihw_infos {
6356     my ($conf, $drive) = @_;
6357
6358     my $maxdev = 0;
6359
6360     if ($conf->{scsihw} && ($conf->{scsihw} =~ m/^lsi/)) {
6361         $maxdev = 7;
6362     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6363         $maxdev = 1;
6364     } else {
6365         $maxdev = 256;
6366     }
6367
6368     my $controller = int($drive->{index} / $maxdev);
6369     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6370
6371     return ($maxdev, $controller, $controller_prefix);
6372 }
6373
6374 # bash completion helper
6375
6376 sub complete_backup_archives {
6377     my ($cmdname, $pname, $cvalue) = @_;
6378
6379     my $cfg = PVE::Storage::config();
6380
6381     my $storeid;
6382
6383     if ($cvalue =~ m/^([^:]+):/) {
6384         $storeid = $1;
6385     }
6386
6387     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6388
6389     my $res = [];
6390     foreach my $id (keys %$data) {
6391         foreach my $item (@{$data->{$id}}) {
6392             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6393             push @$res, $item->{volid} if defined($item->{volid});
6394         }
6395     }
6396
6397     return $res;
6398 }
6399
6400 my $complete_vmid_full = sub {
6401     my ($running) = @_;
6402
6403     my $idlist = vmstatus();
6404
6405     my $res = [];
6406
6407     foreach my $id (keys %$idlist) {
6408         my $d = $idlist->{$id};
6409         if (defined($running)) {
6410             next if $d->{template};
6411             next if $running && $d->{status} ne 'running';
6412             next if !$running && $d->{status} eq 'running';
6413         }
6414         push @$res, $id;
6415
6416     }
6417     return $res;
6418 };
6419
6420 sub complete_vmid {
6421     return &$complete_vmid_full();
6422 }
6423
6424 sub complete_vmid_stopped {
6425     return &$complete_vmid_full(0);
6426 }
6427
6428 sub complete_vmid_running {
6429     return &$complete_vmid_full(1);
6430 }
6431
6432 sub complete_storage {
6433
6434     my $cfg = PVE::Storage::config();
6435     my $ids = $cfg->{ids};
6436
6437     my $res = [];
6438     foreach my $sid (keys %$ids) {
6439         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6440         push @$res, $sid;
6441     }
6442
6443     return $res;
6444 }
6445
6446 1;