PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QMPClient;
  31 use PVE::RPCEnvironment;
  32 use Time::HiRes qw(gettimeofday);
  33
  34 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  35
  36 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  37
  38 # Note about locking: we use flock on the config file protect
  39 # against concurent actions.
  40 # Aditionaly, we have a 'lock' setting in the config file. This
  41 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  42 # allowed when such lock is set. But you can ignore this kind of
  43 # lock with the --skiplock flag.
  44
  45 cfs_register_file('/qemu-server/',
  46                   \&parse_vm_config,
  47                   \&write_vm_config);
  48
  49 PVE::JSONSchema::register_standard_option('skiplock', {
  50     description => "Ignore locks - only root is allowed to use this option.",
  51     type => 'boolean',
  52     optional => 1,
  53 });
  54
  55 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  56     description => "Some command save/restore state from this location.",
  57     type => 'string',
  58     maxLength => 128,
  59     optional => 1,
  60 });
  61
  62 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  63     description => "The name of the snapshot.",
  64     type => 'string', format => 'pve-configid',
  65     maxLength => 40,
  66 });
  67
  68 #no warnings 'redefine';
  69
  70 sub cgroups_write {
  71    my ($controller, $vmid, $option, $value) = @_;
  72
  73    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  74    PVE::ProcFSTools::write_proc_entry($path, $value);
  75
  76 }
  77
  78 my $nodename = PVE::INotify::nodename();
  79
  80 mkdir "/etc/pve/nodes/$nodename";
  81 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  82 mkdir $confdir;
  83
  84 my $var_run_tmpdir = "/var/run/qemu-server";
  85 mkdir $var_run_tmpdir;
  86
  87 my $lock_dir = "/var/lock/qemu-server";
  88 mkdir $lock_dir;
  89
  90 my $pcisysfs = "/sys/bus/pci";
  91
  92 my $confdesc = {
  93     onboot => {
  94         optional => 1,
  95         type => 'boolean',
  96         description => "Specifies whether a VM will be started during system bootup.",
  97         default => 0,
  98     },
  99     autostart => {
 100         optional => 1,
 101         type => 'boolean',
 102         description => "Automatic restart after crash (currently ignored).",
 103         default => 0,
 104     },
 105     hotplug => {
 106         optional => 1,
 107         type => 'string', format => 'pve-hotplug-features',
 108         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 109         default => 'network,disk,usb',
 110     },
 111     reboot => {
 112         optional => 1,
 113         type => 'boolean',
 114         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 115         default => 1,
 116     },
 117     lock => {
 118         optional => 1,
 119         type => 'string',
 120         description => "Lock/unlock the VM.",
 121         enum => [qw(migrate backup snapshot rollback)],
 122     },
 123     cpulimit => {
 124         optional => 1,
 125         type => 'number',
 126         description => "Limit of CPU usage. Note if the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 127         minimum => 0,
 128         maximum => 128,
 129         default => 0,
 130     },
 131     cpuunits => {
 132         optional => 1,
 133         type => 'integer',
 134         description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 135         minimum => 0,
 136         maximum => 500000,
 137         default => 1000,
 138     },
 139     memory => {
 140         optional => 1,
 141         type => 'integer',
 142         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 143         minimum => 16,
 144         default => 512,
 145     },
 146     balloon => {
 147         optional => 1,
 148         type => 'integer',
 149         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 150         minimum => 0,
 151     },
 152     shares => {
 153         optional => 1,
 154         type => 'integer',
 155         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 156         minimum => 0,
 157         maximum => 50000,
 158         default => 1000,
 159     },
 160     keyboard => {
 161         optional => 1,
 162         type => 'string',
 163         description => "Keybord layout for vnc server. Default is read from the datacenter configuration file.",
 164         enum => PVE::Tools::kvmkeymaplist(),
 165         default => 'en-us',
 166     },
 167     name => {
 168         optional => 1,
 169         type => 'string', format => 'dns-name',
 170         description => "Set a name for the VM. Only used on the configuration web interface.",
 171     },
 172     scsihw => {
 173         optional => 1,
 174         type => 'string',
 175         description => "scsi controller model",
 176         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 177         default => 'lsi',
 178     },
 179     description => {
 180         optional => 1,
 181         type => 'string',
 182         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 183     },
 184     ostype => {
 185         optional => 1,
 186         type => 'string',
 187         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 l24 l26 solaris)],
 188         description => <<EODESC,
 189 Used to enable special optimization/features for specific
 190 operating systems:
 191
 192 other  => unspecified OS
 193 wxp    => Microsoft Windows XP
 194 w2k    => Microsoft Windows 2000
 195 w2k3   => Microsoft Windows 2003
 196 w2k8   => Microsoft Windows 2008
 197 wvista => Microsoft Windows Vista
 198 win7   => Microsoft Windows 7
 199 win8   => Microsoft Windows 8/2012
 200 l24    => Linux 2.4 Kernel
 201 l26    => Linux 2.6/3.X Kernel
 202 solaris => solaris/opensolaris/openindiania kernel
 203
 204 other|l24|l26|solaris                       ... no special behaviour
 205 wxp|w2k|w2k3|w2k8|wvista|win7|win8  ... use --localtime switch
 206 EODESC
 207     },
 208     boot => {
 209         optional => 1,
 210         type => 'string',
 211         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 212         pattern => '[acdn]{1,4}',
 213         default => 'cdn',
 214     },
 215     bootdisk => {
 216         optional => 1,
 217         type => 'string', format => 'pve-qm-bootdisk',
 218         description => "Enable booting from specified disk.",
 219         pattern => '(ide|sata|scsi|virtio)\d+',
 220     },
 221     smp => {
 222         optional => 1,
 223         type => 'integer',
 224         description => "The number of CPUs. Please use option -sockets instead.",
 225         minimum => 1,
 226         default => 1,
 227     },
 228     sockets => {
 229         optional => 1,
 230         type => 'integer',
 231         description => "The number of CPU sockets.",
 232         minimum => 1,
 233         default => 1,
 234     },
 235     cores => {
 236         optional => 1,
 237         type => 'integer',
 238         description => "The number of cores per socket.",
 239         minimum => 1,
 240         default => 1,
 241     },
 242     numa => {
 243         optional => 1,
 244         type => 'boolean',
 245         description => "Enable/disable Numa.",
 246         default => 0,
 247     },
 248     vcpus => {
 249         optional => 1,
 250         type => 'integer',
 251         description => "Number of hotplugged vcpus.",
 252         minimum => 1,
 253         default => 0,
 254     },
 255     acpi => {
 256         optional => 1,
 257         type => 'boolean',
 258         description => "Enable/disable ACPI.",
 259         default => 1,
 260     },
 261     agent => {
 262         optional => 1,
 263         type => 'boolean',
 264         description => "Enable/disable Qemu GuestAgent.",
 265         default => 0,
 266     },
 267     kvm => {
 268         optional => 1,
 269         type => 'boolean',
 270         description => "Enable/disable KVM hardware virtualization.",
 271         default => 1,
 272     },
 273     tdf => {
 274         optional => 1,
 275         type => 'boolean',
 276         description => "Enable/disable time drift fix.",
 277         default => 0,
 278     },
 279     localtime => {
 280         optional => 1,
 281         type => 'boolean',
 282         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 283     },
 284     freeze => {
 285         optional => 1,
 286         type => 'boolean',
 287         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 288     },
 289     vga => {
 290         optional => 1,
 291         type => 'string',
 292         description => "Select VGA type. If you want to use high resolution modes (>= 1280x1024x16) then you should use option 'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and 'cirrur' for other OS types. Option 'qxl' enables the SPICE display sever. You can also run without any graphic card using a serial devive as terminal.",
 293         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 294     },
 295     watchdog => {
 296         optional => 1,
 297         type => 'string', format => 'pve-qm-watchdog',
 298         typetext => '[[model=]i6300esb|ib700] [,[action=]reset|shutdown|poweroff|pause|debug|none]',
 299         description => "Create a virtual hardware watchdog device.  Once enabled (by a guest action), the watchdog must be periodically polled by an agent inside the guest or else the guest will be restarted (or execute the action specified)",
 300     },
 301     startdate => {
 302         optional => 1,
 303         type => 'string',
 304         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 305         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 306         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 307         default => 'now',
 308     },
 309     startup =>  get_standard_option('pve-startup-order'),
 310     template => {
 311         optional => 1,
 312         type => 'boolean',
 313         description => "Enable/disable Template.",
 314         default => 0,
 315     },
 316     args => {
 317         optional => 1,
 318         type => 'string',
 319         description => <<EODESCR,
 320 Note: this option is for experts only. It allows you to pass arbitrary arguments to kvm, for example:
 321
 322 args: -no-reboot -no-hpet
 323 EODESCR
 324     },
 325     tablet => {
 326         optional => 1,
 327         type => 'boolean',
 328         default => 1,
 329         description => "Enable/disable the usb tablet device. This device is usually needed to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with normal VNC clients. If you're running lots of console-only guests on one host, you may consider disabling this to save some context switches. This is turned of by default if you use spice (vga=qxl).",
 330     },
 331     migrate_speed => {
 332         optional => 1,
 333         type => 'integer',
 334         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 335         minimum => 0,
 336         default => 0,
 337     },
 338     migrate_downtime => {
 339         optional => 1,
 340         type => 'number',
 341         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 342         minimum => 0,
 343         default => 0.1,
 344     },
 345     cdrom => {
 346         optional => 1,
 347         type => 'string', format => 'pve-qm-drive',
 348         typetext => 'volume',
 349         description => "This is an alias for option -ide2",
 350     },
 351     cpu => {
 352         optional => 1,
 353         description => "Emulated CPU type.",
 354         type => 'string',
 355         enum => [ qw(486 athlon pentium pentium2 pentium3 coreduo core2duo kvm32 kvm64 qemu32 qemu64 phenom Conroe Penryn Nehalem Westmere SandyBridge IvyBridge Haswell Broadwell Opteron_G1 Opteron_G2 Opteron_G3 Opteron_G4 Opteron_G5 host) ],
 356         default => 'kvm64',
 357     },
 358     parent => get_standard_option('pve-snapshot-name', {
 359         optional => 1,
 360         description => "Parent snapshot name. This is used internally, and should not be modified.",
 361     }),
 362     snaptime => {
 363         optional => 1,
 364         description => "Timestamp for snapshots.",
 365         type => 'integer',
 366         minimum => 0,
 367     },
 368     vmstate => {
 369         optional => 1,
 370         type => 'string', format => 'pve-volume-id',
 371         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 372     },
 373     machine => {
 374         description => "Specific the Qemu machine type.",
 375         type => 'string',
 376         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
 377         maxLength => 40,
 378         optional => 1,
 379     },
 380     smbios1 => {
 381         description => "Specify SMBIOS type 1 fields.",
 382         type => 'string', format => 'pve-qm-smbios1',
 383         typetext => "[manufacturer=str][,product=str][,version=str][,serial=str] [,uuid=uuid][,sku=str][,family=str]",
 384         maxLength => 256,
 385         optional => 1,
 386     },
 387     protection => {
 388         optional => 1,
 389         type => 'boolean',
 390         description => "Sets the protection flag of the VM. This will prevent the remove operation.",
 391         default => 0,
 392     },
 393 };
 394
 395 # what about other qemu settings ?
 396 #cpu => 'string',
 397 #machine => 'string',
 398 #fda => 'file',
 399 #fdb => 'file',
 400 #mtdblock => 'file',
 401 #sd => 'file',
 402 #pflash => 'file',
 403 #snapshot => 'bool',
 404 #bootp => 'file',
 405 ##tftp => 'dir',
 406 ##smb => 'dir',
 407 #kernel => 'file',
 408 #append => 'string',
 409 #initrd => 'file',
 410 ##soundhw => 'string',
 411
 412 while (my ($k, $v) = each %$confdesc) {
 413     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 414 }
 415
 416 my $MAX_IDE_DISKS = 4;
 417 my $MAX_SCSI_DISKS = 14;
 418 my $MAX_VIRTIO_DISKS = 16;
 419 my $MAX_SATA_DISKS = 6;
 420 my $MAX_USB_DEVICES = 5;
 421 my $MAX_NETS = 32;
 422 my $MAX_UNUSED_DISKS = 8;
 423 my $MAX_HOSTPCI_DEVICES = 4;
 424 my $MAX_SERIAL_PORTS = 4;
 425 my $MAX_PARALLEL_PORTS = 3;
 426 my $MAX_NUMA = 8;
 427 my $MAX_MEM = 4194304;
 428 my $STATICMEM = 1024;
 429
 430 my $numadesc = {
 431     optional => 1,
 432     type => 'string', format => 'pve-qm-numanode',
 433     typetext => "cpus=<id[-id],memory=<mb>[[,hostnodes=<id[-id]>] [,policy=<preferred|bind|interleave>]]",
 434     description => "numa topology",
 435 };
 436 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 437
 438 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 439     $confdesc->{"numa$i"} = $numadesc;
 440 }
 441
 442 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 443                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 444                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 445 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 446
 447 my $netdesc = {
 448     optional => 1,
 449     type => 'string', format => 'pve-qm-net',
 450     typetext => "MODEL=XX:XX:XX:XX:XX:XX [,bridge=<dev>][,queues=<nbqueues>][,rate=<mbps>] [,tag=<vlanid>][,firewall=0|1],link_down=0|1]",
 451     description => <<EODESCR,
 452 Specify network devices.
 453
 454 MODEL is one of: $nic_model_list_txt
 455
 456 XX:XX:XX:XX:XX:XX should be an unique MAC address. This is
 457 automatically generated if not specified.
 458
 459 The bridge parameter can be used to automatically add the interface to a bridge device. The Proxmox VE standard bridge is called 'vmbr0'.
 460
 461 Option 'rate' is used to limit traffic bandwidth from and to this interface. It is specified as floating point number, unit is 'Megabytes per second'.
 462
 463 If you specify no bridge, we create a kvm 'user' (NATed) network device, which provides DHCP and DNS services. The following addresses are used:
 464
 465 10.0.2.2   Gateway
 466 10.0.2.3   DNS Server
 467 10.0.2.4   SMB Server
 468
 469 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 470
 471 EODESCR
 472 };
 473 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 474
 475 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 476     $confdesc->{"net$i"} = $netdesc;
 477 }
 478
 479 my $drivename_hash;
 480
 481 my $idedesc = {
 482     optional => 1,
 483     type => 'string', format => 'pve-qm-drive',
 484     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads] [,discard=ignore|on] [,serial=serial][,model=model]',
 485     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
 486 };
 487 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
 488
 489 my $scsidesc = {
 490     optional => 1,
 491     type => 'string', format => 'pve-qm-drive',
 492     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads] [,discard=ignore|on] [,iothread=on] [,queues=<nbqueues>] [,serial=serial]',
 493     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
 494 };
 495 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 496
 497 my $satadesc = {
 498     optional => 1,
 499     type => 'string', format => 'pve-qm-drive',
 500     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads]  [,discard=ignore|on] [,serial=serial]',
 501     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
 502 };
 503 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
 504
 505 my $virtiodesc = {
 506     optional => 1,
 507     type => 'string', format => 'pve-qm-drive',
 508     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads]  [,discard=ignore|on] [,iothread=on] [,serial=serial]',
 509     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
 510 };
 511 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
 512
 513 my $usbdesc = {
 514     optional => 1,
 515     type => 'string', format => 'pve-qm-usb-device',
 516     typetext => 'host=HOSTUSBDEVICE|spice',
 517     description => <<EODESCR,
 518 Configure an USB device (n is 0 to 4). This can be used to
 519 pass-through usb devices to the guest. HOSTUSBDEVICE syntax is:
 520
 521 'bus-port(.port)*' (decimal numbers) or
 522 'vendor_id:product_id' (hexadeciaml numbers)
 523
 524 You can use the 'lsusb -t' command to list existing usb devices.
 525
 526 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 527
 528 The value 'spice' can be used to add a usb redirection devices for spice.
 529
 530 EODESCR
 531 };
 532 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
 533
 534 my $hostpcidesc = {
 535         optional => 1,
 536         type => 'string', format => 'pve-qm-hostpci',
 537         typetext => "[host=]HOSTPCIDEVICE [,rombar=on|off] [,pcie=0|1] [,x-vga=on|off]",
 538         description => <<EODESCR,
 539 Map host pci devices. HOSTPCIDEVICE syntax is:
 540
 541 'bus:dev.func' (hexadecimal numbers)
 542
 543 You can us the 'lspci' command to list existing pci devices.
 544
 545 The 'rombar' option determines whether or not the device's ROM will be visible in the guest's memory map (default is 'on').
 546
 547 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 548
 549 Experimental: user reported problems with this option.
 550 EODESCR
 551 };
 552 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
 553
 554 my $serialdesc = {
 555         optional => 1,
 556         type => 'string',
 557         pattern => '(/dev/.+|socket)',
 558         description =>  <<EODESCR,
 559 Create a serial device inside the VM (n is 0 to 3), and pass through a host serial device (i.e. /dev/ttyS0), or create a unix socket on the host side (use 'qm terminal' to open a terminal connection).
 560
 561 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 562
 563 Experimental: user reported problems with this option.
 564 EODESCR
 565 };
 566
 567 my $paralleldesc= {
 568         optional => 1,
 569         type => 'string',
 570         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
 571         description =>  <<EODESCR,
 572 Map host parallel devices (n is 0 to 2).
 573
 574 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 575
 576 Experimental: user reported problems with this option.
 577 EODESCR
 578 };
 579
 580 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
 581     $confdesc->{"parallel$i"} = $paralleldesc;
 582 }
 583
 584 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
 585     $confdesc->{"serial$i"} = $serialdesc;
 586 }
 587
 588 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
 589     $confdesc->{"hostpci$i"} = $hostpcidesc;
 590 }
 591
 592 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
 593     $drivename_hash->{"ide$i"} = 1;
 594     $confdesc->{"ide$i"} = $idedesc;
 595 }
 596
 597 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
 598     $drivename_hash->{"sata$i"} = 1;
 599     $confdesc->{"sata$i"} = $satadesc;
 600 }
 601
 602 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
 603     $drivename_hash->{"scsi$i"} = 1;
 604     $confdesc->{"scsi$i"} = $scsidesc ;
 605 }
 606
 607 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
 608     $drivename_hash->{"virtio$i"} = 1;
 609     $confdesc->{"virtio$i"} = $virtiodesc;
 610 }
 611
 612 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
 613     $confdesc->{"usb$i"} = $usbdesc;
 614 }
 615
 616 my $unuseddesc = {
 617     optional => 1,
 618     type => 'string', format => 'pve-volume-id',
 619     description => "Reference to unused volumes.",
 620 };
 621
 622 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
 623     $confdesc->{"unused$i"} = $unuseddesc;
 624 }
 625
 626 my $kvm_api_version = 0;
 627
 628 sub kvm_version {
 629
 630     return $kvm_api_version if $kvm_api_version;
 631
 632     my $fh = IO::File->new("</dev/kvm") ||
 633         return 0;
 634
 635     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
 636         $kvm_api_version = $v;
 637     }
 638
 639     $fh->close();
 640
 641     return  $kvm_api_version;
 642 }
 643
 644 my $kvm_user_version;
 645
 646 sub kvm_user_version {
 647
 648     return $kvm_user_version if $kvm_user_version;
 649
 650     $kvm_user_version = 'unknown';
 651
 652     my $tmp = `kvm -help 2>/dev/null`;
 653
 654     if ($tmp =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
 655         $kvm_user_version = $2;
 656     }
 657
 658     return $kvm_user_version;
 659
 660 }
 661
 662 my $kernel_has_vhost_net = -c '/dev/vhost-net';
 663
 664 sub disknames {
 665     # order is important - used to autoselect boot disk
 666     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
 667             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
 668             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
 669             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))));
 670 }
 671
 672 sub valid_drivename {
 673     my $dev = shift;
 674
 675     return defined($drivename_hash->{$dev});
 676 }
 677
 678 sub option_exists {
 679     my $key = shift;
 680     return defined($confdesc->{$key});
 681 }
 682
 683 sub nic_models {
 684     return $nic_model_list;
 685 }
 686
 687 sub os_list_description {
 688
 689     return {
 690         other => 'Other',
 691         wxp => 'Windows XP',
 692         w2k => 'Windows 2000',
 693         w2k3 =>, 'Windows 2003',
 694         w2k8 => 'Windows 2008',
 695         wvista => 'Windows Vista',
 696         win7 => 'Windows 7',
 697         win8 => 'Windows 8/2012',
 698         l24 => 'Linux 2.4',
 699         l26 => 'Linux 2.6',
 700     };
 701 }
 702
 703 my $cdrom_path;
 704
 705 sub get_cdrom_path {
 706
 707     return  $cdrom_path if $cdrom_path;
 708
 709     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
 710     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
 711     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
 712 }
 713
 714 sub get_iso_path {
 715     my ($storecfg, $vmid, $cdrom) = @_;
 716
 717     if ($cdrom eq 'cdrom') {
 718         return get_cdrom_path();
 719     } elsif ($cdrom eq 'none') {
 720         return '';
 721     } elsif ($cdrom =~ m|^/|) {
 722         return $cdrom;
 723     } else {
 724         return PVE::Storage::path($storecfg, $cdrom);
 725     }
 726 }
 727
 728 # try to convert old style file names to volume IDs
 729 sub filename_to_volume_id {
 730     my ($vmid, $file, $media) = @_;
 731
 732     if (!($file eq 'none' || $file eq 'cdrom' ||
 733           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
 734
 735         return undef if $file =~ m|/|;
 736
 737         if ($media && $media eq 'cdrom') {
 738             $file = "local:iso/$file";
 739         } else {
 740             $file = "local:$vmid/$file";
 741         }
 742     }
 743
 744     return $file;
 745 }
 746
 747 sub verify_media_type {
 748     my ($opt, $vtype, $media) = @_;
 749
 750     return if !$media;
 751
 752     my $etype;
 753     if ($media eq 'disk') {
 754         $etype = 'images';
 755     } elsif ($media eq 'cdrom') {
 756         $etype = 'iso';
 757     } else {
 758         die "internal error";
 759     }
 760
 761     return if ($vtype eq $etype);
 762
 763     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
 764 }
 765
 766 sub cleanup_drive_path {
 767     my ($opt, $storecfg, $drive) = @_;
 768
 769     # try to convert filesystem paths to volume IDs
 770
 771     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
 772         ($drive->{file} !~ m|^/dev/.+|) &&
 773         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
 774         ($drive->{file} !~ m/^\d+$/)) {
 775         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
 776         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
 777         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
 778         verify_media_type($opt, $vtype, $drive->{media});
 779         $drive->{file} = $volid;
 780     }
 781
 782     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
 783 }
 784
 785 sub create_conf_nolock {
 786     my ($vmid, $settings) = @_;
 787
 788     my $filename = config_file($vmid);
 789
 790     die "configuration file '$filename' already exists\n" if -f $filename;
 791
 792     my $defaults = load_defaults();
 793
 794     $settings->{name} = "vm$vmid" if !$settings->{name};
 795     $settings->{memory} = $defaults->{memory} if !$settings->{memory};
 796
 797     my $data = '';
 798     foreach my $opt (keys %$settings) {
 799         next if !$confdesc->{$opt};
 800
 801         my $value = $settings->{$opt};
 802         next if !$value;
 803
 804         $data .= "$opt: $value\n";
 805     }
 806
 807     PVE::Tools::file_set_contents($filename, $data);
 808 }
 809
 810 sub parse_hotplug_features {
 811     my ($data) = @_;
 812
 813     my $res = {};
 814
 815     return $res if $data eq '0';
 816
 817     $data = $confdesc->{hotplug}->{default} if $data eq '1';
 818
 819     foreach my $feature (PVE::Tools::split_list($data)) {
 820         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
 821             $res->{$1} = 1;
 822         } else {
 823             warn "ignoring unknown hotplug feature '$feature'\n";
 824         }
 825     }
 826     return $res;
 827 }
 828
 829 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
 830 sub pve_verify_hotplug_features {
 831     my ($value, $noerr) = @_;
 832
 833     return $value if parse_hotplug_features($value);
 834
 835     return undef if $noerr;
 836
 837     die "unable to parse hotplug option\n";
 838 }
 839
 840 my $parse_size = sub {
 841     my ($value) = @_;
 842
 843     return undef if $value !~ m/^(\d+(\.\d+)?)([KMG])?$/;
 844     my ($size, $unit) = ($1, $3);
 845     if ($unit) {
 846         if ($unit eq 'K') {
 847             $size = $size * 1024;
 848         } elsif ($unit eq 'M') {
 849             $size = $size * 1024 * 1024;
 850         } elsif ($unit eq 'G') {
 851             $size = $size * 1024 * 1024 * 1024;
 852         }
 853     }
 854     return int($size);
 855 };
 856
 857 my $format_size = sub {
 858     my ($size) = @_;
 859
 860     $size = int($size);
 861
 862     my $kb = int($size/1024);
 863     return $size if $kb*1024 != $size;
 864
 865     my $mb = int($kb/1024);
 866     return "${kb}K" if $mb*1024 != $kb;
 867
 868     my $gb = int($mb/1024);
 869     return "${mb}M" if $gb*1024 != $mb;
 870
 871     return "${gb}G";
 872 };
 873
 874 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
 875 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
 876 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
 877 #        [,aio=native|threads][,discard=ignore|on][,iothread=on]
 878 #        [,serial=serial][,model=model]
 879
 880 sub parse_drive {
 881     my ($key, $data) = @_;
 882
 883     my $res = {};
 884
 885     # $key may be undefined - used to verify JSON parameters
 886     if (!defined($key)) {
 887         $res->{interface} = 'unknown'; # should not harm when used to verify parameters
 888         $res->{index} = 0;
 889     } elsif ($key =~ m/^([^\d]+)(\d+)$/) {
 890         $res->{interface} = $1;
 891         $res->{index} = $2;
 892     } else {
 893         return undef;
 894     }
 895
 896     foreach my $p (split (/,/, $data)) {
 897         next if $p =~ m/^\s*$/;
 898
 899         if ($p =~ m/^(file|volume|cyls|heads|secs|trans|media|snapshot|cache|format|rerror|werror|backup|aio|bps|mbps|mbps_max|bps_rd|mbps_rd|mbps_rd_max|bps_wr|mbps_wr|mbps_wr_max|iops|iops_max|iops_rd|iops_rd_max|iops_wr|iops_wr_max|size|discard|iothread|queues|serial|model)=(.+)$/) {
 900             my ($k, $v) = ($1, $2);
 901
 902             $k = 'file' if $k eq 'volume';
 903
 904             return undef if defined $res->{$k};
 905
 906             if ($k eq 'bps' || $k eq 'bps_rd' || $k eq 'bps_wr') {
 907                 return undef if !$v || $v !~ m/^\d+/;
 908                 $k = "m$k";
 909                 $v = sprintf("%.3f", $v / (1024*1024));
 910             }
 911             $res->{$k} = $v;
 912         } else {
 913             if (!$res->{file} && $p !~ m/=/) {
 914                 $res->{file} = $p;
 915             } else {
 916                 return undef;
 917             }
 918         }
 919     }
 920
 921     return undef if !$res->{file};
 922
 923     return undef if $res->{cache} &&
 924         $res->{cache} !~ m/^(off|none|writethrough|writeback|unsafe|directsync)$/;
 925     return undef if $res->{snapshot} && $res->{snapshot} !~ m/^(on|off)$/;
 926     return undef if $res->{cyls} && $res->{cyls} !~ m/^\d+$/;
 927     return undef if $res->{heads} && $res->{heads} !~ m/^\d+$/;
 928     return undef if $res->{secs} && $res->{secs} !~ m/^\d+$/;
 929     return undef if $res->{media} && $res->{media} !~ m/^(disk|cdrom)$/;
 930     return undef if $res->{trans} && $res->{trans} !~ m/^(none|lba|auto)$/;
 931     return undef if $res->{format} && $res->{format} !~ m/^(raw|cow|qcow|qed|qcow2|vmdk|cloop)$/;
 932     return undef if $res->{rerror} && $res->{rerror} !~ m/^(ignore|report|stop)$/;
 933     return undef if $res->{werror} && $res->{werror} !~ m/^(enospc|ignore|report|stop)$/;
 934     return undef if $res->{backup} && $res->{backup} !~ m/^(yes|no)$/;
 935     return undef if $res->{aio} && $res->{aio} !~ m/^(native|threads)$/;
 936     return undef if $res->{discard} && $res->{discard} !~ m/^(ignore|on)$/;
 937     return undef if $res->{iothread} && $res->{iothread} !~ m/^(on)$/;
 938     return undef if $res->{queues} && ($res->{queues} !~ m/^\d+$/ || $res->{queues} < 2);
 939
 940     return undef if $res->{mbps_rd} && $res->{mbps};
 941     return undef if $res->{mbps_wr} && $res->{mbps};
 942
 943     return undef if $res->{mbps} && $res->{mbps} !~ m/^\d+(\.\d+)?$/;
 944     return undef if $res->{mbps_max} && $res->{mbps_max} !~ m/^\d+(\.\d+)?$/;
 945     return undef if $res->{mbps_rd} && $res->{mbps_rd} !~ m/^\d+(\.\d+)?$/;
 946     return undef if $res->{mbps_rd_max} && $res->{mbps_rd_max} !~ m/^\d+(\.\d+)?$/;
 947     return undef if $res->{mbps_wr} && $res->{mbps_wr} !~ m/^\d+(\.\d+)?$/;
 948     return undef if $res->{mbps_wr_max} && $res->{mbps_wr_max} !~ m/^\d+(\.\d+)?$/;
 949
 950     return undef if $res->{iops_rd} && $res->{iops};
 951     return undef if $res->{iops_wr} && $res->{iops};
 952
 953
 954     return undef if $res->{iops} && $res->{iops} !~ m/^\d+$/;
 955     return undef if $res->{iops_max} && $res->{iops_max} !~ m/^\d+$/;
 956     return undef if $res->{iops_rd} && $res->{iops_rd} !~ m/^\d+$/;
 957     return undef if $res->{iops_rd_max} && $res->{iops_rd_max} !~ m/^\d+$/;
 958     return undef if $res->{iops_wr} && $res->{iops_wr} !~ m/^\d+$/;
 959     return undef if $res->{iops_wr_max} && $res->{iops_wr_max} !~ m/^\d+$/;
 960
 961     if ($res->{size}) {
 962         return undef if !defined($res->{size} = &$parse_size($res->{size}));
 963     }
 964
 965     if ($res->{media} && ($res->{media} eq 'cdrom')) {
 966         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
 967         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
 968         return undef if $res->{interface} eq 'virtio';
 969     }
 970
 971     # rerror does not work with scsi drives
 972     if ($res->{rerror}) {
 973         return undef if $res->{interface} eq 'scsi';
 974     }
 975
 976     return $res;
 977 }
 978
 979 my @qemu_drive_options = qw(heads secs cyls trans media format cache snapshot rerror werror aio discard iops iops_rd iops_wr iops_max iops_rd_max iops_wr_max serial);
 980
 981 sub print_drive {
 982     my ($vmid, $drive) = @_;
 983
 984     my $opts = '';
 985     foreach my $o (@qemu_drive_options, 'mbps', 'mbps_rd', 'mbps_wr', 'mbps_max', 'mbps_rd_max', 'mbps_wr_max', 'backup', 'iothread', 'queues') {
 986         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
 987     }
 988
 989     if ($drive->{size}) {
 990         $opts .= ",size=" . &$format_size($drive->{size});
 991     }
 992
 993     if (my $model = $drive->{model}) {
 994         $opts .= ",model=$model";
 995     }
 996
 997     return "$drive->{file}$opts";
 998 }
 999
1000 sub scsi_inquiry {
1001     my($fh, $noerr) = @_;
1002
1003     my $SG_IO = 0x2285;
1004     my $SG_GET_VERSION_NUM = 0x2282;
1005
1006     my $versionbuf = "\x00" x 8;
1007     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1008     if (!$ret) {
1009         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1010         return undef;
1011     }
1012     my $version = unpack("I", $versionbuf);
1013     if ($version < 30000) {
1014         die "scsi generic interface too old\n"  if !$noerr;
1015         return undef;
1016     }
1017
1018     my $buf = "\x00" x 36;
1019     my $sensebuf = "\x00" x 8;
1020     my $cmd = pack("C x3 C x1", 0x12, 36);
1021
1022     # see /usr/include/scsi/sg.h
1023     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1024
1025     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1026                       length($sensebuf), 0, length($buf), $buf,
1027                       $cmd, $sensebuf, 6000);
1028
1029     $ret = ioctl($fh, $SG_IO, $packet);
1030     if (!$ret) {
1031         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1032         return undef;
1033     }
1034
1035     my @res = unpack($sg_io_hdr_t, $packet);
1036     if ($res[17] || $res[18]) {
1037         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1038         return undef;
1039     }
1040
1041     my $res = {};
1042     (my $byte0, my $byte1, $res->{vendor},
1043      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1044
1045     $res->{removable} = $byte1 & 128 ? 1 : 0;
1046     $res->{type} = $byte0 & 31;
1047
1048     return $res;
1049 }
1050
1051 sub path_is_scsi {
1052     my ($path) = @_;
1053
1054     my $fh = IO::File->new("+<$path") || return undef;
1055     my $res = scsi_inquiry($fh, 1);
1056     close($fh);
1057
1058     return $res;
1059 }
1060
1061 sub machine_type_is_q35 {
1062     my ($conf) = @_;
1063
1064     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1065 }
1066
1067 sub print_tabletdevice_full {
1068     my ($conf) = @_;
1069
1070     my $q35 = machine_type_is_q35($conf);
1071
1072     # we use uhci for old VMs because tablet driver was buggy in older qemu
1073     my $usbbus = $q35 ? "ehci" : "uhci";
1074
1075     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1076 }
1077
1078 sub print_drivedevice_full {
1079     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1080
1081     my $device = '';
1082     my $maxdev = 0;
1083
1084     if ($drive->{interface} eq 'virtio') {
1085         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1086         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1087         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1088     } elsif ($drive->{interface} eq 'scsi') {
1089
1090         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1091         my $unit = $drive->{index} % $maxdev;
1092         my $devicetype = 'hd';
1093         my $path = '';
1094         if (drive_is_cdrom($drive)) {
1095               $devicetype = 'cd';
1096         } else {
1097               if ($drive->{file} =~ m|^/|) {
1098                   $path = $drive->{file};
1099               } else {
1100                   $path = PVE::Storage::path($storecfg, $drive->{file});
1101               }
1102
1103               if($path =~ m/^iscsi\:\/\//){
1104                   $devicetype = 'generic';
1105               } else {
1106                   if (my $info = path_is_scsi($path)) {
1107                       if ($info->{type} == 0) {
1108                           $devicetype = 'block';
1109                       } elsif ($info->{type} == 1) { # tape
1110                           $devicetype = 'generic';
1111                       }
1112                   }
1113               }
1114          }
1115
1116         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1117             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1118         } else {
1119             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1120         }
1121
1122     } elsif ($drive->{interface} eq 'ide'){
1123         $maxdev = 2;
1124         my $controller = int($drive->{index} / $maxdev);
1125         my $unit = $drive->{index} % $maxdev;
1126         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1127
1128         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1129         if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
1130             $device .= ",model=$model";
1131         }
1132     } elsif ($drive->{interface} eq 'sata'){
1133         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1134         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1135         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1136     } elsif ($drive->{interface} eq 'usb') {
1137         die "implement me";
1138         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1139     } else {
1140         die "unsupported interface type";
1141     }
1142
1143     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1144
1145     return $device;
1146 }
1147
1148 sub get_initiator_name {
1149     my $initiator;
1150
1151     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1152     while (defined(my $line = <$fh>)) {
1153         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1154         $initiator = $1;
1155         last;
1156     }
1157     $fh->close();
1158
1159     return $initiator;
1160 }
1161
1162 sub print_drive_full {
1163     my ($storecfg, $vmid, $drive) = @_;
1164
1165     my $path;
1166     my $volid = $drive->{file};
1167     my $format;
1168
1169     if (drive_is_cdrom($drive)) {
1170         $path = get_iso_path($storecfg, $vmid, $volid);
1171     } else {
1172         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1173         if ($storeid) {
1174             $path = PVE::Storage::path($storecfg, $volid);
1175             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1176             $format = qemu_img_format($scfg, $volname);
1177         } else {
1178             $path = $volid;
1179         }
1180    }
1181
1182     my $opts = '';
1183     foreach my $o (@qemu_drive_options) {
1184         next if $o eq 'bootindex';
1185         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1186     }
1187
1188     $opts .= ",format=$format" if $format && !$drive->{format};
1189
1190     foreach my $o (qw(bps bps_rd bps_wr)) {
1191         my $v = $drive->{"m$o"};
1192         $opts .= ",$o=" . int($v*1024*1024) if $v;
1193     }
1194
1195     my $cache_direct = 0;
1196
1197     if (my $cache = $drive->{cache}) {
1198         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1199     } elsif (!drive_is_cdrom($drive)) {
1200         $opts .= ",cache=none";
1201         $cache_direct = 1;
1202     }
1203
1204     # aio native works only with O_DIRECT
1205     if (!$drive->{aio}) {
1206         if($cache_direct) {
1207             $opts .= ",aio=native";
1208         } else {
1209             $opts .= ",aio=threads";
1210         }
1211     }
1212
1213     my $detectzeroes = $drive->{discard} ? "unmap" : "on";
1214     $opts .= ",detect-zeroes=$detectzeroes" if !drive_is_cdrom($drive);
1215
1216     my $pathinfo = $path ? "file=$path," : '';
1217
1218     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1219 }
1220
1221 sub print_netdevice_full {
1222     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1223
1224     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1225
1226     my $device = $net->{model};
1227     if ($net->{model} eq 'virtio') {
1228          $device = 'virtio-net-pci';
1229      };
1230
1231     my $pciaddr = print_pci_addr("$netid", $bridges);
1232     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1233     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1234         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1235         my $vectors = $net->{queues} * 2 + 2;
1236         $tmpstr .= ",vectors=$vectors,mq=on";
1237     }
1238     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1239
1240     if ($use_old_bios_files) {
1241         my $romfile;
1242         if ($device eq 'virtio-net-pci') {
1243             $romfile = 'pxe-virtio.rom';
1244         } elsif ($device eq 'e1000') {
1245             $romfile = 'pxe-e1000.rom';
1246         } elsif ($device eq 'ne2k') {
1247             $romfile = 'pxe-ne2k_pci.rom';
1248         } elsif ($device eq 'pcnet') {
1249             $romfile = 'pxe-pcnet.rom';
1250         } elsif ($device eq 'rtl8139') {
1251             $romfile = 'pxe-rtl8139.rom';
1252         }
1253         $tmpstr .= ",romfile=$romfile" if $romfile;
1254     }
1255
1256     return $tmpstr;
1257 }
1258
1259 sub print_netdev_full {
1260     my ($vmid, $conf, $net, $netid) = @_;
1261
1262     my $i = '';
1263     if ($netid =~ m/^net(\d+)$/) {
1264         $i = int($1);
1265     }
1266
1267     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1268
1269     my $ifname = "tap${vmid}i$i";
1270
1271     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1272     die "interface name '$ifname' is too long (max 15 character)\n"
1273         if length($ifname) >= 16;
1274
1275     my $vhostparam = '';
1276     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1277
1278     my $vmname = $conf->{name} || "vm$vmid";
1279
1280     my $netdev = "";
1281
1282     if ($net->{bridge}) {
1283         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/pve-bridge,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1284     } else {
1285         $netdev = "type=user,id=$netid,hostname=$vmname";
1286     }
1287
1288     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1289
1290     return $netdev;
1291 }
1292
1293 sub drive_is_cdrom {
1294     my ($drive) = @_;
1295
1296     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1297
1298 }
1299
1300 sub parse_numa {
1301     my ($data) = @_;
1302
1303     my $res = {};
1304
1305     foreach my $kvp (split(/,/, $data)) {
1306
1307         if ($kvp =~ m/^memory=(\S+)$/) {
1308             $res->{memory} = $1;
1309         } elsif ($kvp =~ m/^policy=(preferred|bind|interleave)$/) {
1310             $res->{policy} = $1;
1311         } elsif ($kvp =~ m/^cpus=(\d+)(-(\d+))?$/) {
1312             $res->{cpus}->{start} = $1;
1313             $res->{cpus}->{end} = $3;
1314         } elsif ($kvp =~ m/^hostnodes=(\d+)(-(\d+))?$/) {
1315             $res->{hostnodes}->{start} = $1;
1316             $res->{hostnodes}->{end} = $3;
1317         } else {
1318             return undef;
1319         }
1320     }
1321
1322     return $res;
1323 }
1324
1325 sub parse_hostpci {
1326     my ($value) = @_;
1327
1328     return undef if !$value;
1329
1330
1331     my @list = split(/,/, $value);
1332     my $found;
1333
1334     my $res = {};
1335     foreach my $kv (@list) {
1336
1337         if ($kv =~ m/^(host=)?([a-f0-9]{2}:[a-f0-9]{2})(\.([a-f0-9]))?$/) {
1338             $found = 1;
1339             if(defined($4)){
1340                 push @{$res->{pciid}}, { id => $2 , function => $4};
1341
1342             }else{
1343                 my $pcidevices = lspci($2);
1344                 $res->{pciid} = $pcidevices->{$2};
1345             }
1346         } elsif ($kv =~ m/^rombar=(on|off)$/) {
1347             $res->{rombar} = $1;
1348         } elsif ($kv =~ m/^x-vga=(on|off)$/) {
1349             $res->{'x-vga'} = $1;
1350         } elsif ($kv =~ m/^pcie=(\d+)$/) {
1351             $res->{pcie} = 1 if $1 == 1;
1352         } else {
1353             warn "unknown hostpci setting '$kv'\n";
1354         }
1355     }
1356
1357     return undef if !$found;
1358
1359     return $res;
1360 }
1361
1362 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1363 sub parse_net {
1364     my ($data) = @_;
1365
1366     my $res = {};
1367
1368     foreach my $kvp (split(/,/, $data)) {
1369
1370         if ($kvp =~ m/^(ne2k_pci|e1000|e1000-82540em|e1000-82544gc|e1000-82545em|rtl8139|pcnet|virtio|ne2k_isa|i82551|i82557b|i82559er|vmxnet3)(=([0-9a-f]{2}(:[0-9a-f]{2}){5}))?$/i) {
1371             my $model = lc($1);
1372             my $mac = defined($3) ? uc($3) : PVE::Tools::random_ether_addr();
1373             $res->{model} = $model;
1374             $res->{macaddr} = $mac;
1375         } elsif ($kvp =~ m/^bridge=(\S+)$/) {
1376             $res->{bridge} = $1;
1377         } elsif ($kvp =~ m/^queues=(\d+)$/) {
1378             $res->{queues} = $1;
1379         } elsif ($kvp =~ m/^rate=(\d+(\.\d+)?)$/) {
1380             $res->{rate} = $1;
1381         } elsif ($kvp =~ m/^tag=(\d+)$/) {
1382             $res->{tag} = $1;
1383         } elsif ($kvp =~ m/^firewall=([01])$/) {
1384             $res->{firewall} = $1;
1385         } elsif ($kvp =~ m/^link_down=([01])$/) {
1386             $res->{link_down} = $1;
1387         } else {
1388             return undef;
1389         }
1390
1391     }
1392
1393     return undef if !$res->{model};
1394
1395     return $res;
1396 }
1397
1398 sub print_net {
1399     my $net = shift;
1400
1401     my $res = "$net->{model}";
1402     $res .= "=$net->{macaddr}" if $net->{macaddr};
1403     $res .= ",bridge=$net->{bridge}" if $net->{bridge};
1404     $res .= ",rate=$net->{rate}" if $net->{rate};
1405     $res .= ",tag=$net->{tag}" if $net->{tag};
1406     $res .= ",firewall=1" if $net->{firewall};
1407     $res .= ",link_down=1" if $net->{link_down};
1408     $res .= ",queues=$net->{queues}" if $net->{queues};
1409
1410     return $res;
1411 }
1412
1413 sub add_random_macs {
1414     my ($settings) = @_;
1415
1416     foreach my $opt (keys %$settings) {
1417         next if $opt !~ m/^net(\d+)$/;
1418         my $net = parse_net($settings->{$opt});
1419         next if !$net;
1420         $settings->{$opt} = print_net($net);
1421     }
1422 }
1423
1424 sub add_unused_volume {
1425     my ($config, $volid) = @_;
1426
1427     my $key;
1428     for (my $ind = $MAX_UNUSED_DISKS - 1; $ind >= 0; $ind--) {
1429         my $test = "unused$ind";
1430         if (my $vid = $config->{$test}) {
1431             return if $vid eq $volid; # do not add duplicates
1432         } else {
1433             $key = $test;
1434         }
1435     }
1436
1437     die "To many unused volume - please delete them first.\n" if !$key;
1438
1439     $config->{$key} = $volid;
1440
1441     return $key;
1442 }
1443
1444 sub vm_is_volid_owner {
1445     my ($storecfg, $vmid, $volid) = @_;
1446
1447     if ($volid !~  m|^/|) {
1448         my ($path, $owner);
1449         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1450         if ($owner && ($owner == $vmid)) {
1451             return 1;
1452         }
1453     }
1454
1455     return undef;
1456 }
1457
1458 sub split_flagged_list {
1459     my $text = shift || '';
1460     $text =~ s/[,;]/ /g;
1461     $text =~ s/^\s+//;
1462     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
1463 }
1464
1465 sub join_flagged_list {
1466     my ($how, $lst) = @_;
1467     join $how, map { $lst->{$_} . $_ } keys %$lst;
1468 }
1469
1470 sub vmconfig_delete_pending_option {
1471     my ($conf, $key, $force) = @_;
1472
1473     delete $conf->{pending}->{$key};
1474     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1475     $pending_delete_hash->{$key} = $force ? '!' : '';
1476     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1477 }
1478
1479 sub vmconfig_undelete_pending_option {
1480     my ($conf, $key) = @_;
1481
1482     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1483     delete $pending_delete_hash->{$key};
1484
1485     if (%$pending_delete_hash) {
1486         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1487     } else {
1488         delete $conf->{pending}->{delete};
1489     }
1490 }
1491
1492 sub vmconfig_register_unused_drive {
1493     my ($storecfg, $vmid, $conf, $drive) = @_;
1494
1495     if (!drive_is_cdrom($drive)) {
1496         my $volid = $drive->{file};
1497         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1498             add_unused_volume($conf, $volid, $vmid);
1499         }
1500     }
1501 }
1502
1503 sub vmconfig_cleanup_pending {
1504     my ($conf) = @_;
1505
1506     # remove pending changes when nothing changed
1507     my $changes;
1508     foreach my $opt (keys %{$conf->{pending}}) {
1509         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
1510             $changes = 1;
1511             delete $conf->{pending}->{$opt};
1512         }
1513     }
1514
1515     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
1516     my $pending_delete_hash = {};
1517     while (my ($opt, $force) = each %$current_delete_hash) {
1518         if (defined($conf->{$opt})) {
1519             $pending_delete_hash->{$opt} = $force;
1520         } else {
1521             $changes = 1;
1522         }
1523     }
1524
1525     if (%$pending_delete_hash) {
1526         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1527     } else {
1528         delete $conf->{pending}->{delete};
1529     }
1530
1531     return $changes;
1532 }
1533
1534 my $valid_smbios1_options = {
1535     manufacturer => '\S+',
1536     product => '\S+',
1537     version => '\S+',
1538     serial => '\S+',
1539     uuid => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1540     sku => '\S+',
1541     family => '\S+',
1542 };
1543
1544 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
1545 sub parse_smbios1 {
1546     my ($data) = @_;
1547
1548     my $res = {};
1549
1550     foreach my $kvp (split(/,/, $data)) {
1551         return undef if $kvp !~ m/^(\S+)=(.+)$/;
1552         my ($k, $v) = split(/=/, $kvp);
1553         return undef if !defined($k) || !defined($v);
1554         return undef if !$valid_smbios1_options->{$k};
1555         return undef if $v !~ m/^$valid_smbios1_options->{$k}$/;
1556         $res->{$k} = $v;
1557     }
1558
1559     return $res;
1560 }
1561
1562 sub print_smbios1 {
1563     my ($smbios1) = @_;
1564
1565     my $data = '';
1566     foreach my $k (keys %$smbios1) {
1567         next if !defined($smbios1->{$k});
1568         next if !$valid_smbios1_options->{$k};
1569         $data .= ',' if $data;
1570         $data .= "$k=$smbios1->{$k}";
1571     }
1572     return $data;
1573 }
1574
1575 PVE::JSONSchema::register_format('pve-qm-smbios1', \&verify_smbios1);
1576 sub verify_smbios1 {
1577     my ($value, $noerr) = @_;
1578
1579     return $value if parse_smbios1($value);
1580
1581     return undef if $noerr;
1582
1583     die "unable to parse smbios (type 1) options\n";
1584 }
1585
1586 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
1587 sub verify_bootdisk {
1588     my ($value, $noerr) = @_;
1589
1590     return $value if valid_drivename($value);
1591
1592     return undef if $noerr;
1593
1594     die "invalid boot disk '$value'\n";
1595 }
1596
1597 PVE::JSONSchema::register_format('pve-qm-numanode', \&verify_numa);
1598 sub verify_numa {
1599     my ($value, $noerr) = @_;
1600
1601     return $value if parse_numa($value);
1602
1603     return undef if $noerr;
1604
1605     die "unable to parse numa options\n";
1606 }
1607
1608 PVE::JSONSchema::register_format('pve-qm-net', \&verify_net);
1609 sub verify_net {
1610     my ($value, $noerr) = @_;
1611
1612     return $value if parse_net($value);
1613
1614     return undef if $noerr;
1615
1616     die "unable to parse network options\n";
1617 }
1618
1619 PVE::JSONSchema::register_format('pve-qm-drive', \&verify_drive);
1620 sub verify_drive {
1621     my ($value, $noerr) = @_;
1622
1623     return $value if parse_drive(undef, $value);
1624
1625     return undef if $noerr;
1626
1627     die "unable to parse drive options\n";
1628 }
1629
1630 PVE::JSONSchema::register_format('pve-qm-hostpci', \&verify_hostpci);
1631 sub verify_hostpci {
1632     my ($value, $noerr) = @_;
1633
1634     return $value if parse_hostpci($value);
1635
1636     return undef if $noerr;
1637
1638     die "unable to parse pci id\n";
1639 }
1640
1641 PVE::JSONSchema::register_format('pve-qm-watchdog', \&verify_watchdog);
1642 sub verify_watchdog {
1643     my ($value, $noerr) = @_;
1644
1645     return $value if parse_watchdog($value);
1646
1647     return undef if $noerr;
1648
1649     die "unable to parse watchdog options\n";
1650 }
1651
1652 sub parse_watchdog {
1653     my ($value) = @_;
1654
1655     return undef if !$value;
1656
1657     my $res = {};
1658
1659     foreach my $p (split(/,/, $value)) {
1660         next if $p =~ m/^\s*$/;
1661
1662         if ($p =~ m/^(model=)?(i6300esb|ib700)$/) {
1663             $res->{model} = $2;
1664         } elsif ($p =~ m/^(action=)?(reset|shutdown|poweroff|pause|debug|none)$/) {
1665             $res->{action} = $2;
1666         } else {
1667             return undef;
1668         }
1669     }
1670
1671     return $res;
1672 }
1673
1674 sub parse_usb_device {
1675     my ($value) = @_;
1676
1677     return undef if !$value;
1678
1679     my @dl = split(/,/, $value);
1680     my $found;
1681
1682     my $res = {};
1683     foreach my $v (@dl) {
1684         if ($v =~ m/^host=(0x)?([0-9A-Fa-f]{4}):(0x)?([0-9A-Fa-f]{4})$/) {
1685             $found = 1;
1686             $res->{vendorid} = $2;
1687             $res->{productid} = $4;
1688         } elsif ($v =~ m/^host=(\d+)\-(\d+(\.\d+)*)$/) {
1689             $found = 1;
1690             $res->{hostbus} = $1;
1691             $res->{hostport} = $2;
1692         } elsif ($v =~ m/^spice$/) {
1693             $found = 1;
1694             $res->{spice} = 1;
1695         } else {
1696             return undef;
1697         }
1698     }
1699     return undef if !$found;
1700
1701     return $res;
1702 }
1703
1704 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
1705 sub verify_usb_device {
1706     my ($value, $noerr) = @_;
1707
1708     return $value if parse_usb_device($value);
1709
1710     return undef if $noerr;
1711
1712     die "unable to parse usb device\n";
1713 }
1714
1715 # add JSON properties for create and set function
1716 sub json_config_properties {
1717     my $prop = shift;
1718
1719     foreach my $opt (keys %$confdesc) {
1720         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
1721         $prop->{$opt} = $confdesc->{$opt};
1722     }
1723
1724     return $prop;
1725 }
1726
1727 sub check_type {
1728     my ($key, $value) = @_;
1729
1730     die "unknown setting '$key'\n" if !$confdesc->{$key};
1731
1732     my $type = $confdesc->{$key}->{type};
1733
1734     if (!defined($value)) {
1735         die "got undefined value\n";
1736     }
1737
1738     if ($value =~ m/[\n\r]/) {
1739         die "property contains a line feed\n";
1740     }
1741
1742     if ($type eq 'boolean') {
1743         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
1744         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
1745         die "type check ('boolean') failed - got '$value'\n";
1746     } elsif ($type eq 'integer') {
1747         return int($1) if $value =~ m/^(\d+)$/;
1748         die "type check ('integer') failed - got '$value'\n";
1749     } elsif ($type eq 'number') {
1750         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
1751         die "type check ('number') failed - got '$value'\n";
1752     } elsif ($type eq 'string') {
1753         if (my $fmt = $confdesc->{$key}->{format}) {
1754             if ($fmt eq 'pve-qm-drive') {
1755                 # special case - we need to pass $key to parse_drive()
1756                 my $drive = parse_drive($key, $value);
1757                 return $value if $drive;
1758                 die "unable to parse drive options\n";
1759             }
1760             PVE::JSONSchema::check_format($fmt, $value);
1761             return $value;
1762         }
1763         $value =~ s/^\"(.*)\"$/$1/;
1764         return $value;
1765     } else {
1766         die "internal error"
1767     }
1768 }
1769
1770 sub lock_config_full {
1771     my ($vmid, $timeout, $code, @param) = @_;
1772
1773     my $filename = config_file_lock($vmid);
1774
1775     my $res = lock_file($filename, $timeout, $code, @param);
1776
1777     die $@ if $@;
1778
1779     return $res;
1780 }
1781
1782 sub lock_config_mode {
1783     my ($vmid, $timeout, $shared, $code, @param) = @_;
1784
1785     my $filename = config_file_lock($vmid);
1786
1787     my $res = lock_file_full($filename, $timeout, $shared, $code, @param);
1788
1789     die $@ if $@;
1790
1791     return $res;
1792 }
1793
1794 sub lock_config {
1795     my ($vmid, $code, @param) = @_;
1796
1797     return lock_config_full($vmid, 10, $code, @param);
1798 }
1799
1800 sub cfs_config_path {
1801     my ($vmid, $node) = @_;
1802
1803     $node = $nodename if !$node;
1804     return "nodes/$node/qemu-server/$vmid.conf";
1805 }
1806
1807 sub check_iommu_support{
1808     #fixme : need to check IOMMU support
1809     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
1810
1811     my $iommu=1;
1812     return $iommu;
1813
1814 }
1815
1816 sub config_file {
1817     my ($vmid, $node) = @_;
1818
1819     my $cfspath = cfs_config_path($vmid, $node);
1820     return "/etc/pve/$cfspath";
1821 }
1822
1823 sub config_file_lock {
1824     my ($vmid) = @_;
1825
1826     return "$lock_dir/lock-$vmid.conf";
1827 }
1828
1829 sub touch_config {
1830     my ($vmid) = @_;
1831
1832     my $conf = config_file($vmid);
1833     utime undef, undef, $conf;
1834 }
1835
1836 sub destroy_vm {
1837     my ($storecfg, $vmid, $keep_empty_config) = @_;
1838
1839     my $conffile = config_file($vmid);
1840
1841     my $conf = load_config($vmid);
1842
1843     check_lock($conf);
1844
1845     # only remove disks owned by this VM
1846     foreach_drive($conf, sub {
1847         my ($ds, $drive) = @_;
1848
1849         return if drive_is_cdrom($drive);
1850
1851         my $volid = $drive->{file};
1852
1853         return if !$volid || $volid =~ m|^/|;
1854
1855         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
1856         return if !$path || !$owner || ($owner != $vmid);
1857
1858         PVE::Storage::vdisk_free($storecfg, $volid);
1859     });
1860
1861     if ($keep_empty_config) {
1862         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
1863     } else {
1864         unlink $conffile;
1865     }
1866
1867     # also remove unused disk
1868     eval {
1869         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
1870
1871         eval {
1872             PVE::Storage::foreach_volid($dl, sub {
1873                 my ($volid, $sid, $volname, $d) = @_;
1874                 PVE::Storage::vdisk_free($storecfg, $volid);
1875             });
1876         };
1877         warn $@ if $@;
1878
1879     };
1880     warn $@ if $@;
1881 }
1882
1883 sub load_config {
1884     my ($vmid, $node) = @_;
1885
1886     my $cfspath = cfs_config_path($vmid, $node);
1887
1888     my $conf = PVE::Cluster::cfs_read_file($cfspath);
1889
1890     die "no such VM ('$vmid')\n" if !defined($conf);
1891
1892     return $conf;
1893 }
1894
1895 sub parse_vm_config {
1896     my ($filename, $raw) = @_;
1897
1898     return undef if !defined($raw);
1899
1900     my $res = {
1901         digest => Digest::SHA::sha1_hex($raw),
1902         snapshots => {},
1903         pending => {},
1904     };
1905
1906     $filename =~ m|/qemu-server/(\d+)\.conf$|
1907         || die "got strange filename '$filename'";
1908
1909     my $vmid = $1;
1910
1911     my $conf = $res;
1912     my $descr;
1913     my $section = '';
1914
1915     my @lines = split(/\n/, $raw);
1916     foreach my $line (@lines) {
1917         next if $line =~ m/^\s*$/;
1918
1919         if ($line =~ m/^\[PENDING\]\s*$/i) {
1920             $section = 'pending';
1921             if (defined($descr)) {
1922                 $descr =~ s/\s+$//;
1923                 $conf->{description} = $descr;
1924             }
1925             $descr = undef;
1926             $conf = $res->{$section} = {};
1927             next;
1928
1929         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
1930             $section = $1;
1931             if (defined($descr)) {
1932                 $descr =~ s/\s+$//;
1933                 $conf->{description} = $descr;
1934             }
1935             $descr = undef;
1936             $conf = $res->{snapshots}->{$section} = {};
1937             next;
1938         }
1939
1940         if ($line =~ m/^\#(.*)\s*$/) {
1941             $descr = '' if !defined($descr);
1942             $descr .= PVE::Tools::decode_text($1) . "\n";
1943             next;
1944         }
1945
1946         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
1947             $descr = '' if !defined($descr);
1948             $descr .= PVE::Tools::decode_text($2);
1949         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
1950             $conf->{snapstate} = $1;
1951         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
1952             my $key = $1;
1953             my $value = $2;
1954             $conf->{$key} = $value;
1955         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
1956             my $value = $1;
1957             if ($section eq 'pending') {
1958                 $conf->{delete} = $value; # we parse this later
1959             } else {
1960                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
1961             }
1962         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
1963             my $key = $1;
1964             my $value = $2;
1965             eval { $value = check_type($key, $value); };
1966             if ($@) {
1967                 warn "vm $vmid - unable to parse value of '$key' - $@";
1968             } else {
1969                 my $fmt = $confdesc->{$key}->{format};
1970                 if ($fmt && $fmt eq 'pve-qm-drive') {
1971                     my $v = parse_drive($key, $value);
1972                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
1973                         $v->{file} = $volid;
1974                         $value = print_drive($vmid, $v);
1975                     } else {
1976                         warn "vm $vmid - unable to parse value of '$key'\n";
1977                         next;
1978                     }
1979                 }
1980
1981                 if ($key eq 'cdrom') {
1982                     $conf->{ide2} = $value;
1983                 } else {
1984                     $conf->{$key} = $value;
1985                 }
1986             }
1987         }
1988     }
1989
1990     if (defined($descr)) {
1991         $descr =~ s/\s+$//;
1992         $conf->{description} = $descr;
1993     }
1994     delete $res->{snapstate}; # just to be sure
1995
1996     return $res;
1997 }
1998
1999 sub write_vm_config {
2000     my ($filename, $conf) = @_;
2001
2002     delete $conf->{snapstate}; # just to be sure
2003
2004     if ($conf->{cdrom}) {
2005         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2006         $conf->{ide2} = $conf->{cdrom};
2007         delete $conf->{cdrom};
2008     }
2009
2010     # we do not use 'smp' any longer
2011     if ($conf->{sockets}) {
2012         delete $conf->{smp};
2013     } elsif ($conf->{smp}) {
2014         $conf->{sockets} = $conf->{smp};
2015         delete $conf->{cores};
2016         delete $conf->{smp};
2017     }
2018
2019     my $used_volids = {};
2020
2021     my $cleanup_config = sub {
2022         my ($cref, $pending, $snapname) = @_;
2023
2024         foreach my $key (keys %$cref) {
2025             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2026                 $key eq 'snapstate' || $key eq 'pending';
2027             my $value = $cref->{$key};
2028             if ($key eq 'delete') {
2029                 die "propertry 'delete' is only allowed in [PENDING]\n"
2030                     if !$pending;
2031                 # fixme: check syntax?
2032                 next;
2033             }
2034             eval { $value = check_type($key, $value); };
2035             die "unable to parse value of '$key' - $@" if $@;
2036
2037             $cref->{$key} = $value;
2038
2039             if (!$snapname && valid_drivename($key)) {
2040                 my $drive = parse_drive($key, $value);
2041                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2042             }
2043         }
2044     };
2045
2046     &$cleanup_config($conf);
2047
2048     &$cleanup_config($conf->{pending}, 1);
2049
2050     foreach my $snapname (keys %{$conf->{snapshots}}) {
2051         die "internal error" if $snapname eq 'pending';
2052         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2053     }
2054
2055     # remove 'unusedX' settings if we re-add a volume
2056     foreach my $key (keys %$conf) {
2057         my $value = $conf->{$key};
2058         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2059             delete $conf->{$key};
2060         }
2061     }
2062
2063     my $generate_raw_config = sub {
2064         my ($conf, $pending) = @_;
2065
2066         my $raw = '';
2067
2068         # add description as comment to top of file
2069         if (defined(my $descr = $conf->{description})) {
2070             if ($descr) {
2071                 foreach my $cl (split(/\n/, $descr)) {
2072                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2073                 }
2074             } else {
2075                 $raw .= "#\n" if $pending;
2076             }
2077         }
2078
2079         foreach my $key (sort keys %$conf) {
2080             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2081             $raw .= "$key: $conf->{$key}\n";
2082         }
2083         return $raw;
2084     };
2085
2086     my $raw = &$generate_raw_config($conf);
2087
2088     if (scalar(keys %{$conf->{pending}})){
2089         $raw .= "\n[PENDING]\n";
2090         $raw .= &$generate_raw_config($conf->{pending}, 1);
2091     }
2092
2093     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2094         $raw .= "\n[$snapname]\n";
2095         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2096     }
2097
2098     return $raw;
2099 }
2100
2101 sub update_config_nolock {
2102     my ($vmid, $conf, $skiplock) = @_;
2103
2104     check_lock($conf) if !$skiplock;
2105
2106     my $cfspath = cfs_config_path($vmid);
2107
2108     PVE::Cluster::cfs_write_file($cfspath, $conf);
2109 }
2110
2111 sub update_config {
2112     my ($vmid, $conf, $skiplock) = @_;
2113
2114     lock_config($vmid, &update_config_nolock, $conf, $skiplock);
2115 }
2116
2117 sub load_defaults {
2118
2119     my $res = {};
2120
2121     # we use static defaults from our JSON schema configuration
2122     foreach my $key (keys %$confdesc) {
2123         if (defined(my $default = $confdesc->{$key}->{default})) {
2124             $res->{$key} = $default;
2125         }
2126     }
2127
2128     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2129     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2130
2131     return $res;
2132 }
2133
2134 sub config_list {
2135     my $vmlist = PVE::Cluster::get_vmlist();
2136     my $res = {};
2137     return $res if !$vmlist || !$vmlist->{ids};
2138     my $ids = $vmlist->{ids};
2139
2140     foreach my $vmid (keys %$ids) {
2141         my $d = $ids->{$vmid};
2142         next if !$d->{node} || $d->{node} ne $nodename;
2143         next if !$d->{type} || $d->{type} ne 'qemu';
2144         $res->{$vmid}->{exists} = 1;
2145     }
2146     return $res;
2147 }
2148
2149 # test if VM uses local resources (to prevent migration)
2150 sub check_local_resources {
2151     my ($conf, $noerr) = @_;
2152
2153     my $loc_res = 0;
2154
2155     $loc_res = 1 if $conf->{hostusb}; # old syntax
2156     $loc_res = 1 if $conf->{hostpci}; # old syntax
2157
2158     foreach my $k (keys %$conf) {
2159         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2160         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2161     }
2162
2163     die "VM uses local resources\n" if $loc_res && !$noerr;
2164
2165     return $loc_res;
2166 }
2167
2168 # check if used storages are available on all nodes (use by migrate)
2169 sub check_storage_availability {
2170     my ($storecfg, $conf, $node) = @_;
2171
2172     foreach_drive($conf, sub {
2173         my ($ds, $drive) = @_;
2174
2175         my $volid = $drive->{file};
2176         return if !$volid;
2177
2178         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2179         return if !$sid;
2180
2181         # check if storage is available on both nodes
2182         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2183         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2184    });
2185 }
2186
2187 # list nodes where all VM images are available (used by has_feature API)
2188 sub shared_nodes {
2189     my ($conf, $storecfg) = @_;
2190
2191     my $nodelist = PVE::Cluster::get_nodelist();
2192     my $nodehash = { map { $_ => 1 } @$nodelist };
2193     my $nodename = PVE::INotify::nodename();
2194
2195     foreach_drive($conf, sub {
2196         my ($ds, $drive) = @_;
2197
2198         my $volid = $drive->{file};
2199         return if !$volid;
2200
2201         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2202         if ($storeid) {
2203             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2204             if ($scfg->{disable}) {
2205                 $nodehash = {};
2206             } elsif (my $avail = $scfg->{nodes}) {
2207                 foreach my $node (keys %$nodehash) {
2208                     delete $nodehash->{$node} if !$avail->{$node};
2209                 }
2210             } elsif (!$scfg->{shared}) {
2211                 foreach my $node (keys %$nodehash) {
2212                     delete $nodehash->{$node} if $node ne $nodename
2213                 }
2214             }
2215         }
2216     });
2217
2218     return $nodehash
2219 }
2220
2221 sub check_lock {
2222     my ($conf) = @_;
2223
2224     die "VM is locked ($conf->{lock})\n" if $conf->{lock};
2225 }
2226
2227 sub check_cmdline {
2228     my ($pidfile, $pid) = @_;
2229
2230     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2231     if (defined($fh)) {
2232         my $line = <$fh>;
2233         $fh->close;
2234         return undef if !$line;
2235         my @param = split(/\0/, $line);
2236
2237         my $cmd = $param[0];
2238         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2239
2240         for (my $i = 0; $i < scalar (@param); $i++) {
2241             my $p = $param[$i];
2242             next if !$p;
2243             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2244                 my $p = $param[$i+1];
2245                 return 1 if $p && ($p eq $pidfile);
2246                 return undef;
2247             }
2248         }
2249     }
2250     return undef;
2251 }
2252
2253 sub check_running {
2254     my ($vmid, $nocheck, $node) = @_;
2255
2256     my $filename = config_file($vmid, $node);
2257
2258     die "unable to find configuration file for VM $vmid - no such machine\n"
2259         if !$nocheck && ! -f $filename;
2260
2261     my $pidfile = pidfile_name($vmid);
2262
2263     if (my $fd = IO::File->new("<$pidfile")) {
2264         my $st = stat($fd);
2265         my $line = <$fd>;
2266         close($fd);
2267
2268         my $mtime = $st->mtime;
2269         if ($mtime > time()) {
2270             warn "file '$filename' modified in future\n";
2271         }
2272
2273         if ($line =~ m/^(\d+)$/) {
2274             my $pid = $1;
2275             if (check_cmdline($pidfile, $pid)) {
2276                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2277                     return $pid;
2278                 }
2279             }
2280         }
2281     }
2282
2283     return undef;
2284 }
2285
2286 sub vzlist {
2287
2288     my $vzlist = config_list();
2289
2290     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2291
2292     while (defined(my $de = $fd->read)) {
2293         next if $de !~ m/^(\d+)\.pid$/;
2294         my $vmid = $1;
2295         next if !defined($vzlist->{$vmid});
2296         if (my $pid = check_running($vmid)) {
2297             $vzlist->{$vmid}->{pid} = $pid;
2298         }
2299     }
2300
2301     return $vzlist;
2302 }
2303
2304 sub disksize {
2305     my ($storecfg, $conf) = @_;
2306
2307     my $bootdisk = $conf->{bootdisk};
2308     return undef if !$bootdisk;
2309     return undef if !valid_drivename($bootdisk);
2310
2311     return undef if !$conf->{$bootdisk};
2312
2313     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2314     return undef if !defined($drive);
2315
2316     return undef if drive_is_cdrom($drive);
2317
2318     my $volid = $drive->{file};
2319     return undef if !$volid;
2320
2321     return $drive->{size};
2322 }
2323
2324 my $last_proc_pid_stat;
2325
2326 # get VM status information
2327 # This must be fast and should not block ($full == false)
2328 # We only query KVM using QMP if $full == true (this can be slow)
2329 sub vmstatus {
2330     my ($opt_vmid, $full) = @_;
2331
2332     my $res = {};
2333
2334     my $storecfg = PVE::Storage::config();
2335
2336     my $list = vzlist();
2337     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2338
2339     my $cpucount = $cpuinfo->{cpus} || 1;
2340
2341     foreach my $vmid (keys %$list) {
2342         next if $opt_vmid && ($vmid ne $opt_vmid);
2343
2344         my $cfspath = cfs_config_path($vmid);
2345         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2346
2347         my $d = {};
2348         $d->{pid} = $list->{$vmid}->{pid};
2349
2350         # fixme: better status?
2351         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2352
2353         my $size = disksize($storecfg, $conf);
2354         if (defined($size)) {
2355             $d->{disk} = 0; # no info available
2356             $d->{maxdisk} = $size;
2357         } else {
2358             $d->{disk} = 0;
2359             $d->{maxdisk} = 0;
2360         }
2361
2362         $d->{cpus} = ($conf->{sockets} || 1) * ($conf->{cores} || 1);
2363         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2364         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2365
2366         $d->{name} = $conf->{name} || "VM $vmid";
2367         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024) : 0;
2368
2369         if ($conf->{balloon}) {
2370             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2371             $d->{shares} = defined($conf->{shares}) ? $conf->{shares} : 1000;
2372         }
2373
2374         $d->{uptime} = 0;
2375         $d->{cpu} = 0;
2376         $d->{mem} = 0;
2377
2378         $d->{netout} = 0;
2379         $d->{netin} = 0;
2380
2381         $d->{diskread} = 0;
2382         $d->{diskwrite} = 0;
2383
2384         $d->{template} = is_template($conf);
2385
2386         $res->{$vmid} = $d;
2387     }
2388
2389     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2390     foreach my $dev (keys %$netdev) {
2391         next if $dev !~ m/^tap([1-9]\d*)i/;
2392         my $vmid = $1;
2393         my $d = $res->{$vmid};
2394         next if !$d;
2395
2396         $d->{netout} += $netdev->{$dev}->{receive};
2397         $d->{netin} += $netdev->{$dev}->{transmit};
2398
2399         if ($full) {
2400             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2401             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2402         }
2403
2404     }
2405
2406     my $ctime = gettimeofday;
2407
2408     foreach my $vmid (keys %$list) {
2409
2410         my $d = $res->{$vmid};
2411         my $pid = $d->{pid};
2412         next if !$pid;
2413
2414         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2415         next if !$pstat; # not running
2416
2417         my $used = $pstat->{utime} + $pstat->{stime};
2418
2419         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2420
2421         if ($pstat->{vsize}) {
2422             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2423         }
2424
2425         my $old = $last_proc_pid_stat->{$pid};
2426         if (!$old) {
2427             $last_proc_pid_stat->{$pid} = {
2428                 time => $ctime,
2429                 used => $used,
2430                 cpu => 0,
2431             };
2432             next;
2433         }
2434
2435         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2436
2437         if ($dtime > 1000) {
2438             my $dutime = $used -  $old->{used};
2439
2440             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2441             $last_proc_pid_stat->{$pid} = {
2442                 time => $ctime,
2443                 used => $used,
2444                 cpu => $d->{cpu},
2445             };
2446         } else {
2447             $d->{cpu} = $old->{cpu};
2448         }
2449     }
2450
2451     return $res if !$full;
2452
2453     my $qmpclient = PVE::QMPClient->new();
2454
2455     my $ballooncb = sub {
2456         my ($vmid, $resp) = @_;
2457
2458         my $info = $resp->{'return'};
2459         return if !$info->{max_mem};
2460
2461         my $d = $res->{$vmid};
2462
2463         # use memory assigned to VM
2464         $d->{maxmem} = $info->{max_mem};
2465         $d->{balloon} = $info->{actual};
2466
2467         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2468             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2469             $d->{freemem} = $info->{free_mem};
2470         }
2471
2472         $d->{ballooninfo} = $info;
2473     };
2474
2475     my $blockstatscb = sub {
2476         my ($vmid, $resp) = @_;
2477         my $data = $resp->{'return'} || [];
2478         my $totalrdbytes = 0;
2479         my $totalwrbytes = 0;
2480
2481         for my $blockstat (@$data) {
2482             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2483             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2484
2485             $blockstat->{device} =~ s/drive-//;
2486             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2487         }
2488         $res->{$vmid}->{diskread} = $totalrdbytes;
2489         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2490     };
2491
2492     my $statuscb = sub {
2493         my ($vmid, $resp) = @_;
2494
2495         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2496         # this fails if ballon driver is not loaded, so this must be
2497         # the last commnand (following command are aborted if this fails).
2498         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2499
2500         my $status = 'unknown';
2501         if (!defined($status = $resp->{'return'}->{status})) {
2502             warn "unable to get VM status\n";
2503             return;
2504         }
2505
2506         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2507     };
2508
2509     foreach my $vmid (keys %$list) {
2510         next if $opt_vmid && ($vmid ne $opt_vmid);
2511         next if !$res->{$vmid}->{pid}; # not running
2512         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2513     }
2514
2515     $qmpclient->queue_execute(undef, 1);
2516
2517     foreach my $vmid (keys %$list) {
2518         next if $opt_vmid && ($vmid ne $opt_vmid);
2519         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2520     }
2521
2522     return $res;
2523 }
2524
2525 sub foreach_dimm {
2526     my ($conf, $vmid, $memory, $sockets, $func) = @_;
2527
2528     my $dimm_id = 0;
2529     my $current_size = 1024;
2530     my $dimm_size = 512;
2531     return if $current_size == $memory;
2532
2533     for (my $j = 0; $j < 8; $j++) {
2534         for (my $i = 0; $i < 32; $i++) {
2535             my $name = "dimm${dimm_id}";
2536             $dimm_id++;
2537             my $numanode = $i % $sockets;
2538             $current_size += $dimm_size;
2539             &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
2540             return  $current_size if $current_size >= $memory;
2541         }
2542         $dimm_size *= 2;
2543     }
2544 }
2545
2546 sub foreach_reverse_dimm {
2547     my ($conf, $vmid, $memory, $sockets, $func) = @_;
2548
2549     my $dimm_id = 253;
2550     my $current_size = 4177920;
2551     my $dimm_size = 65536;
2552     return if $current_size == $memory;
2553
2554     for (my $j = 0; $j < 8; $j++) {
2555         for (my $i = 0; $i < 32; $i++) {
2556             my $name = "dimm${dimm_id}";
2557             $dimm_id--;
2558             my $numanode = $i % $sockets;
2559             $current_size -= $dimm_size;
2560             &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
2561             return  $current_size if $current_size <= $memory;
2562         }
2563         $dimm_size /= 2;
2564     }
2565 }
2566
2567 sub foreach_drive {
2568     my ($conf, $func) = @_;
2569
2570     foreach my $ds (keys %$conf) {
2571         next if !valid_drivename($ds);
2572
2573         my $drive = parse_drive($ds, $conf->{$ds});
2574         next if !$drive;
2575
2576         &$func($ds, $drive);
2577     }
2578 }
2579
2580 sub foreach_volid {
2581     my ($conf, $func) = @_;
2582
2583     my $volhash = {};
2584
2585     my $test_volid = sub {
2586         my ($volid, $is_cdrom) = @_;
2587
2588         return if !$volid;
2589
2590         $volhash->{$volid} = $is_cdrom || 0;
2591     };
2592
2593     foreach_drive($conf, sub {
2594         my ($ds, $drive) = @_;
2595         &$test_volid($drive->{file}, drive_is_cdrom($drive));
2596     });
2597
2598     foreach my $snapname (keys %{$conf->{snapshots}}) {
2599         my $snap = $conf->{snapshots}->{$snapname};
2600         &$test_volid($snap->{vmstate}, 0);
2601         foreach_drive($snap, sub {
2602             my ($ds, $drive) = @_;
2603             &$test_volid($drive->{file}, drive_is_cdrom($drive));
2604         });
2605     }
2606
2607     foreach my $volid (keys %$volhash) {
2608         &$func($volid, $volhash->{$volid});
2609     }
2610 }
2611
2612 sub vga_conf_has_spice {
2613     my ($vga) = @_;
2614
2615     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
2616
2617     return $1 || 1;
2618 }
2619
2620 sub config_to_command {
2621     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2622
2623     my $cmd = [];
2624     my $globalFlags = [];
2625     my $machineFlags = [];
2626     my $rtcFlags = [];
2627     my $cpuFlags = [];
2628     my $devices = [];
2629     my $pciaddr = '';
2630     my $bridges = {};
2631     my $kvmver = kvm_user_version();
2632     my $vernum = 0; # unknown
2633     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
2634         $vernum = $1*1000000+$2*1000;
2635     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
2636         $vernum = $1*1000000+$2*1000+$3;
2637     }
2638
2639     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
2640
2641     my $have_ovz = -f '/proc/vz/vestat';
2642
2643     my $q35 = machine_type_is_q35($conf);
2644     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2645     my $machine_type = $forcemachine || $conf->{machine};
2646     my $use_old_bios_files = undef;
2647     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
2648
2649     my $cpuunits = defined($conf->{cpuunits}) ?
2650             $conf->{cpuunits} : $defaults->{cpuunits};
2651
2652     push @$cmd, '/usr/bin/systemd-run';
2653     push @$cmd, '--scope';
2654     push @$cmd, '--slice', "qemu";
2655     push @$cmd, '--unit', $vmid;
2656     push @$cmd, '-p', "CPUShares=$cpuunits";
2657     if ($conf->{cpulimit}) {
2658         my $cpulimit = int($conf->{cpulimit} * 100);
2659         push @$cmd, '-p', "CPUQuota=$cpulimit\%";
2660     }
2661
2662     push @$cmd, '/usr/bin/kvm';
2663
2664     push @$cmd, '-id', $vmid;
2665
2666     my $use_virtio = 0;
2667
2668     my $qmpsocket = qmp_socket($vmid);
2669     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2670     push @$cmd, '-mon', "chardev=qmp,mode=control";
2671
2672     my $socket = vnc_socket($vmid);
2673     push @$cmd,  '-vnc', "unix:$socket,x509,password";
2674
2675     push @$cmd, '-pidfile' , pidfile_name($vmid);
2676
2677     push @$cmd, '-daemonize';
2678
2679     if ($conf->{smbios1}) {
2680         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
2681     }
2682
2683     if ($q35) {
2684         # the q35 chipset support native usb2, so we enable usb controller
2685         # by default for this machine type
2686         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
2687     } else {
2688         $pciaddr = print_pci_addr("piix3", $bridges);
2689         push @$devices, '-device', "piix3-usb-uhci,id=uhci$pciaddr.0x2";
2690
2691         my $use_usb2 = 0;
2692         for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2693             next if !$conf->{"usb$i"};
2694             $use_usb2 = 1;
2695         }
2696         # include usb device config
2697         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-usb.cfg' if $use_usb2;
2698     }
2699
2700     my $vga = $conf->{vga};
2701
2702     my $qxlnum = vga_conf_has_spice($vga);
2703     $vga = 'qxl' if $qxlnum;
2704
2705     if (!$vga) {
2706         if ($conf->{ostype} && ($conf->{ostype} eq 'win8' ||
2707                                 $conf->{ostype} eq 'win7' ||
2708                                 $conf->{ostype} eq 'w2k8')) {
2709             $vga = 'std';
2710         } else {
2711             $vga = 'cirrus';
2712         }
2713     }
2714
2715     # enable absolute mouse coordinates (needed by vnc)
2716     my $tablet;
2717     if (defined($conf->{tablet})) {
2718         $tablet = $conf->{tablet};
2719     } else {
2720         $tablet = $defaults->{tablet};
2721         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
2722         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
2723     }
2724
2725     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
2726
2727     # host pci devices
2728     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
2729         my $d = parse_hostpci($conf->{"hostpci$i"});
2730         next if !$d;
2731
2732         my $pcie = $d->{pcie};
2733         if($pcie){
2734             die "q35 machine model is not enabled" if !$q35;
2735             $pciaddr = print_pcie_addr("hostpci$i");
2736         }else{
2737             $pciaddr = print_pci_addr("hostpci$i", $bridges);
2738         }
2739
2740         my $rombar = $d->{rombar} && $d->{rombar} eq 'off' ? ",rombar=0" : "";
2741         my $xvga = $d->{'x-vga'} && $d->{'x-vga'} eq 'on' ? ",x-vga=on" : "";
2742         if ($xvga && $xvga ne '') {
2743             push @$cpuFlags, 'kvm=off';
2744             $vga = 'none';
2745         }
2746         my $pcidevices = $d->{pciid};
2747         my $multifunction = 1 if @$pcidevices > 1;
2748
2749         my $j=0;
2750         foreach my $pcidevice (@$pcidevices) {
2751
2752             my $id = "hostpci$i";
2753             $id .= ".$j" if $multifunction;
2754             my $addr = $pciaddr;
2755             $addr .= ".$j" if $multifunction;
2756             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
2757
2758             if($j == 0){
2759                 $devicestr .= "$rombar$xvga";
2760                 $devicestr .= ",multifunction=on" if $multifunction;
2761             }
2762
2763             push @$devices, '-device', $devicestr;
2764             $j++;
2765         }
2766     }
2767
2768     # usb devices
2769     for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2770         my $d = parse_usb_device($conf->{"usb$i"});
2771         next if !$d;
2772         if ($d->{vendorid} && $d->{productid}) {
2773             push @$devices, '-device', "usb-host,vendorid=0x$d->{vendorid},productid=0x$d->{productid}";
2774         } elsif (defined($d->{hostbus}) && defined($d->{hostport})) {
2775             push @$devices, '-device', "usb-host,hostbus=$d->{hostbus},hostport=$d->{hostport}";
2776         } elsif ($d->{spice}) {
2777             # usb redir support for spice
2778             push @$devices, '-chardev', "spicevmc,id=usbredirchardev$i,name=usbredir";
2779             push @$devices, '-device', "usb-redir,chardev=usbredirchardev$i,id=usbredirdev$i,bus=ehci.0";
2780         }
2781     }
2782
2783     # serial devices
2784     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2785         if (my $path = $conf->{"serial$i"}) {
2786             if ($path eq 'socket') {
2787                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
2788                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
2789                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2790             } else {
2791                 die "no such serial device\n" if ! -c $path;
2792                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
2793                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2794             }
2795         }
2796     }
2797
2798     # parallel devices
2799     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
2800         if (my $path = $conf->{"parallel$i"}) {
2801             die "no such parallel device\n" if ! -c $path;
2802             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
2803             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
2804             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
2805         }
2806     }
2807
2808     my $vmname = $conf->{name} || "vm$vmid";
2809
2810     push @$cmd, '-name', $vmname;
2811
2812     my $sockets = 1;
2813     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
2814     $sockets = $conf->{sockets} if  $conf->{sockets};
2815
2816     my $cores = $conf->{cores} || 1;
2817
2818     my $maxcpus = $sockets * $cores;
2819
2820     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
2821
2822     my $allowed_vcpus = $cpuinfo->{cpus};
2823
2824     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
2825         if ($allowed_vcpus < $maxcpus);
2826
2827     push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
2828
2829     push @$cmd, '-nodefaults';
2830
2831     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2832
2833     my $bootindex_hash = {};
2834     my $i = 1;
2835     foreach my $o (split(//, $bootorder)) {
2836         $bootindex_hash->{$o} = $i*100;
2837         $i++;
2838     }
2839
2840     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000";
2841
2842     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
2843
2844     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
2845
2846     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
2847
2848     # time drift fix
2849     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
2850
2851     my $nokvm = defined($conf->{kvm}) && $conf->{kvm} == 0 ? 1 : 0;
2852     my $useLocaltime = $conf->{localtime};
2853
2854     if (my $ost = $conf->{ostype}) {
2855         # other, wxp, w2k, w2k3, w2k8, wvista, win7, win8, l24, l26, solaris
2856
2857         if ($ost =~ m/^w/) { # windows
2858             $useLocaltime = 1 if !defined($conf->{localtime});
2859
2860             # use time drift fix when acpi is enabled
2861             if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
2862                 $tdf = 1 if !defined($conf->{tdf});
2863             }
2864         }
2865
2866         if ($ost eq 'win7' || $ost eq 'win8' || $ost eq 'w2k8' ||
2867             $ost eq 'wvista') {
2868             push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
2869             push @$cmd, '-no-hpet';
2870             if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
2871                 push @$cpuFlags , 'hv_spinlocks=0x1fff' if !$nokvm;
2872                 push @$cpuFlags , 'hv_vapic' if !$nokvm;
2873                 push @$cpuFlags , 'hv_time' if !$nokvm;
2874
2875             } else {
2876                 push @$cpuFlags , 'hv_spinlocks=0xffff' if !$nokvm;
2877             }
2878         }
2879
2880         if ($ost eq 'win7' || $ost eq 'win8') {
2881             push @$cpuFlags , 'hv_relaxed' if !$nokvm;
2882         }
2883     }
2884
2885     push @$rtcFlags, 'driftfix=slew' if $tdf;
2886
2887     if ($nokvm) {
2888         push @$machineFlags, 'accel=tcg';
2889     } else {
2890         die "No accelerator found!\n" if !$cpuinfo->{hvm};
2891     }
2892
2893     if ($machine_type) {
2894         push @$machineFlags, "type=${machine_type}";
2895     }
2896
2897     if ($conf->{startdate}) {
2898         push @$rtcFlags, "base=$conf->{startdate}";
2899     } elsif ($useLocaltime) {
2900         push @$rtcFlags, 'base=localtime';
2901     }
2902
2903     my $cpu = $nokvm ? "qemu64" : "kvm64";
2904     $cpu = $conf->{cpu} if $conf->{cpu};
2905
2906     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
2907
2908     push @$cpuFlags , '-x2apic'
2909         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
2910
2911     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
2912
2913     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
2914
2915     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
2916
2917         push @$cpuFlags , '+kvm_pv_unhalt' if !$nokvm;
2918         push @$cpuFlags , '+kvm_pv_eoi' if !$nokvm;
2919         push @$cpuFlags , '-kvm_steal_time' if !$nokvm;
2920     }
2921
2922     push @$cpuFlags, 'enforce' if $cpu ne 'host' && !$nokvm;
2923
2924     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
2925
2926     push @$cmd, '-cpu', $cpu;
2927
2928     my $memory = $conf->{memory} || $defaults->{memory};
2929     my $static_memory = 0;
2930     my $dimm_memory = 0;
2931
2932     if ($hotplug_features->{memory}) {
2933         die "Numa need to be enabled for memory hotplug\n" if !$conf->{numa};
2934         die "Total memory is bigger than ${MAX_MEM}MB\n" if $memory > $MAX_MEM;
2935         $static_memory = $STATICMEM;
2936         die "minimum memory must be ${static_memory}MB\n" if($memory < $static_memory);
2937         $dimm_memory = $memory - $static_memory;
2938         push @$cmd, '-m', "size=${static_memory},slots=255,maxmem=${MAX_MEM}M";
2939
2940     } else {
2941
2942         $static_memory = $memory;
2943         push @$cmd, '-m', $static_memory;
2944     }
2945
2946     if ($conf->{numa}) {
2947
2948         my $numa_totalmemory = undef;
2949         for (my $i = 0; $i < $MAX_NUMA; $i++) {
2950             next if !$conf->{"numa$i"};
2951             my $numa = parse_numa($conf->{"numa$i"});
2952             next if !$numa;
2953             # memory
2954             die "missing numa node$i memory value\n" if !$numa->{memory};
2955             my $numa_memory = $numa->{memory};
2956             $numa_totalmemory += $numa_memory;
2957             my $numa_object = "memory-backend-ram,id=ram-node$i,size=${numa_memory}M";
2958
2959             # cpus
2960             my $cpus_start = $numa->{cpus}->{start};
2961             die "missing numa node$i cpus\n" if !defined($cpus_start);
2962             my $cpus_end = $numa->{cpus}->{end} if defined($numa->{cpus}->{end});
2963             my $cpus = $cpus_start;
2964             if (defined($cpus_end)) {
2965                 $cpus .= "-$cpus_end";
2966                 die "numa node$i :  cpu range $cpus is incorrect\n" if $cpus_end <= $cpus_start;
2967             }
2968
2969             # hostnodes
2970             my $hostnodes_start = $numa->{hostnodes}->{start};
2971             if (defined($hostnodes_start)) {
2972                 my $hostnodes_end = $numa->{hostnodes}->{end} if defined($numa->{hostnodes}->{end});
2973                 my $hostnodes = $hostnodes_start;
2974                 if (defined($hostnodes_end)) {
2975                     $hostnodes .= "-$hostnodes_end";
2976                     die "host node $hostnodes range is incorrect\n" if $hostnodes_end <= $hostnodes_start;
2977                 }
2978
2979                 my $hostnodes_end_range = defined($hostnodes_end) ? $hostnodes_end : $hostnodes_start;
2980                 for (my $i = $hostnodes_start; $i <= $hostnodes_end_range; $i++ ) {
2981                     die "host numa node$i don't exist\n" if ! -d "/sys/devices/system/node/node$i/";
2982                 }
2983
2984                 # policy
2985                 my $policy = $numa->{policy};
2986                 die "you need to define a policy for hostnode $hostnodes\n" if !$policy;
2987                 $numa_object .= ",host-nodes=$hostnodes,policy=$policy";
2988             }
2989
2990             push @$cmd, '-object', $numa_object;
2991             push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
2992         }
2993
2994         die "total memory for NUMA nodes must be equal to vm static memory\n"
2995             if $numa_totalmemory && $numa_totalmemory != $static_memory;
2996
2997         #if no custom tology, we split memory and cores across numa nodes
2998         if(!$numa_totalmemory) {
2999
3000             my $numa_memory = ($static_memory / $sockets) . "M";
3001
3002             for (my $i = 0; $i < $sockets; $i++)  {
3003
3004                 my $cpustart = ($cores * $i);
3005                 my $cpuend = ($cpustart + $cores - 1) if $cores && $cores > 1;
3006                 my $cpus = $cpustart;
3007                 $cpus .= "-$cpuend" if $cpuend;
3008
3009                 push @$cmd, '-object', "memory-backend-ram,size=$numa_memory,id=ram-node$i";
3010                 push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
3011             }
3012         }
3013     }
3014
3015     if ($hotplug_features->{memory}) {
3016         foreach_dimm($conf, $vmid, $memory, $sockets, sub {
3017             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3018             push @$cmd, "-object" , "memory-backend-ram,id=mem-$name,size=${dimm_size}M";
3019             push @$cmd, "-device", "pc-dimm,id=$name,memdev=mem-$name,node=$numanode";
3020
3021             #if dimm_memory is not aligned to dimm map
3022             if($current_size > $memory) {
3023                  $conf->{memory} = $current_size;
3024                  update_config_nolock($vmid, $conf, 1);
3025             }
3026         });
3027     }
3028
3029     push @$cmd, '-S' if $conf->{freeze};
3030
3031     # set keyboard layout
3032     my $kb = $conf->{keyboard} || $defaults->{keyboard};
3033     push @$cmd, '-k', $kb if $kb;
3034
3035     # enable sound
3036     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3037     #push @$cmd, '-soundhw', 'es1370';
3038     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3039
3040     if($conf->{agent}) {
3041         my $qgasocket = qmp_socket($vmid, 1);
3042         my $pciaddr = print_pci_addr("qga0", $bridges);
3043         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3044         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3045         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3046     }
3047
3048     my $spice_port;
3049
3050     if ($qxlnum) {
3051         if ($qxlnum > 1) {
3052             if ($conf->{ostype} && $conf->{ostype} =~ m/^w/){
3053                 for(my $i = 1; $i < $qxlnum; $i++){
3054                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3055                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3056                 }
3057             } else {
3058                 # assume other OS works like Linux
3059                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3060                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3061             }
3062         }
3063
3064         my $pciaddr = print_pci_addr("spice", $bridges);
3065
3066         my $nodename = PVE::INotify::nodename();
3067         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3068         $spice_port = PVE::Tools::next_spice_port($pfamily);
3069
3070         push @$devices, '-spice', "tls-port=${spice_port},addr=localhost,tls-ciphers=DES-CBC3-SHA,seamless-migration=on";
3071
3072         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3073         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3074         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3075     }
3076
3077     # enable balloon by default, unless explicitly disabled
3078     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3079         $pciaddr = print_pci_addr("balloon0", $bridges);
3080         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3081     }
3082
3083     if ($conf->{watchdog}) {
3084         my $wdopts = parse_watchdog($conf->{watchdog});
3085         $pciaddr = print_pci_addr("watchdog", $bridges);
3086         my $watchdog = $wdopts->{model} || 'i6300esb';
3087         push @$devices, '-device', "$watchdog$pciaddr";
3088         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3089     }
3090
3091     my $vollist = [];
3092     my $scsicontroller = {};
3093     my $ahcicontroller = {};
3094     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3095
3096     # Add iscsi initiator name if available
3097     if (my $initiator = get_initiator_name()) {
3098         push @$devices, '-iscsi', "initiator-name=$initiator";
3099     }
3100
3101     foreach_drive($conf, sub {
3102         my ($ds, $drive) = @_;
3103
3104         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3105             push @$vollist, $drive->{file};
3106         }
3107
3108         $use_virtio = 1 if $ds =~ m/^virtio/;
3109
3110         if (drive_is_cdrom ($drive)) {
3111             if ($bootindex_hash->{d}) {
3112                 $drive->{bootindex} = $bootindex_hash->{d};
3113                 $bootindex_hash->{d} += 1;
3114             }
3115         } else {
3116             if ($bootindex_hash->{c}) {
3117                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3118                 $bootindex_hash->{c} += 1;
3119             }
3120         }
3121
3122         if($drive->{interface} eq 'virtio'){
3123            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3124         }
3125
3126         if ($drive->{interface} eq 'scsi') {
3127
3128             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3129
3130             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3131             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3132
3133             my $iothread = '';
3134             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3135                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3136                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3137             }
3138
3139             my $queues = '';
3140             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3141                 $queues = ",num_queues=$drive->{queues}";
3142             }
3143
3144             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3145             $scsicontroller->{$controller}=1;
3146         }
3147
3148         if ($drive->{interface} eq 'sata') {
3149            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3150            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3151            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3152            $ahcicontroller->{$controller}=1;
3153         }
3154
3155         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3156         push @$devices, '-drive',$drive_cmd;
3157         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3158     });
3159
3160     for (my $i = 0; $i < $MAX_NETS; $i++) {
3161          next if !$conf->{"net$i"};
3162          my $d = parse_net($conf->{"net$i"});
3163          next if !$d;
3164
3165          $use_virtio = 1 if $d->{model} eq 'virtio';
3166
3167          if ($bootindex_hash->{n}) {
3168             $d->{bootindex} = $bootindex_hash->{n};
3169             $bootindex_hash->{n} += 1;
3170          }
3171
3172          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3173          push @$devices, '-netdev', $netdevfull;
3174
3175          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3176          push @$devices, '-device', $netdevicefull;
3177     }
3178
3179     if (!$q35) {
3180         # add pci bridges
3181         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3182            $bridges->{1} = 1;
3183            $bridges->{2} = 1;
3184         }
3185
3186         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3187
3188         while (my ($k, $v) = each %$bridges) {
3189             $pciaddr = print_pci_addr("pci.$k");
3190             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3191         }
3192     }
3193
3194     # add custom args
3195     if ($conf->{args}) {
3196         my $aa = PVE::Tools::split_args($conf->{args});
3197         push @$cmd, @$aa;
3198     }
3199
3200     push @$cmd, @$devices;
3201     push @$cmd, '-rtc', join(',', @$rtcFlags)
3202         if scalar(@$rtcFlags);
3203     push @$cmd, '-machine', join(',', @$machineFlags)
3204         if scalar(@$machineFlags);
3205     push @$cmd, '-global', join(',', @$globalFlags)
3206         if scalar(@$globalFlags);
3207
3208     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3209 }
3210
3211 sub vnc_socket {
3212     my ($vmid) = @_;
3213     return "${var_run_tmpdir}/$vmid.vnc";
3214 }
3215
3216 sub spice_port {
3217     my ($vmid) = @_;
3218
3219     my $res = vm_mon_cmd($vmid, 'query-spice');
3220
3221     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3222 }
3223
3224 sub qmp_socket {
3225     my ($vmid, $qga) = @_;
3226     my $sockettype = $qga ? 'qga' : 'qmp';
3227     return "${var_run_tmpdir}/$vmid.$sockettype";
3228 }
3229
3230 sub pidfile_name {
3231     my ($vmid) = @_;
3232     return "${var_run_tmpdir}/$vmid.pid";
3233 }
3234
3235 sub vm_devices_list {
3236     my ($vmid) = @_;
3237
3238     my $res = vm_mon_cmd($vmid, 'query-pci');
3239     my $devices = {};
3240     foreach my $pcibus (@$res) {
3241         foreach my $device (@{$pcibus->{devices}}) {
3242             next if !$device->{'qdev_id'};
3243             if ($device->{'pci_bridge'}) {
3244                 $devices->{$device->{'qdev_id'}} = 1;
3245                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3246                     next if !$bridge_device->{'qdev_id'};
3247                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3248                     $devices->{$device->{'qdev_id'}}++;
3249                 }
3250             } else {
3251                 $devices->{$device->{'qdev_id'}} = 1;
3252             }
3253         }
3254     }
3255
3256     my $resblock = vm_mon_cmd($vmid, 'query-block');
3257     foreach my $block (@$resblock) {
3258         if($block->{device} =~ m/^drive-(\S+)/){
3259                 $devices->{$1} = 1;
3260         }
3261     }
3262
3263     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3264     foreach my $mice (@$resmice) {
3265         if ($mice->{name} eq 'QEMU HID Tablet') {
3266             $devices->{tablet} = 1;
3267             last;
3268         }
3269     }
3270
3271     return $devices;
3272 }
3273
3274 sub vm_deviceplug {
3275     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3276
3277     my $q35 = machine_type_is_q35($conf);
3278
3279     my $devices_list = vm_devices_list($vmid);
3280     return 1 if defined($devices_list->{$deviceid});
3281
3282     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3283
3284     if ($deviceid eq 'tablet') {
3285
3286         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3287
3288     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3289
3290         qemu_iothread_add($vmid, $deviceid, $device);
3291
3292         qemu_driveadd($storecfg, $vmid, $device);
3293         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3294
3295         qemu_deviceadd($vmid, $devicefull);
3296         eval { qemu_deviceaddverify($vmid, $deviceid); };
3297         if (my $err = $@) {
3298             eval { qemu_drivedel($vmid, $deviceid); };
3299             warn $@ if $@;
3300             die $err;
3301         }
3302
3303     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3304
3305
3306         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3307         my $pciaddr = print_pci_addr($deviceid);
3308         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3309
3310         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3311
3312         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3313             qemu_iothread_add($vmid, $deviceid, $device);
3314             $devicefull .= ",iothread=iothread-$deviceid";
3315         }
3316
3317         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3318             $devicefull .= ",num_queues=$device->{queues}";
3319         }
3320
3321         qemu_deviceadd($vmid, $devicefull);
3322         qemu_deviceaddverify($vmid, $deviceid);
3323
3324     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3325
3326         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3327         qemu_driveadd($storecfg, $vmid, $device);
3328
3329         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3330         eval { qemu_deviceadd($vmid, $devicefull); };
3331         if (my $err = $@) {
3332             eval { qemu_drivedel($vmid, $deviceid); };
3333             warn $@ if $@;
3334             die $err;
3335         }
3336
3337     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3338
3339         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3340         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid);
3341         qemu_deviceadd($vmid, $netdevicefull);
3342         eval { qemu_deviceaddverify($vmid, $deviceid); };
3343         if (my $err = $@) {
3344             eval { qemu_netdevdel($vmid, $deviceid); };
3345             warn $@ if $@;
3346             die $err;
3347         }
3348
3349     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3350
3351         my $bridgeid = $2;
3352         my $pciaddr = print_pci_addr($deviceid);
3353         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3354
3355         qemu_deviceadd($vmid, $devicefull);
3356         qemu_deviceaddverify($vmid, $deviceid);
3357
3358     } else {
3359         die "can't hotplug device '$deviceid'\n";
3360     }
3361
3362     return 1;
3363 }
3364
3365 # fixme: this should raise exceptions on error!
3366 sub vm_deviceunplug {
3367     my ($vmid, $conf, $deviceid) = @_;
3368
3369     my $devices_list = vm_devices_list($vmid);
3370     return 1 if !defined($devices_list->{$deviceid});
3371
3372     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3373
3374     if ($deviceid eq 'tablet') {
3375
3376         qemu_devicedel($vmid, $deviceid);
3377
3378     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3379
3380         qemu_devicedel($vmid, $deviceid);
3381         qemu_devicedelverify($vmid, $deviceid);
3382         qemu_drivedel($vmid, $deviceid);
3383         qemu_iothread_del($conf, $vmid, $deviceid);
3384
3385     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3386
3387         qemu_devicedel($vmid, $deviceid);
3388         qemu_devicedelverify($vmid, $deviceid);
3389         qemu_iothread_del($conf, $vmid, $deviceid);
3390
3391     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3392
3393         #qemu 2.3 segfault on drive_del with virtioscsi + iothread
3394         my $device = parse_drive($deviceid, $conf->{$deviceid});
3395         die "virtioscsi with iothread is not hot-unplugglable currently" if $device->{iothread};
3396
3397         qemu_devicedel($vmid, $deviceid);
3398         qemu_drivedel($vmid, $deviceid);
3399         qemu_deletescsihw($conf, $vmid, $deviceid);
3400
3401     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3402
3403         qemu_devicedel($vmid, $deviceid);
3404         qemu_devicedelverify($vmid, $deviceid);
3405         qemu_netdevdel($vmid, $deviceid);
3406
3407     } else {
3408         die "can't unplug device '$deviceid'\n";
3409     }
3410
3411     return 1;
3412 }
3413
3414 sub qemu_deviceadd {
3415     my ($vmid, $devicefull) = @_;
3416
3417     $devicefull = "driver=".$devicefull;
3418     my %options =  split(/[=,]/, $devicefull);
3419
3420     vm_mon_cmd($vmid, "device_add" , %options);
3421 }
3422
3423 sub qemu_devicedel {
3424     my ($vmid, $deviceid) = @_;
3425
3426     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3427 }
3428
3429 sub qemu_iothread_add {
3430     my($vmid, $deviceid, $device) = @_;
3431
3432     if ($device->{iothread}) {
3433         my $iothreads = vm_iothreads_list($vmid);
3434         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3435     }
3436 }
3437
3438 sub qemu_iothread_del {
3439     my($conf, $vmid, $deviceid) = @_;
3440
3441     my $device = parse_drive($deviceid, $conf->{$deviceid});
3442     if ($device->{iothread}) {
3443         my $iothreads = vm_iothreads_list($vmid);
3444         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3445     }
3446 }
3447
3448 sub qemu_objectadd {
3449     my($vmid, $objectid, $qomtype) = @_;
3450
3451     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3452
3453     return 1;
3454 }
3455
3456 sub qemu_objectdel {
3457     my($vmid, $objectid) = @_;
3458
3459     vm_mon_cmd($vmid, "object-del", id => $objectid);
3460
3461     return 1;
3462 }
3463
3464 sub qemu_driveadd {
3465     my ($storecfg, $vmid, $device) = @_;
3466
3467     my $drive = print_drive_full($storecfg, $vmid, $device);
3468     $drive =~ s/\\/\\\\/g;
3469     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3470
3471     # If the command succeeds qemu prints: "OK"
3472     return 1 if $ret =~ m/OK/s;
3473
3474     die "adding drive failed: $ret\n";
3475 }
3476
3477 sub qemu_drivedel {
3478     my($vmid, $deviceid) = @_;
3479
3480     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3481     $ret =~ s/^\s+//;
3482
3483     return 1 if $ret eq "";
3484
3485     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3486     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3487
3488     die "deleting drive $deviceid failed : $ret\n";
3489 }
3490
3491 sub qemu_deviceaddverify {
3492     my ($vmid, $deviceid) = @_;
3493
3494     for (my $i = 0; $i <= 5; $i++) {
3495          my $devices_list = vm_devices_list($vmid);
3496          return 1 if defined($devices_list->{$deviceid});
3497          sleep 1;
3498     }
3499
3500     die "error on hotplug device '$deviceid'\n";
3501 }
3502
3503
3504 sub qemu_devicedelverify {
3505     my ($vmid, $deviceid) = @_;
3506
3507     # need to verify that the device is correctly removed as device_del
3508     # is async and empty return is not reliable
3509
3510     for (my $i = 0; $i <= 5; $i++) {
3511          my $devices_list = vm_devices_list($vmid);
3512          return 1 if !defined($devices_list->{$deviceid});
3513          sleep 1;
3514     }
3515
3516     die "error on hot-unplugging device '$deviceid'\n";
3517 }
3518
3519 sub qemu_findorcreatescsihw {
3520     my ($storecfg, $conf, $vmid, $device) = @_;
3521
3522     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3523
3524     my $scsihwid="$controller_prefix$controller";
3525     my $devices_list = vm_devices_list($vmid);
3526
3527     if(!defined($devices_list->{$scsihwid})) {
3528         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3529     }
3530
3531     return 1;
3532 }
3533
3534 sub qemu_deletescsihw {
3535     my ($conf, $vmid, $opt) = @_;
3536
3537     my $device = parse_drive($opt, $conf->{$opt});
3538
3539     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3540         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3541         return 1;
3542     }
3543
3544     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3545
3546     my $devices_list = vm_devices_list($vmid);
3547     foreach my $opt (keys %{$devices_list}) {
3548         if (PVE::QemuServer::valid_drivename($opt)) {
3549             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3550             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3551                 return 1;
3552             }
3553         }
3554     }
3555
3556     my $scsihwid="scsihw$controller";
3557
3558     vm_deviceunplug($vmid, $conf, $scsihwid);
3559
3560     return 1;
3561 }
3562
3563 sub qemu_add_pci_bridge {
3564     my ($storecfg, $conf, $vmid, $device) = @_;
3565
3566     my $bridges = {};
3567
3568     my $bridgeid;
3569
3570     print_pci_addr($device, $bridges);
3571
3572     while (my ($k, $v) = each %$bridges) {
3573         $bridgeid = $k;
3574     }
3575     return 1 if !defined($bridgeid) || $bridgeid < 1;
3576
3577     my $bridge = "pci.$bridgeid";
3578     my $devices_list = vm_devices_list($vmid);
3579
3580     if (!defined($devices_list->{$bridge})) {
3581         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3582     }
3583
3584     return 1;
3585 }
3586
3587 sub qemu_set_link_status {
3588     my ($vmid, $device, $up) = @_;
3589
3590     vm_mon_cmd($vmid, "set_link", name => $device,
3591                up => $up ? JSON::true : JSON::false);
3592 }
3593
3594 sub qemu_netdevadd {
3595     my ($vmid, $conf, $device, $deviceid) = @_;
3596
3597     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid);
3598     my %options =  split(/[=,]/, $netdev);
3599
3600     vm_mon_cmd($vmid, "netdev_add",  %options);
3601     return 1;
3602 }
3603
3604 sub qemu_netdevdel {
3605     my ($vmid, $deviceid) = @_;
3606
3607     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3608 }
3609
3610 sub qemu_cpu_hotplug {
3611     my ($vmid, $conf, $vcpus) = @_;
3612
3613     my $sockets = 1;
3614     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3615     $sockets = $conf->{sockets} if  $conf->{sockets};
3616     my $cores = $conf->{cores} || 1;
3617     my $maxcpus = $sockets * $cores;
3618
3619     $vcpus = $maxcpus if !$vcpus;
3620
3621     die "you can't add more vcpus than maxcpus\n"
3622         if $vcpus > $maxcpus;
3623
3624     my $currentvcpus = $conf->{vcpus} || $maxcpus;
3625     die "online cpu unplug is not yet possible\n"
3626         if $vcpus < $currentvcpus;
3627
3628     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3629     die "vcpus in running vm is different than configuration\n"
3630         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
3631
3632     for (my $i = $currentvcpus; $i < $vcpus; $i++) {
3633         vm_mon_cmd($vmid, "cpu-add", id => int($i));
3634     }
3635 }
3636
3637 sub qemu_memory_hotplug {
3638     my ($vmid, $conf, $defaults, $opt, $value) = @_;
3639
3640     return $value if !check_running($vmid);
3641
3642     my $memory = $conf->{memory} || $defaults->{memory};
3643     $value = $defaults->{memory} if !$value;
3644     return $value if $value == $memory;
3645
3646     my $static_memory = $STATICMEM;
3647     my $dimm_memory = $memory - $static_memory;
3648
3649     die "memory can't be lower than $static_memory MB" if $value < $static_memory;
3650     die "you cannot add more memory than $MAX_MEM MB!\n" if $memory > $MAX_MEM;
3651
3652
3653     my $sockets = 1;
3654     $sockets = $conf->{sockets} if $conf->{sockets};
3655
3656     if($value > $memory) {
3657
3658         foreach_dimm($conf, $vmid, $value, $sockets, sub {
3659             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3660
3661                 return if $current_size <= $conf->{memory};
3662
3663                 eval { vm_mon_cmd($vmid, "object-add", 'qom-type' => "memory-backend-ram", id => "mem-$name", props => { size => int($dimm_size*1024*1024) } ) };
3664                 if (my $err = $@) {
3665                     eval { qemu_objectdel($vmid, "mem-$name"); };
3666                     die $err;
3667                 }
3668
3669                 eval { vm_mon_cmd($vmid, "device_add", driver => "pc-dimm", id => "$name", memdev => "mem-$name", node => $numanode) };
3670                 if (my $err = $@) {
3671                     eval { qemu_objectdel($vmid, "mem-$name"); };
3672                     die $err;
3673                 }
3674                 #update conf after each succesful module hotplug
3675                 $conf->{memory} = $current_size;
3676                 update_config_nolock($vmid, $conf, 1);
3677         });
3678
3679     } else {
3680
3681         foreach_reverse_dimm($conf, $vmid, $value, $sockets, sub {
3682             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3683
3684                 return if $current_size >= $conf->{memory};
3685                 print "try to unplug memory dimm $name\n";
3686
3687                 my $retry = 0;
3688                 while (1) {
3689                     eval { qemu_devicedel($vmid, $name) };
3690                     sleep 3;
3691                     my $dimm_list = qemu_dimm_list($vmid);
3692                     last if !$dimm_list->{$name};
3693                     raise_param_exc({ $name => "error unplug memory module" }) if $retry > 5;
3694                     $retry++;
3695                 }
3696
3697                 #update conf after each succesful module unplug
3698                 $conf->{memory} = $current_size;
3699
3700                 eval { qemu_objectdel($vmid, "mem-$name"); };
3701                 update_config_nolock($vmid, $conf, 1);
3702         });
3703     }
3704 }
3705
3706 sub qemu_dimm_list {
3707     my ($vmid) = @_;
3708
3709     my $dimmarray = vm_mon_cmd_nocheck($vmid, "query-memory-devices");
3710     my $dimms = {};
3711
3712     foreach my $dimm (@$dimmarray) {
3713
3714         $dimms->{$dimm->{data}->{id}}->{id} = $dimm->{data}->{id};
3715         $dimms->{$dimm->{data}->{id}}->{node} = $dimm->{data}->{node};
3716         $dimms->{$dimm->{data}->{id}}->{addr} = $dimm->{data}->{addr};
3717         $dimms->{$dimm->{data}->{id}}->{size} = $dimm->{data}->{size};
3718         $dimms->{$dimm->{data}->{id}}->{slot} = $dimm->{data}->{slot};
3719     }
3720     return $dimms;
3721 }
3722
3723 sub qemu_block_set_io_throttle {
3724     my ($vmid, $deviceid, $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr) = @_;
3725
3726     return if !check_running($vmid) ;
3727
3728     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid, bps => int($bps), bps_rd => int($bps_rd), bps_wr => int($bps_wr), iops => int($iops), iops_rd => int($iops_rd), iops_wr => int($iops_wr));
3729
3730 }
3731
3732 # old code, only used to shutdown old VM after update
3733 sub __read_avail {
3734     my ($fh, $timeout) = @_;
3735
3736     my $sel = new IO::Select;
3737     $sel->add($fh);
3738
3739     my $res = '';
3740     my $buf;
3741
3742     my @ready;
3743     while (scalar (@ready = $sel->can_read($timeout))) {
3744         my $count;
3745         if ($count = $fh->sysread($buf, 8192)) {
3746             if ($buf =~ /^(.*)\(qemu\) $/s) {
3747                 $res .= $1;
3748                 last;
3749             } else {
3750                 $res .= $buf;
3751             }
3752         } else {
3753             if (!defined($count)) {
3754                 die "$!\n";
3755             }
3756             last;
3757         }
3758     }
3759
3760     die "monitor read timeout\n" if !scalar(@ready);
3761
3762     return $res;
3763 }
3764
3765 # old code, only used to shutdown old VM after update
3766 sub vm_monitor_command {
3767     my ($vmid, $cmdstr, $nocheck) = @_;
3768
3769     my $res;
3770
3771     eval {
3772         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
3773
3774         my $sname = "${var_run_tmpdir}/$vmid.mon";
3775
3776         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
3777             die "unable to connect to VM $vmid socket - $!\n";
3778
3779         my $timeout = 3;
3780
3781         # hack: migrate sometime blocks the monitor (when migrate_downtime
3782         # is set)
3783         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3784             $timeout = 60*60; # 1 hour
3785         }
3786
3787         # read banner;
3788         my $data = __read_avail($sock, $timeout);
3789
3790         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
3791             die "got unexpected qemu monitor banner\n";
3792         }
3793
3794         my $sel = new IO::Select;
3795         $sel->add($sock);
3796
3797         if (!scalar(my @ready = $sel->can_write($timeout))) {
3798             die "monitor write error - timeout";
3799         }
3800
3801         my $fullcmd = "$cmdstr\r";
3802
3803         # syslog('info', "VM $vmid monitor command: $cmdstr");
3804
3805         my $b;
3806         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
3807             die "monitor write error - $!";
3808         }
3809
3810         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
3811
3812         $timeout = 20;
3813
3814         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3815             $timeout = 60*60; # 1 hour
3816         } elsif ($cmdstr =~ m/^(eject|change)/) {
3817             $timeout = 60; # note: cdrom mount command is slow
3818         }
3819         if ($res = __read_avail($sock, $timeout)) {
3820
3821             my @lines = split("\r?\n", $res);
3822
3823             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
3824
3825             $res = join("\n", @lines);
3826             $res .= "\n";
3827         }
3828     };
3829
3830     my $err = $@;
3831
3832     if ($err) {
3833         syslog("err", "VM $vmid monitor command failed - $err");
3834         die $err;
3835     }
3836
3837     return $res;
3838 }
3839
3840 sub qemu_block_resize {
3841     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
3842
3843     my $running = check_running($vmid);
3844
3845     return if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
3846
3847     return if !$running;
3848
3849     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
3850
3851 }
3852
3853 sub qemu_volume_snapshot {
3854     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3855
3856     my $running = check_running($vmid);
3857
3858     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
3859         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
3860     } else {
3861         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
3862     }
3863 }
3864
3865 sub qemu_volume_snapshot_delete {
3866     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3867
3868     my $running = check_running($vmid);
3869
3870     return if !PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
3871
3872     return if !$running;
3873
3874     vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
3875 }
3876
3877 sub set_migration_caps {
3878     my ($vmid) = @_;
3879
3880     my $cap_ref = [];
3881
3882     my $enabled_cap = {
3883         "auto-converge" => 1,
3884         "xbzrle" => 1,
3885         "x-rdma-pin-all" => 0,
3886         "zero-blocks" => 0,
3887         "compress" => 0
3888     };
3889
3890     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
3891
3892     for my $supported_capability (@$supported_capabilities) {
3893         push @$cap_ref, {
3894             capability => $supported_capability->{capability},
3895             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
3896         };
3897     }
3898
3899     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
3900 }
3901
3902 my $fast_plug_option = {
3903     'lock' => 1,
3904     'name' => 1,
3905     'onboot' => 1,
3906     'shares' => 1,
3907     'startup' => 1,
3908     'description' => 1,
3909 };
3910
3911 # hotplug changes in [PENDING]
3912 # $selection hash can be used to only apply specified options, for
3913 # example: { cores => 1 } (only apply changed 'cores')
3914 # $errors ref is used to return error messages
3915 sub vmconfig_hotplug_pending {
3916     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
3917
3918     my $defaults = load_defaults();
3919
3920     # commit values which do not have any impact on running VM first
3921     # Note: those option cannot raise errors, we we do not care about
3922     # $selection and always apply them.
3923
3924     my $add_error = sub {
3925         my ($opt, $msg) = @_;
3926         $errors->{$opt} = "hotplug problem - $msg";
3927     };
3928
3929     my $changes = 0;
3930     foreach my $opt (keys %{$conf->{pending}}) { # add/change
3931         if ($fast_plug_option->{$opt}) {
3932             $conf->{$opt} = $conf->{pending}->{$opt};
3933             delete $conf->{pending}->{$opt};
3934             $changes = 1;
3935         }
3936     }
3937
3938     if ($changes) {
3939         update_config_nolock($vmid, $conf, 1);
3940         $conf = load_config($vmid); # update/reload
3941     }
3942
3943     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3944
3945     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
3946     while (my ($opt, $force) = each %$pending_delete_hash) {
3947         next if $selection && !$selection->{$opt};
3948         eval {
3949             if ($opt eq 'hotplug') {
3950                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
3951             } elsif ($opt eq 'tablet') {
3952                 die "skip\n" if !$hotplug_features->{usb};
3953                 if ($defaults->{tablet}) {
3954                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
3955                 } else {
3956                     vm_deviceunplug($vmid, $conf, $opt);
3957                 }
3958             } elsif ($opt eq 'vcpus') {
3959                 die "skip\n" if !$hotplug_features->{cpu};
3960                 qemu_cpu_hotplug($vmid, $conf, undef);
3961             } elsif ($opt eq 'balloon') {
3962                 # enable balloon device is not hotpluggable
3963                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
3964             } elsif ($fast_plug_option->{$opt}) {
3965                 # do nothing
3966             } elsif ($opt =~ m/^net(\d+)$/) {
3967                 die "skip\n" if !$hotplug_features->{network};
3968                 vm_deviceunplug($vmid, $conf, $opt);
3969             } elsif (valid_drivename($opt)) {
3970                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
3971                 vm_deviceunplug($vmid, $conf, $opt);
3972                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
3973             } elsif ($opt =~ m/^memory$/) {
3974                 die "skip\n" if !$hotplug_features->{memory};
3975                 qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
3976             } elsif ($opt eq 'cpuunits') {
3977                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
3978             } elsif ($opt eq 'cpulimit') {
3979                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
3980             } else {
3981                 die "skip\n";
3982             }
3983         };
3984         if (my $err = $@) {
3985             &$add_error($opt, $err) if $err ne "skip\n";
3986         } else {
3987             # save new config if hotplug was successful
3988             delete $conf->{$opt};
3989             vmconfig_undelete_pending_option($conf, $opt);
3990             update_config_nolock($vmid, $conf, 1);
3991             $conf = load_config($vmid); # update/reload
3992         }
3993     }
3994
3995     foreach my $opt (keys %{$conf->{pending}}) {
3996         next if $selection && !$selection->{$opt};
3997         my $value = $conf->{pending}->{$opt};
3998         eval {
3999             if ($opt eq 'hotplug') {
4000                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4001             } elsif ($opt eq 'tablet') {
4002                 die "skip\n" if !$hotplug_features->{usb};
4003                 if ($value == 1) {
4004                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4005                 } elsif ($value == 0) {
4006                     vm_deviceunplug($vmid, $conf, $opt);
4007                 }
4008             } elsif ($opt eq 'vcpus') {
4009                 die "skip\n" if !$hotplug_features->{cpu};
4010                 qemu_cpu_hotplug($vmid, $conf, $value);
4011             } elsif ($opt eq 'balloon') {
4012                 # enable/disable balloning device is not hotpluggable
4013                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4014                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4015                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4016
4017                 # allow manual ballooning if shares is set to zero
4018                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4019                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4020                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4021                 }
4022             } elsif ($opt =~ m/^net(\d+)$/) {
4023                 # some changes can be done without hotplug
4024                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4025                                     $vmid, $opt, $value);
4026             } elsif (valid_drivename($opt)) {
4027                 # some changes can be done without hotplug
4028                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4029                                      $vmid, $opt, $value, 1);
4030             } elsif ($opt =~ m/^memory$/) { #dimms
4031                 die "skip\n" if !$hotplug_features->{memory};
4032                 $value = qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4033             } elsif ($opt eq 'cpuunits') {
4034                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4035             } elsif ($opt eq 'cpulimit') {
4036                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4037                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4038             } else {
4039                 die "skip\n";  # skip non-hot-pluggable options
4040             }
4041         };
4042         if (my $err = $@) {
4043             &$add_error($opt, $err) if $err ne "skip\n";
4044         } else {
4045             # save new config if hotplug was successful
4046             $conf->{$opt} = $value;
4047             delete $conf->{pending}->{$opt};
4048             update_config_nolock($vmid, $conf, 1);
4049             $conf = load_config($vmid); # update/reload
4050         }
4051     }
4052 }
4053
4054 sub try_deallocate_drive {
4055     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4056
4057     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4058         my $volid = $drive->{file};
4059         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4060             my $sid = PVE::Storage::parse_volume_id($volid);
4061             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4062
4063             # check if the disk is really unused
4064             my $used_paths = PVE::QemuServer::get_used_paths($vmid, $storecfg, $conf, 1, $key);
4065             my $path = PVE::Storage::path($storecfg, $volid);
4066             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4067                    if $used_paths->{$path};
4068             PVE::Storage::vdisk_free($storecfg, $volid);
4069             return 1;
4070         } else {
4071             # If vm is not owner of this disk remove from config
4072             return 1;
4073         }
4074     }
4075
4076     return undef;
4077 }
4078
4079 sub vmconfig_delete_or_detach_drive {
4080     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4081
4082     my $drive = parse_drive($opt, $conf->{$opt});
4083
4084     my $rpcenv = PVE::RPCEnvironment::get();
4085     my $authuser = $rpcenv->get_user();
4086
4087     if ($force) {
4088         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4089         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4090     } else {
4091         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4092     }
4093 }
4094
4095 sub vmconfig_apply_pending {
4096     my ($vmid, $conf, $storecfg) = @_;
4097
4098     # cold plug
4099
4100     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4101     while (my ($opt, $force) = each %$pending_delete_hash) {
4102         die "internal error" if $opt =~ m/^unused/;
4103         $conf = load_config($vmid); # update/reload
4104         if (!defined($conf->{$opt})) {
4105             vmconfig_undelete_pending_option($conf, $opt);
4106             update_config_nolock($vmid, $conf, 1);
4107         } elsif (valid_drivename($opt)) {
4108             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4109             vmconfig_undelete_pending_option($conf, $opt);
4110             delete $conf->{$opt};
4111             update_config_nolock($vmid, $conf, 1);
4112         } else {
4113             vmconfig_undelete_pending_option($conf, $opt);
4114             delete $conf->{$opt};
4115             update_config_nolock($vmid, $conf, 1);
4116         }
4117     }
4118
4119     $conf = load_config($vmid); # update/reload
4120
4121     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4122         $conf = load_config($vmid); # update/reload
4123
4124         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4125             # skip if nothing changed
4126         } elsif (valid_drivename($opt)) {
4127             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4128                 if defined($conf->{$opt});
4129             $conf->{$opt} = $conf->{pending}->{$opt};
4130         } else {
4131             $conf->{$opt} = $conf->{pending}->{$opt};
4132         }
4133
4134         delete $conf->{pending}->{$opt};
4135         update_config_nolock($vmid, $conf, 1);
4136     }
4137 }
4138
4139 my $safe_num_ne = sub {
4140     my ($a, $b) = @_;
4141
4142     return 0 if !defined($a) && !defined($b);
4143     return 1 if !defined($a);
4144     return 1 if !defined($b);
4145
4146     return $a != $b;
4147 };
4148
4149 my $safe_string_ne = sub {
4150     my ($a, $b) = @_;
4151
4152     return 0 if !defined($a) && !defined($b);
4153     return 1 if !defined($a);
4154     return 1 if !defined($b);
4155
4156     return $a ne $b;
4157 };
4158
4159 sub vmconfig_update_net {
4160     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4161
4162     my $newnet = parse_net($value);
4163
4164     if ($conf->{$opt}) {
4165         my $oldnet = parse_net($conf->{$opt});
4166
4167         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4168             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4169             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4170             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4171
4172             # for non online change, we try to hot-unplug
4173             die "skip\n" if !$hotplug;
4174             vm_deviceunplug($vmid, $conf, $opt);
4175         } else {
4176
4177             die "internal error" if $opt !~ m/net(\d+)/;
4178             my $iface = "tap${vmid}i$1";
4179
4180             if (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4181                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4182             }
4183
4184             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4185                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4186                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4187                 PVE::Network::tap_unplug($iface);
4188                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall});
4189             }
4190
4191             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4192                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4193             }
4194
4195             return 1;
4196         }
4197     }
4198
4199     if ($hotplug) {
4200         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4201     } else {
4202         die "skip\n";
4203     }
4204 }
4205
4206 sub vmconfig_update_disk {
4207     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4208
4209     # fixme: do we need force?
4210
4211     my $drive = parse_drive($opt, $value);
4212
4213     if ($conf->{$opt}) {
4214
4215         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4216
4217             my $media = $drive->{media} || 'disk';
4218             my $oldmedia = $old_drive->{media} || 'disk';
4219             die "unable to change media type\n" if $media ne $oldmedia;
4220
4221             if (!drive_is_cdrom($old_drive)) {
4222
4223                 if ($drive->{file} ne $old_drive->{file}) {
4224
4225                     die "skip\n" if !$hotplug;
4226
4227                     # unplug and register as unused
4228                     vm_deviceunplug($vmid, $conf, $opt);
4229                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4230
4231                 } else {
4232                     # update existing disk
4233
4234                     # skip non hotpluggable value
4235                     if (&$safe_num_ne($drive->{discard}, $old_drive->{discard}) ||
4236                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4237                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4238                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4239                         die "skip\n";
4240                     }
4241
4242                     # apply throttle
4243                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4244                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4245                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4246                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4247                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4248                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4249                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4250                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4251                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4252                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4253                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4254                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max})) {
4255
4256                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4257                                                    ($drive->{mbps} || 0)*1024*1024,
4258                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4259                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4260                                                    $drive->{iops} || 0,
4261                                                    $drive->{iops_rd} || 0,
4262                                                    $drive->{iops_wr} || 0,
4263                                                    ($drive->{mbps_max} || 0)*1024*1024,
4264                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4265                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4266                                                    $drive->{iops_max} || 0,
4267                                                    $drive->{iops_rd_max} || 0,
4268                                                    $drive->{iops_wr_max} || 0);
4269
4270                     }
4271
4272                     return 1;
4273                 }
4274
4275             } else { # cdrom
4276
4277                 if ($drive->{file} eq 'none') {
4278                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4279                 } else {
4280                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4281                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4282                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4283                 }
4284
4285                 return 1;
4286             }
4287         }
4288     }
4289
4290     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4291     # hotplug new disks
4292     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4293 }
4294
4295 sub vm_start {
4296     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4297         $forcemachine, $spice_ticket) = @_;
4298
4299     lock_config($vmid, sub {
4300         my $conf = load_config($vmid, $migratedfrom);
4301
4302         die "you can't start a vm if it's a template\n" if is_template($conf);
4303
4304         check_lock($conf) if !$skiplock;
4305
4306         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4307
4308         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4309             vmconfig_apply_pending($vmid, $conf, $storecfg);
4310             $conf = load_config($vmid); # update/reload
4311         }
4312
4313         my $defaults = load_defaults();
4314
4315         # set environment variable useful inside network script
4316         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4317
4318         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4319
4320         my $migrate_port = 0;
4321         my $migrate_uri;
4322         if ($statefile) {
4323             if ($statefile eq 'tcp') {
4324                 my $localip = "localhost";
4325                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4326                 my $nodename = PVE::INotify::nodename();
4327                 if ($datacenterconf->{migration_unsecure}) {
4328                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4329                         $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4330                 }
4331                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4332                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4333                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4334                 push @$cmd, '-incoming', $migrate_uri;
4335                 push @$cmd, '-S';
4336             } else {
4337                 push @$cmd, '-loadstate', $statefile;
4338             }
4339         } elsif ($paused) {
4340             push @$cmd, '-S';
4341         }
4342
4343         # host pci devices
4344         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4345           my $d = parse_hostpci($conf->{"hostpci$i"});
4346           next if !$d;
4347           my $pcidevices = $d->{pciid};
4348           foreach my $pcidevice (@$pcidevices) {
4349                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4350
4351                 my $info = pci_device_info("0000:$pciid");
4352                 die "IOMMU not present\n" if !check_iommu_support();
4353                 die "no pci device info for device '$pciid'\n" if !$info;
4354                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4355                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4356           }
4357         }
4358
4359         PVE::Storage::activate_volumes($storecfg, $vollist);
4360
4361         eval  { run_command($cmd, timeout => $statefile ? undef : 30,
4362                     umask => 0077); };
4363         my $err = $@;
4364         die "start failed: $err" if $err;
4365
4366         print "migration listens on $migrate_uri\n" if $migrate_uri;
4367
4368         if ($statefile && $statefile ne 'tcp')  {
4369             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4370             warn $@ if $@;
4371         }
4372
4373         if ($migratedfrom) {
4374
4375             eval {
4376                 set_migration_caps($vmid);
4377             };
4378             warn $@ if $@;
4379
4380             if ($spice_port) {
4381                 print "spice listens on port $spice_port\n";
4382                 if ($spice_ticket) {
4383                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4384                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4385                 }
4386             }
4387
4388         } else {
4389
4390             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4391                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4392                     if $conf->{balloon};
4393             }
4394
4395             foreach my $opt (keys %$conf) {
4396                 next if $opt !~  m/^net\d+$/;
4397                 my $nicconf = parse_net($conf->{$opt});
4398                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4399             }
4400         }
4401
4402         vm_mon_cmd_nocheck($vmid, 'qom-set',
4403                     path => "machine/peripheral/balloon0",
4404                     property => "guest-stats-polling-interval",
4405                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4406
4407     });
4408 }
4409
4410 sub vm_mon_cmd {
4411     my ($vmid, $execute, %params) = @_;
4412
4413     my $cmd = { execute => $execute, arguments => \%params };
4414     vm_qmp_command($vmid, $cmd);
4415 }
4416
4417 sub vm_mon_cmd_nocheck {
4418     my ($vmid, $execute, %params) = @_;
4419
4420     my $cmd = { execute => $execute, arguments => \%params };
4421     vm_qmp_command($vmid, $cmd, 1);
4422 }
4423
4424 sub vm_qmp_command {
4425     my ($vmid, $cmd, $nocheck) = @_;
4426
4427     my $res;
4428
4429     my $timeout;
4430     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4431         $timeout = $cmd->{arguments}->{timeout};
4432         delete $cmd->{arguments}->{timeout};
4433     }
4434
4435     eval {
4436         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4437         my $sname = qmp_socket($vmid);
4438         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4439             my $qmpclient = PVE::QMPClient->new();
4440
4441             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4442         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
4443             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
4444                 if scalar(%{$cmd->{arguments}});
4445             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
4446         } else {
4447             die "unable to open monitor socket\n";
4448         }
4449     };
4450     if (my $err = $@) {
4451         syslog("err", "VM $vmid qmp command failed - $err");
4452         die $err;
4453     }
4454
4455     return $res;
4456 }
4457
4458 sub vm_human_monitor_command {
4459     my ($vmid, $cmdline) = @_;
4460
4461     my $res;
4462
4463     my $cmd = {
4464         execute => 'human-monitor-command',
4465         arguments => { 'command-line' => $cmdline},
4466     };
4467
4468     return vm_qmp_command($vmid, $cmd);
4469 }
4470
4471 sub vm_commandline {
4472     my ($storecfg, $vmid) = @_;
4473
4474     my $conf = load_config($vmid);
4475
4476     my $defaults = load_defaults();
4477
4478     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
4479
4480     return join(' ', @$cmd);
4481 }
4482
4483 sub vm_reset {
4484     my ($vmid, $skiplock) = @_;
4485
4486     lock_config($vmid, sub {
4487
4488         my $conf = load_config($vmid);
4489
4490         check_lock($conf) if !$skiplock;
4491
4492         vm_mon_cmd($vmid, "system_reset");
4493     });
4494 }
4495
4496 sub get_vm_volumes {
4497     my ($conf) = @_;
4498
4499     my $vollist = [];
4500     foreach_volid($conf, sub {
4501         my ($volid, $is_cdrom) = @_;
4502
4503         return if $volid =~ m|^/|;
4504
4505         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4506         return if !$sid;
4507
4508         push @$vollist, $volid;
4509     });
4510
4511     return $vollist;
4512 }
4513
4514 sub vm_stop_cleanup {
4515     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
4516
4517     eval {
4518
4519         if (!$keepActive) {
4520             my $vollist = get_vm_volumes($conf);
4521             PVE::Storage::deactivate_volumes($storecfg, $vollist);
4522         }
4523
4524         foreach my $ext (qw(mon qmp pid vnc qga)) {
4525             unlink "/var/run/qemu-server/${vmid}.$ext";
4526         }
4527
4528         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
4529     };
4530     warn $@ if $@; # avoid errors - just warn
4531 }
4532
4533 # Note: use $nockeck to skip tests if VM configuration file exists.
4534 # We need that when migration VMs to other nodes (files already moved)
4535 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
4536 sub vm_stop {
4537     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
4538
4539     $force = 1 if !defined($force) && !$shutdown;
4540
4541     if ($migratedfrom){
4542         my $pid = check_running($vmid, $nocheck, $migratedfrom);
4543         kill 15, $pid if $pid;
4544         my $conf = load_config($vmid, $migratedfrom);
4545         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
4546         return;
4547     }
4548
4549     lock_config($vmid, sub {
4550
4551         my $pid = check_running($vmid, $nocheck);
4552         return if !$pid;
4553
4554         my $conf;
4555         if (!$nocheck) {
4556             $conf = load_config($vmid);
4557             check_lock($conf) if !$skiplock;
4558             if (!defined($timeout) && $shutdown && $conf->{startup}) {
4559                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
4560                 $timeout = $opts->{down} if $opts->{down};
4561             }
4562         }
4563
4564         $timeout = 60 if !defined($timeout);
4565
4566         eval {
4567             if ($shutdown) {
4568                 if (defined($conf) && $conf->{agent}) {
4569                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
4570                 } else {
4571                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
4572                 }
4573             } else {
4574                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
4575             }
4576         };
4577         my $err = $@;
4578
4579         if (!$err) {
4580             my $count = 0;
4581             while (($count < $timeout) && check_running($vmid, $nocheck)) {
4582                 $count++;
4583                 sleep 1;
4584             }
4585
4586             if ($count >= $timeout) {
4587                 if ($force) {
4588                     warn "VM still running - terminating now with SIGTERM\n";
4589                     kill 15, $pid;
4590                 } else {
4591                     die "VM quit/powerdown failed - got timeout\n";
4592                 }
4593             } else {
4594                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4595                 return;
4596             }
4597         } else {
4598             if ($force) {
4599                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
4600                 kill 15, $pid;
4601             } else {
4602                 die "VM quit/powerdown failed\n";
4603             }
4604         }
4605
4606         # wait again
4607         $timeout = 10;
4608
4609         my $count = 0;
4610         while (($count < $timeout) && check_running($vmid, $nocheck)) {
4611             $count++;
4612             sleep 1;
4613         }
4614
4615         if ($count >= $timeout) {
4616             warn "VM still running - terminating now with SIGKILL\n";
4617             kill 9, $pid;
4618             sleep 1;
4619         }
4620
4621         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4622    });
4623 }
4624
4625 sub vm_suspend {
4626     my ($vmid, $skiplock) = @_;
4627
4628     lock_config($vmid, sub {
4629
4630         my $conf = load_config($vmid);
4631
4632         check_lock($conf) if !($skiplock || ($conf->{lock} && $conf->{lock} eq 'backup'));
4633
4634         vm_mon_cmd($vmid, "stop");
4635     });
4636 }
4637
4638 sub vm_resume {
4639     my ($vmid, $skiplock, $nocheck) = @_;
4640
4641     lock_config($vmid, sub {
4642
4643         if (!$nocheck) {
4644
4645             my $conf = load_config($vmid);
4646
4647             check_lock($conf) if !($skiplock || ($conf->{lock} && $conf->{lock} eq 'backup'));
4648
4649             vm_mon_cmd($vmid, "cont");
4650
4651         } else {
4652             vm_mon_cmd_nocheck($vmid, "cont");
4653         }
4654     });
4655 }
4656
4657 sub vm_sendkey {
4658     my ($vmid, $skiplock, $key) = @_;
4659
4660     lock_config($vmid, sub {
4661
4662         my $conf = load_config($vmid);
4663
4664         # there is no qmp command, so we use the human monitor command
4665         vm_human_monitor_command($vmid, "sendkey $key");
4666     });
4667 }
4668
4669 sub vm_destroy {
4670     my ($storecfg, $vmid, $skiplock) = @_;
4671
4672     lock_config($vmid, sub {
4673
4674         my $conf = load_config($vmid);
4675
4676         check_lock($conf) if !$skiplock;
4677
4678         if (!check_running($vmid)) {
4679             destroy_vm($storecfg, $vmid);
4680         } else {
4681             die "VM $vmid is running - destroy failed\n";
4682         }
4683     });
4684 }
4685
4686 # pci helpers
4687
4688 sub file_write {
4689     my ($filename, $buf) = @_;
4690
4691     my $fh = IO::File->new($filename, "w");
4692     return undef if !$fh;
4693
4694     my $res = print $fh $buf;
4695
4696     $fh->close();
4697
4698     return $res;
4699 }
4700
4701 sub pci_device_info {
4702     my ($name) = @_;
4703
4704     my $res;
4705
4706     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
4707     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
4708
4709     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
4710     return undef if !defined($irq) || $irq !~ m/^\d+$/;
4711
4712     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
4713     return undef if !defined($vendor) || $vendor !~ s/^0x//;
4714
4715     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
4716     return undef if !defined($product) || $product !~ s/^0x//;
4717
4718     $res = {
4719         name => $name,
4720         vendor => $vendor,
4721         product => $product,
4722         domain => $domain,
4723         bus => $bus,
4724         slot => $slot,
4725         func => $func,
4726         irq => $irq,
4727         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
4728     };
4729
4730     return $res;
4731 }
4732
4733 sub pci_dev_reset {
4734     my ($dev) = @_;
4735
4736     my $name = $dev->{name};
4737
4738     my $fn = "$pcisysfs/devices/$name/reset";
4739
4740     return file_write($fn, "1");
4741 }
4742
4743 sub pci_dev_bind_to_vfio {
4744     my ($dev) = @_;
4745
4746     my $name = $dev->{name};
4747
4748     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4749
4750     if (!-d $vfio_basedir) {
4751         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4752     }
4753     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4754
4755     my $testdir = "$vfio_basedir/$name";
4756     return 1 if -d $testdir;
4757
4758     my $data = "$dev->{vendor} $dev->{product}";
4759     return undef if !file_write("$vfio_basedir/new_id", $data);
4760
4761     my $fn = "$pcisysfs/devices/$name/driver/unbind";
4762     if (!file_write($fn, $name)) {
4763         return undef if -f $fn;
4764     }
4765
4766     $fn = "$vfio_basedir/bind";
4767     if (! -d $testdir) {
4768         return undef if !file_write($fn, $name);
4769     }
4770
4771     return -d $testdir;
4772 }
4773
4774 sub pci_dev_group_bind_to_vfio {
4775     my ($pciid) = @_;
4776
4777     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4778
4779     if (!-d $vfio_basedir) {
4780         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4781     }
4782     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4783
4784     # get IOMMU group devices
4785     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
4786       my @devs = grep /^0000:/, readdir($D);
4787     closedir($D);
4788
4789     foreach my $pciid (@devs) {
4790         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
4791
4792         # pci bridges, switches or root ports are not supported
4793         # they have a pci_bus subdirectory so skip them
4794         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
4795
4796         my $info = pci_device_info($1);
4797         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
4798     }
4799
4800     return 1;
4801 }
4802
4803 sub print_pci_addr {
4804     my ($id, $bridges) = @_;
4805
4806     my $res = '';
4807     my $devices = {
4808         piix3 => { bus => 0, addr => 1 },
4809         #addr2 : first videocard
4810         balloon0 => { bus => 0, addr => 3 },
4811         watchdog => { bus => 0, addr => 4 },
4812         scsihw0 => { bus => 0, addr => 5 },
4813         'pci.3' => { bus => 0, addr => 5 }, #can also be used for virtio-scsi-single bridge
4814         scsihw1 => { bus => 0, addr => 6 },
4815         ahci0 => { bus => 0, addr => 7 },
4816         qga0 => { bus => 0, addr => 8 },
4817         spice => { bus => 0, addr => 9 },
4818         virtio0 => { bus => 0, addr => 10 },
4819         virtio1 => { bus => 0, addr => 11 },
4820         virtio2 => { bus => 0, addr => 12 },
4821         virtio3 => { bus => 0, addr => 13 },
4822         virtio4 => { bus => 0, addr => 14 },
4823         virtio5 => { bus => 0, addr => 15 },
4824         hostpci0 => { bus => 0, addr => 16 },
4825         hostpci1 => { bus => 0, addr => 17 },
4826         net0 => { bus => 0, addr => 18 },
4827         net1 => { bus => 0, addr => 19 },
4828         net2 => { bus => 0, addr => 20 },
4829         net3 => { bus => 0, addr => 21 },
4830         net4 => { bus => 0, addr => 22 },
4831         net5 => { bus => 0, addr => 23 },
4832         vga1 => { bus => 0, addr => 24 },
4833         vga2 => { bus => 0, addr => 25 },
4834         vga3 => { bus => 0, addr => 26 },
4835         hostpci2 => { bus => 0, addr => 27 },
4836         hostpci3 => { bus => 0, addr => 28 },
4837         #addr29 : usb-host (pve-usb.cfg)
4838         'pci.1' => { bus => 0, addr => 30 },
4839         'pci.2' => { bus => 0, addr => 31 },
4840         'net6' => { bus => 1, addr => 1 },
4841         'net7' => { bus => 1, addr => 2 },
4842         'net8' => { bus => 1, addr => 3 },
4843         'net9' => { bus => 1, addr => 4 },
4844         'net10' => { bus => 1, addr => 5 },
4845         'net11' => { bus => 1, addr => 6 },
4846         'net12' => { bus => 1, addr => 7 },
4847         'net13' => { bus => 1, addr => 8 },
4848         'net14' => { bus => 1, addr => 9 },
4849         'net15' => { bus => 1, addr => 10 },
4850         'net16' => { bus => 1, addr => 11 },
4851         'net17' => { bus => 1, addr => 12 },
4852         'net18' => { bus => 1, addr => 13 },
4853         'net19' => { bus => 1, addr => 14 },
4854         'net20' => { bus => 1, addr => 15 },
4855         'net21' => { bus => 1, addr => 16 },
4856         'net22' => { bus => 1, addr => 17 },
4857         'net23' => { bus => 1, addr => 18 },
4858         'net24' => { bus => 1, addr => 19 },
4859         'net25' => { bus => 1, addr => 20 },
4860         'net26' => { bus => 1, addr => 21 },
4861         'net27' => { bus => 1, addr => 22 },
4862         'net28' => { bus => 1, addr => 23 },
4863         'net29' => { bus => 1, addr => 24 },
4864         'net30' => { bus => 1, addr => 25 },
4865         'net31' => { bus => 1, addr => 26 },
4866         'virtio6' => { bus => 2, addr => 1 },
4867         'virtio7' => { bus => 2, addr => 2 },
4868         'virtio8' => { bus => 2, addr => 3 },
4869         'virtio9' => { bus => 2, addr => 4 },
4870         'virtio10' => { bus => 2, addr => 5 },
4871         'virtio11' => { bus => 2, addr => 6 },
4872         'virtio12' => { bus => 2, addr => 7 },
4873         'virtio13' => { bus => 2, addr => 8 },
4874         'virtio14' => { bus => 2, addr => 9 },
4875         'virtio15' => { bus => 2, addr => 10 },
4876         'virtioscsi0' => { bus => 3, addr => 1 },
4877         'virtioscsi1' => { bus => 3, addr => 2 },
4878         'virtioscsi2' => { bus => 3, addr => 3 },
4879         'virtioscsi3' => { bus => 3, addr => 4 },
4880         'virtioscsi4' => { bus => 3, addr => 5 },
4881         'virtioscsi5' => { bus => 3, addr => 6 },
4882         'virtioscsi6' => { bus => 3, addr => 7 },
4883         'virtioscsi7' => { bus => 3, addr => 8 },
4884         'virtioscsi8' => { bus => 3, addr => 9 },
4885         'virtioscsi9' => { bus => 3, addr => 10 },
4886         'virtioscsi10' => { bus => 3, addr => 11 },
4887         'virtioscsi11' => { bus => 3, addr => 12 },
4888         'virtioscsi12' => { bus => 3, addr => 13 },
4889         'virtioscsi13' => { bus => 3, addr => 14 },
4890         'virtioscsi14' => { bus => 3, addr => 15 },
4891         'virtioscsi15' => { bus => 3, addr => 16 },
4892         'virtioscsi16' => { bus => 3, addr => 17 },
4893         'virtioscsi17' => { bus => 3, addr => 18 },
4894         'virtioscsi18' => { bus => 3, addr => 19 },
4895         'virtioscsi19' => { bus => 3, addr => 20 },
4896         'virtioscsi20' => { bus => 3, addr => 21 },
4897         'virtioscsi21' => { bus => 3, addr => 22 },
4898         'virtioscsi22' => { bus => 3, addr => 23 },
4899         'virtioscsi23' => { bus => 3, addr => 24 },
4900         'virtioscsi24' => { bus => 3, addr => 25 },
4901         'virtioscsi25' => { bus => 3, addr => 26 },
4902         'virtioscsi26' => { bus => 3, addr => 27 },
4903         'virtioscsi27' => { bus => 3, addr => 28 },
4904         'virtioscsi28' => { bus => 3, addr => 29 },
4905         'virtioscsi29' => { bus => 3, addr => 30 },
4906         'virtioscsi30' => { bus => 3, addr => 31 },
4907
4908     };
4909
4910     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
4911            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
4912            my $bus = $devices->{$id}->{bus};
4913            $res = ",bus=pci.$bus,addr=$addr";
4914            $bridges->{$bus} = 1 if $bridges;
4915     }
4916     return $res;
4917
4918 }
4919
4920 sub print_pcie_addr {
4921     my ($id) = @_;
4922
4923     my $res = '';
4924     my $devices = {
4925         hostpci0 => { bus => "ich9-pcie-port-1", addr => 0 },
4926         hostpci1 => { bus => "ich9-pcie-port-2", addr => 0 },
4927         hostpci2 => { bus => "ich9-pcie-port-3", addr => 0 },
4928         hostpci3 => { bus => "ich9-pcie-port-4", addr => 0 },
4929     };
4930
4931     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
4932            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
4933            my $bus = $devices->{$id}->{bus};
4934            $res = ",bus=$bus,addr=$addr";
4935     }
4936     return $res;
4937
4938 }
4939
4940 # vzdump restore implementaion
4941
4942 sub tar_archive_read_firstfile {
4943     my $archive = shift;
4944
4945     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
4946
4947     # try to detect archive type first
4948     my $pid = open (TMP, "tar tf '$archive'|") ||
4949         die "unable to open file '$archive'\n";
4950     my $firstfile = <TMP>;
4951     kill 15, $pid;
4952     close TMP;
4953
4954     die "ERROR: archive contaions no data\n" if !$firstfile;
4955     chomp $firstfile;
4956
4957     return $firstfile;
4958 }
4959
4960 sub tar_restore_cleanup {
4961     my ($storecfg, $statfile) = @_;
4962
4963     print STDERR "starting cleanup\n";
4964
4965     if (my $fd = IO::File->new($statfile, "r")) {
4966         while (defined(my $line = <$fd>)) {
4967             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
4968                 my $volid = $2;
4969                 eval {
4970                     if ($volid =~ m|^/|) {
4971                         unlink $volid || die 'unlink failed\n';
4972                     } else {
4973                         PVE::Storage::vdisk_free($storecfg, $volid);
4974                     }
4975                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
4976                 };
4977                 print STDERR "unable to cleanup '$volid' - $@" if $@;
4978             } else {
4979                 print STDERR "unable to parse line in statfile - $line";
4980             }
4981         }
4982         $fd->close();
4983     }
4984 }
4985
4986 sub restore_archive {
4987     my ($archive, $vmid, $user, $opts) = @_;
4988
4989     my $format = $opts->{format};
4990     my $comp;
4991
4992     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
4993         $format = 'tar' if !$format;
4994         $comp = 'gzip';
4995     } elsif ($archive =~ m/\.tar$/) {
4996         $format = 'tar' if !$format;
4997     } elsif ($archive =~ m/.tar.lzo$/) {
4998         $format = 'tar' if !$format;
4999         $comp = 'lzop';
5000     } elsif ($archive =~ m/\.vma$/) {
5001         $format = 'vma' if !$format;
5002     } elsif ($archive =~ m/\.vma\.gz$/) {
5003         $format = 'vma' if !$format;
5004         $comp = 'gzip';
5005     } elsif ($archive =~ m/\.vma\.lzo$/) {
5006         $format = 'vma' if !$format;
5007         $comp = 'lzop';
5008     } else {
5009         $format = 'vma' if !$format; # default
5010     }
5011
5012     # try to detect archive format
5013     if ($format eq 'tar') {
5014         return restore_tar_archive($archive, $vmid, $user, $opts);
5015     } else {
5016         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5017     }
5018 }
5019
5020 sub restore_update_config_line {
5021     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5022
5023     return if $line =~ m/^\#qmdump\#/;
5024     return if $line =~ m/^\#vzdump\#/;
5025     return if $line =~ m/^lock:/;
5026     return if $line =~ m/^unused\d+:/;
5027     return if $line =~ m/^parent:/;
5028     return if $line =~ m/^template:/; # restored VM is never a template
5029
5030     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5031         # try to convert old 1.X settings
5032         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5033         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5034             my ($model, $macaddr) = split(/\=/, $devconfig);
5035             $macaddr = PVE::Tools::random_ether_addr() if !$macaddr || $unique;
5036             my $net = {
5037                 model => $model,
5038                 bridge => "vmbr$ind",
5039                 macaddr => $macaddr,
5040             };
5041             my $netstr = print_net($net);
5042
5043             print $outfd "net$cookie->{netcount}: $netstr\n";
5044             $cookie->{netcount}++;
5045         }
5046     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5047         my ($id, $netstr) = ($1, $2);
5048         my $net = parse_net($netstr);
5049         $net->{macaddr} = PVE::Tools::random_ether_addr() if $net->{macaddr};
5050         $netstr = print_net($net);
5051         print $outfd "$id: $netstr\n";
5052     } elsif ($line =~ m/^((ide|scsi|virtio|sata)\d+):\s*(\S+)\s*$/) {
5053         my $virtdev = $1;
5054         my $value = $3;
5055         if ($line =~ m/backup=no/) {
5056             print $outfd "#$line";
5057         } elsif ($virtdev && $map->{$virtdev}) {
5058             my $di = parse_drive($virtdev, $value);
5059             delete $di->{format}; # format can change on restore
5060             $di->{file} = $map->{$virtdev};
5061             $value = print_drive($vmid, $di);
5062             print $outfd "$virtdev: $value\n";
5063         } else {
5064             print $outfd $line;
5065         }
5066     } else {
5067         print $outfd $line;
5068     }
5069 }
5070
5071 sub scan_volids {
5072     my ($cfg, $vmid) = @_;
5073
5074     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5075
5076     my $volid_hash = {};
5077     foreach my $storeid (keys %$info) {
5078         foreach my $item (@{$info->{$storeid}}) {
5079             next if !($item->{volid} && $item->{size});
5080             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5081             $volid_hash->{$item->{volid}} = $item;
5082         }
5083     }
5084
5085     return $volid_hash;
5086 }
5087
5088 sub get_used_paths {
5089     my ($vmid, $storecfg, $conf, $scan_snapshots, $skip_drive) = @_;
5090
5091     my $used_path = {};
5092
5093     my $scan_config = sub {
5094         my ($cref, $snapname) = @_;
5095
5096         foreach my $key (keys %$cref) {
5097             my $value = $cref->{$key};
5098             if (valid_drivename($key)) {
5099                 next if $skip_drive && $key eq $skip_drive;
5100                 my $drive = parse_drive($key, $value);
5101                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5102                 if ($drive->{file} =~ m!^/!) {
5103                     $used_path->{$drive->{file}}++; # = 1;
5104                 } else {
5105                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5106                     next if !$storeid;
5107                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5108                     next if !$scfg;
5109                     my $path = PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5110                     $used_path->{$path}++; # = 1;
5111                 }
5112             }
5113         }
5114     };
5115
5116     &$scan_config($conf);
5117
5118     undef $skip_drive;
5119
5120     if ($scan_snapshots) {
5121         foreach my $snapname (keys %{$conf->{snapshots}}) {
5122             &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5123         }
5124     }
5125
5126     return $used_path;
5127 }
5128
5129 sub update_disksize {
5130     my ($vmid, $conf, $volid_hash) = @_;
5131
5132     my $changes;
5133
5134     my $used = {};
5135
5136     # Note: it is allowed to define multiple storages with same path (alias), so
5137     # we need to check both 'volid' and real 'path' (two different volid can point
5138     # to the same path).
5139
5140     my $usedpath = {};
5141
5142     # update size info
5143     foreach my $opt (keys %$conf) {
5144         if (valid_drivename($opt)) {
5145             my $drive = parse_drive($opt, $conf->{$opt});
5146             my $volid = $drive->{file};
5147             next if !$volid;
5148
5149             $used->{$volid} = 1;
5150             if ($volid_hash->{$volid} &&
5151                 (my $path = $volid_hash->{$volid}->{path})) {
5152                 $usedpath->{$path} = 1;
5153             }
5154
5155             next if drive_is_cdrom($drive);
5156             next if !$volid_hash->{$volid};
5157
5158             $drive->{size} = $volid_hash->{$volid}->{size};
5159             my $new = print_drive($vmid, $drive);
5160             if ($new ne $conf->{$opt}) {
5161                 $changes = 1;
5162                 $conf->{$opt} = $new;
5163             }
5164         }
5165     }
5166
5167     # remove 'unusedX' entry if volume is used
5168     foreach my $opt (keys %$conf) {
5169         next if $opt !~ m/^unused\d+$/;
5170         my $volid = $conf->{$opt};
5171         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5172         if ($used->{$volid} || ($path && $usedpath->{$path})) {
5173             $changes = 1;
5174             delete $conf->{$opt};
5175         }
5176     }
5177
5178     foreach my $volid (sort keys %$volid_hash) {
5179         next if $volid =~ m/vm-$vmid-state-/;
5180         next if $used->{$volid};
5181         my $path = $volid_hash->{$volid}->{path};
5182         next if !$path; # just to be sure
5183         next if $usedpath->{$path};
5184         $changes = 1;
5185         add_unused_volume($conf, $volid);
5186         $usedpath->{$path} = 1; # avoid to add more than once (aliases)
5187     }
5188
5189     return $changes;
5190 }
5191
5192 sub rescan {
5193     my ($vmid, $nolock) = @_;
5194
5195     my $cfg = PVE::Cluster::cfs_read_file("storage.cfg");
5196
5197     my $volid_hash = scan_volids($cfg, $vmid);
5198
5199     my $updatefn =  sub {
5200         my ($vmid) = @_;
5201
5202         my $conf = load_config($vmid);
5203
5204         check_lock($conf);
5205
5206         my $vm_volids = {};
5207         foreach my $volid (keys %$volid_hash) {
5208             my $info = $volid_hash->{$volid};
5209             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5210         }
5211
5212         my $changes = update_disksize($vmid, $conf, $vm_volids);
5213
5214         update_config_nolock($vmid, $conf, 1) if $changes;
5215     };
5216
5217     if (defined($vmid)) {
5218         if ($nolock) {
5219             &$updatefn($vmid);
5220         } else {
5221             lock_config($vmid, $updatefn, $vmid);
5222         }
5223     } else {
5224         my $vmlist = config_list();
5225         foreach my $vmid (keys %$vmlist) {
5226             if ($nolock) {
5227                 &$updatefn($vmid);
5228             } else {
5229                 lock_config($vmid, $updatefn, $vmid);
5230             }
5231         }
5232     }
5233 }
5234
5235 sub restore_vma_archive {
5236     my ($archive, $vmid, $user, $opts, $comp) = @_;
5237
5238     my $input = $archive eq '-' ? "<&STDIN" : undef;
5239     my $readfrom = $archive;
5240
5241     my $uncomp = '';
5242     if ($comp) {
5243         $readfrom = '-';
5244         my $qarchive = PVE::Tools::shellquote($archive);
5245         if ($comp eq 'gzip') {
5246             $uncomp = "zcat $qarchive|";
5247         } elsif ($comp eq 'lzop') {
5248             $uncomp = "lzop -d -c $qarchive|";
5249         } else {
5250             die "unknown compression method '$comp'\n";
5251         }
5252
5253     }
5254
5255     my $tmpdir = "/var/tmp/vzdumptmp$$";
5256     rmtree $tmpdir;
5257
5258     # disable interrupts (always do cleanups)
5259     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5260         warn "got interrupt - ignored\n";
5261     };
5262
5263     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5264     POSIX::mkfifo($mapfifo, 0600);
5265     my $fifofh;
5266
5267     my $openfifo = sub {
5268         open($fifofh, '>', $mapfifo) || die $!;
5269     };
5270
5271     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5272
5273     my $oldtimeout;
5274     my $timeout = 5;
5275
5276     my $devinfo = {};
5277
5278     my $rpcenv = PVE::RPCEnvironment::get();
5279
5280     my $conffile = config_file($vmid);
5281     my $tmpfn = "$conffile.$$.tmp";
5282
5283     # Note: $oldconf is undef if VM does not exists
5284     my $oldconf = PVE::Cluster::cfs_read_file(cfs_config_path($vmid));
5285
5286     my $print_devmap = sub {
5287         my $virtdev_hash = {};
5288
5289         my $cfgfn = "$tmpdir/qemu-server.conf";
5290
5291         # we can read the config - that is already extracted
5292         my $fh = IO::File->new($cfgfn, "r") ||
5293             "unable to read qemu-server.conf - $!\n";
5294
5295         while (defined(my $line = <$fh>)) {
5296             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5297                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5298                 die "archive does not contain data for drive '$virtdev'\n"
5299                     if !$devinfo->{$devname};
5300                 if (defined($opts->{storage})) {
5301                     $storeid = $opts->{storage} || 'local';
5302                 } elsif (!$storeid) {
5303                     $storeid = 'local';
5304                 }
5305                 $format = 'raw' if !$format;
5306                 $devinfo->{$devname}->{devname} = $devname;
5307                 $devinfo->{$devname}->{virtdev} = $virtdev;
5308                 $devinfo->{$devname}->{format} = $format;
5309                 $devinfo->{$devname}->{storeid} = $storeid;
5310
5311                 # check permission on storage
5312                 my $pool = $opts->{pool}; # todo: do we need that?
5313                 if ($user ne 'root@pam') {
5314                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5315                 }
5316
5317                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5318             }
5319         }
5320
5321         foreach my $devname (keys %$devinfo) {
5322             die "found no device mapping information for device '$devname'\n"
5323                 if !$devinfo->{$devname}->{virtdev};
5324         }
5325
5326         my $cfg = cfs_read_file('storage.cfg');
5327
5328         # create empty/temp config
5329         if ($oldconf) {
5330             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5331             foreach_drive($oldconf, sub {
5332                 my ($ds, $drive) = @_;
5333
5334                 return if drive_is_cdrom($drive);
5335
5336                 my $volid = $drive->{file};
5337
5338                 return if !$volid || $volid =~ m|^/|;
5339
5340                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5341                 return if !$path || !$owner || ($owner != $vmid);
5342
5343                 # Note: only delete disk we want to restore
5344                 # other volumes will become unused
5345                 if ($virtdev_hash->{$ds}) {
5346                     PVE::Storage::vdisk_free($cfg, $volid);
5347                 }
5348             });
5349         }
5350
5351         my $map = {};
5352         foreach my $virtdev (sort keys %$virtdev_hash) {
5353             my $d = $virtdev_hash->{$virtdev};
5354             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5355             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5356
5357             # test if requested format is supported
5358             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5359             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5360             $d->{format} = $defFormat if !$supported;
5361
5362             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5363                                                   $d->{format}, undef, $alloc_size);
5364             print STDERR "new volume ID is '$volid'\n";
5365             $d->{volid} = $volid;
5366             my $path = PVE::Storage::path($cfg, $volid);
5367
5368             PVE::Storage::activate_volumes($cfg,[$volid]);
5369
5370             my $write_zeros = 1;
5371             # fixme: what other storages types initialize volumes with zero?
5372             if ($scfg->{type} eq 'dir' || $scfg->{type} eq 'nfs' || $scfg->{type} eq 'glusterfs' ||
5373                 $scfg->{type} eq 'sheepdog' || $scfg->{type} eq 'rbd') {
5374                 $write_zeros = 0;
5375             }
5376
5377             print $fifofh "${write_zeros}:$d->{devname}=$path\n";
5378
5379             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5380             $map->{$virtdev} = $volid;
5381         }
5382
5383         $fh->seek(0, 0) || die "seek failed - $!\n";
5384
5385         my $outfd = new IO::File ($tmpfn, "w") ||
5386             die "unable to write config for VM $vmid\n";
5387
5388         my $cookie = { netcount => 0 };
5389         while (defined(my $line = <$fh>)) {
5390             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5391         }
5392
5393         $fh->close();
5394         $outfd->close();
5395     };
5396
5397     eval {
5398         # enable interrupts
5399         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5400             die "interrupted by signal\n";
5401         };
5402         local $SIG{ALRM} = sub { die "got timeout\n"; };
5403
5404         $oldtimeout = alarm($timeout);
5405
5406         my $parser = sub {
5407             my $line = shift;
5408
5409             print "$line\n";
5410
5411             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5412                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5413                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5414             } elsif ($line =~ m/^CTIME: /) {
5415                 # we correctly received the vma config, so we can disable
5416                 # the timeout now for disk allocation (set to 10 minutes, so
5417                 # that we always timeout if something goes wrong)
5418                 alarm(600);
5419                 &$print_devmap();
5420                 print $fifofh "done\n";
5421                 my $tmp = $oldtimeout || 0;
5422                 $oldtimeout = undef;
5423                 alarm($tmp);
5424                 close($fifofh);
5425             }
5426         };
5427
5428         print "restore vma archive: $cmd\n";
5429         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5430     };
5431     my $err = $@;
5432
5433     alarm($oldtimeout) if $oldtimeout;
5434
5435     my $vollist = [];
5436     foreach my $devname (keys %$devinfo) {
5437         my $volid = $devinfo->{$devname}->{volid};
5438         push @$vollist, $volid if $volid;
5439     }
5440
5441     my $cfg = cfs_read_file('storage.cfg');
5442     PVE::Storage::deactivate_volumes($cfg, $vollist);
5443
5444     unlink $mapfifo;
5445
5446     if ($err) {
5447         rmtree $tmpdir;
5448         unlink $tmpfn;
5449
5450         foreach my $devname (keys %$devinfo) {
5451             my $volid = $devinfo->{$devname}->{volid};
5452             next if !$volid;
5453             eval {
5454                 if ($volid =~ m|^/|) {
5455                     unlink $volid || die 'unlink failed\n';
5456                 } else {
5457                     PVE::Storage::vdisk_free($cfg, $volid);
5458                 }
5459                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5460             };
5461             print STDERR "unable to cleanup '$volid' - $@" if $@;
5462         }
5463         die $err;
5464     }
5465
5466     rmtree $tmpdir;
5467
5468     rename($tmpfn, $conffile) ||
5469         die "unable to commit configuration file '$conffile'\n";
5470
5471     PVE::Cluster::cfs_update(); # make sure we read new file
5472
5473     eval { rescan($vmid, 1); };
5474     warn $@ if $@;
5475 }
5476
5477 sub restore_tar_archive {
5478     my ($archive, $vmid, $user, $opts) = @_;
5479
5480     if ($archive ne '-') {
5481         my $firstfile = tar_archive_read_firstfile($archive);
5482         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5483             if $firstfile ne 'qemu-server.conf';
5484     }
5485
5486     my $storecfg = cfs_read_file('storage.cfg');
5487
5488     # destroy existing data - keep empty config
5489     my $vmcfgfn = config_file($vmid);
5490     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5491
5492     my $tocmd = "/usr/lib/qemu-server/qmextract";
5493
5494     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5495     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5496     $tocmd .= ' --prealloc' if $opts->{prealloc};
5497     $tocmd .= ' --info' if $opts->{info};
5498
5499     # tar option "xf" does not autodetect compression when read from STDIN,
5500     # so we pipe to zcat
5501     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5502         PVE::Tools::shellquote("--to-command=$tocmd");
5503
5504     my $tmpdir = "/var/tmp/vzdumptmp$$";
5505     mkpath $tmpdir;
5506
5507     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5508     local $ENV{VZDUMP_VMID} = $vmid;
5509     local $ENV{VZDUMP_USER} = $user;
5510
5511     my $conffile = config_file($vmid);
5512     my $tmpfn = "$conffile.$$.tmp";
5513
5514     # disable interrupts (always do cleanups)
5515     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5516         print STDERR "got interrupt - ignored\n";
5517     };
5518
5519     eval {
5520         # enable interrupts
5521         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5522             die "interrupted by signal\n";
5523         };
5524
5525         if ($archive eq '-') {
5526             print "extracting archive from STDIN\n";
5527             run_command($cmd, input => "<&STDIN");
5528         } else {
5529             print "extracting archive '$archive'\n";
5530             run_command($cmd);
5531         }
5532
5533         return if $opts->{info};
5534
5535         # read new mapping
5536         my $map = {};
5537         my $statfile = "$tmpdir/qmrestore.stat";
5538         if (my $fd = IO::File->new($statfile, "r")) {
5539             while (defined (my $line = <$fd>)) {
5540                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5541                     $map->{$1} = $2 if $1;
5542                 } else {
5543                     print STDERR "unable to parse line in statfile - $line\n";
5544                 }
5545             }
5546             $fd->close();
5547         }
5548
5549         my $confsrc = "$tmpdir/qemu-server.conf";
5550
5551         my $srcfd = new IO::File($confsrc, "r") ||
5552             die "unable to open file '$confsrc'\n";
5553
5554         my $outfd = new IO::File ($tmpfn, "w") ||
5555             die "unable to write config for VM $vmid\n";
5556
5557         my $cookie = { netcount => 0 };
5558         while (defined (my $line = <$srcfd>)) {
5559             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5560         }
5561
5562         $srcfd->close();
5563         $outfd->close();
5564     };
5565     my $err = $@;
5566
5567     if ($err) {
5568
5569         unlink $tmpfn;
5570
5571         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
5572
5573         die $err;
5574     }
5575
5576     rmtree $tmpdir;
5577
5578     rename $tmpfn, $conffile ||
5579         die "unable to commit configuration file '$conffile'\n";
5580
5581     PVE::Cluster::cfs_update(); # make sure we read new file
5582
5583     eval { rescan($vmid, 1); };
5584     warn $@ if $@;
5585 };
5586
5587
5588 # Internal snapshots
5589
5590 # NOTE: Snapshot create/delete involves several non-atomic
5591 # action, and can take a long time.
5592 # So we try to avoid locking the file and use 'lock' variable
5593 # inside the config file instead.
5594
5595 my $snapshot_copy_config = sub {
5596     my ($source, $dest) = @_;
5597
5598     foreach my $k (keys %$source) {
5599         next if $k eq 'snapshots';
5600         next if $k eq 'snapstate';
5601         next if $k eq 'snaptime';
5602         next if $k eq 'vmstate';
5603         next if $k eq 'lock';
5604         next if $k eq 'digest';
5605         next if $k eq 'description';
5606         next if $k =~ m/^unused\d+$/;
5607
5608         $dest->{$k} = $source->{$k};
5609     }
5610 };
5611
5612 my $snapshot_apply_config = sub {
5613     my ($conf, $snap) = @_;
5614
5615     # copy snapshot list
5616     my $newconf = {
5617         snapshots => $conf->{snapshots},
5618     };
5619
5620     # keep description and list of unused disks
5621     foreach my $k (keys %$conf) {
5622         next if !($k =~ m/^unused\d+$/ || $k eq 'description');
5623         $newconf->{$k} = $conf->{$k};
5624     }
5625
5626     &$snapshot_copy_config($snap, $newconf);
5627
5628     return $newconf;
5629 };
5630
5631 sub foreach_writable_storage {
5632     my ($conf, $func) = @_;
5633
5634     my $sidhash = {};
5635
5636     foreach my $ds (keys %$conf) {
5637         next if !valid_drivename($ds);
5638
5639         my $drive = parse_drive($ds, $conf->{$ds});
5640         next if !$drive;
5641         next if drive_is_cdrom($drive);
5642
5643         my $volid = $drive->{file};
5644
5645         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5646         $sidhash->{$sid} = $sid if $sid;
5647     }
5648
5649     foreach my $sid (sort keys %$sidhash) {
5650         &$func($sid);
5651     }
5652 }
5653
5654 my $alloc_vmstate_volid = sub {
5655     my ($storecfg, $vmid, $conf, $snapname) = @_;
5656
5657     # Note: we try to be smart when selecting a $target storage
5658
5659     my $target;
5660
5661     # search shared storage first
5662     foreach_writable_storage($conf, sub {
5663         my ($sid) = @_;
5664         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
5665         return if !$scfg->{shared};
5666
5667         $target = $sid if !$target || $scfg->{path}; # prefer file based storage
5668     });
5669
5670     if (!$target) {
5671         # now search local storage
5672         foreach_writable_storage($conf, sub {
5673             my ($sid) = @_;
5674             my $scfg = PVE::Storage::storage_config($storecfg, $sid);
5675             return if $scfg->{shared};
5676
5677             $target = $sid if !$target || $scfg->{path}; # prefer file based storage;
5678         });
5679     }
5680
5681     $target = 'local' if !$target;
5682
5683     my $driver_state_size = 500; # assume 32MB is enough to safe all driver state;
5684     # we abort live save after $conf->{memory}, so we need at max twice that space
5685     my $size = $conf->{memory}*2 + $driver_state_size;
5686
5687     my $name = "vm-$vmid-state-$snapname";
5688     my $scfg = PVE::Storage::storage_config($storecfg, $target);
5689     $name .= ".raw" if $scfg->{path}; # add filename extension for file base storage
5690     my $volid = PVE::Storage::vdisk_alloc($storecfg, $target, $vmid, 'raw', $name, $size*1024);
5691
5692     return $volid;
5693 };
5694
5695 my $snapshot_prepare = sub {
5696     my ($vmid, $snapname, $save_vmstate, $comment) = @_;
5697
5698     my $snap;
5699
5700     my $updatefn =  sub {
5701
5702         my $conf = load_config($vmid);
5703
5704         die "you can't take a snapshot if it's a template\n"
5705             if is_template($conf);
5706
5707         check_lock($conf);
5708
5709         $conf->{lock} = 'snapshot';
5710
5711         die "snapshot name '$snapname' already used\n"
5712             if defined($conf->{snapshots}->{$snapname});
5713
5714         my $storecfg = PVE::Storage::config();
5715         die "snapshot feature is not available" if !has_feature('snapshot', $conf, $storecfg);
5716
5717         $snap = $conf->{snapshots}->{$snapname} = {};
5718
5719         if ($save_vmstate && check_running($vmid)) {
5720             $snap->{vmstate} = &$alloc_vmstate_volid($storecfg, $vmid, $conf, $snapname);
5721         }
5722
5723         &$snapshot_copy_config($conf, $snap);
5724
5725         $snap->{snapstate} = "prepare";
5726         $snap->{snaptime} = time();
5727         $snap->{description} = $comment if $comment;
5728
5729         # always overwrite machine if we save vmstate. This makes sure we
5730         # can restore it later using correct machine type
5731         $snap->{machine} = get_current_qemu_machine($vmid) if $snap->{vmstate};
5732
5733         update_config_nolock($vmid, $conf, 1);
5734     };
5735
5736     lock_config($vmid, $updatefn);
5737
5738     return $snap;
5739 };
5740
5741 my $snapshot_commit = sub {
5742     my ($vmid, $snapname) = @_;
5743
5744     my $updatefn = sub {
5745
5746         my $conf = load_config($vmid);
5747
5748         die "missing snapshot lock\n"
5749             if !($conf->{lock} && $conf->{lock} eq 'snapshot');
5750
5751         my $has_machine_config = defined($conf->{machine});
5752
5753         my $snap = $conf->{snapshots}->{$snapname};
5754
5755         die "snapshot '$snapname' does not exist\n" if !defined($snap);
5756
5757         die "wrong snapshot state\n"
5758             if !($snap->{snapstate} && $snap->{snapstate} eq "prepare");
5759
5760         delete $snap->{snapstate};
5761         delete $conf->{lock};
5762
5763         my $newconf = &$snapshot_apply_config($conf, $snap);
5764
5765         delete $newconf->{machine} if !$has_machine_config;
5766
5767         $newconf->{parent} = $snapname;
5768
5769         update_config_nolock($vmid, $newconf, 1);
5770     };
5771
5772     lock_config($vmid, $updatefn);
5773 };
5774
5775 sub snapshot_rollback {
5776     my ($vmid, $snapname) = @_;
5777
5778     my $prepare = 1;
5779
5780     my $storecfg = PVE::Storage::config();
5781
5782     my $conf = load_config($vmid);
5783
5784     my $get_snapshot_config = sub {
5785
5786         die "you can't rollback if vm is a template\n" if is_template($conf);
5787
5788         my $res = $conf->{snapshots}->{$snapname};
5789
5790         die "snapshot '$snapname' does not exist\n" if !defined($res);
5791
5792         return $res;
5793     };
5794
5795     my $snap = &$get_snapshot_config();
5796
5797     foreach_drive($snap, sub {
5798         my ($ds, $drive) = @_;
5799
5800         return if drive_is_cdrom($drive);
5801
5802         my $volid = $drive->{file};
5803
5804         PVE::Storage::volume_rollback_is_possible($storecfg, $volid, $snapname);
5805     });
5806
5807     my $updatefn = sub {
5808
5809         $conf = load_config($vmid);
5810
5811         $snap = &$get_snapshot_config();
5812
5813         die "unable to rollback to incomplete snapshot (snapstate = $snap->{snapstate})\n"
5814             if $snap->{snapstate};
5815
5816         if ($prepare) {
5817             check_lock($conf);
5818             vm_stop($storecfg, $vmid, undef, undef, 5, undef, undef);
5819         }
5820
5821         die "unable to rollback vm $vmid: vm is running\n"
5822             if check_running($vmid);
5823
5824         if ($prepare) {
5825             $conf->{lock} = 'rollback';
5826         } else {
5827             die "got wrong lock\n" if !($conf->{lock} && $conf->{lock} eq 'rollback');
5828             delete $conf->{lock};
5829         }
5830
5831         my $forcemachine;
5832
5833         if (!$prepare) {
5834             my $has_machine_config = defined($conf->{machine});
5835
5836             # copy snapshot config to current config
5837             $conf = &$snapshot_apply_config($conf, $snap);
5838             $conf->{parent} = $snapname;
5839
5840             # Note: old code did not store 'machine', so we try to be smart
5841             # and guess the snapshot was generated with kvm 1.4 (pc-i440fx-1.4).
5842             $forcemachine = $conf->{machine} || 'pc-i440fx-1.4';
5843             # we remove the 'machine' configuration if not explicitly specified
5844             # in the original config.
5845             delete $conf->{machine} if $snap->{vmstate} && !$has_machine_config;
5846         }
5847
5848         update_config_nolock($vmid, $conf, 1);
5849
5850         if (!$prepare && $snap->{vmstate}) {
5851             my $statefile = PVE::Storage::path($storecfg, $snap->{vmstate});
5852             vm_start($storecfg, $vmid, $statefile, undef, undef, undef, $forcemachine);
5853         }
5854     };
5855
5856     lock_config($vmid, $updatefn);
5857
5858     foreach_drive($snap, sub {
5859         my ($ds, $drive) = @_;
5860
5861         return if drive_is_cdrom($drive);
5862
5863         my $volid = $drive->{file};
5864         my $device = "drive-$ds";
5865
5866         PVE::Storage::volume_snapshot_rollback($storecfg, $volid, $snapname);
5867     });
5868
5869     $prepare = 0;
5870     lock_config($vmid, $updatefn);
5871 }
5872
5873 my $savevm_wait = sub {
5874     my ($vmid) = @_;
5875
5876     for(;;) {
5877         my $stat = vm_mon_cmd_nocheck($vmid, "query-savevm");
5878         if (!$stat->{status}) {
5879             die "savevm not active\n";
5880         } elsif ($stat->{status} eq 'active') {
5881             sleep(1);
5882             next;
5883         } elsif ($stat->{status} eq 'completed') {
5884             last;
5885         } else {
5886             die "query-savevm returned status '$stat->{status}'\n";
5887         }
5888     }
5889 };
5890
5891 sub do_snapshots_with_qemu {
5892     my ($storecfg, $volid) = @_;
5893
5894     my $storage_name = PVE::Storage::parse_volume_id($volid);
5895
5896     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
5897         && !$storecfg->{ids}->{$storage_name}->{krbd}){
5898         return 1;
5899     }
5900
5901     if ($volid =~ m/\.(qcow2|qed)$/){
5902         return 1;
5903     }
5904
5905     return undef;
5906 }
5907
5908 sub snapshot_create {
5909     my ($vmid, $snapname, $save_vmstate, $comment) = @_;
5910
5911     my $snap = &$snapshot_prepare($vmid, $snapname, $save_vmstate, $comment);
5912
5913     $save_vmstate = 0 if !$snap->{vmstate}; # vm is not running
5914
5915     my $config = load_config($vmid);
5916
5917     my $running = check_running($vmid);
5918
5919     my $freezefs = $running && $config->{agent};
5920     $freezefs = 0 if $snap->{vmstate}; # not needed if we save RAM
5921
5922     my $drivehash = {};
5923
5924     if ($freezefs) {
5925         eval { vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
5926         warn "guest-fsfreeze-freeze problems - $@" if $@;
5927     }
5928
5929     eval {
5930         # create internal snapshots of all drives
5931
5932         my $storecfg = PVE::Storage::config();
5933
5934         if ($running) {
5935             if ($snap->{vmstate}) {
5936                 my $path = PVE::Storage::path($storecfg, $snap->{vmstate});
5937                 vm_mon_cmd($vmid, "savevm-start", statefile => $path);
5938                 &$savevm_wait($vmid);
5939             } else {
5940                 vm_mon_cmd($vmid, "savevm-start");
5941             }
5942         };
5943
5944         foreach_drive($snap, sub {
5945             my ($ds, $drive) = @_;
5946
5947             return if drive_is_cdrom($drive);
5948
5949             my $volid = $drive->{file};
5950             my $device = "drive-$ds";
5951
5952             qemu_volume_snapshot($vmid, $device, $storecfg, $volid, $snapname);
5953             $drivehash->{$ds} = 1;
5954        });
5955     };
5956     my $err = $@;
5957
5958     if ($running) {
5959         eval { vm_mon_cmd($vmid, "savevm-end")  };
5960         warn $@ if $@;
5961
5962         if ($freezefs) {
5963             eval { vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
5964             warn "guest-fsfreeze-thaw problems - $@" if $@;
5965         }
5966
5967         # savevm-end is async, we need to wait
5968         for (;;) {
5969             my $stat = vm_mon_cmd_nocheck($vmid, "query-savevm");
5970             if (!$stat->{bytes}) {
5971                 last;
5972             } else {
5973                 print "savevm not yet finished\n";
5974                 sleep(1);
5975                 next;
5976             }
5977         }
5978     }
5979
5980     if ($err) {
5981         warn "snapshot create failed: starting cleanup\n";
5982         eval { snapshot_delete($vmid, $snapname, 0, $drivehash); };
5983         warn $@ if $@;
5984         die $err;
5985     }
5986
5987     &$snapshot_commit($vmid, $snapname);
5988 }
5989
5990 # Note: $drivehash is only set when called from snapshot_create.
5991 sub snapshot_delete {
5992     my ($vmid, $snapname, $force, $drivehash) = @_;
5993
5994     my $prepare = 1;
5995
5996     my $snap;
5997     my $unused = [];
5998
5999     my $unlink_parent = sub {
6000         my ($confref, $new_parent) = @_;
6001
6002         if ($confref->{parent} && $confref->{parent} eq $snapname) {
6003             if ($new_parent) {
6004                 $confref->{parent} = $new_parent;
6005             } else {
6006                 delete $confref->{parent};
6007             }
6008         }
6009     };
6010
6011     my $updatefn =  sub {
6012         my ($remove_drive) = @_;
6013
6014         my $conf = load_config($vmid);
6015
6016         if (!$drivehash) {
6017             check_lock($conf);
6018             die "you can't delete a snapshot if vm is a template\n"
6019                 if is_template($conf);
6020         }
6021
6022         $snap = $conf->{snapshots}->{$snapname};
6023
6024         die "snapshot '$snapname' does not exist\n" if !defined($snap);
6025
6026         # remove parent refs
6027         if (!$prepare) {
6028             &$unlink_parent($conf, $snap->{parent});
6029             foreach my $sn (keys %{$conf->{snapshots}}) {
6030                 next if $sn eq $snapname;
6031                 &$unlink_parent($conf->{snapshots}->{$sn}, $snap->{parent});
6032             }
6033         }
6034
6035         if ($remove_drive) {
6036             if ($remove_drive eq 'vmstate') {
6037                 delete $snap->{$remove_drive};
6038             } else {
6039                 my $drive = parse_drive($remove_drive, $snap->{$remove_drive});
6040                 my $volid = $drive->{file};
6041                 delete $snap->{$remove_drive};
6042                 add_unused_volume($conf, $volid);
6043             }
6044         }
6045
6046         if ($prepare) {
6047             $snap->{snapstate} = 'delete';
6048         } else {
6049             delete $conf->{snapshots}->{$snapname};
6050             delete $conf->{lock} if $drivehash;
6051             foreach my $volid (@$unused) {
6052                 add_unused_volume($conf, $volid);
6053             }
6054         }
6055
6056         update_config_nolock($vmid, $conf, 1);
6057     };
6058
6059     lock_config($vmid, $updatefn);
6060
6061     # now remove vmstate file
6062
6063     my $storecfg = PVE::Storage::config();
6064
6065     if ($snap->{vmstate}) {
6066         eval {  PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
6067         if (my $err = $@) {
6068             die $err if !$force;
6069             warn $err;
6070         }
6071         # save changes (remove vmstate from snapshot)
6072         lock_config($vmid, $updatefn, 'vmstate') if !$force;
6073     };
6074
6075     # now remove all internal snapshots
6076     foreach_drive($snap, sub {
6077         my ($ds, $drive) = @_;
6078
6079         return if drive_is_cdrom($drive);
6080
6081         my $volid = $drive->{file};
6082         my $device = "drive-$ds";
6083
6084         if (!$drivehash || $drivehash->{$ds}) {
6085             eval { qemu_volume_snapshot_delete($vmid, $device, $storecfg, $volid, $snapname); };
6086             if (my $err = $@) {
6087                 die $err if !$force;
6088                 warn $err;
6089             }
6090         }
6091
6092         # save changes (remove drive fron snapshot)
6093         lock_config($vmid, $updatefn, $ds) if !$force;
6094         push @$unused, $volid;
6095     });
6096
6097     # now cleanup config
6098     $prepare = 0;
6099     lock_config($vmid, $updatefn);
6100 }
6101
6102 sub has_feature {
6103     my ($feature, $conf, $storecfg, $snapname, $running) = @_;
6104
6105     my $err;
6106     foreach_drive($conf, sub {
6107         my ($ds, $drive) = @_;
6108
6109         return if drive_is_cdrom($drive);
6110         my $volid = $drive->{file};
6111         $err = 1 if !PVE::Storage::volume_has_feature($storecfg, $feature, $volid, $snapname, $running);
6112     });
6113
6114     return $err ? 0 : 1;
6115 }
6116
6117 sub template_create {
6118     my ($vmid, $conf, $disk) = @_;
6119
6120     my $storecfg = PVE::Storage::config();
6121
6122     foreach_drive($conf, sub {
6123         my ($ds, $drive) = @_;
6124
6125         return if drive_is_cdrom($drive);
6126         return if $disk && $ds ne $disk;
6127
6128         my $volid = $drive->{file};
6129         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6130
6131         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6132         $drive->{file} = $voliddst;
6133         $conf->{$ds} = print_drive($vmid, $drive);
6134         update_config_nolock($vmid, $conf, 1);
6135     });
6136 }
6137
6138 sub is_template {
6139     my ($conf) = @_;
6140
6141     return 1 if defined $conf->{template} && $conf->{template} == 1;
6142 }
6143
6144 sub qemu_img_convert {
6145     my ($src_volid, $dst_volid, $size, $snapname) = @_;
6146
6147     my $storecfg = PVE::Storage::config();
6148     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6149     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6150
6151     if ($src_storeid && $dst_storeid) {
6152         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6153         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6154
6155         my $src_format = qemu_img_format($src_scfg, $src_volname);
6156         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6157
6158         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6159         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6160
6161         my $cmd = [];
6162         push @$cmd, '/usr/bin/qemu-img', 'convert', '-t', 'writeback', '-p', '-n';
6163         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
6164         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path, $dst_path;
6165
6166         my $parser = sub {
6167             my $line = shift;
6168             if($line =~ m/\((\S+)\/100\%\)/){
6169                 my $percent = $1;
6170                 my $transferred = int($size * $percent / 100);
6171                 my $remaining = $size - $transferred;
6172
6173                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6174             }
6175
6176         };
6177
6178         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6179         my $err = $@;
6180         die "copy failed: $err" if $err;
6181     }
6182 }
6183
6184 sub qemu_img_format {
6185     my ($scfg, $volname) = @_;
6186
6187     if ($scfg->{path} && $volname =~ m/\.(raw|cow|qcow|qcow2|qed|vmdk|cloop)$/) {
6188         return $1;
6189     } else {
6190         return "raw";
6191     }
6192 }
6193
6194 sub qemu_drive_mirror {
6195     my ($vmid, $drive, $dst_volid, $vmiddst) = @_;
6196
6197     my $storecfg = PVE::Storage::config();
6198     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6199
6200     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6201
6202     my $format = qemu_img_format($dst_scfg, $dst_volname);
6203
6204     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6205
6206     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $dst_path };
6207     $opts->{format} = $format if $format;
6208
6209     print "drive mirror is starting (scanning bitmap) : this step can take some minutes/hours, depend of disk size and storage speed\n";
6210
6211     eval {
6212     vm_mon_cmd($vmid, "drive-mirror", %$opts);
6213         while (1) {
6214             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6215             my $stat = @$stats[0];
6216             die "mirroring job seem to have die. Maybe do you have bad sectors?" if !$stat;
6217             die "error job is not mirroring" if $stat->{type} ne "mirror";
6218
6219             my $busy = $stat->{busy};
6220             my $ready = $stat->{ready};
6221
6222             if (my $total = $stat->{len}) {
6223                 my $transferred = $stat->{offset} || 0;
6224                 my $remaining = $total - $transferred;
6225                 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6226
6227                 print "transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6228             }
6229
6230
6231             if ($stat->{ready} eq 'true') {
6232
6233                 last if $vmiddst != $vmid;
6234
6235                 # try to switch the disk if source and destination are on the same guest
6236                 eval { vm_mon_cmd($vmid, "block-job-complete", device => "drive-$drive") };
6237                 last if !$@;
6238                 die $@ if $@ !~ m/cannot be completed/;
6239             }
6240             sleep 1;
6241         }
6242
6243
6244     };
6245     my $err = $@;
6246
6247     my $cancel_job = sub {
6248         vm_mon_cmd($vmid, "block-job-cancel", device => "drive-$drive");
6249         while (1) {
6250             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6251             my $stat = @$stats[0];
6252             last if !$stat;
6253             sleep 1;
6254         }
6255     };
6256
6257     if ($err) {
6258         eval { &$cancel_job(); };
6259         die "mirroring error: $err";
6260     }
6261
6262     if ($vmiddst != $vmid) {
6263         # if we clone a disk for a new target vm, we don't switch the disk
6264         &$cancel_job(); # so we call block-job-cancel
6265     }
6266 }
6267
6268 sub clone_disk {
6269     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6270         $newvmid, $storage, $format, $full, $newvollist) = @_;
6271
6272     my $newvolid;
6273
6274     if (!$full) {
6275         print "create linked clone of drive $drivename ($drive->{file})\n";
6276         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6277         push @$newvollist, $newvolid;
6278     } else {
6279         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6280         $storeid = $storage if $storage;
6281
6282         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6283         if (!$format) {
6284             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6285             $format = qemu_img_format($scfg, $volname);
6286         }
6287
6288         # test if requested format is supported - else use default
6289         my $supported = grep { $_ eq $format } @$validFormats;
6290         $format = $defFormat if !$supported;
6291
6292         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6293
6294         print "create full clone of drive $drivename ($drive->{file})\n";
6295         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $format, undef, ($size/1024));
6296         push @$newvollist, $newvolid;
6297
6298         PVE::Storage::activate_volumes($storecfg, $newvollist);
6299
6300         if (!$running || $snapname) {
6301             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname);
6302         } else {
6303             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid);
6304         }
6305     }
6306
6307     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6308
6309     my $disk = $drive;
6310     $disk->{format} = undef;
6311     $disk->{file} = $newvolid;
6312     $disk->{size} = $size;
6313
6314     return $disk;
6315 }
6316
6317 # this only works if VM is running
6318 sub get_current_qemu_machine {
6319     my ($vmid) = @_;
6320
6321     my $cmd = { execute => 'query-machines', arguments => {} };
6322     my $res = vm_qmp_command($vmid, $cmd);
6323
6324     my ($current, $default);
6325     foreach my $e (@$res) {
6326         $default = $e->{name} if $e->{'is-default'};
6327         $current = $e->{name} if $e->{'is-current'};
6328     }
6329
6330     # fallback to the default machine if current is not supported by qemu
6331     return $current || $default || 'pc';
6332 }
6333
6334 sub qemu_machine_feature_enabled {
6335     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6336
6337     my $current_major;
6338     my $current_minor;
6339
6340     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6341
6342         $current_major = $3;
6343         $current_minor = $4;
6344
6345     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6346
6347         $current_major = $1;
6348         $current_minor = $2;
6349     }
6350
6351     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
6352
6353
6354 }
6355
6356 sub qemu_machine_pxe {
6357     my ($vmid, $conf, $machine) = @_;
6358
6359     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6360
6361     foreach my $opt (keys %$conf) {
6362         next if $opt !~ m/^net(\d+)$/;
6363         my $net = PVE::QemuServer::parse_net($conf->{$opt});
6364         next if !$net;
6365         my $romfile = PVE::QemuServer::vm_mon_cmd_nocheck($vmid, 'qom-get', path => $opt, property => 'romfile');
6366         return $machine.".pxe" if $romfile =~ m/pxe/;
6367         last;
6368     }
6369
6370 }
6371
6372 sub qemu_use_old_bios_files {
6373     my ($machine_type) = @_;
6374
6375     return if !$machine_type;
6376
6377     my $use_old_bios_files = undef;
6378
6379     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6380         $machine_type = $1;
6381         $use_old_bios_files = 1;
6382     } else {
6383         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6384         # load new efi bios files on migration. So this hack is required to allow
6385         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6386         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6387         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, undef, 2, 4);
6388     }
6389
6390     return ($use_old_bios_files, $machine_type);
6391 }
6392
6393 sub lspci {
6394
6395     my $devices = {};
6396
6397     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6398             my (undef, $id, $function) = @_;
6399             my $res = { id => $id, function => $function};
6400             push @{$devices->{$id}}, $res;
6401     });
6402
6403     return $devices;
6404 }
6405
6406 sub vm_iothreads_list {
6407     my ($vmid) = @_;
6408
6409     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6410
6411     my $iothreads = {};
6412     foreach my $iothread (@$res) {
6413         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6414     }
6415
6416     return $iothreads;
6417 }
6418
6419 sub scsihw_infos {
6420     my ($conf, $drive) = @_;
6421
6422     my $maxdev = 0;
6423
6424     if ($conf->{scsihw} && ($conf->{scsihw} =~ m/^lsi/)) {
6425         $maxdev = 7;
6426     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6427         $maxdev = 1;
6428     } else {
6429         $maxdev = 256;
6430     }
6431
6432     my $controller = int($drive->{index} / $maxdev);
6433     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6434
6435     return ($maxdev, $controller, $controller_prefix);
6436 }
6437
6438 # bash completion helper
6439
6440 sub complete_backup_archives {
6441     my ($cmdname, $pname, $cvalue) = @_;
6442
6443     my $cfg = PVE::Storage::config();
6444
6445     my $storeid;
6446
6447     if ($cvalue =~ m/^([^:]+):/) {
6448         $storeid = $1;
6449     }
6450
6451     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6452
6453     my $res = [];
6454     foreach my $id (keys %$data) {
6455         foreach my $item (@{$data->{$id}}) {
6456             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6457             push @$res, $item->{volid} if defined($item->{volid});
6458         }
6459     }
6460
6461     return $res;
6462 }
6463
6464 my $complete_vmid_full = sub {
6465     my ($running) = @_;
6466
6467     my $idlist = vmstatus();
6468
6469     my $res = [];
6470
6471     foreach my $id (keys %$idlist) {
6472         my $d = $idlist->{$id};
6473         if (defined($running)) {
6474             next if $d->{template};
6475             next if $running && $d->{status} ne 'running';
6476             next if !$running && $d->{status} eq 'running';
6477         }
6478         push @$res, $id;
6479
6480     }
6481     return $res;
6482 };
6483
6484 sub complete_vmid {
6485     return &$complete_vmid_full();
6486 }
6487
6488 sub complete_vmid_stopped {
6489     return &$complete_vmid_full(0);
6490 }
6491
6492 sub complete_vmid_running {
6493     return &$complete_vmid_full(1);
6494 }
6495
6496 sub complete_storage {
6497
6498     my $cfg = PVE::Storage::config();
6499     my $ids = $cfg->{ids};
6500
6501     my $res = [];
6502     foreach my $sid (keys %$ids) {
6503         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6504         push @$res, $sid;
6505     }
6506
6507     return $res;
6508 }
6509
6510 1;