PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QMPClient;
  31 use PVE::RPCEnvironment;
  32 use Time::HiRes qw(gettimeofday);
  33
  34 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  35
  36 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  37
  38 # Note about locking: we use flock on the config file protect
  39 # against concurent actions.
  40 # Aditionaly, we have a 'lock' setting in the config file. This
  41 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  42 # allowed when such lock is set. But you can ignore this kind of
  43 # lock with the --skiplock flag.
  44
  45 cfs_register_file('/qemu-server/',
  46                   \&parse_vm_config,
  47                   \&write_vm_config);
  48
  49 PVE::JSONSchema::register_standard_option('skiplock', {
  50     description => "Ignore locks - only root is allowed to use this option.",
  51     type => 'boolean',
  52     optional => 1,
  53 });
  54
  55 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  56     description => "Some command save/restore state from this location.",
  57     type => 'string',
  58     maxLength => 128,
  59     optional => 1,
  60 });
  61
  62 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  63     description => "The name of the snapshot.",
  64     type => 'string', format => 'pve-configid',
  65     maxLength => 40,
  66 });
  67
  68 #no warnings 'redefine';
  69
  70 sub cgroups_write {
  71    my ($controller, $vmid, $option, $value) = @_;
  72
  73    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  74    PVE::ProcFSTools::write_proc_entry($path, $value);
  75
  76 }
  77
  78 my $nodename = PVE::INotify::nodename();
  79
  80 mkdir "/etc/pve/nodes/$nodename";
  81 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  82 mkdir $confdir;
  83
  84 my $var_run_tmpdir = "/var/run/qemu-server";
  85 mkdir $var_run_tmpdir;
  86
  87 my $lock_dir = "/var/lock/qemu-server";
  88 mkdir $lock_dir;
  89
  90 my $pcisysfs = "/sys/bus/pci";
  91
  92 my $confdesc = {
  93     onboot => {
  94         optional => 1,
  95         type => 'boolean',
  96         description => "Specifies whether a VM will be started during system bootup.",
  97         default => 0,
  98     },
  99     autostart => {
 100         optional => 1,
 101         type => 'boolean',
 102         description => "Automatic restart after crash (currently ignored).",
 103         default => 0,
 104     },
 105     hotplug => {
 106         optional => 1,
 107         type => 'string', format => 'pve-hotplug-features',
 108         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 109         default => 'network,disk,usb',
 110     },
 111     reboot => {
 112         optional => 1,
 113         type => 'boolean',
 114         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 115         default => 1,
 116     },
 117     lock => {
 118         optional => 1,
 119         type => 'string',
 120         description => "Lock/unlock the VM.",
 121         enum => [qw(migrate backup snapshot rollback)],
 122     },
 123     cpulimit => {
 124         optional => 1,
 125         type => 'number',
 126         description => "Limit of CPU usage. Note if the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 127         minimum => 0,
 128         maximum => 128,
 129         default => 0,
 130     },
 131     cpuunits => {
 132         optional => 1,
 133         type => 'integer',
 134         description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 135         minimum => 0,
 136         maximum => 500000,
 137         default => 1000,
 138     },
 139     memory => {
 140         optional => 1,
 141         type => 'integer',
 142         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 143         minimum => 16,
 144         default => 512,
 145     },
 146     balloon => {
 147         optional => 1,
 148         type => 'integer',
 149         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 150         minimum => 0,
 151     },
 152     shares => {
 153         optional => 1,
 154         type => 'integer',
 155         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 156         minimum => 0,
 157         maximum => 50000,
 158         default => 1000,
 159     },
 160     keyboard => {
 161         optional => 1,
 162         type => 'string',
 163         description => "Keybord layout for vnc server. Default is read from the datacenter configuration file.",
 164         enum => PVE::Tools::kvmkeymaplist(),
 165         default => 'en-us',
 166     },
 167     name => {
 168         optional => 1,
 169         type => 'string', format => 'dns-name',
 170         description => "Set a name for the VM. Only used on the configuration web interface.",
 171     },
 172     scsihw => {
 173         optional => 1,
 174         type => 'string',
 175         description => "scsi controller model",
 176         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 177         default => 'lsi',
 178     },
 179     description => {
 180         optional => 1,
 181         type => 'string',
 182         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 183     },
 184     ostype => {
 185         optional => 1,
 186         type => 'string',
 187         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 l24 l26 solaris)],
 188         description => <<EODESC,
 189 Used to enable special optimization/features for specific
 190 operating systems:
 191
 192 other  => unspecified OS
 193 wxp    => Microsoft Windows XP
 194 w2k    => Microsoft Windows 2000
 195 w2k3   => Microsoft Windows 2003
 196 w2k8   => Microsoft Windows 2008
 197 wvista => Microsoft Windows Vista
 198 win7   => Microsoft Windows 7
 199 win8   => Microsoft Windows 8/2012
 200 l24    => Linux 2.4 Kernel
 201 l26    => Linux 2.6/3.X Kernel
 202 solaris => solaris/opensolaris/openindiania kernel
 203
 204 other|l24|l26|solaris                       ... no special behaviour
 205 wxp|w2k|w2k3|w2k8|wvista|win7|win8  ... use --localtime switch
 206 EODESC
 207     },
 208     boot => {
 209         optional => 1,
 210         type => 'string',
 211         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 212         pattern => '[acdn]{1,4}',
 213         default => 'cdn',
 214     },
 215     bootdisk => {
 216         optional => 1,
 217         type => 'string', format => 'pve-qm-bootdisk',
 218         description => "Enable booting from specified disk.",
 219         pattern => '(ide|sata|scsi|virtio)\d+',
 220     },
 221     smp => {
 222         optional => 1,
 223         type => 'integer',
 224         description => "The number of CPUs. Please use option -sockets instead.",
 225         minimum => 1,
 226         default => 1,
 227     },
 228     sockets => {
 229         optional => 1,
 230         type => 'integer',
 231         description => "The number of CPU sockets.",
 232         minimum => 1,
 233         default => 1,
 234     },
 235     cores => {
 236         optional => 1,
 237         type => 'integer',
 238         description => "The number of cores per socket.",
 239         minimum => 1,
 240         default => 1,
 241     },
 242     numa => {
 243         optional => 1,
 244         type => 'boolean',
 245         description => "Enable/disable Numa.",
 246         default => 0,
 247     },
 248     vcpus => {
 249         optional => 1,
 250         type => 'integer',
 251         description => "Number of hotplugged vcpus.",
 252         minimum => 1,
 253         default => 0,
 254     },
 255     acpi => {
 256         optional => 1,
 257         type => 'boolean',
 258         description => "Enable/disable ACPI.",
 259         default => 1,
 260     },
 261     agent => {
 262         optional => 1,
 263         type => 'boolean',
 264         description => "Enable/disable Qemu GuestAgent.",
 265         default => 0,
 266     },
 267     kvm => {
 268         optional => 1,
 269         type => 'boolean',
 270         description => "Enable/disable KVM hardware virtualization.",
 271         default => 1,
 272     },
 273     tdf => {
 274         optional => 1,
 275         type => 'boolean',
 276         description => "Enable/disable time drift fix.",
 277         default => 0,
 278     },
 279     localtime => {
 280         optional => 1,
 281         type => 'boolean',
 282         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 283     },
 284     freeze => {
 285         optional => 1,
 286         type => 'boolean',
 287         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 288     },
 289     vga => {
 290         optional => 1,
 291         type => 'string',
 292         description => "Select VGA type. If you want to use high resolution modes (>= 1280x1024x16) then you should use option 'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and 'cirrur' for other OS types. Option 'qxl' enables the SPICE display sever. You can also run without any graphic card using a serial devive as terminal.",
 293         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 294     },
 295     watchdog => {
 296         optional => 1,
 297         type => 'string', format => 'pve-qm-watchdog',
 298         typetext => '[[model=]i6300esb|ib700] [,[action=]reset|shutdown|poweroff|pause|debug|none]',
 299         description => "Create a virtual hardware watchdog device.  Once enabled (by a guest action), the watchdog must be periodically polled by an agent inside the guest or else the guest will be restarted (or execute the action specified)",
 300     },
 301     startdate => {
 302         optional => 1,
 303         type => 'string',
 304         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 305         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 306         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 307         default => 'now',
 308     },
 309     startup =>  get_standard_option('pve-startup-order'),
 310     template => {
 311         optional => 1,
 312         type => 'boolean',
 313         description => "Enable/disable Template.",
 314         default => 0,
 315     },
 316     args => {
 317         optional => 1,
 318         type => 'string',
 319         description => <<EODESCR,
 320 Note: this option is for experts only. It allows you to pass arbitrary arguments to kvm, for example:
 321
 322 args: -no-reboot -no-hpet
 323 EODESCR
 324     },
 325     tablet => {
 326         optional => 1,
 327         type => 'boolean',
 328         default => 1,
 329         description => "Enable/disable the usb tablet device. This device is usually needed to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with normal VNC clients. If you're running lots of console-only guests on one host, you may consider disabling this to save some context switches. This is turned of by default if you use spice (vga=qxl).",
 330     },
 331     migrate_speed => {
 332         optional => 1,
 333         type => 'integer',
 334         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 335         minimum => 0,
 336         default => 0,
 337     },
 338     migrate_downtime => {
 339         optional => 1,
 340         type => 'number',
 341         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 342         minimum => 0,
 343         default => 0.1,
 344     },
 345     cdrom => {
 346         optional => 1,
 347         type => 'string', format => 'pve-qm-drive',
 348         typetext => 'volume',
 349         description => "This is an alias for option -ide2",
 350     },
 351     cpu => {
 352         optional => 1,
 353         description => "Emulated CPU type.",
 354         type => 'string',
 355         enum => [ qw(486 athlon pentium pentium2 pentium3 coreduo core2duo kvm32 kvm64 qemu32 qemu64 phenom Conroe Penryn Nehalem Westmere SandyBridge IvyBridge Haswell Broadwell Opteron_G1 Opteron_G2 Opteron_G3 Opteron_G4 Opteron_G5 host) ],
 356         default => 'kvm64',
 357     },
 358     parent => get_standard_option('pve-snapshot-name', {
 359         optional => 1,
 360         description => "Parent snapshot name. This is used internally, and should not be modified.",
 361     }),
 362     snaptime => {
 363         optional => 1,
 364         description => "Timestamp for snapshots.",
 365         type => 'integer',
 366         minimum => 0,
 367     },
 368     vmstate => {
 369         optional => 1,
 370         type => 'string', format => 'pve-volume-id',
 371         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 372     },
 373     machine => {
 374         description => "Specific the Qemu machine type.",
 375         type => 'string',
 376         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
 377         maxLength => 40,
 378         optional => 1,
 379     },
 380     smbios1 => {
 381         description => "Specify SMBIOS type 1 fields.",
 382         type => 'string', format => 'pve-qm-smbios1',
 383         typetext => "[manufacturer=str][,product=str][,version=str][,serial=str] [,uuid=uuid][,sku=str][,family=str]",
 384         maxLength => 256,
 385         optional => 1,
 386     },
 387     protection => {
 388         optional => 1,
 389         type => 'boolean',
 390         description => "Sets the protection flag of the VM. This will prevent the remove operation.",
 391         default => 0,
 392     },
 393 };
 394
 395 # what about other qemu settings ?
 396 #cpu => 'string',
 397 #machine => 'string',
 398 #fda => 'file',
 399 #fdb => 'file',
 400 #mtdblock => 'file',
 401 #sd => 'file',
 402 #pflash => 'file',
 403 #snapshot => 'bool',
 404 #bootp => 'file',
 405 ##tftp => 'dir',
 406 ##smb => 'dir',
 407 #kernel => 'file',
 408 #append => 'string',
 409 #initrd => 'file',
 410 ##soundhw => 'string',
 411
 412 while (my ($k, $v) = each %$confdesc) {
 413     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 414 }
 415
 416 my $MAX_IDE_DISKS = 4;
 417 my $MAX_SCSI_DISKS = 14;
 418 my $MAX_VIRTIO_DISKS = 16;
 419 my $MAX_SATA_DISKS = 6;
 420 my $MAX_USB_DEVICES = 5;
 421 my $MAX_NETS = 32;
 422 my $MAX_UNUSED_DISKS = 8;
 423 my $MAX_HOSTPCI_DEVICES = 4;
 424 my $MAX_SERIAL_PORTS = 4;
 425 my $MAX_PARALLEL_PORTS = 3;
 426 my $MAX_NUMA = 8;
 427 my $MAX_MEM = 4194304;
 428 my $STATICMEM = 1024;
 429
 430 my $numadesc = {
 431     optional => 1,
 432     type => 'string', format => 'pve-qm-numanode',
 433     typetext => "cpus=<id[-id],memory=<mb>[[,hostnodes=<id[-id]>] [,policy=<preferred|bind|interleave>]]",
 434     description => "numa topology",
 435 };
 436 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 437
 438 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 439     $confdesc->{"numa$i"} = $numadesc;
 440 }
 441
 442 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 443                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 444                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 445 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 446
 447 my $netdesc = {
 448     optional => 1,
 449     type => 'string', format => 'pve-qm-net',
 450     typetext => "MODEL=XX:XX:XX:XX:XX:XX [,bridge=<dev>][,queues=<nbqueues>][,rate=<mbps>] [,tag=<vlanid>][,firewall=0|1],link_down=0|1]",
 451     description => <<EODESCR,
 452 Specify network devices.
 453
 454 MODEL is one of: $nic_model_list_txt
 455
 456 XX:XX:XX:XX:XX:XX should be an unique MAC address. This is
 457 automatically generated if not specified.
 458
 459 The bridge parameter can be used to automatically add the interface to a bridge device. The Proxmox VE standard bridge is called 'vmbr0'.
 460
 461 Option 'rate' is used to limit traffic bandwidth from and to this interface. It is specified as floating point number, unit is 'Megabytes per second'.
 462
 463 If you specify no bridge, we create a kvm 'user' (NATed) network device, which provides DHCP and DNS services. The following addresses are used:
 464
 465 10.0.2.2   Gateway
 466 10.0.2.3   DNS Server
 467 10.0.2.4   SMB Server
 468
 469 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 470
 471 EODESCR
 472 };
 473 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 474
 475 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 476     $confdesc->{"net$i"} = $netdesc;
 477 }
 478
 479 my $drivename_hash;
 480
 481 my $idedesc = {
 482     optional => 1,
 483     type => 'string', format => 'pve-qm-drive',
 484     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads] [,discard=ignore|on] [,serial=serial][,model=model]',
 485     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
 486 };
 487 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
 488
 489 my $scsidesc = {
 490     optional => 1,
 491     type => 'string', format => 'pve-qm-drive',
 492     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads] [,discard=ignore|on] [,iothread=on] [,queues=<nbqueues>] [,serial=serial]',
 493     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
 494 };
 495 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 496
 497 my $satadesc = {
 498     optional => 1,
 499     type => 'string', format => 'pve-qm-drive',
 500     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads]  [,discard=ignore|on] [,serial=serial]',
 501     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
 502 };
 503 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
 504
 505 my $virtiodesc = {
 506     optional => 1,
 507     type => 'string', format => 'pve-qm-drive',
 508     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads]  [,discard=ignore|on] [,iothread=on] [,serial=serial]',
 509     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
 510 };
 511 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
 512
 513 my $usbdesc = {
 514     optional => 1,
 515     type => 'string', format => 'pve-qm-usb-device',
 516     typetext => 'host=HOSTUSBDEVICE|spice',
 517     description => <<EODESCR,
 518 Configure an USB device (n is 0 to 4). This can be used to
 519 pass-through usb devices to the guest. HOSTUSBDEVICE syntax is:
 520
 521 'bus-port(.port)*' (decimal numbers) or
 522 'vendor_id:product_id' (hexadeciaml numbers)
 523
 524 You can use the 'lsusb -t' command to list existing usb devices.
 525
 526 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 527
 528 The value 'spice' can be used to add a usb redirection devices for spice.
 529
 530 EODESCR
 531 };
 532 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
 533
 534 my $hostpcidesc = {
 535         optional => 1,
 536         type => 'string', format => 'pve-qm-hostpci',
 537         typetext => "[host=]HOSTPCIDEVICE [,rombar=on|off] [,pcie=0|1] [,x-vga=on|off]",
 538         description => <<EODESCR,
 539 Map host pci devices. HOSTPCIDEVICE syntax is:
 540
 541 'bus:dev.func' (hexadecimal numbers)
 542
 543 You can us the 'lspci' command to list existing pci devices.
 544
 545 The 'rombar' option determines whether or not the device's ROM will be visible in the guest's memory map (default is 'on').
 546
 547 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 548
 549 Experimental: user reported problems with this option.
 550 EODESCR
 551 };
 552 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
 553
 554 my $serialdesc = {
 555         optional => 1,
 556         type => 'string',
 557         pattern => '(/dev/.+|socket)',
 558         description =>  <<EODESCR,
 559 Create a serial device inside the VM (n is 0 to 3), and pass through a host serial device (i.e. /dev/ttyS0), or create a unix socket on the host side (use 'qm terminal' to open a terminal connection).
 560
 561 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 562
 563 Experimental: user reported problems with this option.
 564 EODESCR
 565 };
 566
 567 my $paralleldesc= {
 568         optional => 1,
 569         type => 'string',
 570         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
 571         description =>  <<EODESCR,
 572 Map host parallel devices (n is 0 to 2).
 573
 574 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 575
 576 Experimental: user reported problems with this option.
 577 EODESCR
 578 };
 579
 580 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
 581     $confdesc->{"parallel$i"} = $paralleldesc;
 582 }
 583
 584 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
 585     $confdesc->{"serial$i"} = $serialdesc;
 586 }
 587
 588 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
 589     $confdesc->{"hostpci$i"} = $hostpcidesc;
 590 }
 591
 592 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
 593     $drivename_hash->{"ide$i"} = 1;
 594     $confdesc->{"ide$i"} = $idedesc;
 595 }
 596
 597 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
 598     $drivename_hash->{"sata$i"} = 1;
 599     $confdesc->{"sata$i"} = $satadesc;
 600 }
 601
 602 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
 603     $drivename_hash->{"scsi$i"} = 1;
 604     $confdesc->{"scsi$i"} = $scsidesc ;
 605 }
 606
 607 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
 608     $drivename_hash->{"virtio$i"} = 1;
 609     $confdesc->{"virtio$i"} = $virtiodesc;
 610 }
 611
 612 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
 613     $confdesc->{"usb$i"} = $usbdesc;
 614 }
 615
 616 my $unuseddesc = {
 617     optional => 1,
 618     type => 'string', format => 'pve-volume-id',
 619     description => "Reference to unused volumes.",
 620 };
 621
 622 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
 623     $confdesc->{"unused$i"} = $unuseddesc;
 624 }
 625
 626 my $kvm_api_version = 0;
 627
 628 sub kvm_version {
 629
 630     return $kvm_api_version if $kvm_api_version;
 631
 632     my $fh = IO::File->new("</dev/kvm") ||
 633         return 0;
 634
 635     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
 636         $kvm_api_version = $v;
 637     }
 638
 639     $fh->close();
 640
 641     return  $kvm_api_version;
 642 }
 643
 644 my $kvm_user_version;
 645
 646 sub kvm_user_version {
 647
 648     return $kvm_user_version if $kvm_user_version;
 649
 650     $kvm_user_version = 'unknown';
 651
 652     my $tmp = `kvm -help 2>/dev/null`;
 653
 654     if ($tmp =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
 655         $kvm_user_version = $2;
 656     }
 657
 658     return $kvm_user_version;
 659
 660 }
 661
 662 my $kernel_has_vhost_net = -c '/dev/vhost-net';
 663
 664 sub disknames {
 665     # order is important - used to autoselect boot disk
 666     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
 667             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
 668             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
 669             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))));
 670 }
 671
 672 sub valid_drivename {
 673     my $dev = shift;
 674
 675     return defined($drivename_hash->{$dev});
 676 }
 677
 678 sub option_exists {
 679     my $key = shift;
 680     return defined($confdesc->{$key});
 681 }
 682
 683 sub nic_models {
 684     return $nic_model_list;
 685 }
 686
 687 sub os_list_description {
 688
 689     return {
 690         other => 'Other',
 691         wxp => 'Windows XP',
 692         w2k => 'Windows 2000',
 693         w2k3 =>, 'Windows 2003',
 694         w2k8 => 'Windows 2008',
 695         wvista => 'Windows Vista',
 696         win7 => 'Windows 7',
 697         win8 => 'Windows 8/2012',
 698         l24 => 'Linux 2.4',
 699         l26 => 'Linux 2.6',
 700     };
 701 }
 702
 703 my $cdrom_path;
 704
 705 sub get_cdrom_path {
 706
 707     return  $cdrom_path if $cdrom_path;
 708
 709     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
 710     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
 711     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
 712 }
 713
 714 sub get_iso_path {
 715     my ($storecfg, $vmid, $cdrom) = @_;
 716
 717     if ($cdrom eq 'cdrom') {
 718         return get_cdrom_path();
 719     } elsif ($cdrom eq 'none') {
 720         return '';
 721     } elsif ($cdrom =~ m|^/|) {
 722         return $cdrom;
 723     } else {
 724         return PVE::Storage::path($storecfg, $cdrom);
 725     }
 726 }
 727
 728 # try to convert old style file names to volume IDs
 729 sub filename_to_volume_id {
 730     my ($vmid, $file, $media) = @_;
 731
 732     if (!($file eq 'none' || $file eq 'cdrom' ||
 733           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
 734
 735         return undef if $file =~ m|/|;
 736
 737         if ($media && $media eq 'cdrom') {
 738             $file = "local:iso/$file";
 739         } else {
 740             $file = "local:$vmid/$file";
 741         }
 742     }
 743
 744     return $file;
 745 }
 746
 747 sub verify_media_type {
 748     my ($opt, $vtype, $media) = @_;
 749
 750     return if !$media;
 751
 752     my $etype;
 753     if ($media eq 'disk') {
 754         $etype = 'images';
 755     } elsif ($media eq 'cdrom') {
 756         $etype = 'iso';
 757     } else {
 758         die "internal error";
 759     }
 760
 761     return if ($vtype eq $etype);
 762
 763     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
 764 }
 765
 766 sub cleanup_drive_path {
 767     my ($opt, $storecfg, $drive) = @_;
 768
 769     # try to convert filesystem paths to volume IDs
 770
 771     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
 772         ($drive->{file} !~ m|^/dev/.+|) &&
 773         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
 774         ($drive->{file} !~ m/^\d+$/)) {
 775         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
 776         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
 777         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
 778         verify_media_type($opt, $vtype, $drive->{media});
 779         $drive->{file} = $volid;
 780     }
 781
 782     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
 783 }
 784
 785 sub create_conf_nolock {
 786     my ($vmid, $settings) = @_;
 787
 788     my $filename = config_file($vmid);
 789
 790     die "configuration file '$filename' already exists\n" if -f $filename;
 791
 792     my $defaults = load_defaults();
 793
 794     $settings->{name} = "vm$vmid" if !$settings->{name};
 795     $settings->{memory} = $defaults->{memory} if !$settings->{memory};
 796
 797     my $data = '';
 798     foreach my $opt (keys %$settings) {
 799         next if !$confdesc->{$opt};
 800
 801         my $value = $settings->{$opt};
 802         next if !$value;
 803
 804         $data .= "$opt: $value\n";
 805     }
 806
 807     PVE::Tools::file_set_contents($filename, $data);
 808 }
 809
 810 sub parse_hotplug_features {
 811     my ($data) = @_;
 812
 813     my $res = {};
 814
 815     return $res if $data eq '0';
 816
 817     $data = $confdesc->{hotplug}->{default} if $data eq '1';
 818
 819     foreach my $feature (PVE::Tools::split_list($data)) {
 820         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
 821             $res->{$1} = 1;
 822         } else {
 823             warn "ignoring unknown hotplug feature '$feature'\n";
 824         }
 825     }
 826     return $res;
 827 }
 828
 829 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
 830 sub pve_verify_hotplug_features {
 831     my ($value, $noerr) = @_;
 832
 833     return $value if parse_hotplug_features($value);
 834
 835     return undef if $noerr;
 836
 837     die "unable to parse hotplug option\n";
 838 }
 839
 840 my $parse_size = sub {
 841     my ($value) = @_;
 842
 843     return undef if $value !~ m/^(\d+(\.\d+)?)([KMG])?$/;
 844     my ($size, $unit) = ($1, $3);
 845     if ($unit) {
 846         if ($unit eq 'K') {
 847             $size = $size * 1024;
 848         } elsif ($unit eq 'M') {
 849             $size = $size * 1024 * 1024;
 850         } elsif ($unit eq 'G') {
 851             $size = $size * 1024 * 1024 * 1024;
 852         }
 853     }
 854     return int($size);
 855 };
 856
 857 my $format_size = sub {
 858     my ($size) = @_;
 859
 860     $size = int($size);
 861
 862     my $kb = int($size/1024);
 863     return $size if $kb*1024 != $size;
 864
 865     my $mb = int($kb/1024);
 866     return "${kb}K" if $mb*1024 != $kb;
 867
 868     my $gb = int($mb/1024);
 869     return "${mb}M" if $gb*1024 != $mb;
 870
 871     return "${gb}G";
 872 };
 873
 874 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
 875 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
 876 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
 877 #        [,aio=native|threads][,discard=ignore|on][,iothread=on]
 878 #        [,serial=serial][,model=model]
 879
 880 sub parse_drive {
 881     my ($key, $data) = @_;
 882
 883     my $res = {};
 884
 885     # $key may be undefined - used to verify JSON parameters
 886     if (!defined($key)) {
 887         $res->{interface} = 'unknown'; # should not harm when used to verify parameters
 888         $res->{index} = 0;
 889     } elsif ($key =~ m/^([^\d]+)(\d+)$/) {
 890         $res->{interface} = $1;
 891         $res->{index} = $2;
 892     } else {
 893         return undef;
 894     }
 895
 896     foreach my $p (split (/,/, $data)) {
 897         next if $p =~ m/^\s*$/;
 898
 899         if ($p =~ m/^(file|volume|cyls|heads|secs|trans|media|snapshot|cache|format|rerror|werror|backup|aio|bps|mbps|mbps_max|bps_rd|mbps_rd|mbps_rd_max|bps_wr|mbps_wr|mbps_wr_max|iops|iops_max|iops_rd|iops_rd_max|iops_wr|iops_wr_max|size|discard|iothread|queues|serial|model)=(.+)$/) {
 900             my ($k, $v) = ($1, $2);
 901
 902             $k = 'file' if $k eq 'volume';
 903
 904             return undef if defined $res->{$k};
 905
 906             if ($k eq 'bps' || $k eq 'bps_rd' || $k eq 'bps_wr') {
 907                 return undef if !$v || $v !~ m/^\d+/;
 908                 $k = "m$k";
 909                 $v = sprintf("%.3f", $v / (1024*1024));
 910             }
 911             $res->{$k} = $v;
 912         } else {
 913             if (!$res->{file} && $p !~ m/=/) {
 914                 $res->{file} = $p;
 915             } else {
 916                 return undef;
 917             }
 918         }
 919     }
 920
 921     return undef if !$res->{file};
 922
 923     return undef if $res->{cache} &&
 924         $res->{cache} !~ m/^(off|none|writethrough|writeback|unsafe|directsync)$/;
 925     return undef if $res->{snapshot} && $res->{snapshot} !~ m/^(on|off)$/;
 926     return undef if $res->{cyls} && $res->{cyls} !~ m/^\d+$/;
 927     return undef if $res->{heads} && $res->{heads} !~ m/^\d+$/;
 928     return undef if $res->{secs} && $res->{secs} !~ m/^\d+$/;
 929     return undef if $res->{media} && $res->{media} !~ m/^(disk|cdrom)$/;
 930     return undef if $res->{trans} && $res->{trans} !~ m/^(none|lba|auto)$/;
 931     return undef if $res->{format} && $res->{format} !~ m/^(raw|cow|qcow|qed|qcow2|vmdk|cloop)$/;
 932     return undef if $res->{rerror} && $res->{rerror} !~ m/^(ignore|report|stop)$/;
 933     return undef if $res->{werror} && $res->{werror} !~ m/^(enospc|ignore|report|stop)$/;
 934     return undef if $res->{backup} && $res->{backup} !~ m/^(yes|no)$/;
 935     return undef if $res->{aio} && $res->{aio} !~ m/^(native|threads)$/;
 936     return undef if $res->{discard} && $res->{discard} !~ m/^(ignore|on)$/;
 937     return undef if $res->{iothread} && $res->{iothread} !~ m/^(on)$/;
 938     return undef if $res->{queues} && ($res->{queues} !~ m/^\d+$/ || $res->{queues} < 2);
 939
 940     return undef if $res->{mbps_rd} && $res->{mbps};
 941     return undef if $res->{mbps_wr} && $res->{mbps};
 942
 943     return undef if $res->{mbps} && $res->{mbps} !~ m/^\d+(\.\d+)?$/;
 944     return undef if $res->{mbps_max} && $res->{mbps_max} !~ m/^\d+(\.\d+)?$/;
 945     return undef if $res->{mbps_rd} && $res->{mbps_rd} !~ m/^\d+(\.\d+)?$/;
 946     return undef if $res->{mbps_rd_max} && $res->{mbps_rd_max} !~ m/^\d+(\.\d+)?$/;
 947     return undef if $res->{mbps_wr} && $res->{mbps_wr} !~ m/^\d+(\.\d+)?$/;
 948     return undef if $res->{mbps_wr_max} && $res->{mbps_wr_max} !~ m/^\d+(\.\d+)?$/;
 949
 950     return undef if $res->{iops_rd} && $res->{iops};
 951     return undef if $res->{iops_wr} && $res->{iops};
 952
 953
 954     return undef if $res->{iops} && $res->{iops} !~ m/^\d+$/;
 955     return undef if $res->{iops_max} && $res->{iops_max} !~ m/^\d+$/;
 956     return undef if $res->{iops_rd} && $res->{iops_rd} !~ m/^\d+$/;
 957     return undef if $res->{iops_rd_max} && $res->{iops_rd_max} !~ m/^\d+$/;
 958     return undef if $res->{iops_wr} && $res->{iops_wr} !~ m/^\d+$/;
 959     return undef if $res->{iops_wr_max} && $res->{iops_wr_max} !~ m/^\d+$/;
 960
 961     if ($res->{size}) {
 962         return undef if !defined($res->{size} = &$parse_size($res->{size}));
 963     }
 964
 965     if ($res->{media} && ($res->{media} eq 'cdrom')) {
 966         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
 967         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
 968         return undef if $res->{interface} eq 'virtio';
 969     }
 970
 971     # rerror does not work with scsi drives
 972     if ($res->{rerror}) {
 973         return undef if $res->{interface} eq 'scsi';
 974     }
 975
 976     return $res;
 977 }
 978
 979 my @qemu_drive_options = qw(heads secs cyls trans media format cache snapshot rerror werror aio discard iops iops_rd iops_wr iops_max iops_rd_max iops_wr_max serial);
 980
 981 sub print_drive {
 982     my ($vmid, $drive) = @_;
 983
 984     my $opts = '';
 985     foreach my $o (@qemu_drive_options, 'mbps', 'mbps_rd', 'mbps_wr', 'mbps_max', 'mbps_rd_max', 'mbps_wr_max', 'backup', 'iothread', 'queues') {
 986         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
 987     }
 988
 989     if ($drive->{size}) {
 990         $opts .= ",size=" . &$format_size($drive->{size});
 991     }
 992
 993     if (my $model = $drive->{model}) {
 994         $opts .= ",model=$model";
 995     }
 996
 997     return "$drive->{file}$opts";
 998 }
 999
1000 sub scsi_inquiry {
1001     my($fh, $noerr) = @_;
1002
1003     my $SG_IO = 0x2285;
1004     my $SG_GET_VERSION_NUM = 0x2282;
1005
1006     my $versionbuf = "\x00" x 8;
1007     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1008     if (!$ret) {
1009         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1010         return undef;
1011     }
1012     my $version = unpack("I", $versionbuf);
1013     if ($version < 30000) {
1014         die "scsi generic interface too old\n"  if !$noerr;
1015         return undef;
1016     }
1017
1018     my $buf = "\x00" x 36;
1019     my $sensebuf = "\x00" x 8;
1020     my $cmd = pack("C x3 C x1", 0x12, 36);
1021
1022     # see /usr/include/scsi/sg.h
1023     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1024
1025     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1026                       length($sensebuf), 0, length($buf), $buf,
1027                       $cmd, $sensebuf, 6000);
1028
1029     $ret = ioctl($fh, $SG_IO, $packet);
1030     if (!$ret) {
1031         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1032         return undef;
1033     }
1034
1035     my @res = unpack($sg_io_hdr_t, $packet);
1036     if ($res[17] || $res[18]) {
1037         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1038         return undef;
1039     }
1040
1041     my $res = {};
1042     (my $byte0, my $byte1, $res->{vendor},
1043      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1044
1045     $res->{removable} = $byte1 & 128 ? 1 : 0;
1046     $res->{type} = $byte0 & 31;
1047
1048     return $res;
1049 }
1050
1051 sub path_is_scsi {
1052     my ($path) = @_;
1053
1054     my $fh = IO::File->new("+<$path") || return undef;
1055     my $res = scsi_inquiry($fh, 1);
1056     close($fh);
1057
1058     return $res;
1059 }
1060
1061 sub machine_type_is_q35 {
1062     my ($conf) = @_;
1063
1064     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1065 }
1066
1067 sub print_tabletdevice_full {
1068     my ($conf) = @_;
1069
1070     my $q35 = machine_type_is_q35($conf);
1071
1072     # we use uhci for old VMs because tablet driver was buggy in older qemu
1073     my $usbbus = $q35 ? "ehci" : "uhci";
1074
1075     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1076 }
1077
1078 sub print_drivedevice_full {
1079     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1080
1081     my $device = '';
1082     my $maxdev = 0;
1083
1084     if ($drive->{interface} eq 'virtio') {
1085         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1086         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1087         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1088     } elsif ($drive->{interface} eq 'scsi') {
1089
1090         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1091         my $unit = $drive->{index} % $maxdev;
1092         my $devicetype = 'hd';
1093         my $path = '';
1094         if (drive_is_cdrom($drive)) {
1095               $devicetype = 'cd';
1096         } else {
1097               if ($drive->{file} =~ m|^/|) {
1098                   $path = $drive->{file};
1099               } else {
1100                   $path = PVE::Storage::path($storecfg, $drive->{file});
1101               }
1102
1103               if($path =~ m/^iscsi\:\/\//){
1104                   $devicetype = 'generic';
1105               } else {
1106                   if (my $info = path_is_scsi($path)) {
1107                       if ($info->{type} == 0) {
1108                           $devicetype = 'block';
1109                       } elsif ($info->{type} == 1) { # tape
1110                           $devicetype = 'generic';
1111                       }
1112                   }
1113               }
1114          }
1115
1116         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1117             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1118         } else {
1119             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1120         }
1121
1122     } elsif ($drive->{interface} eq 'ide'){
1123         $maxdev = 2;
1124         my $controller = int($drive->{index} / $maxdev);
1125         my $unit = $drive->{index} % $maxdev;
1126         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1127
1128         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1129         if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
1130             $device .= ",model=$model";
1131         }
1132     } elsif ($drive->{interface} eq 'sata'){
1133         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1134         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1135         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1136     } elsif ($drive->{interface} eq 'usb') {
1137         die "implement me";
1138         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1139     } else {
1140         die "unsupported interface type";
1141     }
1142
1143     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1144
1145     return $device;
1146 }
1147
1148 sub get_initiator_name {
1149     my $initiator;
1150
1151     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1152     while (defined(my $line = <$fh>)) {
1153         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1154         $initiator = $1;
1155         last;
1156     }
1157     $fh->close();
1158
1159     return $initiator;
1160 }
1161
1162 sub print_drive_full {
1163     my ($storecfg, $vmid, $drive) = @_;
1164
1165     my $path;
1166     my $volid = $drive->{file};
1167     my $format;
1168
1169     if (drive_is_cdrom($drive)) {
1170         $path = get_iso_path($storecfg, $vmid, $volid);
1171     } else {
1172         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1173         if ($storeid) {
1174             $path = PVE::Storage::path($storecfg, $volid);
1175             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1176             $format = qemu_img_format($scfg, $volname);
1177         } else {
1178             $path = $volid;
1179         }
1180    }
1181
1182     my $opts = '';
1183     foreach my $o (@qemu_drive_options) {
1184         next if $o eq 'bootindex';
1185         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1186     }
1187
1188     $opts .= ",format=$format" if $format && !$drive->{format};
1189
1190     foreach my $o (qw(bps bps_rd bps_wr)) {
1191         my $v = $drive->{"m$o"};
1192         $opts .= ",$o=" . int($v*1024*1024) if $v;
1193     }
1194
1195     my $cache_direct = 0;
1196
1197     if (my $cache = $drive->{cache}) {
1198         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1199     } elsif (!drive_is_cdrom($drive)) {
1200         $opts .= ",cache=none";
1201         $cache_direct = 1;
1202     }
1203
1204     # aio native works only with O_DIRECT
1205     if (!$drive->{aio}) {
1206         if($cache_direct) {
1207             $opts .= ",aio=native";
1208         } else {
1209             $opts .= ",aio=threads";
1210         }
1211     }
1212
1213     my $detectzeroes = $drive->{discard} ? "unmap" : "on";
1214     $opts .= ",detect-zeroes=$detectzeroes" if !drive_is_cdrom($drive);
1215
1216     my $pathinfo = $path ? "file=$path," : '';
1217
1218     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1219 }
1220
1221 sub print_netdevice_full {
1222     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1223
1224     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1225
1226     my $device = $net->{model};
1227     if ($net->{model} eq 'virtio') {
1228          $device = 'virtio-net-pci';
1229      };
1230
1231     my $pciaddr = print_pci_addr("$netid", $bridges);
1232     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1233     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1234         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1235         my $vectors = $net->{queues} * 2 + 2;
1236         $tmpstr .= ",vectors=$vectors,mq=on";
1237     }
1238     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1239
1240     if ($use_old_bios_files) {
1241         my $romfile;
1242         if ($device eq 'virtio-net-pci') {
1243             $romfile = 'pxe-virtio.rom';
1244         } elsif ($device eq 'e1000') {
1245             $romfile = 'pxe-e1000.rom';
1246         } elsif ($device eq 'ne2k') {
1247             $romfile = 'pxe-ne2k_pci.rom';
1248         } elsif ($device eq 'pcnet') {
1249             $romfile = 'pxe-pcnet.rom';
1250         } elsif ($device eq 'rtl8139') {
1251             $romfile = 'pxe-rtl8139.rom';
1252         }
1253         $tmpstr .= ",romfile=$romfile" if $romfile;
1254     }
1255
1256     return $tmpstr;
1257 }
1258
1259 sub print_netdev_full {
1260     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1261
1262     my $i = '';
1263     if ($netid =~ m/^net(\d+)$/) {
1264         $i = int($1);
1265     }
1266
1267     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1268
1269     my $ifname = "tap${vmid}i$i";
1270
1271     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1272     die "interface name '$ifname' is too long (max 15 character)\n"
1273         if length($ifname) >= 16;
1274
1275     my $vhostparam = '';
1276     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1277
1278     my $vmname = $conf->{name} || "vm$vmid";
1279
1280     my $netdev = "";
1281     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1282
1283     if ($net->{bridge}) {
1284         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1285     } else {
1286         $netdev = "type=user,id=$netid,hostname=$vmname";
1287     }
1288
1289     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1290
1291     return $netdev;
1292 }
1293
1294 sub drive_is_cdrom {
1295     my ($drive) = @_;
1296
1297     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1298
1299 }
1300
1301 sub parse_numa {
1302     my ($data) = @_;
1303
1304     my $res = {};
1305
1306     foreach my $kvp (split(/,/, $data)) {
1307
1308         if ($kvp =~ m/^memory=(\S+)$/) {
1309             $res->{memory} = $1;
1310         } elsif ($kvp =~ m/^policy=(preferred|bind|interleave)$/) {
1311             $res->{policy} = $1;
1312         } elsif ($kvp =~ m/^cpus=(\d+)(-(\d+))?$/) {
1313             $res->{cpus}->{start} = $1;
1314             $res->{cpus}->{end} = $3;
1315         } elsif ($kvp =~ m/^hostnodes=(\d+)(-(\d+))?$/) {
1316             $res->{hostnodes}->{start} = $1;
1317             $res->{hostnodes}->{end} = $3;
1318         } else {
1319             return undef;
1320         }
1321     }
1322
1323     return $res;
1324 }
1325
1326 sub parse_hostpci {
1327     my ($value) = @_;
1328
1329     return undef if !$value;
1330
1331
1332     my @list = split(/,/, $value);
1333     my $found;
1334
1335     my $res = {};
1336     foreach my $kv (@list) {
1337
1338         if ($kv =~ m/^(host=)?([a-f0-9]{2}:[a-f0-9]{2})(\.([a-f0-9]))?$/) {
1339             $found = 1;
1340             if(defined($4)){
1341                 push @{$res->{pciid}}, { id => $2 , function => $4};
1342
1343             }else{
1344                 my $pcidevices = lspci($2);
1345                 $res->{pciid} = $pcidevices->{$2};
1346             }
1347         } elsif ($kv =~ m/^rombar=(on|off)$/) {
1348             $res->{rombar} = $1;
1349         } elsif ($kv =~ m/^x-vga=(on|off)$/) {
1350             $res->{'x-vga'} = $1;
1351         } elsif ($kv =~ m/^pcie=(\d+)$/) {
1352             $res->{pcie} = 1 if $1 == 1;
1353         } else {
1354             warn "unknown hostpci setting '$kv'\n";
1355         }
1356     }
1357
1358     return undef if !$found;
1359
1360     return $res;
1361 }
1362
1363 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1364 sub parse_net {
1365     my ($data) = @_;
1366
1367     my $res = {};
1368
1369     foreach my $kvp (split(/,/, $data)) {
1370
1371         if ($kvp =~ m/^(ne2k_pci|e1000|e1000-82540em|e1000-82544gc|e1000-82545em|rtl8139|pcnet|virtio|ne2k_isa|i82551|i82557b|i82559er|vmxnet3)(=([0-9a-f]{2}(:[0-9a-f]{2}){5}))?$/i) {
1372             my $model = lc($1);
1373             my $mac = defined($3) ? uc($3) : PVE::Tools::random_ether_addr();
1374             $res->{model} = $model;
1375             $res->{macaddr} = $mac;
1376         } elsif ($kvp =~ m/^bridge=(\S+)$/) {
1377             $res->{bridge} = $1;
1378         } elsif ($kvp =~ m/^queues=(\d+)$/) {
1379             $res->{queues} = $1;
1380         } elsif ($kvp =~ m/^rate=(\d+(\.\d+)?)$/) {
1381             $res->{rate} = $1;
1382         } elsif ($kvp =~ m/^tag=(\d+)$/) {
1383             $res->{tag} = $1;
1384         } elsif ($kvp =~ m/^firewall=([01])$/) {
1385             $res->{firewall} = $1;
1386         } elsif ($kvp =~ m/^link_down=([01])$/) {
1387             $res->{link_down} = $1;
1388         } else {
1389             return undef;
1390         }
1391
1392     }
1393
1394     return undef if !$res->{model};
1395
1396     return $res;
1397 }
1398
1399 sub print_net {
1400     my $net = shift;
1401
1402     my $res = "$net->{model}";
1403     $res .= "=$net->{macaddr}" if $net->{macaddr};
1404     $res .= ",bridge=$net->{bridge}" if $net->{bridge};
1405     $res .= ",rate=$net->{rate}" if $net->{rate};
1406     $res .= ",tag=$net->{tag}" if $net->{tag};
1407     $res .= ",firewall=1" if $net->{firewall};
1408     $res .= ",link_down=1" if $net->{link_down};
1409     $res .= ",queues=$net->{queues}" if $net->{queues};
1410
1411     return $res;
1412 }
1413
1414 sub add_random_macs {
1415     my ($settings) = @_;
1416
1417     foreach my $opt (keys %$settings) {
1418         next if $opt !~ m/^net(\d+)$/;
1419         my $net = parse_net($settings->{$opt});
1420         next if !$net;
1421         $settings->{$opt} = print_net($net);
1422     }
1423 }
1424
1425 sub add_unused_volume {
1426     my ($config, $volid) = @_;
1427
1428     my $key;
1429     for (my $ind = $MAX_UNUSED_DISKS - 1; $ind >= 0; $ind--) {
1430         my $test = "unused$ind";
1431         if (my $vid = $config->{$test}) {
1432             return if $vid eq $volid; # do not add duplicates
1433         } else {
1434             $key = $test;
1435         }
1436     }
1437
1438     die "To many unused volume - please delete them first.\n" if !$key;
1439
1440     $config->{$key} = $volid;
1441
1442     return $key;
1443 }
1444
1445 sub vm_is_volid_owner {
1446     my ($storecfg, $vmid, $volid) = @_;
1447
1448     if ($volid !~  m|^/|) {
1449         my ($path, $owner);
1450         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1451         if ($owner && ($owner == $vmid)) {
1452             return 1;
1453         }
1454     }
1455
1456     return undef;
1457 }
1458
1459 sub split_flagged_list {
1460     my $text = shift || '';
1461     $text =~ s/[,;]/ /g;
1462     $text =~ s/^\s+//;
1463     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
1464 }
1465
1466 sub join_flagged_list {
1467     my ($how, $lst) = @_;
1468     join $how, map { $lst->{$_} . $_ } keys %$lst;
1469 }
1470
1471 sub vmconfig_delete_pending_option {
1472     my ($conf, $key, $force) = @_;
1473
1474     delete $conf->{pending}->{$key};
1475     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1476     $pending_delete_hash->{$key} = $force ? '!' : '';
1477     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1478 }
1479
1480 sub vmconfig_undelete_pending_option {
1481     my ($conf, $key) = @_;
1482
1483     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1484     delete $pending_delete_hash->{$key};
1485
1486     if (%$pending_delete_hash) {
1487         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1488     } else {
1489         delete $conf->{pending}->{delete};
1490     }
1491 }
1492
1493 sub vmconfig_register_unused_drive {
1494     my ($storecfg, $vmid, $conf, $drive) = @_;
1495
1496     if (!drive_is_cdrom($drive)) {
1497         my $volid = $drive->{file};
1498         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1499             add_unused_volume($conf, $volid, $vmid);
1500         }
1501     }
1502 }
1503
1504 sub vmconfig_cleanup_pending {
1505     my ($conf) = @_;
1506
1507     # remove pending changes when nothing changed
1508     my $changes;
1509     foreach my $opt (keys %{$conf->{pending}}) {
1510         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
1511             $changes = 1;
1512             delete $conf->{pending}->{$opt};
1513         }
1514     }
1515
1516     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
1517     my $pending_delete_hash = {};
1518     while (my ($opt, $force) = each %$current_delete_hash) {
1519         if (defined($conf->{$opt})) {
1520             $pending_delete_hash->{$opt} = $force;
1521         } else {
1522             $changes = 1;
1523         }
1524     }
1525
1526     if (%$pending_delete_hash) {
1527         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1528     } else {
1529         delete $conf->{pending}->{delete};
1530     }
1531
1532     return $changes;
1533 }
1534
1535 my $valid_smbios1_options = {
1536     manufacturer => '\S+',
1537     product => '\S+',
1538     version => '\S+',
1539     serial => '\S+',
1540     uuid => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1541     sku => '\S+',
1542     family => '\S+',
1543 };
1544
1545 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
1546 sub parse_smbios1 {
1547     my ($data) = @_;
1548
1549     my $res = {};
1550
1551     foreach my $kvp (split(/,/, $data)) {
1552         return undef if $kvp !~ m/^(\S+)=(.+)$/;
1553         my ($k, $v) = split(/=/, $kvp);
1554         return undef if !defined($k) || !defined($v);
1555         return undef if !$valid_smbios1_options->{$k};
1556         return undef if $v !~ m/^$valid_smbios1_options->{$k}$/;
1557         $res->{$k} = $v;
1558     }
1559
1560     return $res;
1561 }
1562
1563 sub print_smbios1 {
1564     my ($smbios1) = @_;
1565
1566     my $data = '';
1567     foreach my $k (keys %$smbios1) {
1568         next if !defined($smbios1->{$k});
1569         next if !$valid_smbios1_options->{$k};
1570         $data .= ',' if $data;
1571         $data .= "$k=$smbios1->{$k}";
1572     }
1573     return $data;
1574 }
1575
1576 PVE::JSONSchema::register_format('pve-qm-smbios1', \&verify_smbios1);
1577 sub verify_smbios1 {
1578     my ($value, $noerr) = @_;
1579
1580     return $value if parse_smbios1($value);
1581
1582     return undef if $noerr;
1583
1584     die "unable to parse smbios (type 1) options\n";
1585 }
1586
1587 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
1588 sub verify_bootdisk {
1589     my ($value, $noerr) = @_;
1590
1591     return $value if valid_drivename($value);
1592
1593     return undef if $noerr;
1594
1595     die "invalid boot disk '$value'\n";
1596 }
1597
1598 PVE::JSONSchema::register_format('pve-qm-numanode', \&verify_numa);
1599 sub verify_numa {
1600     my ($value, $noerr) = @_;
1601
1602     return $value if parse_numa($value);
1603
1604     return undef if $noerr;
1605
1606     die "unable to parse numa options\n";
1607 }
1608
1609 PVE::JSONSchema::register_format('pve-qm-net', \&verify_net);
1610 sub verify_net {
1611     my ($value, $noerr) = @_;
1612
1613     return $value if parse_net($value);
1614
1615     return undef if $noerr;
1616
1617     die "unable to parse network options\n";
1618 }
1619
1620 PVE::JSONSchema::register_format('pve-qm-drive', \&verify_drive);
1621 sub verify_drive {
1622     my ($value, $noerr) = @_;
1623
1624     return $value if parse_drive(undef, $value);
1625
1626     return undef if $noerr;
1627
1628     die "unable to parse drive options\n";
1629 }
1630
1631 PVE::JSONSchema::register_format('pve-qm-hostpci', \&verify_hostpci);
1632 sub verify_hostpci {
1633     my ($value, $noerr) = @_;
1634
1635     return $value if parse_hostpci($value);
1636
1637     return undef if $noerr;
1638
1639     die "unable to parse pci id\n";
1640 }
1641
1642 PVE::JSONSchema::register_format('pve-qm-watchdog', \&verify_watchdog);
1643 sub verify_watchdog {
1644     my ($value, $noerr) = @_;
1645
1646     return $value if parse_watchdog($value);
1647
1648     return undef if $noerr;
1649
1650     die "unable to parse watchdog options\n";
1651 }
1652
1653 sub parse_watchdog {
1654     my ($value) = @_;
1655
1656     return undef if !$value;
1657
1658     my $res = {};
1659
1660     foreach my $p (split(/,/, $value)) {
1661         next if $p =~ m/^\s*$/;
1662
1663         if ($p =~ m/^(model=)?(i6300esb|ib700)$/) {
1664             $res->{model} = $2;
1665         } elsif ($p =~ m/^(action=)?(reset|shutdown|poweroff|pause|debug|none)$/) {
1666             $res->{action} = $2;
1667         } else {
1668             return undef;
1669         }
1670     }
1671
1672     return $res;
1673 }
1674
1675 sub parse_usb_device {
1676     my ($value) = @_;
1677
1678     return undef if !$value;
1679
1680     my @dl = split(/,/, $value);
1681     my $found;
1682
1683     my $res = {};
1684     foreach my $v (@dl) {
1685         if ($v =~ m/^host=(0x)?([0-9A-Fa-f]{4}):(0x)?([0-9A-Fa-f]{4})$/) {
1686             $found = 1;
1687             $res->{vendorid} = $2;
1688             $res->{productid} = $4;
1689         } elsif ($v =~ m/^host=(\d+)\-(\d+(\.\d+)*)$/) {
1690             $found = 1;
1691             $res->{hostbus} = $1;
1692             $res->{hostport} = $2;
1693         } elsif ($v =~ m/^spice$/) {
1694             $found = 1;
1695             $res->{spice} = 1;
1696         } else {
1697             return undef;
1698         }
1699     }
1700     return undef if !$found;
1701
1702     return $res;
1703 }
1704
1705 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
1706 sub verify_usb_device {
1707     my ($value, $noerr) = @_;
1708
1709     return $value if parse_usb_device($value);
1710
1711     return undef if $noerr;
1712
1713     die "unable to parse usb device\n";
1714 }
1715
1716 # add JSON properties for create and set function
1717 sub json_config_properties {
1718     my $prop = shift;
1719
1720     foreach my $opt (keys %$confdesc) {
1721         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
1722         $prop->{$opt} = $confdesc->{$opt};
1723     }
1724
1725     return $prop;
1726 }
1727
1728 sub check_type {
1729     my ($key, $value) = @_;
1730
1731     die "unknown setting '$key'\n" if !$confdesc->{$key};
1732
1733     my $type = $confdesc->{$key}->{type};
1734
1735     if (!defined($value)) {
1736         die "got undefined value\n";
1737     }
1738
1739     if ($value =~ m/[\n\r]/) {
1740         die "property contains a line feed\n";
1741     }
1742
1743     if ($type eq 'boolean') {
1744         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
1745         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
1746         die "type check ('boolean') failed - got '$value'\n";
1747     } elsif ($type eq 'integer') {
1748         return int($1) if $value =~ m/^(\d+)$/;
1749         die "type check ('integer') failed - got '$value'\n";
1750     } elsif ($type eq 'number') {
1751         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
1752         die "type check ('number') failed - got '$value'\n";
1753     } elsif ($type eq 'string') {
1754         if (my $fmt = $confdesc->{$key}->{format}) {
1755             if ($fmt eq 'pve-qm-drive') {
1756                 # special case - we need to pass $key to parse_drive()
1757                 my $drive = parse_drive($key, $value);
1758                 return $value if $drive;
1759                 die "unable to parse drive options\n";
1760             }
1761             PVE::JSONSchema::check_format($fmt, $value);
1762             return $value;
1763         }
1764         $value =~ s/^\"(.*)\"$/$1/;
1765         return $value;
1766     } else {
1767         die "internal error"
1768     }
1769 }
1770
1771 sub lock_config_full {
1772     my ($vmid, $timeout, $code, @param) = @_;
1773
1774     my $filename = config_file_lock($vmid);
1775
1776     my $res = lock_file($filename, $timeout, $code, @param);
1777
1778     die $@ if $@;
1779
1780     return $res;
1781 }
1782
1783 sub lock_config_mode {
1784     my ($vmid, $timeout, $shared, $code, @param) = @_;
1785
1786     my $filename = config_file_lock($vmid);
1787
1788     my $res = lock_file_full($filename, $timeout, $shared, $code, @param);
1789
1790     die $@ if $@;
1791
1792     return $res;
1793 }
1794
1795 sub lock_config {
1796     my ($vmid, $code, @param) = @_;
1797
1798     return lock_config_full($vmid, 10, $code, @param);
1799 }
1800
1801 sub cfs_config_path {
1802     my ($vmid, $node) = @_;
1803
1804     $node = $nodename if !$node;
1805     return "nodes/$node/qemu-server/$vmid.conf";
1806 }
1807
1808 sub check_iommu_support{
1809     #fixme : need to check IOMMU support
1810     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
1811
1812     my $iommu=1;
1813     return $iommu;
1814
1815 }
1816
1817 sub config_file {
1818     my ($vmid, $node) = @_;
1819
1820     my $cfspath = cfs_config_path($vmid, $node);
1821     return "/etc/pve/$cfspath";
1822 }
1823
1824 sub config_file_lock {
1825     my ($vmid) = @_;
1826
1827     return "$lock_dir/lock-$vmid.conf";
1828 }
1829
1830 sub touch_config {
1831     my ($vmid) = @_;
1832
1833     my $conf = config_file($vmid);
1834     utime undef, undef, $conf;
1835 }
1836
1837 sub destroy_vm {
1838     my ($storecfg, $vmid, $keep_empty_config) = @_;
1839
1840     my $conffile = config_file($vmid);
1841
1842     my $conf = load_config($vmid);
1843
1844     check_lock($conf);
1845
1846     # only remove disks owned by this VM
1847     foreach_drive($conf, sub {
1848         my ($ds, $drive) = @_;
1849
1850         return if drive_is_cdrom($drive);
1851
1852         my $volid = $drive->{file};
1853
1854         return if !$volid || $volid =~ m|^/|;
1855
1856         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
1857         return if !$path || !$owner || ($owner != $vmid);
1858
1859         PVE::Storage::vdisk_free($storecfg, $volid);
1860     });
1861
1862     if ($keep_empty_config) {
1863         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
1864     } else {
1865         unlink $conffile;
1866     }
1867
1868     # also remove unused disk
1869     eval {
1870         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
1871
1872         eval {
1873             PVE::Storage::foreach_volid($dl, sub {
1874                 my ($volid, $sid, $volname, $d) = @_;
1875                 PVE::Storage::vdisk_free($storecfg, $volid);
1876             });
1877         };
1878         warn $@ if $@;
1879
1880     };
1881     warn $@ if $@;
1882 }
1883
1884 sub load_config {
1885     my ($vmid, $node) = @_;
1886
1887     my $cfspath = cfs_config_path($vmid, $node);
1888
1889     my $conf = PVE::Cluster::cfs_read_file($cfspath);
1890
1891     die "no such VM ('$vmid')\n" if !defined($conf);
1892
1893     return $conf;
1894 }
1895
1896 sub parse_vm_config {
1897     my ($filename, $raw) = @_;
1898
1899     return undef if !defined($raw);
1900
1901     my $res = {
1902         digest => Digest::SHA::sha1_hex($raw),
1903         snapshots => {},
1904         pending => {},
1905     };
1906
1907     $filename =~ m|/qemu-server/(\d+)\.conf$|
1908         || die "got strange filename '$filename'";
1909
1910     my $vmid = $1;
1911
1912     my $conf = $res;
1913     my $descr;
1914     my $section = '';
1915
1916     my @lines = split(/\n/, $raw);
1917     foreach my $line (@lines) {
1918         next if $line =~ m/^\s*$/;
1919
1920         if ($line =~ m/^\[PENDING\]\s*$/i) {
1921             $section = 'pending';
1922             if (defined($descr)) {
1923                 $descr =~ s/\s+$//;
1924                 $conf->{description} = $descr;
1925             }
1926             $descr = undef;
1927             $conf = $res->{$section} = {};
1928             next;
1929
1930         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
1931             $section = $1;
1932             if (defined($descr)) {
1933                 $descr =~ s/\s+$//;
1934                 $conf->{description} = $descr;
1935             }
1936             $descr = undef;
1937             $conf = $res->{snapshots}->{$section} = {};
1938             next;
1939         }
1940
1941         if ($line =~ m/^\#(.*)\s*$/) {
1942             $descr = '' if !defined($descr);
1943             $descr .= PVE::Tools::decode_text($1) . "\n";
1944             next;
1945         }
1946
1947         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
1948             $descr = '' if !defined($descr);
1949             $descr .= PVE::Tools::decode_text($2);
1950         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
1951             $conf->{snapstate} = $1;
1952         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
1953             my $key = $1;
1954             my $value = $2;
1955             $conf->{$key} = $value;
1956         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
1957             my $value = $1;
1958             if ($section eq 'pending') {
1959                 $conf->{delete} = $value; # we parse this later
1960             } else {
1961                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
1962             }
1963         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
1964             my $key = $1;
1965             my $value = $2;
1966             eval { $value = check_type($key, $value); };
1967             if ($@) {
1968                 warn "vm $vmid - unable to parse value of '$key' - $@";
1969             } else {
1970                 my $fmt = $confdesc->{$key}->{format};
1971                 if ($fmt && $fmt eq 'pve-qm-drive') {
1972                     my $v = parse_drive($key, $value);
1973                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
1974                         $v->{file} = $volid;
1975                         $value = print_drive($vmid, $v);
1976                     } else {
1977                         warn "vm $vmid - unable to parse value of '$key'\n";
1978                         next;
1979                     }
1980                 }
1981
1982                 if ($key eq 'cdrom') {
1983                     $conf->{ide2} = $value;
1984                 } else {
1985                     $conf->{$key} = $value;
1986                 }
1987             }
1988         }
1989     }
1990
1991     if (defined($descr)) {
1992         $descr =~ s/\s+$//;
1993         $conf->{description} = $descr;
1994     }
1995     delete $res->{snapstate}; # just to be sure
1996
1997     return $res;
1998 }
1999
2000 sub write_vm_config {
2001     my ($filename, $conf) = @_;
2002
2003     delete $conf->{snapstate}; # just to be sure
2004
2005     if ($conf->{cdrom}) {
2006         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2007         $conf->{ide2} = $conf->{cdrom};
2008         delete $conf->{cdrom};
2009     }
2010
2011     # we do not use 'smp' any longer
2012     if ($conf->{sockets}) {
2013         delete $conf->{smp};
2014     } elsif ($conf->{smp}) {
2015         $conf->{sockets} = $conf->{smp};
2016         delete $conf->{cores};
2017         delete $conf->{smp};
2018     }
2019
2020     my $used_volids = {};
2021
2022     my $cleanup_config = sub {
2023         my ($cref, $pending, $snapname) = @_;
2024
2025         foreach my $key (keys %$cref) {
2026             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2027                 $key eq 'snapstate' || $key eq 'pending';
2028             my $value = $cref->{$key};
2029             if ($key eq 'delete') {
2030                 die "propertry 'delete' is only allowed in [PENDING]\n"
2031                     if !$pending;
2032                 # fixme: check syntax?
2033                 next;
2034             }
2035             eval { $value = check_type($key, $value); };
2036             die "unable to parse value of '$key' - $@" if $@;
2037
2038             $cref->{$key} = $value;
2039
2040             if (!$snapname && valid_drivename($key)) {
2041                 my $drive = parse_drive($key, $value);
2042                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2043             }
2044         }
2045     };
2046
2047     &$cleanup_config($conf);
2048
2049     &$cleanup_config($conf->{pending}, 1);
2050
2051     foreach my $snapname (keys %{$conf->{snapshots}}) {
2052         die "internal error" if $snapname eq 'pending';
2053         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2054     }
2055
2056     # remove 'unusedX' settings if we re-add a volume
2057     foreach my $key (keys %$conf) {
2058         my $value = $conf->{$key};
2059         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2060             delete $conf->{$key};
2061         }
2062     }
2063
2064     my $generate_raw_config = sub {
2065         my ($conf, $pending) = @_;
2066
2067         my $raw = '';
2068
2069         # add description as comment to top of file
2070         if (defined(my $descr = $conf->{description})) {
2071             if ($descr) {
2072                 foreach my $cl (split(/\n/, $descr)) {
2073                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2074                 }
2075             } else {
2076                 $raw .= "#\n" if $pending;
2077             }
2078         }
2079
2080         foreach my $key (sort keys %$conf) {
2081             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2082             $raw .= "$key: $conf->{$key}\n";
2083         }
2084         return $raw;
2085     };
2086
2087     my $raw = &$generate_raw_config($conf);
2088
2089     if (scalar(keys %{$conf->{pending}})){
2090         $raw .= "\n[PENDING]\n";
2091         $raw .= &$generate_raw_config($conf->{pending}, 1);
2092     }
2093
2094     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2095         $raw .= "\n[$snapname]\n";
2096         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2097     }
2098
2099     return $raw;
2100 }
2101
2102 sub update_config_nolock {
2103     my ($vmid, $conf, $skiplock) = @_;
2104
2105     check_lock($conf) if !$skiplock;
2106
2107     my $cfspath = cfs_config_path($vmid);
2108
2109     PVE::Cluster::cfs_write_file($cfspath, $conf);
2110 }
2111
2112 sub update_config {
2113     my ($vmid, $conf, $skiplock) = @_;
2114
2115     lock_config($vmid, &update_config_nolock, $conf, $skiplock);
2116 }
2117
2118 sub load_defaults {
2119
2120     my $res = {};
2121
2122     # we use static defaults from our JSON schema configuration
2123     foreach my $key (keys %$confdesc) {
2124         if (defined(my $default = $confdesc->{$key}->{default})) {
2125             $res->{$key} = $default;
2126         }
2127     }
2128
2129     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2130     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2131
2132     return $res;
2133 }
2134
2135 sub config_list {
2136     my $vmlist = PVE::Cluster::get_vmlist();
2137     my $res = {};
2138     return $res if !$vmlist || !$vmlist->{ids};
2139     my $ids = $vmlist->{ids};
2140
2141     foreach my $vmid (keys %$ids) {
2142         my $d = $ids->{$vmid};
2143         next if !$d->{node} || $d->{node} ne $nodename;
2144         next if !$d->{type} || $d->{type} ne 'qemu';
2145         $res->{$vmid}->{exists} = 1;
2146     }
2147     return $res;
2148 }
2149
2150 # test if VM uses local resources (to prevent migration)
2151 sub check_local_resources {
2152     my ($conf, $noerr) = @_;
2153
2154     my $loc_res = 0;
2155
2156     $loc_res = 1 if $conf->{hostusb}; # old syntax
2157     $loc_res = 1 if $conf->{hostpci}; # old syntax
2158
2159     foreach my $k (keys %$conf) {
2160         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2161         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2162     }
2163
2164     die "VM uses local resources\n" if $loc_res && !$noerr;
2165
2166     return $loc_res;
2167 }
2168
2169 # check if used storages are available on all nodes (use by migrate)
2170 sub check_storage_availability {
2171     my ($storecfg, $conf, $node) = @_;
2172
2173     foreach_drive($conf, sub {
2174         my ($ds, $drive) = @_;
2175
2176         my $volid = $drive->{file};
2177         return if !$volid;
2178
2179         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2180         return if !$sid;
2181
2182         # check if storage is available on both nodes
2183         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2184         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2185    });
2186 }
2187
2188 # list nodes where all VM images are available (used by has_feature API)
2189 sub shared_nodes {
2190     my ($conf, $storecfg) = @_;
2191
2192     my $nodelist = PVE::Cluster::get_nodelist();
2193     my $nodehash = { map { $_ => 1 } @$nodelist };
2194     my $nodename = PVE::INotify::nodename();
2195
2196     foreach_drive($conf, sub {
2197         my ($ds, $drive) = @_;
2198
2199         my $volid = $drive->{file};
2200         return if !$volid;
2201
2202         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2203         if ($storeid) {
2204             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2205             if ($scfg->{disable}) {
2206                 $nodehash = {};
2207             } elsif (my $avail = $scfg->{nodes}) {
2208                 foreach my $node (keys %$nodehash) {
2209                     delete $nodehash->{$node} if !$avail->{$node};
2210                 }
2211             } elsif (!$scfg->{shared}) {
2212                 foreach my $node (keys %$nodehash) {
2213                     delete $nodehash->{$node} if $node ne $nodename
2214                 }
2215             }
2216         }
2217     });
2218
2219     return $nodehash
2220 }
2221
2222 sub check_lock {
2223     my ($conf) = @_;
2224
2225     die "VM is locked ($conf->{lock})\n" if $conf->{lock};
2226 }
2227
2228 sub check_cmdline {
2229     my ($pidfile, $pid) = @_;
2230
2231     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2232     if (defined($fh)) {
2233         my $line = <$fh>;
2234         $fh->close;
2235         return undef if !$line;
2236         my @param = split(/\0/, $line);
2237
2238         my $cmd = $param[0];
2239         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2240
2241         for (my $i = 0; $i < scalar (@param); $i++) {
2242             my $p = $param[$i];
2243             next if !$p;
2244             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2245                 my $p = $param[$i+1];
2246                 return 1 if $p && ($p eq $pidfile);
2247                 return undef;
2248             }
2249         }
2250     }
2251     return undef;
2252 }
2253
2254 sub check_running {
2255     my ($vmid, $nocheck, $node) = @_;
2256
2257     my $filename = config_file($vmid, $node);
2258
2259     die "unable to find configuration file for VM $vmid - no such machine\n"
2260         if !$nocheck && ! -f $filename;
2261
2262     my $pidfile = pidfile_name($vmid);
2263
2264     if (my $fd = IO::File->new("<$pidfile")) {
2265         my $st = stat($fd);
2266         my $line = <$fd>;
2267         close($fd);
2268
2269         my $mtime = $st->mtime;
2270         if ($mtime > time()) {
2271             warn "file '$filename' modified in future\n";
2272         }
2273
2274         if ($line =~ m/^(\d+)$/) {
2275             my $pid = $1;
2276             if (check_cmdline($pidfile, $pid)) {
2277                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2278                     return $pid;
2279                 }
2280             }
2281         }
2282     }
2283
2284     return undef;
2285 }
2286
2287 sub vzlist {
2288
2289     my $vzlist = config_list();
2290
2291     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2292
2293     while (defined(my $de = $fd->read)) {
2294         next if $de !~ m/^(\d+)\.pid$/;
2295         my $vmid = $1;
2296         next if !defined($vzlist->{$vmid});
2297         if (my $pid = check_running($vmid)) {
2298             $vzlist->{$vmid}->{pid} = $pid;
2299         }
2300     }
2301
2302     return $vzlist;
2303 }
2304
2305 sub disksize {
2306     my ($storecfg, $conf) = @_;
2307
2308     my $bootdisk = $conf->{bootdisk};
2309     return undef if !$bootdisk;
2310     return undef if !valid_drivename($bootdisk);
2311
2312     return undef if !$conf->{$bootdisk};
2313
2314     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2315     return undef if !defined($drive);
2316
2317     return undef if drive_is_cdrom($drive);
2318
2319     my $volid = $drive->{file};
2320     return undef if !$volid;
2321
2322     return $drive->{size};
2323 }
2324
2325 my $last_proc_pid_stat;
2326
2327 # get VM status information
2328 # This must be fast and should not block ($full == false)
2329 # We only query KVM using QMP if $full == true (this can be slow)
2330 sub vmstatus {
2331     my ($opt_vmid, $full) = @_;
2332
2333     my $res = {};
2334
2335     my $storecfg = PVE::Storage::config();
2336
2337     my $list = vzlist();
2338     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2339
2340     my $cpucount = $cpuinfo->{cpus} || 1;
2341
2342     foreach my $vmid (keys %$list) {
2343         next if $opt_vmid && ($vmid ne $opt_vmid);
2344
2345         my $cfspath = cfs_config_path($vmid);
2346         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2347
2348         my $d = {};
2349         $d->{pid} = $list->{$vmid}->{pid};
2350
2351         # fixme: better status?
2352         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2353
2354         my $size = disksize($storecfg, $conf);
2355         if (defined($size)) {
2356             $d->{disk} = 0; # no info available
2357             $d->{maxdisk} = $size;
2358         } else {
2359             $d->{disk} = 0;
2360             $d->{maxdisk} = 0;
2361         }
2362
2363         $d->{cpus} = ($conf->{sockets} || 1) * ($conf->{cores} || 1);
2364         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2365         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2366
2367         $d->{name} = $conf->{name} || "VM $vmid";
2368         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024) : 0;
2369
2370         if ($conf->{balloon}) {
2371             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2372             $d->{shares} = defined($conf->{shares}) ? $conf->{shares} : 1000;
2373         }
2374
2375         $d->{uptime} = 0;
2376         $d->{cpu} = 0;
2377         $d->{mem} = 0;
2378
2379         $d->{netout} = 0;
2380         $d->{netin} = 0;
2381
2382         $d->{diskread} = 0;
2383         $d->{diskwrite} = 0;
2384
2385         $d->{template} = is_template($conf);
2386
2387         $res->{$vmid} = $d;
2388     }
2389
2390     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2391     foreach my $dev (keys %$netdev) {
2392         next if $dev !~ m/^tap([1-9]\d*)i/;
2393         my $vmid = $1;
2394         my $d = $res->{$vmid};
2395         next if !$d;
2396
2397         $d->{netout} += $netdev->{$dev}->{receive};
2398         $d->{netin} += $netdev->{$dev}->{transmit};
2399
2400         if ($full) {
2401             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2402             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2403         }
2404
2405     }
2406
2407     my $ctime = gettimeofday;
2408
2409     foreach my $vmid (keys %$list) {
2410
2411         my $d = $res->{$vmid};
2412         my $pid = $d->{pid};
2413         next if !$pid;
2414
2415         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2416         next if !$pstat; # not running
2417
2418         my $used = $pstat->{utime} + $pstat->{stime};
2419
2420         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2421
2422         if ($pstat->{vsize}) {
2423             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2424         }
2425
2426         my $old = $last_proc_pid_stat->{$pid};
2427         if (!$old) {
2428             $last_proc_pid_stat->{$pid} = {
2429                 time => $ctime,
2430                 used => $used,
2431                 cpu => 0,
2432             };
2433             next;
2434         }
2435
2436         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2437
2438         if ($dtime > 1000) {
2439             my $dutime = $used -  $old->{used};
2440
2441             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2442             $last_proc_pid_stat->{$pid} = {
2443                 time => $ctime,
2444                 used => $used,
2445                 cpu => $d->{cpu},
2446             };
2447         } else {
2448             $d->{cpu} = $old->{cpu};
2449         }
2450     }
2451
2452     return $res if !$full;
2453
2454     my $qmpclient = PVE::QMPClient->new();
2455
2456     my $ballooncb = sub {
2457         my ($vmid, $resp) = @_;
2458
2459         my $info = $resp->{'return'};
2460         return if !$info->{max_mem};
2461
2462         my $d = $res->{$vmid};
2463
2464         # use memory assigned to VM
2465         $d->{maxmem} = $info->{max_mem};
2466         $d->{balloon} = $info->{actual};
2467
2468         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2469             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2470             $d->{freemem} = $info->{free_mem};
2471         }
2472
2473         $d->{ballooninfo} = $info;
2474     };
2475
2476     my $blockstatscb = sub {
2477         my ($vmid, $resp) = @_;
2478         my $data = $resp->{'return'} || [];
2479         my $totalrdbytes = 0;
2480         my $totalwrbytes = 0;
2481
2482         for my $blockstat (@$data) {
2483             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2484             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2485
2486             $blockstat->{device} =~ s/drive-//;
2487             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2488         }
2489         $res->{$vmid}->{diskread} = $totalrdbytes;
2490         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2491     };
2492
2493     my $statuscb = sub {
2494         my ($vmid, $resp) = @_;
2495
2496         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2497         # this fails if ballon driver is not loaded, so this must be
2498         # the last commnand (following command are aborted if this fails).
2499         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2500
2501         my $status = 'unknown';
2502         if (!defined($status = $resp->{'return'}->{status})) {
2503             warn "unable to get VM status\n";
2504             return;
2505         }
2506
2507         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2508     };
2509
2510     foreach my $vmid (keys %$list) {
2511         next if $opt_vmid && ($vmid ne $opt_vmid);
2512         next if !$res->{$vmid}->{pid}; # not running
2513         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2514     }
2515
2516     $qmpclient->queue_execute(undef, 1);
2517
2518     foreach my $vmid (keys %$list) {
2519         next if $opt_vmid && ($vmid ne $opt_vmid);
2520         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2521     }
2522
2523     return $res;
2524 }
2525
2526 sub foreach_dimm {
2527     my ($conf, $vmid, $memory, $sockets, $func) = @_;
2528
2529     my $dimm_id = 0;
2530     my $current_size = 1024;
2531     my $dimm_size = 512;
2532     return if $current_size == $memory;
2533
2534     for (my $j = 0; $j < 8; $j++) {
2535         for (my $i = 0; $i < 32; $i++) {
2536             my $name = "dimm${dimm_id}";
2537             $dimm_id++;
2538             my $numanode = $i % $sockets;
2539             $current_size += $dimm_size;
2540             &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
2541             return  $current_size if $current_size >= $memory;
2542         }
2543         $dimm_size *= 2;
2544     }
2545 }
2546
2547 sub foreach_reverse_dimm {
2548     my ($conf, $vmid, $memory, $sockets, $func) = @_;
2549
2550     my $dimm_id = 253;
2551     my $current_size = 4177920;
2552     my $dimm_size = 65536;
2553     return if $current_size == $memory;
2554
2555     for (my $j = 0; $j < 8; $j++) {
2556         for (my $i = 0; $i < 32; $i++) {
2557             my $name = "dimm${dimm_id}";
2558             $dimm_id--;
2559             my $numanode = $i % $sockets;
2560             $current_size -= $dimm_size;
2561             &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
2562             return  $current_size if $current_size <= $memory;
2563         }
2564         $dimm_size /= 2;
2565     }
2566 }
2567
2568 sub foreach_drive {
2569     my ($conf, $func) = @_;
2570
2571     foreach my $ds (keys %$conf) {
2572         next if !valid_drivename($ds);
2573
2574         my $drive = parse_drive($ds, $conf->{$ds});
2575         next if !$drive;
2576
2577         &$func($ds, $drive);
2578     }
2579 }
2580
2581 sub foreach_volid {
2582     my ($conf, $func) = @_;
2583
2584     my $volhash = {};
2585
2586     my $test_volid = sub {
2587         my ($volid, $is_cdrom) = @_;
2588
2589         return if !$volid;
2590
2591         $volhash->{$volid} = $is_cdrom || 0;
2592     };
2593
2594     foreach_drive($conf, sub {
2595         my ($ds, $drive) = @_;
2596         &$test_volid($drive->{file}, drive_is_cdrom($drive));
2597     });
2598
2599     foreach my $snapname (keys %{$conf->{snapshots}}) {
2600         my $snap = $conf->{snapshots}->{$snapname};
2601         &$test_volid($snap->{vmstate}, 0);
2602         foreach_drive($snap, sub {
2603             my ($ds, $drive) = @_;
2604             &$test_volid($drive->{file}, drive_is_cdrom($drive));
2605         });
2606     }
2607
2608     foreach my $volid (keys %$volhash) {
2609         &$func($volid, $volhash->{$volid});
2610     }
2611 }
2612
2613 sub vga_conf_has_spice {
2614     my ($vga) = @_;
2615
2616     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
2617
2618     return $1 || 1;
2619 }
2620
2621 sub config_to_command {
2622     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2623
2624     my $cmd = [];
2625     my $globalFlags = [];
2626     my $machineFlags = [];
2627     my $rtcFlags = [];
2628     my $cpuFlags = [];
2629     my $devices = [];
2630     my $pciaddr = '';
2631     my $bridges = {};
2632     my $kvmver = kvm_user_version();
2633     my $vernum = 0; # unknown
2634     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
2635         $vernum = $1*1000000+$2*1000;
2636     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
2637         $vernum = $1*1000000+$2*1000+$3;
2638     }
2639
2640     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
2641
2642     my $have_ovz = -f '/proc/vz/vestat';
2643
2644     my $q35 = machine_type_is_q35($conf);
2645     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2646     my $machine_type = $forcemachine || $conf->{machine};
2647     my $use_old_bios_files = undef;
2648     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
2649
2650     my $cpuunits = defined($conf->{cpuunits}) ?
2651             $conf->{cpuunits} : $defaults->{cpuunits};
2652
2653     push @$cmd, '/usr/bin/systemd-run';
2654     push @$cmd, '--scope';
2655     push @$cmd, '--slice', "qemu";
2656     push @$cmd, '--unit', $vmid;
2657     push @$cmd, '-p', "CPUShares=$cpuunits";
2658     if ($conf->{cpulimit}) {
2659         my $cpulimit = int($conf->{cpulimit} * 100);
2660         push @$cmd, '-p', "CPUQuota=$cpulimit\%";
2661     }
2662
2663     push @$cmd, '/usr/bin/kvm';
2664
2665     push @$cmd, '-id', $vmid;
2666
2667     my $use_virtio = 0;
2668
2669     my $qmpsocket = qmp_socket($vmid);
2670     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2671     push @$cmd, '-mon', "chardev=qmp,mode=control";
2672
2673     my $socket = vnc_socket($vmid);
2674     push @$cmd,  '-vnc', "unix:$socket,x509,password";
2675
2676     push @$cmd, '-pidfile' , pidfile_name($vmid);
2677
2678     push @$cmd, '-daemonize';
2679
2680     if ($conf->{smbios1}) {
2681         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
2682     }
2683
2684     if ($q35) {
2685         # the q35 chipset support native usb2, so we enable usb controller
2686         # by default for this machine type
2687         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
2688     } else {
2689         $pciaddr = print_pci_addr("piix3", $bridges);
2690         push @$devices, '-device', "piix3-usb-uhci,id=uhci$pciaddr.0x2";
2691
2692         my $use_usb2 = 0;
2693         for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2694             next if !$conf->{"usb$i"};
2695             $use_usb2 = 1;
2696         }
2697         # include usb device config
2698         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-usb.cfg' if $use_usb2;
2699     }
2700
2701     my $vga = $conf->{vga};
2702
2703     my $qxlnum = vga_conf_has_spice($vga);
2704     $vga = 'qxl' if $qxlnum;
2705
2706     if (!$vga) {
2707         if ($conf->{ostype} && ($conf->{ostype} eq 'win8' ||
2708                                 $conf->{ostype} eq 'win7' ||
2709                                 $conf->{ostype} eq 'w2k8')) {
2710             $vga = 'std';
2711         } else {
2712             $vga = 'cirrus';
2713         }
2714     }
2715
2716     # enable absolute mouse coordinates (needed by vnc)
2717     my $tablet;
2718     if (defined($conf->{tablet})) {
2719         $tablet = $conf->{tablet};
2720     } else {
2721         $tablet = $defaults->{tablet};
2722         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
2723         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
2724     }
2725
2726     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
2727
2728     # host pci devices
2729     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
2730         my $d = parse_hostpci($conf->{"hostpci$i"});
2731         next if !$d;
2732
2733         my $pcie = $d->{pcie};
2734         if($pcie){
2735             die "q35 machine model is not enabled" if !$q35;
2736             $pciaddr = print_pcie_addr("hostpci$i");
2737         }else{
2738             $pciaddr = print_pci_addr("hostpci$i", $bridges);
2739         }
2740
2741         my $rombar = $d->{rombar} && $d->{rombar} eq 'off' ? ",rombar=0" : "";
2742         my $xvga = $d->{'x-vga'} && $d->{'x-vga'} eq 'on' ? ",x-vga=on" : "";
2743         if ($xvga && $xvga ne '') {
2744             push @$cpuFlags, 'kvm=off';
2745             $vga = 'none';
2746         }
2747         my $pcidevices = $d->{pciid};
2748         my $multifunction = 1 if @$pcidevices > 1;
2749
2750         my $j=0;
2751         foreach my $pcidevice (@$pcidevices) {
2752
2753             my $id = "hostpci$i";
2754             $id .= ".$j" if $multifunction;
2755             my $addr = $pciaddr;
2756             $addr .= ".$j" if $multifunction;
2757             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
2758
2759             if($j == 0){
2760                 $devicestr .= "$rombar$xvga";
2761                 $devicestr .= ",multifunction=on" if $multifunction;
2762             }
2763
2764             push @$devices, '-device', $devicestr;
2765             $j++;
2766         }
2767     }
2768
2769     # usb devices
2770     for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2771         my $d = parse_usb_device($conf->{"usb$i"});
2772         next if !$d;
2773         if ($d->{vendorid} && $d->{productid}) {
2774             push @$devices, '-device', "usb-host,vendorid=0x$d->{vendorid},productid=0x$d->{productid}";
2775         } elsif (defined($d->{hostbus}) && defined($d->{hostport})) {
2776             push @$devices, '-device', "usb-host,hostbus=$d->{hostbus},hostport=$d->{hostport}";
2777         } elsif ($d->{spice}) {
2778             # usb redir support for spice
2779             push @$devices, '-chardev', "spicevmc,id=usbredirchardev$i,name=usbredir";
2780             push @$devices, '-device', "usb-redir,chardev=usbredirchardev$i,id=usbredirdev$i,bus=ehci.0";
2781         }
2782     }
2783
2784     # serial devices
2785     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2786         if (my $path = $conf->{"serial$i"}) {
2787             if ($path eq 'socket') {
2788                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
2789                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
2790                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2791             } else {
2792                 die "no such serial device\n" if ! -c $path;
2793                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
2794                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2795             }
2796         }
2797     }
2798
2799     # parallel devices
2800     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
2801         if (my $path = $conf->{"parallel$i"}) {
2802             die "no such parallel device\n" if ! -c $path;
2803             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
2804             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
2805             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
2806         }
2807     }
2808
2809     my $vmname = $conf->{name} || "vm$vmid";
2810
2811     push @$cmd, '-name', $vmname;
2812
2813     my $sockets = 1;
2814     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
2815     $sockets = $conf->{sockets} if  $conf->{sockets};
2816
2817     my $cores = $conf->{cores} || 1;
2818
2819     my $maxcpus = $sockets * $cores;
2820
2821     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
2822
2823     my $allowed_vcpus = $cpuinfo->{cpus};
2824
2825     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
2826         if ($allowed_vcpus < $maxcpus);
2827
2828     push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
2829
2830     push @$cmd, '-nodefaults';
2831
2832     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2833
2834     my $bootindex_hash = {};
2835     my $i = 1;
2836     foreach my $o (split(//, $bootorder)) {
2837         $bootindex_hash->{$o} = $i*100;
2838         $i++;
2839     }
2840
2841     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000";
2842
2843     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
2844
2845     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
2846
2847     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
2848
2849     # time drift fix
2850     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
2851
2852     my $nokvm = defined($conf->{kvm}) && $conf->{kvm} == 0 ? 1 : 0;
2853     my $useLocaltime = $conf->{localtime};
2854
2855     if (my $ost = $conf->{ostype}) {
2856         # other, wxp, w2k, w2k3, w2k8, wvista, win7, win8, l24, l26, solaris
2857
2858         if ($ost =~ m/^w/) { # windows
2859             $useLocaltime = 1 if !defined($conf->{localtime});
2860
2861             # use time drift fix when acpi is enabled
2862             if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
2863                 $tdf = 1 if !defined($conf->{tdf});
2864             }
2865         }
2866
2867         if ($ost eq 'win7' || $ost eq 'win8' || $ost eq 'w2k8' ||
2868             $ost eq 'wvista') {
2869             push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
2870             push @$cmd, '-no-hpet';
2871             if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
2872                 push @$cpuFlags , 'hv_spinlocks=0x1fff' if !$nokvm;
2873                 push @$cpuFlags , 'hv_vapic' if !$nokvm;
2874                 push @$cpuFlags , 'hv_time' if !$nokvm;
2875
2876             } else {
2877                 push @$cpuFlags , 'hv_spinlocks=0xffff' if !$nokvm;
2878             }
2879         }
2880
2881         if ($ost eq 'win7' || $ost eq 'win8') {
2882             push @$cpuFlags , 'hv_relaxed' if !$nokvm;
2883         }
2884     }
2885
2886     push @$rtcFlags, 'driftfix=slew' if $tdf;
2887
2888     if ($nokvm) {
2889         push @$machineFlags, 'accel=tcg';
2890     } else {
2891         die "No accelerator found!\n" if !$cpuinfo->{hvm};
2892     }
2893
2894     if ($machine_type) {
2895         push @$machineFlags, "type=${machine_type}";
2896     }
2897
2898     if ($conf->{startdate}) {
2899         push @$rtcFlags, "base=$conf->{startdate}";
2900     } elsif ($useLocaltime) {
2901         push @$rtcFlags, 'base=localtime';
2902     }
2903
2904     my $cpu = $nokvm ? "qemu64" : "kvm64";
2905     $cpu = $conf->{cpu} if $conf->{cpu};
2906
2907     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
2908
2909     push @$cpuFlags , '-x2apic'
2910         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
2911
2912     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
2913
2914     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
2915
2916     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
2917
2918         push @$cpuFlags , '+kvm_pv_unhalt' if !$nokvm;
2919         push @$cpuFlags , '+kvm_pv_eoi' if !$nokvm;
2920         push @$cpuFlags , '-kvm_steal_time' if !$nokvm;
2921     }
2922
2923     push @$cpuFlags, 'enforce' if $cpu ne 'host' && !$nokvm;
2924
2925     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
2926
2927     push @$cmd, '-cpu', $cpu;
2928
2929     my $memory = $conf->{memory} || $defaults->{memory};
2930     my $static_memory = 0;
2931     my $dimm_memory = 0;
2932
2933     if ($hotplug_features->{memory}) {
2934         die "Numa need to be enabled for memory hotplug\n" if !$conf->{numa};
2935         die "Total memory is bigger than ${MAX_MEM}MB\n" if $memory > $MAX_MEM;
2936         $static_memory = $STATICMEM;
2937         die "minimum memory must be ${static_memory}MB\n" if($memory < $static_memory);
2938         $dimm_memory = $memory - $static_memory;
2939         push @$cmd, '-m', "size=${static_memory},slots=255,maxmem=${MAX_MEM}M";
2940
2941     } else {
2942
2943         $static_memory = $memory;
2944         push @$cmd, '-m', $static_memory;
2945     }
2946
2947     if ($conf->{numa}) {
2948
2949         my $numa_totalmemory = undef;
2950         for (my $i = 0; $i < $MAX_NUMA; $i++) {
2951             next if !$conf->{"numa$i"};
2952             my $numa = parse_numa($conf->{"numa$i"});
2953             next if !$numa;
2954             # memory
2955             die "missing numa node$i memory value\n" if !$numa->{memory};
2956             my $numa_memory = $numa->{memory};
2957             $numa_totalmemory += $numa_memory;
2958             my $numa_object = "memory-backend-ram,id=ram-node$i,size=${numa_memory}M";
2959
2960             # cpus
2961             my $cpus_start = $numa->{cpus}->{start};
2962             die "missing numa node$i cpus\n" if !defined($cpus_start);
2963             my $cpus_end = $numa->{cpus}->{end} if defined($numa->{cpus}->{end});
2964             my $cpus = $cpus_start;
2965             if (defined($cpus_end)) {
2966                 $cpus .= "-$cpus_end";
2967                 die "numa node$i :  cpu range $cpus is incorrect\n" if $cpus_end <= $cpus_start;
2968             }
2969
2970             # hostnodes
2971             my $hostnodes_start = $numa->{hostnodes}->{start};
2972             if (defined($hostnodes_start)) {
2973                 my $hostnodes_end = $numa->{hostnodes}->{end} if defined($numa->{hostnodes}->{end});
2974                 my $hostnodes = $hostnodes_start;
2975                 if (defined($hostnodes_end)) {
2976                     $hostnodes .= "-$hostnodes_end";
2977                     die "host node $hostnodes range is incorrect\n" if $hostnodes_end <= $hostnodes_start;
2978                 }
2979
2980                 my $hostnodes_end_range = defined($hostnodes_end) ? $hostnodes_end : $hostnodes_start;
2981                 for (my $i = $hostnodes_start; $i <= $hostnodes_end_range; $i++ ) {
2982                     die "host numa node$i don't exist\n" if ! -d "/sys/devices/system/node/node$i/";
2983                 }
2984
2985                 # policy
2986                 my $policy = $numa->{policy};
2987                 die "you need to define a policy for hostnode $hostnodes\n" if !$policy;
2988                 $numa_object .= ",host-nodes=$hostnodes,policy=$policy";
2989             }
2990
2991             push @$cmd, '-object', $numa_object;
2992             push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
2993         }
2994
2995         die "total memory for NUMA nodes must be equal to vm static memory\n"
2996             if $numa_totalmemory && $numa_totalmemory != $static_memory;
2997
2998         #if no custom tology, we split memory and cores across numa nodes
2999         if(!$numa_totalmemory) {
3000
3001             my $numa_memory = ($static_memory / $sockets) . "M";
3002
3003             for (my $i = 0; $i < $sockets; $i++)  {
3004
3005                 my $cpustart = ($cores * $i);
3006                 my $cpuend = ($cpustart + $cores - 1) if $cores && $cores > 1;
3007                 my $cpus = $cpustart;
3008                 $cpus .= "-$cpuend" if $cpuend;
3009
3010                 push @$cmd, '-object', "memory-backend-ram,size=$numa_memory,id=ram-node$i";
3011                 push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
3012             }
3013         }
3014     }
3015
3016     if ($hotplug_features->{memory}) {
3017         foreach_dimm($conf, $vmid, $memory, $sockets, sub {
3018             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3019             push @$cmd, "-object" , "memory-backend-ram,id=mem-$name,size=${dimm_size}M";
3020             push @$cmd, "-device", "pc-dimm,id=$name,memdev=mem-$name,node=$numanode";
3021
3022             #if dimm_memory is not aligned to dimm map
3023             if($current_size > $memory) {
3024                  $conf->{memory} = $current_size;
3025                  update_config_nolock($vmid, $conf, 1);
3026             }
3027         });
3028     }
3029
3030     push @$cmd, '-S' if $conf->{freeze};
3031
3032     # set keyboard layout
3033     my $kb = $conf->{keyboard} || $defaults->{keyboard};
3034     push @$cmd, '-k', $kb if $kb;
3035
3036     # enable sound
3037     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3038     #push @$cmd, '-soundhw', 'es1370';
3039     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3040
3041     if($conf->{agent}) {
3042         my $qgasocket = qmp_socket($vmid, 1);
3043         my $pciaddr = print_pci_addr("qga0", $bridges);
3044         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3045         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3046         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3047     }
3048
3049     my $spice_port;
3050
3051     if ($qxlnum) {
3052         if ($qxlnum > 1) {
3053             if ($conf->{ostype} && $conf->{ostype} =~ m/^w/){
3054                 for(my $i = 1; $i < $qxlnum; $i++){
3055                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3056                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3057                 }
3058             } else {
3059                 # assume other OS works like Linux
3060                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3061                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3062             }
3063         }
3064
3065         my $pciaddr = print_pci_addr("spice", $bridges);
3066
3067         my $nodename = PVE::INotify::nodename();
3068         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3069         $spice_port = PVE::Tools::next_spice_port($pfamily);
3070
3071         push @$devices, '-spice', "tls-port=${spice_port},addr=localhost,tls-ciphers=DES-CBC3-SHA,seamless-migration=on";
3072
3073         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3074         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3075         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3076     }
3077
3078     # enable balloon by default, unless explicitly disabled
3079     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3080         $pciaddr = print_pci_addr("balloon0", $bridges);
3081         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3082     }
3083
3084     if ($conf->{watchdog}) {
3085         my $wdopts = parse_watchdog($conf->{watchdog});
3086         $pciaddr = print_pci_addr("watchdog", $bridges);
3087         my $watchdog = $wdopts->{model} || 'i6300esb';
3088         push @$devices, '-device', "$watchdog$pciaddr";
3089         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3090     }
3091
3092     my $vollist = [];
3093     my $scsicontroller = {};
3094     my $ahcicontroller = {};
3095     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3096
3097     # Add iscsi initiator name if available
3098     if (my $initiator = get_initiator_name()) {
3099         push @$devices, '-iscsi', "initiator-name=$initiator";
3100     }
3101
3102     foreach_drive($conf, sub {
3103         my ($ds, $drive) = @_;
3104
3105         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3106             push @$vollist, $drive->{file};
3107         }
3108
3109         $use_virtio = 1 if $ds =~ m/^virtio/;
3110
3111         if (drive_is_cdrom ($drive)) {
3112             if ($bootindex_hash->{d}) {
3113                 $drive->{bootindex} = $bootindex_hash->{d};
3114                 $bootindex_hash->{d} += 1;
3115             }
3116         } else {
3117             if ($bootindex_hash->{c}) {
3118                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3119                 $bootindex_hash->{c} += 1;
3120             }
3121         }
3122
3123         if($drive->{interface} eq 'virtio'){
3124            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3125         }
3126
3127         if ($drive->{interface} eq 'scsi') {
3128
3129             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3130
3131             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3132             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3133
3134             my $iothread = '';
3135             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3136                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3137                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3138             }
3139
3140             my $queues = '';
3141             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3142                 $queues = ",num_queues=$drive->{queues}";
3143             }
3144
3145             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3146             $scsicontroller->{$controller}=1;
3147         }
3148
3149         if ($drive->{interface} eq 'sata') {
3150            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3151            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3152            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3153            $ahcicontroller->{$controller}=1;
3154         }
3155
3156         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3157         push @$devices, '-drive',$drive_cmd;
3158         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3159     });
3160
3161     for (my $i = 0; $i < $MAX_NETS; $i++) {
3162          next if !$conf->{"net$i"};
3163          my $d = parse_net($conf->{"net$i"});
3164          next if !$d;
3165
3166          $use_virtio = 1 if $d->{model} eq 'virtio';
3167
3168          if ($bootindex_hash->{n}) {
3169             $d->{bootindex} = $bootindex_hash->{n};
3170             $bootindex_hash->{n} += 1;
3171          }
3172
3173          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3174          push @$devices, '-netdev', $netdevfull;
3175
3176          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3177          push @$devices, '-device', $netdevicefull;
3178     }
3179
3180     if (!$q35) {
3181         # add pci bridges
3182         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3183            $bridges->{1} = 1;
3184            $bridges->{2} = 1;
3185         }
3186
3187         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3188
3189         while (my ($k, $v) = each %$bridges) {
3190             $pciaddr = print_pci_addr("pci.$k");
3191             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3192         }
3193     }
3194
3195     # add custom args
3196     if ($conf->{args}) {
3197         my $aa = PVE::Tools::split_args($conf->{args});
3198         push @$cmd, @$aa;
3199     }
3200
3201     push @$cmd, @$devices;
3202     push @$cmd, '-rtc', join(',', @$rtcFlags)
3203         if scalar(@$rtcFlags);
3204     push @$cmd, '-machine', join(',', @$machineFlags)
3205         if scalar(@$machineFlags);
3206     push @$cmd, '-global', join(',', @$globalFlags)
3207         if scalar(@$globalFlags);
3208
3209     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3210 }
3211
3212 sub vnc_socket {
3213     my ($vmid) = @_;
3214     return "${var_run_tmpdir}/$vmid.vnc";
3215 }
3216
3217 sub spice_port {
3218     my ($vmid) = @_;
3219
3220     my $res = vm_mon_cmd($vmid, 'query-spice');
3221
3222     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3223 }
3224
3225 sub qmp_socket {
3226     my ($vmid, $qga) = @_;
3227     my $sockettype = $qga ? 'qga' : 'qmp';
3228     return "${var_run_tmpdir}/$vmid.$sockettype";
3229 }
3230
3231 sub pidfile_name {
3232     my ($vmid) = @_;
3233     return "${var_run_tmpdir}/$vmid.pid";
3234 }
3235
3236 sub vm_devices_list {
3237     my ($vmid) = @_;
3238
3239     my $res = vm_mon_cmd($vmid, 'query-pci');
3240     my $devices = {};
3241     foreach my $pcibus (@$res) {
3242         foreach my $device (@{$pcibus->{devices}}) {
3243             next if !$device->{'qdev_id'};
3244             if ($device->{'pci_bridge'}) {
3245                 $devices->{$device->{'qdev_id'}} = 1;
3246                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3247                     next if !$bridge_device->{'qdev_id'};
3248                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3249                     $devices->{$device->{'qdev_id'}}++;
3250                 }
3251             } else {
3252                 $devices->{$device->{'qdev_id'}} = 1;
3253             }
3254         }
3255     }
3256
3257     my $resblock = vm_mon_cmd($vmid, 'query-block');
3258     foreach my $block (@$resblock) {
3259         if($block->{device} =~ m/^drive-(\S+)/){
3260                 $devices->{$1} = 1;
3261         }
3262     }
3263
3264     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3265     foreach my $mice (@$resmice) {
3266         if ($mice->{name} eq 'QEMU HID Tablet') {
3267             $devices->{tablet} = 1;
3268             last;
3269         }
3270     }
3271
3272     return $devices;
3273 }
3274
3275 sub vm_deviceplug {
3276     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3277
3278     my $q35 = machine_type_is_q35($conf);
3279
3280     my $devices_list = vm_devices_list($vmid);
3281     return 1 if defined($devices_list->{$deviceid});
3282
3283     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3284
3285     if ($deviceid eq 'tablet') {
3286
3287         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3288
3289     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3290
3291         qemu_iothread_add($vmid, $deviceid, $device);
3292
3293         qemu_driveadd($storecfg, $vmid, $device);
3294         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3295
3296         qemu_deviceadd($vmid, $devicefull);
3297         eval { qemu_deviceaddverify($vmid, $deviceid); };
3298         if (my $err = $@) {
3299             eval { qemu_drivedel($vmid, $deviceid); };
3300             warn $@ if $@;
3301             die $err;
3302         }
3303
3304     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3305
3306
3307         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3308         my $pciaddr = print_pci_addr($deviceid);
3309         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3310
3311         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3312
3313         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3314             qemu_iothread_add($vmid, $deviceid, $device);
3315             $devicefull .= ",iothread=iothread-$deviceid";
3316         }
3317
3318         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3319             $devicefull .= ",num_queues=$device->{queues}";
3320         }
3321
3322         qemu_deviceadd($vmid, $devicefull);
3323         qemu_deviceaddverify($vmid, $deviceid);
3324
3325     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3326
3327         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3328         qemu_driveadd($storecfg, $vmid, $device);
3329
3330         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3331         eval { qemu_deviceadd($vmid, $devicefull); };
3332         if (my $err = $@) {
3333             eval { qemu_drivedel($vmid, $deviceid); };
3334             warn $@ if $@;
3335             die $err;
3336         }
3337
3338     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3339
3340         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3341
3342         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
3343         my $use_old_bios_files = undef;
3344         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3345
3346         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3347         qemu_deviceadd($vmid, $netdevicefull);
3348         eval { qemu_deviceaddverify($vmid, $deviceid); };
3349         if (my $err = $@) {
3350             eval { qemu_netdevdel($vmid, $deviceid); };
3351             warn $@ if $@;
3352             die $err;
3353         }
3354
3355     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3356
3357         my $bridgeid = $2;
3358         my $pciaddr = print_pci_addr($deviceid);
3359         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3360
3361         qemu_deviceadd($vmid, $devicefull);
3362         qemu_deviceaddverify($vmid, $deviceid);
3363
3364     } else {
3365         die "can't hotplug device '$deviceid'\n";
3366     }
3367
3368     return 1;
3369 }
3370
3371 # fixme: this should raise exceptions on error!
3372 sub vm_deviceunplug {
3373     my ($vmid, $conf, $deviceid) = @_;
3374
3375     my $devices_list = vm_devices_list($vmid);
3376     return 1 if !defined($devices_list->{$deviceid});
3377
3378     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3379
3380     if ($deviceid eq 'tablet') {
3381
3382         qemu_devicedel($vmid, $deviceid);
3383
3384     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3385
3386         qemu_devicedel($vmid, $deviceid);
3387         qemu_devicedelverify($vmid, $deviceid);
3388         qemu_drivedel($vmid, $deviceid);
3389         qemu_iothread_del($conf, $vmid, $deviceid);
3390
3391     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3392
3393         qemu_devicedel($vmid, $deviceid);
3394         qemu_devicedelverify($vmid, $deviceid);
3395         qemu_iothread_del($conf, $vmid, $deviceid);
3396
3397     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3398
3399         #qemu 2.3 segfault on drive_del with virtioscsi + iothread
3400         my $device = parse_drive($deviceid, $conf->{$deviceid});
3401         die "virtioscsi with iothread is not hot-unplugglable currently" if $device->{iothread};
3402
3403         qemu_devicedel($vmid, $deviceid);
3404         qemu_drivedel($vmid, $deviceid);
3405         qemu_deletescsihw($conf, $vmid, $deviceid);
3406
3407     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3408
3409         qemu_devicedel($vmid, $deviceid);
3410         qemu_devicedelverify($vmid, $deviceid);
3411         qemu_netdevdel($vmid, $deviceid);
3412
3413     } else {
3414         die "can't unplug device '$deviceid'\n";
3415     }
3416
3417     return 1;
3418 }
3419
3420 sub qemu_deviceadd {
3421     my ($vmid, $devicefull) = @_;
3422
3423     $devicefull = "driver=".$devicefull;
3424     my %options =  split(/[=,]/, $devicefull);
3425
3426     vm_mon_cmd($vmid, "device_add" , %options);
3427 }
3428
3429 sub qemu_devicedel {
3430     my ($vmid, $deviceid) = @_;
3431
3432     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3433 }
3434
3435 sub qemu_iothread_add {
3436     my($vmid, $deviceid, $device) = @_;
3437
3438     if ($device->{iothread}) {
3439         my $iothreads = vm_iothreads_list($vmid);
3440         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3441     }
3442 }
3443
3444 sub qemu_iothread_del {
3445     my($conf, $vmid, $deviceid) = @_;
3446
3447     my $device = parse_drive($deviceid, $conf->{$deviceid});
3448     if ($device->{iothread}) {
3449         my $iothreads = vm_iothreads_list($vmid);
3450         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3451     }
3452 }
3453
3454 sub qemu_objectadd {
3455     my($vmid, $objectid, $qomtype) = @_;
3456
3457     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3458
3459     return 1;
3460 }
3461
3462 sub qemu_objectdel {
3463     my($vmid, $objectid) = @_;
3464
3465     vm_mon_cmd($vmid, "object-del", id => $objectid);
3466
3467     return 1;
3468 }
3469
3470 sub qemu_driveadd {
3471     my ($storecfg, $vmid, $device) = @_;
3472
3473     my $drive = print_drive_full($storecfg, $vmid, $device);
3474     $drive =~ s/\\/\\\\/g;
3475     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3476
3477     # If the command succeeds qemu prints: "OK"
3478     return 1 if $ret =~ m/OK/s;
3479
3480     die "adding drive failed: $ret\n";
3481 }
3482
3483 sub qemu_drivedel {
3484     my($vmid, $deviceid) = @_;
3485
3486     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3487     $ret =~ s/^\s+//;
3488
3489     return 1 if $ret eq "";
3490
3491     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3492     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3493
3494     die "deleting drive $deviceid failed : $ret\n";
3495 }
3496
3497 sub qemu_deviceaddverify {
3498     my ($vmid, $deviceid) = @_;
3499
3500     for (my $i = 0; $i <= 5; $i++) {
3501          my $devices_list = vm_devices_list($vmid);
3502          return 1 if defined($devices_list->{$deviceid});
3503          sleep 1;
3504     }
3505
3506     die "error on hotplug device '$deviceid'\n";
3507 }
3508
3509
3510 sub qemu_devicedelverify {
3511     my ($vmid, $deviceid) = @_;
3512
3513     # need to verify that the device is correctly removed as device_del
3514     # is async and empty return is not reliable
3515
3516     for (my $i = 0; $i <= 5; $i++) {
3517          my $devices_list = vm_devices_list($vmid);
3518          return 1 if !defined($devices_list->{$deviceid});
3519          sleep 1;
3520     }
3521
3522     die "error on hot-unplugging device '$deviceid'\n";
3523 }
3524
3525 sub qemu_findorcreatescsihw {
3526     my ($storecfg, $conf, $vmid, $device) = @_;
3527
3528     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3529
3530     my $scsihwid="$controller_prefix$controller";
3531     my $devices_list = vm_devices_list($vmid);
3532
3533     if(!defined($devices_list->{$scsihwid})) {
3534         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3535     }
3536
3537     return 1;
3538 }
3539
3540 sub qemu_deletescsihw {
3541     my ($conf, $vmid, $opt) = @_;
3542
3543     my $device = parse_drive($opt, $conf->{$opt});
3544
3545     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3546         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3547         return 1;
3548     }
3549
3550     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3551
3552     my $devices_list = vm_devices_list($vmid);
3553     foreach my $opt (keys %{$devices_list}) {
3554         if (PVE::QemuServer::valid_drivename($opt)) {
3555             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3556             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3557                 return 1;
3558             }
3559         }
3560     }
3561
3562     my $scsihwid="scsihw$controller";
3563
3564     vm_deviceunplug($vmid, $conf, $scsihwid);
3565
3566     return 1;
3567 }
3568
3569 sub qemu_add_pci_bridge {
3570     my ($storecfg, $conf, $vmid, $device) = @_;
3571
3572     my $bridges = {};
3573
3574     my $bridgeid;
3575
3576     print_pci_addr($device, $bridges);
3577
3578     while (my ($k, $v) = each %$bridges) {
3579         $bridgeid = $k;
3580     }
3581     return 1 if !defined($bridgeid) || $bridgeid < 1;
3582
3583     my $bridge = "pci.$bridgeid";
3584     my $devices_list = vm_devices_list($vmid);
3585
3586     if (!defined($devices_list->{$bridge})) {
3587         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3588     }
3589
3590     return 1;
3591 }
3592
3593 sub qemu_set_link_status {
3594     my ($vmid, $device, $up) = @_;
3595
3596     vm_mon_cmd($vmid, "set_link", name => $device,
3597                up => $up ? JSON::true : JSON::false);
3598 }
3599
3600 sub qemu_netdevadd {
3601     my ($vmid, $conf, $device, $deviceid) = @_;
3602
3603     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
3604     my %options =  split(/[=,]/, $netdev);
3605
3606     vm_mon_cmd($vmid, "netdev_add",  %options);
3607     return 1;
3608 }
3609
3610 sub qemu_netdevdel {
3611     my ($vmid, $deviceid) = @_;
3612
3613     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3614 }
3615
3616 sub qemu_cpu_hotplug {
3617     my ($vmid, $conf, $vcpus) = @_;
3618
3619     my $sockets = 1;
3620     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3621     $sockets = $conf->{sockets} if  $conf->{sockets};
3622     my $cores = $conf->{cores} || 1;
3623     my $maxcpus = $sockets * $cores;
3624
3625     $vcpus = $maxcpus if !$vcpus;
3626
3627     die "you can't add more vcpus than maxcpus\n"
3628         if $vcpus > $maxcpus;
3629
3630     my $currentvcpus = $conf->{vcpus} || $maxcpus;
3631     die "online cpu unplug is not yet possible\n"
3632         if $vcpus < $currentvcpus;
3633
3634     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3635     die "vcpus in running vm is different than configuration\n"
3636         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
3637
3638     for (my $i = $currentvcpus; $i < $vcpus; $i++) {
3639         vm_mon_cmd($vmid, "cpu-add", id => int($i));
3640     }
3641 }
3642
3643 sub qemu_memory_hotplug {
3644     my ($vmid, $conf, $defaults, $opt, $value) = @_;
3645
3646     return $value if !check_running($vmid);
3647
3648     my $memory = $conf->{memory} || $defaults->{memory};
3649     $value = $defaults->{memory} if !$value;
3650     return $value if $value == $memory;
3651
3652     my $static_memory = $STATICMEM;
3653     my $dimm_memory = $memory - $static_memory;
3654
3655     die "memory can't be lower than $static_memory MB" if $value < $static_memory;
3656     die "you cannot add more memory than $MAX_MEM MB!\n" if $memory > $MAX_MEM;
3657
3658
3659     my $sockets = 1;
3660     $sockets = $conf->{sockets} if $conf->{sockets};
3661
3662     if($value > $memory) {
3663
3664         foreach_dimm($conf, $vmid, $value, $sockets, sub {
3665             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3666
3667                 return if $current_size <= $conf->{memory};
3668
3669                 eval { vm_mon_cmd($vmid, "object-add", 'qom-type' => "memory-backend-ram", id => "mem-$name", props => { size => int($dimm_size*1024*1024) } ) };
3670                 if (my $err = $@) {
3671                     eval { qemu_objectdel($vmid, "mem-$name"); };
3672                     die $err;
3673                 }
3674
3675                 eval { vm_mon_cmd($vmid, "device_add", driver => "pc-dimm", id => "$name", memdev => "mem-$name", node => $numanode) };
3676                 if (my $err = $@) {
3677                     eval { qemu_objectdel($vmid, "mem-$name"); };
3678                     die $err;
3679                 }
3680                 #update conf after each succesful module hotplug
3681                 $conf->{memory} = $current_size;
3682                 update_config_nolock($vmid, $conf, 1);
3683         });
3684
3685     } else {
3686
3687         foreach_reverse_dimm($conf, $vmid, $value, $sockets, sub {
3688             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3689
3690                 return if $current_size >= $conf->{memory};
3691                 print "try to unplug memory dimm $name\n";
3692
3693                 my $retry = 0;
3694                 while (1) {
3695                     eval { qemu_devicedel($vmid, $name) };
3696                     sleep 3;
3697                     my $dimm_list = qemu_dimm_list($vmid);
3698                     last if !$dimm_list->{$name};
3699                     raise_param_exc({ $name => "error unplug memory module" }) if $retry > 5;
3700                     $retry++;
3701                 }
3702
3703                 #update conf after each succesful module unplug
3704                 $conf->{memory} = $current_size;
3705
3706                 eval { qemu_objectdel($vmid, "mem-$name"); };
3707                 update_config_nolock($vmid, $conf, 1);
3708         });
3709     }
3710 }
3711
3712 sub qemu_dimm_list {
3713     my ($vmid) = @_;
3714
3715     my $dimmarray = vm_mon_cmd_nocheck($vmid, "query-memory-devices");
3716     my $dimms = {};
3717
3718     foreach my $dimm (@$dimmarray) {
3719
3720         $dimms->{$dimm->{data}->{id}}->{id} = $dimm->{data}->{id};
3721         $dimms->{$dimm->{data}->{id}}->{node} = $dimm->{data}->{node};
3722         $dimms->{$dimm->{data}->{id}}->{addr} = $dimm->{data}->{addr};
3723         $dimms->{$dimm->{data}->{id}}->{size} = $dimm->{data}->{size};
3724         $dimms->{$dimm->{data}->{id}}->{slot} = $dimm->{data}->{slot};
3725     }
3726     return $dimms;
3727 }
3728
3729 sub qemu_block_set_io_throttle {
3730     my ($vmid, $deviceid, $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr) = @_;
3731
3732     return if !check_running($vmid) ;
3733
3734     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid, bps => int($bps), bps_rd => int($bps_rd), bps_wr => int($bps_wr), iops => int($iops), iops_rd => int($iops_rd), iops_wr => int($iops_wr));
3735
3736 }
3737
3738 # old code, only used to shutdown old VM after update
3739 sub __read_avail {
3740     my ($fh, $timeout) = @_;
3741
3742     my $sel = new IO::Select;
3743     $sel->add($fh);
3744
3745     my $res = '';
3746     my $buf;
3747
3748     my @ready;
3749     while (scalar (@ready = $sel->can_read($timeout))) {
3750         my $count;
3751         if ($count = $fh->sysread($buf, 8192)) {
3752             if ($buf =~ /^(.*)\(qemu\) $/s) {
3753                 $res .= $1;
3754                 last;
3755             } else {
3756                 $res .= $buf;
3757             }
3758         } else {
3759             if (!defined($count)) {
3760                 die "$!\n";
3761             }
3762             last;
3763         }
3764     }
3765
3766     die "monitor read timeout\n" if !scalar(@ready);
3767
3768     return $res;
3769 }
3770
3771 # old code, only used to shutdown old VM after update
3772 sub vm_monitor_command {
3773     my ($vmid, $cmdstr, $nocheck) = @_;
3774
3775     my $res;
3776
3777     eval {
3778         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
3779
3780         my $sname = "${var_run_tmpdir}/$vmid.mon";
3781
3782         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
3783             die "unable to connect to VM $vmid socket - $!\n";
3784
3785         my $timeout = 3;
3786
3787         # hack: migrate sometime blocks the monitor (when migrate_downtime
3788         # is set)
3789         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3790             $timeout = 60*60; # 1 hour
3791         }
3792
3793         # read banner;
3794         my $data = __read_avail($sock, $timeout);
3795
3796         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
3797             die "got unexpected qemu monitor banner\n";
3798         }
3799
3800         my $sel = new IO::Select;
3801         $sel->add($sock);
3802
3803         if (!scalar(my @ready = $sel->can_write($timeout))) {
3804             die "monitor write error - timeout";
3805         }
3806
3807         my $fullcmd = "$cmdstr\r";
3808
3809         # syslog('info', "VM $vmid monitor command: $cmdstr");
3810
3811         my $b;
3812         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
3813             die "monitor write error - $!";
3814         }
3815
3816         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
3817
3818         $timeout = 20;
3819
3820         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3821             $timeout = 60*60; # 1 hour
3822         } elsif ($cmdstr =~ m/^(eject|change)/) {
3823             $timeout = 60; # note: cdrom mount command is slow
3824         }
3825         if ($res = __read_avail($sock, $timeout)) {
3826
3827             my @lines = split("\r?\n", $res);
3828
3829             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
3830
3831             $res = join("\n", @lines);
3832             $res .= "\n";
3833         }
3834     };
3835
3836     my $err = $@;
3837
3838     if ($err) {
3839         syslog("err", "VM $vmid monitor command failed - $err");
3840         die $err;
3841     }
3842
3843     return $res;
3844 }
3845
3846 sub qemu_block_resize {
3847     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
3848
3849     my $running = check_running($vmid);
3850
3851     return if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
3852
3853     return if !$running;
3854
3855     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
3856
3857 }
3858
3859 sub qemu_volume_snapshot {
3860     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3861
3862     my $running = check_running($vmid);
3863
3864     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
3865         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
3866     } else {
3867         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
3868     }
3869 }
3870
3871 sub qemu_volume_snapshot_delete {
3872     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3873
3874     my $running = check_running($vmid);
3875
3876     return if !PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
3877
3878     return if !$running;
3879
3880     vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
3881 }
3882
3883 sub set_migration_caps {
3884     my ($vmid) = @_;
3885
3886     my $cap_ref = [];
3887
3888     my $enabled_cap = {
3889         "auto-converge" => 1,
3890         "xbzrle" => 1,
3891         "x-rdma-pin-all" => 0,
3892         "zero-blocks" => 0,
3893         "compress" => 0
3894     };
3895
3896     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
3897
3898     for my $supported_capability (@$supported_capabilities) {
3899         push @$cap_ref, {
3900             capability => $supported_capability->{capability},
3901             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
3902         };
3903     }
3904
3905     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
3906 }
3907
3908 my $fast_plug_option = {
3909     'lock' => 1,
3910     'name' => 1,
3911     'onboot' => 1,
3912     'shares' => 1,
3913     'startup' => 1,
3914     'description' => 1,
3915 };
3916
3917 # hotplug changes in [PENDING]
3918 # $selection hash can be used to only apply specified options, for
3919 # example: { cores => 1 } (only apply changed 'cores')
3920 # $errors ref is used to return error messages
3921 sub vmconfig_hotplug_pending {
3922     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
3923
3924     my $defaults = load_defaults();
3925
3926     # commit values which do not have any impact on running VM first
3927     # Note: those option cannot raise errors, we we do not care about
3928     # $selection and always apply them.
3929
3930     my $add_error = sub {
3931         my ($opt, $msg) = @_;
3932         $errors->{$opt} = "hotplug problem - $msg";
3933     };
3934
3935     my $changes = 0;
3936     foreach my $opt (keys %{$conf->{pending}}) { # add/change
3937         if ($fast_plug_option->{$opt}) {
3938             $conf->{$opt} = $conf->{pending}->{$opt};
3939             delete $conf->{pending}->{$opt};
3940             $changes = 1;
3941         }
3942     }
3943
3944     if ($changes) {
3945         update_config_nolock($vmid, $conf, 1);
3946         $conf = load_config($vmid); # update/reload
3947     }
3948
3949     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3950
3951     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
3952     while (my ($opt, $force) = each %$pending_delete_hash) {
3953         next if $selection && !$selection->{$opt};
3954         eval {
3955             if ($opt eq 'hotplug') {
3956                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
3957             } elsif ($opt eq 'tablet') {
3958                 die "skip\n" if !$hotplug_features->{usb};
3959                 if ($defaults->{tablet}) {
3960                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
3961                 } else {
3962                     vm_deviceunplug($vmid, $conf, $opt);
3963                 }
3964             } elsif ($opt eq 'vcpus') {
3965                 die "skip\n" if !$hotplug_features->{cpu};
3966                 qemu_cpu_hotplug($vmid, $conf, undef);
3967             } elsif ($opt eq 'balloon') {
3968                 # enable balloon device is not hotpluggable
3969                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
3970             } elsif ($fast_plug_option->{$opt}) {
3971                 # do nothing
3972             } elsif ($opt =~ m/^net(\d+)$/) {
3973                 die "skip\n" if !$hotplug_features->{network};
3974                 vm_deviceunplug($vmid, $conf, $opt);
3975             } elsif (valid_drivename($opt)) {
3976                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
3977                 vm_deviceunplug($vmid, $conf, $opt);
3978                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
3979             } elsif ($opt =~ m/^memory$/) {
3980                 die "skip\n" if !$hotplug_features->{memory};
3981                 qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
3982             } elsif ($opt eq 'cpuunits') {
3983                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
3984             } elsif ($opt eq 'cpulimit') {
3985                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
3986             } else {
3987                 die "skip\n";
3988             }
3989         };
3990         if (my $err = $@) {
3991             &$add_error($opt, $err) if $err ne "skip\n";
3992         } else {
3993             # save new config if hotplug was successful
3994             delete $conf->{$opt};
3995             vmconfig_undelete_pending_option($conf, $opt);
3996             update_config_nolock($vmid, $conf, 1);
3997             $conf = load_config($vmid); # update/reload
3998         }
3999     }
4000
4001     foreach my $opt (keys %{$conf->{pending}}) {
4002         next if $selection && !$selection->{$opt};
4003         my $value = $conf->{pending}->{$opt};
4004         eval {
4005             if ($opt eq 'hotplug') {
4006                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4007             } elsif ($opt eq 'tablet') {
4008                 die "skip\n" if !$hotplug_features->{usb};
4009                 if ($value == 1) {
4010                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4011                 } elsif ($value == 0) {
4012                     vm_deviceunplug($vmid, $conf, $opt);
4013                 }
4014             } elsif ($opt eq 'vcpus') {
4015                 die "skip\n" if !$hotplug_features->{cpu};
4016                 qemu_cpu_hotplug($vmid, $conf, $value);
4017             } elsif ($opt eq 'balloon') {
4018                 # enable/disable balloning device is not hotpluggable
4019                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4020                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4021                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4022
4023                 # allow manual ballooning if shares is set to zero
4024                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4025                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4026                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4027                 }
4028             } elsif ($opt =~ m/^net(\d+)$/) {
4029                 # some changes can be done without hotplug
4030                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4031                                     $vmid, $opt, $value);
4032             } elsif (valid_drivename($opt)) {
4033                 # some changes can be done without hotplug
4034                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4035                                      $vmid, $opt, $value, 1);
4036             } elsif ($opt =~ m/^memory$/) { #dimms
4037                 die "skip\n" if !$hotplug_features->{memory};
4038                 $value = qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4039             } elsif ($opt eq 'cpuunits') {
4040                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4041             } elsif ($opt eq 'cpulimit') {
4042                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4043                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4044             } else {
4045                 die "skip\n";  # skip non-hot-pluggable options
4046             }
4047         };
4048         if (my $err = $@) {
4049             &$add_error($opt, $err) if $err ne "skip\n";
4050         } else {
4051             # save new config if hotplug was successful
4052             $conf->{$opt} = $value;
4053             delete $conf->{pending}->{$opt};
4054             update_config_nolock($vmid, $conf, 1);
4055             $conf = load_config($vmid); # update/reload
4056         }
4057     }
4058 }
4059
4060 sub try_deallocate_drive {
4061     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4062
4063     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4064         my $volid = $drive->{file};
4065         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4066             my $sid = PVE::Storage::parse_volume_id($volid);
4067             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4068
4069             # check if the disk is really unused
4070             my $used_paths = PVE::QemuServer::get_used_paths($vmid, $storecfg, $conf, 1, $key);
4071             my $path = PVE::Storage::path($storecfg, $volid);
4072             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4073                    if $used_paths->{$path};
4074             PVE::Storage::vdisk_free($storecfg, $volid);
4075             return 1;
4076         } else {
4077             # If vm is not owner of this disk remove from config
4078             return 1;
4079         }
4080     }
4081
4082     return undef;
4083 }
4084
4085 sub vmconfig_delete_or_detach_drive {
4086     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4087
4088     my $drive = parse_drive($opt, $conf->{$opt});
4089
4090     my $rpcenv = PVE::RPCEnvironment::get();
4091     my $authuser = $rpcenv->get_user();
4092
4093     if ($force) {
4094         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4095         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4096     } else {
4097         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4098     }
4099 }
4100
4101 sub vmconfig_apply_pending {
4102     my ($vmid, $conf, $storecfg) = @_;
4103
4104     # cold plug
4105
4106     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4107     while (my ($opt, $force) = each %$pending_delete_hash) {
4108         die "internal error" if $opt =~ m/^unused/;
4109         $conf = load_config($vmid); # update/reload
4110         if (!defined($conf->{$opt})) {
4111             vmconfig_undelete_pending_option($conf, $opt);
4112             update_config_nolock($vmid, $conf, 1);
4113         } elsif (valid_drivename($opt)) {
4114             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4115             vmconfig_undelete_pending_option($conf, $opt);
4116             delete $conf->{$opt};
4117             update_config_nolock($vmid, $conf, 1);
4118         } else {
4119             vmconfig_undelete_pending_option($conf, $opt);
4120             delete $conf->{$opt};
4121             update_config_nolock($vmid, $conf, 1);
4122         }
4123     }
4124
4125     $conf = load_config($vmid); # update/reload
4126
4127     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4128         $conf = load_config($vmid); # update/reload
4129
4130         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4131             # skip if nothing changed
4132         } elsif (valid_drivename($opt)) {
4133             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4134                 if defined($conf->{$opt});
4135             $conf->{$opt} = $conf->{pending}->{$opt};
4136         } else {
4137             $conf->{$opt} = $conf->{pending}->{$opt};
4138         }
4139
4140         delete $conf->{pending}->{$opt};
4141         update_config_nolock($vmid, $conf, 1);
4142     }
4143 }
4144
4145 my $safe_num_ne = sub {
4146     my ($a, $b) = @_;
4147
4148     return 0 if !defined($a) && !defined($b);
4149     return 1 if !defined($a);
4150     return 1 if !defined($b);
4151
4152     return $a != $b;
4153 };
4154
4155 my $safe_string_ne = sub {
4156     my ($a, $b) = @_;
4157
4158     return 0 if !defined($a) && !defined($b);
4159     return 1 if !defined($a);
4160     return 1 if !defined($b);
4161
4162     return $a ne $b;
4163 };
4164
4165 sub vmconfig_update_net {
4166     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4167
4168     my $newnet = parse_net($value);
4169
4170     if ($conf->{$opt}) {
4171         my $oldnet = parse_net($conf->{$opt});
4172
4173         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4174             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4175             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4176             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4177
4178             # for non online change, we try to hot-unplug
4179             die "skip\n" if !$hotplug;
4180             vm_deviceunplug($vmid, $conf, $opt);
4181         } else {
4182
4183             die "internal error" if $opt !~ m/net(\d+)/;
4184             my $iface = "tap${vmid}i$1";
4185
4186             if (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4187                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4188             }
4189
4190             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4191                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4192                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4193                 PVE::Network::tap_unplug($iface);
4194                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall});
4195             }
4196
4197             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4198                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4199             }
4200
4201             return 1;
4202         }
4203     }
4204
4205     if ($hotplug) {
4206         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4207     } else {
4208         die "skip\n";
4209     }
4210 }
4211
4212 sub vmconfig_update_disk {
4213     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4214
4215     # fixme: do we need force?
4216
4217     my $drive = parse_drive($opt, $value);
4218
4219     if ($conf->{$opt}) {
4220
4221         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4222
4223             my $media = $drive->{media} || 'disk';
4224             my $oldmedia = $old_drive->{media} || 'disk';
4225             die "unable to change media type\n" if $media ne $oldmedia;
4226
4227             if (!drive_is_cdrom($old_drive)) {
4228
4229                 if ($drive->{file} ne $old_drive->{file}) {
4230
4231                     die "skip\n" if !$hotplug;
4232
4233                     # unplug and register as unused
4234                     vm_deviceunplug($vmid, $conf, $opt);
4235                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4236
4237                 } else {
4238                     # update existing disk
4239
4240                     # skip non hotpluggable value
4241                     if (&$safe_num_ne($drive->{discard}, $old_drive->{discard}) ||
4242                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4243                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4244                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4245                         die "skip\n";
4246                     }
4247
4248                     # apply throttle
4249                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4250                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4251                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4252                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4253                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4254                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4255                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4256                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4257                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4258                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4259                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4260                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max})) {
4261
4262                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4263                                                    ($drive->{mbps} || 0)*1024*1024,
4264                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4265                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4266                                                    $drive->{iops} || 0,
4267                                                    $drive->{iops_rd} || 0,
4268                                                    $drive->{iops_wr} || 0,
4269                                                    ($drive->{mbps_max} || 0)*1024*1024,
4270                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4271                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4272                                                    $drive->{iops_max} || 0,
4273                                                    $drive->{iops_rd_max} || 0,
4274                                                    $drive->{iops_wr_max} || 0);
4275
4276                     }
4277
4278                     return 1;
4279                 }
4280
4281             } else { # cdrom
4282
4283                 if ($drive->{file} eq 'none') {
4284                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4285                 } else {
4286                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4287                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4288                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4289                 }
4290
4291                 return 1;
4292             }
4293         }
4294     }
4295
4296     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4297     # hotplug new disks
4298     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4299 }
4300
4301 sub vm_start {
4302     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4303         $forcemachine, $spice_ticket) = @_;
4304
4305     lock_config($vmid, sub {
4306         my $conf = load_config($vmid, $migratedfrom);
4307
4308         die "you can't start a vm if it's a template\n" if is_template($conf);
4309
4310         check_lock($conf) if !$skiplock;
4311
4312         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4313
4314         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4315             vmconfig_apply_pending($vmid, $conf, $storecfg);
4316             $conf = load_config($vmid); # update/reload
4317         }
4318
4319         my $defaults = load_defaults();
4320
4321         # set environment variable useful inside network script
4322         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4323
4324         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4325
4326         my $migrate_port = 0;
4327         my $migrate_uri;
4328         if ($statefile) {
4329             if ($statefile eq 'tcp') {
4330                 my $localip = "localhost";
4331                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4332                 my $nodename = PVE::INotify::nodename();
4333                 if ($datacenterconf->{migration_unsecure}) {
4334                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4335                         $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4336                 }
4337                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4338                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4339                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4340                 push @$cmd, '-incoming', $migrate_uri;
4341                 push @$cmd, '-S';
4342             } else {
4343                 push @$cmd, '-loadstate', $statefile;
4344             }
4345         } elsif ($paused) {
4346             push @$cmd, '-S';
4347         }
4348
4349         # host pci devices
4350         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4351           my $d = parse_hostpci($conf->{"hostpci$i"});
4352           next if !$d;
4353           my $pcidevices = $d->{pciid};
4354           foreach my $pcidevice (@$pcidevices) {
4355                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4356
4357                 my $info = pci_device_info("0000:$pciid");
4358                 die "IOMMU not present\n" if !check_iommu_support();
4359                 die "no pci device info for device '$pciid'\n" if !$info;
4360                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4361                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4362           }
4363         }
4364
4365         PVE::Storage::activate_volumes($storecfg, $vollist);
4366
4367         eval  { run_command($cmd, timeout => $statefile ? undef : 30,
4368                     umask => 0077); };
4369         my $err = $@;
4370         die "start failed: $err" if $err;
4371
4372         print "migration listens on $migrate_uri\n" if $migrate_uri;
4373
4374         if ($statefile && $statefile ne 'tcp')  {
4375             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4376             warn $@ if $@;
4377         }
4378
4379         if ($migratedfrom) {
4380
4381             eval {
4382                 set_migration_caps($vmid);
4383             };
4384             warn $@ if $@;
4385
4386             if ($spice_port) {
4387                 print "spice listens on port $spice_port\n";
4388                 if ($spice_ticket) {
4389                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4390                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4391                 }
4392             }
4393
4394         } else {
4395
4396             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4397                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4398                     if $conf->{balloon};
4399             }
4400
4401             foreach my $opt (keys %$conf) {
4402                 next if $opt !~  m/^net\d+$/;
4403                 my $nicconf = parse_net($conf->{$opt});
4404                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4405             }
4406         }
4407
4408         vm_mon_cmd_nocheck($vmid, 'qom-set',
4409                     path => "machine/peripheral/balloon0",
4410                     property => "guest-stats-polling-interval",
4411                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4412
4413     });
4414 }
4415
4416 sub vm_mon_cmd {
4417     my ($vmid, $execute, %params) = @_;
4418
4419     my $cmd = { execute => $execute, arguments => \%params };
4420     vm_qmp_command($vmid, $cmd);
4421 }
4422
4423 sub vm_mon_cmd_nocheck {
4424     my ($vmid, $execute, %params) = @_;
4425
4426     my $cmd = { execute => $execute, arguments => \%params };
4427     vm_qmp_command($vmid, $cmd, 1);
4428 }
4429
4430 sub vm_qmp_command {
4431     my ($vmid, $cmd, $nocheck) = @_;
4432
4433     my $res;
4434
4435     my $timeout;
4436     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4437         $timeout = $cmd->{arguments}->{timeout};
4438         delete $cmd->{arguments}->{timeout};
4439     }
4440
4441     eval {
4442         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4443         my $sname = qmp_socket($vmid);
4444         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4445             my $qmpclient = PVE::QMPClient->new();
4446
4447             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4448         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
4449             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
4450                 if scalar(%{$cmd->{arguments}});
4451             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
4452         } else {
4453             die "unable to open monitor socket\n";
4454         }
4455     };
4456     if (my $err = $@) {
4457         syslog("err", "VM $vmid qmp command failed - $err");
4458         die $err;
4459     }
4460
4461     return $res;
4462 }
4463
4464 sub vm_human_monitor_command {
4465     my ($vmid, $cmdline) = @_;
4466
4467     my $res;
4468
4469     my $cmd = {
4470         execute => 'human-monitor-command',
4471         arguments => { 'command-line' => $cmdline},
4472     };
4473
4474     return vm_qmp_command($vmid, $cmd);
4475 }
4476
4477 sub vm_commandline {
4478     my ($storecfg, $vmid) = @_;
4479
4480     my $conf = load_config($vmid);
4481
4482     my $defaults = load_defaults();
4483
4484     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
4485
4486     return join(' ', @$cmd);
4487 }
4488
4489 sub vm_reset {
4490     my ($vmid, $skiplock) = @_;
4491
4492     lock_config($vmid, sub {
4493
4494         my $conf = load_config($vmid);
4495
4496         check_lock($conf) if !$skiplock;
4497
4498         vm_mon_cmd($vmid, "system_reset");
4499     });
4500 }
4501
4502 sub get_vm_volumes {
4503     my ($conf) = @_;
4504
4505     my $vollist = [];
4506     foreach_volid($conf, sub {
4507         my ($volid, $is_cdrom) = @_;
4508
4509         return if $volid =~ m|^/|;
4510
4511         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4512         return if !$sid;
4513
4514         push @$vollist, $volid;
4515     });
4516
4517     return $vollist;
4518 }
4519
4520 sub vm_stop_cleanup {
4521     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
4522
4523     eval {
4524
4525         if (!$keepActive) {
4526             my $vollist = get_vm_volumes($conf);
4527             PVE::Storage::deactivate_volumes($storecfg, $vollist);
4528         }
4529
4530         foreach my $ext (qw(mon qmp pid vnc qga)) {
4531             unlink "/var/run/qemu-server/${vmid}.$ext";
4532         }
4533
4534         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
4535     };
4536     warn $@ if $@; # avoid errors - just warn
4537 }
4538
4539 # Note: use $nockeck to skip tests if VM configuration file exists.
4540 # We need that when migration VMs to other nodes (files already moved)
4541 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
4542 sub vm_stop {
4543     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
4544
4545     $force = 1 if !defined($force) && !$shutdown;
4546
4547     if ($migratedfrom){
4548         my $pid = check_running($vmid, $nocheck, $migratedfrom);
4549         kill 15, $pid if $pid;
4550         my $conf = load_config($vmid, $migratedfrom);
4551         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
4552         return;
4553     }
4554
4555     lock_config($vmid, sub {
4556
4557         my $pid = check_running($vmid, $nocheck);
4558         return if !$pid;
4559
4560         my $conf;
4561         if (!$nocheck) {
4562             $conf = load_config($vmid);
4563             check_lock($conf) if !$skiplock;
4564             if (!defined($timeout) && $shutdown && $conf->{startup}) {
4565                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
4566                 $timeout = $opts->{down} if $opts->{down};
4567             }
4568         }
4569
4570         $timeout = 60 if !defined($timeout);
4571
4572         eval {
4573             if ($shutdown) {
4574                 if (defined($conf) && $conf->{agent}) {
4575                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
4576                 } else {
4577                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
4578                 }
4579             } else {
4580                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
4581             }
4582         };
4583         my $err = $@;
4584
4585         if (!$err) {
4586             my $count = 0;
4587             while (($count < $timeout) && check_running($vmid, $nocheck)) {
4588                 $count++;
4589                 sleep 1;
4590             }
4591
4592             if ($count >= $timeout) {
4593                 if ($force) {
4594                     warn "VM still running - terminating now with SIGTERM\n";
4595                     kill 15, $pid;
4596                 } else {
4597                     die "VM quit/powerdown failed - got timeout\n";
4598                 }
4599             } else {
4600                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4601                 return;
4602             }
4603         } else {
4604             if ($force) {
4605                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
4606                 kill 15, $pid;
4607             } else {
4608                 die "VM quit/powerdown failed\n";
4609             }
4610         }
4611
4612         # wait again
4613         $timeout = 10;
4614
4615         my $count = 0;
4616         while (($count < $timeout) && check_running($vmid, $nocheck)) {
4617             $count++;
4618             sleep 1;
4619         }
4620
4621         if ($count >= $timeout) {
4622             warn "VM still running - terminating now with SIGKILL\n";
4623             kill 9, $pid;
4624             sleep 1;
4625         }
4626
4627         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4628    });
4629 }
4630
4631 sub vm_suspend {
4632     my ($vmid, $skiplock) = @_;
4633
4634     lock_config($vmid, sub {
4635
4636         my $conf = load_config($vmid);
4637
4638         check_lock($conf) if !($skiplock || ($conf->{lock} && $conf->{lock} eq 'backup'));
4639
4640         vm_mon_cmd($vmid, "stop");
4641     });
4642 }
4643
4644 sub vm_resume {
4645     my ($vmid, $skiplock, $nocheck) = @_;
4646
4647     lock_config($vmid, sub {
4648
4649         if (!$nocheck) {
4650
4651             my $conf = load_config($vmid);
4652
4653             check_lock($conf) if !($skiplock || ($conf->{lock} && $conf->{lock} eq 'backup'));
4654
4655             vm_mon_cmd($vmid, "cont");
4656
4657         } else {
4658             vm_mon_cmd_nocheck($vmid, "cont");
4659         }
4660     });
4661 }
4662
4663 sub vm_sendkey {
4664     my ($vmid, $skiplock, $key) = @_;
4665
4666     lock_config($vmid, sub {
4667
4668         my $conf = load_config($vmid);
4669
4670         # there is no qmp command, so we use the human monitor command
4671         vm_human_monitor_command($vmid, "sendkey $key");
4672     });
4673 }
4674
4675 sub vm_destroy {
4676     my ($storecfg, $vmid, $skiplock) = @_;
4677
4678     lock_config($vmid, sub {
4679
4680         my $conf = load_config($vmid);
4681
4682         check_lock($conf) if !$skiplock;
4683
4684         if (!check_running($vmid)) {
4685             destroy_vm($storecfg, $vmid);
4686         } else {
4687             die "VM $vmid is running - destroy failed\n";
4688         }
4689     });
4690 }
4691
4692 # pci helpers
4693
4694 sub file_write {
4695     my ($filename, $buf) = @_;
4696
4697     my $fh = IO::File->new($filename, "w");
4698     return undef if !$fh;
4699
4700     my $res = print $fh $buf;
4701
4702     $fh->close();
4703
4704     return $res;
4705 }
4706
4707 sub pci_device_info {
4708     my ($name) = @_;
4709
4710     my $res;
4711
4712     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
4713     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
4714
4715     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
4716     return undef if !defined($irq) || $irq !~ m/^\d+$/;
4717
4718     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
4719     return undef if !defined($vendor) || $vendor !~ s/^0x//;
4720
4721     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
4722     return undef if !defined($product) || $product !~ s/^0x//;
4723
4724     $res = {
4725         name => $name,
4726         vendor => $vendor,
4727         product => $product,
4728         domain => $domain,
4729         bus => $bus,
4730         slot => $slot,
4731         func => $func,
4732         irq => $irq,
4733         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
4734     };
4735
4736     return $res;
4737 }
4738
4739 sub pci_dev_reset {
4740     my ($dev) = @_;
4741
4742     my $name = $dev->{name};
4743
4744     my $fn = "$pcisysfs/devices/$name/reset";
4745
4746     return file_write($fn, "1");
4747 }
4748
4749 sub pci_dev_bind_to_vfio {
4750     my ($dev) = @_;
4751
4752     my $name = $dev->{name};
4753
4754     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4755
4756     if (!-d $vfio_basedir) {
4757         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4758     }
4759     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4760
4761     my $testdir = "$vfio_basedir/$name";
4762     return 1 if -d $testdir;
4763
4764     my $data = "$dev->{vendor} $dev->{product}";
4765     return undef if !file_write("$vfio_basedir/new_id", $data);
4766
4767     my $fn = "$pcisysfs/devices/$name/driver/unbind";
4768     if (!file_write($fn, $name)) {
4769         return undef if -f $fn;
4770     }
4771
4772     $fn = "$vfio_basedir/bind";
4773     if (! -d $testdir) {
4774         return undef if !file_write($fn, $name);
4775     }
4776
4777     return -d $testdir;
4778 }
4779
4780 sub pci_dev_group_bind_to_vfio {
4781     my ($pciid) = @_;
4782
4783     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4784
4785     if (!-d $vfio_basedir) {
4786         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4787     }
4788     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4789
4790     # get IOMMU group devices
4791     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
4792       my @devs = grep /^0000:/, readdir($D);
4793     closedir($D);
4794
4795     foreach my $pciid (@devs) {
4796         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
4797
4798         # pci bridges, switches or root ports are not supported
4799         # they have a pci_bus subdirectory so skip them
4800         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
4801
4802         my $info = pci_device_info($1);
4803         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
4804     }
4805
4806     return 1;
4807 }
4808
4809 sub print_pci_addr {
4810     my ($id, $bridges) = @_;
4811
4812     my $res = '';
4813     my $devices = {
4814         piix3 => { bus => 0, addr => 1 },
4815         #addr2 : first videocard
4816         balloon0 => { bus => 0, addr => 3 },
4817         watchdog => { bus => 0, addr => 4 },
4818         scsihw0 => { bus => 0, addr => 5 },
4819         'pci.3' => { bus => 0, addr => 5 }, #can also be used for virtio-scsi-single bridge
4820         scsihw1 => { bus => 0, addr => 6 },
4821         ahci0 => { bus => 0, addr => 7 },
4822         qga0 => { bus => 0, addr => 8 },
4823         spice => { bus => 0, addr => 9 },
4824         virtio0 => { bus => 0, addr => 10 },
4825         virtio1 => { bus => 0, addr => 11 },
4826         virtio2 => { bus => 0, addr => 12 },
4827         virtio3 => { bus => 0, addr => 13 },
4828         virtio4 => { bus => 0, addr => 14 },
4829         virtio5 => { bus => 0, addr => 15 },
4830         hostpci0 => { bus => 0, addr => 16 },
4831         hostpci1 => { bus => 0, addr => 17 },
4832         net0 => { bus => 0, addr => 18 },
4833         net1 => { bus => 0, addr => 19 },
4834         net2 => { bus => 0, addr => 20 },
4835         net3 => { bus => 0, addr => 21 },
4836         net4 => { bus => 0, addr => 22 },
4837         net5 => { bus => 0, addr => 23 },
4838         vga1 => { bus => 0, addr => 24 },
4839         vga2 => { bus => 0, addr => 25 },
4840         vga3 => { bus => 0, addr => 26 },
4841         hostpci2 => { bus => 0, addr => 27 },
4842         hostpci3 => { bus => 0, addr => 28 },
4843         #addr29 : usb-host (pve-usb.cfg)
4844         'pci.1' => { bus => 0, addr => 30 },
4845         'pci.2' => { bus => 0, addr => 31 },
4846         'net6' => { bus => 1, addr => 1 },
4847         'net7' => { bus => 1, addr => 2 },
4848         'net8' => { bus => 1, addr => 3 },
4849         'net9' => { bus => 1, addr => 4 },
4850         'net10' => { bus => 1, addr => 5 },
4851         'net11' => { bus => 1, addr => 6 },
4852         'net12' => { bus => 1, addr => 7 },
4853         'net13' => { bus => 1, addr => 8 },
4854         'net14' => { bus => 1, addr => 9 },
4855         'net15' => { bus => 1, addr => 10 },
4856         'net16' => { bus => 1, addr => 11 },
4857         'net17' => { bus => 1, addr => 12 },
4858         'net18' => { bus => 1, addr => 13 },
4859         'net19' => { bus => 1, addr => 14 },
4860         'net20' => { bus => 1, addr => 15 },
4861         'net21' => { bus => 1, addr => 16 },
4862         'net22' => { bus => 1, addr => 17 },
4863         'net23' => { bus => 1, addr => 18 },
4864         'net24' => { bus => 1, addr => 19 },
4865         'net25' => { bus => 1, addr => 20 },
4866         'net26' => { bus => 1, addr => 21 },
4867         'net27' => { bus => 1, addr => 22 },
4868         'net28' => { bus => 1, addr => 23 },
4869         'net29' => { bus => 1, addr => 24 },
4870         'net30' => { bus => 1, addr => 25 },
4871         'net31' => { bus => 1, addr => 26 },
4872         'virtio6' => { bus => 2, addr => 1 },
4873         'virtio7' => { bus => 2, addr => 2 },
4874         'virtio8' => { bus => 2, addr => 3 },
4875         'virtio9' => { bus => 2, addr => 4 },
4876         'virtio10' => { bus => 2, addr => 5 },
4877         'virtio11' => { bus => 2, addr => 6 },
4878         'virtio12' => { bus => 2, addr => 7 },
4879         'virtio13' => { bus => 2, addr => 8 },
4880         'virtio14' => { bus => 2, addr => 9 },
4881         'virtio15' => { bus => 2, addr => 10 },
4882         'virtioscsi0' => { bus => 3, addr => 1 },
4883         'virtioscsi1' => { bus => 3, addr => 2 },
4884         'virtioscsi2' => { bus => 3, addr => 3 },
4885         'virtioscsi3' => { bus => 3, addr => 4 },
4886         'virtioscsi4' => { bus => 3, addr => 5 },
4887         'virtioscsi5' => { bus => 3, addr => 6 },
4888         'virtioscsi6' => { bus => 3, addr => 7 },
4889         'virtioscsi7' => { bus => 3, addr => 8 },
4890         'virtioscsi8' => { bus => 3, addr => 9 },
4891         'virtioscsi9' => { bus => 3, addr => 10 },
4892         'virtioscsi10' => { bus => 3, addr => 11 },
4893         'virtioscsi11' => { bus => 3, addr => 12 },
4894         'virtioscsi12' => { bus => 3, addr => 13 },
4895         'virtioscsi13' => { bus => 3, addr => 14 },
4896         'virtioscsi14' => { bus => 3, addr => 15 },
4897         'virtioscsi15' => { bus => 3, addr => 16 },
4898         'virtioscsi16' => { bus => 3, addr => 17 },
4899         'virtioscsi17' => { bus => 3, addr => 18 },
4900         'virtioscsi18' => { bus => 3, addr => 19 },
4901         'virtioscsi19' => { bus => 3, addr => 20 },
4902         'virtioscsi20' => { bus => 3, addr => 21 },
4903         'virtioscsi21' => { bus => 3, addr => 22 },
4904         'virtioscsi22' => { bus => 3, addr => 23 },
4905         'virtioscsi23' => { bus => 3, addr => 24 },
4906         'virtioscsi24' => { bus => 3, addr => 25 },
4907         'virtioscsi25' => { bus => 3, addr => 26 },
4908         'virtioscsi26' => { bus => 3, addr => 27 },
4909         'virtioscsi27' => { bus => 3, addr => 28 },
4910         'virtioscsi28' => { bus => 3, addr => 29 },
4911         'virtioscsi29' => { bus => 3, addr => 30 },
4912         'virtioscsi30' => { bus => 3, addr => 31 },
4913
4914     };
4915
4916     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
4917            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
4918            my $bus = $devices->{$id}->{bus};
4919            $res = ",bus=pci.$bus,addr=$addr";
4920            $bridges->{$bus} = 1 if $bridges;
4921     }
4922     return $res;
4923
4924 }
4925
4926 sub print_pcie_addr {
4927     my ($id) = @_;
4928
4929     my $res = '';
4930     my $devices = {
4931         hostpci0 => { bus => "ich9-pcie-port-1", addr => 0 },
4932         hostpci1 => { bus => "ich9-pcie-port-2", addr => 0 },
4933         hostpci2 => { bus => "ich9-pcie-port-3", addr => 0 },
4934         hostpci3 => { bus => "ich9-pcie-port-4", addr => 0 },
4935     };
4936
4937     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
4938            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
4939            my $bus = $devices->{$id}->{bus};
4940            $res = ",bus=$bus,addr=$addr";
4941     }
4942     return $res;
4943
4944 }
4945
4946 # vzdump restore implementaion
4947
4948 sub tar_archive_read_firstfile {
4949     my $archive = shift;
4950
4951     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
4952
4953     # try to detect archive type first
4954     my $pid = open (TMP, "tar tf '$archive'|") ||
4955         die "unable to open file '$archive'\n";
4956     my $firstfile = <TMP>;
4957     kill 15, $pid;
4958     close TMP;
4959
4960     die "ERROR: archive contaions no data\n" if !$firstfile;
4961     chomp $firstfile;
4962
4963     return $firstfile;
4964 }
4965
4966 sub tar_restore_cleanup {
4967     my ($storecfg, $statfile) = @_;
4968
4969     print STDERR "starting cleanup\n";
4970
4971     if (my $fd = IO::File->new($statfile, "r")) {
4972         while (defined(my $line = <$fd>)) {
4973             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
4974                 my $volid = $2;
4975                 eval {
4976                     if ($volid =~ m|^/|) {
4977                         unlink $volid || die 'unlink failed\n';
4978                     } else {
4979                         PVE::Storage::vdisk_free($storecfg, $volid);
4980                     }
4981                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
4982                 };
4983                 print STDERR "unable to cleanup '$volid' - $@" if $@;
4984             } else {
4985                 print STDERR "unable to parse line in statfile - $line";
4986             }
4987         }
4988         $fd->close();
4989     }
4990 }
4991
4992 sub restore_archive {
4993     my ($archive, $vmid, $user, $opts) = @_;
4994
4995     my $format = $opts->{format};
4996     my $comp;
4997
4998     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
4999         $format = 'tar' if !$format;
5000         $comp = 'gzip';
5001     } elsif ($archive =~ m/\.tar$/) {
5002         $format = 'tar' if !$format;
5003     } elsif ($archive =~ m/.tar.lzo$/) {
5004         $format = 'tar' if !$format;
5005         $comp = 'lzop';
5006     } elsif ($archive =~ m/\.vma$/) {
5007         $format = 'vma' if !$format;
5008     } elsif ($archive =~ m/\.vma\.gz$/) {
5009         $format = 'vma' if !$format;
5010         $comp = 'gzip';
5011     } elsif ($archive =~ m/\.vma\.lzo$/) {
5012         $format = 'vma' if !$format;
5013         $comp = 'lzop';
5014     } else {
5015         $format = 'vma' if !$format; # default
5016     }
5017
5018     # try to detect archive format
5019     if ($format eq 'tar') {
5020         return restore_tar_archive($archive, $vmid, $user, $opts);
5021     } else {
5022         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5023     }
5024 }
5025
5026 sub restore_update_config_line {
5027     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5028
5029     return if $line =~ m/^\#qmdump\#/;
5030     return if $line =~ m/^\#vzdump\#/;
5031     return if $line =~ m/^lock:/;
5032     return if $line =~ m/^unused\d+:/;
5033     return if $line =~ m/^parent:/;
5034     return if $line =~ m/^template:/; # restored VM is never a template
5035
5036     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5037         # try to convert old 1.X settings
5038         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5039         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5040             my ($model, $macaddr) = split(/\=/, $devconfig);
5041             $macaddr = PVE::Tools::random_ether_addr() if !$macaddr || $unique;
5042             my $net = {
5043                 model => $model,
5044                 bridge => "vmbr$ind",
5045                 macaddr => $macaddr,
5046             };
5047             my $netstr = print_net($net);
5048
5049             print $outfd "net$cookie->{netcount}: $netstr\n";
5050             $cookie->{netcount}++;
5051         }
5052     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5053         my ($id, $netstr) = ($1, $2);
5054         my $net = parse_net($netstr);
5055         $net->{macaddr} = PVE::Tools::random_ether_addr() if $net->{macaddr};
5056         $netstr = print_net($net);
5057         print $outfd "$id: $netstr\n";
5058     } elsif ($line =~ m/^((ide|scsi|virtio|sata)\d+):\s*(\S+)\s*$/) {
5059         my $virtdev = $1;
5060         my $value = $3;
5061         if ($line =~ m/backup=no/) {
5062             print $outfd "#$line";
5063         } elsif ($virtdev && $map->{$virtdev}) {
5064             my $di = parse_drive($virtdev, $value);
5065             delete $di->{format}; # format can change on restore
5066             $di->{file} = $map->{$virtdev};
5067             $value = print_drive($vmid, $di);
5068             print $outfd "$virtdev: $value\n";
5069         } else {
5070             print $outfd $line;
5071         }
5072     } else {
5073         print $outfd $line;
5074     }
5075 }
5076
5077 sub scan_volids {
5078     my ($cfg, $vmid) = @_;
5079
5080     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5081
5082     my $volid_hash = {};
5083     foreach my $storeid (keys %$info) {
5084         foreach my $item (@{$info->{$storeid}}) {
5085             next if !($item->{volid} && $item->{size});
5086             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5087             $volid_hash->{$item->{volid}} = $item;
5088         }
5089     }
5090
5091     return $volid_hash;
5092 }
5093
5094 sub get_used_paths {
5095     my ($vmid, $storecfg, $conf, $scan_snapshots, $skip_drive) = @_;
5096
5097     my $used_path = {};
5098
5099     my $scan_config = sub {
5100         my ($cref, $snapname) = @_;
5101
5102         foreach my $key (keys %$cref) {
5103             my $value = $cref->{$key};
5104             if (valid_drivename($key)) {
5105                 next if $skip_drive && $key eq $skip_drive;
5106                 my $drive = parse_drive($key, $value);
5107                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5108                 if ($drive->{file} =~ m!^/!) {
5109                     $used_path->{$drive->{file}}++; # = 1;
5110                 } else {
5111                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5112                     next if !$storeid;
5113                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5114                     next if !$scfg;
5115                     my $path = PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5116                     $used_path->{$path}++; # = 1;
5117                 }
5118             }
5119         }
5120     };
5121
5122     &$scan_config($conf);
5123
5124     undef $skip_drive;
5125
5126     if ($scan_snapshots) {
5127         foreach my $snapname (keys %{$conf->{snapshots}}) {
5128             &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5129         }
5130     }
5131
5132     return $used_path;
5133 }
5134
5135 sub update_disksize {
5136     my ($vmid, $conf, $volid_hash) = @_;
5137
5138     my $changes;
5139
5140     my $used = {};
5141
5142     # Note: it is allowed to define multiple storages with same path (alias), so
5143     # we need to check both 'volid' and real 'path' (two different volid can point
5144     # to the same path).
5145
5146     my $usedpath = {};
5147
5148     # update size info
5149     foreach my $opt (keys %$conf) {
5150         if (valid_drivename($opt)) {
5151             my $drive = parse_drive($opt, $conf->{$opt});
5152             my $volid = $drive->{file};
5153             next if !$volid;
5154
5155             $used->{$volid} = 1;
5156             if ($volid_hash->{$volid} &&
5157                 (my $path = $volid_hash->{$volid}->{path})) {
5158                 $usedpath->{$path} = 1;
5159             }
5160
5161             next if drive_is_cdrom($drive);
5162             next if !$volid_hash->{$volid};
5163
5164             $drive->{size} = $volid_hash->{$volid}->{size};
5165             my $new = print_drive($vmid, $drive);
5166             if ($new ne $conf->{$opt}) {
5167                 $changes = 1;
5168                 $conf->{$opt} = $new;
5169             }
5170         }
5171     }
5172
5173     # remove 'unusedX' entry if volume is used
5174     foreach my $opt (keys %$conf) {
5175         next if $opt !~ m/^unused\d+$/;
5176         my $volid = $conf->{$opt};
5177         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5178         if ($used->{$volid} || ($path && $usedpath->{$path})) {
5179             $changes = 1;
5180             delete $conf->{$opt};
5181         }
5182     }
5183
5184     foreach my $volid (sort keys %$volid_hash) {
5185         next if $volid =~ m/vm-$vmid-state-/;
5186         next if $used->{$volid};
5187         my $path = $volid_hash->{$volid}->{path};
5188         next if !$path; # just to be sure
5189         next if $usedpath->{$path};
5190         $changes = 1;
5191         add_unused_volume($conf, $volid);
5192         $usedpath->{$path} = 1; # avoid to add more than once (aliases)
5193     }
5194
5195     return $changes;
5196 }
5197
5198 sub rescan {
5199     my ($vmid, $nolock) = @_;
5200
5201     my $cfg = PVE::Cluster::cfs_read_file("storage.cfg");
5202
5203     my $volid_hash = scan_volids($cfg, $vmid);
5204
5205     my $updatefn =  sub {
5206         my ($vmid) = @_;
5207
5208         my $conf = load_config($vmid);
5209
5210         check_lock($conf);
5211
5212         my $vm_volids = {};
5213         foreach my $volid (keys %$volid_hash) {
5214             my $info = $volid_hash->{$volid};
5215             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5216         }
5217
5218         my $changes = update_disksize($vmid, $conf, $vm_volids);
5219
5220         update_config_nolock($vmid, $conf, 1) if $changes;
5221     };
5222
5223     if (defined($vmid)) {
5224         if ($nolock) {
5225             &$updatefn($vmid);
5226         } else {
5227             lock_config($vmid, $updatefn, $vmid);
5228         }
5229     } else {
5230         my $vmlist = config_list();
5231         foreach my $vmid (keys %$vmlist) {
5232             if ($nolock) {
5233                 &$updatefn($vmid);
5234             } else {
5235                 lock_config($vmid, $updatefn, $vmid);
5236             }
5237         }
5238     }
5239 }
5240
5241 sub restore_vma_archive {
5242     my ($archive, $vmid, $user, $opts, $comp) = @_;
5243
5244     my $input = $archive eq '-' ? "<&STDIN" : undef;
5245     my $readfrom = $archive;
5246
5247     my $uncomp = '';
5248     if ($comp) {
5249         $readfrom = '-';
5250         my $qarchive = PVE::Tools::shellquote($archive);
5251         if ($comp eq 'gzip') {
5252             $uncomp = "zcat $qarchive|";
5253         } elsif ($comp eq 'lzop') {
5254             $uncomp = "lzop -d -c $qarchive|";
5255         } else {
5256             die "unknown compression method '$comp'\n";
5257         }
5258
5259     }
5260
5261     my $tmpdir = "/var/tmp/vzdumptmp$$";
5262     rmtree $tmpdir;
5263
5264     # disable interrupts (always do cleanups)
5265     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5266         warn "got interrupt - ignored\n";
5267     };
5268
5269     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5270     POSIX::mkfifo($mapfifo, 0600);
5271     my $fifofh;
5272
5273     my $openfifo = sub {
5274         open($fifofh, '>', $mapfifo) || die $!;
5275     };
5276
5277     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5278
5279     my $oldtimeout;
5280     my $timeout = 5;
5281
5282     my $devinfo = {};
5283
5284     my $rpcenv = PVE::RPCEnvironment::get();
5285
5286     my $conffile = config_file($vmid);
5287     my $tmpfn = "$conffile.$$.tmp";
5288
5289     # Note: $oldconf is undef if VM does not exists
5290     my $oldconf = PVE::Cluster::cfs_read_file(cfs_config_path($vmid));
5291
5292     my $print_devmap = sub {
5293         my $virtdev_hash = {};
5294
5295         my $cfgfn = "$tmpdir/qemu-server.conf";
5296
5297         # we can read the config - that is already extracted
5298         my $fh = IO::File->new($cfgfn, "r") ||
5299             "unable to read qemu-server.conf - $!\n";
5300
5301         while (defined(my $line = <$fh>)) {
5302             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5303                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5304                 die "archive does not contain data for drive '$virtdev'\n"
5305                     if !$devinfo->{$devname};
5306                 if (defined($opts->{storage})) {
5307                     $storeid = $opts->{storage} || 'local';
5308                 } elsif (!$storeid) {
5309                     $storeid = 'local';
5310                 }
5311                 $format = 'raw' if !$format;
5312                 $devinfo->{$devname}->{devname} = $devname;
5313                 $devinfo->{$devname}->{virtdev} = $virtdev;
5314                 $devinfo->{$devname}->{format} = $format;
5315                 $devinfo->{$devname}->{storeid} = $storeid;
5316
5317                 # check permission on storage
5318                 my $pool = $opts->{pool}; # todo: do we need that?
5319                 if ($user ne 'root@pam') {
5320                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5321                 }
5322
5323                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5324             }
5325         }
5326
5327         foreach my $devname (keys %$devinfo) {
5328             die "found no device mapping information for device '$devname'\n"
5329                 if !$devinfo->{$devname}->{virtdev};
5330         }
5331
5332         my $cfg = cfs_read_file('storage.cfg');
5333
5334         # create empty/temp config
5335         if ($oldconf) {
5336             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5337             foreach_drive($oldconf, sub {
5338                 my ($ds, $drive) = @_;
5339
5340                 return if drive_is_cdrom($drive);
5341
5342                 my $volid = $drive->{file};
5343
5344                 return if !$volid || $volid =~ m|^/|;
5345
5346                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5347                 return if !$path || !$owner || ($owner != $vmid);
5348
5349                 # Note: only delete disk we want to restore
5350                 # other volumes will become unused
5351                 if ($virtdev_hash->{$ds}) {
5352                     PVE::Storage::vdisk_free($cfg, $volid);
5353                 }
5354             });
5355         }
5356
5357         my $map = {};
5358         foreach my $virtdev (sort keys %$virtdev_hash) {
5359             my $d = $virtdev_hash->{$virtdev};
5360             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5361             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5362
5363             # test if requested format is supported
5364             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5365             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5366             $d->{format} = $defFormat if !$supported;
5367
5368             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5369                                                   $d->{format}, undef, $alloc_size);
5370             print STDERR "new volume ID is '$volid'\n";
5371             $d->{volid} = $volid;
5372             my $path = PVE::Storage::path($cfg, $volid);
5373
5374             PVE::Storage::activate_volumes($cfg,[$volid]);
5375
5376             my $write_zeros = 1;
5377             # fixme: what other storages types initialize volumes with zero?
5378             if ($scfg->{type} eq 'dir' || $scfg->{type} eq 'nfs' || $scfg->{type} eq 'glusterfs' ||
5379                 $scfg->{type} eq 'sheepdog' || $scfg->{type} eq 'rbd') {
5380                 $write_zeros = 0;
5381             }
5382
5383             print $fifofh "${write_zeros}:$d->{devname}=$path\n";
5384
5385             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5386             $map->{$virtdev} = $volid;
5387         }
5388
5389         $fh->seek(0, 0) || die "seek failed - $!\n";
5390
5391         my $outfd = new IO::File ($tmpfn, "w") ||
5392             die "unable to write config for VM $vmid\n";
5393
5394         my $cookie = { netcount => 0 };
5395         while (defined(my $line = <$fh>)) {
5396             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5397         }
5398
5399         $fh->close();
5400         $outfd->close();
5401     };
5402
5403     eval {
5404         # enable interrupts
5405         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5406             die "interrupted by signal\n";
5407         };
5408         local $SIG{ALRM} = sub { die "got timeout\n"; };
5409
5410         $oldtimeout = alarm($timeout);
5411
5412         my $parser = sub {
5413             my $line = shift;
5414
5415             print "$line\n";
5416
5417             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5418                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5419                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5420             } elsif ($line =~ m/^CTIME: /) {
5421                 # we correctly received the vma config, so we can disable
5422                 # the timeout now for disk allocation (set to 10 minutes, so
5423                 # that we always timeout if something goes wrong)
5424                 alarm(600);
5425                 &$print_devmap();
5426                 print $fifofh "done\n";
5427                 my $tmp = $oldtimeout || 0;
5428                 $oldtimeout = undef;
5429                 alarm($tmp);
5430                 close($fifofh);
5431             }
5432         };
5433
5434         print "restore vma archive: $cmd\n";
5435         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5436     };
5437     my $err = $@;
5438
5439     alarm($oldtimeout) if $oldtimeout;
5440
5441     my $vollist = [];
5442     foreach my $devname (keys %$devinfo) {
5443         my $volid = $devinfo->{$devname}->{volid};
5444         push @$vollist, $volid if $volid;
5445     }
5446
5447     my $cfg = cfs_read_file('storage.cfg');
5448     PVE::Storage::deactivate_volumes($cfg, $vollist);
5449
5450     unlink $mapfifo;
5451
5452     if ($err) {
5453         rmtree $tmpdir;
5454         unlink $tmpfn;
5455
5456         foreach my $devname (keys %$devinfo) {
5457             my $volid = $devinfo->{$devname}->{volid};
5458             next if !$volid;
5459             eval {
5460                 if ($volid =~ m|^/|) {
5461                     unlink $volid || die 'unlink failed\n';
5462                 } else {
5463                     PVE::Storage::vdisk_free($cfg, $volid);
5464                 }
5465                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5466             };
5467             print STDERR "unable to cleanup '$volid' - $@" if $@;
5468         }
5469         die $err;
5470     }
5471
5472     rmtree $tmpdir;
5473
5474     rename($tmpfn, $conffile) ||
5475         die "unable to commit configuration file '$conffile'\n";
5476
5477     PVE::Cluster::cfs_update(); # make sure we read new file
5478
5479     eval { rescan($vmid, 1); };
5480     warn $@ if $@;
5481 }
5482
5483 sub restore_tar_archive {
5484     my ($archive, $vmid, $user, $opts) = @_;
5485
5486     if ($archive ne '-') {
5487         my $firstfile = tar_archive_read_firstfile($archive);
5488         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5489             if $firstfile ne 'qemu-server.conf';
5490     }
5491
5492     my $storecfg = cfs_read_file('storage.cfg');
5493
5494     # destroy existing data - keep empty config
5495     my $vmcfgfn = config_file($vmid);
5496     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5497
5498     my $tocmd = "/usr/lib/qemu-server/qmextract";
5499
5500     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5501     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5502     $tocmd .= ' --prealloc' if $opts->{prealloc};
5503     $tocmd .= ' --info' if $opts->{info};
5504
5505     # tar option "xf" does not autodetect compression when read from STDIN,
5506     # so we pipe to zcat
5507     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5508         PVE::Tools::shellquote("--to-command=$tocmd");
5509
5510     my $tmpdir = "/var/tmp/vzdumptmp$$";
5511     mkpath $tmpdir;
5512
5513     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5514     local $ENV{VZDUMP_VMID} = $vmid;
5515     local $ENV{VZDUMP_USER} = $user;
5516
5517     my $conffile = config_file($vmid);
5518     my $tmpfn = "$conffile.$$.tmp";
5519
5520     # disable interrupts (always do cleanups)
5521     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5522         print STDERR "got interrupt - ignored\n";
5523     };
5524
5525     eval {
5526         # enable interrupts
5527         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5528             die "interrupted by signal\n";
5529         };
5530
5531         if ($archive eq '-') {
5532             print "extracting archive from STDIN\n";
5533             run_command($cmd, input => "<&STDIN");
5534         } else {
5535             print "extracting archive '$archive'\n";
5536             run_command($cmd);
5537         }
5538
5539         return if $opts->{info};
5540
5541         # read new mapping
5542         my $map = {};
5543         my $statfile = "$tmpdir/qmrestore.stat";
5544         if (my $fd = IO::File->new($statfile, "r")) {
5545             while (defined (my $line = <$fd>)) {
5546                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5547                     $map->{$1} = $2 if $1;
5548                 } else {
5549                     print STDERR "unable to parse line in statfile - $line\n";
5550                 }
5551             }
5552             $fd->close();
5553         }
5554
5555         my $confsrc = "$tmpdir/qemu-server.conf";
5556
5557         my $srcfd = new IO::File($confsrc, "r") ||
5558             die "unable to open file '$confsrc'\n";
5559
5560         my $outfd = new IO::File ($tmpfn, "w") ||
5561             die "unable to write config for VM $vmid\n";
5562
5563         my $cookie = { netcount => 0 };
5564         while (defined (my $line = <$srcfd>)) {
5565             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5566         }
5567
5568         $srcfd->close();
5569         $outfd->close();
5570     };
5571     my $err = $@;
5572
5573     if ($err) {
5574
5575         unlink $tmpfn;
5576
5577         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
5578
5579         die $err;
5580     }
5581
5582     rmtree $tmpdir;
5583
5584     rename $tmpfn, $conffile ||
5585         die "unable to commit configuration file '$conffile'\n";
5586
5587     PVE::Cluster::cfs_update(); # make sure we read new file
5588
5589     eval { rescan($vmid, 1); };
5590     warn $@ if $@;
5591 };
5592
5593
5594 # Internal snapshots
5595
5596 # NOTE: Snapshot create/delete involves several non-atomic
5597 # action, and can take a long time.
5598 # So we try to avoid locking the file and use 'lock' variable
5599 # inside the config file instead.
5600
5601 my $snapshot_copy_config = sub {
5602     my ($source, $dest) = @_;
5603
5604     foreach my $k (keys %$source) {
5605         next if $k eq 'snapshots';
5606         next if $k eq 'snapstate';
5607         next if $k eq 'snaptime';
5608         next if $k eq 'vmstate';
5609         next if $k eq 'lock';
5610         next if $k eq 'digest';
5611         next if $k eq 'description';
5612         next if $k =~ m/^unused\d+$/;
5613
5614         $dest->{$k} = $source->{$k};
5615     }
5616 };
5617
5618 my $snapshot_apply_config = sub {
5619     my ($conf, $snap) = @_;
5620
5621     # copy snapshot list
5622     my $newconf = {
5623         snapshots => $conf->{snapshots},
5624     };
5625
5626     # keep description and list of unused disks
5627     foreach my $k (keys %$conf) {
5628         next if !($k =~ m/^unused\d+$/ || $k eq 'description');
5629         $newconf->{$k} = $conf->{$k};
5630     }
5631
5632     &$snapshot_copy_config($snap, $newconf);
5633
5634     return $newconf;
5635 };
5636
5637 sub foreach_writable_storage {
5638     my ($conf, $func) = @_;
5639
5640     my $sidhash = {};
5641
5642     foreach my $ds (keys %$conf) {
5643         next if !valid_drivename($ds);
5644
5645         my $drive = parse_drive($ds, $conf->{$ds});
5646         next if !$drive;
5647         next if drive_is_cdrom($drive);
5648
5649         my $volid = $drive->{file};
5650
5651         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5652         $sidhash->{$sid} = $sid if $sid;
5653     }
5654
5655     foreach my $sid (sort keys %$sidhash) {
5656         &$func($sid);
5657     }
5658 }
5659
5660 my $alloc_vmstate_volid = sub {
5661     my ($storecfg, $vmid, $conf, $snapname) = @_;
5662
5663     # Note: we try to be smart when selecting a $target storage
5664
5665     my $target;
5666
5667     # search shared storage first
5668     foreach_writable_storage($conf, sub {
5669         my ($sid) = @_;
5670         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
5671         return if !$scfg->{shared};
5672
5673         $target = $sid if !$target || $scfg->{path}; # prefer file based storage
5674     });
5675
5676     if (!$target) {
5677         # now search local storage
5678         foreach_writable_storage($conf, sub {
5679             my ($sid) = @_;
5680             my $scfg = PVE::Storage::storage_config($storecfg, $sid);
5681             return if $scfg->{shared};
5682
5683             $target = $sid if !$target || $scfg->{path}; # prefer file based storage;
5684         });
5685     }
5686
5687     $target = 'local' if !$target;
5688
5689     my $driver_state_size = 500; # assume 32MB is enough to safe all driver state;
5690     # we abort live save after $conf->{memory}, so we need at max twice that space
5691     my $size = $conf->{memory}*2 + $driver_state_size;
5692
5693     my $name = "vm-$vmid-state-$snapname";
5694     my $scfg = PVE::Storage::storage_config($storecfg, $target);
5695     $name .= ".raw" if $scfg->{path}; # add filename extension for file base storage
5696     my $volid = PVE::Storage::vdisk_alloc($storecfg, $target, $vmid, 'raw', $name, $size*1024);
5697
5698     return $volid;
5699 };
5700
5701 my $snapshot_prepare = sub {
5702     my ($vmid, $snapname, $save_vmstate, $comment) = @_;
5703
5704     my $snap;
5705
5706     my $updatefn =  sub {
5707
5708         my $conf = load_config($vmid);
5709
5710         die "you can't take a snapshot if it's a template\n"
5711             if is_template($conf);
5712
5713         check_lock($conf);
5714
5715         $conf->{lock} = 'snapshot';
5716
5717         die "snapshot name '$snapname' already used\n"
5718             if defined($conf->{snapshots}->{$snapname});
5719
5720         my $storecfg = PVE::Storage::config();
5721         die "snapshot feature is not available" if !has_feature('snapshot', $conf, $storecfg);
5722
5723         $snap = $conf->{snapshots}->{$snapname} = {};
5724
5725         if ($save_vmstate && check_running($vmid)) {
5726             $snap->{vmstate} = &$alloc_vmstate_volid($storecfg, $vmid, $conf, $snapname);
5727         }
5728
5729         &$snapshot_copy_config($conf, $snap);
5730
5731         $snap->{snapstate} = "prepare";
5732         $snap->{snaptime} = time();
5733         $snap->{description} = $comment if $comment;
5734
5735         # always overwrite machine if we save vmstate. This makes sure we
5736         # can restore it later using correct machine type
5737         $snap->{machine} = get_current_qemu_machine($vmid) if $snap->{vmstate};
5738
5739         update_config_nolock($vmid, $conf, 1);
5740     };
5741
5742     lock_config($vmid, $updatefn);
5743
5744     return $snap;
5745 };
5746
5747 my $snapshot_commit = sub {
5748     my ($vmid, $snapname) = @_;
5749
5750     my $updatefn = sub {
5751
5752         my $conf = load_config($vmid);
5753
5754         die "missing snapshot lock\n"
5755             if !($conf->{lock} && $conf->{lock} eq 'snapshot');
5756
5757         my $has_machine_config = defined($conf->{machine});
5758
5759         my $snap = $conf->{snapshots}->{$snapname};
5760
5761         die "snapshot '$snapname' does not exist\n" if !defined($snap);
5762
5763         die "wrong snapshot state\n"
5764             if !($snap->{snapstate} && $snap->{snapstate} eq "prepare");
5765
5766         delete $snap->{snapstate};
5767         delete $conf->{lock};
5768
5769         my $newconf = &$snapshot_apply_config($conf, $snap);
5770
5771         delete $newconf->{machine} if !$has_machine_config;
5772
5773         $newconf->{parent} = $snapname;
5774
5775         update_config_nolock($vmid, $newconf, 1);
5776     };
5777
5778     lock_config($vmid, $updatefn);
5779 };
5780
5781 sub snapshot_rollback {
5782     my ($vmid, $snapname) = @_;
5783
5784     my $prepare = 1;
5785
5786     my $storecfg = PVE::Storage::config();
5787
5788     my $conf = load_config($vmid);
5789
5790     my $get_snapshot_config = sub {
5791
5792         die "you can't rollback if vm is a template\n" if is_template($conf);
5793
5794         my $res = $conf->{snapshots}->{$snapname};
5795
5796         die "snapshot '$snapname' does not exist\n" if !defined($res);
5797
5798         return $res;
5799     };
5800
5801     my $snap = &$get_snapshot_config();
5802
5803     foreach_drive($snap, sub {
5804         my ($ds, $drive) = @_;
5805
5806         return if drive_is_cdrom($drive);
5807
5808         my $volid = $drive->{file};
5809
5810         PVE::Storage::volume_rollback_is_possible($storecfg, $volid, $snapname);
5811     });
5812
5813     my $updatefn = sub {
5814
5815         $conf = load_config($vmid);
5816
5817         $snap = &$get_snapshot_config();
5818
5819         die "unable to rollback to incomplete snapshot (snapstate = $snap->{snapstate})\n"
5820             if $snap->{snapstate};
5821
5822         if ($prepare) {
5823             check_lock($conf);
5824             vm_stop($storecfg, $vmid, undef, undef, 5, undef, undef);
5825         }
5826
5827         die "unable to rollback vm $vmid: vm is running\n"
5828             if check_running($vmid);
5829
5830         if ($prepare) {
5831             $conf->{lock} = 'rollback';
5832         } else {
5833             die "got wrong lock\n" if !($conf->{lock} && $conf->{lock} eq 'rollback');
5834             delete $conf->{lock};
5835         }
5836
5837         my $forcemachine;
5838
5839         if (!$prepare) {
5840             my $has_machine_config = defined($conf->{machine});
5841
5842             # copy snapshot config to current config
5843             $conf = &$snapshot_apply_config($conf, $snap);
5844             $conf->{parent} = $snapname;
5845
5846             # Note: old code did not store 'machine', so we try to be smart
5847             # and guess the snapshot was generated with kvm 1.4 (pc-i440fx-1.4).
5848             $forcemachine = $conf->{machine} || 'pc-i440fx-1.4';
5849             # we remove the 'machine' configuration if not explicitly specified
5850             # in the original config.
5851             delete $conf->{machine} if $snap->{vmstate} && !$has_machine_config;
5852         }
5853
5854         update_config_nolock($vmid, $conf, 1);
5855
5856         if (!$prepare && $snap->{vmstate}) {
5857             my $statefile = PVE::Storage::path($storecfg, $snap->{vmstate});
5858             vm_start($storecfg, $vmid, $statefile, undef, undef, undef, $forcemachine);
5859         }
5860     };
5861
5862     lock_config($vmid, $updatefn);
5863
5864     foreach_drive($snap, sub {
5865         my ($ds, $drive) = @_;
5866
5867         return if drive_is_cdrom($drive);
5868
5869         my $volid = $drive->{file};
5870         my $device = "drive-$ds";
5871
5872         PVE::Storage::volume_snapshot_rollback($storecfg, $volid, $snapname);
5873     });
5874
5875     $prepare = 0;
5876     lock_config($vmid, $updatefn);
5877 }
5878
5879 my $savevm_wait = sub {
5880     my ($vmid) = @_;
5881
5882     for(;;) {
5883         my $stat = vm_mon_cmd_nocheck($vmid, "query-savevm");
5884         if (!$stat->{status}) {
5885             die "savevm not active\n";
5886         } elsif ($stat->{status} eq 'active') {
5887             sleep(1);
5888             next;
5889         } elsif ($stat->{status} eq 'completed') {
5890             last;
5891         } else {
5892             die "query-savevm returned status '$stat->{status}'\n";
5893         }
5894     }
5895 };
5896
5897 sub do_snapshots_with_qemu {
5898     my ($storecfg, $volid) = @_;
5899
5900     my $storage_name = PVE::Storage::parse_volume_id($volid);
5901
5902     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
5903         && !$storecfg->{ids}->{$storage_name}->{krbd}){
5904         return 1;
5905     }
5906
5907     if ($volid =~ m/\.(qcow2|qed)$/){
5908         return 1;
5909     }
5910
5911     return undef;
5912 }
5913
5914 sub snapshot_create {
5915     my ($vmid, $snapname, $save_vmstate, $comment) = @_;
5916
5917     my $snap = &$snapshot_prepare($vmid, $snapname, $save_vmstate, $comment);
5918
5919     $save_vmstate = 0 if !$snap->{vmstate}; # vm is not running
5920
5921     my $config = load_config($vmid);
5922
5923     my $running = check_running($vmid);
5924
5925     my $freezefs = $running && $config->{agent};
5926     $freezefs = 0 if $snap->{vmstate}; # not needed if we save RAM
5927
5928     my $drivehash = {};
5929
5930     if ($freezefs) {
5931         eval { vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
5932         warn "guest-fsfreeze-freeze problems - $@" if $@;
5933     }
5934
5935     eval {
5936         # create internal snapshots of all drives
5937
5938         my $storecfg = PVE::Storage::config();
5939
5940         if ($running) {
5941             if ($snap->{vmstate}) {
5942                 my $path = PVE::Storage::path($storecfg, $snap->{vmstate});
5943                 vm_mon_cmd($vmid, "savevm-start", statefile => $path);
5944                 &$savevm_wait($vmid);
5945             } else {
5946                 vm_mon_cmd($vmid, "savevm-start");
5947             }
5948         };
5949
5950         foreach_drive($snap, sub {
5951             my ($ds, $drive) = @_;
5952
5953             return if drive_is_cdrom($drive);
5954
5955             my $volid = $drive->{file};
5956             my $device = "drive-$ds";
5957
5958             qemu_volume_snapshot($vmid, $device, $storecfg, $volid, $snapname);
5959             $drivehash->{$ds} = 1;
5960        });
5961     };
5962     my $err = $@;
5963
5964     if ($running) {
5965         eval { vm_mon_cmd($vmid, "savevm-end")  };
5966         warn $@ if $@;
5967
5968         if ($freezefs) {
5969             eval { vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
5970             warn "guest-fsfreeze-thaw problems - $@" if $@;
5971         }
5972
5973         # savevm-end is async, we need to wait
5974         for (;;) {
5975             my $stat = vm_mon_cmd_nocheck($vmid, "query-savevm");
5976             if (!$stat->{bytes}) {
5977                 last;
5978             } else {
5979                 print "savevm not yet finished\n";
5980                 sleep(1);
5981                 next;
5982             }
5983         }
5984     }
5985
5986     if ($err) {
5987         warn "snapshot create failed: starting cleanup\n";
5988         eval { snapshot_delete($vmid, $snapname, 0, $drivehash); };
5989         warn $@ if $@;
5990         die $err;
5991     }
5992
5993     &$snapshot_commit($vmid, $snapname);
5994 }
5995
5996 # Note: $drivehash is only set when called from snapshot_create.
5997 sub snapshot_delete {
5998     my ($vmid, $snapname, $force, $drivehash) = @_;
5999
6000     my $prepare = 1;
6001
6002     my $snap;
6003     my $unused = [];
6004
6005     my $unlink_parent = sub {
6006         my ($confref, $new_parent) = @_;
6007
6008         if ($confref->{parent} && $confref->{parent} eq $snapname) {
6009             if ($new_parent) {
6010                 $confref->{parent} = $new_parent;
6011             } else {
6012                 delete $confref->{parent};
6013             }
6014         }
6015     };
6016
6017     my $updatefn =  sub {
6018         my ($remove_drive) = @_;
6019
6020         my $conf = load_config($vmid);
6021
6022         if (!$drivehash) {
6023             check_lock($conf);
6024             die "you can't delete a snapshot if vm is a template\n"
6025                 if is_template($conf);
6026         }
6027
6028         $snap = $conf->{snapshots}->{$snapname};
6029
6030         die "snapshot '$snapname' does not exist\n" if !defined($snap);
6031
6032         # remove parent refs
6033         if (!$prepare) {
6034             &$unlink_parent($conf, $snap->{parent});
6035             foreach my $sn (keys %{$conf->{snapshots}}) {
6036                 next if $sn eq $snapname;
6037                 &$unlink_parent($conf->{snapshots}->{$sn}, $snap->{parent});
6038             }
6039         }
6040
6041         if ($remove_drive) {
6042             if ($remove_drive eq 'vmstate') {
6043                 delete $snap->{$remove_drive};
6044             } else {
6045                 my $drive = parse_drive($remove_drive, $snap->{$remove_drive});
6046                 my $volid = $drive->{file};
6047                 delete $snap->{$remove_drive};
6048                 add_unused_volume($conf, $volid);
6049             }
6050         }
6051
6052         if ($prepare) {
6053             $snap->{snapstate} = 'delete';
6054         } else {
6055             delete $conf->{snapshots}->{$snapname};
6056             delete $conf->{lock} if $drivehash;
6057             foreach my $volid (@$unused) {
6058                 add_unused_volume($conf, $volid);
6059             }
6060         }
6061
6062         update_config_nolock($vmid, $conf, 1);
6063     };
6064
6065     lock_config($vmid, $updatefn);
6066
6067     # now remove vmstate file
6068
6069     my $storecfg = PVE::Storage::config();
6070
6071     if ($snap->{vmstate}) {
6072         eval {  PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
6073         if (my $err = $@) {
6074             die $err if !$force;
6075             warn $err;
6076         }
6077         # save changes (remove vmstate from snapshot)
6078         lock_config($vmid, $updatefn, 'vmstate') if !$force;
6079     };
6080
6081     # now remove all internal snapshots
6082     foreach_drive($snap, sub {
6083         my ($ds, $drive) = @_;
6084
6085         return if drive_is_cdrom($drive);
6086
6087         my $volid = $drive->{file};
6088         my $device = "drive-$ds";
6089
6090         if (!$drivehash || $drivehash->{$ds}) {
6091             eval { qemu_volume_snapshot_delete($vmid, $device, $storecfg, $volid, $snapname); };
6092             if (my $err = $@) {
6093                 die $err if !$force;
6094                 warn $err;
6095             }
6096         }
6097
6098         # save changes (remove drive fron snapshot)
6099         lock_config($vmid, $updatefn, $ds) if !$force;
6100         push @$unused, $volid;
6101     });
6102
6103     # now cleanup config
6104     $prepare = 0;
6105     lock_config($vmid, $updatefn);
6106 }
6107
6108 sub has_feature {
6109     my ($feature, $conf, $storecfg, $snapname, $running) = @_;
6110
6111     my $err;
6112     foreach_drive($conf, sub {
6113         my ($ds, $drive) = @_;
6114
6115         return if drive_is_cdrom($drive);
6116         my $volid = $drive->{file};
6117         $err = 1 if !PVE::Storage::volume_has_feature($storecfg, $feature, $volid, $snapname, $running);
6118     });
6119
6120     return $err ? 0 : 1;
6121 }
6122
6123 sub template_create {
6124     my ($vmid, $conf, $disk) = @_;
6125
6126     my $storecfg = PVE::Storage::config();
6127
6128     foreach_drive($conf, sub {
6129         my ($ds, $drive) = @_;
6130
6131         return if drive_is_cdrom($drive);
6132         return if $disk && $ds ne $disk;
6133
6134         my $volid = $drive->{file};
6135         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6136
6137         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6138         $drive->{file} = $voliddst;
6139         $conf->{$ds} = print_drive($vmid, $drive);
6140         update_config_nolock($vmid, $conf, 1);
6141     });
6142 }
6143
6144 sub is_template {
6145     my ($conf) = @_;
6146
6147     return 1 if defined $conf->{template} && $conf->{template} == 1;
6148 }
6149
6150 sub qemu_img_convert {
6151     my ($src_volid, $dst_volid, $size, $snapname) = @_;
6152
6153     my $storecfg = PVE::Storage::config();
6154     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6155     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6156
6157     if ($src_storeid && $dst_storeid) {
6158         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6159         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6160
6161         my $src_format = qemu_img_format($src_scfg, $src_volname);
6162         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6163
6164         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6165         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6166
6167         my $cmd = [];
6168         push @$cmd, '/usr/bin/qemu-img', 'convert', '-t', 'writeback', '-p', '-n';
6169         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
6170         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path, $dst_path;
6171
6172         my $parser = sub {
6173             my $line = shift;
6174             if($line =~ m/\((\S+)\/100\%\)/){
6175                 my $percent = $1;
6176                 my $transferred = int($size * $percent / 100);
6177                 my $remaining = $size - $transferred;
6178
6179                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6180             }
6181
6182         };
6183
6184         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6185         my $err = $@;
6186         die "copy failed: $err" if $err;
6187     }
6188 }
6189
6190 sub qemu_img_format {
6191     my ($scfg, $volname) = @_;
6192
6193     if ($scfg->{path} && $volname =~ m/\.(raw|cow|qcow|qcow2|qed|vmdk|cloop)$/) {
6194         return $1;
6195     } else {
6196         return "raw";
6197     }
6198 }
6199
6200 sub qemu_drive_mirror {
6201     my ($vmid, $drive, $dst_volid, $vmiddst) = @_;
6202
6203     my $storecfg = PVE::Storage::config();
6204     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6205
6206     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6207
6208     my $format = qemu_img_format($dst_scfg, $dst_volname);
6209
6210     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6211
6212     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $dst_path };
6213     $opts->{format} = $format if $format;
6214
6215     print "drive mirror is starting (scanning bitmap) : this step can take some minutes/hours, depend of disk size and storage speed\n";
6216
6217     eval {
6218     vm_mon_cmd($vmid, "drive-mirror", %$opts);
6219         while (1) {
6220             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6221             my $stat = @$stats[0];
6222             die "mirroring job seem to have die. Maybe do you have bad sectors?" if !$stat;
6223             die "error job is not mirroring" if $stat->{type} ne "mirror";
6224
6225             my $busy = $stat->{busy};
6226             my $ready = $stat->{ready};
6227
6228             if (my $total = $stat->{len}) {
6229                 my $transferred = $stat->{offset} || 0;
6230                 my $remaining = $total - $transferred;
6231                 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6232
6233                 print "transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6234             }
6235
6236
6237             if ($stat->{ready} eq 'true') {
6238
6239                 last if $vmiddst != $vmid;
6240
6241                 # try to switch the disk if source and destination are on the same guest
6242                 eval { vm_mon_cmd($vmid, "block-job-complete", device => "drive-$drive") };
6243                 last if !$@;
6244                 die $@ if $@ !~ m/cannot be completed/;
6245             }
6246             sleep 1;
6247         }
6248
6249
6250     };
6251     my $err = $@;
6252
6253     my $cancel_job = sub {
6254         vm_mon_cmd($vmid, "block-job-cancel", device => "drive-$drive");
6255         while (1) {
6256             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6257             my $stat = @$stats[0];
6258             last if !$stat;
6259             sleep 1;
6260         }
6261     };
6262
6263     if ($err) {
6264         eval { &$cancel_job(); };
6265         die "mirroring error: $err";
6266     }
6267
6268     if ($vmiddst != $vmid) {
6269         # if we clone a disk for a new target vm, we don't switch the disk
6270         &$cancel_job(); # so we call block-job-cancel
6271     }
6272 }
6273
6274 sub clone_disk {
6275     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6276         $newvmid, $storage, $format, $full, $newvollist) = @_;
6277
6278     my $newvolid;
6279
6280     if (!$full) {
6281         print "create linked clone of drive $drivename ($drive->{file})\n";
6282         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6283         push @$newvollist, $newvolid;
6284     } else {
6285         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6286         $storeid = $storage if $storage;
6287
6288         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6289         if (!$format) {
6290             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6291             $format = qemu_img_format($scfg, $volname);
6292         }
6293
6294         # test if requested format is supported - else use default
6295         my $supported = grep { $_ eq $format } @$validFormats;
6296         $format = $defFormat if !$supported;
6297
6298         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6299
6300         print "create full clone of drive $drivename ($drive->{file})\n";
6301         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $format, undef, ($size/1024));
6302         push @$newvollist, $newvolid;
6303
6304         PVE::Storage::activate_volumes($storecfg, $newvollist);
6305
6306         if (!$running || $snapname) {
6307             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname);
6308         } else {
6309             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid);
6310         }
6311     }
6312
6313     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6314
6315     my $disk = $drive;
6316     $disk->{format} = undef;
6317     $disk->{file} = $newvolid;
6318     $disk->{size} = $size;
6319
6320     return $disk;
6321 }
6322
6323 # this only works if VM is running
6324 sub get_current_qemu_machine {
6325     my ($vmid) = @_;
6326
6327     my $cmd = { execute => 'query-machines', arguments => {} };
6328     my $res = vm_qmp_command($vmid, $cmd);
6329
6330     my ($current, $default);
6331     foreach my $e (@$res) {
6332         $default = $e->{name} if $e->{'is-default'};
6333         $current = $e->{name} if $e->{'is-current'};
6334     }
6335
6336     # fallback to the default machine if current is not supported by qemu
6337     return $current || $default || 'pc';
6338 }
6339
6340 sub qemu_machine_feature_enabled {
6341     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6342
6343     my $current_major;
6344     my $current_minor;
6345
6346     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6347
6348         $current_major = $3;
6349         $current_minor = $4;
6350
6351     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6352
6353         $current_major = $1;
6354         $current_minor = $2;
6355     }
6356
6357     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
6358
6359
6360 }
6361
6362 sub qemu_machine_pxe {
6363     my ($vmid, $conf, $machine) = @_;
6364
6365     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6366
6367     foreach my $opt (keys %$conf) {
6368         next if $opt !~ m/^net(\d+)$/;
6369         my $net = PVE::QemuServer::parse_net($conf->{$opt});
6370         next if !$net;
6371         my $romfile = PVE::QemuServer::vm_mon_cmd_nocheck($vmid, 'qom-get', path => $opt, property => 'romfile');
6372         return $machine.".pxe" if $romfile =~ m/pxe/;
6373         last;
6374     }
6375
6376 }
6377
6378 sub qemu_use_old_bios_files {
6379     my ($machine_type) = @_;
6380
6381     return if !$machine_type;
6382
6383     my $use_old_bios_files = undef;
6384
6385     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6386         $machine_type = $1;
6387         $use_old_bios_files = 1;
6388     } else {
6389         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6390         # load new efi bios files on migration. So this hack is required to allow
6391         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6392         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6393         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, undef, 2, 4);
6394     }
6395
6396     return ($use_old_bios_files, $machine_type);
6397 }
6398
6399 sub lspci {
6400
6401     my $devices = {};
6402
6403     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6404             my (undef, $id, $function) = @_;
6405             my $res = { id => $id, function => $function};
6406             push @{$devices->{$id}}, $res;
6407     });
6408
6409     return $devices;
6410 }
6411
6412 sub vm_iothreads_list {
6413     my ($vmid) = @_;
6414
6415     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6416
6417     my $iothreads = {};
6418     foreach my $iothread (@$res) {
6419         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6420     }
6421
6422     return $iothreads;
6423 }
6424
6425 sub scsihw_infos {
6426     my ($conf, $drive) = @_;
6427
6428     my $maxdev = 0;
6429
6430     if ($conf->{scsihw} && ($conf->{scsihw} =~ m/^lsi/)) {
6431         $maxdev = 7;
6432     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6433         $maxdev = 1;
6434     } else {
6435         $maxdev = 256;
6436     }
6437
6438     my $controller = int($drive->{index} / $maxdev);
6439     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6440
6441     return ($maxdev, $controller, $controller_prefix);
6442 }
6443
6444 # bash completion helper
6445
6446 sub complete_backup_archives {
6447     my ($cmdname, $pname, $cvalue) = @_;
6448
6449     my $cfg = PVE::Storage::config();
6450
6451     my $storeid;
6452
6453     if ($cvalue =~ m/^([^:]+):/) {
6454         $storeid = $1;
6455     }
6456
6457     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6458
6459     my $res = [];
6460     foreach my $id (keys %$data) {
6461         foreach my $item (@{$data->{$id}}) {
6462             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6463             push @$res, $item->{volid} if defined($item->{volid});
6464         }
6465     }
6466
6467     return $res;
6468 }
6469
6470 my $complete_vmid_full = sub {
6471     my ($running) = @_;
6472
6473     my $idlist = vmstatus();
6474
6475     my $res = [];
6476
6477     foreach my $id (keys %$idlist) {
6478         my $d = $idlist->{$id};
6479         if (defined($running)) {
6480             next if $d->{template};
6481             next if $running && $d->{status} ne 'running';
6482             next if !$running && $d->{status} eq 'running';
6483         }
6484         push @$res, $id;
6485
6486     }
6487     return $res;
6488 };
6489
6490 sub complete_vmid {
6491     return &$complete_vmid_full();
6492 }
6493
6494 sub complete_vmid_stopped {
6495     return &$complete_vmid_full(0);
6496 }
6497
6498 sub complete_vmid_running {
6499     return &$complete_vmid_full(1);
6500 }
6501
6502 sub complete_storage {
6503
6504     my $cfg = PVE::Storage::config();
6505     my $ids = $cfg->{ids};
6506
6507     my $res = [];
6508     foreach my $sid (keys %$ids) {
6509         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6510         push @$res, $sid;
6511     }
6512
6513     return $res;
6514 }
6515
6516 1;