PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QMPClient;
  31 use PVE::RPCEnvironment;
  32 use Time::HiRes qw(gettimeofday);
  33
  34 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  35
  36 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  37
  38 # Note about locking: we use flock on the config file protect
  39 # against concurent actions.
  40 # Aditionaly, we have a 'lock' setting in the config file. This
  41 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  42 # allowed when such lock is set. But you can ignore this kind of
  43 # lock with the --skiplock flag.
  44
  45 cfs_register_file('/qemu-server/',
  46                   \&parse_vm_config,
  47                   \&write_vm_config);
  48
  49 PVE::JSONSchema::register_standard_option('skiplock', {
  50     description => "Ignore locks - only root is allowed to use this option.",
  51     type => 'boolean',
  52     optional => 1,
  53 });
  54
  55 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  56     description => "Some command save/restore state from this location.",
  57     type => 'string',
  58     maxLength => 128,
  59     optional => 1,
  60 });
  61
  62 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  63     description => "The name of the snapshot.",
  64     type => 'string', format => 'pve-configid',
  65     maxLength => 40,
  66 });
  67
  68 #no warnings 'redefine';
  69
  70 sub cgroups_write {
  71    my ($controller, $vmid, $option, $value) = @_;
  72
  73    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  74    PVE::ProcFSTools::write_proc_entry($path, $value);
  75
  76 }
  77
  78 my $nodename = PVE::INotify::nodename();
  79
  80 mkdir "/etc/pve/nodes/$nodename";
  81 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  82 mkdir $confdir;
  83
  84 my $var_run_tmpdir = "/var/run/qemu-server";
  85 mkdir $var_run_tmpdir;
  86
  87 my $lock_dir = "/var/lock/qemu-server";
  88 mkdir $lock_dir;
  89
  90 my $pcisysfs = "/sys/bus/pci";
  91
  92 my $confdesc = {
  93     onboot => {
  94         optional => 1,
  95         type => 'boolean',
  96         description => "Specifies whether a VM will be started during system bootup.",
  97         default => 0,
  98     },
  99     autostart => {
 100         optional => 1,
 101         type => 'boolean',
 102         description => "Automatic restart after crash (currently ignored).",
 103         default => 0,
 104     },
 105     hotplug => {
 106         optional => 1,
 107         type => 'string', format => 'pve-hotplug-features',
 108         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 109         default => 'network,disk,usb',
 110     },
 111     reboot => {
 112         optional => 1,
 113         type => 'boolean',
 114         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 115         default => 1,
 116     },
 117     lock => {
 118         optional => 1,
 119         type => 'string',
 120         description => "Lock/unlock the VM.",
 121         enum => [qw(migrate backup snapshot rollback)],
 122     },
 123     cpulimit => {
 124         optional => 1,
 125         type => 'number',
 126         description => "Limit of CPU usage. Note if the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 127         minimum => 0,
 128         maximum => 128,
 129         default => 0,
 130     },
 131     cpuunits => {
 132         optional => 1,
 133         type => 'integer',
 134         description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 135         minimum => 0,
 136         maximum => 500000,
 137         default => 1000,
 138     },
 139     memory => {
 140         optional => 1,
 141         type => 'integer',
 142         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 143         minimum => 16,
 144         default => 512,
 145     },
 146     balloon => {
 147         optional => 1,
 148         type => 'integer',
 149         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 150         minimum => 0,
 151     },
 152     shares => {
 153         optional => 1,
 154         type => 'integer',
 155         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 156         minimum => 0,
 157         maximum => 50000,
 158         default => 1000,
 159     },
 160     keyboard => {
 161         optional => 1,
 162         type => 'string',
 163         description => "Keybord layout for vnc server. Default is read from the datacenter configuration file.",
 164         enum => PVE::Tools::kvmkeymaplist(),
 165         default => 'en-us',
 166     },
 167     name => {
 168         optional => 1,
 169         type => 'string', format => 'dns-name',
 170         description => "Set a name for the VM. Only used on the configuration web interface.",
 171     },
 172     scsihw => {
 173         optional => 1,
 174         type => 'string',
 175         description => "scsi controller model",
 176         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 177         default => 'lsi',
 178     },
 179     description => {
 180         optional => 1,
 181         type => 'string',
 182         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 183     },
 184     ostype => {
 185         optional => 1,
 186         type => 'string',
 187         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 l24 l26 solaris)],
 188         description => <<EODESC,
 189 Used to enable special optimization/features for specific
 190 operating systems:
 191
 192 other  => unspecified OS
 193 wxp    => Microsoft Windows XP
 194 w2k    => Microsoft Windows 2000
 195 w2k3   => Microsoft Windows 2003
 196 w2k8   => Microsoft Windows 2008
 197 wvista => Microsoft Windows Vista
 198 win7   => Microsoft Windows 7
 199 win8   => Microsoft Windows 8/2012
 200 l24    => Linux 2.4 Kernel
 201 l26    => Linux 2.6/3.X Kernel
 202 solaris => solaris/opensolaris/openindiania kernel
 203
 204 other|l24|l26|solaris                       ... no special behaviour
 205 wxp|w2k|w2k3|w2k8|wvista|win7|win8  ... use --localtime switch
 206 EODESC
 207     },
 208     boot => {
 209         optional => 1,
 210         type => 'string',
 211         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 212         pattern => '[acdn]{1,4}',
 213         default => 'cdn',
 214     },
 215     bootdisk => {
 216         optional => 1,
 217         type => 'string', format => 'pve-qm-bootdisk',
 218         description => "Enable booting from specified disk.",
 219         pattern => '(ide|sata|scsi|virtio)\d+',
 220     },
 221     smp => {
 222         optional => 1,
 223         type => 'integer',
 224         description => "The number of CPUs. Please use option -sockets instead.",
 225         minimum => 1,
 226         default => 1,
 227     },
 228     sockets => {
 229         optional => 1,
 230         type => 'integer',
 231         description => "The number of CPU sockets.",
 232         minimum => 1,
 233         default => 1,
 234     },
 235     cores => {
 236         optional => 1,
 237         type => 'integer',
 238         description => "The number of cores per socket.",
 239         minimum => 1,
 240         default => 1,
 241     },
 242     numa => {
 243         optional => 1,
 244         type => 'boolean',
 245         description => "Enable/disable Numa.",
 246         default => 0,
 247     },
 248     vcpus => {
 249         optional => 1,
 250         type => 'integer',
 251         description => "Number of hotplugged vcpus.",
 252         minimum => 1,
 253         default => 0,
 254     },
 255     acpi => {
 256         optional => 1,
 257         type => 'boolean',
 258         description => "Enable/disable ACPI.",
 259         default => 1,
 260     },
 261     agent => {
 262         optional => 1,
 263         type => 'boolean',
 264         description => "Enable/disable Qemu GuestAgent.",
 265         default => 0,
 266     },
 267     kvm => {
 268         optional => 1,
 269         type => 'boolean',
 270         description => "Enable/disable KVM hardware virtualization.",
 271         default => 1,
 272     },
 273     tdf => {
 274         optional => 1,
 275         type => 'boolean',
 276         description => "Enable/disable time drift fix.",
 277         default => 0,
 278     },
 279     localtime => {
 280         optional => 1,
 281         type => 'boolean',
 282         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 283     },
 284     freeze => {
 285         optional => 1,
 286         type => 'boolean',
 287         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 288     },
 289     vga => {
 290         optional => 1,
 291         type => 'string',
 292         description => "Select VGA type. If you want to use high resolution modes (>= 1280x1024x16) then you should use option 'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and 'cirrur' for other OS types. Option 'qxl' enables the SPICE display sever. You can also run without any graphic card using a serial devive as terminal.",
 293         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 294     },
 295     watchdog => {
 296         optional => 1,
 297         type => 'string', format => 'pve-qm-watchdog',
 298         typetext => '[[model=]i6300esb|ib700] [,[action=]reset|shutdown|poweroff|pause|debug|none]',
 299         description => "Create a virtual hardware watchdog device.  Once enabled (by a guest action), the watchdog must be periodically polled by an agent inside the guest or else the guest will be restarted (or execute the action specified)",
 300     },
 301     startdate => {
 302         optional => 1,
 303         type => 'string',
 304         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 305         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 306         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 307         default => 'now',
 308     },
 309     startup =>  get_standard_option('pve-startup-order'),
 310     template => {
 311         optional => 1,
 312         type => 'boolean',
 313         description => "Enable/disable Template.",
 314         default => 0,
 315     },
 316     args => {
 317         optional => 1,
 318         type => 'string',
 319         description => <<EODESCR,
 320 Note: this option is for experts only. It allows you to pass arbitrary arguments to kvm, for example:
 321
 322 args: -no-reboot -no-hpet
 323 EODESCR
 324     },
 325     tablet => {
 326         optional => 1,
 327         type => 'boolean',
 328         default => 1,
 329         description => "Enable/disable the usb tablet device. This device is usually needed to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with normal VNC clients. If you're running lots of console-only guests on one host, you may consider disabling this to save some context switches. This is turned of by default if you use spice (vga=qxl).",
 330     },
 331     migrate_speed => {
 332         optional => 1,
 333         type => 'integer',
 334         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 335         minimum => 0,
 336         default => 0,
 337     },
 338     migrate_downtime => {
 339         optional => 1,
 340         type => 'number',
 341         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 342         minimum => 0,
 343         default => 0.1,
 344     },
 345     cdrom => {
 346         optional => 1,
 347         type => 'string', format => 'pve-qm-drive',
 348         typetext => 'volume',
 349         description => "This is an alias for option -ide2",
 350     },
 351     cpu => {
 352         optional => 1,
 353         description => "Emulated CPU type.",
 354         type => 'string',
 355         enum => [ qw(486 athlon pentium pentium2 pentium3 coreduo core2duo kvm32 kvm64 qemu32 qemu64 phenom Conroe Penryn Nehalem Westmere SandyBridge IvyBridge Haswell Broadwell Opteron_G1 Opteron_G2 Opteron_G3 Opteron_G4 Opteron_G5 host) ],
 356         default => 'kvm64',
 357     },
 358     parent => get_standard_option('pve-snapshot-name', {
 359         optional => 1,
 360         description => "Parent snapshot name. This is used internally, and should not be modified.",
 361     }),
 362     snaptime => {
 363         optional => 1,
 364         description => "Timestamp for snapshots.",
 365         type => 'integer',
 366         minimum => 0,
 367     },
 368     vmstate => {
 369         optional => 1,
 370         type => 'string', format => 'pve-volume-id',
 371         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 372     },
 373     machine => {
 374         description => "Specific the Qemu machine type.",
 375         type => 'string',
 376         pattern => '(pc|pc(-i440fx)?-\d+\.\d+|q35|pc-q35-\d+\.\d+)',
 377         maxLength => 40,
 378         optional => 1,
 379     },
 380     smbios1 => {
 381         description => "Specify SMBIOS type 1 fields.",
 382         type => 'string', format => 'pve-qm-smbios1',
 383         typetext => "[manufacturer=str][,product=str][,version=str][,serial=str] [,uuid=uuid][,sku=str][,family=str]",
 384         maxLength => 256,
 385         optional => 1,
 386     },
 387     protection => {
 388         optional => 1,
 389         type => 'boolean',
 390         description => "Sets the protection flag of the VM. This will prevent the remove operation.",
 391         default => 0,
 392     },
 393 };
 394
 395 # what about other qemu settings ?
 396 #cpu => 'string',
 397 #machine => 'string',
 398 #fda => 'file',
 399 #fdb => 'file',
 400 #mtdblock => 'file',
 401 #sd => 'file',
 402 #pflash => 'file',
 403 #snapshot => 'bool',
 404 #bootp => 'file',
 405 ##tftp => 'dir',
 406 ##smb => 'dir',
 407 #kernel => 'file',
 408 #append => 'string',
 409 #initrd => 'file',
 410 ##soundhw => 'string',
 411
 412 while (my ($k, $v) = each %$confdesc) {
 413     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 414 }
 415
 416 my $MAX_IDE_DISKS = 4;
 417 my $MAX_SCSI_DISKS = 14;
 418 my $MAX_VIRTIO_DISKS = 16;
 419 my $MAX_SATA_DISKS = 6;
 420 my $MAX_USB_DEVICES = 5;
 421 my $MAX_NETS = 32;
 422 my $MAX_UNUSED_DISKS = 8;
 423 my $MAX_HOSTPCI_DEVICES = 4;
 424 my $MAX_SERIAL_PORTS = 4;
 425 my $MAX_PARALLEL_PORTS = 3;
 426 my $MAX_NUMA = 8;
 427 my $MAX_MEM = 4194304;
 428 my $STATICMEM = 1024;
 429
 430 my $numadesc = {
 431     optional => 1,
 432     type => 'string', format => 'pve-qm-numanode',
 433     typetext => "cpus=<id[-id],memory=<mb>[[,hostnodes=<id[-id]>] [,policy=<preferred|bind|interleave>]]",
 434     description => "numa topology",
 435 };
 436 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 437
 438 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 439     $confdesc->{"numa$i"} = $numadesc;
 440 }
 441
 442 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 443                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 444                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 445 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 446
 447 my $netdesc = {
 448     optional => 1,
 449     type => 'string', format => 'pve-qm-net',
 450     typetext => "MODEL=XX:XX:XX:XX:XX:XX [,bridge=<dev>][,queues=<nbqueues>][,rate=<mbps>] [,tag=<vlanid>][,firewall=0|1],link_down=0|1]",
 451     description => <<EODESCR,
 452 Specify network devices.
 453
 454 MODEL is one of: $nic_model_list_txt
 455
 456 XX:XX:XX:XX:XX:XX should be an unique MAC address. This is
 457 automatically generated if not specified.
 458
 459 The bridge parameter can be used to automatically add the interface to a bridge device. The Proxmox VE standard bridge is called 'vmbr0'.
 460
 461 Option 'rate' is used to limit traffic bandwidth from and to this interface. It is specified as floating point number, unit is 'Megabytes per second'.
 462
 463 If you specify no bridge, we create a kvm 'user' (NATed) network device, which provides DHCP and DNS services. The following addresses are used:
 464
 465 10.0.2.2   Gateway
 466 10.0.2.3   DNS Server
 467 10.0.2.4   SMB Server
 468
 469 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 470
 471 EODESCR
 472 };
 473 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 474
 475 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 476     $confdesc->{"net$i"} = $netdesc;
 477 }
 478
 479 my $drivename_hash;
 480
 481 my $idedesc = {
 482     optional => 1,
 483     type => 'string', format => 'pve-qm-drive',
 484     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads] [,discard=ignore|on] [,serial=serial][,model=model]',
 485     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
 486 };
 487 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
 488
 489 my $scsidesc = {
 490     optional => 1,
 491     type => 'string', format => 'pve-qm-drive',
 492     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads] [,discard=ignore|on] [,iothread=on] [,queues=<nbqueues>] [,serial=serial]',
 493     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
 494 };
 495 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 496
 497 my $satadesc = {
 498     optional => 1,
 499     type => 'string', format => 'pve-qm-drive',
 500     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads]  [,discard=ignore|on] [,serial=serial]',
 501     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
 502 };
 503 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
 504
 505 my $virtiodesc = {
 506     optional => 1,
 507     type => 'string', format => 'pve-qm-drive',
 508     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads]  [,discard=ignore|on] [,iothread=on] [,serial=serial]',
 509     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
 510 };
 511 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
 512
 513 my $usbdesc = {
 514     optional => 1,
 515     type => 'string', format => 'pve-qm-usb-device',
 516     typetext => 'host=HOSTUSBDEVICE|spice',
 517     description => <<EODESCR,
 518 Configure an USB device (n is 0 to 4). This can be used to
 519 pass-through usb devices to the guest. HOSTUSBDEVICE syntax is:
 520
 521 'bus-port(.port)*' (decimal numbers) or
 522 'vendor_id:product_id' (hexadeciaml numbers)
 523
 524 You can use the 'lsusb -t' command to list existing usb devices.
 525
 526 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 527
 528 The value 'spice' can be used to add a usb redirection devices for spice.
 529
 530 EODESCR
 531 };
 532 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
 533
 534 my $hostpcidesc = {
 535         optional => 1,
 536         type => 'string', format => 'pve-qm-hostpci',
 537         typetext => "[host=]HOSTPCIDEVICE [,rombar=on|off] [,pcie=0|1] [,x-vga=on|off]",
 538         description => <<EODESCR,
 539 Map host pci devices. HOSTPCIDEVICE syntax is:
 540
 541 'bus:dev.func' (hexadecimal numbers)
 542
 543 You can us the 'lspci' command to list existing pci devices.
 544
 545 The 'rombar' option determines whether or not the device's ROM will be visible in the guest's memory map (default is 'on').
 546
 547 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 548
 549 Experimental: user reported problems with this option.
 550 EODESCR
 551 };
 552 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
 553
 554 my $serialdesc = {
 555         optional => 1,
 556         type => 'string',
 557         pattern => '(/dev/.+|socket)',
 558         description =>  <<EODESCR,
 559 Create a serial device inside the VM (n is 0 to 3), and pass through a host serial device (i.e. /dev/ttyS0), or create a unix socket on the host side (use 'qm terminal' to open a terminal connection).
 560
 561 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 562
 563 Experimental: user reported problems with this option.
 564 EODESCR
 565 };
 566
 567 my $paralleldesc= {
 568         optional => 1,
 569         type => 'string',
 570         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
 571         description =>  <<EODESCR,
 572 Map host parallel devices (n is 0 to 2).
 573
 574 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 575
 576 Experimental: user reported problems with this option.
 577 EODESCR
 578 };
 579
 580 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
 581     $confdesc->{"parallel$i"} = $paralleldesc;
 582 }
 583
 584 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
 585     $confdesc->{"serial$i"} = $serialdesc;
 586 }
 587
 588 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
 589     $confdesc->{"hostpci$i"} = $hostpcidesc;
 590 }
 591
 592 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
 593     $drivename_hash->{"ide$i"} = 1;
 594     $confdesc->{"ide$i"} = $idedesc;
 595 }
 596
 597 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
 598     $drivename_hash->{"sata$i"} = 1;
 599     $confdesc->{"sata$i"} = $satadesc;
 600 }
 601
 602 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
 603     $drivename_hash->{"scsi$i"} = 1;
 604     $confdesc->{"scsi$i"} = $scsidesc ;
 605 }
 606
 607 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
 608     $drivename_hash->{"virtio$i"} = 1;
 609     $confdesc->{"virtio$i"} = $virtiodesc;
 610 }
 611
 612 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
 613     $confdesc->{"usb$i"} = $usbdesc;
 614 }
 615
 616 my $unuseddesc = {
 617     optional => 1,
 618     type => 'string', format => 'pve-volume-id',
 619     description => "Reference to unused volumes.",
 620 };
 621
 622 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
 623     $confdesc->{"unused$i"} = $unuseddesc;
 624 }
 625
 626 my $kvm_api_version = 0;
 627
 628 sub kvm_version {
 629
 630     return $kvm_api_version if $kvm_api_version;
 631
 632     my $fh = IO::File->new("</dev/kvm") ||
 633         return 0;
 634
 635     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
 636         $kvm_api_version = $v;
 637     }
 638
 639     $fh->close();
 640
 641     return  $kvm_api_version;
 642 }
 643
 644 my $kvm_user_version;
 645
 646 sub kvm_user_version {
 647
 648     return $kvm_user_version if $kvm_user_version;
 649
 650     $kvm_user_version = 'unknown';
 651
 652     my $tmp = `kvm -help 2>/dev/null`;
 653
 654     if ($tmp =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
 655         $kvm_user_version = $2;
 656     }
 657
 658     return $kvm_user_version;
 659
 660 }
 661
 662 my $kernel_has_vhost_net = -c '/dev/vhost-net';
 663
 664 sub disknames {
 665     # order is important - used to autoselect boot disk
 666     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
 667             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
 668             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
 669             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))));
 670 }
 671
 672 sub valid_drivename {
 673     my $dev = shift;
 674
 675     return defined($drivename_hash->{$dev});
 676 }
 677
 678 sub option_exists {
 679     my $key = shift;
 680     return defined($confdesc->{$key});
 681 }
 682
 683 sub nic_models {
 684     return $nic_model_list;
 685 }
 686
 687 sub os_list_description {
 688
 689     return {
 690         other => 'Other',
 691         wxp => 'Windows XP',
 692         w2k => 'Windows 2000',
 693         w2k3 =>, 'Windows 2003',
 694         w2k8 => 'Windows 2008',
 695         wvista => 'Windows Vista',
 696         win7 => 'Windows 7',
 697         win8 => 'Windows 8/2012',
 698         l24 => 'Linux 2.4',
 699         l26 => 'Linux 2.6',
 700     };
 701 }
 702
 703 my $cdrom_path;
 704
 705 sub get_cdrom_path {
 706
 707     return  $cdrom_path if $cdrom_path;
 708
 709     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
 710     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
 711     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
 712 }
 713
 714 sub get_iso_path {
 715     my ($storecfg, $vmid, $cdrom) = @_;
 716
 717     if ($cdrom eq 'cdrom') {
 718         return get_cdrom_path();
 719     } elsif ($cdrom eq 'none') {
 720         return '';
 721     } elsif ($cdrom =~ m|^/|) {
 722         return $cdrom;
 723     } else {
 724         return PVE::Storage::path($storecfg, $cdrom);
 725     }
 726 }
 727
 728 # try to convert old style file names to volume IDs
 729 sub filename_to_volume_id {
 730     my ($vmid, $file, $media) = @_;
 731
 732     if (!($file eq 'none' || $file eq 'cdrom' ||
 733           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
 734
 735         return undef if $file =~ m|/|;
 736
 737         if ($media && $media eq 'cdrom') {
 738             $file = "local:iso/$file";
 739         } else {
 740             $file = "local:$vmid/$file";
 741         }
 742     }
 743
 744     return $file;
 745 }
 746
 747 sub verify_media_type {
 748     my ($opt, $vtype, $media) = @_;
 749
 750     return if !$media;
 751
 752     my $etype;
 753     if ($media eq 'disk') {
 754         $etype = 'images';
 755     } elsif ($media eq 'cdrom') {
 756         $etype = 'iso';
 757     } else {
 758         die "internal error";
 759     }
 760
 761     return if ($vtype eq $etype);
 762
 763     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
 764 }
 765
 766 sub cleanup_drive_path {
 767     my ($opt, $storecfg, $drive) = @_;
 768
 769     # try to convert filesystem paths to volume IDs
 770
 771     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
 772         ($drive->{file} !~ m|^/dev/.+|) &&
 773         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
 774         ($drive->{file} !~ m/^\d+$/)) {
 775         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
 776         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
 777         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
 778         verify_media_type($opt, $vtype, $drive->{media});
 779         $drive->{file} = $volid;
 780     }
 781
 782     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
 783 }
 784
 785 sub create_conf_nolock {
 786     my ($vmid, $settings) = @_;
 787
 788     my $filename = config_file($vmid);
 789
 790     die "configuration file '$filename' already exists\n" if -f $filename;
 791
 792     my $defaults = load_defaults();
 793
 794     $settings->{name} = "vm$vmid" if !$settings->{name};
 795     $settings->{memory} = $defaults->{memory} if !$settings->{memory};
 796
 797     my $data = '';
 798     foreach my $opt (keys %$settings) {
 799         next if !$confdesc->{$opt};
 800
 801         my $value = $settings->{$opt};
 802         next if !$value;
 803
 804         $data .= "$opt: $value\n";
 805     }
 806
 807     PVE::Tools::file_set_contents($filename, $data);
 808 }
 809
 810 sub parse_hotplug_features {
 811     my ($data) = @_;
 812
 813     my $res = {};
 814
 815     return $res if $data eq '0';
 816
 817     $data = $confdesc->{hotplug}->{default} if $data eq '1';
 818
 819     foreach my $feature (PVE::Tools::split_list($data)) {
 820         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
 821             $res->{$1} = 1;
 822         } else {
 823             warn "ignoring unknown hotplug feature '$feature'\n";
 824         }
 825     }
 826     return $res;
 827 }
 828
 829 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
 830 sub pve_verify_hotplug_features {
 831     my ($value, $noerr) = @_;
 832
 833     return $value if parse_hotplug_features($value);
 834
 835     return undef if $noerr;
 836
 837     die "unable to parse hotplug option\n";
 838 }
 839
 840 my $parse_size = sub {
 841     my ($value) = @_;
 842
 843     return undef if $value !~ m/^(\d+(\.\d+)?)([KMG])?$/;
 844     my ($size, $unit) = ($1, $3);
 845     if ($unit) {
 846         if ($unit eq 'K') {
 847             $size = $size * 1024;
 848         } elsif ($unit eq 'M') {
 849             $size = $size * 1024 * 1024;
 850         } elsif ($unit eq 'G') {
 851             $size = $size * 1024 * 1024 * 1024;
 852         }
 853     }
 854     return int($size);
 855 };
 856
 857 my $format_size = sub {
 858     my ($size) = @_;
 859
 860     $size = int($size);
 861
 862     my $kb = int($size/1024);
 863     return $size if $kb*1024 != $size;
 864
 865     my $mb = int($kb/1024);
 866     return "${kb}K" if $mb*1024 != $kb;
 867
 868     my $gb = int($mb/1024);
 869     return "${mb}M" if $gb*1024 != $mb;
 870
 871     return "${gb}G";
 872 };
 873
 874 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
 875 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
 876 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
 877 #        [,aio=native|threads][,discard=ignore|on][,iothread=on]
 878 #        [,serial=serial][,model=model]
 879
 880 sub parse_drive {
 881     my ($key, $data) = @_;
 882
 883     my $res = {};
 884
 885     # $key may be undefined - used to verify JSON parameters
 886     if (!defined($key)) {
 887         $res->{interface} = 'unknown'; # should not harm when used to verify parameters
 888         $res->{index} = 0;
 889     } elsif ($key =~ m/^([^\d]+)(\d+)$/) {
 890         $res->{interface} = $1;
 891         $res->{index} = $2;
 892     } else {
 893         return undef;
 894     }
 895
 896     foreach my $p (split (/,/, $data)) {
 897         next if $p =~ m/^\s*$/;
 898
 899         if ($p =~ m/^(file|volume|cyls|heads|secs|trans|media|snapshot|cache|format|rerror|werror|backup|aio|bps|mbps|mbps_max|bps_rd|mbps_rd|mbps_rd_max|bps_wr|mbps_wr|mbps_wr_max|iops|iops_max|iops_rd|iops_rd_max|iops_wr|iops_wr_max|size|discard|iothread|queues|serial|model)=(.+)$/) {
 900             my ($k, $v) = ($1, $2);
 901
 902             $k = 'file' if $k eq 'volume';
 903
 904             return undef if defined $res->{$k};
 905
 906             if ($k eq 'bps' || $k eq 'bps_rd' || $k eq 'bps_wr') {
 907                 return undef if !$v || $v !~ m/^\d+/;
 908                 $k = "m$k";
 909                 $v = sprintf("%.3f", $v / (1024*1024));
 910             }
 911             $res->{$k} = $v;
 912         } else {
 913             if (!$res->{file} && $p !~ m/=/) {
 914                 $res->{file} = $p;
 915             } else {
 916                 return undef;
 917             }
 918         }
 919     }
 920
 921     return undef if !$res->{file};
 922
 923     return undef if $res->{cache} &&
 924         $res->{cache} !~ m/^(off|none|writethrough|writeback|unsafe|directsync)$/;
 925     return undef if $res->{snapshot} && $res->{snapshot} !~ m/^(on|off)$/;
 926     return undef if $res->{cyls} && $res->{cyls} !~ m/^\d+$/;
 927     return undef if $res->{heads} && $res->{heads} !~ m/^\d+$/;
 928     return undef if $res->{secs} && $res->{secs} !~ m/^\d+$/;
 929     return undef if $res->{media} && $res->{media} !~ m/^(disk|cdrom)$/;
 930     return undef if $res->{trans} && $res->{trans} !~ m/^(none|lba|auto)$/;
 931     return undef if $res->{format} && $res->{format} !~ m/^(raw|cow|qcow|qed|qcow2|vmdk|cloop)$/;
 932     return undef if $res->{rerror} && $res->{rerror} !~ m/^(ignore|report|stop)$/;
 933     return undef if $res->{werror} && $res->{werror} !~ m/^(enospc|ignore|report|stop)$/;
 934     return undef if $res->{backup} && $res->{backup} !~ m/^(yes|no)$/;
 935     return undef if $res->{aio} && $res->{aio} !~ m/^(native|threads)$/;
 936     return undef if $res->{discard} && $res->{discard} !~ m/^(ignore|on)$/;
 937     return undef if $res->{iothread} && $res->{iothread} !~ m/^(on)$/;
 938     return undef if $res->{queues} && ($res->{queues} !~ m/^\d+$/ || $res->{queues} < 2);
 939
 940     return undef if $res->{mbps_rd} && $res->{mbps};
 941     return undef if $res->{mbps_wr} && $res->{mbps};
 942
 943     return undef if $res->{mbps} && $res->{mbps} !~ m/^\d+(\.\d+)?$/;
 944     return undef if $res->{mbps_max} && $res->{mbps_max} !~ m/^\d+(\.\d+)?$/;
 945     return undef if $res->{mbps_rd} && $res->{mbps_rd} !~ m/^\d+(\.\d+)?$/;
 946     return undef if $res->{mbps_rd_max} && $res->{mbps_rd_max} !~ m/^\d+(\.\d+)?$/;
 947     return undef if $res->{mbps_wr} && $res->{mbps_wr} !~ m/^\d+(\.\d+)?$/;
 948     return undef if $res->{mbps_wr_max} && $res->{mbps_wr_max} !~ m/^\d+(\.\d+)?$/;
 949
 950     return undef if $res->{iops_rd} && $res->{iops};
 951     return undef if $res->{iops_wr} && $res->{iops};
 952
 953
 954     return undef if $res->{iops} && $res->{iops} !~ m/^\d+$/;
 955     return undef if $res->{iops_max} && $res->{iops_max} !~ m/^\d+$/;
 956     return undef if $res->{iops_rd} && $res->{iops_rd} !~ m/^\d+$/;
 957     return undef if $res->{iops_rd_max} && $res->{iops_rd_max} !~ m/^\d+$/;
 958     return undef if $res->{iops_wr} && $res->{iops_wr} !~ m/^\d+$/;
 959     return undef if $res->{iops_wr_max} && $res->{iops_wr_max} !~ m/^\d+$/;
 960
 961     if ($res->{size}) {
 962         return undef if !defined($res->{size} = &$parse_size($res->{size}));
 963     }
 964
 965     if ($res->{media} && ($res->{media} eq 'cdrom')) {
 966         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
 967         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
 968         return undef if $res->{interface} eq 'virtio';
 969     }
 970
 971     # rerror does not work with scsi drives
 972     if ($res->{rerror}) {
 973         return undef if $res->{interface} eq 'scsi';
 974     }
 975
 976     return $res;
 977 }
 978
 979 my @qemu_drive_options = qw(heads secs cyls trans media format cache snapshot rerror werror aio discard iops iops_rd iops_wr iops_max iops_rd_max iops_wr_max serial);
 980
 981 sub print_drive {
 982     my ($vmid, $drive) = @_;
 983
 984     my $opts = '';
 985     foreach my $o (@qemu_drive_options, 'mbps', 'mbps_rd', 'mbps_wr', 'mbps_max', 'mbps_rd_max', 'mbps_wr_max', 'backup', 'iothread', 'queues') {
 986         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
 987     }
 988
 989     if ($drive->{size}) {
 990         $opts .= ",size=" . &$format_size($drive->{size});
 991     }
 992
 993     if (my $model = $drive->{model}) {
 994         $opts .= ",model=$model";
 995     }
 996
 997     return "$drive->{file}$opts";
 998 }
 999
1000 sub scsi_inquiry {
1001     my($fh, $noerr) = @_;
1002
1003     my $SG_IO = 0x2285;
1004     my $SG_GET_VERSION_NUM = 0x2282;
1005
1006     my $versionbuf = "\x00" x 8;
1007     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1008     if (!$ret) {
1009         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1010         return undef;
1011     }
1012     my $version = unpack("I", $versionbuf);
1013     if ($version < 30000) {
1014         die "scsi generic interface too old\n"  if !$noerr;
1015         return undef;
1016     }
1017
1018     my $buf = "\x00" x 36;
1019     my $sensebuf = "\x00" x 8;
1020     my $cmd = pack("C x3 C x1", 0x12, 36);
1021
1022     # see /usr/include/scsi/sg.h
1023     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1024
1025     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1026                       length($sensebuf), 0, length($buf), $buf,
1027                       $cmd, $sensebuf, 6000);
1028
1029     $ret = ioctl($fh, $SG_IO, $packet);
1030     if (!$ret) {
1031         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1032         return undef;
1033     }
1034
1035     my @res = unpack($sg_io_hdr_t, $packet);
1036     if ($res[17] || $res[18]) {
1037         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1038         return undef;
1039     }
1040
1041     my $res = {};
1042     (my $byte0, my $byte1, $res->{vendor},
1043      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1044
1045     $res->{removable} = $byte1 & 128 ? 1 : 0;
1046     $res->{type} = $byte0 & 31;
1047
1048     return $res;
1049 }
1050
1051 sub path_is_scsi {
1052     my ($path) = @_;
1053
1054     my $fh = IO::File->new("+<$path") || return undef;
1055     my $res = scsi_inquiry($fh, 1);
1056     close($fh);
1057
1058     return $res;
1059 }
1060
1061 sub machine_type_is_q35 {
1062     my ($conf) = @_;
1063
1064     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1065 }
1066
1067 sub print_tabletdevice_full {
1068     my ($conf) = @_;
1069
1070     my $q35 = machine_type_is_q35($conf);
1071
1072     # we use uhci for old VMs because tablet driver was buggy in older qemu
1073     my $usbbus = $q35 ? "ehci" : "uhci";
1074
1075     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1076 }
1077
1078 sub print_drivedevice_full {
1079     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1080
1081     my $device = '';
1082     my $maxdev = 0;
1083
1084     if ($drive->{interface} eq 'virtio') {
1085         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1086         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1087         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1088     } elsif ($drive->{interface} eq 'scsi') {
1089
1090         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1091         my $unit = $drive->{index} % $maxdev;
1092         my $devicetype = 'hd';
1093         my $path = '';
1094         if (drive_is_cdrom($drive)) {
1095               $devicetype = 'cd';
1096         } else {
1097               if ($drive->{file} =~ m|^/|) {
1098                   $path = $drive->{file};
1099               } else {
1100                   $path = PVE::Storage::path($storecfg, $drive->{file});
1101               }
1102
1103               if($path =~ m/^iscsi\:\/\//){
1104                   $devicetype = 'generic';
1105               } else {
1106                   if (my $info = path_is_scsi($path)) {
1107                       if ($info->{type} == 0) {
1108                           $devicetype = 'block';
1109                       } elsif ($info->{type} == 1) { # tape
1110                           $devicetype = 'generic';
1111                       }
1112                   }
1113               }
1114          }
1115
1116         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1117             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1118         } else {
1119             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1120         }
1121
1122     } elsif ($drive->{interface} eq 'ide'){
1123         $maxdev = 2;
1124         my $controller = int($drive->{index} / $maxdev);
1125         my $unit = $drive->{index} % $maxdev;
1126         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1127
1128         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1129         if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
1130             $device .= ",model=$model";
1131         }
1132     } elsif ($drive->{interface} eq 'sata'){
1133         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1134         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1135         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1136     } elsif ($drive->{interface} eq 'usb') {
1137         die "implement me";
1138         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1139     } else {
1140         die "unsupported interface type";
1141     }
1142
1143     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1144
1145     return $device;
1146 }
1147
1148 sub get_initiator_name {
1149     my $initiator;
1150
1151     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1152     while (defined(my $line = <$fh>)) {
1153         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1154         $initiator = $1;
1155         last;
1156     }
1157     $fh->close();
1158
1159     return $initiator;
1160 }
1161
1162 sub print_drive_full {
1163     my ($storecfg, $vmid, $drive) = @_;
1164
1165     my $path;
1166     my $volid = $drive->{file};
1167     my $format;
1168
1169     if (drive_is_cdrom($drive)) {
1170         $path = get_iso_path($storecfg, $vmid, $volid);
1171     } else {
1172         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1173         if ($storeid) {
1174             $path = PVE::Storage::path($storecfg, $volid);
1175             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1176             $format = qemu_img_format($scfg, $volname);
1177         } else {
1178             $path = $volid;
1179         }
1180    }
1181
1182     my $opts = '';
1183     foreach my $o (@qemu_drive_options) {
1184         next if $o eq 'bootindex';
1185         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1186     }
1187
1188     $opts .= ",format=$format" if $format && !$drive->{format};
1189
1190     foreach my $o (qw(bps bps_rd bps_wr)) {
1191         my $v = $drive->{"m$o"};
1192         $opts .= ",$o=" . int($v*1024*1024) if $v;
1193     }
1194
1195     my $cache_direct = 0;
1196
1197     if (my $cache = $drive->{cache}) {
1198         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1199     } elsif (!drive_is_cdrom($drive)) {
1200         $opts .= ",cache=none";
1201         $cache_direct = 1;
1202     }
1203
1204     # aio native works only with O_DIRECT
1205     if (!$drive->{aio}) {
1206         if($cache_direct) {
1207             $opts .= ",aio=native";
1208         } else {
1209             $opts .= ",aio=threads";
1210         }
1211     }
1212
1213     my $detectzeroes = $drive->{discard} ? "unmap" : "on";
1214     $opts .= ",detect-zeroes=$detectzeroes" if !drive_is_cdrom($drive);
1215
1216     my $pathinfo = $path ? "file=$path," : '';
1217
1218     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1219 }
1220
1221 sub print_netdevice_full {
1222     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1223
1224     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1225
1226     my $device = $net->{model};
1227     if ($net->{model} eq 'virtio') {
1228          $device = 'virtio-net-pci';
1229      };
1230
1231     my $pciaddr = print_pci_addr("$netid", $bridges);
1232     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1233     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1234         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1235         my $vectors = $net->{queues} * 2 + 2;
1236         $tmpstr .= ",vectors=$vectors,mq=on";
1237     }
1238     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1239
1240     if ($use_old_bios_files) {
1241         my $romfile;
1242         if ($device eq 'virtio-net-pci') {
1243             $romfile = 'pxe-virtio.rom';
1244         } elsif ($device eq 'e1000') {
1245             $romfile = 'pxe-e1000.rom';
1246         } elsif ($device eq 'ne2k') {
1247             $romfile = 'pxe-ne2k_pci.rom';
1248         } elsif ($device eq 'pcnet') {
1249             $romfile = 'pxe-pcnet.rom';
1250         } elsif ($device eq 'rtl8139') {
1251             $romfile = 'pxe-rtl8139.rom';
1252         }
1253         $tmpstr .= ",romfile=$romfile" if $romfile;
1254     }
1255
1256     return $tmpstr;
1257 }
1258
1259 sub print_netdev_full {
1260     my ($vmid, $conf, $net, $netid) = @_;
1261
1262     my $i = '';
1263     if ($netid =~ m/^net(\d+)$/) {
1264         $i = int($1);
1265     }
1266
1267     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1268
1269     my $ifname = "tap${vmid}i$i";
1270
1271     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1272     die "interface name '$ifname' is too long (max 15 character)\n"
1273         if length($ifname) >= 16;
1274
1275     my $vhostparam = '';
1276     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1277
1278     my $vmname = $conf->{name} || "vm$vmid";
1279
1280     my $netdev = "";
1281
1282     if ($net->{bridge}) {
1283         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/pve-bridge,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1284     } else {
1285         $netdev = "type=user,id=$netid,hostname=$vmname";
1286     }
1287
1288     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1289
1290     return $netdev;
1291 }
1292
1293 sub drive_is_cdrom {
1294     my ($drive) = @_;
1295
1296     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1297
1298 }
1299
1300 sub parse_numa {
1301     my ($data) = @_;
1302
1303     my $res = {};
1304
1305     foreach my $kvp (split(/,/, $data)) {
1306
1307         if ($kvp =~ m/^memory=(\S+)$/) {
1308             $res->{memory} = $1;
1309         } elsif ($kvp =~ m/^policy=(preferred|bind|interleave)$/) {
1310             $res->{policy} = $1;
1311         } elsif ($kvp =~ m/^cpus=(\d+)(-(\d+))?$/) {
1312             $res->{cpus}->{start} = $1;
1313             $res->{cpus}->{end} = $3;
1314         } elsif ($kvp =~ m/^hostnodes=(\d+)(-(\d+))?$/) {
1315             $res->{hostnodes}->{start} = $1;
1316             $res->{hostnodes}->{end} = $3;
1317         } else {
1318             return undef;
1319         }
1320     }
1321
1322     return $res;
1323 }
1324
1325 sub parse_hostpci {
1326     my ($value) = @_;
1327
1328     return undef if !$value;
1329
1330
1331     my @list = split(/,/, $value);
1332     my $found;
1333
1334     my $res = {};
1335     foreach my $kv (@list) {
1336
1337         if ($kv =~ m/^(host=)?([a-f0-9]{2}:[a-f0-9]{2})(\.([a-f0-9]))?$/) {
1338             $found = 1;
1339             if(defined($4)){
1340                 push @{$res->{pciid}}, { id => $2 , function => $4};
1341
1342             }else{
1343                 my $pcidevices = lspci($2);
1344                 $res->{pciid} = $pcidevices->{$2};
1345             }
1346         } elsif ($kv =~ m/^rombar=(on|off)$/) {
1347             $res->{rombar} = $1;
1348         } elsif ($kv =~ m/^x-vga=(on|off)$/) {
1349             $res->{'x-vga'} = $1;
1350         } elsif ($kv =~ m/^pcie=(\d+)$/) {
1351             $res->{pcie} = 1 if $1 == 1;
1352         } else {
1353             warn "unknown hostpci setting '$kv'\n";
1354         }
1355     }
1356
1357     return undef if !$found;
1358
1359     return $res;
1360 }
1361
1362 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1363 sub parse_net {
1364     my ($data) = @_;
1365
1366     my $res = {};
1367
1368     foreach my $kvp (split(/,/, $data)) {
1369
1370         if ($kvp =~ m/^(ne2k_pci|e1000|e1000-82540em|e1000-82544gc|e1000-82545em|rtl8139|pcnet|virtio|ne2k_isa|i82551|i82557b|i82559er|vmxnet3)(=([0-9a-f]{2}(:[0-9a-f]{2}){5}))?$/i) {
1371             my $model = lc($1);
1372             my $mac = defined($3) ? uc($3) : PVE::Tools::random_ether_addr();
1373             $res->{model} = $model;
1374             $res->{macaddr} = $mac;
1375         } elsif ($kvp =~ m/^bridge=(\S+)$/) {
1376             $res->{bridge} = $1;
1377         } elsif ($kvp =~ m/^queues=(\d+)$/) {
1378             $res->{queues} = $1;
1379         } elsif ($kvp =~ m/^rate=(\d+(\.\d+)?)$/) {
1380             $res->{rate} = $1;
1381         } elsif ($kvp =~ m/^tag=(\d+)$/) {
1382             $res->{tag} = $1;
1383         } elsif ($kvp =~ m/^firewall=([01])$/) {
1384             $res->{firewall} = $1;
1385         } elsif ($kvp =~ m/^link_down=([01])$/) {
1386             $res->{link_down} = $1;
1387         } else {
1388             return undef;
1389         }
1390
1391     }
1392
1393     return undef if !$res->{model};
1394
1395     return $res;
1396 }
1397
1398 sub print_net {
1399     my $net = shift;
1400
1401     my $res = "$net->{model}";
1402     $res .= "=$net->{macaddr}" if $net->{macaddr};
1403     $res .= ",bridge=$net->{bridge}" if $net->{bridge};
1404     $res .= ",rate=$net->{rate}" if $net->{rate};
1405     $res .= ",tag=$net->{tag}" if $net->{tag};
1406     $res .= ",firewall=1" if $net->{firewall};
1407     $res .= ",link_down=1" if $net->{link_down};
1408     $res .= ",queues=$net->{queues}" if $net->{queues};
1409
1410     return $res;
1411 }
1412
1413 sub add_random_macs {
1414     my ($settings) = @_;
1415
1416     foreach my $opt (keys %$settings) {
1417         next if $opt !~ m/^net(\d+)$/;
1418         my $net = parse_net($settings->{$opt});
1419         next if !$net;
1420         $settings->{$opt} = print_net($net);
1421     }
1422 }
1423
1424 sub add_unused_volume {
1425     my ($config, $volid) = @_;
1426
1427     my $key;
1428     for (my $ind = $MAX_UNUSED_DISKS - 1; $ind >= 0; $ind--) {
1429         my $test = "unused$ind";
1430         if (my $vid = $config->{$test}) {
1431             return if $vid eq $volid; # do not add duplicates
1432         } else {
1433             $key = $test;
1434         }
1435     }
1436
1437     die "To many unused volume - please delete them first.\n" if !$key;
1438
1439     $config->{$key} = $volid;
1440
1441     return $key;
1442 }
1443
1444 sub vm_is_volid_owner {
1445     my ($storecfg, $vmid, $volid) = @_;
1446
1447     if ($volid !~  m|^/|) {
1448         my ($path, $owner);
1449         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1450         if ($owner && ($owner == $vmid)) {
1451             return 1;
1452         }
1453     }
1454
1455     return undef;
1456 }
1457
1458 sub split_flagged_list {
1459     my $text = shift || '';
1460     $text =~ s/[,;]/ /g;
1461     $text =~ s/^\s+//;
1462     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
1463 }
1464
1465 sub join_flagged_list {
1466     my ($how, $lst) = @_;
1467     join $how, map { $lst->{$_} . $_ } keys %$lst;
1468 }
1469
1470 sub vmconfig_delete_pending_option {
1471     my ($conf, $key, $force) = @_;
1472
1473     delete $conf->{pending}->{$key};
1474     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1475     $pending_delete_hash->{$key} = $force ? '!' : '';
1476     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1477 }
1478
1479 sub vmconfig_undelete_pending_option {
1480     my ($conf, $key) = @_;
1481
1482     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1483     delete $pending_delete_hash->{$key};
1484
1485     if (%$pending_delete_hash) {
1486         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1487     } else {
1488         delete $conf->{pending}->{delete};
1489     }
1490 }
1491
1492 sub vmconfig_register_unused_drive {
1493     my ($storecfg, $vmid, $conf, $drive) = @_;
1494
1495     if (!drive_is_cdrom($drive)) {
1496         my $volid = $drive->{file};
1497         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1498             add_unused_volume($conf, $volid, $vmid);
1499         }
1500     }
1501 }
1502
1503 sub vmconfig_cleanup_pending {
1504     my ($conf) = @_;
1505
1506     # remove pending changes when nothing changed
1507     my $changes;
1508     foreach my $opt (keys %{$conf->{pending}}) {
1509         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
1510             $changes = 1;
1511             delete $conf->{pending}->{$opt};
1512         }
1513     }
1514
1515     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
1516     my $pending_delete_hash = {};
1517     while (my ($opt, $force) = each %$current_delete_hash) {
1518         if (defined($conf->{$opt})) {
1519             $pending_delete_hash->{$opt} = $force;
1520         } else {
1521             $changes = 1;
1522         }
1523     }
1524
1525     if (%$pending_delete_hash) {
1526         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1527     } else {
1528         delete $conf->{pending}->{delete};
1529     }
1530
1531     return $changes;
1532 }
1533
1534 my $valid_smbios1_options = {
1535     manufacturer => '\S+',
1536     product => '\S+',
1537     version => '\S+',
1538     serial => '\S+',
1539     uuid => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1540     sku => '\S+',
1541     family => '\S+',
1542 };
1543
1544 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
1545 sub parse_smbios1 {
1546     my ($data) = @_;
1547
1548     my $res = {};
1549
1550     foreach my $kvp (split(/,/, $data)) {
1551         return undef if $kvp !~ m/^(\S+)=(.+)$/;
1552         my ($k, $v) = split(/=/, $kvp);
1553         return undef if !defined($k) || !defined($v);
1554         return undef if !$valid_smbios1_options->{$k};
1555         return undef if $v !~ m/^$valid_smbios1_options->{$k}$/;
1556         $res->{$k} = $v;
1557     }
1558
1559     return $res;
1560 }
1561
1562 sub print_smbios1 {
1563     my ($smbios1) = @_;
1564
1565     my $data = '';
1566     foreach my $k (keys %$smbios1) {
1567         next if !defined($smbios1->{$k});
1568         next if !$valid_smbios1_options->{$k};
1569         $data .= ',' if $data;
1570         $data .= "$k=$smbios1->{$k}";
1571     }
1572     return $data;
1573 }
1574
1575 PVE::JSONSchema::register_format('pve-qm-smbios1', \&verify_smbios1);
1576 sub verify_smbios1 {
1577     my ($value, $noerr) = @_;
1578
1579     return $value if parse_smbios1($value);
1580
1581     return undef if $noerr;
1582
1583     die "unable to parse smbios (type 1) options\n";
1584 }
1585
1586 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
1587 sub verify_bootdisk {
1588     my ($value, $noerr) = @_;
1589
1590     return $value if valid_drivename($value);
1591
1592     return undef if $noerr;
1593
1594     die "invalid boot disk '$value'\n";
1595 }
1596
1597 PVE::JSONSchema::register_format('pve-qm-numanode', \&verify_numa);
1598 sub verify_numa {
1599     my ($value, $noerr) = @_;
1600
1601     return $value if parse_numa($value);
1602
1603     return undef if $noerr;
1604
1605     die "unable to parse numa options\n";
1606 }
1607
1608 PVE::JSONSchema::register_format('pve-qm-net', \&verify_net);
1609 sub verify_net {
1610     my ($value, $noerr) = @_;
1611
1612     return $value if parse_net($value);
1613
1614     return undef if $noerr;
1615
1616     die "unable to parse network options\n";
1617 }
1618
1619 PVE::JSONSchema::register_format('pve-qm-drive', \&verify_drive);
1620 sub verify_drive {
1621     my ($value, $noerr) = @_;
1622
1623     return $value if parse_drive(undef, $value);
1624
1625     return undef if $noerr;
1626
1627     die "unable to parse drive options\n";
1628 }
1629
1630 PVE::JSONSchema::register_format('pve-qm-hostpci', \&verify_hostpci);
1631 sub verify_hostpci {
1632     my ($value, $noerr) = @_;
1633
1634     return $value if parse_hostpci($value);
1635
1636     return undef if $noerr;
1637
1638     die "unable to parse pci id\n";
1639 }
1640
1641 PVE::JSONSchema::register_format('pve-qm-watchdog', \&verify_watchdog);
1642 sub verify_watchdog {
1643     my ($value, $noerr) = @_;
1644
1645     return $value if parse_watchdog($value);
1646
1647     return undef if $noerr;
1648
1649     die "unable to parse watchdog options\n";
1650 }
1651
1652 sub parse_watchdog {
1653     my ($value) = @_;
1654
1655     return undef if !$value;
1656
1657     my $res = {};
1658
1659     foreach my $p (split(/,/, $value)) {
1660         next if $p =~ m/^\s*$/;
1661
1662         if ($p =~ m/^(model=)?(i6300esb|ib700)$/) {
1663             $res->{model} = $2;
1664         } elsif ($p =~ m/^(action=)?(reset|shutdown|poweroff|pause|debug|none)$/) {
1665             $res->{action} = $2;
1666         } else {
1667             return undef;
1668         }
1669     }
1670
1671     return $res;
1672 }
1673
1674 sub parse_usb_device {
1675     my ($value) = @_;
1676
1677     return undef if !$value;
1678
1679     my @dl = split(/,/, $value);
1680     my $found;
1681
1682     my $res = {};
1683     foreach my $v (@dl) {
1684         if ($v =~ m/^host=(0x)?([0-9A-Fa-f]{4}):(0x)?([0-9A-Fa-f]{4})$/) {
1685             $found = 1;
1686             $res->{vendorid} = $2;
1687             $res->{productid} = $4;
1688         } elsif ($v =~ m/^host=(\d+)\-(\d+(\.\d+)*)$/) {
1689             $found = 1;
1690             $res->{hostbus} = $1;
1691             $res->{hostport} = $2;
1692         } elsif ($v =~ m/^spice$/) {
1693             $found = 1;
1694             $res->{spice} = 1;
1695         } else {
1696             return undef;
1697         }
1698     }
1699     return undef if !$found;
1700
1701     return $res;
1702 }
1703
1704 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
1705 sub verify_usb_device {
1706     my ($value, $noerr) = @_;
1707
1708     return $value if parse_usb_device($value);
1709
1710     return undef if $noerr;
1711
1712     die "unable to parse usb device\n";
1713 }
1714
1715 # add JSON properties for create and set function
1716 sub json_config_properties {
1717     my $prop = shift;
1718
1719     foreach my $opt (keys %$confdesc) {
1720         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
1721         $prop->{$opt} = $confdesc->{$opt};
1722     }
1723
1724     return $prop;
1725 }
1726
1727 sub check_type {
1728     my ($key, $value) = @_;
1729
1730     die "unknown setting '$key'\n" if !$confdesc->{$key};
1731
1732     my $type = $confdesc->{$key}->{type};
1733
1734     if (!defined($value)) {
1735         die "got undefined value\n";
1736     }
1737
1738     if ($value =~ m/[\n\r]/) {
1739         die "property contains a line feed\n";
1740     }
1741
1742     if ($type eq 'boolean') {
1743         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
1744         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
1745         die "type check ('boolean') failed - got '$value'\n";
1746     } elsif ($type eq 'integer') {
1747         return int($1) if $value =~ m/^(\d+)$/;
1748         die "type check ('integer') failed - got '$value'\n";
1749     } elsif ($type eq 'number') {
1750         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
1751         die "type check ('number') failed - got '$value'\n";
1752     } elsif ($type eq 'string') {
1753         if (my $fmt = $confdesc->{$key}->{format}) {
1754             if ($fmt eq 'pve-qm-drive') {
1755                 # special case - we need to pass $key to parse_drive()
1756                 my $drive = parse_drive($key, $value);
1757                 return $value if $drive;
1758                 die "unable to parse drive options\n";
1759             }
1760             PVE::JSONSchema::check_format($fmt, $value);
1761             return $value;
1762         }
1763         $value =~ s/^\"(.*)\"$/$1/;
1764         return $value;
1765     } else {
1766         die "internal error"
1767     }
1768 }
1769
1770 sub lock_config_full {
1771     my ($vmid, $timeout, $code, @param) = @_;
1772
1773     my $filename = config_file_lock($vmid);
1774
1775     my $res = lock_file($filename, $timeout, $code, @param);
1776
1777     die $@ if $@;
1778
1779     return $res;
1780 }
1781
1782 sub lock_config_mode {
1783     my ($vmid, $timeout, $shared, $code, @param) = @_;
1784
1785     my $filename = config_file_lock($vmid);
1786
1787     my $res = lock_file_full($filename, $timeout, $shared, $code, @param);
1788
1789     die $@ if $@;
1790
1791     return $res;
1792 }
1793
1794 sub lock_config {
1795     my ($vmid, $code, @param) = @_;
1796
1797     return lock_config_full($vmid, 10, $code, @param);
1798 }
1799
1800 sub cfs_config_path {
1801     my ($vmid, $node) = @_;
1802
1803     $node = $nodename if !$node;
1804     return "nodes/$node/qemu-server/$vmid.conf";
1805 }
1806
1807 sub check_iommu_support{
1808     #fixme : need to check IOMMU support
1809     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
1810
1811     my $iommu=1;
1812     return $iommu;
1813
1814 }
1815
1816 sub config_file {
1817     my ($vmid, $node) = @_;
1818
1819     my $cfspath = cfs_config_path($vmid, $node);
1820     return "/etc/pve/$cfspath";
1821 }
1822
1823 sub config_file_lock {
1824     my ($vmid) = @_;
1825
1826     return "$lock_dir/lock-$vmid.conf";
1827 }
1828
1829 sub touch_config {
1830     my ($vmid) = @_;
1831
1832     my $conf = config_file($vmid);
1833     utime undef, undef, $conf;
1834 }
1835
1836 sub destroy_vm {
1837     my ($storecfg, $vmid, $keep_empty_config) = @_;
1838
1839     my $conffile = config_file($vmid);
1840
1841     my $conf = load_config($vmid);
1842
1843     check_lock($conf);
1844
1845     # only remove disks owned by this VM
1846     foreach_drive($conf, sub {
1847         my ($ds, $drive) = @_;
1848
1849         return if drive_is_cdrom($drive);
1850
1851         my $volid = $drive->{file};
1852
1853         return if !$volid || $volid =~ m|^/|;
1854
1855         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
1856         return if !$path || !$owner || ($owner != $vmid);
1857
1858         PVE::Storage::vdisk_free($storecfg, $volid);
1859     });
1860
1861     if ($keep_empty_config) {
1862         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
1863     } else {
1864         unlink $conffile;
1865     }
1866
1867     # also remove unused disk
1868     eval {
1869         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
1870
1871         eval {
1872             PVE::Storage::foreach_volid($dl, sub {
1873                 my ($volid, $sid, $volname, $d) = @_;
1874                 PVE::Storage::vdisk_free($storecfg, $volid);
1875             });
1876         };
1877         warn $@ if $@;
1878
1879     };
1880     warn $@ if $@;
1881 }
1882
1883 sub load_config {
1884     my ($vmid, $node) = @_;
1885
1886     my $cfspath = cfs_config_path($vmid, $node);
1887
1888     my $conf = PVE::Cluster::cfs_read_file($cfspath);
1889
1890     die "no such VM ('$vmid')\n" if !defined($conf);
1891
1892     return $conf;
1893 }
1894
1895 sub parse_vm_config {
1896     my ($filename, $raw) = @_;
1897
1898     return undef if !defined($raw);
1899
1900     my $res = {
1901         digest => Digest::SHA::sha1_hex($raw),
1902         snapshots => {},
1903         pending => {},
1904     };
1905
1906     $filename =~ m|/qemu-server/(\d+)\.conf$|
1907         || die "got strange filename '$filename'";
1908
1909     my $vmid = $1;
1910
1911     my $conf = $res;
1912     my $descr;
1913     my $section = '';
1914
1915     my @lines = split(/\n/, $raw);
1916     foreach my $line (@lines) {
1917         next if $line =~ m/^\s*$/;
1918
1919         if ($line =~ m/^\[PENDING\]\s*$/i) {
1920             $section = 'pending';
1921             if (defined($descr)) {
1922                 $descr =~ s/\s+$//;
1923                 $conf->{description} = $descr;
1924             }
1925             $descr = undef;
1926             $conf = $res->{$section} = {};
1927             next;
1928
1929         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
1930             $section = $1;
1931             if (defined($descr)) {
1932                 $descr =~ s/\s+$//;
1933                 $conf->{description} = $descr;
1934             }
1935             $descr = undef;
1936             $conf = $res->{snapshots}->{$section} = {};
1937             next;
1938         }
1939
1940         if ($line =~ m/^\#(.*)\s*$/) {
1941             $descr = '' if !defined($descr);
1942             $descr .= PVE::Tools::decode_text($1) . "\n";
1943             next;
1944         }
1945
1946         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
1947             $descr = '' if !defined($descr);
1948             $descr .= PVE::Tools::decode_text($2);
1949         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
1950             $conf->{snapstate} = $1;
1951         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
1952             my $key = $1;
1953             my $value = $2;
1954             $conf->{$key} = $value;
1955         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
1956             my $value = $1;
1957             if ($section eq 'pending') {
1958                 $conf->{delete} = $value; # we parse this later
1959             } else {
1960                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
1961             }
1962         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
1963             my $key = $1;
1964             my $value = $2;
1965             eval { $value = check_type($key, $value); };
1966             if ($@) {
1967                 warn "vm $vmid - unable to parse value of '$key' - $@";
1968             } else {
1969                 my $fmt = $confdesc->{$key}->{format};
1970                 if ($fmt && $fmt eq 'pve-qm-drive') {
1971                     my $v = parse_drive($key, $value);
1972                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
1973                         $v->{file} = $volid;
1974                         $value = print_drive($vmid, $v);
1975                     } else {
1976                         warn "vm $vmid - unable to parse value of '$key'\n";
1977                         next;
1978                     }
1979                 }
1980
1981                 if ($key eq 'cdrom') {
1982                     $conf->{ide2} = $value;
1983                 } else {
1984                     $conf->{$key} = $value;
1985                 }
1986             }
1987         }
1988     }
1989
1990     if (defined($descr)) {
1991         $descr =~ s/\s+$//;
1992         $conf->{description} = $descr;
1993     }
1994     delete $res->{snapstate}; # just to be sure
1995
1996     return $res;
1997 }
1998
1999 sub write_vm_config {
2000     my ($filename, $conf) = @_;
2001
2002     delete $conf->{snapstate}; # just to be sure
2003
2004     if ($conf->{cdrom}) {
2005         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2006         $conf->{ide2} = $conf->{cdrom};
2007         delete $conf->{cdrom};
2008     }
2009
2010     # we do not use 'smp' any longer
2011     if ($conf->{sockets}) {
2012         delete $conf->{smp};
2013     } elsif ($conf->{smp}) {
2014         $conf->{sockets} = $conf->{smp};
2015         delete $conf->{cores};
2016         delete $conf->{smp};
2017     }
2018
2019     my $used_volids = {};
2020
2021     my $cleanup_config = sub {
2022         my ($cref, $pending, $snapname) = @_;
2023
2024         foreach my $key (keys %$cref) {
2025             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2026                 $key eq 'snapstate' || $key eq 'pending';
2027             my $value = $cref->{$key};
2028             if ($key eq 'delete') {
2029                 die "propertry 'delete' is only allowed in [PENDING]\n"
2030                     if !$pending;
2031                 # fixme: check syntax?
2032                 next;
2033             }
2034             eval { $value = check_type($key, $value); };
2035             die "unable to parse value of '$key' - $@" if $@;
2036
2037             $cref->{$key} = $value;
2038
2039             if (!$snapname && valid_drivename($key)) {
2040                 my $drive = parse_drive($key, $value);
2041                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2042             }
2043         }
2044     };
2045
2046     &$cleanup_config($conf);
2047
2048     &$cleanup_config($conf->{pending}, 1);
2049
2050     foreach my $snapname (keys %{$conf->{snapshots}}) {
2051         die "internal error" if $snapname eq 'pending';
2052         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2053     }
2054
2055     # remove 'unusedX' settings if we re-add a volume
2056     foreach my $key (keys %$conf) {
2057         my $value = $conf->{$key};
2058         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2059             delete $conf->{$key};
2060         }
2061     }
2062
2063     my $generate_raw_config = sub {
2064         my ($conf, $pending) = @_;
2065
2066         my $raw = '';
2067
2068         # add description as comment to top of file
2069         if (defined(my $descr = $conf->{description})) {
2070             if ($descr) {
2071                 foreach my $cl (split(/\n/, $descr)) {
2072                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2073                 }
2074             } else {
2075                 $raw .= "#\n" if $pending;
2076             }
2077         }
2078
2079         foreach my $key (sort keys %$conf) {
2080             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2081             $raw .= "$key: $conf->{$key}\n";
2082         }
2083         return $raw;
2084     };
2085
2086     my $raw = &$generate_raw_config($conf);
2087
2088     if (scalar(keys %{$conf->{pending}})){
2089         $raw .= "\n[PENDING]\n";
2090         $raw .= &$generate_raw_config($conf->{pending}, 1);
2091     }
2092
2093     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2094         $raw .= "\n[$snapname]\n";
2095         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2096     }
2097
2098     return $raw;
2099 }
2100
2101 sub update_config_nolock {
2102     my ($vmid, $conf, $skiplock) = @_;
2103
2104     check_lock($conf) if !$skiplock;
2105
2106     my $cfspath = cfs_config_path($vmid);
2107
2108     PVE::Cluster::cfs_write_file($cfspath, $conf);
2109 }
2110
2111 sub update_config {
2112     my ($vmid, $conf, $skiplock) = @_;
2113
2114     lock_config($vmid, &update_config_nolock, $conf, $skiplock);
2115 }
2116
2117 sub load_defaults {
2118
2119     my $res = {};
2120
2121     # we use static defaults from our JSON schema configuration
2122     foreach my $key (keys %$confdesc) {
2123         if (defined(my $default = $confdesc->{$key}->{default})) {
2124             $res->{$key} = $default;
2125         }
2126     }
2127
2128     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2129     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2130
2131     return $res;
2132 }
2133
2134 sub config_list {
2135     my $vmlist = PVE::Cluster::get_vmlist();
2136     my $res = {};
2137     return $res if !$vmlist || !$vmlist->{ids};
2138     my $ids = $vmlist->{ids};
2139
2140     foreach my $vmid (keys %$ids) {
2141         my $d = $ids->{$vmid};
2142         next if !$d->{node} || $d->{node} ne $nodename;
2143         next if !$d->{type} || $d->{type} ne 'qemu';
2144         $res->{$vmid}->{exists} = 1;
2145     }
2146     return $res;
2147 }
2148
2149 # test if VM uses local resources (to prevent migration)
2150 sub check_local_resources {
2151     my ($conf, $noerr) = @_;
2152
2153     my $loc_res = 0;
2154
2155     $loc_res = 1 if $conf->{hostusb}; # old syntax
2156     $loc_res = 1 if $conf->{hostpci}; # old syntax
2157
2158     foreach my $k (keys %$conf) {
2159         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2160         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2161     }
2162
2163     die "VM uses local resources\n" if $loc_res && !$noerr;
2164
2165     return $loc_res;
2166 }
2167
2168 # check if used storages are available on all nodes (use by migrate)
2169 sub check_storage_availability {
2170     my ($storecfg, $conf, $node) = @_;
2171
2172     foreach_drive($conf, sub {
2173         my ($ds, $drive) = @_;
2174
2175         my $volid = $drive->{file};
2176         return if !$volid;
2177
2178         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2179         return if !$sid;
2180
2181         # check if storage is available on both nodes
2182         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2183         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2184    });
2185 }
2186
2187 # list nodes where all VM images are available (used by has_feature API)
2188 sub shared_nodes {
2189     my ($conf, $storecfg) = @_;
2190
2191     my $nodelist = PVE::Cluster::get_nodelist();
2192     my $nodehash = { map { $_ => 1 } @$nodelist };
2193     my $nodename = PVE::INotify::nodename();
2194
2195     foreach_drive($conf, sub {
2196         my ($ds, $drive) = @_;
2197
2198         my $volid = $drive->{file};
2199         return if !$volid;
2200
2201         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2202         if ($storeid) {
2203             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2204             if ($scfg->{disable}) {
2205                 $nodehash = {};
2206             } elsif (my $avail = $scfg->{nodes}) {
2207                 foreach my $node (keys %$nodehash) {
2208                     delete $nodehash->{$node} if !$avail->{$node};
2209                 }
2210             } elsif (!$scfg->{shared}) {
2211                 foreach my $node (keys %$nodehash) {
2212                     delete $nodehash->{$node} if $node ne $nodename
2213                 }
2214             }
2215         }
2216     });
2217
2218     return $nodehash
2219 }
2220
2221 sub check_lock {
2222     my ($conf) = @_;
2223
2224     die "VM is locked ($conf->{lock})\n" if $conf->{lock};
2225 }
2226
2227 sub check_cmdline {
2228     my ($pidfile, $pid) = @_;
2229
2230     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2231     if (defined($fh)) {
2232         my $line = <$fh>;
2233         $fh->close;
2234         return undef if !$line;
2235         my @param = split(/\0/, $line);
2236
2237         my $cmd = $param[0];
2238         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2239
2240         for (my $i = 0; $i < scalar (@param); $i++) {
2241             my $p = $param[$i];
2242             next if !$p;
2243             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2244                 my $p = $param[$i+1];
2245                 return 1 if $p && ($p eq $pidfile);
2246                 return undef;
2247             }
2248         }
2249     }
2250     return undef;
2251 }
2252
2253 sub check_running {
2254     my ($vmid, $nocheck, $node) = @_;
2255
2256     my $filename = config_file($vmid, $node);
2257
2258     die "unable to find configuration file for VM $vmid - no such machine\n"
2259         if !$nocheck && ! -f $filename;
2260
2261     my $pidfile = pidfile_name($vmid);
2262
2263     if (my $fd = IO::File->new("<$pidfile")) {
2264         my $st = stat($fd);
2265         my $line = <$fd>;
2266         close($fd);
2267
2268         my $mtime = $st->mtime;
2269         if ($mtime > time()) {
2270             warn "file '$filename' modified in future\n";
2271         }
2272
2273         if ($line =~ m/^(\d+)$/) {
2274             my $pid = $1;
2275             if (check_cmdline($pidfile, $pid)) {
2276                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2277                     return $pid;
2278                 }
2279             }
2280         }
2281     }
2282
2283     return undef;
2284 }
2285
2286 sub vzlist {
2287
2288     my $vzlist = config_list();
2289
2290     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2291
2292     while (defined(my $de = $fd->read)) {
2293         next if $de !~ m/^(\d+)\.pid$/;
2294         my $vmid = $1;
2295         next if !defined($vzlist->{$vmid});
2296         if (my $pid = check_running($vmid)) {
2297             $vzlist->{$vmid}->{pid} = $pid;
2298         }
2299     }
2300
2301     return $vzlist;
2302 }
2303
2304 sub disksize {
2305     my ($storecfg, $conf) = @_;
2306
2307     my $bootdisk = $conf->{bootdisk};
2308     return undef if !$bootdisk;
2309     return undef if !valid_drivename($bootdisk);
2310
2311     return undef if !$conf->{$bootdisk};
2312
2313     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2314     return undef if !defined($drive);
2315
2316     return undef if drive_is_cdrom($drive);
2317
2318     my $volid = $drive->{file};
2319     return undef if !$volid;
2320
2321     return $drive->{size};
2322 }
2323
2324 my $last_proc_pid_stat;
2325
2326 # get VM status information
2327 # This must be fast and should not block ($full == false)
2328 # We only query KVM using QMP if $full == true (this can be slow)
2329 sub vmstatus {
2330     my ($opt_vmid, $full) = @_;
2331
2332     my $res = {};
2333
2334     my $storecfg = PVE::Storage::config();
2335
2336     my $list = vzlist();
2337     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2338
2339     my $cpucount = $cpuinfo->{cpus} || 1;
2340
2341     foreach my $vmid (keys %$list) {
2342         next if $opt_vmid && ($vmid ne $opt_vmid);
2343
2344         my $cfspath = cfs_config_path($vmid);
2345         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2346
2347         my $d = {};
2348         $d->{pid} = $list->{$vmid}->{pid};
2349
2350         # fixme: better status?
2351         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2352
2353         my $size = disksize($storecfg, $conf);
2354         if (defined($size)) {
2355             $d->{disk} = 0; # no info available
2356             $d->{maxdisk} = $size;
2357         } else {
2358             $d->{disk} = 0;
2359             $d->{maxdisk} = 0;
2360         }
2361
2362         $d->{cpus} = ($conf->{sockets} || 1) * ($conf->{cores} || 1);
2363         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2364         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2365
2366         $d->{name} = $conf->{name} || "VM $vmid";
2367         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024) : 0;
2368
2369         if ($conf->{balloon}) {
2370             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2371             $d->{shares} = defined($conf->{shares}) ? $conf->{shares} : 1000;
2372         }
2373
2374         $d->{uptime} = 0;
2375         $d->{cpu} = 0;
2376         $d->{mem} = 0;
2377
2378         $d->{netout} = 0;
2379         $d->{netin} = 0;
2380
2381         $d->{diskread} = 0;
2382         $d->{diskwrite} = 0;
2383
2384         $d->{template} = is_template($conf);
2385
2386         $res->{$vmid} = $d;
2387     }
2388
2389     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2390     foreach my $dev (keys %$netdev) {
2391         next if $dev !~ m/^tap([1-9]\d*)i/;
2392         my $vmid = $1;
2393         my $d = $res->{$vmid};
2394         next if !$d;
2395
2396         $d->{netout} += $netdev->{$dev}->{receive};
2397         $d->{netin} += $netdev->{$dev}->{transmit};
2398
2399         if ($full) {
2400             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2401             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2402         }
2403
2404     }
2405
2406     my $ctime = gettimeofday;
2407
2408     foreach my $vmid (keys %$list) {
2409
2410         my $d = $res->{$vmid};
2411         my $pid = $d->{pid};
2412         next if !$pid;
2413
2414         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2415         next if !$pstat; # not running
2416
2417         my $used = $pstat->{utime} + $pstat->{stime};
2418
2419         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2420
2421         if ($pstat->{vsize}) {
2422             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2423         }
2424
2425         my $old = $last_proc_pid_stat->{$pid};
2426         if (!$old) {
2427             $last_proc_pid_stat->{$pid} = {
2428                 time => $ctime,
2429                 used => $used,
2430                 cpu => 0,
2431             };
2432             next;
2433         }
2434
2435         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2436
2437         if ($dtime > 1000) {
2438             my $dutime = $used -  $old->{used};
2439
2440             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2441             $last_proc_pid_stat->{$pid} = {
2442                 time => $ctime,
2443                 used => $used,
2444                 cpu => $d->{cpu},
2445             };
2446         } else {
2447             $d->{cpu} = $old->{cpu};
2448         }
2449     }
2450
2451     return $res if !$full;
2452
2453     my $qmpclient = PVE::QMPClient->new();
2454
2455     my $ballooncb = sub {
2456         my ($vmid, $resp) = @_;
2457
2458         my $info = $resp->{'return'};
2459         return if !$info->{max_mem};
2460
2461         my $d = $res->{$vmid};
2462
2463         # use memory assigned to VM
2464         $d->{maxmem} = $info->{max_mem};
2465         $d->{balloon} = $info->{actual};
2466
2467         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2468             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2469             $d->{freemem} = $info->{free_mem};
2470         }
2471
2472         $d->{ballooninfo} = $info;
2473     };
2474
2475     my $blockstatscb = sub {
2476         my ($vmid, $resp) = @_;
2477         my $data = $resp->{'return'} || [];
2478         my $totalrdbytes = 0;
2479         my $totalwrbytes = 0;
2480
2481         for my $blockstat (@$data) {
2482             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2483             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2484
2485             $blockstat->{device} =~ s/drive-//;
2486             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2487         }
2488         $res->{$vmid}->{diskread} = $totalrdbytes;
2489         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2490     };
2491
2492     my $statuscb = sub {
2493         my ($vmid, $resp) = @_;
2494
2495         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2496         # this fails if ballon driver is not loaded, so this must be
2497         # the last commnand (following command are aborted if this fails).
2498         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2499
2500         my $status = 'unknown';
2501         if (!defined($status = $resp->{'return'}->{status})) {
2502             warn "unable to get VM status\n";
2503             return;
2504         }
2505
2506         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2507     };
2508
2509     foreach my $vmid (keys %$list) {
2510         next if $opt_vmid && ($vmid ne $opt_vmid);
2511         next if !$res->{$vmid}->{pid}; # not running
2512         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2513     }
2514
2515     $qmpclient->queue_execute(undef, 1);
2516
2517     foreach my $vmid (keys %$list) {
2518         next if $opt_vmid && ($vmid ne $opt_vmid);
2519         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2520     }
2521
2522     return $res;
2523 }
2524
2525 sub foreach_dimm {
2526     my ($conf, $vmid, $memory, $sockets, $func) = @_;
2527
2528     my $dimm_id = 0;
2529     my $current_size = 1024;
2530     my $dimm_size = 512;
2531     return if $current_size == $memory;
2532
2533     for (my $j = 0; $j < 8; $j++) {
2534         for (my $i = 0; $i < 32; $i++) {
2535             my $name = "dimm${dimm_id}";
2536             $dimm_id++;
2537             my $numanode = $i % $sockets;
2538             $current_size += $dimm_size;
2539             &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
2540             return  $current_size if $current_size >= $memory;
2541         }
2542         $dimm_size *= 2;
2543     }
2544 }
2545
2546 sub foreach_reverse_dimm {
2547     my ($conf, $vmid, $memory, $sockets, $func) = @_;
2548
2549     my $dimm_id = 253;
2550     my $current_size = 4177920;
2551     my $dimm_size = 65536;
2552     return if $current_size == $memory;
2553
2554     for (my $j = 0; $j < 8; $j++) {
2555         for (my $i = 0; $i < 32; $i++) {
2556             my $name = "dimm${dimm_id}";
2557             $dimm_id--;
2558             my $numanode = $i % $sockets;
2559             $current_size -= $dimm_size;
2560             &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
2561             return  $current_size if $current_size <= $memory;
2562         }
2563         $dimm_size /= 2;
2564     }
2565 }
2566
2567 sub foreach_drive {
2568     my ($conf, $func) = @_;
2569
2570     foreach my $ds (keys %$conf) {
2571         next if !valid_drivename($ds);
2572
2573         my $drive = parse_drive($ds, $conf->{$ds});
2574         next if !$drive;
2575
2576         &$func($ds, $drive);
2577     }
2578 }
2579
2580 sub foreach_volid {
2581     my ($conf, $func) = @_;
2582
2583     my $volhash = {};
2584
2585     my $test_volid = sub {
2586         my ($volid, $is_cdrom) = @_;
2587
2588         return if !$volid;
2589
2590         $volhash->{$volid} = $is_cdrom || 0;
2591     };
2592
2593     foreach_drive($conf, sub {
2594         my ($ds, $drive) = @_;
2595         &$test_volid($drive->{file}, drive_is_cdrom($drive));
2596     });
2597
2598     foreach my $snapname (keys %{$conf->{snapshots}}) {
2599         my $snap = $conf->{snapshots}->{$snapname};
2600         &$test_volid($snap->{vmstate}, 0);
2601         foreach_drive($snap, sub {
2602             my ($ds, $drive) = @_;
2603             &$test_volid($drive->{file}, drive_is_cdrom($drive));
2604         });
2605     }
2606
2607     foreach my $volid (keys %$volhash) {
2608         &$func($volid, $volhash->{$volid});
2609     }
2610 }
2611
2612 sub vga_conf_has_spice {
2613     my ($vga) = @_;
2614
2615     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
2616
2617     return $1 || 1;
2618 }
2619
2620 sub config_to_command {
2621     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2622
2623     my $cmd = [];
2624     my $globalFlags = [];
2625     my $machineFlags = [];
2626     my $rtcFlags = [];
2627     my $cpuFlags = [];
2628     my $devices = [];
2629     my $pciaddr = '';
2630     my $bridges = {};
2631     my $kvmver = kvm_user_version();
2632     my $vernum = 0; # unknown
2633     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
2634         $vernum = $1*1000000+$2*1000;
2635     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
2636         $vernum = $1*1000000+$2*1000+$3;
2637     }
2638
2639     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
2640
2641     my $have_ovz = -f '/proc/vz/vestat';
2642
2643     my $q35 = machine_type_is_q35($conf);
2644     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2645     my $machine_type = $forcemachine || $conf->{machine};
2646
2647     my $cpuunits = defined($conf->{cpuunits}) ?
2648             $conf->{cpuunits} : $defaults->{cpuunits};
2649
2650     push @$cmd, '/usr/bin/systemd-run';
2651     push @$cmd, '--scope';
2652     push @$cmd, '--slice', "qemu";
2653     push @$cmd, '--unit', $vmid;
2654     push @$cmd, '-p', "CPUShares=$cpuunits";
2655     if ($conf->{cpulimit}) {
2656         my $cpulimit = int($conf->{cpulimit} * 100);
2657         push @$cmd, '-p', "CPUQuota=$cpulimit\%";
2658     }
2659
2660     # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
2661     # load new efi bios files on migration. So this hack is required to allow
2662     # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
2663     # updrading from proxmox-ve-3.X to proxmox-ve 4.0
2664     my $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
2665
2666     push @$cmd, '/usr/bin/kvm';
2667
2668     push @$cmd, '-id', $vmid;
2669
2670     my $use_virtio = 0;
2671
2672     my $qmpsocket = qmp_socket($vmid);
2673     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2674     push @$cmd, '-mon', "chardev=qmp,mode=control";
2675
2676     my $socket = vnc_socket($vmid);
2677     push @$cmd,  '-vnc', "unix:$socket,x509,password";
2678
2679     push @$cmd, '-pidfile' , pidfile_name($vmid);
2680
2681     push @$cmd, '-daemonize';
2682
2683     if ($conf->{smbios1}) {
2684         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
2685     }
2686
2687     if ($q35) {
2688         # the q35 chipset support native usb2, so we enable usb controller
2689         # by default for this machine type
2690         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
2691     } else {
2692         $pciaddr = print_pci_addr("piix3", $bridges);
2693         push @$devices, '-device', "piix3-usb-uhci,id=uhci$pciaddr.0x2";
2694
2695         my $use_usb2 = 0;
2696         for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2697             next if !$conf->{"usb$i"};
2698             $use_usb2 = 1;
2699         }
2700         # include usb device config
2701         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-usb.cfg' if $use_usb2;
2702     }
2703
2704     my $vga = $conf->{vga};
2705
2706     my $qxlnum = vga_conf_has_spice($vga);
2707     $vga = 'qxl' if $qxlnum;
2708
2709     if (!$vga) {
2710         if ($conf->{ostype} && ($conf->{ostype} eq 'win8' ||
2711                                 $conf->{ostype} eq 'win7' ||
2712                                 $conf->{ostype} eq 'w2k8')) {
2713             $vga = 'std';
2714         } else {
2715             $vga = 'cirrus';
2716         }
2717     }
2718
2719     # enable absolute mouse coordinates (needed by vnc)
2720     my $tablet;
2721     if (defined($conf->{tablet})) {
2722         $tablet = $conf->{tablet};
2723     } else {
2724         $tablet = $defaults->{tablet};
2725         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
2726         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
2727     }
2728
2729     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
2730
2731     # host pci devices
2732     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
2733         my $d = parse_hostpci($conf->{"hostpci$i"});
2734         next if !$d;
2735
2736         my $pcie = $d->{pcie};
2737         if($pcie){
2738             die "q35 machine model is not enabled" if !$q35;
2739             $pciaddr = print_pcie_addr("hostpci$i");
2740         }else{
2741             $pciaddr = print_pci_addr("hostpci$i", $bridges);
2742         }
2743
2744         my $rombar = $d->{rombar} && $d->{rombar} eq 'off' ? ",rombar=0" : "";
2745         my $xvga = $d->{'x-vga'} && $d->{'x-vga'} eq 'on' ? ",x-vga=on" : "";
2746         if ($xvga && $xvga ne '') {
2747             push @$cpuFlags, 'kvm=off';
2748             $vga = 'none';
2749         }
2750         my $pcidevices = $d->{pciid};
2751         my $multifunction = 1 if @$pcidevices > 1;
2752
2753         my $j=0;
2754         foreach my $pcidevice (@$pcidevices) {
2755
2756             my $id = "hostpci$i";
2757             $id .= ".$j" if $multifunction;
2758             my $addr = $pciaddr;
2759             $addr .= ".$j" if $multifunction;
2760             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
2761
2762             if($j == 0){
2763                 $devicestr .= "$rombar$xvga";
2764                 $devicestr .= ",multifunction=on" if $multifunction;
2765             }
2766
2767             push @$devices, '-device', $devicestr;
2768             $j++;
2769         }
2770     }
2771
2772     # usb devices
2773     for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2774         my $d = parse_usb_device($conf->{"usb$i"});
2775         next if !$d;
2776         if ($d->{vendorid} && $d->{productid}) {
2777             push @$devices, '-device', "usb-host,vendorid=0x$d->{vendorid},productid=0x$d->{productid}";
2778         } elsif (defined($d->{hostbus}) && defined($d->{hostport})) {
2779             push @$devices, '-device', "usb-host,hostbus=$d->{hostbus},hostport=$d->{hostport}";
2780         } elsif ($d->{spice}) {
2781             # usb redir support for spice
2782             push @$devices, '-chardev', "spicevmc,id=usbredirchardev$i,name=usbredir";
2783             push @$devices, '-device', "usb-redir,chardev=usbredirchardev$i,id=usbredirdev$i,bus=ehci.0";
2784         }
2785     }
2786
2787     # serial devices
2788     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2789         if (my $path = $conf->{"serial$i"}) {
2790             if ($path eq 'socket') {
2791                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
2792                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
2793                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2794             } else {
2795                 die "no such serial device\n" if ! -c $path;
2796                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
2797                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2798             }
2799         }
2800     }
2801
2802     # parallel devices
2803     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
2804         if (my $path = $conf->{"parallel$i"}) {
2805             die "no such parallel device\n" if ! -c $path;
2806             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
2807             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
2808             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
2809         }
2810     }
2811
2812     my $vmname = $conf->{name} || "vm$vmid";
2813
2814     push @$cmd, '-name', $vmname;
2815
2816     my $sockets = 1;
2817     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
2818     $sockets = $conf->{sockets} if  $conf->{sockets};
2819
2820     my $cores = $conf->{cores} || 1;
2821
2822     my $maxcpus = $sockets * $cores;
2823
2824     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
2825
2826     my $allowed_vcpus = $cpuinfo->{cpus};
2827
2828     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
2829         if ($allowed_vcpus < $maxcpus);
2830
2831     push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
2832
2833     push @$cmd, '-nodefaults';
2834
2835     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2836
2837     my $bootindex_hash = {};
2838     my $i = 1;
2839     foreach my $o (split(//, $bootorder)) {
2840         $bootindex_hash->{$o} = $i*100;
2841         $i++;
2842     }
2843
2844     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000";
2845
2846     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
2847
2848     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
2849
2850     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
2851
2852     # time drift fix
2853     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
2854
2855     my $nokvm = defined($conf->{kvm}) && $conf->{kvm} == 0 ? 1 : 0;
2856     my $useLocaltime = $conf->{localtime};
2857
2858     if (my $ost = $conf->{ostype}) {
2859         # other, wxp, w2k, w2k3, w2k8, wvista, win7, win8, l24, l26, solaris
2860
2861         if ($ost =~ m/^w/) { # windows
2862             $useLocaltime = 1 if !defined($conf->{localtime});
2863
2864             # use time drift fix when acpi is enabled
2865             if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
2866                 $tdf = 1 if !defined($conf->{tdf});
2867             }
2868         }
2869
2870         if ($ost eq 'win7' || $ost eq 'win8' || $ost eq 'w2k8' ||
2871             $ost eq 'wvista') {
2872             push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
2873             push @$cmd, '-no-hpet';
2874             if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
2875                 push @$cpuFlags , 'hv_spinlocks=0x1fff' if !$nokvm;
2876                 push @$cpuFlags , 'hv_vapic' if !$nokvm;
2877                 push @$cpuFlags , 'hv_time' if !$nokvm;
2878
2879             } else {
2880                 push @$cpuFlags , 'hv_spinlocks=0xffff' if !$nokvm;
2881             }
2882         }
2883
2884         if ($ost eq 'win7' || $ost eq 'win8') {
2885             push @$cpuFlags , 'hv_relaxed' if !$nokvm;
2886         }
2887     }
2888
2889     push @$rtcFlags, 'driftfix=slew' if $tdf;
2890
2891     if ($nokvm) {
2892         push @$machineFlags, 'accel=tcg';
2893     } else {
2894         die "No accelerator found!\n" if !$cpuinfo->{hvm};
2895     }
2896
2897     if ($machine_type) {
2898         push @$machineFlags, "type=${machine_type}";
2899     }
2900
2901     if ($conf->{startdate}) {
2902         push @$rtcFlags, "base=$conf->{startdate}";
2903     } elsif ($useLocaltime) {
2904         push @$rtcFlags, 'base=localtime';
2905     }
2906
2907     my $cpu = $nokvm ? "qemu64" : "kvm64";
2908     $cpu = $conf->{cpu} if $conf->{cpu};
2909
2910     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
2911
2912     push @$cpuFlags , '-x2apic'
2913         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
2914
2915     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
2916
2917     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
2918
2919     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
2920
2921         push @$cpuFlags , '+kvm_pv_unhalt' if !$nokvm;
2922         push @$cpuFlags , '+kvm_pv_eoi' if !$nokvm;
2923         push @$cpuFlags , '-kvm_steal_time' if !$nokvm;
2924     }
2925
2926     push @$cpuFlags, 'enforce' if $cpu ne 'host' && !$nokvm;
2927
2928     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
2929
2930     push @$cmd, '-cpu', $cpu;
2931
2932     my $memory = $conf->{memory} || $defaults->{memory};
2933     my $static_memory = 0;
2934     my $dimm_memory = 0;
2935
2936     if ($hotplug_features->{memory}) {
2937         die "Numa need to be enabled for memory hotplug\n" if !$conf->{numa};
2938         die "Total memory is bigger than ${MAX_MEM}MB\n" if $memory > $MAX_MEM;
2939         $static_memory = $STATICMEM;
2940         die "minimum memory must be ${static_memory}MB\n" if($memory < $static_memory);
2941         $dimm_memory = $memory - $static_memory;
2942         push @$cmd, '-m', "size=${static_memory},slots=255,maxmem=${MAX_MEM}M";
2943
2944     } else {
2945
2946         $static_memory = $memory;
2947         push @$cmd, '-m', $static_memory;
2948     }
2949
2950     if ($conf->{numa}) {
2951
2952         my $numa_totalmemory = undef;
2953         for (my $i = 0; $i < $MAX_NUMA; $i++) {
2954             next if !$conf->{"numa$i"};
2955             my $numa = parse_numa($conf->{"numa$i"});
2956             next if !$numa;
2957             # memory
2958             die "missing numa node$i memory value\n" if !$numa->{memory};
2959             my $numa_memory = $numa->{memory};
2960             $numa_totalmemory += $numa_memory;
2961             my $numa_object = "memory-backend-ram,id=ram-node$i,size=${numa_memory}M";
2962
2963             # cpus
2964             my $cpus_start = $numa->{cpus}->{start};
2965             die "missing numa node$i cpus\n" if !defined($cpus_start);
2966             my $cpus_end = $numa->{cpus}->{end} if defined($numa->{cpus}->{end});
2967             my $cpus = $cpus_start;
2968             if (defined($cpus_end)) {
2969                 $cpus .= "-$cpus_end";
2970                 die "numa node$i :  cpu range $cpus is incorrect\n" if $cpus_end <= $cpus_start;
2971             }
2972
2973             # hostnodes
2974             my $hostnodes_start = $numa->{hostnodes}->{start};
2975             if (defined($hostnodes_start)) {
2976                 my $hostnodes_end = $numa->{hostnodes}->{end} if defined($numa->{hostnodes}->{end});
2977                 my $hostnodes = $hostnodes_start;
2978                 if (defined($hostnodes_end)) {
2979                     $hostnodes .= "-$hostnodes_end";
2980                     die "host node $hostnodes range is incorrect\n" if $hostnodes_end <= $hostnodes_start;
2981                 }
2982
2983                 my $hostnodes_end_range = defined($hostnodes_end) ? $hostnodes_end : $hostnodes_start;
2984                 for (my $i = $hostnodes_start; $i <= $hostnodes_end_range; $i++ ) {
2985                     die "host numa node$i don't exist\n" if ! -d "/sys/devices/system/node/node$i/";
2986                 }
2987
2988                 # policy
2989                 my $policy = $numa->{policy};
2990                 die "you need to define a policy for hostnode $hostnodes\n" if !$policy;
2991                 $numa_object .= ",host-nodes=$hostnodes,policy=$policy";
2992             }
2993
2994             push @$cmd, '-object', $numa_object;
2995             push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
2996         }
2997
2998         die "total memory for NUMA nodes must be equal to vm static memory\n"
2999             if $numa_totalmemory && $numa_totalmemory != $static_memory;
3000
3001         #if no custom tology, we split memory and cores across numa nodes
3002         if(!$numa_totalmemory) {
3003
3004             my $numa_memory = ($static_memory / $sockets) . "M";
3005
3006             for (my $i = 0; $i < $sockets; $i++)  {
3007
3008                 my $cpustart = ($cores * $i);
3009                 my $cpuend = ($cpustart + $cores - 1) if $cores && $cores > 1;
3010                 my $cpus = $cpustart;
3011                 $cpus .= "-$cpuend" if $cpuend;
3012
3013                 push @$cmd, '-object', "memory-backend-ram,size=$numa_memory,id=ram-node$i";
3014                 push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
3015             }
3016         }
3017     }
3018
3019     if ($hotplug_features->{memory}) {
3020         foreach_dimm($conf, $vmid, $memory, $sockets, sub {
3021             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3022             push @$cmd, "-object" , "memory-backend-ram,id=mem-$name,size=${dimm_size}M";
3023             push @$cmd, "-device", "pc-dimm,id=$name,memdev=mem-$name,node=$numanode";
3024
3025             #if dimm_memory is not aligned to dimm map
3026             if($current_size > $memory) {
3027                  $conf->{memory} = $current_size;
3028                  update_config_nolock($vmid, $conf, 1);
3029             }
3030         });
3031     }
3032
3033     push @$cmd, '-S' if $conf->{freeze};
3034
3035     # set keyboard layout
3036     my $kb = $conf->{keyboard} || $defaults->{keyboard};
3037     push @$cmd, '-k', $kb if $kb;
3038
3039     # enable sound
3040     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3041     #push @$cmd, '-soundhw', 'es1370';
3042     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3043
3044     if($conf->{agent}) {
3045         my $qgasocket = qmp_socket($vmid, 1);
3046         my $pciaddr = print_pci_addr("qga0", $bridges);
3047         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3048         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3049         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3050     }
3051
3052     my $spice_port;
3053
3054     if ($qxlnum) {
3055         if ($qxlnum > 1) {
3056             if ($conf->{ostype} && $conf->{ostype} =~ m/^w/){
3057                 for(my $i = 1; $i < $qxlnum; $i++){
3058                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3059                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3060                 }
3061             } else {
3062                 # assume other OS works like Linux
3063                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3064                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3065             }
3066         }
3067
3068         my $pciaddr = print_pci_addr("spice", $bridges);
3069
3070         my $nodename = PVE::INotify::nodename();
3071         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3072         $spice_port = PVE::Tools::next_spice_port($pfamily);
3073
3074         push @$devices, '-spice', "tls-port=${spice_port},addr=localhost,tls-ciphers=DES-CBC3-SHA,seamless-migration=on";
3075
3076         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3077         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3078         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3079     }
3080
3081     # enable balloon by default, unless explicitly disabled
3082     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3083         $pciaddr = print_pci_addr("balloon0", $bridges);
3084         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3085     }
3086
3087     if ($conf->{watchdog}) {
3088         my $wdopts = parse_watchdog($conf->{watchdog});
3089         $pciaddr = print_pci_addr("watchdog", $bridges);
3090         my $watchdog = $wdopts->{model} || 'i6300esb';
3091         push @$devices, '-device', "$watchdog$pciaddr";
3092         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3093     }
3094
3095     my $vollist = [];
3096     my $scsicontroller = {};
3097     my $ahcicontroller = {};
3098     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3099
3100     # Add iscsi initiator name if available
3101     if (my $initiator = get_initiator_name()) {
3102         push @$devices, '-iscsi', "initiator-name=$initiator";
3103     }
3104
3105     foreach_drive($conf, sub {
3106         my ($ds, $drive) = @_;
3107
3108         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3109             push @$vollist, $drive->{file};
3110         }
3111
3112         $use_virtio = 1 if $ds =~ m/^virtio/;
3113
3114         if (drive_is_cdrom ($drive)) {
3115             if ($bootindex_hash->{d}) {
3116                 $drive->{bootindex} = $bootindex_hash->{d};
3117                 $bootindex_hash->{d} += 1;
3118             }
3119         } else {
3120             if ($bootindex_hash->{c}) {
3121                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3122                 $bootindex_hash->{c} += 1;
3123             }
3124         }
3125
3126         if($drive->{interface} eq 'virtio'){
3127            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3128         }
3129
3130         if ($drive->{interface} eq 'scsi') {
3131
3132             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3133
3134             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3135             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3136
3137             my $iothread = '';
3138             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3139                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3140                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3141             }
3142
3143             my $queues = '';
3144             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3145                 $queues = ",num_queues=$drive->{queues}";
3146             }
3147
3148             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3149             $scsicontroller->{$controller}=1;
3150         }
3151
3152         if ($drive->{interface} eq 'sata') {
3153            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3154            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3155            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3156            $ahcicontroller->{$controller}=1;
3157         }
3158
3159         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3160         push @$devices, '-drive',$drive_cmd;
3161         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3162     });
3163
3164     for (my $i = 0; $i < $MAX_NETS; $i++) {
3165          next if !$conf->{"net$i"};
3166          my $d = parse_net($conf->{"net$i"});
3167          next if !$d;
3168
3169          $use_virtio = 1 if $d->{model} eq 'virtio';
3170
3171          if ($bootindex_hash->{n}) {
3172             $d->{bootindex} = $bootindex_hash->{n};
3173             $bootindex_hash->{n} += 1;
3174          }
3175
3176          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3177          push @$devices, '-netdev', $netdevfull;
3178
3179          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3180          push @$devices, '-device', $netdevicefull;
3181     }
3182
3183     if (!$q35) {
3184         # add pci bridges
3185         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3186            $bridges->{1} = 1;
3187            $bridges->{2} = 1;
3188         }
3189
3190         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3191
3192         while (my ($k, $v) = each %$bridges) {
3193             $pciaddr = print_pci_addr("pci.$k");
3194             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3195         }
3196     }
3197
3198     # add custom args
3199     if ($conf->{args}) {
3200         my $aa = PVE::Tools::split_args($conf->{args});
3201         push @$cmd, @$aa;
3202     }
3203
3204     push @$cmd, @$devices;
3205     push @$cmd, '-rtc', join(',', @$rtcFlags)
3206         if scalar(@$rtcFlags);
3207     push @$cmd, '-machine', join(',', @$machineFlags)
3208         if scalar(@$machineFlags);
3209     push @$cmd, '-global', join(',', @$globalFlags)
3210         if scalar(@$globalFlags);
3211
3212     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3213 }
3214
3215 sub vnc_socket {
3216     my ($vmid) = @_;
3217     return "${var_run_tmpdir}/$vmid.vnc";
3218 }
3219
3220 sub spice_port {
3221     my ($vmid) = @_;
3222
3223     my $res = vm_mon_cmd($vmid, 'query-spice');
3224
3225     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3226 }
3227
3228 sub qmp_socket {
3229     my ($vmid, $qga) = @_;
3230     my $sockettype = $qga ? 'qga' : 'qmp';
3231     return "${var_run_tmpdir}/$vmid.$sockettype";
3232 }
3233
3234 sub pidfile_name {
3235     my ($vmid) = @_;
3236     return "${var_run_tmpdir}/$vmid.pid";
3237 }
3238
3239 sub vm_devices_list {
3240     my ($vmid) = @_;
3241
3242     my $res = vm_mon_cmd($vmid, 'query-pci');
3243     my $devices = {};
3244     foreach my $pcibus (@$res) {
3245         foreach my $device (@{$pcibus->{devices}}) {
3246             next if !$device->{'qdev_id'};
3247             if ($device->{'pci_bridge'}) {
3248                 $devices->{$device->{'qdev_id'}} = 1;
3249                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3250                     next if !$bridge_device->{'qdev_id'};
3251                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3252                     $devices->{$device->{'qdev_id'}}++;
3253                 }
3254             } else {
3255                 $devices->{$device->{'qdev_id'}} = 1;
3256             }
3257         }
3258     }
3259
3260     my $resblock = vm_mon_cmd($vmid, 'query-block');
3261     foreach my $block (@$resblock) {
3262         if($block->{device} =~ m/^drive-(\S+)/){
3263                 $devices->{$1} = 1;
3264         }
3265     }
3266
3267     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3268     foreach my $mice (@$resmice) {
3269         if ($mice->{name} eq 'QEMU HID Tablet') {
3270             $devices->{tablet} = 1;
3271             last;
3272         }
3273     }
3274
3275     return $devices;
3276 }
3277
3278 sub vm_deviceplug {
3279     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3280
3281     my $q35 = machine_type_is_q35($conf);
3282
3283     my $devices_list = vm_devices_list($vmid);
3284     return 1 if defined($devices_list->{$deviceid});
3285
3286     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3287
3288     if ($deviceid eq 'tablet') {
3289
3290         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3291
3292     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3293
3294         qemu_iothread_add($vmid, $deviceid, $device);
3295
3296         qemu_driveadd($storecfg, $vmid, $device);
3297         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3298
3299         qemu_deviceadd($vmid, $devicefull);
3300         eval { qemu_deviceaddverify($vmid, $deviceid); };
3301         if (my $err = $@) {
3302             eval { qemu_drivedel($vmid, $deviceid); };
3303             warn $@ if $@;
3304             die $err;
3305         }
3306
3307     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3308
3309
3310         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3311         my $pciaddr = print_pci_addr($deviceid);
3312         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3313
3314         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3315
3316         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3317             qemu_iothread_add($vmid, $deviceid, $device);
3318             $devicefull .= ",iothread=iothread-$deviceid";
3319         }
3320
3321         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3322             $devicefull .= ",num_queues=$device->{queues}";
3323         }
3324
3325         qemu_deviceadd($vmid, $devicefull);
3326         qemu_deviceaddverify($vmid, $deviceid);
3327
3328     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3329
3330         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3331         qemu_driveadd($storecfg, $vmid, $device);
3332
3333         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3334         eval { qemu_deviceadd($vmid, $devicefull); };
3335         if (my $err = $@) {
3336             eval { qemu_drivedel($vmid, $deviceid); };
3337             warn $@ if $@;
3338             die $err;
3339         }
3340
3341     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3342
3343         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3344         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid);
3345         qemu_deviceadd($vmid, $netdevicefull);
3346         eval { qemu_deviceaddverify($vmid, $deviceid); };
3347         if (my $err = $@) {
3348             eval { qemu_netdevdel($vmid, $deviceid); };
3349             warn $@ if $@;
3350             die $err;
3351         }
3352
3353     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3354
3355         my $bridgeid = $2;
3356         my $pciaddr = print_pci_addr($deviceid);
3357         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3358
3359         qemu_deviceadd($vmid, $devicefull);
3360         qemu_deviceaddverify($vmid, $deviceid);
3361
3362     } else {
3363         die "can't hotplug device '$deviceid'\n";
3364     }
3365
3366     return 1;
3367 }
3368
3369 # fixme: this should raise exceptions on error!
3370 sub vm_deviceunplug {
3371     my ($vmid, $conf, $deviceid) = @_;
3372
3373     my $devices_list = vm_devices_list($vmid);
3374     return 1 if !defined($devices_list->{$deviceid});
3375
3376     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3377
3378     if ($deviceid eq 'tablet') {
3379
3380         qemu_devicedel($vmid, $deviceid);
3381
3382     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3383
3384         qemu_devicedel($vmid, $deviceid);
3385         qemu_devicedelverify($vmid, $deviceid);
3386         qemu_drivedel($vmid, $deviceid);
3387         qemu_iothread_del($conf, $vmid, $deviceid);
3388
3389     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3390
3391         qemu_devicedel($vmid, $deviceid);
3392         qemu_devicedelverify($vmid, $deviceid);
3393         qemu_iothread_del($conf, $vmid, $deviceid);
3394
3395     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3396
3397         #qemu 2.3 segfault on drive_del with virtioscsi + iothread
3398         my $device = parse_drive($deviceid, $conf->{$deviceid});
3399         die "virtioscsi with iothread is not hot-unplugglable currently" if $device->{iothread};
3400
3401         qemu_devicedel($vmid, $deviceid);
3402         qemu_drivedel($vmid, $deviceid);
3403         qemu_deletescsihw($conf, $vmid, $deviceid);
3404
3405     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3406
3407         qemu_devicedel($vmid, $deviceid);
3408         qemu_devicedelverify($vmid, $deviceid);
3409         qemu_netdevdel($vmid, $deviceid);
3410
3411     } else {
3412         die "can't unplug device '$deviceid'\n";
3413     }
3414
3415     return 1;
3416 }
3417
3418 sub qemu_deviceadd {
3419     my ($vmid, $devicefull) = @_;
3420
3421     $devicefull = "driver=".$devicefull;
3422     my %options =  split(/[=,]/, $devicefull);
3423
3424     vm_mon_cmd($vmid, "device_add" , %options);
3425 }
3426
3427 sub qemu_devicedel {
3428     my ($vmid, $deviceid) = @_;
3429
3430     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3431 }
3432
3433 sub qemu_iothread_add {
3434     my($vmid, $deviceid, $device) = @_;
3435
3436     if ($device->{iothread}) {
3437         my $iothreads = vm_iothreads_list($vmid);
3438         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3439     }
3440 }
3441
3442 sub qemu_iothread_del {
3443     my($conf, $vmid, $deviceid) = @_;
3444
3445     my $device = parse_drive($deviceid, $conf->{$deviceid});
3446     if ($device->{iothread}) {
3447         my $iothreads = vm_iothreads_list($vmid);
3448         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3449     }
3450 }
3451
3452 sub qemu_objectadd {
3453     my($vmid, $objectid, $qomtype) = @_;
3454
3455     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3456
3457     return 1;
3458 }
3459
3460 sub qemu_objectdel {
3461     my($vmid, $objectid) = @_;
3462
3463     vm_mon_cmd($vmid, "object-del", id => $objectid);
3464
3465     return 1;
3466 }
3467
3468 sub qemu_driveadd {
3469     my ($storecfg, $vmid, $device) = @_;
3470
3471     my $drive = print_drive_full($storecfg, $vmid, $device);
3472     $drive =~ s/\\/\\\\/g;
3473     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3474
3475     # If the command succeeds qemu prints: "OK"
3476     return 1 if $ret =~ m/OK/s;
3477
3478     die "adding drive failed: $ret\n";
3479 }
3480
3481 sub qemu_drivedel {
3482     my($vmid, $deviceid) = @_;
3483
3484     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3485     $ret =~ s/^\s+//;
3486
3487     return 1 if $ret eq "";
3488
3489     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3490     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3491
3492     die "deleting drive $deviceid failed : $ret\n";
3493 }
3494
3495 sub qemu_deviceaddverify {
3496     my ($vmid, $deviceid) = @_;
3497
3498     for (my $i = 0; $i <= 5; $i++) {
3499          my $devices_list = vm_devices_list($vmid);
3500          return 1 if defined($devices_list->{$deviceid});
3501          sleep 1;
3502     }
3503
3504     die "error on hotplug device '$deviceid'\n";
3505 }
3506
3507
3508 sub qemu_devicedelverify {
3509     my ($vmid, $deviceid) = @_;
3510
3511     # need to verify that the device is correctly removed as device_del
3512     # is async and empty return is not reliable
3513
3514     for (my $i = 0; $i <= 5; $i++) {
3515          my $devices_list = vm_devices_list($vmid);
3516          return 1 if !defined($devices_list->{$deviceid});
3517          sleep 1;
3518     }
3519
3520     die "error on hot-unplugging device '$deviceid'\n";
3521 }
3522
3523 sub qemu_findorcreatescsihw {
3524     my ($storecfg, $conf, $vmid, $device) = @_;
3525
3526     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3527
3528     my $scsihwid="$controller_prefix$controller";
3529     my $devices_list = vm_devices_list($vmid);
3530
3531     if(!defined($devices_list->{$scsihwid})) {
3532         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3533     }
3534
3535     return 1;
3536 }
3537
3538 sub qemu_deletescsihw {
3539     my ($conf, $vmid, $opt) = @_;
3540
3541     my $device = parse_drive($opt, $conf->{$opt});
3542
3543     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3544         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3545         return 1;
3546     }
3547
3548     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3549
3550     my $devices_list = vm_devices_list($vmid);
3551     foreach my $opt (keys %{$devices_list}) {
3552         if (PVE::QemuServer::valid_drivename($opt)) {
3553             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3554             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3555                 return 1;
3556             }
3557         }
3558     }
3559
3560     my $scsihwid="scsihw$controller";
3561
3562     vm_deviceunplug($vmid, $conf, $scsihwid);
3563
3564     return 1;
3565 }
3566
3567 sub qemu_add_pci_bridge {
3568     my ($storecfg, $conf, $vmid, $device) = @_;
3569
3570     my $bridges = {};
3571
3572     my $bridgeid;
3573
3574     print_pci_addr($device, $bridges);
3575
3576     while (my ($k, $v) = each %$bridges) {
3577         $bridgeid = $k;
3578     }
3579     return 1 if !defined($bridgeid) || $bridgeid < 1;
3580
3581     my $bridge = "pci.$bridgeid";
3582     my $devices_list = vm_devices_list($vmid);
3583
3584     if (!defined($devices_list->{$bridge})) {
3585         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3586     }
3587
3588     return 1;
3589 }
3590
3591 sub qemu_set_link_status {
3592     my ($vmid, $device, $up) = @_;
3593
3594     vm_mon_cmd($vmid, "set_link", name => $device,
3595                up => $up ? JSON::true : JSON::false);
3596 }
3597
3598 sub qemu_netdevadd {
3599     my ($vmid, $conf, $device, $deviceid) = @_;
3600
3601     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid);
3602     my %options =  split(/[=,]/, $netdev);
3603
3604     vm_mon_cmd($vmid, "netdev_add",  %options);
3605     return 1;
3606 }
3607
3608 sub qemu_netdevdel {
3609     my ($vmid, $deviceid) = @_;
3610
3611     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3612 }
3613
3614 sub qemu_cpu_hotplug {
3615     my ($vmid, $conf, $vcpus) = @_;
3616
3617     my $sockets = 1;
3618     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3619     $sockets = $conf->{sockets} if  $conf->{sockets};
3620     my $cores = $conf->{cores} || 1;
3621     my $maxcpus = $sockets * $cores;
3622
3623     $vcpus = $maxcpus if !$vcpus;
3624
3625     die "you can't add more vcpus than maxcpus\n"
3626         if $vcpus > $maxcpus;
3627
3628     my $currentvcpus = $conf->{vcpus} || $maxcpus;
3629     die "online cpu unplug is not yet possible\n"
3630         if $vcpus < $currentvcpus;
3631
3632     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3633     die "vcpus in running vm is different than configuration\n"
3634         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
3635
3636     for (my $i = $currentvcpus; $i < $vcpus; $i++) {
3637         vm_mon_cmd($vmid, "cpu-add", id => int($i));
3638     }
3639 }
3640
3641 sub qemu_memory_hotplug {
3642     my ($vmid, $conf, $defaults, $opt, $value) = @_;
3643
3644     return $value if !check_running($vmid);
3645
3646     my $memory = $conf->{memory} || $defaults->{memory};
3647     $value = $defaults->{memory} if !$value;
3648     return $value if $value == $memory;
3649
3650     my $static_memory = $STATICMEM;
3651     my $dimm_memory = $memory - $static_memory;
3652
3653     die "memory can't be lower than $static_memory MB" if $value < $static_memory;
3654     die "you cannot add more memory than $MAX_MEM MB!\n" if $memory > $MAX_MEM;
3655
3656
3657     my $sockets = 1;
3658     $sockets = $conf->{sockets} if $conf->{sockets};
3659
3660     if($value > $memory) {
3661
3662         foreach_dimm($conf, $vmid, $value, $sockets, sub {
3663             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3664
3665                 return if $current_size <= $conf->{memory};
3666
3667                 eval { vm_mon_cmd($vmid, "object-add", 'qom-type' => "memory-backend-ram", id => "mem-$name", props => { size => int($dimm_size*1024*1024) } ) };
3668                 if (my $err = $@) {
3669                     eval { qemu_objectdel($vmid, "mem-$name"); };
3670                     die $err;
3671                 }
3672
3673                 eval { vm_mon_cmd($vmid, "device_add", driver => "pc-dimm", id => "$name", memdev => "mem-$name", node => $numanode) };
3674                 if (my $err = $@) {
3675                     eval { qemu_objectdel($vmid, "mem-$name"); };
3676                     die $err;
3677                 }
3678                 #update conf after each succesful module hotplug
3679                 $conf->{memory} = $current_size;
3680                 update_config_nolock($vmid, $conf, 1);
3681         });
3682
3683     } else {
3684
3685         foreach_reverse_dimm($conf, $vmid, $value, $sockets, sub {
3686             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3687
3688                 return if $current_size >= $conf->{memory};
3689                 print "try to unplug memory dimm $name\n";
3690
3691                 my $retry = 0;
3692                 while (1) {
3693                     eval { qemu_devicedel($vmid, $name) };
3694                     sleep 3;
3695                     my $dimm_list = qemu_dimm_list($vmid);
3696                     last if !$dimm_list->{$name};
3697                     raise_param_exc({ $name => "error unplug memory module" }) if $retry > 5;
3698                     $retry++;
3699                 }
3700
3701                 #update conf after each succesful module unplug
3702                 $conf->{memory} = $current_size;
3703
3704                 eval { qemu_objectdel($vmid, "mem-$name"); };
3705                 update_config_nolock($vmid, $conf, 1);
3706         });
3707     }
3708 }
3709
3710 sub qemu_dimm_list {
3711     my ($vmid) = @_;
3712
3713     my $dimmarray = vm_mon_cmd_nocheck($vmid, "query-memory-devices");
3714     my $dimms = {};
3715
3716     foreach my $dimm (@$dimmarray) {
3717
3718         $dimms->{$dimm->{data}->{id}}->{id} = $dimm->{data}->{id};
3719         $dimms->{$dimm->{data}->{id}}->{node} = $dimm->{data}->{node};
3720         $dimms->{$dimm->{data}->{id}}->{addr} = $dimm->{data}->{addr};
3721         $dimms->{$dimm->{data}->{id}}->{size} = $dimm->{data}->{size};
3722         $dimms->{$dimm->{data}->{id}}->{slot} = $dimm->{data}->{slot};
3723     }
3724     return $dimms;
3725 }
3726
3727 sub qemu_block_set_io_throttle {
3728     my ($vmid, $deviceid, $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr) = @_;
3729
3730     return if !check_running($vmid) ;
3731
3732     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid, bps => int($bps), bps_rd => int($bps_rd), bps_wr => int($bps_wr), iops => int($iops), iops_rd => int($iops_rd), iops_wr => int($iops_wr));
3733
3734 }
3735
3736 # old code, only used to shutdown old VM after update
3737 sub __read_avail {
3738     my ($fh, $timeout) = @_;
3739
3740     my $sel = new IO::Select;
3741     $sel->add($fh);
3742
3743     my $res = '';
3744     my $buf;
3745
3746     my @ready;
3747     while (scalar (@ready = $sel->can_read($timeout))) {
3748         my $count;
3749         if ($count = $fh->sysread($buf, 8192)) {
3750             if ($buf =~ /^(.*)\(qemu\) $/s) {
3751                 $res .= $1;
3752                 last;
3753             } else {
3754                 $res .= $buf;
3755             }
3756         } else {
3757             if (!defined($count)) {
3758                 die "$!\n";
3759             }
3760             last;
3761         }
3762     }
3763
3764     die "monitor read timeout\n" if !scalar(@ready);
3765
3766     return $res;
3767 }
3768
3769 # old code, only used to shutdown old VM after update
3770 sub vm_monitor_command {
3771     my ($vmid, $cmdstr, $nocheck) = @_;
3772
3773     my $res;
3774
3775     eval {
3776         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
3777
3778         my $sname = "${var_run_tmpdir}/$vmid.mon";
3779
3780         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
3781             die "unable to connect to VM $vmid socket - $!\n";
3782
3783         my $timeout = 3;
3784
3785         # hack: migrate sometime blocks the monitor (when migrate_downtime
3786         # is set)
3787         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3788             $timeout = 60*60; # 1 hour
3789         }
3790
3791         # read banner;
3792         my $data = __read_avail($sock, $timeout);
3793
3794         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
3795             die "got unexpected qemu monitor banner\n";
3796         }
3797
3798         my $sel = new IO::Select;
3799         $sel->add($sock);
3800
3801         if (!scalar(my @ready = $sel->can_write($timeout))) {
3802             die "monitor write error - timeout";
3803         }
3804
3805         my $fullcmd = "$cmdstr\r";
3806
3807         # syslog('info', "VM $vmid monitor command: $cmdstr");
3808
3809         my $b;
3810         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
3811             die "monitor write error - $!";
3812         }
3813
3814         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
3815
3816         $timeout = 20;
3817
3818         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3819             $timeout = 60*60; # 1 hour
3820         } elsif ($cmdstr =~ m/^(eject|change)/) {
3821             $timeout = 60; # note: cdrom mount command is slow
3822         }
3823         if ($res = __read_avail($sock, $timeout)) {
3824
3825             my @lines = split("\r?\n", $res);
3826
3827             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
3828
3829             $res = join("\n", @lines);
3830             $res .= "\n";
3831         }
3832     };
3833
3834     my $err = $@;
3835
3836     if ($err) {
3837         syslog("err", "VM $vmid monitor command failed - $err");
3838         die $err;
3839     }
3840
3841     return $res;
3842 }
3843
3844 sub qemu_block_resize {
3845     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
3846
3847     my $running = check_running($vmid);
3848
3849     return if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
3850
3851     return if !$running;
3852
3853     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
3854
3855 }
3856
3857 sub qemu_volume_snapshot {
3858     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3859
3860     my $running = check_running($vmid);
3861
3862     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
3863         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
3864     } else {
3865         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
3866     }
3867 }
3868
3869 sub qemu_volume_snapshot_delete {
3870     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3871
3872     my $running = check_running($vmid);
3873
3874     return if !PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
3875
3876     return if !$running;
3877
3878     vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
3879 }
3880
3881 sub set_migration_caps {
3882     my ($vmid) = @_;
3883
3884     my $cap_ref = [];
3885
3886     my $enabled_cap = {
3887         "auto-converge" => 1,
3888         "xbzrle" => 1,
3889         "x-rdma-pin-all" => 0,
3890         "zero-blocks" => 0,
3891         "compress" => 0
3892     };
3893
3894     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
3895
3896     for my $supported_capability (@$supported_capabilities) {
3897         push @$cap_ref, {
3898             capability => $supported_capability->{capability},
3899             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
3900         };
3901     }
3902
3903     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
3904 }
3905
3906 my $fast_plug_option = {
3907     'lock' => 1,
3908     'name' => 1,
3909     'onboot' => 1,
3910     'shares' => 1,
3911     'startup' => 1,
3912     'description' => 1,
3913 };
3914
3915 # hotplug changes in [PENDING]
3916 # $selection hash can be used to only apply specified options, for
3917 # example: { cores => 1 } (only apply changed 'cores')
3918 # $errors ref is used to return error messages
3919 sub vmconfig_hotplug_pending {
3920     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
3921
3922     my $defaults = load_defaults();
3923
3924     # commit values which do not have any impact on running VM first
3925     # Note: those option cannot raise errors, we we do not care about
3926     # $selection and always apply them.
3927
3928     my $add_error = sub {
3929         my ($opt, $msg) = @_;
3930         $errors->{$opt} = "hotplug problem - $msg";
3931     };
3932
3933     my $changes = 0;
3934     foreach my $opt (keys %{$conf->{pending}}) { # add/change
3935         if ($fast_plug_option->{$opt}) {
3936             $conf->{$opt} = $conf->{pending}->{$opt};
3937             delete $conf->{pending}->{$opt};
3938             $changes = 1;
3939         }
3940     }
3941
3942     if ($changes) {
3943         update_config_nolock($vmid, $conf, 1);
3944         $conf = load_config($vmid); # update/reload
3945     }
3946
3947     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3948
3949     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
3950     while (my ($opt, $force) = each %$pending_delete_hash) {
3951         next if $selection && !$selection->{$opt};
3952         eval {
3953             if ($opt eq 'hotplug') {
3954                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
3955             } elsif ($opt eq 'tablet') {
3956                 die "skip\n" if !$hotplug_features->{usb};
3957                 if ($defaults->{tablet}) {
3958                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
3959                 } else {
3960                     vm_deviceunplug($vmid, $conf, $opt);
3961                 }
3962             } elsif ($opt eq 'vcpus') {
3963                 die "skip\n" if !$hotplug_features->{cpu};
3964                 qemu_cpu_hotplug($vmid, $conf, undef);
3965             } elsif ($opt eq 'balloon') {
3966                 # enable balloon device is not hotpluggable
3967                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
3968             } elsif ($fast_plug_option->{$opt}) {
3969                 # do nothing
3970             } elsif ($opt =~ m/^net(\d+)$/) {
3971                 die "skip\n" if !$hotplug_features->{network};
3972                 vm_deviceunplug($vmid, $conf, $opt);
3973             } elsif (valid_drivename($opt)) {
3974                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
3975                 vm_deviceunplug($vmid, $conf, $opt);
3976                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
3977             } elsif ($opt =~ m/^memory$/) {
3978                 die "skip\n" if !$hotplug_features->{memory};
3979                 qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
3980             } elsif ($opt eq 'cpuunits') {
3981                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
3982             } elsif ($opt eq 'cpulimit') {
3983                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
3984             } else {
3985                 die "skip\n";
3986             }
3987         };
3988         if (my $err = $@) {
3989             &$add_error($opt, $err) if $err ne "skip\n";
3990         } else {
3991             # save new config if hotplug was successful
3992             delete $conf->{$opt};
3993             vmconfig_undelete_pending_option($conf, $opt);
3994             update_config_nolock($vmid, $conf, 1);
3995             $conf = load_config($vmid); # update/reload
3996         }
3997     }
3998
3999     foreach my $opt (keys %{$conf->{pending}}) {
4000         next if $selection && !$selection->{$opt};
4001         my $value = $conf->{pending}->{$opt};
4002         eval {
4003             if ($opt eq 'hotplug') {
4004                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4005             } elsif ($opt eq 'tablet') {
4006                 die "skip\n" if !$hotplug_features->{usb};
4007                 if ($value == 1) {
4008                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4009                 } elsif ($value == 0) {
4010                     vm_deviceunplug($vmid, $conf, $opt);
4011                 }
4012             } elsif ($opt eq 'vcpus') {
4013                 die "skip\n" if !$hotplug_features->{cpu};
4014                 qemu_cpu_hotplug($vmid, $conf, $value);
4015             } elsif ($opt eq 'balloon') {
4016                 # enable/disable balloning device is not hotpluggable
4017                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4018                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4019                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4020
4021                 # allow manual ballooning if shares is set to zero
4022                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4023                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4024                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4025                 }
4026             } elsif ($opt =~ m/^net(\d+)$/) {
4027                 # some changes can be done without hotplug
4028                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4029                                     $vmid, $opt, $value);
4030             } elsif (valid_drivename($opt)) {
4031                 # some changes can be done without hotplug
4032                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4033                                      $vmid, $opt, $value, 1);
4034             } elsif ($opt =~ m/^memory$/) { #dimms
4035                 die "skip\n" if !$hotplug_features->{memory};
4036                 $value = qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4037             } elsif ($opt eq 'cpuunits') {
4038                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4039             } elsif ($opt eq 'cpulimit') {
4040                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4041                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4042             } else {
4043                 die "skip\n";  # skip non-hot-pluggable options
4044             }
4045         };
4046         if (my $err = $@) {
4047             &$add_error($opt, $err) if $err ne "skip\n";
4048         } else {
4049             # save new config if hotplug was successful
4050             $conf->{$opt} = $value;
4051             delete $conf->{pending}->{$opt};
4052             update_config_nolock($vmid, $conf, 1);
4053             $conf = load_config($vmid); # update/reload
4054         }
4055     }
4056 }
4057
4058 sub try_deallocate_drive {
4059     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4060
4061     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4062         my $volid = $drive->{file};
4063         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4064             my $sid = PVE::Storage::parse_volume_id($volid);
4065             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4066
4067             # check if the disk is really unused
4068             my $used_paths = PVE::QemuServer::get_used_paths($vmid, $storecfg, $conf, 1, $key);
4069             my $path = PVE::Storage::path($storecfg, $volid);
4070             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4071                    if $used_paths->{$path};
4072             PVE::Storage::vdisk_free($storecfg, $volid);
4073             return 1;
4074         } else {
4075             # If vm is not owner of this disk remove from config
4076             return 1;
4077         }
4078     }
4079
4080     return undef;
4081 }
4082
4083 sub vmconfig_delete_or_detach_drive {
4084     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4085
4086     my $drive = parse_drive($opt, $conf->{$opt});
4087
4088     my $rpcenv = PVE::RPCEnvironment::get();
4089     my $authuser = $rpcenv->get_user();
4090
4091     if ($force) {
4092         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4093         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4094     } else {
4095         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4096     }
4097 }
4098
4099 sub vmconfig_apply_pending {
4100     my ($vmid, $conf, $storecfg) = @_;
4101
4102     # cold plug
4103
4104     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4105     while (my ($opt, $force) = each %$pending_delete_hash) {
4106         die "internal error" if $opt =~ m/^unused/;
4107         $conf = load_config($vmid); # update/reload
4108         if (!defined($conf->{$opt})) {
4109             vmconfig_undelete_pending_option($conf, $opt);
4110             update_config_nolock($vmid, $conf, 1);
4111         } elsif (valid_drivename($opt)) {
4112             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4113             vmconfig_undelete_pending_option($conf, $opt);
4114             delete $conf->{$opt};
4115             update_config_nolock($vmid, $conf, 1);
4116         } else {
4117             vmconfig_undelete_pending_option($conf, $opt);
4118             delete $conf->{$opt};
4119             update_config_nolock($vmid, $conf, 1);
4120         }
4121     }
4122
4123     $conf = load_config($vmid); # update/reload
4124
4125     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4126         $conf = load_config($vmid); # update/reload
4127
4128         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4129             # skip if nothing changed
4130         } elsif (valid_drivename($opt)) {
4131             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4132                 if defined($conf->{$opt});
4133             $conf->{$opt} = $conf->{pending}->{$opt};
4134         } else {
4135             $conf->{$opt} = $conf->{pending}->{$opt};
4136         }
4137
4138         delete $conf->{pending}->{$opt};
4139         update_config_nolock($vmid, $conf, 1);
4140     }
4141 }
4142
4143 my $safe_num_ne = sub {
4144     my ($a, $b) = @_;
4145
4146     return 0 if !defined($a) && !defined($b);
4147     return 1 if !defined($a);
4148     return 1 if !defined($b);
4149
4150     return $a != $b;
4151 };
4152
4153 my $safe_string_ne = sub {
4154     my ($a, $b) = @_;
4155
4156     return 0 if !defined($a) && !defined($b);
4157     return 1 if !defined($a);
4158     return 1 if !defined($b);
4159
4160     return $a ne $b;
4161 };
4162
4163 sub vmconfig_update_net {
4164     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4165
4166     my $newnet = parse_net($value);
4167
4168     if ($conf->{$opt}) {
4169         my $oldnet = parse_net($conf->{$opt});
4170
4171         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4172             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4173             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4174             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4175
4176             # for non online change, we try to hot-unplug
4177             die "skip\n" if !$hotplug;
4178             vm_deviceunplug($vmid, $conf, $opt);
4179         } else {
4180
4181             die "internal error" if $opt !~ m/net(\d+)/;
4182             my $iface = "tap${vmid}i$1";
4183
4184             if (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4185                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4186             }
4187
4188             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4189                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4190                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4191                 PVE::Network::tap_unplug($iface);
4192                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall});
4193             }
4194
4195             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4196                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4197             }
4198
4199             return 1;
4200         }
4201     }
4202
4203     if ($hotplug) {
4204         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4205     } else {
4206         die "skip\n";
4207     }
4208 }
4209
4210 sub vmconfig_update_disk {
4211     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4212
4213     # fixme: do we need force?
4214
4215     my $drive = parse_drive($opt, $value);
4216
4217     if ($conf->{$opt}) {
4218
4219         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4220
4221             my $media = $drive->{media} || 'disk';
4222             my $oldmedia = $old_drive->{media} || 'disk';
4223             die "unable to change media type\n" if $media ne $oldmedia;
4224
4225             if (!drive_is_cdrom($old_drive)) {
4226
4227                 if ($drive->{file} ne $old_drive->{file}) {
4228
4229                     die "skip\n" if !$hotplug;
4230
4231                     # unplug and register as unused
4232                     vm_deviceunplug($vmid, $conf, $opt);
4233                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4234
4235                 } else {
4236                     # update existing disk
4237
4238                     # skip non hotpluggable value
4239                     if (&$safe_num_ne($drive->{discard}, $old_drive->{discard}) ||
4240                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4241                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4242                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4243                         die "skip\n";
4244                     }
4245
4246                     # apply throttle
4247                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4248                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4249                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4250                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4251                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4252                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4253                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4254                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4255                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4256                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4257                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4258                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max})) {
4259
4260                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4261                                                    ($drive->{mbps} || 0)*1024*1024,
4262                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4263                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4264                                                    $drive->{iops} || 0,
4265                                                    $drive->{iops_rd} || 0,
4266                                                    $drive->{iops_wr} || 0,
4267                                                    ($drive->{mbps_max} || 0)*1024*1024,
4268                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4269                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4270                                                    $drive->{iops_max} || 0,
4271                                                    $drive->{iops_rd_max} || 0,
4272                                                    $drive->{iops_wr_max} || 0);
4273
4274                     }
4275
4276                     return 1;
4277                 }
4278
4279             } else { # cdrom
4280
4281                 if ($drive->{file} eq 'none') {
4282                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4283                 } else {
4284                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4285                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4286                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4287                 }
4288
4289                 return 1;
4290             }
4291         }
4292     }
4293
4294     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4295     # hotplug new disks
4296     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4297 }
4298
4299 sub vm_start {
4300     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4301         $forcemachine, $spice_ticket) = @_;
4302
4303     lock_config($vmid, sub {
4304         my $conf = load_config($vmid, $migratedfrom);
4305
4306         die "you can't start a vm if it's a template\n" if is_template($conf);
4307
4308         check_lock($conf) if !$skiplock;
4309
4310         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4311
4312         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4313             vmconfig_apply_pending($vmid, $conf, $storecfg);
4314             $conf = load_config($vmid); # update/reload
4315         }
4316
4317         my $defaults = load_defaults();
4318
4319         # set environment variable useful inside network script
4320         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4321
4322         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4323
4324         my $migrate_port = 0;
4325         my $migrate_uri;
4326         if ($statefile) {
4327             if ($statefile eq 'tcp') {
4328                 my $localip = "localhost";
4329                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4330                 my $nodename = PVE::INotify::nodename();
4331                 if ($datacenterconf->{migration_unsecure}) {
4332                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4333                 }
4334                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4335                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4336                 $migrate_uri = "tcp:[${localip}]:${migrate_port}";
4337                 push @$cmd, '-incoming', $migrate_uri;
4338                 push @$cmd, '-S';
4339             } else {
4340                 push @$cmd, '-loadstate', $statefile;
4341             }
4342         } elsif ($paused) {
4343             push @$cmd, '-S';
4344         }
4345
4346         # host pci devices
4347         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4348           my $d = parse_hostpci($conf->{"hostpci$i"});
4349           next if !$d;
4350           my $pcidevices = $d->{pciid};
4351           foreach my $pcidevice (@$pcidevices) {
4352                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4353
4354                 my $info = pci_device_info("0000:$pciid");
4355                 die "IOMMU not present\n" if !check_iommu_support();
4356                 die "no pci device info for device '$pciid'\n" if !$info;
4357                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4358                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4359           }
4360         }
4361
4362         PVE::Storage::activate_volumes($storecfg, $vollist);
4363
4364         eval  { run_command($cmd, timeout => $statefile ? undef : 30,
4365                     umask => 0077); };
4366         my $err = $@;
4367         die "start failed: $err" if $err;
4368
4369         print "migration listens on $migrate_uri\n" if $migrate_uri;
4370
4371         if ($statefile && $statefile ne 'tcp')  {
4372             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4373             warn $@ if $@;
4374         }
4375
4376         if ($migratedfrom) {
4377
4378             eval {
4379                 set_migration_caps($vmid);
4380             };
4381             warn $@ if $@;
4382
4383             if ($spice_port) {
4384                 print "spice listens on port $spice_port\n";
4385                 if ($spice_ticket) {
4386                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4387                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4388                 }
4389             }
4390
4391         } else {
4392
4393             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4394                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4395                     if $conf->{balloon};
4396             }
4397
4398             foreach my $opt (keys %$conf) {
4399                 next if $opt !~  m/^net\d+$/;
4400                 my $nicconf = parse_net($conf->{$opt});
4401                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4402             }
4403         }
4404
4405         vm_mon_cmd_nocheck($vmid, 'qom-set',
4406                     path => "machine/peripheral/balloon0",
4407                     property => "guest-stats-polling-interval",
4408                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4409
4410     });
4411 }
4412
4413 sub vm_mon_cmd {
4414     my ($vmid, $execute, %params) = @_;
4415
4416     my $cmd = { execute => $execute, arguments => \%params };
4417     vm_qmp_command($vmid, $cmd);
4418 }
4419
4420 sub vm_mon_cmd_nocheck {
4421     my ($vmid, $execute, %params) = @_;
4422
4423     my $cmd = { execute => $execute, arguments => \%params };
4424     vm_qmp_command($vmid, $cmd, 1);
4425 }
4426
4427 sub vm_qmp_command {
4428     my ($vmid, $cmd, $nocheck) = @_;
4429
4430     my $res;
4431
4432     my $timeout;
4433     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4434         $timeout = $cmd->{arguments}->{timeout};
4435         delete $cmd->{arguments}->{timeout};
4436     }
4437
4438     eval {
4439         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4440         my $sname = qmp_socket($vmid);
4441         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4442             my $qmpclient = PVE::QMPClient->new();
4443
4444             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4445         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
4446             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
4447                 if scalar(%{$cmd->{arguments}});
4448             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
4449         } else {
4450             die "unable to open monitor socket\n";
4451         }
4452     };
4453     if (my $err = $@) {
4454         syslog("err", "VM $vmid qmp command failed - $err");
4455         die $err;
4456     }
4457
4458     return $res;
4459 }
4460
4461 sub vm_human_monitor_command {
4462     my ($vmid, $cmdline) = @_;
4463
4464     my $res;
4465
4466     my $cmd = {
4467         execute => 'human-monitor-command',
4468         arguments => { 'command-line' => $cmdline},
4469     };
4470
4471     return vm_qmp_command($vmid, $cmd);
4472 }
4473
4474 sub vm_commandline {
4475     my ($storecfg, $vmid) = @_;
4476
4477     my $conf = load_config($vmid);
4478
4479     my $defaults = load_defaults();
4480
4481     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
4482
4483     return join(' ', @$cmd);
4484 }
4485
4486 sub vm_reset {
4487     my ($vmid, $skiplock) = @_;
4488
4489     lock_config($vmid, sub {
4490
4491         my $conf = load_config($vmid);
4492
4493         check_lock($conf) if !$skiplock;
4494
4495         vm_mon_cmd($vmid, "system_reset");
4496     });
4497 }
4498
4499 sub get_vm_volumes {
4500     my ($conf) = @_;
4501
4502     my $vollist = [];
4503     foreach_volid($conf, sub {
4504         my ($volid, $is_cdrom) = @_;
4505
4506         return if $volid =~ m|^/|;
4507
4508         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4509         return if !$sid;
4510
4511         push @$vollist, $volid;
4512     });
4513
4514     return $vollist;
4515 }
4516
4517 sub vm_stop_cleanup {
4518     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
4519
4520     eval {
4521
4522         if (!$keepActive) {
4523             my $vollist = get_vm_volumes($conf);
4524             PVE::Storage::deactivate_volumes($storecfg, $vollist);
4525         }
4526
4527         foreach my $ext (qw(mon qmp pid vnc qga)) {
4528             unlink "/var/run/qemu-server/${vmid}.$ext";
4529         }
4530
4531         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
4532     };
4533     warn $@ if $@; # avoid errors - just warn
4534 }
4535
4536 # Note: use $nockeck to skip tests if VM configuration file exists.
4537 # We need that when migration VMs to other nodes (files already moved)
4538 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
4539 sub vm_stop {
4540     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
4541
4542     $force = 1 if !defined($force) && !$shutdown;
4543
4544     if ($migratedfrom){
4545         my $pid = check_running($vmid, $nocheck, $migratedfrom);
4546         kill 15, $pid if $pid;
4547         my $conf = load_config($vmid, $migratedfrom);
4548         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
4549         return;
4550     }
4551
4552     lock_config($vmid, sub {
4553
4554         my $pid = check_running($vmid, $nocheck);
4555         return if !$pid;
4556
4557         my $conf;
4558         if (!$nocheck) {
4559             $conf = load_config($vmid);
4560             check_lock($conf) if !$skiplock;
4561             if (!defined($timeout) && $shutdown && $conf->{startup}) {
4562                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
4563                 $timeout = $opts->{down} if $opts->{down};
4564             }
4565         }
4566
4567         $timeout = 60 if !defined($timeout);
4568
4569         eval {
4570             if ($shutdown) {
4571                 if (defined($conf) && $conf->{agent}) {
4572                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
4573                 } else {
4574                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
4575                 }
4576             } else {
4577                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
4578             }
4579         };
4580         my $err = $@;
4581
4582         if (!$err) {
4583             my $count = 0;
4584             while (($count < $timeout) && check_running($vmid, $nocheck)) {
4585                 $count++;
4586                 sleep 1;
4587             }
4588
4589             if ($count >= $timeout) {
4590                 if ($force) {
4591                     warn "VM still running - terminating now with SIGTERM\n";
4592                     kill 15, $pid;
4593                 } else {
4594                     die "VM quit/powerdown failed - got timeout\n";
4595                 }
4596             } else {
4597                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4598                 return;
4599             }
4600         } else {
4601             if ($force) {
4602                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
4603                 kill 15, $pid;
4604             } else {
4605                 die "VM quit/powerdown failed\n";
4606             }
4607         }
4608
4609         # wait again
4610         $timeout = 10;
4611
4612         my $count = 0;
4613         while (($count < $timeout) && check_running($vmid, $nocheck)) {
4614             $count++;
4615             sleep 1;
4616         }
4617
4618         if ($count >= $timeout) {
4619             warn "VM still running - terminating now with SIGKILL\n";
4620             kill 9, $pid;
4621             sleep 1;
4622         }
4623
4624         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4625    });
4626 }
4627
4628 sub vm_suspend {
4629     my ($vmid, $skiplock) = @_;
4630
4631     lock_config($vmid, sub {
4632
4633         my $conf = load_config($vmid);
4634
4635         check_lock($conf) if !($skiplock || ($conf->{lock} && $conf->{lock} eq 'backup'));
4636
4637         vm_mon_cmd($vmid, "stop");
4638     });
4639 }
4640
4641 sub vm_resume {
4642     my ($vmid, $skiplock, $nocheck) = @_;
4643
4644     lock_config($vmid, sub {
4645
4646         if (!$nocheck) {
4647
4648             my $conf = load_config($vmid);
4649
4650             check_lock($conf) if !($skiplock || ($conf->{lock} && $conf->{lock} eq 'backup'));
4651
4652             vm_mon_cmd($vmid, "cont");
4653
4654         } else {
4655             vm_mon_cmd_nocheck($vmid, "cont");
4656         }
4657     });
4658 }
4659
4660 sub vm_sendkey {
4661     my ($vmid, $skiplock, $key) = @_;
4662
4663     lock_config($vmid, sub {
4664
4665         my $conf = load_config($vmid);
4666
4667         # there is no qmp command, so we use the human monitor command
4668         vm_human_monitor_command($vmid, "sendkey $key");
4669     });
4670 }
4671
4672 sub vm_destroy {
4673     my ($storecfg, $vmid, $skiplock) = @_;
4674
4675     lock_config($vmid, sub {
4676
4677         my $conf = load_config($vmid);
4678
4679         check_lock($conf) if !$skiplock;
4680
4681         if (!check_running($vmid)) {
4682             destroy_vm($storecfg, $vmid);
4683         } else {
4684             die "VM $vmid is running - destroy failed\n";
4685         }
4686     });
4687 }
4688
4689 # pci helpers
4690
4691 sub file_write {
4692     my ($filename, $buf) = @_;
4693
4694     my $fh = IO::File->new($filename, "w");
4695     return undef if !$fh;
4696
4697     my $res = print $fh $buf;
4698
4699     $fh->close();
4700
4701     return $res;
4702 }
4703
4704 sub pci_device_info {
4705     my ($name) = @_;
4706
4707     my $res;
4708
4709     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
4710     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
4711
4712     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
4713     return undef if !defined($irq) || $irq !~ m/^\d+$/;
4714
4715     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
4716     return undef if !defined($vendor) || $vendor !~ s/^0x//;
4717
4718     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
4719     return undef if !defined($product) || $product !~ s/^0x//;
4720
4721     $res = {
4722         name => $name,
4723         vendor => $vendor,
4724         product => $product,
4725         domain => $domain,
4726         bus => $bus,
4727         slot => $slot,
4728         func => $func,
4729         irq => $irq,
4730         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
4731     };
4732
4733     return $res;
4734 }
4735
4736 sub pci_dev_reset {
4737     my ($dev) = @_;
4738
4739     my $name = $dev->{name};
4740
4741     my $fn = "$pcisysfs/devices/$name/reset";
4742
4743     return file_write($fn, "1");
4744 }
4745
4746 sub pci_dev_bind_to_vfio {
4747     my ($dev) = @_;
4748
4749     my $name = $dev->{name};
4750
4751     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4752
4753     if (!-d $vfio_basedir) {
4754         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4755     }
4756     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4757
4758     my $testdir = "$vfio_basedir/$name";
4759     return 1 if -d $testdir;
4760
4761     my $data = "$dev->{vendor} $dev->{product}";
4762     return undef if !file_write("$vfio_basedir/new_id", $data);
4763
4764     my $fn = "$pcisysfs/devices/$name/driver/unbind";
4765     if (!file_write($fn, $name)) {
4766         return undef if -f $fn;
4767     }
4768
4769     $fn = "$vfio_basedir/bind";
4770     if (! -d $testdir) {
4771         return undef if !file_write($fn, $name);
4772     }
4773
4774     return -d $testdir;
4775 }
4776
4777 sub pci_dev_group_bind_to_vfio {
4778     my ($pciid) = @_;
4779
4780     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4781
4782     if (!-d $vfio_basedir) {
4783         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4784     }
4785     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4786
4787     # get IOMMU group devices
4788     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
4789       my @devs = grep /^0000:/, readdir($D);
4790     closedir($D);
4791
4792     foreach my $pciid (@devs) {
4793         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
4794
4795         # pci bridges, switches or root ports are not supported
4796         # they have a pci_bus subdirectory so skip them
4797         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
4798
4799         my $info = pci_device_info($1);
4800         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
4801     }
4802
4803     return 1;
4804 }
4805
4806 sub print_pci_addr {
4807     my ($id, $bridges) = @_;
4808
4809     my $res = '';
4810     my $devices = {
4811         piix3 => { bus => 0, addr => 1 },
4812         #addr2 : first videocard
4813         balloon0 => { bus => 0, addr => 3 },
4814         watchdog => { bus => 0, addr => 4 },
4815         scsihw0 => { bus => 0, addr => 5 },
4816         'pci.3' => { bus => 0, addr => 5 }, #can also be used for virtio-scsi-single bridge
4817         scsihw1 => { bus => 0, addr => 6 },
4818         ahci0 => { bus => 0, addr => 7 },
4819         qga0 => { bus => 0, addr => 8 },
4820         spice => { bus => 0, addr => 9 },
4821         virtio0 => { bus => 0, addr => 10 },
4822         virtio1 => { bus => 0, addr => 11 },
4823         virtio2 => { bus => 0, addr => 12 },
4824         virtio3 => { bus => 0, addr => 13 },
4825         virtio4 => { bus => 0, addr => 14 },
4826         virtio5 => { bus => 0, addr => 15 },
4827         hostpci0 => { bus => 0, addr => 16 },
4828         hostpci1 => { bus => 0, addr => 17 },
4829         net0 => { bus => 0, addr => 18 },
4830         net1 => { bus => 0, addr => 19 },
4831         net2 => { bus => 0, addr => 20 },
4832         net3 => { bus => 0, addr => 21 },
4833         net4 => { bus => 0, addr => 22 },
4834         net5 => { bus => 0, addr => 23 },
4835         vga1 => { bus => 0, addr => 24 },
4836         vga2 => { bus => 0, addr => 25 },
4837         vga3 => { bus => 0, addr => 26 },
4838         hostpci2 => { bus => 0, addr => 27 },
4839         hostpci3 => { bus => 0, addr => 28 },
4840         #addr29 : usb-host (pve-usb.cfg)
4841         'pci.1' => { bus => 0, addr => 30 },
4842         'pci.2' => { bus => 0, addr => 31 },
4843         'net6' => { bus => 1, addr => 1 },
4844         'net7' => { bus => 1, addr => 2 },
4845         'net8' => { bus => 1, addr => 3 },
4846         'net9' => { bus => 1, addr => 4 },
4847         'net10' => { bus => 1, addr => 5 },
4848         'net11' => { bus => 1, addr => 6 },
4849         'net12' => { bus => 1, addr => 7 },
4850         'net13' => { bus => 1, addr => 8 },
4851         'net14' => { bus => 1, addr => 9 },
4852         'net15' => { bus => 1, addr => 10 },
4853         'net16' => { bus => 1, addr => 11 },
4854         'net17' => { bus => 1, addr => 12 },
4855         'net18' => { bus => 1, addr => 13 },
4856         'net19' => { bus => 1, addr => 14 },
4857         'net20' => { bus => 1, addr => 15 },
4858         'net21' => { bus => 1, addr => 16 },
4859         'net22' => { bus => 1, addr => 17 },
4860         'net23' => { bus => 1, addr => 18 },
4861         'net24' => { bus => 1, addr => 19 },
4862         'net25' => { bus => 1, addr => 20 },
4863         'net26' => { bus => 1, addr => 21 },
4864         'net27' => { bus => 1, addr => 22 },
4865         'net28' => { bus => 1, addr => 23 },
4866         'net29' => { bus => 1, addr => 24 },
4867         'net30' => { bus => 1, addr => 25 },
4868         'net31' => { bus => 1, addr => 26 },
4869         'virtio6' => { bus => 2, addr => 1 },
4870         'virtio7' => { bus => 2, addr => 2 },
4871         'virtio8' => { bus => 2, addr => 3 },
4872         'virtio9' => { bus => 2, addr => 4 },
4873         'virtio10' => { bus => 2, addr => 5 },
4874         'virtio11' => { bus => 2, addr => 6 },
4875         'virtio12' => { bus => 2, addr => 7 },
4876         'virtio13' => { bus => 2, addr => 8 },
4877         'virtio14' => { bus => 2, addr => 9 },
4878         'virtio15' => { bus => 2, addr => 10 },
4879         'virtioscsi0' => { bus => 3, addr => 1 },
4880         'virtioscsi1' => { bus => 3, addr => 2 },
4881         'virtioscsi2' => { bus => 3, addr => 3 },
4882         'virtioscsi3' => { bus => 3, addr => 4 },
4883         'virtioscsi4' => { bus => 3, addr => 5 },
4884         'virtioscsi5' => { bus => 3, addr => 6 },
4885         'virtioscsi6' => { bus => 3, addr => 7 },
4886         'virtioscsi7' => { bus => 3, addr => 8 },
4887         'virtioscsi8' => { bus => 3, addr => 9 },
4888         'virtioscsi9' => { bus => 3, addr => 10 },
4889         'virtioscsi10' => { bus => 3, addr => 11 },
4890         'virtioscsi11' => { bus => 3, addr => 12 },
4891         'virtioscsi12' => { bus => 3, addr => 13 },
4892         'virtioscsi13' => { bus => 3, addr => 14 },
4893         'virtioscsi14' => { bus => 3, addr => 15 },
4894         'virtioscsi15' => { bus => 3, addr => 16 },
4895         'virtioscsi16' => { bus => 3, addr => 17 },
4896         'virtioscsi17' => { bus => 3, addr => 18 },
4897         'virtioscsi18' => { bus => 3, addr => 19 },
4898         'virtioscsi19' => { bus => 3, addr => 20 },
4899         'virtioscsi20' => { bus => 3, addr => 21 },
4900         'virtioscsi21' => { bus => 3, addr => 22 },
4901         'virtioscsi22' => { bus => 3, addr => 23 },
4902         'virtioscsi23' => { bus => 3, addr => 24 },
4903         'virtioscsi24' => { bus => 3, addr => 25 },
4904         'virtioscsi25' => { bus => 3, addr => 26 },
4905         'virtioscsi26' => { bus => 3, addr => 27 },
4906         'virtioscsi27' => { bus => 3, addr => 28 },
4907         'virtioscsi28' => { bus => 3, addr => 29 },
4908         'virtioscsi29' => { bus => 3, addr => 30 },
4909         'virtioscsi30' => { bus => 3, addr => 31 },
4910
4911     };
4912
4913     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
4914            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
4915            my $bus = $devices->{$id}->{bus};
4916            $res = ",bus=pci.$bus,addr=$addr";
4917            $bridges->{$bus} = 1 if $bridges;
4918     }
4919     return $res;
4920
4921 }
4922
4923 sub print_pcie_addr {
4924     my ($id) = @_;
4925
4926     my $res = '';
4927     my $devices = {
4928         hostpci0 => { bus => "ich9-pcie-port-1", addr => 0 },
4929         hostpci1 => { bus => "ich9-pcie-port-2", addr => 0 },
4930         hostpci2 => { bus => "ich9-pcie-port-3", addr => 0 },
4931         hostpci3 => { bus => "ich9-pcie-port-4", addr => 0 },
4932     };
4933
4934     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
4935            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
4936            my $bus = $devices->{$id}->{bus};
4937            $res = ",bus=$bus,addr=$addr";
4938     }
4939     return $res;
4940
4941 }
4942
4943 # vzdump restore implementaion
4944
4945 sub tar_archive_read_firstfile {
4946     my $archive = shift;
4947
4948     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
4949
4950     # try to detect archive type first
4951     my $pid = open (TMP, "tar tf '$archive'|") ||
4952         die "unable to open file '$archive'\n";
4953     my $firstfile = <TMP>;
4954     kill 15, $pid;
4955     close TMP;
4956
4957     die "ERROR: archive contaions no data\n" if !$firstfile;
4958     chomp $firstfile;
4959
4960     return $firstfile;
4961 }
4962
4963 sub tar_restore_cleanup {
4964     my ($storecfg, $statfile) = @_;
4965
4966     print STDERR "starting cleanup\n";
4967
4968     if (my $fd = IO::File->new($statfile, "r")) {
4969         while (defined(my $line = <$fd>)) {
4970             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
4971                 my $volid = $2;
4972                 eval {
4973                     if ($volid =~ m|^/|) {
4974                         unlink $volid || die 'unlink failed\n';
4975                     } else {
4976                         PVE::Storage::vdisk_free($storecfg, $volid);
4977                     }
4978                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
4979                 };
4980                 print STDERR "unable to cleanup '$volid' - $@" if $@;
4981             } else {
4982                 print STDERR "unable to parse line in statfile - $line";
4983             }
4984         }
4985         $fd->close();
4986     }
4987 }
4988
4989 sub restore_archive {
4990     my ($archive, $vmid, $user, $opts) = @_;
4991
4992     my $format = $opts->{format};
4993     my $comp;
4994
4995     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
4996         $format = 'tar' if !$format;
4997         $comp = 'gzip';
4998     } elsif ($archive =~ m/\.tar$/) {
4999         $format = 'tar' if !$format;
5000     } elsif ($archive =~ m/.tar.lzo$/) {
5001         $format = 'tar' if !$format;
5002         $comp = 'lzop';
5003     } elsif ($archive =~ m/\.vma$/) {
5004         $format = 'vma' if !$format;
5005     } elsif ($archive =~ m/\.vma\.gz$/) {
5006         $format = 'vma' if !$format;
5007         $comp = 'gzip';
5008     } elsif ($archive =~ m/\.vma\.lzo$/) {
5009         $format = 'vma' if !$format;
5010         $comp = 'lzop';
5011     } else {
5012         $format = 'vma' if !$format; # default
5013     }
5014
5015     # try to detect archive format
5016     if ($format eq 'tar') {
5017         return restore_tar_archive($archive, $vmid, $user, $opts);
5018     } else {
5019         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5020     }
5021 }
5022
5023 sub restore_update_config_line {
5024     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5025
5026     return if $line =~ m/^\#qmdump\#/;
5027     return if $line =~ m/^\#vzdump\#/;
5028     return if $line =~ m/^lock:/;
5029     return if $line =~ m/^unused\d+:/;
5030     return if $line =~ m/^parent:/;
5031     return if $line =~ m/^template:/; # restored VM is never a template
5032
5033     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5034         # try to convert old 1.X settings
5035         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5036         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5037             my ($model, $macaddr) = split(/\=/, $devconfig);
5038             $macaddr = PVE::Tools::random_ether_addr() if !$macaddr || $unique;
5039             my $net = {
5040                 model => $model,
5041                 bridge => "vmbr$ind",
5042                 macaddr => $macaddr,
5043             };
5044             my $netstr = print_net($net);
5045
5046             print $outfd "net$cookie->{netcount}: $netstr\n";
5047             $cookie->{netcount}++;
5048         }
5049     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5050         my ($id, $netstr) = ($1, $2);
5051         my $net = parse_net($netstr);
5052         $net->{macaddr} = PVE::Tools::random_ether_addr() if $net->{macaddr};
5053         $netstr = print_net($net);
5054         print $outfd "$id: $netstr\n";
5055     } elsif ($line =~ m/^((ide|scsi|virtio|sata)\d+):\s*(\S+)\s*$/) {
5056         my $virtdev = $1;
5057         my $value = $3;
5058         if ($line =~ m/backup=no/) {
5059             print $outfd "#$line";
5060         } elsif ($virtdev && $map->{$virtdev}) {
5061             my $di = parse_drive($virtdev, $value);
5062             delete $di->{format}; # format can change on restore
5063             $di->{file} = $map->{$virtdev};
5064             $value = print_drive($vmid, $di);
5065             print $outfd "$virtdev: $value\n";
5066         } else {
5067             print $outfd $line;
5068         }
5069     } else {
5070         print $outfd $line;
5071     }
5072 }
5073
5074 sub scan_volids {
5075     my ($cfg, $vmid) = @_;
5076
5077     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5078
5079     my $volid_hash = {};
5080     foreach my $storeid (keys %$info) {
5081         foreach my $item (@{$info->{$storeid}}) {
5082             next if !($item->{volid} && $item->{size});
5083             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5084             $volid_hash->{$item->{volid}} = $item;
5085         }
5086     }
5087
5088     return $volid_hash;
5089 }
5090
5091 sub get_used_paths {
5092     my ($vmid, $storecfg, $conf, $scan_snapshots, $skip_drive) = @_;
5093
5094     my $used_path = {};
5095
5096     my $scan_config = sub {
5097         my ($cref, $snapname) = @_;
5098
5099         foreach my $key (keys %$cref) {
5100             my $value = $cref->{$key};
5101             if (valid_drivename($key)) {
5102                 next if $skip_drive && $key eq $skip_drive;
5103                 my $drive = parse_drive($key, $value);
5104                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5105                 if ($drive->{file} =~ m!^/!) {
5106                     $used_path->{$drive->{file}}++; # = 1;
5107                 } else {
5108                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5109                     next if !$storeid;
5110                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5111                     next if !$scfg;
5112                     my $path = PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5113                     $used_path->{$path}++; # = 1;
5114                 }
5115             }
5116         }
5117     };
5118
5119     &$scan_config($conf);
5120
5121     undef $skip_drive;
5122
5123     if ($scan_snapshots) {
5124         foreach my $snapname (keys %{$conf->{snapshots}}) {
5125             &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5126         }
5127     }
5128
5129     return $used_path;
5130 }
5131
5132 sub update_disksize {
5133     my ($vmid, $conf, $volid_hash) = @_;
5134
5135     my $changes;
5136
5137     my $used = {};
5138
5139     # Note: it is allowed to define multiple storages with same path (alias), so
5140     # we need to check both 'volid' and real 'path' (two different volid can point
5141     # to the same path).
5142
5143     my $usedpath = {};
5144
5145     # update size info
5146     foreach my $opt (keys %$conf) {
5147         if (valid_drivename($opt)) {
5148             my $drive = parse_drive($opt, $conf->{$opt});
5149             my $volid = $drive->{file};
5150             next if !$volid;
5151
5152             $used->{$volid} = 1;
5153             if ($volid_hash->{$volid} &&
5154                 (my $path = $volid_hash->{$volid}->{path})) {
5155                 $usedpath->{$path} = 1;
5156             }
5157
5158             next if drive_is_cdrom($drive);
5159             next if !$volid_hash->{$volid};
5160
5161             $drive->{size} = $volid_hash->{$volid}->{size};
5162             my $new = print_drive($vmid, $drive);
5163             if ($new ne $conf->{$opt}) {
5164                 $changes = 1;
5165                 $conf->{$opt} = $new;
5166             }
5167         }
5168     }
5169
5170     # remove 'unusedX' entry if volume is used
5171     foreach my $opt (keys %$conf) {
5172         next if $opt !~ m/^unused\d+$/;
5173         my $volid = $conf->{$opt};
5174         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5175         if ($used->{$volid} || ($path && $usedpath->{$path})) {
5176             $changes = 1;
5177             delete $conf->{$opt};
5178         }
5179     }
5180
5181     foreach my $volid (sort keys %$volid_hash) {
5182         next if $volid =~ m/vm-$vmid-state-/;
5183         next if $used->{$volid};
5184         my $path = $volid_hash->{$volid}->{path};
5185         next if !$path; # just to be sure
5186         next if $usedpath->{$path};
5187         $changes = 1;
5188         add_unused_volume($conf, $volid);
5189         $usedpath->{$path} = 1; # avoid to add more than once (aliases)
5190     }
5191
5192     return $changes;
5193 }
5194
5195 sub rescan {
5196     my ($vmid, $nolock) = @_;
5197
5198     my $cfg = PVE::Cluster::cfs_read_file("storage.cfg");
5199
5200     my $volid_hash = scan_volids($cfg, $vmid);
5201
5202     my $updatefn =  sub {
5203         my ($vmid) = @_;
5204
5205         my $conf = load_config($vmid);
5206
5207         check_lock($conf);
5208
5209         my $vm_volids = {};
5210         foreach my $volid (keys %$volid_hash) {
5211             my $info = $volid_hash->{$volid};
5212             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5213         }
5214
5215         my $changes = update_disksize($vmid, $conf, $vm_volids);
5216
5217         update_config_nolock($vmid, $conf, 1) if $changes;
5218     };
5219
5220     if (defined($vmid)) {
5221         if ($nolock) {
5222             &$updatefn($vmid);
5223         } else {
5224             lock_config($vmid, $updatefn, $vmid);
5225         }
5226     } else {
5227         my $vmlist = config_list();
5228         foreach my $vmid (keys %$vmlist) {
5229             if ($nolock) {
5230                 &$updatefn($vmid);
5231             } else {
5232                 lock_config($vmid, $updatefn, $vmid);
5233             }
5234         }
5235     }
5236 }
5237
5238 sub restore_vma_archive {
5239     my ($archive, $vmid, $user, $opts, $comp) = @_;
5240
5241     my $input = $archive eq '-' ? "<&STDIN" : undef;
5242     my $readfrom = $archive;
5243
5244     my $uncomp = '';
5245     if ($comp) {
5246         $readfrom = '-';
5247         my $qarchive = PVE::Tools::shellquote($archive);
5248         if ($comp eq 'gzip') {
5249             $uncomp = "zcat $qarchive|";
5250         } elsif ($comp eq 'lzop') {
5251             $uncomp = "lzop -d -c $qarchive|";
5252         } else {
5253             die "unknown compression method '$comp'\n";
5254         }
5255
5256     }
5257
5258     my $tmpdir = "/var/tmp/vzdumptmp$$";
5259     rmtree $tmpdir;
5260
5261     # disable interrupts (always do cleanups)
5262     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5263         warn "got interrupt - ignored\n";
5264     };
5265
5266     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5267     POSIX::mkfifo($mapfifo, 0600);
5268     my $fifofh;
5269
5270     my $openfifo = sub {
5271         open($fifofh, '>', $mapfifo) || die $!;
5272     };
5273
5274     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5275
5276     my $oldtimeout;
5277     my $timeout = 5;
5278
5279     my $devinfo = {};
5280
5281     my $rpcenv = PVE::RPCEnvironment::get();
5282
5283     my $conffile = config_file($vmid);
5284     my $tmpfn = "$conffile.$$.tmp";
5285
5286     # Note: $oldconf is undef if VM does not exists
5287     my $oldconf = PVE::Cluster::cfs_read_file(cfs_config_path($vmid));
5288
5289     my $print_devmap = sub {
5290         my $virtdev_hash = {};
5291
5292         my $cfgfn = "$tmpdir/qemu-server.conf";
5293
5294         # we can read the config - that is already extracted
5295         my $fh = IO::File->new($cfgfn, "r") ||
5296             "unable to read qemu-server.conf - $!\n";
5297
5298         while (defined(my $line = <$fh>)) {
5299             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5300                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5301                 die "archive does not contain data for drive '$virtdev'\n"
5302                     if !$devinfo->{$devname};
5303                 if (defined($opts->{storage})) {
5304                     $storeid = $opts->{storage} || 'local';
5305                 } elsif (!$storeid) {
5306                     $storeid = 'local';
5307                 }
5308                 $format = 'raw' if !$format;
5309                 $devinfo->{$devname}->{devname} = $devname;
5310                 $devinfo->{$devname}->{virtdev} = $virtdev;
5311                 $devinfo->{$devname}->{format} = $format;
5312                 $devinfo->{$devname}->{storeid} = $storeid;
5313
5314                 # check permission on storage
5315                 my $pool = $opts->{pool}; # todo: do we need that?
5316                 if ($user ne 'root@pam') {
5317                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5318                 }
5319
5320                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5321             }
5322         }
5323
5324         foreach my $devname (keys %$devinfo) {
5325             die "found no device mapping information for device '$devname'\n"
5326                 if !$devinfo->{$devname}->{virtdev};
5327         }
5328
5329         my $cfg = cfs_read_file('storage.cfg');
5330
5331         # create empty/temp config
5332         if ($oldconf) {
5333             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5334             foreach_drive($oldconf, sub {
5335                 my ($ds, $drive) = @_;
5336
5337                 return if drive_is_cdrom($drive);
5338
5339                 my $volid = $drive->{file};
5340
5341                 return if !$volid || $volid =~ m|^/|;
5342
5343                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5344                 return if !$path || !$owner || ($owner != $vmid);
5345
5346                 # Note: only delete disk we want to restore
5347                 # other volumes will become unused
5348                 if ($virtdev_hash->{$ds}) {
5349                     PVE::Storage::vdisk_free($cfg, $volid);
5350                 }
5351             });
5352         }
5353
5354         my $map = {};
5355         foreach my $virtdev (sort keys %$virtdev_hash) {
5356             my $d = $virtdev_hash->{$virtdev};
5357             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5358             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5359
5360             # test if requested format is supported
5361             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5362             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5363             $d->{format} = $defFormat if !$supported;
5364
5365             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5366                                                   $d->{format}, undef, $alloc_size);
5367             print STDERR "new volume ID is '$volid'\n";
5368             $d->{volid} = $volid;
5369             my $path = PVE::Storage::path($cfg, $volid);
5370
5371             PVE::Storage::activate_volumes($cfg,[$volid]);
5372
5373             my $write_zeros = 1;
5374             # fixme: what other storages types initialize volumes with zero?
5375             if ($scfg->{type} eq 'dir' || $scfg->{type} eq 'nfs' || $scfg->{type} eq 'glusterfs' ||
5376                 $scfg->{type} eq 'sheepdog' || $scfg->{type} eq 'rbd') {
5377                 $write_zeros = 0;
5378             }
5379
5380             print $fifofh "${write_zeros}:$d->{devname}=$path\n";
5381
5382             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5383             $map->{$virtdev} = $volid;
5384         }
5385
5386         $fh->seek(0, 0) || die "seek failed - $!\n";
5387
5388         my $outfd = new IO::File ($tmpfn, "w") ||
5389             die "unable to write config for VM $vmid\n";
5390
5391         my $cookie = { netcount => 0 };
5392         while (defined(my $line = <$fh>)) {
5393             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5394         }
5395
5396         $fh->close();
5397         $outfd->close();
5398     };
5399
5400     eval {
5401         # enable interrupts
5402         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5403             die "interrupted by signal\n";
5404         };
5405         local $SIG{ALRM} = sub { die "got timeout\n"; };
5406
5407         $oldtimeout = alarm($timeout);
5408
5409         my $parser = sub {
5410             my $line = shift;
5411
5412             print "$line\n";
5413
5414             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5415                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5416                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5417             } elsif ($line =~ m/^CTIME: /) {
5418                 # we correctly received the vma config, so we can disable
5419                 # the timeout now for disk allocation (set to 10 minutes, so
5420                 # that we always timeout if something goes wrong)
5421                 alarm(600);
5422                 &$print_devmap();
5423                 print $fifofh "done\n";
5424                 my $tmp = $oldtimeout || 0;
5425                 $oldtimeout = undef;
5426                 alarm($tmp);
5427                 close($fifofh);
5428             }
5429         };
5430
5431         print "restore vma archive: $cmd\n";
5432         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5433     };
5434     my $err = $@;
5435
5436     alarm($oldtimeout) if $oldtimeout;
5437
5438     my $vollist = [];
5439     foreach my $devname (keys %$devinfo) {
5440         my $volid = $devinfo->{$devname}->{volid};
5441         push @$vollist, $volid if $volid;
5442     }
5443
5444     my $cfg = cfs_read_file('storage.cfg');
5445     PVE::Storage::deactivate_volumes($cfg, $vollist);
5446
5447     unlink $mapfifo;
5448
5449     if ($err) {
5450         rmtree $tmpdir;
5451         unlink $tmpfn;
5452
5453         foreach my $devname (keys %$devinfo) {
5454             my $volid = $devinfo->{$devname}->{volid};
5455             next if !$volid;
5456             eval {
5457                 if ($volid =~ m|^/|) {
5458                     unlink $volid || die 'unlink failed\n';
5459                 } else {
5460                     PVE::Storage::vdisk_free($cfg, $volid);
5461                 }
5462                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5463             };
5464             print STDERR "unable to cleanup '$volid' - $@" if $@;
5465         }
5466         die $err;
5467     }
5468
5469     rmtree $tmpdir;
5470
5471     rename($tmpfn, $conffile) ||
5472         die "unable to commit configuration file '$conffile'\n";
5473
5474     PVE::Cluster::cfs_update(); # make sure we read new file
5475
5476     eval { rescan($vmid, 1); };
5477     warn $@ if $@;
5478 }
5479
5480 sub restore_tar_archive {
5481     my ($archive, $vmid, $user, $opts) = @_;
5482
5483     if ($archive ne '-') {
5484         my $firstfile = tar_archive_read_firstfile($archive);
5485         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5486             if $firstfile ne 'qemu-server.conf';
5487     }
5488
5489     my $storecfg = cfs_read_file('storage.cfg');
5490
5491     # destroy existing data - keep empty config
5492     my $vmcfgfn = config_file($vmid);
5493     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5494
5495     my $tocmd = "/usr/lib/qemu-server/qmextract";
5496
5497     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5498     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5499     $tocmd .= ' --prealloc' if $opts->{prealloc};
5500     $tocmd .= ' --info' if $opts->{info};
5501
5502     # tar option "xf" does not autodetect compression when read from STDIN,
5503     # so we pipe to zcat
5504     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5505         PVE::Tools::shellquote("--to-command=$tocmd");
5506
5507     my $tmpdir = "/var/tmp/vzdumptmp$$";
5508     mkpath $tmpdir;
5509
5510     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5511     local $ENV{VZDUMP_VMID} = $vmid;
5512     local $ENV{VZDUMP_USER} = $user;
5513
5514     my $conffile = config_file($vmid);
5515     my $tmpfn = "$conffile.$$.tmp";
5516
5517     # disable interrupts (always do cleanups)
5518     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5519         print STDERR "got interrupt - ignored\n";
5520     };
5521
5522     eval {
5523         # enable interrupts
5524         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5525             die "interrupted by signal\n";
5526         };
5527
5528         if ($archive eq '-') {
5529             print "extracting archive from STDIN\n";
5530             run_command($cmd, input => "<&STDIN");
5531         } else {
5532             print "extracting archive '$archive'\n";
5533             run_command($cmd);
5534         }
5535
5536         return if $opts->{info};
5537
5538         # read new mapping
5539         my $map = {};
5540         my $statfile = "$tmpdir/qmrestore.stat";
5541         if (my $fd = IO::File->new($statfile, "r")) {
5542             while (defined (my $line = <$fd>)) {
5543                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5544                     $map->{$1} = $2 if $1;
5545                 } else {
5546                     print STDERR "unable to parse line in statfile - $line\n";
5547                 }
5548             }
5549             $fd->close();
5550         }
5551
5552         my $confsrc = "$tmpdir/qemu-server.conf";
5553
5554         my $srcfd = new IO::File($confsrc, "r") ||
5555             die "unable to open file '$confsrc'\n";
5556
5557         my $outfd = new IO::File ($tmpfn, "w") ||
5558             die "unable to write config for VM $vmid\n";
5559
5560         my $cookie = { netcount => 0 };
5561         while (defined (my $line = <$srcfd>)) {
5562             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5563         }
5564
5565         $srcfd->close();
5566         $outfd->close();
5567     };
5568     my $err = $@;
5569
5570     if ($err) {
5571
5572         unlink $tmpfn;
5573
5574         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
5575
5576         die $err;
5577     }
5578
5579     rmtree $tmpdir;
5580
5581     rename $tmpfn, $conffile ||
5582         die "unable to commit configuration file '$conffile'\n";
5583
5584     PVE::Cluster::cfs_update(); # make sure we read new file
5585
5586     eval { rescan($vmid, 1); };
5587     warn $@ if $@;
5588 };
5589
5590
5591 # Internal snapshots
5592
5593 # NOTE: Snapshot create/delete involves several non-atomic
5594 # action, and can take a long time.
5595 # So we try to avoid locking the file and use 'lock' variable
5596 # inside the config file instead.
5597
5598 my $snapshot_copy_config = sub {
5599     my ($source, $dest) = @_;
5600
5601     foreach my $k (keys %$source) {
5602         next if $k eq 'snapshots';
5603         next if $k eq 'snapstate';
5604         next if $k eq 'snaptime';
5605         next if $k eq 'vmstate';
5606         next if $k eq 'lock';
5607         next if $k eq 'digest';
5608         next if $k eq 'description';
5609         next if $k =~ m/^unused\d+$/;
5610
5611         $dest->{$k} = $source->{$k};
5612     }
5613 };
5614
5615 my $snapshot_apply_config = sub {
5616     my ($conf, $snap) = @_;
5617
5618     # copy snapshot list
5619     my $newconf = {
5620         snapshots => $conf->{snapshots},
5621     };
5622
5623     # keep description and list of unused disks
5624     foreach my $k (keys %$conf) {
5625         next if !($k =~ m/^unused\d+$/ || $k eq 'description');
5626         $newconf->{$k} = $conf->{$k};
5627     }
5628
5629     &$snapshot_copy_config($snap, $newconf);
5630
5631     return $newconf;
5632 };
5633
5634 sub foreach_writable_storage {
5635     my ($conf, $func) = @_;
5636
5637     my $sidhash = {};
5638
5639     foreach my $ds (keys %$conf) {
5640         next if !valid_drivename($ds);
5641
5642         my $drive = parse_drive($ds, $conf->{$ds});
5643         next if !$drive;
5644         next if drive_is_cdrom($drive);
5645
5646         my $volid = $drive->{file};
5647
5648         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5649         $sidhash->{$sid} = $sid if $sid;
5650     }
5651
5652     foreach my $sid (sort keys %$sidhash) {
5653         &$func($sid);
5654     }
5655 }
5656
5657 my $alloc_vmstate_volid = sub {
5658     my ($storecfg, $vmid, $conf, $snapname) = @_;
5659
5660     # Note: we try to be smart when selecting a $target storage
5661
5662     my $target;
5663
5664     # search shared storage first
5665     foreach_writable_storage($conf, sub {
5666         my ($sid) = @_;
5667         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
5668         return if !$scfg->{shared};
5669
5670         $target = $sid if !$target || $scfg->{path}; # prefer file based storage
5671     });
5672
5673     if (!$target) {
5674         # now search local storage
5675         foreach_writable_storage($conf, sub {
5676             my ($sid) = @_;
5677             my $scfg = PVE::Storage::storage_config($storecfg, $sid);
5678             return if $scfg->{shared};
5679
5680             $target = $sid if !$target || $scfg->{path}; # prefer file based storage;
5681         });
5682     }
5683
5684     $target = 'local' if !$target;
5685
5686     my $driver_state_size = 500; # assume 32MB is enough to safe all driver state;
5687     # we abort live save after $conf->{memory}, so we need at max twice that space
5688     my $size = $conf->{memory}*2 + $driver_state_size;
5689
5690     my $name = "vm-$vmid-state-$snapname";
5691     my $scfg = PVE::Storage::storage_config($storecfg, $target);
5692     $name .= ".raw" if $scfg->{path}; # add filename extension for file base storage
5693     my $volid = PVE::Storage::vdisk_alloc($storecfg, $target, $vmid, 'raw', $name, $size*1024);
5694
5695     return $volid;
5696 };
5697
5698 my $snapshot_prepare = sub {
5699     my ($vmid, $snapname, $save_vmstate, $comment) = @_;
5700
5701     my $snap;
5702
5703     my $updatefn =  sub {
5704
5705         my $conf = load_config($vmid);
5706
5707         die "you can't take a snapshot if it's a template\n"
5708             if is_template($conf);
5709
5710         check_lock($conf);
5711
5712         $conf->{lock} = 'snapshot';
5713
5714         die "snapshot name '$snapname' already used\n"
5715             if defined($conf->{snapshots}->{$snapname});
5716
5717         my $storecfg = PVE::Storage::config();
5718         die "snapshot feature is not available" if !has_feature('snapshot', $conf, $storecfg);
5719
5720         $snap = $conf->{snapshots}->{$snapname} = {};
5721
5722         if ($save_vmstate && check_running($vmid)) {
5723             $snap->{vmstate} = &$alloc_vmstate_volid($storecfg, $vmid, $conf, $snapname);
5724         }
5725
5726         &$snapshot_copy_config($conf, $snap);
5727
5728         $snap->{snapstate} = "prepare";
5729         $snap->{snaptime} = time();
5730         $snap->{description} = $comment if $comment;
5731
5732         # always overwrite machine if we save vmstate. This makes sure we
5733         # can restore it later using correct machine type
5734         $snap->{machine} = get_current_qemu_machine($vmid) if $snap->{vmstate};
5735
5736         update_config_nolock($vmid, $conf, 1);
5737     };
5738
5739     lock_config($vmid, $updatefn);
5740
5741     return $snap;
5742 };
5743
5744 my $snapshot_commit = sub {
5745     my ($vmid, $snapname) = @_;
5746
5747     my $updatefn = sub {
5748
5749         my $conf = load_config($vmid);
5750
5751         die "missing snapshot lock\n"
5752             if !($conf->{lock} && $conf->{lock} eq 'snapshot');
5753
5754         my $has_machine_config = defined($conf->{machine});
5755
5756         my $snap = $conf->{snapshots}->{$snapname};
5757
5758         die "snapshot '$snapname' does not exist\n" if !defined($snap);
5759
5760         die "wrong snapshot state\n"
5761             if !($snap->{snapstate} && $snap->{snapstate} eq "prepare");
5762
5763         delete $snap->{snapstate};
5764         delete $conf->{lock};
5765
5766         my $newconf = &$snapshot_apply_config($conf, $snap);
5767
5768         delete $newconf->{machine} if !$has_machine_config;
5769
5770         $newconf->{parent} = $snapname;
5771
5772         update_config_nolock($vmid, $newconf, 1);
5773     };
5774
5775     lock_config($vmid, $updatefn);
5776 };
5777
5778 sub snapshot_rollback {
5779     my ($vmid, $snapname) = @_;
5780
5781     my $prepare = 1;
5782
5783     my $storecfg = PVE::Storage::config();
5784
5785     my $conf = load_config($vmid);
5786
5787     my $get_snapshot_config = sub {
5788
5789         die "you can't rollback if vm is a template\n" if is_template($conf);
5790
5791         my $res = $conf->{snapshots}->{$snapname};
5792
5793         die "snapshot '$snapname' does not exist\n" if !defined($res);
5794
5795         return $res;
5796     };
5797
5798     my $snap = &$get_snapshot_config();
5799
5800     foreach_drive($snap, sub {
5801         my ($ds, $drive) = @_;
5802
5803         return if drive_is_cdrom($drive);
5804
5805         my $volid = $drive->{file};
5806
5807         PVE::Storage::volume_rollback_is_possible($storecfg, $volid, $snapname);
5808     });
5809
5810     my $updatefn = sub {
5811
5812         $conf = load_config($vmid);
5813
5814         $snap = &$get_snapshot_config();
5815
5816         die "unable to rollback to incomplete snapshot (snapstate = $snap->{snapstate})\n"
5817             if $snap->{snapstate};
5818
5819         if ($prepare) {
5820             check_lock($conf);
5821             vm_stop($storecfg, $vmid, undef, undef, 5, undef, undef);
5822         }
5823
5824         die "unable to rollback vm $vmid: vm is running\n"
5825             if check_running($vmid);
5826
5827         if ($prepare) {
5828             $conf->{lock} = 'rollback';
5829         } else {
5830             die "got wrong lock\n" if !($conf->{lock} && $conf->{lock} eq 'rollback');
5831             delete $conf->{lock};
5832         }
5833
5834         my $forcemachine;
5835
5836         if (!$prepare) {
5837             my $has_machine_config = defined($conf->{machine});
5838
5839             # copy snapshot config to current config
5840             $conf = &$snapshot_apply_config($conf, $snap);
5841             $conf->{parent} = $snapname;
5842
5843             # Note: old code did not store 'machine', so we try to be smart
5844             # and guess the snapshot was generated with kvm 1.4 (pc-i440fx-1.4).
5845             $forcemachine = $conf->{machine} || 'pc-i440fx-1.4';
5846             # we remove the 'machine' configuration if not explicitly specified
5847             # in the original config.
5848             delete $conf->{machine} if $snap->{vmstate} && !$has_machine_config;
5849         }
5850
5851         update_config_nolock($vmid, $conf, 1);
5852
5853         if (!$prepare && $snap->{vmstate}) {
5854             my $statefile = PVE::Storage::path($storecfg, $snap->{vmstate});
5855             vm_start($storecfg, $vmid, $statefile, undef, undef, undef, $forcemachine);
5856         }
5857     };
5858
5859     lock_config($vmid, $updatefn);
5860
5861     foreach_drive($snap, sub {
5862         my ($ds, $drive) = @_;
5863
5864         return if drive_is_cdrom($drive);
5865
5866         my $volid = $drive->{file};
5867         my $device = "drive-$ds";
5868
5869         PVE::Storage::volume_snapshot_rollback($storecfg, $volid, $snapname);
5870     });
5871
5872     $prepare = 0;
5873     lock_config($vmid, $updatefn);
5874 }
5875
5876 my $savevm_wait = sub {
5877     my ($vmid) = @_;
5878
5879     for(;;) {
5880         my $stat = vm_mon_cmd_nocheck($vmid, "query-savevm");
5881         if (!$stat->{status}) {
5882             die "savevm not active\n";
5883         } elsif ($stat->{status} eq 'active') {
5884             sleep(1);
5885             next;
5886         } elsif ($stat->{status} eq 'completed') {
5887             last;
5888         } else {
5889             die "query-savevm returned status '$stat->{status}'\n";
5890         }
5891     }
5892 };
5893
5894 sub do_snapshots_with_qemu {
5895     my ($storecfg, $volid) = @_;
5896
5897     my $storage_name = PVE::Storage::parse_volume_id($volid);
5898
5899     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
5900         && !$storecfg->{ids}->{$storage_name}->{krbd}){
5901         return 1;
5902     }
5903
5904     if ($volid =~ m/\.(qcow2|qed)$/){
5905         return 1;
5906     }
5907
5908     return undef;
5909 }
5910
5911 sub snapshot_create {
5912     my ($vmid, $snapname, $save_vmstate, $comment) = @_;
5913
5914     my $snap = &$snapshot_prepare($vmid, $snapname, $save_vmstate, $comment);
5915
5916     $save_vmstate = 0 if !$snap->{vmstate}; # vm is not running
5917
5918     my $config = load_config($vmid);
5919
5920     my $running = check_running($vmid);
5921
5922     my $freezefs = $running && $config->{agent};
5923     $freezefs = 0 if $snap->{vmstate}; # not needed if we save RAM
5924
5925     my $drivehash = {};
5926
5927     if ($freezefs) {
5928         eval { vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
5929         warn "guest-fsfreeze-freeze problems - $@" if $@;
5930     }
5931
5932     eval {
5933         # create internal snapshots of all drives
5934
5935         my $storecfg = PVE::Storage::config();
5936
5937         if ($running) {
5938             if ($snap->{vmstate}) {
5939                 my $path = PVE::Storage::path($storecfg, $snap->{vmstate});
5940                 vm_mon_cmd($vmid, "savevm-start", statefile => $path);
5941                 &$savevm_wait($vmid);
5942             } else {
5943                 vm_mon_cmd($vmid, "savevm-start");
5944             }
5945         };
5946
5947         foreach_drive($snap, sub {
5948             my ($ds, $drive) = @_;
5949
5950             return if drive_is_cdrom($drive);
5951
5952             my $volid = $drive->{file};
5953             my $device = "drive-$ds";
5954
5955             qemu_volume_snapshot($vmid, $device, $storecfg, $volid, $snapname);
5956             $drivehash->{$ds} = 1;
5957        });
5958     };
5959     my $err = $@;
5960
5961     if ($running) {
5962         eval { vm_mon_cmd($vmid, "savevm-end")  };
5963         warn $@ if $@;
5964
5965         if ($freezefs) {
5966             eval { vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
5967             warn "guest-fsfreeze-thaw problems - $@" if $@;
5968         }
5969
5970         # savevm-end is async, we need to wait
5971         for (;;) {
5972             my $stat = vm_mon_cmd_nocheck($vmid, "query-savevm");
5973             if (!$stat->{bytes}) {
5974                 last;
5975             } else {
5976                 print "savevm not yet finished\n";
5977                 sleep(1);
5978                 next;
5979             }
5980         }
5981     }
5982
5983     if ($err) {
5984         warn "snapshot create failed: starting cleanup\n";
5985         eval { snapshot_delete($vmid, $snapname, 0, $drivehash); };
5986         warn $@ if $@;
5987         die $err;
5988     }
5989
5990     &$snapshot_commit($vmid, $snapname);
5991 }
5992
5993 # Note: $drivehash is only set when called from snapshot_create.
5994 sub snapshot_delete {
5995     my ($vmid, $snapname, $force, $drivehash) = @_;
5996
5997     my $prepare = 1;
5998
5999     my $snap;
6000     my $unused = [];
6001
6002     my $unlink_parent = sub {
6003         my ($confref, $new_parent) = @_;
6004
6005         if ($confref->{parent} && $confref->{parent} eq $snapname) {
6006             if ($new_parent) {
6007                 $confref->{parent} = $new_parent;
6008             } else {
6009                 delete $confref->{parent};
6010             }
6011         }
6012     };
6013
6014     my $updatefn =  sub {
6015         my ($remove_drive) = @_;
6016
6017         my $conf = load_config($vmid);
6018
6019         if (!$drivehash) {
6020             check_lock($conf);
6021             die "you can't delete a snapshot if vm is a template\n"
6022                 if is_template($conf);
6023         }
6024
6025         $snap = $conf->{snapshots}->{$snapname};
6026
6027         die "snapshot '$snapname' does not exist\n" if !defined($snap);
6028
6029         # remove parent refs
6030         if (!$prepare) {
6031             &$unlink_parent($conf, $snap->{parent});
6032             foreach my $sn (keys %{$conf->{snapshots}}) {
6033                 next if $sn eq $snapname;
6034                 &$unlink_parent($conf->{snapshots}->{$sn}, $snap->{parent});
6035             }
6036         }
6037
6038         if ($remove_drive) {
6039             if ($remove_drive eq 'vmstate') {
6040                 delete $snap->{$remove_drive};
6041             } else {
6042                 my $drive = parse_drive($remove_drive, $snap->{$remove_drive});
6043                 my $volid = $drive->{file};
6044                 delete $snap->{$remove_drive};
6045                 add_unused_volume($conf, $volid);
6046             }
6047         }
6048
6049         if ($prepare) {
6050             $snap->{snapstate} = 'delete';
6051         } else {
6052             delete $conf->{snapshots}->{$snapname};
6053             delete $conf->{lock} if $drivehash;
6054             foreach my $volid (@$unused) {
6055                 add_unused_volume($conf, $volid);
6056             }
6057         }
6058
6059         update_config_nolock($vmid, $conf, 1);
6060     };
6061
6062     lock_config($vmid, $updatefn);
6063
6064     # now remove vmstate file
6065
6066     my $storecfg = PVE::Storage::config();
6067
6068     if ($snap->{vmstate}) {
6069         eval {  PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
6070         if (my $err = $@) {
6071             die $err if !$force;
6072             warn $err;
6073         }
6074         # save changes (remove vmstate from snapshot)
6075         lock_config($vmid, $updatefn, 'vmstate') if !$force;
6076     };
6077
6078     # now remove all internal snapshots
6079     foreach_drive($snap, sub {
6080         my ($ds, $drive) = @_;
6081
6082         return if drive_is_cdrom($drive);
6083
6084         my $volid = $drive->{file};
6085         my $device = "drive-$ds";
6086
6087         if (!$drivehash || $drivehash->{$ds}) {
6088             eval { qemu_volume_snapshot_delete($vmid, $device, $storecfg, $volid, $snapname); };
6089             if (my $err = $@) {
6090                 die $err if !$force;
6091                 warn $err;
6092             }
6093         }
6094
6095         # save changes (remove drive fron snapshot)
6096         lock_config($vmid, $updatefn, $ds) if !$force;
6097         push @$unused, $volid;
6098     });
6099
6100     # now cleanup config
6101     $prepare = 0;
6102     lock_config($vmid, $updatefn);
6103 }
6104
6105 sub has_feature {
6106     my ($feature, $conf, $storecfg, $snapname, $running) = @_;
6107
6108     my $err;
6109     foreach_drive($conf, sub {
6110         my ($ds, $drive) = @_;
6111
6112         return if drive_is_cdrom($drive);
6113         my $volid = $drive->{file};
6114         $err = 1 if !PVE::Storage::volume_has_feature($storecfg, $feature, $volid, $snapname, $running);
6115     });
6116
6117     return $err ? 0 : 1;
6118 }
6119
6120 sub template_create {
6121     my ($vmid, $conf, $disk) = @_;
6122
6123     my $storecfg = PVE::Storage::config();
6124
6125     foreach_drive($conf, sub {
6126         my ($ds, $drive) = @_;
6127
6128         return if drive_is_cdrom($drive);
6129         return if $disk && $ds ne $disk;
6130
6131         my $volid = $drive->{file};
6132         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6133
6134         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6135         $drive->{file} = $voliddst;
6136         $conf->{$ds} = print_drive($vmid, $drive);
6137         update_config_nolock($vmid, $conf, 1);
6138     });
6139 }
6140
6141 sub is_template {
6142     my ($conf) = @_;
6143
6144     return 1 if defined $conf->{template} && $conf->{template} == 1;
6145 }
6146
6147 sub qemu_img_convert {
6148     my ($src_volid, $dst_volid, $size, $snapname) = @_;
6149
6150     my $storecfg = PVE::Storage::config();
6151     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6152     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6153
6154     if ($src_storeid && $dst_storeid) {
6155         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6156         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6157
6158         my $src_format = qemu_img_format($src_scfg, $src_volname);
6159         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6160
6161         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6162         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6163
6164         my $cmd = [];
6165         push @$cmd, '/usr/bin/qemu-img', 'convert', '-t', 'writeback', '-p', '-n';
6166         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
6167         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path, $dst_path;
6168
6169         my $parser = sub {
6170             my $line = shift;
6171             if($line =~ m/\((\S+)\/100\%\)/){
6172                 my $percent = $1;
6173                 my $transferred = int($size * $percent / 100);
6174                 my $remaining = $size - $transferred;
6175
6176                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6177             }
6178
6179         };
6180
6181         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6182         my $err = $@;
6183         die "copy failed: $err" if $err;
6184     }
6185 }
6186
6187 sub qemu_img_format {
6188     my ($scfg, $volname) = @_;
6189
6190     if ($scfg->{path} && $volname =~ m/\.(raw|cow|qcow|qcow2|qed|vmdk|cloop)$/) {
6191         return $1;
6192     } else {
6193         return "raw";
6194     }
6195 }
6196
6197 sub qemu_drive_mirror {
6198     my ($vmid, $drive, $dst_volid, $vmiddst) = @_;
6199
6200     my $storecfg = PVE::Storage::config();
6201     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6202
6203     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6204
6205     my $format = qemu_img_format($dst_scfg, $dst_volname);
6206
6207     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6208
6209     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $dst_path };
6210     $opts->{format} = $format if $format;
6211
6212     print "drive mirror is starting (scanning bitmap) : this step can take some minutes/hours, depend of disk size and storage speed\n";
6213
6214     eval {
6215     vm_mon_cmd($vmid, "drive-mirror", %$opts);
6216         while (1) {
6217             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6218             my $stat = @$stats[0];
6219             die "mirroring job seem to have die. Maybe do you have bad sectors?" if !$stat;
6220             die "error job is not mirroring" if $stat->{type} ne "mirror";
6221
6222             my $busy = $stat->{busy};
6223             my $ready = $stat->{ready};
6224
6225             if (my $total = $stat->{len}) {
6226                 my $transferred = $stat->{offset} || 0;
6227                 my $remaining = $total - $transferred;
6228                 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6229
6230                 print "transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6231             }
6232
6233
6234             if ($stat->{ready} eq 'true') {
6235
6236                 last if $vmiddst != $vmid;
6237
6238                 # try to switch the disk if source and destination are on the same guest
6239                 eval { vm_mon_cmd($vmid, "block-job-complete", device => "drive-$drive") };
6240                 last if !$@;
6241                 die $@ if $@ !~ m/cannot be completed/;
6242             }
6243             sleep 1;
6244         }
6245
6246
6247     };
6248     my $err = $@;
6249
6250     my $cancel_job = sub {
6251         vm_mon_cmd($vmid, "block-job-cancel", device => "drive-$drive");
6252         while (1) {
6253             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6254             my $stat = @$stats[0];
6255             last if !$stat;
6256             sleep 1;
6257         }
6258     };
6259
6260     if ($err) {
6261         eval { &$cancel_job(); };
6262         die "mirroring error: $err";
6263     }
6264
6265     if ($vmiddst != $vmid) {
6266         # if we clone a disk for a new target vm, we don't switch the disk
6267         &$cancel_job(); # so we call block-job-cancel
6268     }
6269 }
6270
6271 sub clone_disk {
6272     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6273         $newvmid, $storage, $format, $full, $newvollist) = @_;
6274
6275     my $newvolid;
6276
6277     if (!$full) {
6278         print "create linked clone of drive $drivename ($drive->{file})\n";
6279         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6280         push @$newvollist, $newvolid;
6281     } else {
6282         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6283         $storeid = $storage if $storage;
6284
6285         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6286         if (!$format) {
6287             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6288             $format = qemu_img_format($scfg, $volname);
6289         }
6290
6291         # test if requested format is supported - else use default
6292         my $supported = grep { $_ eq $format } @$validFormats;
6293         $format = $defFormat if !$supported;
6294
6295         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6296
6297         print "create full clone of drive $drivename ($drive->{file})\n";
6298         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $format, undef, ($size/1024));
6299         push @$newvollist, $newvolid;
6300
6301         PVE::Storage::activate_volumes($storecfg, $newvollist);
6302
6303         if (!$running || $snapname) {
6304             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname);
6305         } else {
6306             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid);
6307         }
6308     }
6309
6310     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6311
6312     my $disk = $drive;
6313     $disk->{format} = undef;
6314     $disk->{file} = $newvolid;
6315     $disk->{size} = $size;
6316
6317     return $disk;
6318 }
6319
6320 # this only works if VM is running
6321 sub get_current_qemu_machine {
6322     my ($vmid) = @_;
6323
6324     my $cmd = { execute => 'query-machines', arguments => {} };
6325     my $res = vm_qmp_command($vmid, $cmd);
6326
6327     my ($current, $default);
6328     foreach my $e (@$res) {
6329         $default = $e->{name} if $e->{'is-default'};
6330         $current = $e->{name} if $e->{'is-current'};
6331     }
6332
6333     # fallback to the default machine if current is not supported by qemu
6334     return $current || $default || 'pc';
6335 }
6336
6337 sub qemu_machine_feature_enabled {
6338     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6339
6340     my $current_major;
6341     my $current_minor;
6342
6343     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6344
6345         $current_major = $3;
6346         $current_minor = $4;
6347
6348     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6349
6350         $current_major = $1;
6351         $current_minor = $2;
6352     }
6353
6354     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
6355
6356
6357 }
6358
6359 sub lspci {
6360
6361     my $devices = {};
6362
6363     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6364             my (undef, $id, $function) = @_;
6365             my $res = { id => $id, function => $function};
6366             push @{$devices->{$id}}, $res;
6367     });
6368
6369     return $devices;
6370 }
6371
6372 sub vm_iothreads_list {
6373     my ($vmid) = @_;
6374
6375     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6376
6377     my $iothreads = {};
6378     foreach my $iothread (@$res) {
6379         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6380     }
6381
6382     return $iothreads;
6383 }
6384
6385 sub scsihw_infos {
6386     my ($conf, $drive) = @_;
6387
6388     my $maxdev = 0;
6389
6390     if ($conf->{scsihw} && ($conf->{scsihw} =~ m/^lsi/)) {
6391         $maxdev = 7;
6392     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6393         $maxdev = 1;
6394     } else {
6395         $maxdev = 256;
6396     }
6397
6398     my $controller = int($drive->{index} / $maxdev);
6399     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6400
6401     return ($maxdev, $controller, $controller_prefix);
6402 }
6403
6404 # bash completion helper
6405
6406 sub complete_backup_archives {
6407     my ($cmdname, $pname, $cvalue) = @_;
6408
6409     my $cfg = PVE::Storage::config();
6410
6411     my $storeid;
6412
6413     if ($cvalue =~ m/^([^:]+):/) {
6414         $storeid = $1;
6415     }
6416
6417     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6418
6419     my $res = [];
6420     foreach my $id (keys %$data) {
6421         foreach my $item (@{$data->{$id}}) {
6422             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6423             push @$res, $item->{volid} if defined($item->{volid});
6424         }
6425     }
6426
6427     return $res;
6428 }
6429
6430 my $complete_vmid_full = sub {
6431     my ($running) = @_;
6432
6433     my $idlist = vmstatus();
6434
6435     my $res = [];
6436
6437     foreach my $id (keys %$idlist) {
6438         my $d = $idlist->{$id};
6439         if (defined($running)) {
6440             next if $d->{template};
6441             next if $running && $d->{status} ne 'running';
6442             next if !$running && $d->{status} eq 'running';
6443         }
6444         push @$res, $id;
6445
6446     }
6447     return $res;
6448 };
6449
6450 sub complete_vmid {
6451     return &$complete_vmid_full();
6452 }
6453
6454 sub complete_vmid_stopped {
6455     return &$complete_vmid_full(0);
6456 }
6457
6458 sub complete_vmid_running {
6459     return &$complete_vmid_full(1);
6460 }
6461
6462 sub complete_storage {
6463
6464     my $cfg = PVE::Storage::config();
6465     my $ids = $cfg->{ids};
6466
6467     my $res = [];
6468     foreach my $sid (keys %$ids) {
6469         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6470         push @$res, $sid;
6471     }
6472
6473     return $res;
6474 }
6475
6476 1;