PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QMPClient;
  31 use PVE::RPCEnvironment;
  32 use Time::HiRes qw(gettimeofday);
  33
  34 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  35
  36 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  37
  38 # Note about locking: we use flock on the config file protect
  39 # against concurent actions.
  40 # Aditionaly, we have a 'lock' setting in the config file. This
  41 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  42 # allowed when such lock is set. But you can ignore this kind of
  43 # lock with the --skiplock flag.
  44
  45 cfs_register_file('/qemu-server/',
  46                   \&parse_vm_config,
  47                   \&write_vm_config);
  48
  49 PVE::JSONSchema::register_standard_option('skiplock', {
  50     description => "Ignore locks - only root is allowed to use this option.",
  51     type => 'boolean',
  52     optional => 1,
  53 });
  54
  55 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  56     description => "Some command save/restore state from this location.",
  57     type => 'string',
  58     maxLength => 128,
  59     optional => 1,
  60 });
  61
  62 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  63     description => "The name of the snapshot.",
  64     type => 'string', format => 'pve-configid',
  65     maxLength => 40,
  66 });
  67
  68 #no warnings 'redefine';
  69
  70 sub cgroups_write {
  71    my ($controller, $vmid, $option, $value) = @_;
  72
  73    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  74    PVE::ProcFSTools::write_proc_entry($path, $value);
  75
  76 }
  77
  78 my $nodename = PVE::INotify::nodename();
  79
  80 mkdir "/etc/pve/nodes/$nodename";
  81 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  82 mkdir $confdir;
  83
  84 my $var_run_tmpdir = "/var/run/qemu-server";
  85 mkdir $var_run_tmpdir;
  86
  87 my $lock_dir = "/var/lock/qemu-server";
  88 mkdir $lock_dir;
  89
  90 my $pcisysfs = "/sys/bus/pci";
  91
  92 my $confdesc = {
  93     onboot => {
  94         optional => 1,
  95         type => 'boolean',
  96         description => "Specifies whether a VM will be started during system bootup.",
  97         default => 0,
  98     },
  99     autostart => {
 100         optional => 1,
 101         type => 'boolean',
 102         description => "Automatic restart after crash (currently ignored).",
 103         default => 0,
 104     },
 105     hotplug => {
 106         optional => 1,
 107         type => 'string', format => 'pve-hotplug-features',
 108         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 109         default => 'network,disk,usb',
 110     },
 111     reboot => {
 112         optional => 1,
 113         type => 'boolean',
 114         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 115         default => 1,
 116     },
 117     lock => {
 118         optional => 1,
 119         type => 'string',
 120         description => "Lock/unlock the VM.",
 121         enum => [qw(migrate backup snapshot rollback)],
 122     },
 123     cpulimit => {
 124         optional => 1,
 125         type => 'number',
 126         description => "Limit of CPU usage. Note if the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 127         minimum => 0,
 128         maximum => 128,
 129         default => 0,
 130     },
 131     cpuunits => {
 132         optional => 1,
 133         type => 'integer',
 134         description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 135         minimum => 0,
 136         maximum => 500000,
 137         default => 1000,
 138     },
 139     memory => {
 140         optional => 1,
 141         type => 'integer',
 142         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 143         minimum => 16,
 144         default => 512,
 145     },
 146     balloon => {
 147         optional => 1,
 148         type => 'integer',
 149         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 150         minimum => 0,
 151     },
 152     shares => {
 153         optional => 1,
 154         type => 'integer',
 155         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 156         minimum => 0,
 157         maximum => 50000,
 158         default => 1000,
 159     },
 160     keyboard => {
 161         optional => 1,
 162         type => 'string',
 163         description => "Keybord layout for vnc server. Default is read from the datacenter configuration file.",
 164         enum => PVE::Tools::kvmkeymaplist(),
 165         default => 'en-us',
 166     },
 167     name => {
 168         optional => 1,
 169         type => 'string', format => 'dns-name',
 170         description => "Set a name for the VM. Only used on the configuration web interface.",
 171     },
 172     scsihw => {
 173         optional => 1,
 174         type => 'string',
 175         description => "scsi controller model",
 176         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 177         default => 'lsi',
 178     },
 179     description => {
 180         optional => 1,
 181         type => 'string',
 182         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 183     },
 184     ostype => {
 185         optional => 1,
 186         type => 'string',
 187         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 l24 l26 solaris)],
 188         description => <<EODESC,
 189 Used to enable special optimization/features for specific
 190 operating systems:
 191
 192 other  => unspecified OS
 193 wxp    => Microsoft Windows XP
 194 w2k    => Microsoft Windows 2000
 195 w2k3   => Microsoft Windows 2003
 196 w2k8   => Microsoft Windows 2008
 197 wvista => Microsoft Windows Vista
 198 win7   => Microsoft Windows 7
 199 win8   => Microsoft Windows 8/2012
 200 l24    => Linux 2.4 Kernel
 201 l26    => Linux 2.6/3.X Kernel
 202 solaris => solaris/opensolaris/openindiania kernel
 203
 204 other|l24|l26|solaris                       ... no special behaviour
 205 wxp|w2k|w2k3|w2k8|wvista|win7|win8  ... use --localtime switch
 206 EODESC
 207     },
 208     boot => {
 209         optional => 1,
 210         type => 'string',
 211         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 212         pattern => '[acdn]{1,4}',
 213         default => 'cdn',
 214     },
 215     bootdisk => {
 216         optional => 1,
 217         type => 'string', format => 'pve-qm-bootdisk',
 218         description => "Enable booting from specified disk.",
 219         pattern => '(ide|sata|scsi|virtio)\d+',
 220     },
 221     smp => {
 222         optional => 1,
 223         type => 'integer',
 224         description => "The number of CPUs. Please use option -sockets instead.",
 225         minimum => 1,
 226         default => 1,
 227     },
 228     sockets => {
 229         optional => 1,
 230         type => 'integer',
 231         description => "The number of CPU sockets.",
 232         minimum => 1,
 233         default => 1,
 234     },
 235     cores => {
 236         optional => 1,
 237         type => 'integer',
 238         description => "The number of cores per socket.",
 239         minimum => 1,
 240         default => 1,
 241     },
 242     numa => {
 243         optional => 1,
 244         type => 'boolean',
 245         description => "Enable/disable Numa.",
 246         default => 0,
 247     },
 248     vcpus => {
 249         optional => 1,
 250         type => 'integer',
 251         description => "Number of hotplugged vcpus.",
 252         minimum => 1,
 253         default => 0,
 254     },
 255     acpi => {
 256         optional => 1,
 257         type => 'boolean',
 258         description => "Enable/disable ACPI.",
 259         default => 1,
 260     },
 261     agent => {
 262         optional => 1,
 263         type => 'boolean',
 264         description => "Enable/disable Qemu GuestAgent.",
 265         default => 0,
 266     },
 267     kvm => {
 268         optional => 1,
 269         type => 'boolean',
 270         description => "Enable/disable KVM hardware virtualization.",
 271         default => 1,
 272     },
 273     tdf => {
 274         optional => 1,
 275         type => 'boolean',
 276         description => "Enable/disable time drift fix.",
 277         default => 0,
 278     },
 279     localtime => {
 280         optional => 1,
 281         type => 'boolean',
 282         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 283     },
 284     freeze => {
 285         optional => 1,
 286         type => 'boolean',
 287         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 288     },
 289     vga => {
 290         optional => 1,
 291         type => 'string',
 292         description => "Select VGA type. If you want to use high resolution modes (>= 1280x1024x16) then you should use option 'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and 'cirrur' for other OS types. Option 'qxl' enables the SPICE display sever. You can also run without any graphic card using a serial devive as terminal.",
 293         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 294     },
 295     watchdog => {
 296         optional => 1,
 297         type => 'string', format => 'pve-qm-watchdog',
 298         typetext => '[[model=]i6300esb|ib700] [,[action=]reset|shutdown|poweroff|pause|debug|none]',
 299         description => "Create a virtual hardware watchdog device.  Once enabled (by a guest action), the watchdog must be periodically polled by an agent inside the guest or else the guest will be restarted (or execute the action specified)",
 300     },
 301     startdate => {
 302         optional => 1,
 303         type => 'string',
 304         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 305         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 306         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 307         default => 'now',
 308     },
 309     startup =>  get_standard_option('pve-startup-order'),
 310     template => {
 311         optional => 1,
 312         type => 'boolean',
 313         description => "Enable/disable Template.",
 314         default => 0,
 315     },
 316     args => {
 317         optional => 1,
 318         type => 'string',
 319         description => <<EODESCR,
 320 Note: this option is for experts only. It allows you to pass arbitrary arguments to kvm, for example:
 321
 322 args: -no-reboot -no-hpet
 323 EODESCR
 324     },
 325     tablet => {
 326         optional => 1,
 327         type => 'boolean',
 328         default => 1,
 329         description => "Enable/disable the usb tablet device. This device is usually needed to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with normal VNC clients. If you're running lots of console-only guests on one host, you may consider disabling this to save some context switches. This is turned of by default if you use spice (vga=qxl).",
 330     },
 331     migrate_speed => {
 332         optional => 1,
 333         type => 'integer',
 334         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 335         minimum => 0,
 336         default => 0,
 337     },
 338     migrate_downtime => {
 339         optional => 1,
 340         type => 'number',
 341         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 342         minimum => 0,
 343         default => 0.1,
 344     },
 345     cdrom => {
 346         optional => 1,
 347         type => 'string', format => 'pve-qm-drive',
 348         typetext => 'volume',
 349         description => "This is an alias for option -ide2",
 350     },
 351     cpu => {
 352         optional => 1,
 353         description => "Emulated CPU type.",
 354         type => 'string',
 355         enum => [ qw(486 athlon pentium pentium2 pentium3 coreduo core2duo kvm32 kvm64 qemu32 qemu64 phenom Conroe Penryn Nehalem Westmere SandyBridge IvyBridge Haswell Broadwell Opteron_G1 Opteron_G2 Opteron_G3 Opteron_G4 Opteron_G5 host) ],
 356         default => 'kvm64',
 357     },
 358     parent => get_standard_option('pve-snapshot-name', {
 359         optional => 1,
 360         description => "Parent snapshot name. This is used internally, and should not be modified.",
 361     }),
 362     snaptime => {
 363         optional => 1,
 364         description => "Timestamp for snapshots.",
 365         type => 'integer',
 366         minimum => 0,
 367     },
 368     vmstate => {
 369         optional => 1,
 370         type => 'string', format => 'pve-volume-id',
 371         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 372     },
 373     machine => {
 374         description => "Specific the Qemu machine type.",
 375         type => 'string',
 376         pattern => '(pc|pc(-i440fx)?-\d+\.\d+|q35|pc-q35-\d+\.\d+)',
 377         maxLength => 40,
 378         optional => 1,
 379     },
 380     smbios1 => {
 381         description => "Specify SMBIOS type 1 fields.",
 382         type => 'string', format => 'pve-qm-smbios1',
 383         typetext => "[manufacturer=str][,product=str][,version=str][,serial=str] [,uuid=uuid][,sku=str][,family=str]",
 384         maxLength => 256,
 385         optional => 1,
 386     },
 387     protection => {
 388         optional => 1,
 389         type => 'boolean',
 390         description => "Sets the protection flag of the VM. This will prevent the remove operation.",
 391         default => 0,
 392     },
 393 };
 394
 395 # what about other qemu settings ?
 396 #cpu => 'string',
 397 #machine => 'string',
 398 #fda => 'file',
 399 #fdb => 'file',
 400 #mtdblock => 'file',
 401 #sd => 'file',
 402 #pflash => 'file',
 403 #snapshot => 'bool',
 404 #bootp => 'file',
 405 ##tftp => 'dir',
 406 ##smb => 'dir',
 407 #kernel => 'file',
 408 #append => 'string',
 409 #initrd => 'file',
 410 ##soundhw => 'string',
 411
 412 while (my ($k, $v) = each %$confdesc) {
 413     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 414 }
 415
 416 my $MAX_IDE_DISKS = 4;
 417 my $MAX_SCSI_DISKS = 14;
 418 my $MAX_VIRTIO_DISKS = 16;
 419 my $MAX_SATA_DISKS = 6;
 420 my $MAX_USB_DEVICES = 5;
 421 my $MAX_NETS = 32;
 422 my $MAX_UNUSED_DISKS = 8;
 423 my $MAX_HOSTPCI_DEVICES = 4;
 424 my $MAX_SERIAL_PORTS = 4;
 425 my $MAX_PARALLEL_PORTS = 3;
 426 my $MAX_NUMA = 8;
 427 my $MAX_MEM = 4194304;
 428 my $STATICMEM = 1024;
 429
 430 my $numadesc = {
 431     optional => 1,
 432     type => 'string', format => 'pve-qm-numanode',
 433     typetext => "cpus=<id[-id],memory=<mb>[[,hostnodes=<id[-id]>] [,policy=<preferred|bind|interleave>]]",
 434     description => "numa topology",
 435 };
 436 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 437
 438 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 439     $confdesc->{"numa$i"} = $numadesc;
 440 }
 441
 442 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 443                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 444                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 445 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 446
 447 my $netdesc = {
 448     optional => 1,
 449     type => 'string', format => 'pve-qm-net',
 450     typetext => "MODEL=XX:XX:XX:XX:XX:XX [,bridge=<dev>][,queues=<nbqueues>][,rate=<mbps>] [,tag=<vlanid>][,firewall=0|1],link_down=0|1]",
 451     description => <<EODESCR,
 452 Specify network devices.
 453
 454 MODEL is one of: $nic_model_list_txt
 455
 456 XX:XX:XX:XX:XX:XX should be an unique MAC address. This is
 457 automatically generated if not specified.
 458
 459 The bridge parameter can be used to automatically add the interface to a bridge device. The Proxmox VE standard bridge is called 'vmbr0'.
 460
 461 Option 'rate' is used to limit traffic bandwidth from and to this interface. It is specified as floating point number, unit is 'Megabytes per second'.
 462
 463 If you specify no bridge, we create a kvm 'user' (NATed) network device, which provides DHCP and DNS services. The following addresses are used:
 464
 465 10.0.2.2   Gateway
 466 10.0.2.3   DNS Server
 467 10.0.2.4   SMB Server
 468
 469 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 470
 471 EODESCR
 472 };
 473 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 474
 475 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 476     $confdesc->{"net$i"} = $netdesc;
 477 }
 478
 479 my $drivename_hash;
 480
 481 my $idedesc = {
 482     optional => 1,
 483     type => 'string', format => 'pve-qm-drive',
 484     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads] [,discard=ignore|on] [,serial=serial][,model=model]',
 485     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
 486 };
 487 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
 488
 489 my $scsidesc = {
 490     optional => 1,
 491     type => 'string', format => 'pve-qm-drive',
 492     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads] [,discard=ignore|on] [,iothread=on] [,queues=<nbqueues>] [,serial=serial]',
 493     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
 494 };
 495 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 496
 497 my $satadesc = {
 498     optional => 1,
 499     type => 'string', format => 'pve-qm-drive',
 500     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads]  [,discard=ignore|on] [,serial=serial]',
 501     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
 502 };
 503 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
 504
 505 my $virtiodesc = {
 506     optional => 1,
 507     type => 'string', format => 'pve-qm-drive',
 508     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads]  [,discard=ignore|on] [,iothread=on] [,serial=serial]',
 509     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
 510 };
 511 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
 512
 513 my $usbdesc = {
 514     optional => 1,
 515     type => 'string', format => 'pve-qm-usb-device',
 516     typetext => 'host=HOSTUSBDEVICE|spice',
 517     description => <<EODESCR,
 518 Configure an USB device (n is 0 to 4). This can be used to
 519 pass-through usb devices to the guest. HOSTUSBDEVICE syntax is:
 520
 521 'bus-port(.port)*' (decimal numbers) or
 522 'vendor_id:product_id' (hexadeciaml numbers)
 523
 524 You can use the 'lsusb -t' command to list existing usb devices.
 525
 526 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 527
 528 The value 'spice' can be used to add a usb redirection devices for spice.
 529
 530 EODESCR
 531 };
 532 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
 533
 534 my $hostpcidesc = {
 535         optional => 1,
 536         type => 'string', format => 'pve-qm-hostpci',
 537         typetext => "[host=]HOSTPCIDEVICE [,rombar=on|off] [,pcie=0|1] [,x-vga=on|off]",
 538         description => <<EODESCR,
 539 Map host pci devices. HOSTPCIDEVICE syntax is:
 540
 541 'bus:dev.func' (hexadecimal numbers)
 542
 543 You can us the 'lspci' command to list existing pci devices.
 544
 545 The 'rombar' option determines whether or not the device's ROM will be visible in the guest's memory map (default is 'on').
 546
 547 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 548
 549 Experimental: user reported problems with this option.
 550 EODESCR
 551 };
 552 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
 553
 554 my $serialdesc = {
 555         optional => 1,
 556         type => 'string',
 557         pattern => '(/dev/.+|socket)',
 558         description =>  <<EODESCR,
 559 Create a serial device inside the VM (n is 0 to 3), and pass through a host serial device (i.e. /dev/ttyS0), or create a unix socket on the host side (use 'qm terminal' to open a terminal connection).
 560
 561 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 562
 563 Experimental: user reported problems with this option.
 564 EODESCR
 565 };
 566
 567 my $paralleldesc= {
 568         optional => 1,
 569         type => 'string',
 570         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
 571         description =>  <<EODESCR,
 572 Map host parallel devices (n is 0 to 2).
 573
 574 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 575
 576 Experimental: user reported problems with this option.
 577 EODESCR
 578 };
 579
 580 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
 581     $confdesc->{"parallel$i"} = $paralleldesc;
 582 }
 583
 584 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
 585     $confdesc->{"serial$i"} = $serialdesc;
 586 }
 587
 588 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
 589     $confdesc->{"hostpci$i"} = $hostpcidesc;
 590 }
 591
 592 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
 593     $drivename_hash->{"ide$i"} = 1;
 594     $confdesc->{"ide$i"} = $idedesc;
 595 }
 596
 597 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
 598     $drivename_hash->{"sata$i"} = 1;
 599     $confdesc->{"sata$i"} = $satadesc;
 600 }
 601
 602 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
 603     $drivename_hash->{"scsi$i"} = 1;
 604     $confdesc->{"scsi$i"} = $scsidesc ;
 605 }
 606
 607 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
 608     $drivename_hash->{"virtio$i"} = 1;
 609     $confdesc->{"virtio$i"} = $virtiodesc;
 610 }
 611
 612 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
 613     $confdesc->{"usb$i"} = $usbdesc;
 614 }
 615
 616 my $unuseddesc = {
 617     optional => 1,
 618     type => 'string', format => 'pve-volume-id',
 619     description => "Reference to unused volumes.",
 620 };
 621
 622 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
 623     $confdesc->{"unused$i"} = $unuseddesc;
 624 }
 625
 626 my $kvm_api_version = 0;
 627
 628 sub kvm_version {
 629
 630     return $kvm_api_version if $kvm_api_version;
 631
 632     my $fh = IO::File->new("</dev/kvm") ||
 633         return 0;
 634
 635     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
 636         $kvm_api_version = $v;
 637     }
 638
 639     $fh->close();
 640
 641     return  $kvm_api_version;
 642 }
 643
 644 my $kvm_user_version;
 645
 646 sub kvm_user_version {
 647
 648     return $kvm_user_version if $kvm_user_version;
 649
 650     $kvm_user_version = 'unknown';
 651
 652     my $tmp = `kvm -help 2>/dev/null`;
 653
 654     if ($tmp =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
 655         $kvm_user_version = $2;
 656     }
 657
 658     return $kvm_user_version;
 659
 660 }
 661
 662 my $kernel_has_vhost_net = -c '/dev/vhost-net';
 663
 664 sub disknames {
 665     # order is important - used to autoselect boot disk
 666     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
 667             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
 668             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
 669             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))));
 670 }
 671
 672 sub valid_drivename {
 673     my $dev = shift;
 674
 675     return defined($drivename_hash->{$dev});
 676 }
 677
 678 sub option_exists {
 679     my $key = shift;
 680     return defined($confdesc->{$key});
 681 }
 682
 683 sub nic_models {
 684     return $nic_model_list;
 685 }
 686
 687 sub os_list_description {
 688
 689     return {
 690         other => 'Other',
 691         wxp => 'Windows XP',
 692         w2k => 'Windows 2000',
 693         w2k3 =>, 'Windows 2003',
 694         w2k8 => 'Windows 2008',
 695         wvista => 'Windows Vista',
 696         win7 => 'Windows 7',
 697         win8 => 'Windows 8/2012',
 698         l24 => 'Linux 2.4',
 699         l26 => 'Linux 2.6',
 700     };
 701 }
 702
 703 my $cdrom_path;
 704
 705 sub get_cdrom_path {
 706
 707     return  $cdrom_path if $cdrom_path;
 708
 709     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
 710     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
 711     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
 712 }
 713
 714 sub get_iso_path {
 715     my ($storecfg, $vmid, $cdrom) = @_;
 716
 717     if ($cdrom eq 'cdrom') {
 718         return get_cdrom_path();
 719     } elsif ($cdrom eq 'none') {
 720         return '';
 721     } elsif ($cdrom =~ m|^/|) {
 722         return $cdrom;
 723     } else {
 724         return PVE::Storage::path($storecfg, $cdrom);
 725     }
 726 }
 727
 728 # try to convert old style file names to volume IDs
 729 sub filename_to_volume_id {
 730     my ($vmid, $file, $media) = @_;
 731
 732     if (!($file eq 'none' || $file eq 'cdrom' ||
 733           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
 734
 735         return undef if $file =~ m|/|;
 736
 737         if ($media && $media eq 'cdrom') {
 738             $file = "local:iso/$file";
 739         } else {
 740             $file = "local:$vmid/$file";
 741         }
 742     }
 743
 744     return $file;
 745 }
 746
 747 sub verify_media_type {
 748     my ($opt, $vtype, $media) = @_;
 749
 750     return if !$media;
 751
 752     my $etype;
 753     if ($media eq 'disk') {
 754         $etype = 'images';
 755     } elsif ($media eq 'cdrom') {
 756         $etype = 'iso';
 757     } else {
 758         die "internal error";
 759     }
 760
 761     return if ($vtype eq $etype);
 762
 763     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
 764 }
 765
 766 sub cleanup_drive_path {
 767     my ($opt, $storecfg, $drive) = @_;
 768
 769     # try to convert filesystem paths to volume IDs
 770
 771     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
 772         ($drive->{file} !~ m|^/dev/.+|) &&
 773         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
 774         ($drive->{file} !~ m/^\d+$/)) {
 775         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
 776         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
 777         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
 778         verify_media_type($opt, $vtype, $drive->{media});
 779         $drive->{file} = $volid;
 780     }
 781
 782     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
 783 }
 784
 785 sub create_conf_nolock {
 786     my ($vmid, $settings) = @_;
 787
 788     my $filename = config_file($vmid);
 789
 790     die "configuration file '$filename' already exists\n" if -f $filename;
 791
 792     my $defaults = load_defaults();
 793
 794     $settings->{name} = "vm$vmid" if !$settings->{name};
 795     $settings->{memory} = $defaults->{memory} if !$settings->{memory};
 796
 797     my $data = '';
 798     foreach my $opt (keys %$settings) {
 799         next if !$confdesc->{$opt};
 800
 801         my $value = $settings->{$opt};
 802         next if !$value;
 803
 804         $data .= "$opt: $value\n";
 805     }
 806
 807     PVE::Tools::file_set_contents($filename, $data);
 808 }
 809
 810 sub parse_hotplug_features {
 811     my ($data) = @_;
 812
 813     my $res = {};
 814
 815     return $res if $data eq '0';
 816
 817     $data = $confdesc->{hotplug}->{default} if $data eq '1';
 818
 819     foreach my $feature (PVE::Tools::split_list($data)) {
 820         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
 821             $res->{$1} = 1;
 822         } else {
 823             warn "ignoring unknown hotplug feature '$feature'\n";
 824         }
 825     }
 826     return $res;
 827 }
 828
 829 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
 830 sub pve_verify_hotplug_features {
 831     my ($value, $noerr) = @_;
 832
 833     return $value if parse_hotplug_features($value);
 834
 835     return undef if $noerr;
 836
 837     die "unable to parse hotplug option\n";
 838 }
 839
 840 my $parse_size = sub {
 841     my ($value) = @_;
 842
 843     return undef if $value !~ m/^(\d+(\.\d+)?)([KMG])?$/;
 844     my ($size, $unit) = ($1, $3);
 845     if ($unit) {
 846         if ($unit eq 'K') {
 847             $size = $size * 1024;
 848         } elsif ($unit eq 'M') {
 849             $size = $size * 1024 * 1024;
 850         } elsif ($unit eq 'G') {
 851             $size = $size * 1024 * 1024 * 1024;
 852         }
 853     }
 854     return int($size);
 855 };
 856
 857 my $format_size = sub {
 858     my ($size) = @_;
 859
 860     $size = int($size);
 861
 862     my $kb = int($size/1024);
 863     return $size if $kb*1024 != $size;
 864
 865     my $mb = int($kb/1024);
 866     return "${kb}K" if $mb*1024 != $kb;
 867
 868     my $gb = int($mb/1024);
 869     return "${mb}M" if $gb*1024 != $mb;
 870
 871     return "${gb}G";
 872 };
 873
 874 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
 875 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
 876 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
 877 #        [,aio=native|threads][,discard=ignore|on][,iothread=on]
 878 #        [,serial=serial][,model=model]
 879
 880 sub parse_drive {
 881     my ($key, $data) = @_;
 882
 883     my $res = {};
 884
 885     # $key may be undefined - used to verify JSON parameters
 886     if (!defined($key)) {
 887         $res->{interface} = 'unknown'; # should not harm when used to verify parameters
 888         $res->{index} = 0;
 889     } elsif ($key =~ m/^([^\d]+)(\d+)$/) {
 890         $res->{interface} = $1;
 891         $res->{index} = $2;
 892     } else {
 893         return undef;
 894     }
 895
 896     foreach my $p (split (/,/, $data)) {
 897         next if $p =~ m/^\s*$/;
 898
 899         if ($p =~ m/^(file|volume|cyls|heads|secs|trans|media|snapshot|cache|format|rerror|werror|backup|aio|bps|mbps|mbps_max|bps_rd|mbps_rd|mbps_rd_max|bps_wr|mbps_wr|mbps_wr_max|iops|iops_max|iops_rd|iops_rd_max|iops_wr|iops_wr_max|size|discard|iothread|queues|serial|model)=(.+)$/) {
 900             my ($k, $v) = ($1, $2);
 901
 902             $k = 'file' if $k eq 'volume';
 903
 904             return undef if defined $res->{$k};
 905
 906             if ($k eq 'bps' || $k eq 'bps_rd' || $k eq 'bps_wr') {
 907                 return undef if !$v || $v !~ m/^\d+/;
 908                 $k = "m$k";
 909                 $v = sprintf("%.3f", $v / (1024*1024));
 910             }
 911             $res->{$k} = $v;
 912         } else {
 913             if (!$res->{file} && $p !~ m/=/) {
 914                 $res->{file} = $p;
 915             } else {
 916                 return undef;
 917             }
 918         }
 919     }
 920
 921     return undef if !$res->{file};
 922
 923     return undef if $res->{cache} &&
 924         $res->{cache} !~ m/^(off|none|writethrough|writeback|unsafe|directsync)$/;
 925     return undef if $res->{snapshot} && $res->{snapshot} !~ m/^(on|off)$/;
 926     return undef if $res->{cyls} && $res->{cyls} !~ m/^\d+$/;
 927     return undef if $res->{heads} && $res->{heads} !~ m/^\d+$/;
 928     return undef if $res->{secs} && $res->{secs} !~ m/^\d+$/;
 929     return undef if $res->{media} && $res->{media} !~ m/^(disk|cdrom)$/;
 930     return undef if $res->{trans} && $res->{trans} !~ m/^(none|lba|auto)$/;
 931     return undef if $res->{format} && $res->{format} !~ m/^(raw|cow|qcow|qed|qcow2|vmdk|cloop)$/;
 932     return undef if $res->{rerror} && $res->{rerror} !~ m/^(ignore|report|stop)$/;
 933     return undef if $res->{werror} && $res->{werror} !~ m/^(enospc|ignore|report|stop)$/;
 934     return undef if $res->{backup} && $res->{backup} !~ m/^(yes|no)$/;
 935     return undef if $res->{aio} && $res->{aio} !~ m/^(native|threads)$/;
 936     return undef if $res->{discard} && $res->{discard} !~ m/^(ignore|on)$/;
 937     return undef if $res->{iothread} && $res->{iothread} !~ m/^(on)$/;
 938     return undef if $res->{queues} && ($res->{queues} !~ m/^\d+$/ || $res->{queues} < 2);
 939
 940     return undef if $res->{mbps_rd} && $res->{mbps};
 941     return undef if $res->{mbps_wr} && $res->{mbps};
 942
 943     return undef if $res->{mbps} && $res->{mbps} !~ m/^\d+(\.\d+)?$/;
 944     return undef if $res->{mbps_max} && $res->{mbps_max} !~ m/^\d+(\.\d+)?$/;
 945     return undef if $res->{mbps_rd} && $res->{mbps_rd} !~ m/^\d+(\.\d+)?$/;
 946     return undef if $res->{mbps_rd_max} && $res->{mbps_rd_max} !~ m/^\d+(\.\d+)?$/;
 947     return undef if $res->{mbps_wr} && $res->{mbps_wr} !~ m/^\d+(\.\d+)?$/;
 948     return undef if $res->{mbps_wr_max} && $res->{mbps_wr_max} !~ m/^\d+(\.\d+)?$/;
 949
 950     return undef if $res->{iops_rd} && $res->{iops};
 951     return undef if $res->{iops_wr} && $res->{iops};
 952
 953
 954     return undef if $res->{iops} && $res->{iops} !~ m/^\d+$/;
 955     return undef if $res->{iops_max} && $res->{iops_max} !~ m/^\d+$/;
 956     return undef if $res->{iops_rd} && $res->{iops_rd} !~ m/^\d+$/;
 957     return undef if $res->{iops_rd_max} && $res->{iops_rd_max} !~ m/^\d+$/;
 958     return undef if $res->{iops_wr} && $res->{iops_wr} !~ m/^\d+$/;
 959     return undef if $res->{iops_wr_max} && $res->{iops_wr_max} !~ m/^\d+$/;
 960
 961     if ($res->{size}) {
 962         return undef if !defined($res->{size} = &$parse_size($res->{size}));
 963     }
 964
 965     if ($res->{media} && ($res->{media} eq 'cdrom')) {
 966         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
 967         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
 968         return undef if $res->{interface} eq 'virtio';
 969     }
 970
 971     # rerror does not work with scsi drives
 972     if ($res->{rerror}) {
 973         return undef if $res->{interface} eq 'scsi';
 974     }
 975
 976     return $res;
 977 }
 978
 979 my @qemu_drive_options = qw(heads secs cyls trans media format cache snapshot rerror werror aio discard iops iops_rd iops_wr iops_max iops_rd_max iops_wr_max serial);
 980
 981 sub print_drive {
 982     my ($vmid, $drive) = @_;
 983
 984     my $opts = '';
 985     foreach my $o (@qemu_drive_options, 'mbps', 'mbps_rd', 'mbps_wr', 'mbps_max', 'mbps_rd_max', 'mbps_wr_max', 'backup', 'iothread', 'queues') {
 986         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
 987     }
 988
 989     if ($drive->{size}) {
 990         $opts .= ",size=" . &$format_size($drive->{size});
 991     }
 992
 993     if (my $model = $drive->{model}) {
 994         $opts .= ",model=$model";
 995     }
 996
 997     return "$drive->{file}$opts";
 998 }
 999
1000 sub scsi_inquiry {
1001     my($fh, $noerr) = @_;
1002
1003     my $SG_IO = 0x2285;
1004     my $SG_GET_VERSION_NUM = 0x2282;
1005
1006     my $versionbuf = "\x00" x 8;
1007     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1008     if (!$ret) {
1009         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1010         return undef;
1011     }
1012     my $version = unpack("I", $versionbuf);
1013     if ($version < 30000) {
1014         die "scsi generic interface too old\n"  if !$noerr;
1015         return undef;
1016     }
1017
1018     my $buf = "\x00" x 36;
1019     my $sensebuf = "\x00" x 8;
1020     my $cmd = pack("C x3 C x1", 0x12, 36);
1021
1022     # see /usr/include/scsi/sg.h
1023     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1024
1025     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1026                       length($sensebuf), 0, length($buf), $buf,
1027                       $cmd, $sensebuf, 6000);
1028
1029     $ret = ioctl($fh, $SG_IO, $packet);
1030     if (!$ret) {
1031         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1032         return undef;
1033     }
1034
1035     my @res = unpack($sg_io_hdr_t, $packet);
1036     if ($res[17] || $res[18]) {
1037         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1038         return undef;
1039     }
1040
1041     my $res = {};
1042     (my $byte0, my $byte1, $res->{vendor},
1043      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1044
1045     $res->{removable} = $byte1 & 128 ? 1 : 0;
1046     $res->{type} = $byte0 & 31;
1047
1048     return $res;
1049 }
1050
1051 sub path_is_scsi {
1052     my ($path) = @_;
1053
1054     my $fh = IO::File->new("+<$path") || return undef;
1055     my $res = scsi_inquiry($fh, 1);
1056     close($fh);
1057
1058     return $res;
1059 }
1060
1061 sub machine_type_is_q35 {
1062     my ($conf) = @_;
1063
1064     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1065 }
1066
1067 sub print_tabletdevice_full {
1068     my ($conf) = @_;
1069
1070     my $q35 = machine_type_is_q35($conf);
1071
1072     # we use uhci for old VMs because tablet driver was buggy in older qemu
1073     my $usbbus = $q35 ? "ehci" : "uhci";
1074
1075     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1076 }
1077
1078 sub print_drivedevice_full {
1079     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1080
1081     my $device = '';
1082     my $maxdev = 0;
1083
1084     if ($drive->{interface} eq 'virtio') {
1085         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1086         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1087         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1088     } elsif ($drive->{interface} eq 'scsi') {
1089
1090         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1091         my $unit = $drive->{index} % $maxdev;
1092         my $devicetype = 'hd';
1093         my $path = '';
1094         if (drive_is_cdrom($drive)) {
1095               $devicetype = 'cd';
1096         } else {
1097               if ($drive->{file} =~ m|^/|) {
1098                   $path = $drive->{file};
1099               } else {
1100                   $path = PVE::Storage::path($storecfg, $drive->{file});
1101               }
1102
1103               if($path =~ m/^iscsi\:\/\//){
1104                   $devicetype = 'generic';
1105               } else {
1106                   if (my $info = path_is_scsi($path)) {
1107                       if ($info->{type} == 0) {
1108                           $devicetype = 'block';
1109                       } elsif ($info->{type} == 1) { # tape
1110                           $devicetype = 'generic';
1111                       }
1112                   }
1113               }
1114          }
1115
1116         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1117             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1118         } else {
1119             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1120         }
1121
1122     } elsif ($drive->{interface} eq 'ide'){
1123         $maxdev = 2;
1124         my $controller = int($drive->{index} / $maxdev);
1125         my $unit = $drive->{index} % $maxdev;
1126         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1127
1128         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1129         if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
1130             $device .= ",model=$model";
1131         }
1132     } elsif ($drive->{interface} eq 'sata'){
1133         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1134         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1135         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1136     } elsif ($drive->{interface} eq 'usb') {
1137         die "implement me";
1138         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1139     } else {
1140         die "unsupported interface type";
1141     }
1142
1143     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1144
1145     return $device;
1146 }
1147
1148 sub get_initiator_name {
1149     my $initiator;
1150
1151     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1152     while (defined(my $line = <$fh>)) {
1153         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1154         $initiator = $1;
1155         last;
1156     }
1157     $fh->close();
1158
1159     return $initiator;
1160 }
1161
1162 sub print_drive_full {
1163     my ($storecfg, $vmid, $drive) = @_;
1164
1165     my $path;
1166     my $volid = $drive->{file};
1167     my $format;
1168
1169     if (drive_is_cdrom($drive)) {
1170         $path = get_iso_path($storecfg, $vmid, $volid);
1171     } else {
1172         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1173         if ($storeid) {
1174             $path = PVE::Storage::path($storecfg, $volid);
1175             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1176             $format = qemu_img_format($scfg, $volname);
1177         } else {
1178             $path = $volid;
1179         }
1180    }
1181
1182     my $opts = '';
1183     foreach my $o (@qemu_drive_options) {
1184         next if $o eq 'bootindex';
1185         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1186     }
1187
1188     $opts .= ",format=$format" if $format && !$drive->{format};
1189
1190     foreach my $o (qw(bps bps_rd bps_wr)) {
1191         my $v = $drive->{"m$o"};
1192         $opts .= ",$o=" . int($v*1024*1024) if $v;
1193     }
1194
1195     my $cache_direct = 0;
1196
1197     if (my $cache = $drive->{cache}) {
1198         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1199     } elsif (!drive_is_cdrom($drive)) {
1200         $opts .= ",cache=none";
1201         $cache_direct = 1;
1202     }
1203
1204     # aio native works only with O_DIRECT
1205     if (!$drive->{aio}) {
1206         if($cache_direct) {
1207             $opts .= ",aio=native";
1208         } else {
1209             $opts .= ",aio=threads";
1210         }
1211     }
1212
1213     my $detectzeroes = $drive->{discard} ? "unmap" : "on";
1214     $opts .= ",detect-zeroes=$detectzeroes" if !drive_is_cdrom($drive);
1215
1216     my $pathinfo = $path ? "file=$path," : '';
1217
1218     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1219 }
1220
1221 sub print_netdevice_full {
1222     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1223
1224     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1225
1226     my $device = $net->{model};
1227     if ($net->{model} eq 'virtio') {
1228          $device = 'virtio-net-pci';
1229      };
1230
1231     my $pciaddr = print_pci_addr("$netid", $bridges);
1232     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1233     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1234         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1235         my $vectors = $net->{queues} * 2 + 2;
1236         $tmpstr .= ",vectors=$vectors,mq=on";
1237     }
1238     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1239
1240     if ($use_old_bios_files) {
1241         my $romfile;
1242         if ($device eq 'virtio-net-pci') {
1243             $romfile = 'pxe-virtio.rom';
1244         } elsif ($device eq 'e1000') {
1245             $romfile = 'pxe-e1000.rom';
1246         } elsif ($device eq 'ne2k') {
1247             $romfile = 'pxe-ne2k_pci.rom';
1248         } elsif ($device eq 'pcnet') {
1249             $romfile = 'pxe-pcnet.rom';
1250         } elsif ($device eq 'rtl8139') {
1251             $romfile = 'pxe-rtl8139.rom';
1252         }
1253         $tmpstr .= ",romfile=$romfile" if $romfile;
1254     }
1255
1256     return $tmpstr;
1257 }
1258
1259 sub print_netdev_full {
1260     my ($vmid, $conf, $net, $netid) = @_;
1261
1262     my $i = '';
1263     if ($netid =~ m/^net(\d+)$/) {
1264         $i = int($1);
1265     }
1266
1267     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1268
1269     my $ifname = "tap${vmid}i$i";
1270
1271     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1272     die "interface name '$ifname' is too long (max 15 character)\n"
1273         if length($ifname) >= 16;
1274
1275     my $vhostparam = '';
1276     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1277
1278     my $vmname = $conf->{name} || "vm$vmid";
1279
1280     my $netdev = "";
1281
1282     if ($net->{bridge}) {
1283         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/pve-bridge,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1284     } else {
1285         $netdev = "type=user,id=$netid,hostname=$vmname";
1286     }
1287
1288     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1289
1290     return $netdev;
1291 }
1292
1293 sub drive_is_cdrom {
1294     my ($drive) = @_;
1295
1296     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1297
1298 }
1299
1300 sub parse_numa {
1301     my ($data) = @_;
1302
1303     my $res = {};
1304
1305     foreach my $kvp (split(/,/, $data)) {
1306
1307         if ($kvp =~ m/^memory=(\S+)$/) {
1308             $res->{memory} = $1;
1309         } elsif ($kvp =~ m/^policy=(preferred|bind|interleave)$/) {
1310             $res->{policy} = $1;
1311         } elsif ($kvp =~ m/^cpus=(\d+)(-(\d+))?$/) {
1312             $res->{cpus}->{start} = $1;
1313             $res->{cpus}->{end} = $3;
1314         } elsif ($kvp =~ m/^hostnodes=(\d+)(-(\d+))?$/) {
1315             $res->{hostnodes}->{start} = $1;
1316             $res->{hostnodes}->{end} = $3;
1317         } else {
1318             return undef;
1319         }
1320     }
1321
1322     return $res;
1323 }
1324
1325 sub parse_hostpci {
1326     my ($value) = @_;
1327
1328     return undef if !$value;
1329
1330
1331     my @list = split(/,/, $value);
1332     my $found;
1333
1334     my $res = {};
1335     foreach my $kv (@list) {
1336
1337         if ($kv =~ m/^(host=)?([a-f0-9]{2}:[a-f0-9]{2})(\.([a-f0-9]))?$/) {
1338             $found = 1;
1339             if(defined($4)){
1340                 push @{$res->{pciid}}, { id => $2 , function => $4};
1341
1342             }else{
1343                 my $pcidevices = lspci($2);
1344                 $res->{pciid} = $pcidevices->{$2};
1345             }
1346         } elsif ($kv =~ m/^rombar=(on|off)$/) {
1347             $res->{rombar} = $1;
1348         } elsif ($kv =~ m/^x-vga=(on|off)$/) {
1349             $res->{'x-vga'} = $1;
1350         } elsif ($kv =~ m/^pcie=(\d+)$/) {
1351             $res->{pcie} = 1 if $1 == 1;
1352         } else {
1353             warn "unknown hostpci setting '$kv'\n";
1354         }
1355     }
1356
1357     return undef if !$found;
1358
1359     return $res;
1360 }
1361
1362 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1363 sub parse_net {
1364     my ($data) = @_;
1365
1366     my $res = {};
1367
1368     foreach my $kvp (split(/,/, $data)) {
1369
1370         if ($kvp =~ m/^(ne2k_pci|e1000|e1000-82540em|e1000-82544gc|e1000-82545em|rtl8139|pcnet|virtio|ne2k_isa|i82551|i82557b|i82559er|vmxnet3)(=([0-9a-f]{2}(:[0-9a-f]{2}){5}))?$/i) {
1371             my $model = lc($1);
1372             my $mac = defined($3) ? uc($3) : PVE::Tools::random_ether_addr();
1373             $res->{model} = $model;
1374             $res->{macaddr} = $mac;
1375         } elsif ($kvp =~ m/^bridge=(\S+)$/) {
1376             $res->{bridge} = $1;
1377         } elsif ($kvp =~ m/^queues=(\d+)$/) {
1378             $res->{queues} = $1;
1379         } elsif ($kvp =~ m/^rate=(\d+(\.\d+)?)$/) {
1380             $res->{rate} = $1;
1381         } elsif ($kvp =~ m/^tag=(\d+)$/) {
1382             $res->{tag} = $1;
1383         } elsif ($kvp =~ m/^firewall=([01])$/) {
1384             $res->{firewall} = $1;
1385         } elsif ($kvp =~ m/^link_down=([01])$/) {
1386             $res->{link_down} = $1;
1387         } else {
1388             return undef;
1389         }
1390
1391     }
1392
1393     return undef if !$res->{model};
1394
1395     return $res;
1396 }
1397
1398 sub print_net {
1399     my $net = shift;
1400
1401     my $res = "$net->{model}";
1402     $res .= "=$net->{macaddr}" if $net->{macaddr};
1403     $res .= ",bridge=$net->{bridge}" if $net->{bridge};
1404     $res .= ",rate=$net->{rate}" if $net->{rate};
1405     $res .= ",tag=$net->{tag}" if $net->{tag};
1406     $res .= ",firewall=1" if $net->{firewall};
1407     $res .= ",link_down=1" if $net->{link_down};
1408     $res .= ",queues=$net->{queues}" if $net->{queues};
1409
1410     return $res;
1411 }
1412
1413 sub add_random_macs {
1414     my ($settings) = @_;
1415
1416     foreach my $opt (keys %$settings) {
1417         next if $opt !~ m/^net(\d+)$/;
1418         my $net = parse_net($settings->{$opt});
1419         next if !$net;
1420         $settings->{$opt} = print_net($net);
1421     }
1422 }
1423
1424 sub add_unused_volume {
1425     my ($config, $volid) = @_;
1426
1427     my $key;
1428     for (my $ind = $MAX_UNUSED_DISKS - 1; $ind >= 0; $ind--) {
1429         my $test = "unused$ind";
1430         if (my $vid = $config->{$test}) {
1431             return if $vid eq $volid; # do not add duplicates
1432         } else {
1433             $key = $test;
1434         }
1435     }
1436
1437     die "To many unused volume - please delete them first.\n" if !$key;
1438
1439     $config->{$key} = $volid;
1440
1441     return $key;
1442 }
1443
1444 sub vm_is_volid_owner {
1445     my ($storecfg, $vmid, $volid) = @_;
1446
1447     if ($volid !~  m|^/|) {
1448         my ($path, $owner);
1449         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1450         if ($owner && ($owner == $vmid)) {
1451             return 1;
1452         }
1453     }
1454
1455     return undef;
1456 }
1457
1458 sub split_flagged_list {
1459     my $text = shift || '';
1460     $text =~ s/[,;]/ /g;
1461     $text =~ s/^\s+//;
1462     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
1463 }
1464
1465 sub join_flagged_list {
1466     my ($how, $lst) = @_;
1467     join $how, map { $lst->{$_} . $_ } keys %$lst;
1468 }
1469
1470 sub vmconfig_delete_pending_option {
1471     my ($conf, $key, $force) = @_;
1472
1473     delete $conf->{pending}->{$key};
1474     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1475     $pending_delete_hash->{$key} = $force ? '!' : '';
1476     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1477 }
1478
1479 sub vmconfig_undelete_pending_option {
1480     my ($conf, $key) = @_;
1481
1482     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1483     delete $pending_delete_hash->{$key};
1484
1485     if (%$pending_delete_hash) {
1486         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1487     } else {
1488         delete $conf->{pending}->{delete};
1489     }
1490 }
1491
1492 sub vmconfig_register_unused_drive {
1493     my ($storecfg, $vmid, $conf, $drive) = @_;
1494
1495     if (!drive_is_cdrom($drive)) {
1496         my $volid = $drive->{file};
1497         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1498             add_unused_volume($conf, $volid, $vmid);
1499         }
1500     }
1501 }
1502
1503 sub vmconfig_cleanup_pending {
1504     my ($conf) = @_;
1505
1506     # remove pending changes when nothing changed
1507     my $changes;
1508     foreach my $opt (keys %{$conf->{pending}}) {
1509         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
1510             $changes = 1;
1511             delete $conf->{pending}->{$opt};
1512         }
1513     }
1514
1515     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
1516     my $pending_delete_hash = {};
1517     while (my ($opt, $force) = each %$current_delete_hash) {
1518         if (defined($conf->{$opt})) {
1519             $pending_delete_hash->{$opt} = $force;
1520         } else {
1521             $changes = 1;
1522         }
1523     }
1524
1525     if (%$pending_delete_hash) {
1526         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1527     } else {
1528         delete $conf->{pending}->{delete};
1529     }
1530
1531     return $changes;
1532 }
1533
1534 my $valid_smbios1_options = {
1535     manufacturer => '\S+',
1536     product => '\S+',
1537     version => '\S+',
1538     serial => '\S+',
1539     uuid => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1540     sku => '\S+',
1541     family => '\S+',
1542 };
1543
1544 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
1545 sub parse_smbios1 {
1546     my ($data) = @_;
1547
1548     my $res = {};
1549
1550     foreach my $kvp (split(/,/, $data)) {
1551         return undef if $kvp !~ m/^(\S+)=(.+)$/;
1552         my ($k, $v) = split(/=/, $kvp);
1553         return undef if !defined($k) || !defined($v);
1554         return undef if !$valid_smbios1_options->{$k};
1555         return undef if $v !~ m/^$valid_smbios1_options->{$k}$/;
1556         $res->{$k} = $v;
1557     }
1558
1559     return $res;
1560 }
1561
1562 sub print_smbios1 {
1563     my ($smbios1) = @_;
1564
1565     my $data = '';
1566     foreach my $k (keys %$smbios1) {
1567         next if !defined($smbios1->{$k});
1568         next if !$valid_smbios1_options->{$k};
1569         $data .= ',' if $data;
1570         $data .= "$k=$smbios1->{$k}";
1571     }
1572     return $data;
1573 }
1574
1575 PVE::JSONSchema::register_format('pve-qm-smbios1', \&verify_smbios1);
1576 sub verify_smbios1 {
1577     my ($value, $noerr) = @_;
1578
1579     return $value if parse_smbios1($value);
1580
1581     return undef if $noerr;
1582
1583     die "unable to parse smbios (type 1) options\n";
1584 }
1585
1586 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
1587 sub verify_bootdisk {
1588     my ($value, $noerr) = @_;
1589
1590     return $value if valid_drivename($value);
1591
1592     return undef if $noerr;
1593
1594     die "invalid boot disk '$value'\n";
1595 }
1596
1597 PVE::JSONSchema::register_format('pve-qm-numanode', \&verify_numa);
1598 sub verify_numa {
1599     my ($value, $noerr) = @_;
1600
1601     return $value if parse_numa($value);
1602
1603     return undef if $noerr;
1604
1605     die "unable to parse numa options\n";
1606 }
1607
1608 PVE::JSONSchema::register_format('pve-qm-net', \&verify_net);
1609 sub verify_net {
1610     my ($value, $noerr) = @_;
1611
1612     return $value if parse_net($value);
1613
1614     return undef if $noerr;
1615
1616     die "unable to parse network options\n";
1617 }
1618
1619 PVE::JSONSchema::register_format('pve-qm-drive', \&verify_drive);
1620 sub verify_drive {
1621     my ($value, $noerr) = @_;
1622
1623     return $value if parse_drive(undef, $value);
1624
1625     return undef if $noerr;
1626
1627     die "unable to parse drive options\n";
1628 }
1629
1630 PVE::JSONSchema::register_format('pve-qm-hostpci', \&verify_hostpci);
1631 sub verify_hostpci {
1632     my ($value, $noerr) = @_;
1633
1634     return $value if parse_hostpci($value);
1635
1636     return undef if $noerr;
1637
1638     die "unable to parse pci id\n";
1639 }
1640
1641 PVE::JSONSchema::register_format('pve-qm-watchdog', \&verify_watchdog);
1642 sub verify_watchdog {
1643     my ($value, $noerr) = @_;
1644
1645     return $value if parse_watchdog($value);
1646
1647     return undef if $noerr;
1648
1649     die "unable to parse watchdog options\n";
1650 }
1651
1652 sub parse_watchdog {
1653     my ($value) = @_;
1654
1655     return undef if !$value;
1656
1657     my $res = {};
1658
1659     foreach my $p (split(/,/, $value)) {
1660         next if $p =~ m/^\s*$/;
1661
1662         if ($p =~ m/^(model=)?(i6300esb|ib700)$/) {
1663             $res->{model} = $2;
1664         } elsif ($p =~ m/^(action=)?(reset|shutdown|poweroff|pause|debug|none)$/) {
1665             $res->{action} = $2;
1666         } else {
1667             return undef;
1668         }
1669     }
1670
1671     return $res;
1672 }
1673
1674 sub parse_usb_device {
1675     my ($value) = @_;
1676
1677     return undef if !$value;
1678
1679     my @dl = split(/,/, $value);
1680     my $found;
1681
1682     my $res = {};
1683     foreach my $v (@dl) {
1684         if ($v =~ m/^host=(0x)?([0-9A-Fa-f]{4}):(0x)?([0-9A-Fa-f]{4})$/) {
1685             $found = 1;
1686             $res->{vendorid} = $2;
1687             $res->{productid} = $4;
1688         } elsif ($v =~ m/^host=(\d+)\-(\d+(\.\d+)*)$/) {
1689             $found = 1;
1690             $res->{hostbus} = $1;
1691             $res->{hostport} = $2;
1692         } elsif ($v =~ m/^spice$/) {
1693             $found = 1;
1694             $res->{spice} = 1;
1695         } else {
1696             return undef;
1697         }
1698     }
1699     return undef if !$found;
1700
1701     return $res;
1702 }
1703
1704 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
1705 sub verify_usb_device {
1706     my ($value, $noerr) = @_;
1707
1708     return $value if parse_usb_device($value);
1709
1710     return undef if $noerr;
1711
1712     die "unable to parse usb device\n";
1713 }
1714
1715 # add JSON properties for create and set function
1716 sub json_config_properties {
1717     my $prop = shift;
1718
1719     foreach my $opt (keys %$confdesc) {
1720         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
1721         $prop->{$opt} = $confdesc->{$opt};
1722     }
1723
1724     return $prop;
1725 }
1726
1727 sub check_type {
1728     my ($key, $value) = @_;
1729
1730     die "unknown setting '$key'\n" if !$confdesc->{$key};
1731
1732     my $type = $confdesc->{$key}->{type};
1733
1734     if (!defined($value)) {
1735         die "got undefined value\n";
1736     }
1737
1738     if ($value =~ m/[\n\r]/) {
1739         die "property contains a line feed\n";
1740     }
1741
1742     if ($type eq 'boolean') {
1743         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
1744         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
1745         die "type check ('boolean') failed - got '$value'\n";
1746     } elsif ($type eq 'integer') {
1747         return int($1) if $value =~ m/^(\d+)$/;
1748         die "type check ('integer') failed - got '$value'\n";
1749     } elsif ($type eq 'number') {
1750         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
1751         die "type check ('number') failed - got '$value'\n";
1752     } elsif ($type eq 'string') {
1753         if (my $fmt = $confdesc->{$key}->{format}) {
1754             if ($fmt eq 'pve-qm-drive') {
1755                 # special case - we need to pass $key to parse_drive()
1756                 my $drive = parse_drive($key, $value);
1757                 return $value if $drive;
1758                 die "unable to parse drive options\n";
1759             }
1760             PVE::JSONSchema::check_format($fmt, $value);
1761             return $value;
1762         }
1763         $value =~ s/^\"(.*)\"$/$1/;
1764         return $value;
1765     } else {
1766         die "internal error"
1767     }
1768 }
1769
1770 sub lock_config_full {
1771     my ($vmid, $timeout, $code, @param) = @_;
1772
1773     my $filename = config_file_lock($vmid);
1774
1775     my $res = lock_file($filename, $timeout, $code, @param);
1776
1777     die $@ if $@;
1778
1779     return $res;
1780 }
1781
1782 sub lock_config_mode {
1783     my ($vmid, $timeout, $shared, $code, @param) = @_;
1784
1785     my $filename = config_file_lock($vmid);
1786
1787     my $res = lock_file_full($filename, $timeout, $shared, $code, @param);
1788
1789     die $@ if $@;
1790
1791     return $res;
1792 }
1793
1794 sub lock_config {
1795     my ($vmid, $code, @param) = @_;
1796
1797     return lock_config_full($vmid, 10, $code, @param);
1798 }
1799
1800 sub cfs_config_path {
1801     my ($vmid, $node) = @_;
1802
1803     $node = $nodename if !$node;
1804     return "nodes/$node/qemu-server/$vmid.conf";
1805 }
1806
1807 sub check_iommu_support{
1808     #fixme : need to check IOMMU support
1809     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
1810
1811     my $iommu=1;
1812     return $iommu;
1813
1814 }
1815
1816 sub config_file {
1817     my ($vmid, $node) = @_;
1818
1819     my $cfspath = cfs_config_path($vmid, $node);
1820     return "/etc/pve/$cfspath";
1821 }
1822
1823 sub config_file_lock {
1824     my ($vmid) = @_;
1825
1826     return "$lock_dir/lock-$vmid.conf";
1827 }
1828
1829 sub touch_config {
1830     my ($vmid) = @_;
1831
1832     my $conf = config_file($vmid);
1833     utime undef, undef, $conf;
1834 }
1835
1836 sub destroy_vm {
1837     my ($storecfg, $vmid, $keep_empty_config) = @_;
1838
1839     my $conffile = config_file($vmid);
1840
1841     my $conf = load_config($vmid);
1842
1843     check_lock($conf);
1844
1845     # only remove disks owned by this VM
1846     foreach_drive($conf, sub {
1847         my ($ds, $drive) = @_;
1848
1849         return if drive_is_cdrom($drive);
1850
1851         my $volid = $drive->{file};
1852
1853         return if !$volid || $volid =~ m|^/|;
1854
1855         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
1856         return if !$path || !$owner || ($owner != $vmid);
1857
1858         PVE::Storage::vdisk_free($storecfg, $volid);
1859     });
1860
1861     if ($keep_empty_config) {
1862         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
1863     } else {
1864         unlink $conffile;
1865     }
1866
1867     # also remove unused disk
1868     eval {
1869         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
1870
1871         eval {
1872             PVE::Storage::foreach_volid($dl, sub {
1873                 my ($volid, $sid, $volname, $d) = @_;
1874                 PVE::Storage::vdisk_free($storecfg, $volid);
1875             });
1876         };
1877         warn $@ if $@;
1878
1879     };
1880     warn $@ if $@;
1881 }
1882
1883 sub load_config {
1884     my ($vmid, $node) = @_;
1885
1886     my $cfspath = cfs_config_path($vmid, $node);
1887
1888     my $conf = PVE::Cluster::cfs_read_file($cfspath);
1889
1890     die "no such VM ('$vmid')\n" if !defined($conf);
1891
1892     return $conf;
1893 }
1894
1895 sub parse_vm_config {
1896     my ($filename, $raw) = @_;
1897
1898     return undef if !defined($raw);
1899
1900     my $res = {
1901         digest => Digest::SHA::sha1_hex($raw),
1902         snapshots => {},
1903         pending => {},
1904     };
1905
1906     $filename =~ m|/qemu-server/(\d+)\.conf$|
1907         || die "got strange filename '$filename'";
1908
1909     my $vmid = $1;
1910
1911     my $conf = $res;
1912     my $descr;
1913     my $section = '';
1914
1915     my @lines = split(/\n/, $raw);
1916     foreach my $line (@lines) {
1917         next if $line =~ m/^\s*$/;
1918
1919         if ($line =~ m/^\[PENDING\]\s*$/i) {
1920             $section = 'pending';
1921             if (defined($descr)) {
1922                 $descr =~ s/\s+$//;
1923                 $conf->{description} = $descr;
1924             }
1925             $descr = undef;
1926             $conf = $res->{$section} = {};
1927             next;
1928
1929         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
1930             $section = $1;
1931             if (defined($descr)) {
1932                 $descr =~ s/\s+$//;
1933                 $conf->{description} = $descr;
1934             }
1935             $descr = undef;
1936             $conf = $res->{snapshots}->{$section} = {};
1937             next;
1938         }
1939
1940         if ($line =~ m/^\#(.*)\s*$/) {
1941             $descr = '' if !defined($descr);
1942             $descr .= PVE::Tools::decode_text($1) . "\n";
1943             next;
1944         }
1945
1946         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
1947             $descr = '' if !defined($descr);
1948             $descr .= PVE::Tools::decode_text($2);
1949         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
1950             $conf->{snapstate} = $1;
1951         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
1952             my $key = $1;
1953             my $value = $2;
1954             $conf->{$key} = $value;
1955         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
1956             my $value = $1;
1957             if ($section eq 'pending') {
1958                 $conf->{delete} = $value; # we parse this later
1959             } else {
1960                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
1961             }
1962         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
1963             my $key = $1;
1964             my $value = $2;
1965             eval { $value = check_type($key, $value); };
1966             if ($@) {
1967                 warn "vm $vmid - unable to parse value of '$key' - $@";
1968             } else {
1969                 my $fmt = $confdesc->{$key}->{format};
1970                 if ($fmt && $fmt eq 'pve-qm-drive') {
1971                     my $v = parse_drive($key, $value);
1972                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
1973                         $v->{file} = $volid;
1974                         $value = print_drive($vmid, $v);
1975                     } else {
1976                         warn "vm $vmid - unable to parse value of '$key'\n";
1977                         next;
1978                     }
1979                 }
1980
1981                 if ($key eq 'cdrom') {
1982                     $conf->{ide2} = $value;
1983                 } else {
1984                     $conf->{$key} = $value;
1985                 }
1986             }
1987         }
1988     }
1989
1990     if (defined($descr)) {
1991         $descr =~ s/\s+$//;
1992         $conf->{description} = $descr;
1993     }
1994     delete $res->{snapstate}; # just to be sure
1995
1996     return $res;
1997 }
1998
1999 sub write_vm_config {
2000     my ($filename, $conf) = @_;
2001
2002     delete $conf->{snapstate}; # just to be sure
2003
2004     if ($conf->{cdrom}) {
2005         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2006         $conf->{ide2} = $conf->{cdrom};
2007         delete $conf->{cdrom};
2008     }
2009
2010     # we do not use 'smp' any longer
2011     if ($conf->{sockets}) {
2012         delete $conf->{smp};
2013     } elsif ($conf->{smp}) {
2014         $conf->{sockets} = $conf->{smp};
2015         delete $conf->{cores};
2016         delete $conf->{smp};
2017     }
2018
2019     my $used_volids = {};
2020
2021     my $cleanup_config = sub {
2022         my ($cref, $pending, $snapname) = @_;
2023
2024         foreach my $key (keys %$cref) {
2025             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2026                 $key eq 'snapstate' || $key eq 'pending';
2027             my $value = $cref->{$key};
2028             if ($key eq 'delete') {
2029                 die "propertry 'delete' is only allowed in [PENDING]\n"
2030                     if !$pending;
2031                 # fixme: check syntax?
2032                 next;
2033             }
2034             eval { $value = check_type($key, $value); };
2035             die "unable to parse value of '$key' - $@" if $@;
2036
2037             $cref->{$key} = $value;
2038
2039             if (!$snapname && valid_drivename($key)) {
2040                 my $drive = parse_drive($key, $value);
2041                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2042             }
2043         }
2044     };
2045
2046     &$cleanup_config($conf);
2047
2048     &$cleanup_config($conf->{pending}, 1);
2049
2050     foreach my $snapname (keys %{$conf->{snapshots}}) {
2051         die "internal error" if $snapname eq 'pending';
2052         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2053     }
2054
2055     # remove 'unusedX' settings if we re-add a volume
2056     foreach my $key (keys %$conf) {
2057         my $value = $conf->{$key};
2058         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2059             delete $conf->{$key};
2060         }
2061     }
2062
2063     my $generate_raw_config = sub {
2064         my ($conf, $pending) = @_;
2065
2066         my $raw = '';
2067
2068         # add description as comment to top of file
2069         if (defined(my $descr = $conf->{description})) {
2070             if ($descr) {
2071                 foreach my $cl (split(/\n/, $descr)) {
2072                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2073                 }
2074             } else {
2075                 $raw .= "#\n" if $pending;
2076             }
2077         }
2078
2079         foreach my $key (sort keys %$conf) {
2080             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2081             $raw .= "$key: $conf->{$key}\n";
2082         }
2083         return $raw;
2084     };
2085
2086     my $raw = &$generate_raw_config($conf);
2087
2088     if (scalar(keys %{$conf->{pending}})){
2089         $raw .= "\n[PENDING]\n";
2090         $raw .= &$generate_raw_config($conf->{pending}, 1);
2091     }
2092
2093     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2094         $raw .= "\n[$snapname]\n";
2095         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2096     }
2097
2098     return $raw;
2099 }
2100
2101 sub update_config_nolock {
2102     my ($vmid, $conf, $skiplock) = @_;
2103
2104     check_lock($conf) if !$skiplock;
2105
2106     my $cfspath = cfs_config_path($vmid);
2107
2108     PVE::Cluster::cfs_write_file($cfspath, $conf);
2109 }
2110
2111 sub update_config {
2112     my ($vmid, $conf, $skiplock) = @_;
2113
2114     lock_config($vmid, &update_config_nolock, $conf, $skiplock);
2115 }
2116
2117 sub load_defaults {
2118
2119     my $res = {};
2120
2121     # we use static defaults from our JSON schema configuration
2122     foreach my $key (keys %$confdesc) {
2123         if (defined(my $default = $confdesc->{$key}->{default})) {
2124             $res->{$key} = $default;
2125         }
2126     }
2127
2128     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2129     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2130
2131     return $res;
2132 }
2133
2134 sub config_list {
2135     my $vmlist = PVE::Cluster::get_vmlist();
2136     my $res = {};
2137     return $res if !$vmlist || !$vmlist->{ids};
2138     my $ids = $vmlist->{ids};
2139
2140     foreach my $vmid (keys %$ids) {
2141         my $d = $ids->{$vmid};
2142         next if !$d->{node} || $d->{node} ne $nodename;
2143         next if !$d->{type} || $d->{type} ne 'qemu';
2144         $res->{$vmid}->{exists} = 1;
2145     }
2146     return $res;
2147 }
2148
2149 # test if VM uses local resources (to prevent migration)
2150 sub check_local_resources {
2151     my ($conf, $noerr) = @_;
2152
2153     my $loc_res = 0;
2154
2155     $loc_res = 1 if $conf->{hostusb}; # old syntax
2156     $loc_res = 1 if $conf->{hostpci}; # old syntax
2157
2158     foreach my $k (keys %$conf) {
2159         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2160         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2161     }
2162
2163     die "VM uses local resources\n" if $loc_res && !$noerr;
2164
2165     return $loc_res;
2166 }
2167
2168 # check if used storages are available on all nodes (use by migrate)
2169 sub check_storage_availability {
2170     my ($storecfg, $conf, $node) = @_;
2171
2172     foreach_drive($conf, sub {
2173         my ($ds, $drive) = @_;
2174
2175         my $volid = $drive->{file};
2176         return if !$volid;
2177
2178         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2179         return if !$sid;
2180
2181         # check if storage is available on both nodes
2182         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2183         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2184    });
2185 }
2186
2187 # list nodes where all VM images are available (used by has_feature API)
2188 sub shared_nodes {
2189     my ($conf, $storecfg) = @_;
2190
2191     my $nodelist = PVE::Cluster::get_nodelist();
2192     my $nodehash = { map { $_ => 1 } @$nodelist };
2193     my $nodename = PVE::INotify::nodename();
2194
2195     foreach_drive($conf, sub {
2196         my ($ds, $drive) = @_;
2197
2198         my $volid = $drive->{file};
2199         return if !$volid;
2200
2201         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2202         if ($storeid) {
2203             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2204             if ($scfg->{disable}) {
2205                 $nodehash = {};
2206             } elsif (my $avail = $scfg->{nodes}) {
2207                 foreach my $node (keys %$nodehash) {
2208                     delete $nodehash->{$node} if !$avail->{$node};
2209                 }
2210             } elsif (!$scfg->{shared}) {
2211                 foreach my $node (keys %$nodehash) {
2212                     delete $nodehash->{$node} if $node ne $nodename
2213                 }
2214             }
2215         }
2216     });
2217
2218     return $nodehash
2219 }
2220
2221 sub check_lock {
2222     my ($conf) = @_;
2223
2224     die "VM is locked ($conf->{lock})\n" if $conf->{lock};
2225 }
2226
2227 sub check_cmdline {
2228     my ($pidfile, $pid) = @_;
2229
2230     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2231     if (defined($fh)) {
2232         my $line = <$fh>;
2233         $fh->close;
2234         return undef if !$line;
2235         my @param = split(/\0/, $line);
2236
2237         my $cmd = $param[0];
2238         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2239
2240         for (my $i = 0; $i < scalar (@param); $i++) {
2241             my $p = $param[$i];
2242             next if !$p;
2243             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2244                 my $p = $param[$i+1];
2245                 return 1 if $p && ($p eq $pidfile);
2246                 return undef;
2247             }
2248         }
2249     }
2250     return undef;
2251 }
2252
2253 sub check_running {
2254     my ($vmid, $nocheck, $node) = @_;
2255
2256     my $filename = config_file($vmid, $node);
2257
2258     die "unable to find configuration file for VM $vmid - no such machine\n"
2259         if !$nocheck && ! -f $filename;
2260
2261     my $pidfile = pidfile_name($vmid);
2262
2263     if (my $fd = IO::File->new("<$pidfile")) {
2264         my $st = stat($fd);
2265         my $line = <$fd>;
2266         close($fd);
2267
2268         my $mtime = $st->mtime;
2269         if ($mtime > time()) {
2270             warn "file '$filename' modified in future\n";
2271         }
2272
2273         if ($line =~ m/^(\d+)$/) {
2274             my $pid = $1;
2275             if (check_cmdline($pidfile, $pid)) {
2276                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2277                     return $pid;
2278                 }
2279             }
2280         }
2281     }
2282
2283     return undef;
2284 }
2285
2286 sub vzlist {
2287
2288     my $vzlist = config_list();
2289
2290     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2291
2292     while (defined(my $de = $fd->read)) {
2293         next if $de !~ m/^(\d+)\.pid$/;
2294         my $vmid = $1;
2295         next if !defined($vzlist->{$vmid});
2296         if (my $pid = check_running($vmid)) {
2297             $vzlist->{$vmid}->{pid} = $pid;
2298         }
2299     }
2300
2301     return $vzlist;
2302 }
2303
2304 sub disksize {
2305     my ($storecfg, $conf) = @_;
2306
2307     my $bootdisk = $conf->{bootdisk};
2308     return undef if !$bootdisk;
2309     return undef if !valid_drivename($bootdisk);
2310
2311     return undef if !$conf->{$bootdisk};
2312
2313     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2314     return undef if !defined($drive);
2315
2316     return undef if drive_is_cdrom($drive);
2317
2318     my $volid = $drive->{file};
2319     return undef if !$volid;
2320
2321     return $drive->{size};
2322 }
2323
2324 my $last_proc_pid_stat;
2325
2326 # get VM status information
2327 # This must be fast and should not block ($full == false)
2328 # We only query KVM using QMP if $full == true (this can be slow)
2329 sub vmstatus {
2330     my ($opt_vmid, $full) = @_;
2331
2332     my $res = {};
2333
2334     my $storecfg = PVE::Storage::config();
2335
2336     my $list = vzlist();
2337     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2338
2339     my $cpucount = $cpuinfo->{cpus} || 1;
2340
2341     foreach my $vmid (keys %$list) {
2342         next if $opt_vmid && ($vmid ne $opt_vmid);
2343
2344         my $cfspath = cfs_config_path($vmid);
2345         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2346
2347         my $d = {};
2348         $d->{pid} = $list->{$vmid}->{pid};
2349
2350         # fixme: better status?
2351         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2352
2353         my $size = disksize($storecfg, $conf);
2354         if (defined($size)) {
2355             $d->{disk} = 0; # no info available
2356             $d->{maxdisk} = $size;
2357         } else {
2358             $d->{disk} = 0;
2359             $d->{maxdisk} = 0;
2360         }
2361
2362         $d->{cpus} = ($conf->{sockets} || 1) * ($conf->{cores} || 1);
2363         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2364         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2365
2366         $d->{name} = $conf->{name} || "VM $vmid";
2367         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024) : 0;
2368
2369         if ($conf->{balloon}) {
2370             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2371             $d->{shares} = defined($conf->{shares}) ? $conf->{shares} : 1000;
2372         }
2373
2374         $d->{uptime} = 0;
2375         $d->{cpu} = 0;
2376         $d->{mem} = 0;
2377
2378         $d->{netout} = 0;
2379         $d->{netin} = 0;
2380
2381         $d->{diskread} = 0;
2382         $d->{diskwrite} = 0;
2383
2384         $d->{template} = is_template($conf);
2385
2386         $res->{$vmid} = $d;
2387     }
2388
2389     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2390     foreach my $dev (keys %$netdev) {
2391         next if $dev !~ m/^tap([1-9]\d*)i/;
2392         my $vmid = $1;
2393         my $d = $res->{$vmid};
2394         next if !$d;
2395
2396         $d->{netout} += $netdev->{$dev}->{receive};
2397         $d->{netin} += $netdev->{$dev}->{transmit};
2398
2399         if ($full) {
2400             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2401             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2402         }
2403
2404     }
2405
2406     my $ctime = gettimeofday;
2407
2408     foreach my $vmid (keys %$list) {
2409
2410         my $d = $res->{$vmid};
2411         my $pid = $d->{pid};
2412         next if !$pid;
2413
2414         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2415         next if !$pstat; # not running
2416
2417         my $used = $pstat->{utime} + $pstat->{stime};
2418
2419         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2420
2421         if ($pstat->{vsize}) {
2422             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2423         }
2424
2425         my $old = $last_proc_pid_stat->{$pid};
2426         if (!$old) {
2427             $last_proc_pid_stat->{$pid} = {
2428                 time => $ctime,
2429                 used => $used,
2430                 cpu => 0,
2431             };
2432             next;
2433         }
2434
2435         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2436
2437         if ($dtime > 1000) {
2438             my $dutime = $used -  $old->{used};
2439
2440             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2441             $last_proc_pid_stat->{$pid} = {
2442                 time => $ctime,
2443                 used => $used,
2444                 cpu => $d->{cpu},
2445             };
2446         } else {
2447             $d->{cpu} = $old->{cpu};
2448         }
2449     }
2450
2451     return $res if !$full;
2452
2453     my $qmpclient = PVE::QMPClient->new();
2454
2455     my $ballooncb = sub {
2456         my ($vmid, $resp) = @_;
2457
2458         my $info = $resp->{'return'};
2459         return if !$info->{max_mem};
2460
2461         my $d = $res->{$vmid};
2462
2463         # use memory assigned to VM
2464         $d->{maxmem} = $info->{max_mem};
2465         $d->{balloon} = $info->{actual};
2466
2467         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2468             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2469             $d->{freemem} = $info->{free_mem};
2470         }
2471
2472         $d->{ballooninfo} = $info;
2473     };
2474
2475     my $blockstatscb = sub {
2476         my ($vmid, $resp) = @_;
2477         my $data = $resp->{'return'} || [];
2478         my $totalrdbytes = 0;
2479         my $totalwrbytes = 0;
2480
2481         for my $blockstat (@$data) {
2482             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2483             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2484
2485             $blockstat->{device} =~ s/drive-//;
2486             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2487         }
2488         $res->{$vmid}->{diskread} = $totalrdbytes;
2489         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2490     };
2491
2492     my $statuscb = sub {
2493         my ($vmid, $resp) = @_;
2494
2495         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2496         # this fails if ballon driver is not loaded, so this must be
2497         # the last commnand (following command are aborted if this fails).
2498         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2499
2500         my $status = 'unknown';
2501         if (!defined($status = $resp->{'return'}->{status})) {
2502             warn "unable to get VM status\n";
2503             return;
2504         }
2505
2506         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2507     };
2508
2509     foreach my $vmid (keys %$list) {
2510         next if $opt_vmid && ($vmid ne $opt_vmid);
2511         next if !$res->{$vmid}->{pid}; # not running
2512         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2513     }
2514
2515     $qmpclient->queue_execute(undef, 1);
2516
2517     foreach my $vmid (keys %$list) {
2518         next if $opt_vmid && ($vmid ne $opt_vmid);
2519         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2520     }
2521
2522     return $res;
2523 }
2524
2525 sub foreach_dimm {
2526     my ($conf, $vmid, $memory, $sockets, $func) = @_;
2527
2528     my $dimm_id = 0;
2529     my $current_size = 1024;
2530     my $dimm_size = 512;
2531     return if $current_size == $memory;
2532
2533     for (my $j = 0; $j < 8; $j++) {
2534         for (my $i = 0; $i < 32; $i++) {
2535             my $name = "dimm${dimm_id}";
2536             $dimm_id++;
2537             my $numanode = $i % $sockets;
2538             $current_size += $dimm_size;
2539             &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
2540             return  $current_size if $current_size >= $memory;
2541         }
2542         $dimm_size *= 2;
2543     }
2544 }
2545
2546 sub foreach_reverse_dimm {
2547     my ($conf, $vmid, $memory, $sockets, $func) = @_;
2548
2549     my $dimm_id = 253;
2550     my $current_size = 4177920;
2551     my $dimm_size = 65536;
2552     return if $current_size == $memory;
2553
2554     for (my $j = 0; $j < 8; $j++) {
2555         for (my $i = 0; $i < 32; $i++) {
2556             my $name = "dimm${dimm_id}";
2557             $dimm_id--;
2558             my $numanode = $i % $sockets;
2559             $current_size -= $dimm_size;
2560             &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
2561             return  $current_size if $current_size <= $memory;
2562         }
2563         $dimm_size /= 2;
2564     }
2565 }
2566
2567 sub foreach_drive {
2568     my ($conf, $func) = @_;
2569
2570     foreach my $ds (keys %$conf) {
2571         next if !valid_drivename($ds);
2572
2573         my $drive = parse_drive($ds, $conf->{$ds});
2574         next if !$drive;
2575
2576         &$func($ds, $drive);
2577     }
2578 }
2579
2580 sub foreach_volid {
2581     my ($conf, $func) = @_;
2582
2583     my $volhash = {};
2584
2585     my $test_volid = sub {
2586         my ($volid, $is_cdrom) = @_;
2587
2588         return if !$volid;
2589
2590         $volhash->{$volid} = $is_cdrom || 0;
2591     };
2592
2593     foreach_drive($conf, sub {
2594         my ($ds, $drive) = @_;
2595         &$test_volid($drive->{file}, drive_is_cdrom($drive));
2596     });
2597
2598     foreach my $snapname (keys %{$conf->{snapshots}}) {
2599         my $snap = $conf->{snapshots}->{$snapname};
2600         &$test_volid($snap->{vmstate}, 0);
2601         foreach_drive($snap, sub {
2602             my ($ds, $drive) = @_;
2603             &$test_volid($drive->{file}, drive_is_cdrom($drive));
2604         });
2605     }
2606
2607     foreach my $volid (keys %$volhash) {
2608         &$func($volid, $volhash->{$volid});
2609     }
2610 }
2611
2612 sub vga_conf_has_spice {
2613     my ($vga) = @_;
2614
2615     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
2616
2617     return $1 || 1;
2618 }
2619
2620 sub config_to_command {
2621     my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $use_old_bios_files) = @_;
2622
2623     my $cmd = [];
2624     my $globalFlags = [];
2625     my $machineFlags = [];
2626     my $rtcFlags = [];
2627     my $cpuFlags = [];
2628     my $devices = [];
2629     my $pciaddr = '';
2630     my $bridges = {};
2631     my $kvmver = kvm_user_version();
2632     my $vernum = 0; # unknown
2633     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
2634         $vernum = $1*1000000+$2*1000;
2635     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
2636         $vernum = $1*1000000+$2*1000+$3;
2637     }
2638
2639     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
2640
2641     my $have_ovz = -f '/proc/vz/vestat';
2642
2643     my $q35 = machine_type_is_q35($conf);
2644     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2645     my $machine_type = $forcemachine || $conf->{machine};
2646
2647     my $cpuunits = defined($conf->{cpuunits}) ?
2648             $conf->{cpuunits} : $defaults->{cpuunits};
2649
2650     push @$cmd, '/usr/bin/systemd-run';
2651     push @$cmd, '--scope';
2652     push @$cmd, '--slice', "qemu";
2653     push @$cmd, '--unit', $vmid;
2654     push @$cmd, '-p', "CPUShares=$cpuunits";
2655     if ($conf->{cpulimit}) {
2656         my $cpulimit = int($conf->{cpulimit} * 100);
2657         push @$cmd, '-p', "CPUQuota=$cpulimit\%";
2658     }
2659
2660     push @$cmd, '/usr/bin/kvm';
2661
2662     push @$cmd, '-id', $vmid;
2663
2664     my $use_virtio = 0;
2665
2666     my $qmpsocket = qmp_socket($vmid);
2667     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2668     push @$cmd, '-mon', "chardev=qmp,mode=control";
2669
2670     my $socket = vnc_socket($vmid);
2671     push @$cmd,  '-vnc', "unix:$socket,x509,password";
2672
2673     push @$cmd, '-pidfile' , pidfile_name($vmid);
2674
2675     push @$cmd, '-daemonize';
2676
2677     if ($conf->{smbios1}) {
2678         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
2679     }
2680
2681     if ($q35) {
2682         # the q35 chipset support native usb2, so we enable usb controller
2683         # by default for this machine type
2684         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
2685     } else {
2686         $pciaddr = print_pci_addr("piix3", $bridges);
2687         push @$devices, '-device', "piix3-usb-uhci,id=uhci$pciaddr.0x2";
2688
2689         my $use_usb2 = 0;
2690         for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2691             next if !$conf->{"usb$i"};
2692             $use_usb2 = 1;
2693         }
2694         # include usb device config
2695         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-usb.cfg' if $use_usb2;
2696     }
2697
2698     my $vga = $conf->{vga};
2699
2700     my $qxlnum = vga_conf_has_spice($vga);
2701     $vga = 'qxl' if $qxlnum;
2702
2703     if (!$vga) {
2704         if ($conf->{ostype} && ($conf->{ostype} eq 'win8' ||
2705                                 $conf->{ostype} eq 'win7' ||
2706                                 $conf->{ostype} eq 'w2k8')) {
2707             $vga = 'std';
2708         } else {
2709             $vga = 'cirrus';
2710         }
2711     }
2712
2713     # enable absolute mouse coordinates (needed by vnc)
2714     my $tablet;
2715     if (defined($conf->{tablet})) {
2716         $tablet = $conf->{tablet};
2717     } else {
2718         $tablet = $defaults->{tablet};
2719         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
2720         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
2721     }
2722
2723     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
2724
2725     # host pci devices
2726     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
2727         my $d = parse_hostpci($conf->{"hostpci$i"});
2728         next if !$d;
2729
2730         my $pcie = $d->{pcie};
2731         if($pcie){
2732             die "q35 machine model is not enabled" if !$q35;
2733             $pciaddr = print_pcie_addr("hostpci$i");
2734         }else{
2735             $pciaddr = print_pci_addr("hostpci$i", $bridges);
2736         }
2737
2738         my $rombar = $d->{rombar} && $d->{rombar} eq 'off' ? ",rombar=0" : "";
2739         my $xvga = $d->{'x-vga'} && $d->{'x-vga'} eq 'on' ? ",x-vga=on" : "";
2740         if ($xvga && $xvga ne '') {
2741             push @$cpuFlags, 'kvm=off';
2742             $vga = 'none';
2743         }
2744         my $pcidevices = $d->{pciid};
2745         my $multifunction = 1 if @$pcidevices > 1;
2746
2747         my $j=0;
2748         foreach my $pcidevice (@$pcidevices) {
2749
2750             my $id = "hostpci$i";
2751             $id .= ".$j" if $multifunction;
2752             my $addr = $pciaddr;
2753             $addr .= ".$j" if $multifunction;
2754             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
2755
2756             if($j == 0){
2757                 $devicestr .= "$rombar$xvga";
2758                 $devicestr .= ",multifunction=on" if $multifunction;
2759             }
2760
2761             push @$devices, '-device', $devicestr;
2762             $j++;
2763         }
2764     }
2765
2766     # usb devices
2767     for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2768         my $d = parse_usb_device($conf->{"usb$i"});
2769         next if !$d;
2770         if ($d->{vendorid} && $d->{productid}) {
2771             push @$devices, '-device', "usb-host,vendorid=0x$d->{vendorid},productid=0x$d->{productid}";
2772         } elsif (defined($d->{hostbus}) && defined($d->{hostport})) {
2773             push @$devices, '-device', "usb-host,hostbus=$d->{hostbus},hostport=$d->{hostport}";
2774         } elsif ($d->{spice}) {
2775             # usb redir support for spice
2776             push @$devices, '-chardev', "spicevmc,id=usbredirchardev$i,name=usbredir";
2777             push @$devices, '-device', "usb-redir,chardev=usbredirchardev$i,id=usbredirdev$i,bus=ehci.0";
2778         }
2779     }
2780
2781     # serial devices
2782     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2783         if (my $path = $conf->{"serial$i"}) {
2784             if ($path eq 'socket') {
2785                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
2786                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
2787                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2788             } else {
2789                 die "no such serial device\n" if ! -c $path;
2790                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
2791                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2792             }
2793         }
2794     }
2795
2796     # parallel devices
2797     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
2798         if (my $path = $conf->{"parallel$i"}) {
2799             die "no such parallel device\n" if ! -c $path;
2800             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
2801             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
2802             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
2803         }
2804     }
2805
2806     my $vmname = $conf->{name} || "vm$vmid";
2807
2808     push @$cmd, '-name', $vmname;
2809
2810     my $sockets = 1;
2811     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
2812     $sockets = $conf->{sockets} if  $conf->{sockets};
2813
2814     my $cores = $conf->{cores} || 1;
2815
2816     my $maxcpus = $sockets * $cores;
2817
2818     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
2819
2820     my $allowed_vcpus = $cpuinfo->{cpus};
2821
2822     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
2823         if ($allowed_vcpus < $maxcpus);
2824
2825     push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
2826
2827     push @$cmd, '-nodefaults';
2828
2829     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2830
2831     my $bootindex_hash = {};
2832     my $i = 1;
2833     foreach my $o (split(//, $bootorder)) {
2834         $bootindex_hash->{$o} = $i*100;
2835         $i++;
2836     }
2837
2838     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000";
2839
2840     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
2841
2842     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
2843
2844     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
2845
2846     # time drift fix
2847     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
2848
2849     my $nokvm = defined($conf->{kvm}) && $conf->{kvm} == 0 ? 1 : 0;
2850     my $useLocaltime = $conf->{localtime};
2851
2852     if (my $ost = $conf->{ostype}) {
2853         # other, wxp, w2k, w2k3, w2k8, wvista, win7, win8, l24, l26, solaris
2854
2855         if ($ost =~ m/^w/) { # windows
2856             $useLocaltime = 1 if !defined($conf->{localtime});
2857
2858             # use time drift fix when acpi is enabled
2859             if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
2860                 $tdf = 1 if !defined($conf->{tdf});
2861             }
2862         }
2863
2864         if ($ost eq 'win7' || $ost eq 'win8' || $ost eq 'w2k8' ||
2865             $ost eq 'wvista') {
2866             push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
2867             push @$cmd, '-no-hpet';
2868             if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
2869                 push @$cpuFlags , 'hv_spinlocks=0x1fff' if !$nokvm;
2870                 push @$cpuFlags , 'hv_vapic' if !$nokvm;
2871                 push @$cpuFlags , 'hv_time' if !$nokvm;
2872
2873             } else {
2874                 push @$cpuFlags , 'hv_spinlocks=0xffff' if !$nokvm;
2875             }
2876         }
2877
2878         if ($ost eq 'win7' || $ost eq 'win8') {
2879             push @$cpuFlags , 'hv_relaxed' if !$nokvm;
2880         }
2881     }
2882
2883     push @$rtcFlags, 'driftfix=slew' if $tdf;
2884
2885     if ($nokvm) {
2886         push @$machineFlags, 'accel=tcg';
2887     } else {
2888         die "No accelerator found!\n" if !$cpuinfo->{hvm};
2889     }
2890
2891     if ($machine_type) {
2892         push @$machineFlags, "type=${machine_type}";
2893     }
2894
2895     if ($conf->{startdate}) {
2896         push @$rtcFlags, "base=$conf->{startdate}";
2897     } elsif ($useLocaltime) {
2898         push @$rtcFlags, 'base=localtime';
2899     }
2900
2901     my $cpu = $nokvm ? "qemu64" : "kvm64";
2902     $cpu = $conf->{cpu} if $conf->{cpu};
2903
2904     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
2905
2906     push @$cpuFlags , '-x2apic'
2907         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
2908
2909     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
2910
2911     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
2912
2913     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
2914
2915         push @$cpuFlags , '+kvm_pv_unhalt' if !$nokvm;
2916         push @$cpuFlags , '+kvm_pv_eoi' if !$nokvm;
2917         push @$cpuFlags , '-kvm_steal_time' if !$nokvm;
2918     }
2919
2920     push @$cpuFlags, 'enforce' if $cpu ne 'host' && !$nokvm;
2921
2922     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
2923
2924     push @$cmd, '-cpu', $cpu;
2925
2926     my $memory = $conf->{memory} || $defaults->{memory};
2927     my $static_memory = 0;
2928     my $dimm_memory = 0;
2929
2930     if ($hotplug_features->{memory}) {
2931         die "Numa need to be enabled for memory hotplug\n" if !$conf->{numa};
2932         die "Total memory is bigger than ${MAX_MEM}MB\n" if $memory > $MAX_MEM;
2933         $static_memory = $STATICMEM;
2934         die "minimum memory must be ${static_memory}MB\n" if($memory < $static_memory);
2935         $dimm_memory = $memory - $static_memory;
2936         push @$cmd, '-m', "size=${static_memory},slots=255,maxmem=${MAX_MEM}M";
2937
2938     } else {
2939
2940         $static_memory = $memory;
2941         push @$cmd, '-m', $static_memory;
2942     }
2943
2944     if ($conf->{numa}) {
2945
2946         my $numa_totalmemory = undef;
2947         for (my $i = 0; $i < $MAX_NUMA; $i++) {
2948             next if !$conf->{"numa$i"};
2949             my $numa = parse_numa($conf->{"numa$i"});
2950             next if !$numa;
2951             # memory
2952             die "missing numa node$i memory value\n" if !$numa->{memory};
2953             my $numa_memory = $numa->{memory};
2954             $numa_totalmemory += $numa_memory;
2955             my $numa_object = "memory-backend-ram,id=ram-node$i,size=${numa_memory}M";
2956
2957             # cpus
2958             my $cpus_start = $numa->{cpus}->{start};
2959             die "missing numa node$i cpus\n" if !defined($cpus_start);
2960             my $cpus_end = $numa->{cpus}->{end} if defined($numa->{cpus}->{end});
2961             my $cpus = $cpus_start;
2962             if (defined($cpus_end)) {
2963                 $cpus .= "-$cpus_end";
2964                 die "numa node$i :  cpu range $cpus is incorrect\n" if $cpus_end <= $cpus_start;
2965             }
2966
2967             # hostnodes
2968             my $hostnodes_start = $numa->{hostnodes}->{start};
2969             if (defined($hostnodes_start)) {
2970                 my $hostnodes_end = $numa->{hostnodes}->{end} if defined($numa->{hostnodes}->{end});
2971                 my $hostnodes = $hostnodes_start;
2972                 if (defined($hostnodes_end)) {
2973                     $hostnodes .= "-$hostnodes_end";
2974                     die "host node $hostnodes range is incorrect\n" if $hostnodes_end <= $hostnodes_start;
2975                 }
2976
2977                 my $hostnodes_end_range = defined($hostnodes_end) ? $hostnodes_end : $hostnodes_start;
2978                 for (my $i = $hostnodes_start; $i <= $hostnodes_end_range; $i++ ) {
2979                     die "host numa node$i don't exist\n" if ! -d "/sys/devices/system/node/node$i/";
2980                 }
2981
2982                 # policy
2983                 my $policy = $numa->{policy};
2984                 die "you need to define a policy for hostnode $hostnodes\n" if !$policy;
2985                 $numa_object .= ",host-nodes=$hostnodes,policy=$policy";
2986             }
2987
2988             push @$cmd, '-object', $numa_object;
2989             push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
2990         }
2991
2992         die "total memory for NUMA nodes must be equal to vm static memory\n"
2993             if $numa_totalmemory && $numa_totalmemory != $static_memory;
2994
2995         #if no custom tology, we split memory and cores across numa nodes
2996         if(!$numa_totalmemory) {
2997
2998             my $numa_memory = ($static_memory / $sockets) . "M";
2999
3000             for (my $i = 0; $i < $sockets; $i++)  {
3001
3002                 my $cpustart = ($cores * $i);
3003                 my $cpuend = ($cpustart + $cores - 1) if $cores && $cores > 1;
3004                 my $cpus = $cpustart;
3005                 $cpus .= "-$cpuend" if $cpuend;
3006
3007                 push @$cmd, '-object', "memory-backend-ram,size=$numa_memory,id=ram-node$i";
3008                 push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
3009             }
3010         }
3011     }
3012
3013     if ($hotplug_features->{memory}) {
3014         foreach_dimm($conf, $vmid, $memory, $sockets, sub {
3015             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3016             push @$cmd, "-object" , "memory-backend-ram,id=mem-$name,size=${dimm_size}M";
3017             push @$cmd, "-device", "pc-dimm,id=$name,memdev=mem-$name,node=$numanode";
3018
3019             #if dimm_memory is not aligned to dimm map
3020             if($current_size > $memory) {
3021                  $conf->{memory} = $current_size;
3022                  update_config_nolock($vmid, $conf, 1);
3023             }
3024         });
3025     }
3026
3027     push @$cmd, '-S' if $conf->{freeze};
3028
3029     # set keyboard layout
3030     my $kb = $conf->{keyboard} || $defaults->{keyboard};
3031     push @$cmd, '-k', $kb if $kb;
3032
3033     # enable sound
3034     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3035     #push @$cmd, '-soundhw', 'es1370';
3036     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3037
3038     if($conf->{agent}) {
3039         my $qgasocket = qmp_socket($vmid, 1);
3040         my $pciaddr = print_pci_addr("qga0", $bridges);
3041         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3042         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3043         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3044     }
3045
3046     my $spice_port;
3047
3048     if ($qxlnum) {
3049         if ($qxlnum > 1) {
3050             if ($conf->{ostype} && $conf->{ostype} =~ m/^w/){
3051                 for(my $i = 1; $i < $qxlnum; $i++){
3052                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3053                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3054                 }
3055             } else {
3056                 # assume other OS works like Linux
3057                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3058                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3059             }
3060         }
3061
3062         my $pciaddr = print_pci_addr("spice", $bridges);
3063
3064         my $nodename = PVE::INotify::nodename();
3065         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3066         $spice_port = PVE::Tools::next_spice_port($pfamily);
3067
3068         push @$devices, '-spice', "tls-port=${spice_port},addr=localhost,tls-ciphers=DES-CBC3-SHA,seamless-migration=on";
3069
3070         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3071         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3072         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3073     }
3074
3075     # enable balloon by default, unless explicitly disabled
3076     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3077         $pciaddr = print_pci_addr("balloon0", $bridges);
3078         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3079     }
3080
3081     if ($conf->{watchdog}) {
3082         my $wdopts = parse_watchdog($conf->{watchdog});
3083         $pciaddr = print_pci_addr("watchdog", $bridges);
3084         my $watchdog = $wdopts->{model} || 'i6300esb';
3085         push @$devices, '-device', "$watchdog$pciaddr";
3086         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3087     }
3088
3089     my $vollist = [];
3090     my $scsicontroller = {};
3091     my $ahcicontroller = {};
3092     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3093
3094     # Add iscsi initiator name if available
3095     if (my $initiator = get_initiator_name()) {
3096         push @$devices, '-iscsi', "initiator-name=$initiator";
3097     }
3098
3099     foreach_drive($conf, sub {
3100         my ($ds, $drive) = @_;
3101
3102         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3103             push @$vollist, $drive->{file};
3104         }
3105
3106         $use_virtio = 1 if $ds =~ m/^virtio/;
3107
3108         if (drive_is_cdrom ($drive)) {
3109             if ($bootindex_hash->{d}) {
3110                 $drive->{bootindex} = $bootindex_hash->{d};
3111                 $bootindex_hash->{d} += 1;
3112             }
3113         } else {
3114             if ($bootindex_hash->{c}) {
3115                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3116                 $bootindex_hash->{c} += 1;
3117             }
3118         }
3119
3120         if($drive->{interface} eq 'virtio'){
3121            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3122         }
3123
3124         if ($drive->{interface} eq 'scsi') {
3125
3126             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3127
3128             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3129             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3130
3131             my $iothread = '';
3132             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3133                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3134                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3135             }
3136
3137             my $queues = '';
3138             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3139                 $queues = ",num_queues=$drive->{queues}";
3140             }
3141
3142             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3143             $scsicontroller->{$controller}=1;
3144         }
3145
3146         if ($drive->{interface} eq 'sata') {
3147            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3148            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3149            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3150            $ahcicontroller->{$controller}=1;
3151         }
3152
3153         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3154         push @$devices, '-drive',$drive_cmd;
3155         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3156     });
3157
3158     for (my $i = 0; $i < $MAX_NETS; $i++) {
3159          next if !$conf->{"net$i"};
3160          my $d = parse_net($conf->{"net$i"});
3161          next if !$d;
3162
3163          $use_virtio = 1 if $d->{model} eq 'virtio';
3164
3165          if ($bootindex_hash->{n}) {
3166             $d->{bootindex} = $bootindex_hash->{n};
3167             $bootindex_hash->{n} += 1;
3168          }
3169
3170          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3171          push @$devices, '-netdev', $netdevfull;
3172
3173          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3174          push @$devices, '-device', $netdevicefull;
3175     }
3176
3177     if (!$q35) {
3178         # add pci bridges
3179         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3180            $bridges->{1} = 1;
3181            $bridges->{2} = 1;
3182         }
3183
3184         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3185
3186         while (my ($k, $v) = each %$bridges) {
3187             $pciaddr = print_pci_addr("pci.$k");
3188             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3189         }
3190     }
3191
3192     # add custom args
3193     if ($conf->{args}) {
3194         my $aa = PVE::Tools::split_args($conf->{args});
3195         push @$cmd, @$aa;
3196     }
3197
3198     push @$cmd, @$devices;
3199     push @$cmd, '-rtc', join(',', @$rtcFlags)
3200         if scalar(@$rtcFlags);
3201     push @$cmd, '-machine', join(',', @$machineFlags)
3202         if scalar(@$machineFlags);
3203     push @$cmd, '-global', join(',', @$globalFlags)
3204         if scalar(@$globalFlags);
3205
3206     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3207 }
3208
3209 sub vnc_socket {
3210     my ($vmid) = @_;
3211     return "${var_run_tmpdir}/$vmid.vnc";
3212 }
3213
3214 sub spice_port {
3215     my ($vmid) = @_;
3216
3217     my $res = vm_mon_cmd($vmid, 'query-spice');
3218
3219     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3220 }
3221
3222 sub qmp_socket {
3223     my ($vmid, $qga) = @_;
3224     my $sockettype = $qga ? 'qga' : 'qmp';
3225     return "${var_run_tmpdir}/$vmid.$sockettype";
3226 }
3227
3228 sub pidfile_name {
3229     my ($vmid) = @_;
3230     return "${var_run_tmpdir}/$vmid.pid";
3231 }
3232
3233 sub vm_devices_list {
3234     my ($vmid) = @_;
3235
3236     my $res = vm_mon_cmd($vmid, 'query-pci');
3237     my $devices = {};
3238     foreach my $pcibus (@$res) {
3239         foreach my $device (@{$pcibus->{devices}}) {
3240             next if !$device->{'qdev_id'};
3241             if ($device->{'pci_bridge'}) {
3242                 $devices->{$device->{'qdev_id'}} = 1;
3243                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3244                     next if !$bridge_device->{'qdev_id'};
3245                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3246                     $devices->{$device->{'qdev_id'}}++;
3247                 }
3248             } else {
3249                 $devices->{$device->{'qdev_id'}} = 1;
3250             }
3251         }
3252     }
3253
3254     my $resblock = vm_mon_cmd($vmid, 'query-block');
3255     foreach my $block (@$resblock) {
3256         if($block->{device} =~ m/^drive-(\S+)/){
3257                 $devices->{$1} = 1;
3258         }
3259     }
3260
3261     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3262     foreach my $mice (@$resmice) {
3263         if ($mice->{name} eq 'QEMU HID Tablet') {
3264             $devices->{tablet} = 1;
3265             last;
3266         }
3267     }
3268
3269     return $devices;
3270 }
3271
3272 sub vm_deviceplug {
3273     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3274
3275     my $q35 = machine_type_is_q35($conf);
3276
3277     my $devices_list = vm_devices_list($vmid);
3278     return 1 if defined($devices_list->{$deviceid});
3279
3280     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3281
3282     if ($deviceid eq 'tablet') {
3283
3284         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3285
3286     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3287
3288         qemu_iothread_add($vmid, $deviceid, $device);
3289
3290         qemu_driveadd($storecfg, $vmid, $device);
3291         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3292
3293         qemu_deviceadd($vmid, $devicefull);
3294         eval { qemu_deviceaddverify($vmid, $deviceid); };
3295         if (my $err = $@) {
3296             eval { qemu_drivedel($vmid, $deviceid); };
3297             warn $@ if $@;
3298             die $err;
3299         }
3300
3301     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3302
3303
3304         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3305         my $pciaddr = print_pci_addr($deviceid);
3306         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3307
3308         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3309
3310         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3311             qemu_iothread_add($vmid, $deviceid, $device);
3312             $devicefull .= ",iothread=iothread-$deviceid";
3313         }
3314
3315         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3316             $devicefull .= ",num_queues=$device->{queues}";
3317         }
3318
3319         qemu_deviceadd($vmid, $devicefull);
3320         qemu_deviceaddverify($vmid, $deviceid);
3321
3322     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3323
3324         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3325         qemu_driveadd($storecfg, $vmid, $device);
3326
3327         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3328         eval { qemu_deviceadd($vmid, $devicefull); };
3329         if (my $err = $@) {
3330             eval { qemu_drivedel($vmid, $deviceid); };
3331             warn $@ if $@;
3332             die $err;
3333         }
3334
3335     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3336
3337         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3338         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid);
3339         qemu_deviceadd($vmid, $netdevicefull);
3340         eval { qemu_deviceaddverify($vmid, $deviceid); };
3341         if (my $err = $@) {
3342             eval { qemu_netdevdel($vmid, $deviceid); };
3343             warn $@ if $@;
3344             die $err;
3345         }
3346
3347     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3348
3349         my $bridgeid = $2;
3350         my $pciaddr = print_pci_addr($deviceid);
3351         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3352
3353         qemu_deviceadd($vmid, $devicefull);
3354         qemu_deviceaddverify($vmid, $deviceid);
3355
3356     } else {
3357         die "can't hotplug device '$deviceid'\n";
3358     }
3359
3360     return 1;
3361 }
3362
3363 # fixme: this should raise exceptions on error!
3364 sub vm_deviceunplug {
3365     my ($vmid, $conf, $deviceid) = @_;
3366
3367     my $devices_list = vm_devices_list($vmid);
3368     return 1 if !defined($devices_list->{$deviceid});
3369
3370     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3371
3372     if ($deviceid eq 'tablet') {
3373
3374         qemu_devicedel($vmid, $deviceid);
3375
3376     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3377
3378         qemu_devicedel($vmid, $deviceid);
3379         qemu_devicedelverify($vmid, $deviceid);
3380         qemu_drivedel($vmid, $deviceid);
3381         qemu_iothread_del($conf, $vmid, $deviceid);
3382
3383     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3384
3385         qemu_devicedel($vmid, $deviceid);
3386         qemu_devicedelverify($vmid, $deviceid);
3387         qemu_iothread_del($conf, $vmid, $deviceid);
3388
3389     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3390
3391         #qemu 2.3 segfault on drive_del with virtioscsi + iothread
3392         my $device = parse_drive($deviceid, $conf->{$deviceid});
3393         die "virtioscsi with iothread is not hot-unplugglable currently" if $device->{iothread};
3394
3395         qemu_devicedel($vmid, $deviceid);
3396         qemu_drivedel($vmid, $deviceid);
3397         qemu_deletescsihw($conf, $vmid, $deviceid);
3398
3399     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3400
3401         qemu_devicedel($vmid, $deviceid);
3402         qemu_devicedelverify($vmid, $deviceid);
3403         qemu_netdevdel($vmid, $deviceid);
3404
3405     } else {
3406         die "can't unplug device '$deviceid'\n";
3407     }
3408
3409     return 1;
3410 }
3411
3412 sub qemu_deviceadd {
3413     my ($vmid, $devicefull) = @_;
3414
3415     $devicefull = "driver=".$devicefull;
3416     my %options =  split(/[=,]/, $devicefull);
3417
3418     vm_mon_cmd($vmid, "device_add" , %options);
3419 }
3420
3421 sub qemu_devicedel {
3422     my ($vmid, $deviceid) = @_;
3423
3424     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3425 }
3426
3427 sub qemu_iothread_add {
3428     my($vmid, $deviceid, $device) = @_;
3429
3430     if ($device->{iothread}) {
3431         my $iothreads = vm_iothreads_list($vmid);
3432         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3433     }
3434 }
3435
3436 sub qemu_iothread_del {
3437     my($conf, $vmid, $deviceid) = @_;
3438
3439     my $device = parse_drive($deviceid, $conf->{$deviceid});
3440     if ($device->{iothread}) {
3441         my $iothreads = vm_iothreads_list($vmid);
3442         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3443     }
3444 }
3445
3446 sub qemu_objectadd {
3447     my($vmid, $objectid, $qomtype) = @_;
3448
3449     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3450
3451     return 1;
3452 }
3453
3454 sub qemu_objectdel {
3455     my($vmid, $objectid) = @_;
3456
3457     vm_mon_cmd($vmid, "object-del", id => $objectid);
3458
3459     return 1;
3460 }
3461
3462 sub qemu_driveadd {
3463     my ($storecfg, $vmid, $device) = @_;
3464
3465     my $drive = print_drive_full($storecfg, $vmid, $device);
3466     $drive =~ s/\\/\\\\/g;
3467     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3468
3469     # If the command succeeds qemu prints: "OK"
3470     return 1 if $ret =~ m/OK/s;
3471
3472     die "adding drive failed: $ret\n";
3473 }
3474
3475 sub qemu_drivedel {
3476     my($vmid, $deviceid) = @_;
3477
3478     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3479     $ret =~ s/^\s+//;
3480
3481     return 1 if $ret eq "";
3482
3483     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3484     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3485
3486     die "deleting drive $deviceid failed : $ret\n";
3487 }
3488
3489 sub qemu_deviceaddverify {
3490     my ($vmid, $deviceid) = @_;
3491
3492     for (my $i = 0; $i <= 5; $i++) {
3493          my $devices_list = vm_devices_list($vmid);
3494          return 1 if defined($devices_list->{$deviceid});
3495          sleep 1;
3496     }
3497
3498     die "error on hotplug device '$deviceid'\n";
3499 }
3500
3501
3502 sub qemu_devicedelverify {
3503     my ($vmid, $deviceid) = @_;
3504
3505     # need to verify that the device is correctly removed as device_del
3506     # is async and empty return is not reliable
3507
3508     for (my $i = 0; $i <= 5; $i++) {
3509          my $devices_list = vm_devices_list($vmid);
3510          return 1 if !defined($devices_list->{$deviceid});
3511          sleep 1;
3512     }
3513
3514     die "error on hot-unplugging device '$deviceid'\n";
3515 }
3516
3517 sub qemu_findorcreatescsihw {
3518     my ($storecfg, $conf, $vmid, $device) = @_;
3519
3520     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3521
3522     my $scsihwid="$controller_prefix$controller";
3523     my $devices_list = vm_devices_list($vmid);
3524
3525     if(!defined($devices_list->{$scsihwid})) {
3526         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3527     }
3528
3529     return 1;
3530 }
3531
3532 sub qemu_deletescsihw {
3533     my ($conf, $vmid, $opt) = @_;
3534
3535     my $device = parse_drive($opt, $conf->{$opt});
3536
3537     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3538         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3539         return 1;
3540     }
3541
3542     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3543
3544     my $devices_list = vm_devices_list($vmid);
3545     foreach my $opt (keys %{$devices_list}) {
3546         if (PVE::QemuServer::valid_drivename($opt)) {
3547             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3548             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3549                 return 1;
3550             }
3551         }
3552     }
3553
3554     my $scsihwid="scsihw$controller";
3555
3556     vm_deviceunplug($vmid, $conf, $scsihwid);
3557
3558     return 1;
3559 }
3560
3561 sub qemu_add_pci_bridge {
3562     my ($storecfg, $conf, $vmid, $device) = @_;
3563
3564     my $bridges = {};
3565
3566     my $bridgeid;
3567
3568     print_pci_addr($device, $bridges);
3569
3570     while (my ($k, $v) = each %$bridges) {
3571         $bridgeid = $k;
3572     }
3573     return 1 if !defined($bridgeid) || $bridgeid < 1;
3574
3575     my $bridge = "pci.$bridgeid";
3576     my $devices_list = vm_devices_list($vmid);
3577
3578     if (!defined($devices_list->{$bridge})) {
3579         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3580     }
3581
3582     return 1;
3583 }
3584
3585 sub qemu_set_link_status {
3586     my ($vmid, $device, $up) = @_;
3587
3588     vm_mon_cmd($vmid, "set_link", name => $device,
3589                up => $up ? JSON::true : JSON::false);
3590 }
3591
3592 sub qemu_netdevadd {
3593     my ($vmid, $conf, $device, $deviceid) = @_;
3594
3595     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid);
3596     my %options =  split(/[=,]/, $netdev);
3597
3598     vm_mon_cmd($vmid, "netdev_add",  %options);
3599     return 1;
3600 }
3601
3602 sub qemu_netdevdel {
3603     my ($vmid, $deviceid) = @_;
3604
3605     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3606 }
3607
3608 sub qemu_cpu_hotplug {
3609     my ($vmid, $conf, $vcpus) = @_;
3610
3611     my $sockets = 1;
3612     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3613     $sockets = $conf->{sockets} if  $conf->{sockets};
3614     my $cores = $conf->{cores} || 1;
3615     my $maxcpus = $sockets * $cores;
3616
3617     $vcpus = $maxcpus if !$vcpus;
3618
3619     die "you can't add more vcpus than maxcpus\n"
3620         if $vcpus > $maxcpus;
3621
3622     my $currentvcpus = $conf->{vcpus} || $maxcpus;
3623     die "online cpu unplug is not yet possible\n"
3624         if $vcpus < $currentvcpus;
3625
3626     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3627     die "vcpus in running vm is different than configuration\n"
3628         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
3629
3630     for (my $i = $currentvcpus; $i < $vcpus; $i++) {
3631         vm_mon_cmd($vmid, "cpu-add", id => int($i));
3632     }
3633 }
3634
3635 sub qemu_memory_hotplug {
3636     my ($vmid, $conf, $defaults, $opt, $value) = @_;
3637
3638     return $value if !check_running($vmid);
3639
3640     my $memory = $conf->{memory} || $defaults->{memory};
3641     $value = $defaults->{memory} if !$value;
3642     return $value if $value == $memory;
3643
3644     my $static_memory = $STATICMEM;
3645     my $dimm_memory = $memory - $static_memory;
3646
3647     die "memory can't be lower than $static_memory MB" if $value < $static_memory;
3648     die "you cannot add more memory than $MAX_MEM MB!\n" if $memory > $MAX_MEM;
3649
3650
3651     my $sockets = 1;
3652     $sockets = $conf->{sockets} if $conf->{sockets};
3653
3654     if($value > $memory) {
3655
3656         foreach_dimm($conf, $vmid, $value, $sockets, sub {
3657             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3658
3659                 return if $current_size <= $conf->{memory};
3660
3661                 eval { vm_mon_cmd($vmid, "object-add", 'qom-type' => "memory-backend-ram", id => "mem-$name", props => { size => int($dimm_size*1024*1024) } ) };
3662                 if (my $err = $@) {
3663                     eval { qemu_objectdel($vmid, "mem-$name"); };
3664                     die $err;
3665                 }
3666
3667                 eval { vm_mon_cmd($vmid, "device_add", driver => "pc-dimm", id => "$name", memdev => "mem-$name", node => $numanode) };
3668                 if (my $err = $@) {
3669                     eval { qemu_objectdel($vmid, "mem-$name"); };
3670                     die $err;
3671                 }
3672                 #update conf after each succesful module hotplug
3673                 $conf->{memory} = $current_size;
3674                 update_config_nolock($vmid, $conf, 1);
3675         });
3676
3677     } else {
3678
3679         foreach_reverse_dimm($conf, $vmid, $value, $sockets, sub {
3680             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3681
3682                 return if $current_size >= $conf->{memory};
3683                 print "try to unplug memory dimm $name\n";
3684
3685                 my $retry = 0;
3686                 while (1) {
3687                     eval { qemu_devicedel($vmid, $name) };
3688                     sleep 3;
3689                     my $dimm_list = qemu_dimm_list($vmid);
3690                     last if !$dimm_list->{$name};
3691                     raise_param_exc({ $name => "error unplug memory module" }) if $retry > 5;
3692                     $retry++;
3693                 }
3694
3695                 #update conf after each succesful module unplug
3696                 $conf->{memory} = $current_size;
3697
3698                 eval { qemu_objectdel($vmid, "mem-$name"); };
3699                 update_config_nolock($vmid, $conf, 1);
3700         });
3701     }
3702 }
3703
3704 sub qemu_dimm_list {
3705     my ($vmid) = @_;
3706
3707     my $dimmarray = vm_mon_cmd_nocheck($vmid, "query-memory-devices");
3708     my $dimms = {};
3709
3710     foreach my $dimm (@$dimmarray) {
3711
3712         $dimms->{$dimm->{data}->{id}}->{id} = $dimm->{data}->{id};
3713         $dimms->{$dimm->{data}->{id}}->{node} = $dimm->{data}->{node};
3714         $dimms->{$dimm->{data}->{id}}->{addr} = $dimm->{data}->{addr};
3715         $dimms->{$dimm->{data}->{id}}->{size} = $dimm->{data}->{size};
3716         $dimms->{$dimm->{data}->{id}}->{slot} = $dimm->{data}->{slot};
3717     }
3718     return $dimms;
3719 }
3720
3721 sub qemu_block_set_io_throttle {
3722     my ($vmid, $deviceid, $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr) = @_;
3723
3724     return if !check_running($vmid) ;
3725
3726     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid, bps => int($bps), bps_rd => int($bps_rd), bps_wr => int($bps_wr), iops => int($iops), iops_rd => int($iops_rd), iops_wr => int($iops_wr));
3727
3728 }
3729
3730 # old code, only used to shutdown old VM after update
3731 sub __read_avail {
3732     my ($fh, $timeout) = @_;
3733
3734     my $sel = new IO::Select;
3735     $sel->add($fh);
3736
3737     my $res = '';
3738     my $buf;
3739
3740     my @ready;
3741     while (scalar (@ready = $sel->can_read($timeout))) {
3742         my $count;
3743         if ($count = $fh->sysread($buf, 8192)) {
3744             if ($buf =~ /^(.*)\(qemu\) $/s) {
3745                 $res .= $1;
3746                 last;
3747             } else {
3748                 $res .= $buf;
3749             }
3750         } else {
3751             if (!defined($count)) {
3752                 die "$!\n";
3753             }
3754             last;
3755         }
3756     }
3757
3758     die "monitor read timeout\n" if !scalar(@ready);
3759
3760     return $res;
3761 }
3762
3763 # old code, only used to shutdown old VM after update
3764 sub vm_monitor_command {
3765     my ($vmid, $cmdstr, $nocheck) = @_;
3766
3767     my $res;
3768
3769     eval {
3770         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
3771
3772         my $sname = "${var_run_tmpdir}/$vmid.mon";
3773
3774         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
3775             die "unable to connect to VM $vmid socket - $!\n";
3776
3777         my $timeout = 3;
3778
3779         # hack: migrate sometime blocks the monitor (when migrate_downtime
3780         # is set)
3781         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3782             $timeout = 60*60; # 1 hour
3783         }
3784
3785         # read banner;
3786         my $data = __read_avail($sock, $timeout);
3787
3788         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
3789             die "got unexpected qemu monitor banner\n";
3790         }
3791
3792         my $sel = new IO::Select;
3793         $sel->add($sock);
3794
3795         if (!scalar(my @ready = $sel->can_write($timeout))) {
3796             die "monitor write error - timeout";
3797         }
3798
3799         my $fullcmd = "$cmdstr\r";
3800
3801         # syslog('info', "VM $vmid monitor command: $cmdstr");
3802
3803         my $b;
3804         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
3805             die "monitor write error - $!";
3806         }
3807
3808         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
3809
3810         $timeout = 20;
3811
3812         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3813             $timeout = 60*60; # 1 hour
3814         } elsif ($cmdstr =~ m/^(eject|change)/) {
3815             $timeout = 60; # note: cdrom mount command is slow
3816         }
3817         if ($res = __read_avail($sock, $timeout)) {
3818
3819             my @lines = split("\r?\n", $res);
3820
3821             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
3822
3823             $res = join("\n", @lines);
3824             $res .= "\n";
3825         }
3826     };
3827
3828     my $err = $@;
3829
3830     if ($err) {
3831         syslog("err", "VM $vmid monitor command failed - $err");
3832         die $err;
3833     }
3834
3835     return $res;
3836 }
3837
3838 sub qemu_block_resize {
3839     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
3840
3841     my $running = check_running($vmid);
3842
3843     return if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
3844
3845     return if !$running;
3846
3847     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
3848
3849 }
3850
3851 sub qemu_volume_snapshot {
3852     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3853
3854     my $running = check_running($vmid);
3855
3856     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
3857         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
3858     } else {
3859         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
3860     }
3861 }
3862
3863 sub qemu_volume_snapshot_delete {
3864     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3865
3866     my $running = check_running($vmid);
3867
3868     return if !PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
3869
3870     return if !$running;
3871
3872     vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
3873 }
3874
3875 sub set_migration_caps {
3876     my ($vmid) = @_;
3877
3878     my $cap_ref = [];
3879
3880     my $enabled_cap = {
3881         "auto-converge" => 1,
3882         "xbzrle" => 1,
3883         "x-rdma-pin-all" => 0,
3884         "zero-blocks" => 0,
3885         "compress" => 0
3886     };
3887
3888     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
3889
3890     for my $supported_capability (@$supported_capabilities) {
3891         push @$cap_ref, {
3892             capability => $supported_capability->{capability},
3893             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
3894         };
3895     }
3896
3897     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
3898 }
3899
3900 my $fast_plug_option = {
3901     'lock' => 1,
3902     'name' => 1,
3903     'onboot' => 1,
3904     'shares' => 1,
3905     'startup' => 1,
3906     'description' => 1,
3907 };
3908
3909 # hotplug changes in [PENDING]
3910 # $selection hash can be used to only apply specified options, for
3911 # example: { cores => 1 } (only apply changed 'cores')
3912 # $errors ref is used to return error messages
3913 sub vmconfig_hotplug_pending {
3914     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
3915
3916     my $defaults = load_defaults();
3917
3918     # commit values which do not have any impact on running VM first
3919     # Note: those option cannot raise errors, we we do not care about
3920     # $selection and always apply them.
3921
3922     my $add_error = sub {
3923         my ($opt, $msg) = @_;
3924         $errors->{$opt} = "hotplug problem - $msg";
3925     };
3926
3927     my $changes = 0;
3928     foreach my $opt (keys %{$conf->{pending}}) { # add/change
3929         if ($fast_plug_option->{$opt}) {
3930             $conf->{$opt} = $conf->{pending}->{$opt};
3931             delete $conf->{pending}->{$opt};
3932             $changes = 1;
3933         }
3934     }
3935
3936     if ($changes) {
3937         update_config_nolock($vmid, $conf, 1);
3938         $conf = load_config($vmid); # update/reload
3939     }
3940
3941     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3942
3943     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
3944     while (my ($opt, $force) = each %$pending_delete_hash) {
3945         next if $selection && !$selection->{$opt};
3946         eval {
3947             if ($opt eq 'hotplug') {
3948                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
3949             } elsif ($opt eq 'tablet') {
3950                 die "skip\n" if !$hotplug_features->{usb};
3951                 if ($defaults->{tablet}) {
3952                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
3953                 } else {
3954                     vm_deviceunplug($vmid, $conf, $opt);
3955                 }
3956             } elsif ($opt eq 'vcpus') {
3957                 die "skip\n" if !$hotplug_features->{cpu};
3958                 qemu_cpu_hotplug($vmid, $conf, undef);
3959             } elsif ($opt eq 'balloon') {
3960                 # enable balloon device is not hotpluggable
3961                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
3962             } elsif ($fast_plug_option->{$opt}) {
3963                 # do nothing
3964             } elsif ($opt =~ m/^net(\d+)$/) {
3965                 die "skip\n" if !$hotplug_features->{network};
3966                 vm_deviceunplug($vmid, $conf, $opt);
3967             } elsif (valid_drivename($opt)) {
3968                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
3969                 vm_deviceunplug($vmid, $conf, $opt);
3970                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
3971             } elsif ($opt =~ m/^memory$/) {
3972                 die "skip\n" if !$hotplug_features->{memory};
3973                 qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
3974             } elsif ($opt eq 'cpuunits') {
3975                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
3976             } elsif ($opt eq 'cpulimit') {
3977                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
3978             } else {
3979                 die "skip\n";
3980             }
3981         };
3982         if (my $err = $@) {
3983             &$add_error($opt, $err) if $err ne "skip\n";
3984         } else {
3985             # save new config if hotplug was successful
3986             delete $conf->{$opt};
3987             vmconfig_undelete_pending_option($conf, $opt);
3988             update_config_nolock($vmid, $conf, 1);
3989             $conf = load_config($vmid); # update/reload
3990         }
3991     }
3992
3993     foreach my $opt (keys %{$conf->{pending}}) {
3994         next if $selection && !$selection->{$opt};
3995         my $value = $conf->{pending}->{$opt};
3996         eval {
3997             if ($opt eq 'hotplug') {
3998                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
3999             } elsif ($opt eq 'tablet') {
4000                 die "skip\n" if !$hotplug_features->{usb};
4001                 if ($value == 1) {
4002                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4003                 } elsif ($value == 0) {
4004                     vm_deviceunplug($vmid, $conf, $opt);
4005                 }
4006             } elsif ($opt eq 'vcpus') {
4007                 die "skip\n" if !$hotplug_features->{cpu};
4008                 qemu_cpu_hotplug($vmid, $conf, $value);
4009             } elsif ($opt eq 'balloon') {
4010                 # enable/disable balloning device is not hotpluggable
4011                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4012                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4013                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4014
4015                 # allow manual ballooning if shares is set to zero
4016                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4017                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4018                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4019                 }
4020             } elsif ($opt =~ m/^net(\d+)$/) {
4021                 # some changes can be done without hotplug
4022                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4023                                     $vmid, $opt, $value);
4024             } elsif (valid_drivename($opt)) {
4025                 # some changes can be done without hotplug
4026                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4027                                      $vmid, $opt, $value, 1);
4028             } elsif ($opt =~ m/^memory$/) { #dimms
4029                 die "skip\n" if !$hotplug_features->{memory};
4030                 $value = qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4031             } elsif ($opt eq 'cpuunits') {
4032                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4033             } elsif ($opt eq 'cpulimit') {
4034                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4035                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4036             } else {
4037                 die "skip\n";  # skip non-hot-pluggable options
4038             }
4039         };
4040         if (my $err = $@) {
4041             &$add_error($opt, $err) if $err ne "skip\n";
4042         } else {
4043             # save new config if hotplug was successful
4044             $conf->{$opt} = $value;
4045             delete $conf->{pending}->{$opt};
4046             update_config_nolock($vmid, $conf, 1);
4047             $conf = load_config($vmid); # update/reload
4048         }
4049     }
4050 }
4051
4052 sub try_deallocate_drive {
4053     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4054
4055     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4056         my $volid = $drive->{file};
4057         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4058             my $sid = PVE::Storage::parse_volume_id($volid);
4059             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4060
4061             # check if the disk is really unused
4062             my $used_paths = PVE::QemuServer::get_used_paths($vmid, $storecfg, $conf, 1, $key);
4063             my $path = PVE::Storage::path($storecfg, $volid);
4064             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4065                    if $used_paths->{$path};
4066             PVE::Storage::vdisk_free($storecfg, $volid);
4067             return 1;
4068         } else {
4069             # If vm is not owner of this disk remove from config
4070             return 1;
4071         }
4072     }
4073
4074     return undef;
4075 }
4076
4077 sub vmconfig_delete_or_detach_drive {
4078     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4079
4080     my $drive = parse_drive($opt, $conf->{$opt});
4081
4082     my $rpcenv = PVE::RPCEnvironment::get();
4083     my $authuser = $rpcenv->get_user();
4084
4085     if ($force) {
4086         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4087         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4088     } else {
4089         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4090     }
4091 }
4092
4093 sub vmconfig_apply_pending {
4094     my ($vmid, $conf, $storecfg) = @_;
4095
4096     # cold plug
4097
4098     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4099     while (my ($opt, $force) = each %$pending_delete_hash) {
4100         die "internal error" if $opt =~ m/^unused/;
4101         $conf = load_config($vmid); # update/reload
4102         if (!defined($conf->{$opt})) {
4103             vmconfig_undelete_pending_option($conf, $opt);
4104             update_config_nolock($vmid, $conf, 1);
4105         } elsif (valid_drivename($opt)) {
4106             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4107             vmconfig_undelete_pending_option($conf, $opt);
4108             delete $conf->{$opt};
4109             update_config_nolock($vmid, $conf, 1);
4110         } else {
4111             vmconfig_undelete_pending_option($conf, $opt);
4112             delete $conf->{$opt};
4113             update_config_nolock($vmid, $conf, 1);
4114         }
4115     }
4116
4117     $conf = load_config($vmid); # update/reload
4118
4119     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4120         $conf = load_config($vmid); # update/reload
4121
4122         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4123             # skip if nothing changed
4124         } elsif (valid_drivename($opt)) {
4125             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4126                 if defined($conf->{$opt});
4127             $conf->{$opt} = $conf->{pending}->{$opt};
4128         } else {
4129             $conf->{$opt} = $conf->{pending}->{$opt};
4130         }
4131
4132         delete $conf->{pending}->{$opt};
4133         update_config_nolock($vmid, $conf, 1);
4134     }
4135 }
4136
4137 my $safe_num_ne = sub {
4138     my ($a, $b) = @_;
4139
4140     return 0 if !defined($a) && !defined($b);
4141     return 1 if !defined($a);
4142     return 1 if !defined($b);
4143
4144     return $a != $b;
4145 };
4146
4147 my $safe_string_ne = sub {
4148     my ($a, $b) = @_;
4149
4150     return 0 if !defined($a) && !defined($b);
4151     return 1 if !defined($a);
4152     return 1 if !defined($b);
4153
4154     return $a ne $b;
4155 };
4156
4157 sub vmconfig_update_net {
4158     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4159
4160     my $newnet = parse_net($value);
4161
4162     if ($conf->{$opt}) {
4163         my $oldnet = parse_net($conf->{$opt});
4164
4165         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4166             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4167             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4168             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4169
4170             # for non online change, we try to hot-unplug
4171             die "skip\n" if !$hotplug;
4172             vm_deviceunplug($vmid, $conf, $opt);
4173         } else {
4174
4175             die "internal error" if $opt !~ m/net(\d+)/;
4176             my $iface = "tap${vmid}i$1";
4177
4178             if (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4179                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4180             }
4181
4182             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4183                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4184                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4185                 PVE::Network::tap_unplug($iface);
4186                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall});
4187             }
4188
4189             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4190                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4191             }
4192
4193             return 1;
4194         }
4195     }
4196
4197     if ($hotplug) {
4198         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4199     } else {
4200         die "skip\n";
4201     }
4202 }
4203
4204 sub vmconfig_update_disk {
4205     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4206
4207     # fixme: do we need force?
4208
4209     my $drive = parse_drive($opt, $value);
4210
4211     if ($conf->{$opt}) {
4212
4213         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4214
4215             my $media = $drive->{media} || 'disk';
4216             my $oldmedia = $old_drive->{media} || 'disk';
4217             die "unable to change media type\n" if $media ne $oldmedia;
4218
4219             if (!drive_is_cdrom($old_drive)) {
4220
4221                 if ($drive->{file} ne $old_drive->{file}) {
4222
4223                     die "skip\n" if !$hotplug;
4224
4225                     # unplug and register as unused
4226                     vm_deviceunplug($vmid, $conf, $opt);
4227                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4228
4229                 } else {
4230                     # update existing disk
4231
4232                     # skip non hotpluggable value
4233                     if (&$safe_num_ne($drive->{discard}, $old_drive->{discard}) ||
4234                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4235                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4236                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4237                         die "skip\n";
4238                     }
4239
4240                     # apply throttle
4241                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4242                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4243                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4244                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4245                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4246                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4247                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4248                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4249                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4250                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4251                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4252                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max})) {
4253
4254                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4255                                                    ($drive->{mbps} || 0)*1024*1024,
4256                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4257                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4258                                                    $drive->{iops} || 0,
4259                                                    $drive->{iops_rd} || 0,
4260                                                    $drive->{iops_wr} || 0,
4261                                                    ($drive->{mbps_max} || 0)*1024*1024,
4262                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4263                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4264                                                    $drive->{iops_max} || 0,
4265                                                    $drive->{iops_rd_max} || 0,
4266                                                    $drive->{iops_wr_max} || 0);
4267
4268                     }
4269
4270                     return 1;
4271                 }
4272
4273             } else { # cdrom
4274
4275                 if ($drive->{file} eq 'none') {
4276                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4277                 } else {
4278                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4279                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4280                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4281                 }
4282
4283                 return 1;
4284             }
4285         }
4286     }
4287
4288     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4289     # hotplug new disks
4290     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4291 }
4292
4293 sub vm_start {
4294     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4295         $forcemachine, $spice_ticket) = @_;
4296
4297     lock_config($vmid, sub {
4298         my $conf = load_config($vmid, $migratedfrom);
4299
4300         die "you can't start a vm if it's a template\n" if is_template($conf);
4301
4302         check_lock($conf) if !$skiplock;
4303
4304         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4305
4306         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4307             vmconfig_apply_pending($vmid, $conf, $storecfg);
4308             $conf = load_config($vmid); # update/reload
4309         }
4310
4311         my $defaults = load_defaults();
4312
4313         # set environment variable useful inside network script
4314         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4315
4316         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
4317         # load new efi bios files on migration
4318         my $use_old_bios_files;
4319         if ($migratedfrom && $forcemachine && ($forcemachine =~ m/pc-(i440fx|q35)-(\d+)\.(\d+)/)) {
4320             my ($major, $minor) = ($2, $3);
4321             $use_old_bios_files = 1 if ($major <= 2) && ($minor < 4);
4322         }
4323
4324         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine, $use_old_bios_files);
4325
4326         my $migrate_port = 0;
4327         my $migrate_uri;
4328         if ($statefile) {
4329             if ($statefile eq 'tcp') {
4330                 my $localip = "localhost";
4331                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4332                 my $nodename = PVE::INotify::nodename();
4333                 if ($datacenterconf->{migration_unsecure}) {
4334                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4335                 }
4336                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4337                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4338                 $migrate_uri = "tcp:[${localip}]:${migrate_port}";
4339                 push @$cmd, '-incoming', $migrate_uri;
4340                 push @$cmd, '-S';
4341             } else {
4342                 push @$cmd, '-loadstate', $statefile;
4343             }
4344         } elsif ($paused) {
4345             push @$cmd, '-S';
4346         }
4347
4348         # host pci devices
4349         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4350           my $d = parse_hostpci($conf->{"hostpci$i"});
4351           next if !$d;
4352           my $pcidevices = $d->{pciid};
4353           foreach my $pcidevice (@$pcidevices) {
4354                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4355
4356                 my $info = pci_device_info("0000:$pciid");
4357                 die "IOMMU not present\n" if !check_iommu_support();
4358                 die "no pci device info for device '$pciid'\n" if !$info;
4359                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4360                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4361           }
4362         }
4363
4364         PVE::Storage::activate_volumes($storecfg, $vollist);
4365
4366         eval  { run_command($cmd, timeout => $statefile ? undef : 30,
4367                     umask => 0077); };
4368         my $err = $@;
4369         die "start failed: $err" if $err;
4370
4371         print "migration listens on $migrate_uri\n" if $migrate_uri;
4372
4373         if ($statefile && $statefile ne 'tcp')  {
4374             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4375             warn $@ if $@;
4376         }
4377
4378         if ($migratedfrom) {
4379
4380             eval {
4381                 set_migration_caps($vmid);
4382             };
4383             warn $@ if $@;
4384
4385             if ($spice_port) {
4386                 print "spice listens on port $spice_port\n";
4387                 if ($spice_ticket) {
4388                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4389                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4390                 }
4391             }
4392
4393         } else {
4394
4395             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4396                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4397                     if $conf->{balloon};
4398             }
4399
4400             foreach my $opt (keys %$conf) {
4401                 next if $opt !~  m/^net\d+$/;
4402                 my $nicconf = parse_net($conf->{$opt});
4403                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4404             }
4405         }
4406
4407         vm_mon_cmd_nocheck($vmid, 'qom-set',
4408                     path => "machine/peripheral/balloon0",
4409                     property => "guest-stats-polling-interval",
4410                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4411
4412     });
4413 }
4414
4415 sub vm_mon_cmd {
4416     my ($vmid, $execute, %params) = @_;
4417
4418     my $cmd = { execute => $execute, arguments => \%params };
4419     vm_qmp_command($vmid, $cmd);
4420 }
4421
4422 sub vm_mon_cmd_nocheck {
4423     my ($vmid, $execute, %params) = @_;
4424
4425     my $cmd = { execute => $execute, arguments => \%params };
4426     vm_qmp_command($vmid, $cmd, 1);
4427 }
4428
4429 sub vm_qmp_command {
4430     my ($vmid, $cmd, $nocheck) = @_;
4431
4432     my $res;
4433
4434     my $timeout;
4435     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4436         $timeout = $cmd->{arguments}->{timeout};
4437         delete $cmd->{arguments}->{timeout};
4438     }
4439
4440     eval {
4441         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4442         my $sname = qmp_socket($vmid);
4443         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4444             my $qmpclient = PVE::QMPClient->new();
4445
4446             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4447         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
4448             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
4449                 if scalar(%{$cmd->{arguments}});
4450             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
4451         } else {
4452             die "unable to open monitor socket\n";
4453         }
4454     };
4455     if (my $err = $@) {
4456         syslog("err", "VM $vmid qmp command failed - $err");
4457         die $err;
4458     }
4459
4460     return $res;
4461 }
4462
4463 sub vm_human_monitor_command {
4464     my ($vmid, $cmdline) = @_;
4465
4466     my $res;
4467
4468     my $cmd = {
4469         execute => 'human-monitor-command',
4470         arguments => { 'command-line' => $cmdline},
4471     };
4472
4473     return vm_qmp_command($vmid, $cmd);
4474 }
4475
4476 sub vm_commandline {
4477     my ($storecfg, $vmid) = @_;
4478
4479     my $conf = load_config($vmid);
4480
4481     my $defaults = load_defaults();
4482
4483     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
4484
4485     return join(' ', @$cmd);
4486 }
4487
4488 sub vm_reset {
4489     my ($vmid, $skiplock) = @_;
4490
4491     lock_config($vmid, sub {
4492
4493         my $conf = load_config($vmid);
4494
4495         check_lock($conf) if !$skiplock;
4496
4497         vm_mon_cmd($vmid, "system_reset");
4498     });
4499 }
4500
4501 sub get_vm_volumes {
4502     my ($conf) = @_;
4503
4504     my $vollist = [];
4505     foreach_volid($conf, sub {
4506         my ($volid, $is_cdrom) = @_;
4507
4508         return if $volid =~ m|^/|;
4509
4510         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4511         return if !$sid;
4512
4513         push @$vollist, $volid;
4514     });
4515
4516     return $vollist;
4517 }
4518
4519 sub vm_stop_cleanup {
4520     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
4521
4522     eval {
4523
4524         if (!$keepActive) {
4525             my $vollist = get_vm_volumes($conf);
4526             PVE::Storage::deactivate_volumes($storecfg, $vollist);
4527         }
4528
4529         foreach my $ext (qw(mon qmp pid vnc qga)) {
4530             unlink "/var/run/qemu-server/${vmid}.$ext";
4531         }
4532
4533         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
4534     };
4535     warn $@ if $@; # avoid errors - just warn
4536 }
4537
4538 # Note: use $nockeck to skip tests if VM configuration file exists.
4539 # We need that when migration VMs to other nodes (files already moved)
4540 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
4541 sub vm_stop {
4542     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
4543
4544     $force = 1 if !defined($force) && !$shutdown;
4545
4546     if ($migratedfrom){
4547         my $pid = check_running($vmid, $nocheck, $migratedfrom);
4548         kill 15, $pid if $pid;
4549         my $conf = load_config($vmid, $migratedfrom);
4550         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
4551         return;
4552     }
4553
4554     lock_config($vmid, sub {
4555
4556         my $pid = check_running($vmid, $nocheck);
4557         return if !$pid;
4558
4559         my $conf;
4560         if (!$nocheck) {
4561             $conf = load_config($vmid);
4562             check_lock($conf) if !$skiplock;
4563             if (!defined($timeout) && $shutdown && $conf->{startup}) {
4564                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
4565                 $timeout = $opts->{down} if $opts->{down};
4566             }
4567         }
4568
4569         $timeout = 60 if !defined($timeout);
4570
4571         eval {
4572             if ($shutdown) {
4573                 if (defined($conf) && $conf->{agent}) {
4574                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
4575                 } else {
4576                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
4577                 }
4578             } else {
4579                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
4580             }
4581         };
4582         my $err = $@;
4583
4584         if (!$err) {
4585             my $count = 0;
4586             while (($count < $timeout) && check_running($vmid, $nocheck)) {
4587                 $count++;
4588                 sleep 1;
4589             }
4590
4591             if ($count >= $timeout) {
4592                 if ($force) {
4593                     warn "VM still running - terminating now with SIGTERM\n";
4594                     kill 15, $pid;
4595                 } else {
4596                     die "VM quit/powerdown failed - got timeout\n";
4597                 }
4598             } else {
4599                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4600                 return;
4601             }
4602         } else {
4603             if ($force) {
4604                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
4605                 kill 15, $pid;
4606             } else {
4607                 die "VM quit/powerdown failed\n";
4608             }
4609         }
4610
4611         # wait again
4612         $timeout = 10;
4613
4614         my $count = 0;
4615         while (($count < $timeout) && check_running($vmid, $nocheck)) {
4616             $count++;
4617             sleep 1;
4618         }
4619
4620         if ($count >= $timeout) {
4621             warn "VM still running - terminating now with SIGKILL\n";
4622             kill 9, $pid;
4623             sleep 1;
4624         }
4625
4626         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4627    });
4628 }
4629
4630 sub vm_suspend {
4631     my ($vmid, $skiplock) = @_;
4632
4633     lock_config($vmid, sub {
4634
4635         my $conf = load_config($vmid);
4636
4637         check_lock($conf) if !($skiplock || ($conf->{lock} && $conf->{lock} eq 'backup'));
4638
4639         vm_mon_cmd($vmid, "stop");
4640     });
4641 }
4642
4643 sub vm_resume {
4644     my ($vmid, $skiplock, $nocheck) = @_;
4645
4646     lock_config($vmid, sub {
4647
4648         if (!$nocheck) {
4649
4650             my $conf = load_config($vmid);
4651
4652             check_lock($conf) if !($skiplock || ($conf->{lock} && $conf->{lock} eq 'backup'));
4653
4654             vm_mon_cmd($vmid, "cont");
4655
4656         } else {
4657             vm_mon_cmd_nocheck($vmid, "cont");
4658         }
4659     });
4660 }
4661
4662 sub vm_sendkey {
4663     my ($vmid, $skiplock, $key) = @_;
4664
4665     lock_config($vmid, sub {
4666
4667         my $conf = load_config($vmid);
4668
4669         # there is no qmp command, so we use the human monitor command
4670         vm_human_monitor_command($vmid, "sendkey $key");
4671     });
4672 }
4673
4674 sub vm_destroy {
4675     my ($storecfg, $vmid, $skiplock) = @_;
4676
4677     lock_config($vmid, sub {
4678
4679         my $conf = load_config($vmid);
4680
4681         check_lock($conf) if !$skiplock;
4682
4683         if (!check_running($vmid)) {
4684             destroy_vm($storecfg, $vmid);
4685         } else {
4686             die "VM $vmid is running - destroy failed\n";
4687         }
4688     });
4689 }
4690
4691 # pci helpers
4692
4693 sub file_write {
4694     my ($filename, $buf) = @_;
4695
4696     my $fh = IO::File->new($filename, "w");
4697     return undef if !$fh;
4698
4699     my $res = print $fh $buf;
4700
4701     $fh->close();
4702
4703     return $res;
4704 }
4705
4706 sub pci_device_info {
4707     my ($name) = @_;
4708
4709     my $res;
4710
4711     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
4712     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
4713
4714     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
4715     return undef if !defined($irq) || $irq !~ m/^\d+$/;
4716
4717     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
4718     return undef if !defined($vendor) || $vendor !~ s/^0x//;
4719
4720     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
4721     return undef if !defined($product) || $product !~ s/^0x//;
4722
4723     $res = {
4724         name => $name,
4725         vendor => $vendor,
4726         product => $product,
4727         domain => $domain,
4728         bus => $bus,
4729         slot => $slot,
4730         func => $func,
4731         irq => $irq,
4732         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
4733     };
4734
4735     return $res;
4736 }
4737
4738 sub pci_dev_reset {
4739     my ($dev) = @_;
4740
4741     my $name = $dev->{name};
4742
4743     my $fn = "$pcisysfs/devices/$name/reset";
4744
4745     return file_write($fn, "1");
4746 }
4747
4748 sub pci_dev_bind_to_vfio {
4749     my ($dev) = @_;
4750
4751     my $name = $dev->{name};
4752
4753     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4754
4755     if (!-d $vfio_basedir) {
4756         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4757     }
4758     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4759
4760     my $testdir = "$vfio_basedir/$name";
4761     return 1 if -d $testdir;
4762
4763     my $data = "$dev->{vendor} $dev->{product}";
4764     return undef if !file_write("$vfio_basedir/new_id", $data);
4765
4766     my $fn = "$pcisysfs/devices/$name/driver/unbind";
4767     if (!file_write($fn, $name)) {
4768         return undef if -f $fn;
4769     }
4770
4771     $fn = "$vfio_basedir/bind";
4772     if (! -d $testdir) {
4773         return undef if !file_write($fn, $name);
4774     }
4775
4776     return -d $testdir;
4777 }
4778
4779 sub pci_dev_group_bind_to_vfio {
4780     my ($pciid) = @_;
4781
4782     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4783
4784     if (!-d $vfio_basedir) {
4785         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4786     }
4787     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4788
4789     # get IOMMU group devices
4790     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
4791       my @devs = grep /^0000:/, readdir($D);
4792     closedir($D);
4793
4794     foreach my $pciid (@devs) {
4795         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
4796
4797         # pci bridges, switches or root ports are not supported
4798         # they have a pci_bus subdirectory so skip them
4799         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
4800
4801         my $info = pci_device_info($1);
4802         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
4803     }
4804
4805     return 1;
4806 }
4807
4808 sub print_pci_addr {
4809     my ($id, $bridges) = @_;
4810
4811     my $res = '';
4812     my $devices = {
4813         piix3 => { bus => 0, addr => 1 },
4814         #addr2 : first videocard
4815         balloon0 => { bus => 0, addr => 3 },
4816         watchdog => { bus => 0, addr => 4 },
4817         scsihw0 => { bus => 0, addr => 5 },
4818         'pci.3' => { bus => 0, addr => 5 }, #can also be used for virtio-scsi-single bridge
4819         scsihw1 => { bus => 0, addr => 6 },
4820         ahci0 => { bus => 0, addr => 7 },
4821         qga0 => { bus => 0, addr => 8 },
4822         spice => { bus => 0, addr => 9 },
4823         virtio0 => { bus => 0, addr => 10 },
4824         virtio1 => { bus => 0, addr => 11 },
4825         virtio2 => { bus => 0, addr => 12 },
4826         virtio3 => { bus => 0, addr => 13 },
4827         virtio4 => { bus => 0, addr => 14 },
4828         virtio5 => { bus => 0, addr => 15 },
4829         hostpci0 => { bus => 0, addr => 16 },
4830         hostpci1 => { bus => 0, addr => 17 },
4831         net0 => { bus => 0, addr => 18 },
4832         net1 => { bus => 0, addr => 19 },
4833         net2 => { bus => 0, addr => 20 },
4834         net3 => { bus => 0, addr => 21 },
4835         net4 => { bus => 0, addr => 22 },
4836         net5 => { bus => 0, addr => 23 },
4837         vga1 => { bus => 0, addr => 24 },
4838         vga2 => { bus => 0, addr => 25 },
4839         vga3 => { bus => 0, addr => 26 },
4840         hostpci2 => { bus => 0, addr => 27 },
4841         hostpci3 => { bus => 0, addr => 28 },
4842         #addr29 : usb-host (pve-usb.cfg)
4843         'pci.1' => { bus => 0, addr => 30 },
4844         'pci.2' => { bus => 0, addr => 31 },
4845         'net6' => { bus => 1, addr => 1 },
4846         'net7' => { bus => 1, addr => 2 },
4847         'net8' => { bus => 1, addr => 3 },
4848         'net9' => { bus => 1, addr => 4 },
4849         'net10' => { bus => 1, addr => 5 },
4850         'net11' => { bus => 1, addr => 6 },
4851         'net12' => { bus => 1, addr => 7 },
4852         'net13' => { bus => 1, addr => 8 },
4853         'net14' => { bus => 1, addr => 9 },
4854         'net15' => { bus => 1, addr => 10 },
4855         'net16' => { bus => 1, addr => 11 },
4856         'net17' => { bus => 1, addr => 12 },
4857         'net18' => { bus => 1, addr => 13 },
4858         'net19' => { bus => 1, addr => 14 },
4859         'net20' => { bus => 1, addr => 15 },
4860         'net21' => { bus => 1, addr => 16 },
4861         'net22' => { bus => 1, addr => 17 },
4862         'net23' => { bus => 1, addr => 18 },
4863         'net24' => { bus => 1, addr => 19 },
4864         'net25' => { bus => 1, addr => 20 },
4865         'net26' => { bus => 1, addr => 21 },
4866         'net27' => { bus => 1, addr => 22 },
4867         'net28' => { bus => 1, addr => 23 },
4868         'net29' => { bus => 1, addr => 24 },
4869         'net30' => { bus => 1, addr => 25 },
4870         'net31' => { bus => 1, addr => 26 },
4871         'virtio6' => { bus => 2, addr => 1 },
4872         'virtio7' => { bus => 2, addr => 2 },
4873         'virtio8' => { bus => 2, addr => 3 },
4874         'virtio9' => { bus => 2, addr => 4 },
4875         'virtio10' => { bus => 2, addr => 5 },
4876         'virtio11' => { bus => 2, addr => 6 },
4877         'virtio12' => { bus => 2, addr => 7 },
4878         'virtio13' => { bus => 2, addr => 8 },
4879         'virtio14' => { bus => 2, addr => 9 },
4880         'virtio15' => { bus => 2, addr => 10 },
4881         'virtioscsi0' => { bus => 3, addr => 1 },
4882         'virtioscsi1' => { bus => 3, addr => 2 },
4883         'virtioscsi2' => { bus => 3, addr => 3 },
4884         'virtioscsi3' => { bus => 3, addr => 4 },
4885         'virtioscsi4' => { bus => 3, addr => 5 },
4886         'virtioscsi5' => { bus => 3, addr => 6 },
4887         'virtioscsi6' => { bus => 3, addr => 7 },
4888         'virtioscsi7' => { bus => 3, addr => 8 },
4889         'virtioscsi8' => { bus => 3, addr => 9 },
4890         'virtioscsi9' => { bus => 3, addr => 10 },
4891         'virtioscsi10' => { bus => 3, addr => 11 },
4892         'virtioscsi11' => { bus => 3, addr => 12 },
4893         'virtioscsi12' => { bus => 3, addr => 13 },
4894         'virtioscsi13' => { bus => 3, addr => 14 },
4895         'virtioscsi14' => { bus => 3, addr => 15 },
4896         'virtioscsi15' => { bus => 3, addr => 16 },
4897         'virtioscsi16' => { bus => 3, addr => 17 },
4898         'virtioscsi17' => { bus => 3, addr => 18 },
4899         'virtioscsi18' => { bus => 3, addr => 19 },
4900         'virtioscsi19' => { bus => 3, addr => 20 },
4901         'virtioscsi20' => { bus => 3, addr => 21 },
4902         'virtioscsi21' => { bus => 3, addr => 22 },
4903         'virtioscsi22' => { bus => 3, addr => 23 },
4904         'virtioscsi23' => { bus => 3, addr => 24 },
4905         'virtioscsi24' => { bus => 3, addr => 25 },
4906         'virtioscsi25' => { bus => 3, addr => 26 },
4907         'virtioscsi26' => { bus => 3, addr => 27 },
4908         'virtioscsi27' => { bus => 3, addr => 28 },
4909         'virtioscsi28' => { bus => 3, addr => 29 },
4910         'virtioscsi29' => { bus => 3, addr => 30 },
4911         'virtioscsi30' => { bus => 3, addr => 31 },
4912
4913     };
4914
4915     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
4916            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
4917            my $bus = $devices->{$id}->{bus};
4918            $res = ",bus=pci.$bus,addr=$addr";
4919            $bridges->{$bus} = 1 if $bridges;
4920     }
4921     return $res;
4922
4923 }
4924
4925 sub print_pcie_addr {
4926     my ($id) = @_;
4927
4928     my $res = '';
4929     my $devices = {
4930         hostpci0 => { bus => "ich9-pcie-port-1", addr => 0 },
4931         hostpci1 => { bus => "ich9-pcie-port-2", addr => 0 },
4932         hostpci2 => { bus => "ich9-pcie-port-3", addr => 0 },
4933         hostpci3 => { bus => "ich9-pcie-port-4", addr => 0 },
4934     };
4935
4936     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
4937            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
4938            my $bus = $devices->{$id}->{bus};
4939            $res = ",bus=$bus,addr=$addr";
4940     }
4941     return $res;
4942
4943 }
4944
4945 # vzdump restore implementaion
4946
4947 sub tar_archive_read_firstfile {
4948     my $archive = shift;
4949
4950     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
4951
4952     # try to detect archive type first
4953     my $pid = open (TMP, "tar tf '$archive'|") ||
4954         die "unable to open file '$archive'\n";
4955     my $firstfile = <TMP>;
4956     kill 15, $pid;
4957     close TMP;
4958
4959     die "ERROR: archive contaions no data\n" if !$firstfile;
4960     chomp $firstfile;
4961
4962     return $firstfile;
4963 }
4964
4965 sub tar_restore_cleanup {
4966     my ($storecfg, $statfile) = @_;
4967
4968     print STDERR "starting cleanup\n";
4969
4970     if (my $fd = IO::File->new($statfile, "r")) {
4971         while (defined(my $line = <$fd>)) {
4972             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
4973                 my $volid = $2;
4974                 eval {
4975                     if ($volid =~ m|^/|) {
4976                         unlink $volid || die 'unlink failed\n';
4977                     } else {
4978                         PVE::Storage::vdisk_free($storecfg, $volid);
4979                     }
4980                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
4981                 };
4982                 print STDERR "unable to cleanup '$volid' - $@" if $@;
4983             } else {
4984                 print STDERR "unable to parse line in statfile - $line";
4985             }
4986         }
4987         $fd->close();
4988     }
4989 }
4990
4991 sub restore_archive {
4992     my ($archive, $vmid, $user, $opts) = @_;
4993
4994     my $format = $opts->{format};
4995     my $comp;
4996
4997     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
4998         $format = 'tar' if !$format;
4999         $comp = 'gzip';
5000     } elsif ($archive =~ m/\.tar$/) {
5001         $format = 'tar' if !$format;
5002     } elsif ($archive =~ m/.tar.lzo$/) {
5003         $format = 'tar' if !$format;
5004         $comp = 'lzop';
5005     } elsif ($archive =~ m/\.vma$/) {
5006         $format = 'vma' if !$format;
5007     } elsif ($archive =~ m/\.vma\.gz$/) {
5008         $format = 'vma' if !$format;
5009         $comp = 'gzip';
5010     } elsif ($archive =~ m/\.vma\.lzo$/) {
5011         $format = 'vma' if !$format;
5012         $comp = 'lzop';
5013     } else {
5014         $format = 'vma' if !$format; # default
5015     }
5016
5017     # try to detect archive format
5018     if ($format eq 'tar') {
5019         return restore_tar_archive($archive, $vmid, $user, $opts);
5020     } else {
5021         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5022     }
5023 }
5024
5025 sub restore_update_config_line {
5026     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5027
5028     return if $line =~ m/^\#qmdump\#/;
5029     return if $line =~ m/^\#vzdump\#/;
5030     return if $line =~ m/^lock:/;
5031     return if $line =~ m/^unused\d+:/;
5032     return if $line =~ m/^parent:/;
5033     return if $line =~ m/^template:/; # restored VM is never a template
5034
5035     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5036         # try to convert old 1.X settings
5037         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5038         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5039             my ($model, $macaddr) = split(/\=/, $devconfig);
5040             $macaddr = PVE::Tools::random_ether_addr() if !$macaddr || $unique;
5041             my $net = {
5042                 model => $model,
5043                 bridge => "vmbr$ind",
5044                 macaddr => $macaddr,
5045             };
5046             my $netstr = print_net($net);
5047
5048             print $outfd "net$cookie->{netcount}: $netstr\n";
5049             $cookie->{netcount}++;
5050         }
5051     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5052         my ($id, $netstr) = ($1, $2);
5053         my $net = parse_net($netstr);
5054         $net->{macaddr} = PVE::Tools::random_ether_addr() if $net->{macaddr};
5055         $netstr = print_net($net);
5056         print $outfd "$id: $netstr\n";
5057     } elsif ($line =~ m/^((ide|scsi|virtio|sata)\d+):\s*(\S+)\s*$/) {
5058         my $virtdev = $1;
5059         my $value = $3;
5060         if ($line =~ m/backup=no/) {
5061             print $outfd "#$line";
5062         } elsif ($virtdev && $map->{$virtdev}) {
5063             my $di = parse_drive($virtdev, $value);
5064             delete $di->{format}; # format can change on restore
5065             $di->{file} = $map->{$virtdev};
5066             $value = print_drive($vmid, $di);
5067             print $outfd "$virtdev: $value\n";
5068         } else {
5069             print $outfd $line;
5070         }
5071     } else {
5072         print $outfd $line;
5073     }
5074 }
5075
5076 sub scan_volids {
5077     my ($cfg, $vmid) = @_;
5078
5079     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5080
5081     my $volid_hash = {};
5082     foreach my $storeid (keys %$info) {
5083         foreach my $item (@{$info->{$storeid}}) {
5084             next if !($item->{volid} && $item->{size});
5085             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5086             $volid_hash->{$item->{volid}} = $item;
5087         }
5088     }
5089
5090     return $volid_hash;
5091 }
5092
5093 sub get_used_paths {
5094     my ($vmid, $storecfg, $conf, $scan_snapshots, $skip_drive) = @_;
5095
5096     my $used_path = {};
5097
5098     my $scan_config = sub {
5099         my ($cref, $snapname) = @_;
5100
5101         foreach my $key (keys %$cref) {
5102             my $value = $cref->{$key};
5103             if (valid_drivename($key)) {
5104                 next if $skip_drive && $key eq $skip_drive;
5105                 my $drive = parse_drive($key, $value);
5106                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5107                 if ($drive->{file} =~ m!^/!) {
5108                     $used_path->{$drive->{file}}++; # = 1;
5109                 } else {
5110                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5111                     next if !$storeid;
5112                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5113                     next if !$scfg;
5114                     my $path = PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5115                     $used_path->{$path}++; # = 1;
5116                 }
5117             }
5118         }
5119     };
5120
5121     &$scan_config($conf);
5122
5123     undef $skip_drive;
5124
5125     if ($scan_snapshots) {
5126         foreach my $snapname (keys %{$conf->{snapshots}}) {
5127             &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5128         }
5129     }
5130
5131     return $used_path;
5132 }
5133
5134 sub update_disksize {
5135     my ($vmid, $conf, $volid_hash) = @_;
5136
5137     my $changes;
5138
5139     my $used = {};
5140
5141     # Note: it is allowed to define multiple storages with same path (alias), so
5142     # we need to check both 'volid' and real 'path' (two different volid can point
5143     # to the same path).
5144
5145     my $usedpath = {};
5146
5147     # update size info
5148     foreach my $opt (keys %$conf) {
5149         if (valid_drivename($opt)) {
5150             my $drive = parse_drive($opt, $conf->{$opt});
5151             my $volid = $drive->{file};
5152             next if !$volid;
5153
5154             $used->{$volid} = 1;
5155             if ($volid_hash->{$volid} &&
5156                 (my $path = $volid_hash->{$volid}->{path})) {
5157                 $usedpath->{$path} = 1;
5158             }
5159
5160             next if drive_is_cdrom($drive);
5161             next if !$volid_hash->{$volid};
5162
5163             $drive->{size} = $volid_hash->{$volid}->{size};
5164             my $new = print_drive($vmid, $drive);
5165             if ($new ne $conf->{$opt}) {
5166                 $changes = 1;
5167                 $conf->{$opt} = $new;
5168             }
5169         }
5170     }
5171
5172     # remove 'unusedX' entry if volume is used
5173     foreach my $opt (keys %$conf) {
5174         next if $opt !~ m/^unused\d+$/;
5175         my $volid = $conf->{$opt};
5176         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5177         if ($used->{$volid} || ($path && $usedpath->{$path})) {
5178             $changes = 1;
5179             delete $conf->{$opt};
5180         }
5181     }
5182
5183     foreach my $volid (sort keys %$volid_hash) {
5184         next if $volid =~ m/vm-$vmid-state-/;
5185         next if $used->{$volid};
5186         my $path = $volid_hash->{$volid}->{path};
5187         next if !$path; # just to be sure
5188         next if $usedpath->{$path};
5189         $changes = 1;
5190         add_unused_volume($conf, $volid);
5191         $usedpath->{$path} = 1; # avoid to add more than once (aliases)
5192     }
5193
5194     return $changes;
5195 }
5196
5197 sub rescan {
5198     my ($vmid, $nolock) = @_;
5199
5200     my $cfg = PVE::Cluster::cfs_read_file("storage.cfg");
5201
5202     my $volid_hash = scan_volids($cfg, $vmid);
5203
5204     my $updatefn =  sub {
5205         my ($vmid) = @_;
5206
5207         my $conf = load_config($vmid);
5208
5209         check_lock($conf);
5210
5211         my $vm_volids = {};
5212         foreach my $volid (keys %$volid_hash) {
5213             my $info = $volid_hash->{$volid};
5214             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5215         }
5216
5217         my $changes = update_disksize($vmid, $conf, $vm_volids);
5218
5219         update_config_nolock($vmid, $conf, 1) if $changes;
5220     };
5221
5222     if (defined($vmid)) {
5223         if ($nolock) {
5224             &$updatefn($vmid);
5225         } else {
5226             lock_config($vmid, $updatefn, $vmid);
5227         }
5228     } else {
5229         my $vmlist = config_list();
5230         foreach my $vmid (keys %$vmlist) {
5231             if ($nolock) {
5232                 &$updatefn($vmid);
5233             } else {
5234                 lock_config($vmid, $updatefn, $vmid);
5235             }
5236         }
5237     }
5238 }
5239
5240 sub restore_vma_archive {
5241     my ($archive, $vmid, $user, $opts, $comp) = @_;
5242
5243     my $input = $archive eq '-' ? "<&STDIN" : undef;
5244     my $readfrom = $archive;
5245
5246     my $uncomp = '';
5247     if ($comp) {
5248         $readfrom = '-';
5249         my $qarchive = PVE::Tools::shellquote($archive);
5250         if ($comp eq 'gzip') {
5251             $uncomp = "zcat $qarchive|";
5252         } elsif ($comp eq 'lzop') {
5253             $uncomp = "lzop -d -c $qarchive|";
5254         } else {
5255             die "unknown compression method '$comp'\n";
5256         }
5257
5258     }
5259
5260     my $tmpdir = "/var/tmp/vzdumptmp$$";
5261     rmtree $tmpdir;
5262
5263     # disable interrupts (always do cleanups)
5264     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5265         warn "got interrupt - ignored\n";
5266     };
5267
5268     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5269     POSIX::mkfifo($mapfifo, 0600);
5270     my $fifofh;
5271
5272     my $openfifo = sub {
5273         open($fifofh, '>', $mapfifo) || die $!;
5274     };
5275
5276     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5277
5278     my $oldtimeout;
5279     my $timeout = 5;
5280
5281     my $devinfo = {};
5282
5283     my $rpcenv = PVE::RPCEnvironment::get();
5284
5285     my $conffile = config_file($vmid);
5286     my $tmpfn = "$conffile.$$.tmp";
5287
5288     # Note: $oldconf is undef if VM does not exists
5289     my $oldconf = PVE::Cluster::cfs_read_file(cfs_config_path($vmid));
5290
5291     my $print_devmap = sub {
5292         my $virtdev_hash = {};
5293
5294         my $cfgfn = "$tmpdir/qemu-server.conf";
5295
5296         # we can read the config - that is already extracted
5297         my $fh = IO::File->new($cfgfn, "r") ||
5298             "unable to read qemu-server.conf - $!\n";
5299
5300         while (defined(my $line = <$fh>)) {
5301             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5302                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5303                 die "archive does not contain data for drive '$virtdev'\n"
5304                     if !$devinfo->{$devname};
5305                 if (defined($opts->{storage})) {
5306                     $storeid = $opts->{storage} || 'local';
5307                 } elsif (!$storeid) {
5308                     $storeid = 'local';
5309                 }
5310                 $format = 'raw' if !$format;
5311                 $devinfo->{$devname}->{devname} = $devname;
5312                 $devinfo->{$devname}->{virtdev} = $virtdev;
5313                 $devinfo->{$devname}->{format} = $format;
5314                 $devinfo->{$devname}->{storeid} = $storeid;
5315
5316                 # check permission on storage
5317                 my $pool = $opts->{pool}; # todo: do we need that?
5318                 if ($user ne 'root@pam') {
5319                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5320                 }
5321
5322                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5323             }
5324         }
5325
5326         foreach my $devname (keys %$devinfo) {
5327             die "found no device mapping information for device '$devname'\n"
5328                 if !$devinfo->{$devname}->{virtdev};
5329         }
5330
5331         my $cfg = cfs_read_file('storage.cfg');
5332
5333         # create empty/temp config
5334         if ($oldconf) {
5335             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5336             foreach_drive($oldconf, sub {
5337                 my ($ds, $drive) = @_;
5338
5339                 return if drive_is_cdrom($drive);
5340
5341                 my $volid = $drive->{file};
5342
5343                 return if !$volid || $volid =~ m|^/|;
5344
5345                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5346                 return if !$path || !$owner || ($owner != $vmid);
5347
5348                 # Note: only delete disk we want to restore
5349                 # other volumes will become unused
5350                 if ($virtdev_hash->{$ds}) {
5351                     PVE::Storage::vdisk_free($cfg, $volid);
5352                 }
5353             });
5354         }
5355
5356         my $map = {};
5357         foreach my $virtdev (sort keys %$virtdev_hash) {
5358             my $d = $virtdev_hash->{$virtdev};
5359             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5360             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5361
5362             # test if requested format is supported
5363             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5364             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5365             $d->{format} = $defFormat if !$supported;
5366
5367             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5368                                                   $d->{format}, undef, $alloc_size);
5369             print STDERR "new volume ID is '$volid'\n";
5370             $d->{volid} = $volid;
5371             my $path = PVE::Storage::path($cfg, $volid);
5372
5373             PVE::Storage::activate_volumes($cfg,[$volid]);
5374
5375             my $write_zeros = 1;
5376             # fixme: what other storages types initialize volumes with zero?
5377             if ($scfg->{type} eq 'dir' || $scfg->{type} eq 'nfs' || $scfg->{type} eq 'glusterfs' ||
5378                 $scfg->{type} eq 'sheepdog' || $scfg->{type} eq 'rbd') {
5379                 $write_zeros = 0;
5380             }
5381
5382             print $fifofh "${write_zeros}:$d->{devname}=$path\n";
5383
5384             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5385             $map->{$virtdev} = $volid;
5386         }
5387
5388         $fh->seek(0, 0) || die "seek failed - $!\n";
5389
5390         my $outfd = new IO::File ($tmpfn, "w") ||
5391             die "unable to write config for VM $vmid\n";
5392
5393         my $cookie = { netcount => 0 };
5394         while (defined(my $line = <$fh>)) {
5395             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5396         }
5397
5398         $fh->close();
5399         $outfd->close();
5400     };
5401
5402     eval {
5403         # enable interrupts
5404         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5405             die "interrupted by signal\n";
5406         };
5407         local $SIG{ALRM} = sub { die "got timeout\n"; };
5408
5409         $oldtimeout = alarm($timeout);
5410
5411         my $parser = sub {
5412             my $line = shift;
5413
5414             print "$line\n";
5415
5416             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5417                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5418                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5419             } elsif ($line =~ m/^CTIME: /) {
5420                 # we correctly received the vma config, so we can disable
5421                 # the timeout now for disk allocation (set to 10 minutes, so
5422                 # that we always timeout if something goes wrong)
5423                 alarm(600);
5424                 &$print_devmap();
5425                 print $fifofh "done\n";
5426                 my $tmp = $oldtimeout || 0;
5427                 $oldtimeout = undef;
5428                 alarm($tmp);
5429                 close($fifofh);
5430             }
5431         };
5432
5433         print "restore vma archive: $cmd\n";
5434         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5435     };
5436     my $err = $@;
5437
5438     alarm($oldtimeout) if $oldtimeout;
5439
5440     my $vollist = [];
5441     foreach my $devname (keys %$devinfo) {
5442         my $volid = $devinfo->{$devname}->{volid};
5443         push @$vollist, $volid if $volid;
5444     }
5445
5446     my $cfg = cfs_read_file('storage.cfg');
5447     PVE::Storage::deactivate_volumes($cfg, $vollist);
5448
5449     unlink $mapfifo;
5450
5451     if ($err) {
5452         rmtree $tmpdir;
5453         unlink $tmpfn;
5454
5455         foreach my $devname (keys %$devinfo) {
5456             my $volid = $devinfo->{$devname}->{volid};
5457             next if !$volid;
5458             eval {
5459                 if ($volid =~ m|^/|) {
5460                     unlink $volid || die 'unlink failed\n';
5461                 } else {
5462                     PVE::Storage::vdisk_free($cfg, $volid);
5463                 }
5464                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5465             };
5466             print STDERR "unable to cleanup '$volid' - $@" if $@;
5467         }
5468         die $err;
5469     }
5470
5471     rmtree $tmpdir;
5472
5473     rename($tmpfn, $conffile) ||
5474         die "unable to commit configuration file '$conffile'\n";
5475
5476     PVE::Cluster::cfs_update(); # make sure we read new file
5477
5478     eval { rescan($vmid, 1); };
5479     warn $@ if $@;
5480 }
5481
5482 sub restore_tar_archive {
5483     my ($archive, $vmid, $user, $opts) = @_;
5484
5485     if ($archive ne '-') {
5486         my $firstfile = tar_archive_read_firstfile($archive);
5487         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5488             if $firstfile ne 'qemu-server.conf';
5489     }
5490
5491     my $storecfg = cfs_read_file('storage.cfg');
5492
5493     # destroy existing data - keep empty config
5494     my $vmcfgfn = config_file($vmid);
5495     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5496
5497     my $tocmd = "/usr/lib/qemu-server/qmextract";
5498
5499     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5500     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5501     $tocmd .= ' --prealloc' if $opts->{prealloc};
5502     $tocmd .= ' --info' if $opts->{info};
5503
5504     # tar option "xf" does not autodetect compression when read from STDIN,
5505     # so we pipe to zcat
5506     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5507         PVE::Tools::shellquote("--to-command=$tocmd");
5508
5509     my $tmpdir = "/var/tmp/vzdumptmp$$";
5510     mkpath $tmpdir;
5511
5512     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5513     local $ENV{VZDUMP_VMID} = $vmid;
5514     local $ENV{VZDUMP_USER} = $user;
5515
5516     my $conffile = config_file($vmid);
5517     my $tmpfn = "$conffile.$$.tmp";
5518
5519     # disable interrupts (always do cleanups)
5520     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5521         print STDERR "got interrupt - ignored\n";
5522     };
5523
5524     eval {
5525         # enable interrupts
5526         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5527             die "interrupted by signal\n";
5528         };
5529
5530         if ($archive eq '-') {
5531             print "extracting archive from STDIN\n";
5532             run_command($cmd, input => "<&STDIN");
5533         } else {
5534             print "extracting archive '$archive'\n";
5535             run_command($cmd);
5536         }
5537
5538         return if $opts->{info};
5539
5540         # read new mapping
5541         my $map = {};
5542         my $statfile = "$tmpdir/qmrestore.stat";
5543         if (my $fd = IO::File->new($statfile, "r")) {
5544             while (defined (my $line = <$fd>)) {
5545                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5546                     $map->{$1} = $2 if $1;
5547                 } else {
5548                     print STDERR "unable to parse line in statfile - $line\n";
5549                 }
5550             }
5551             $fd->close();
5552         }
5553
5554         my $confsrc = "$tmpdir/qemu-server.conf";
5555
5556         my $srcfd = new IO::File($confsrc, "r") ||
5557             die "unable to open file '$confsrc'\n";
5558
5559         my $outfd = new IO::File ($tmpfn, "w") ||
5560             die "unable to write config for VM $vmid\n";
5561
5562         my $cookie = { netcount => 0 };
5563         while (defined (my $line = <$srcfd>)) {
5564             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5565         }
5566
5567         $srcfd->close();
5568         $outfd->close();
5569     };
5570     my $err = $@;
5571
5572     if ($err) {
5573
5574         unlink $tmpfn;
5575
5576         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
5577
5578         die $err;
5579     }
5580
5581     rmtree $tmpdir;
5582
5583     rename $tmpfn, $conffile ||
5584         die "unable to commit configuration file '$conffile'\n";
5585
5586     PVE::Cluster::cfs_update(); # make sure we read new file
5587
5588     eval { rescan($vmid, 1); };
5589     warn $@ if $@;
5590 };
5591
5592
5593 # Internal snapshots
5594
5595 # NOTE: Snapshot create/delete involves several non-atomic
5596 # action, and can take a long time.
5597 # So we try to avoid locking the file and use 'lock' variable
5598 # inside the config file instead.
5599
5600 my $snapshot_copy_config = sub {
5601     my ($source, $dest) = @_;
5602
5603     foreach my $k (keys %$source) {
5604         next if $k eq 'snapshots';
5605         next if $k eq 'snapstate';
5606         next if $k eq 'snaptime';
5607         next if $k eq 'vmstate';
5608         next if $k eq 'lock';
5609         next if $k eq 'digest';
5610         next if $k eq 'description';
5611         next if $k =~ m/^unused\d+$/;
5612
5613         $dest->{$k} = $source->{$k};
5614     }
5615 };
5616
5617 my $snapshot_apply_config = sub {
5618     my ($conf, $snap) = @_;
5619
5620     # copy snapshot list
5621     my $newconf = {
5622         snapshots => $conf->{snapshots},
5623     };
5624
5625     # keep description and list of unused disks
5626     foreach my $k (keys %$conf) {
5627         next if !($k =~ m/^unused\d+$/ || $k eq 'description');
5628         $newconf->{$k} = $conf->{$k};
5629     }
5630
5631     &$snapshot_copy_config($snap, $newconf);
5632
5633     return $newconf;
5634 };
5635
5636 sub foreach_writable_storage {
5637     my ($conf, $func) = @_;
5638
5639     my $sidhash = {};
5640
5641     foreach my $ds (keys %$conf) {
5642         next if !valid_drivename($ds);
5643
5644         my $drive = parse_drive($ds, $conf->{$ds});
5645         next if !$drive;
5646         next if drive_is_cdrom($drive);
5647
5648         my $volid = $drive->{file};
5649
5650         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5651         $sidhash->{$sid} = $sid if $sid;
5652     }
5653
5654     foreach my $sid (sort keys %$sidhash) {
5655         &$func($sid);
5656     }
5657 }
5658
5659 my $alloc_vmstate_volid = sub {
5660     my ($storecfg, $vmid, $conf, $snapname) = @_;
5661
5662     # Note: we try to be smart when selecting a $target storage
5663
5664     my $target;
5665
5666     # search shared storage first
5667     foreach_writable_storage($conf, sub {
5668         my ($sid) = @_;
5669         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
5670         return if !$scfg->{shared};
5671
5672         $target = $sid if !$target || $scfg->{path}; # prefer file based storage
5673     });
5674
5675     if (!$target) {
5676         # now search local storage
5677         foreach_writable_storage($conf, sub {
5678             my ($sid) = @_;
5679             my $scfg = PVE::Storage::storage_config($storecfg, $sid);
5680             return if $scfg->{shared};
5681
5682             $target = $sid if !$target || $scfg->{path}; # prefer file based storage;
5683         });
5684     }
5685
5686     $target = 'local' if !$target;
5687
5688     my $driver_state_size = 500; # assume 32MB is enough to safe all driver state;
5689     # we abort live save after $conf->{memory}, so we need at max twice that space
5690     my $size = $conf->{memory}*2 + $driver_state_size;
5691
5692     my $name = "vm-$vmid-state-$snapname";
5693     my $scfg = PVE::Storage::storage_config($storecfg, $target);
5694     $name .= ".raw" if $scfg->{path}; # add filename extension for file base storage
5695     my $volid = PVE::Storage::vdisk_alloc($storecfg, $target, $vmid, 'raw', $name, $size*1024);
5696
5697     return $volid;
5698 };
5699
5700 my $snapshot_prepare = sub {
5701     my ($vmid, $snapname, $save_vmstate, $comment) = @_;
5702
5703     my $snap;
5704
5705     my $updatefn =  sub {
5706
5707         my $conf = load_config($vmid);
5708
5709         die "you can't take a snapshot if it's a template\n"
5710             if is_template($conf);
5711
5712         check_lock($conf);
5713
5714         $conf->{lock} = 'snapshot';
5715
5716         die "snapshot name '$snapname' already used\n"
5717             if defined($conf->{snapshots}->{$snapname});
5718
5719         my $storecfg = PVE::Storage::config();
5720         die "snapshot feature is not available" if !has_feature('snapshot', $conf, $storecfg);
5721
5722         $snap = $conf->{snapshots}->{$snapname} = {};
5723
5724         if ($save_vmstate && check_running($vmid)) {
5725             $snap->{vmstate} = &$alloc_vmstate_volid($storecfg, $vmid, $conf, $snapname);
5726         }
5727
5728         &$snapshot_copy_config($conf, $snap);
5729
5730         $snap->{snapstate} = "prepare";
5731         $snap->{snaptime} = time();
5732         $snap->{description} = $comment if $comment;
5733
5734         # always overwrite machine if we save vmstate. This makes sure we
5735         # can restore it later using correct machine type
5736         $snap->{machine} = get_current_qemu_machine($vmid) if $snap->{vmstate};
5737
5738         update_config_nolock($vmid, $conf, 1);
5739     };
5740
5741     lock_config($vmid, $updatefn);
5742
5743     return $snap;
5744 };
5745
5746 my $snapshot_commit = sub {
5747     my ($vmid, $snapname) = @_;
5748
5749     my $updatefn = sub {
5750
5751         my $conf = load_config($vmid);
5752
5753         die "missing snapshot lock\n"
5754             if !($conf->{lock} && $conf->{lock} eq 'snapshot');
5755
5756         my $has_machine_config = defined($conf->{machine});
5757
5758         my $snap = $conf->{snapshots}->{$snapname};
5759
5760         die "snapshot '$snapname' does not exist\n" if !defined($snap);
5761
5762         die "wrong snapshot state\n"
5763             if !($snap->{snapstate} && $snap->{snapstate} eq "prepare");
5764
5765         delete $snap->{snapstate};
5766         delete $conf->{lock};
5767
5768         my $newconf = &$snapshot_apply_config($conf, $snap);
5769
5770         delete $newconf->{machine} if !$has_machine_config;
5771
5772         $newconf->{parent} = $snapname;
5773
5774         update_config_nolock($vmid, $newconf, 1);
5775     };
5776
5777     lock_config($vmid, $updatefn);
5778 };
5779
5780 sub snapshot_rollback {
5781     my ($vmid, $snapname) = @_;
5782
5783     my $prepare = 1;
5784
5785     my $storecfg = PVE::Storage::config();
5786
5787     my $conf = load_config($vmid);
5788
5789     my $get_snapshot_config = sub {
5790
5791         die "you can't rollback if vm is a template\n" if is_template($conf);
5792
5793         my $res = $conf->{snapshots}->{$snapname};
5794
5795         die "snapshot '$snapname' does not exist\n" if !defined($res);
5796
5797         return $res;
5798     };
5799
5800     my $snap = &$get_snapshot_config();
5801
5802     foreach_drive($snap, sub {
5803         my ($ds, $drive) = @_;
5804
5805         return if drive_is_cdrom($drive);
5806
5807         my $volid = $drive->{file};
5808
5809         PVE::Storage::volume_rollback_is_possible($storecfg, $volid, $snapname);
5810     });
5811
5812     my $updatefn = sub {
5813
5814         $conf = load_config($vmid);
5815
5816         $snap = &$get_snapshot_config();
5817
5818         die "unable to rollback to incomplete snapshot (snapstate = $snap->{snapstate})\n"
5819             if $snap->{snapstate};
5820
5821         if ($prepare) {
5822             check_lock($conf);
5823             vm_stop($storecfg, $vmid, undef, undef, 5, undef, undef);
5824         }
5825
5826         die "unable to rollback vm $vmid: vm is running\n"
5827             if check_running($vmid);
5828
5829         if ($prepare) {
5830             $conf->{lock} = 'rollback';
5831         } else {
5832             die "got wrong lock\n" if !($conf->{lock} && $conf->{lock} eq 'rollback');
5833             delete $conf->{lock};
5834         }
5835
5836         my $forcemachine;
5837
5838         if (!$prepare) {
5839             my $has_machine_config = defined($conf->{machine});
5840
5841             # copy snapshot config to current config
5842             $conf = &$snapshot_apply_config($conf, $snap);
5843             $conf->{parent} = $snapname;
5844
5845             # Note: old code did not store 'machine', so we try to be smart
5846             # and guess the snapshot was generated with kvm 1.4 (pc-i440fx-1.4).
5847             $forcemachine = $conf->{machine} || 'pc-i440fx-1.4';
5848             # we remove the 'machine' configuration if not explicitly specified
5849             # in the original config.
5850             delete $conf->{machine} if $snap->{vmstate} && !$has_machine_config;
5851         }
5852
5853         update_config_nolock($vmid, $conf, 1);
5854
5855         if (!$prepare && $snap->{vmstate}) {
5856             my $statefile = PVE::Storage::path($storecfg, $snap->{vmstate});
5857             vm_start($storecfg, $vmid, $statefile, undef, undef, undef, $forcemachine);
5858         }
5859     };
5860
5861     lock_config($vmid, $updatefn);
5862
5863     foreach_drive($snap, sub {
5864         my ($ds, $drive) = @_;
5865
5866         return if drive_is_cdrom($drive);
5867
5868         my $volid = $drive->{file};
5869         my $device = "drive-$ds";
5870
5871         PVE::Storage::volume_snapshot_rollback($storecfg, $volid, $snapname);
5872     });
5873
5874     $prepare = 0;
5875     lock_config($vmid, $updatefn);
5876 }
5877
5878 my $savevm_wait = sub {
5879     my ($vmid) = @_;
5880
5881     for(;;) {
5882         my $stat = vm_mon_cmd_nocheck($vmid, "query-savevm");
5883         if (!$stat->{status}) {
5884             die "savevm not active\n";
5885         } elsif ($stat->{status} eq 'active') {
5886             sleep(1);
5887             next;
5888         } elsif ($stat->{status} eq 'completed') {
5889             last;
5890         } else {
5891             die "query-savevm returned status '$stat->{status}'\n";
5892         }
5893     }
5894 };
5895
5896 sub do_snapshots_with_qemu {
5897     my ($storecfg, $volid) = @_;
5898
5899     my $storage_name = PVE::Storage::parse_volume_id($volid);
5900
5901     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
5902         && !$storecfg->{ids}->{$storage_name}->{krbd}){
5903         return 1;
5904     }
5905
5906     if ($volid =~ m/\.(qcow2|qed)$/){
5907         return 1;
5908     }
5909
5910     return undef;
5911 }
5912
5913 sub snapshot_create {
5914     my ($vmid, $snapname, $save_vmstate, $comment) = @_;
5915
5916     my $snap = &$snapshot_prepare($vmid, $snapname, $save_vmstate, $comment);
5917
5918     $save_vmstate = 0 if !$snap->{vmstate}; # vm is not running
5919
5920     my $config = load_config($vmid);
5921
5922     my $running = check_running($vmid);
5923
5924     my $freezefs = $running && $config->{agent};
5925     $freezefs = 0 if $snap->{vmstate}; # not needed if we save RAM
5926
5927     my $drivehash = {};
5928
5929     if ($freezefs) {
5930         eval { vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
5931         warn "guest-fsfreeze-freeze problems - $@" if $@;
5932     }
5933
5934     eval {
5935         # create internal snapshots of all drives
5936
5937         my $storecfg = PVE::Storage::config();
5938
5939         if ($running) {
5940             if ($snap->{vmstate}) {
5941                 my $path = PVE::Storage::path($storecfg, $snap->{vmstate});
5942                 vm_mon_cmd($vmid, "savevm-start", statefile => $path);
5943                 &$savevm_wait($vmid);
5944             } else {
5945                 vm_mon_cmd($vmid, "savevm-start");
5946             }
5947         };
5948
5949         foreach_drive($snap, sub {
5950             my ($ds, $drive) = @_;
5951
5952             return if drive_is_cdrom($drive);
5953
5954             my $volid = $drive->{file};
5955             my $device = "drive-$ds";
5956
5957             qemu_volume_snapshot($vmid, $device, $storecfg, $volid, $snapname);
5958             $drivehash->{$ds} = 1;
5959        });
5960     };
5961     my $err = $@;
5962
5963     if ($running) {
5964         eval { vm_mon_cmd($vmid, "savevm-end")  };
5965         warn $@ if $@;
5966
5967         if ($freezefs) {
5968             eval { vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
5969             warn "guest-fsfreeze-thaw problems - $@" if $@;
5970         }
5971
5972         # savevm-end is async, we need to wait
5973         for (;;) {
5974             my $stat = vm_mon_cmd_nocheck($vmid, "query-savevm");
5975             if (!$stat->{bytes}) {
5976                 last;
5977             } else {
5978                 print "savevm not yet finished\n";
5979                 sleep(1);
5980                 next;
5981             }
5982         }
5983     }
5984
5985     if ($err) {
5986         warn "snapshot create failed: starting cleanup\n";
5987         eval { snapshot_delete($vmid, $snapname, 0, $drivehash); };
5988         warn $@ if $@;
5989         die $err;
5990     }
5991
5992     &$snapshot_commit($vmid, $snapname);
5993 }
5994
5995 # Note: $drivehash is only set when called from snapshot_create.
5996 sub snapshot_delete {
5997     my ($vmid, $snapname, $force, $drivehash) = @_;
5998
5999     my $prepare = 1;
6000
6001     my $snap;
6002     my $unused = [];
6003
6004     my $unlink_parent = sub {
6005         my ($confref, $new_parent) = @_;
6006
6007         if ($confref->{parent} && $confref->{parent} eq $snapname) {
6008             if ($new_parent) {
6009                 $confref->{parent} = $new_parent;
6010             } else {
6011                 delete $confref->{parent};
6012             }
6013         }
6014     };
6015
6016     my $updatefn =  sub {
6017         my ($remove_drive) = @_;
6018
6019         my $conf = load_config($vmid);
6020
6021         if (!$drivehash) {
6022             check_lock($conf);
6023             die "you can't delete a snapshot if vm is a template\n"
6024                 if is_template($conf);
6025         }
6026
6027         $snap = $conf->{snapshots}->{$snapname};
6028
6029         die "snapshot '$snapname' does not exist\n" if !defined($snap);
6030
6031         # remove parent refs
6032         if (!$prepare) {
6033             &$unlink_parent($conf, $snap->{parent});
6034             foreach my $sn (keys %{$conf->{snapshots}}) {
6035                 next if $sn eq $snapname;
6036                 &$unlink_parent($conf->{snapshots}->{$sn}, $snap->{parent});
6037             }
6038         }
6039
6040         if ($remove_drive) {
6041             if ($remove_drive eq 'vmstate') {
6042                 delete $snap->{$remove_drive};
6043             } else {
6044                 my $drive = parse_drive($remove_drive, $snap->{$remove_drive});
6045                 my $volid = $drive->{file};
6046                 delete $snap->{$remove_drive};
6047                 add_unused_volume($conf, $volid);
6048             }
6049         }
6050
6051         if ($prepare) {
6052             $snap->{snapstate} = 'delete';
6053         } else {
6054             delete $conf->{snapshots}->{$snapname};
6055             delete $conf->{lock} if $drivehash;
6056             foreach my $volid (@$unused) {
6057                 add_unused_volume($conf, $volid);
6058             }
6059         }
6060
6061         update_config_nolock($vmid, $conf, 1);
6062     };
6063
6064     lock_config($vmid, $updatefn);
6065
6066     # now remove vmstate file
6067
6068     my $storecfg = PVE::Storage::config();
6069
6070     if ($snap->{vmstate}) {
6071         eval {  PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
6072         if (my $err = $@) {
6073             die $err if !$force;
6074             warn $err;
6075         }
6076         # save changes (remove vmstate from snapshot)
6077         lock_config($vmid, $updatefn, 'vmstate') if !$force;
6078     };
6079
6080     # now remove all internal snapshots
6081     foreach_drive($snap, sub {
6082         my ($ds, $drive) = @_;
6083
6084         return if drive_is_cdrom($drive);
6085
6086         my $volid = $drive->{file};
6087         my $device = "drive-$ds";
6088
6089         if (!$drivehash || $drivehash->{$ds}) {
6090             eval { qemu_volume_snapshot_delete($vmid, $device, $storecfg, $volid, $snapname); };
6091             if (my $err = $@) {
6092                 die $err if !$force;
6093                 warn $err;
6094             }
6095         }
6096
6097         # save changes (remove drive fron snapshot)
6098         lock_config($vmid, $updatefn, $ds) if !$force;
6099         push @$unused, $volid;
6100     });
6101
6102     # now cleanup config
6103     $prepare = 0;
6104     lock_config($vmid, $updatefn);
6105 }
6106
6107 sub has_feature {
6108     my ($feature, $conf, $storecfg, $snapname, $running) = @_;
6109
6110     my $err;
6111     foreach_drive($conf, sub {
6112         my ($ds, $drive) = @_;
6113
6114         return if drive_is_cdrom($drive);
6115         my $volid = $drive->{file};
6116         $err = 1 if !PVE::Storage::volume_has_feature($storecfg, $feature, $volid, $snapname, $running);
6117     });
6118
6119     return $err ? 0 : 1;
6120 }
6121
6122 sub template_create {
6123     my ($vmid, $conf, $disk) = @_;
6124
6125     my $storecfg = PVE::Storage::config();
6126
6127     foreach_drive($conf, sub {
6128         my ($ds, $drive) = @_;
6129
6130         return if drive_is_cdrom($drive);
6131         return if $disk && $ds ne $disk;
6132
6133         my $volid = $drive->{file};
6134         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6135
6136         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6137         $drive->{file} = $voliddst;
6138         $conf->{$ds} = print_drive($vmid, $drive);
6139         update_config_nolock($vmid, $conf, 1);
6140     });
6141 }
6142
6143 sub is_template {
6144     my ($conf) = @_;
6145
6146     return 1 if defined $conf->{template} && $conf->{template} == 1;
6147 }
6148
6149 sub qemu_img_convert {
6150     my ($src_volid, $dst_volid, $size, $snapname) = @_;
6151
6152     my $storecfg = PVE::Storage::config();
6153     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6154     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6155
6156     if ($src_storeid && $dst_storeid) {
6157         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6158         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6159
6160         my $src_format = qemu_img_format($src_scfg, $src_volname);
6161         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6162
6163         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6164         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6165
6166         my $cmd = [];
6167         push @$cmd, '/usr/bin/qemu-img', 'convert', '-t', 'writeback', '-p', '-n';
6168         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
6169         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path, $dst_path;
6170
6171         my $parser = sub {
6172             my $line = shift;
6173             if($line =~ m/\((\S+)\/100\%\)/){
6174                 my $percent = $1;
6175                 my $transferred = int($size * $percent / 100);
6176                 my $remaining = $size - $transferred;
6177
6178                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6179             }
6180
6181         };
6182
6183         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6184         my $err = $@;
6185         die "copy failed: $err" if $err;
6186     }
6187 }
6188
6189 sub qemu_img_format {
6190     my ($scfg, $volname) = @_;
6191
6192     if ($scfg->{path} && $volname =~ m/\.(raw|cow|qcow|qcow2|qed|vmdk|cloop)$/) {
6193         return $1;
6194     } else {
6195         return "raw";
6196     }
6197 }
6198
6199 sub qemu_drive_mirror {
6200     my ($vmid, $drive, $dst_volid, $vmiddst) = @_;
6201
6202     my $storecfg = PVE::Storage::config();
6203     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6204
6205     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6206
6207     my $format = qemu_img_format($dst_scfg, $dst_volname);
6208
6209     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6210
6211     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $dst_path };
6212     $opts->{format} = $format if $format;
6213
6214     print "drive mirror is starting (scanning bitmap) : this step can take some minutes/hours, depend of disk size and storage speed\n";
6215
6216     eval {
6217     vm_mon_cmd($vmid, "drive-mirror", %$opts);
6218         while (1) {
6219             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6220             my $stat = @$stats[0];
6221             die "mirroring job seem to have die. Maybe do you have bad sectors?" if !$stat;
6222             die "error job is not mirroring" if $stat->{type} ne "mirror";
6223
6224             my $busy = $stat->{busy};
6225             my $ready = $stat->{ready};
6226
6227             if (my $total = $stat->{len}) {
6228                 my $transferred = $stat->{offset} || 0;
6229                 my $remaining = $total - $transferred;
6230                 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6231
6232                 print "transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6233             }
6234
6235
6236             if ($stat->{ready} eq 'true') {
6237
6238                 last if $vmiddst != $vmid;
6239
6240                 # try to switch the disk if source and destination are on the same guest
6241                 eval { vm_mon_cmd($vmid, "block-job-complete", device => "drive-$drive") };
6242                 last if !$@;
6243                 die $@ if $@ !~ m/cannot be completed/;
6244             }
6245             sleep 1;
6246         }
6247
6248
6249     };
6250     my $err = $@;
6251
6252     my $cancel_job = sub {
6253         vm_mon_cmd($vmid, "block-job-cancel", device => "drive-$drive");
6254         while (1) {
6255             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6256             my $stat = @$stats[0];
6257             last if !$stat;
6258             sleep 1;
6259         }
6260     };
6261
6262     if ($err) {
6263         eval { &$cancel_job(); };
6264         die "mirroring error: $err";
6265     }
6266
6267     if ($vmiddst != $vmid) {
6268         # if we clone a disk for a new target vm, we don't switch the disk
6269         &$cancel_job(); # so we call block-job-cancel
6270     }
6271 }
6272
6273 sub clone_disk {
6274     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6275         $newvmid, $storage, $format, $full, $newvollist) = @_;
6276
6277     my $newvolid;
6278
6279     if (!$full) {
6280         print "create linked clone of drive $drivename ($drive->{file})\n";
6281         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6282         push @$newvollist, $newvolid;
6283     } else {
6284         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6285         $storeid = $storage if $storage;
6286
6287         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6288         if (!$format) {
6289             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6290             $format = qemu_img_format($scfg, $volname);
6291         }
6292
6293         # test if requested format is supported - else use default
6294         my $supported = grep { $_ eq $format } @$validFormats;
6295         $format = $defFormat if !$supported;
6296
6297         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6298
6299         print "create full clone of drive $drivename ($drive->{file})\n";
6300         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $format, undef, ($size/1024));
6301         push @$newvollist, $newvolid;
6302
6303         PVE::Storage::activate_volumes($storecfg, $newvollist);
6304
6305         if (!$running || $snapname) {
6306             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname);
6307         } else {
6308             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid);
6309         }
6310     }
6311
6312     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6313
6314     my $disk = $drive;
6315     $disk->{format} = undef;
6316     $disk->{file} = $newvolid;
6317     $disk->{size} = $size;
6318
6319     return $disk;
6320 }
6321
6322 # this only works if VM is running
6323 sub get_current_qemu_machine {
6324     my ($vmid) = @_;
6325
6326     my $cmd = { execute => 'query-machines', arguments => {} };
6327     my $res = vm_qmp_command($vmid, $cmd);
6328
6329     my ($current, $default);
6330     foreach my $e (@$res) {
6331         $default = $e->{name} if $e->{'is-default'};
6332         $current = $e->{name} if $e->{'is-current'};
6333     }
6334
6335     # fallback to the default machine if current is not supported by qemu
6336     return $current || $default || 'pc';
6337 }
6338
6339 sub qemu_machine_feature_enabled {
6340     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6341
6342     my $current_major;
6343     my $current_minor;
6344
6345     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6346
6347         $current_major = $3;
6348         $current_minor = $4;
6349
6350     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6351
6352         $current_major = $1;
6353         $current_minor = $2;
6354     }
6355
6356     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
6357
6358
6359 }
6360
6361 sub lspci {
6362
6363     my $devices = {};
6364
6365     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6366             my (undef, $id, $function) = @_;
6367             my $res = { id => $id, function => $function};
6368             push @{$devices->{$id}}, $res;
6369     });
6370
6371     return $devices;
6372 }
6373
6374 sub vm_iothreads_list {
6375     my ($vmid) = @_;
6376
6377     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6378
6379     my $iothreads = {};
6380     foreach my $iothread (@$res) {
6381         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6382     }
6383
6384     return $iothreads;
6385 }
6386
6387 sub scsihw_infos {
6388     my ($conf, $drive) = @_;
6389
6390     my $maxdev = 0;
6391
6392     if ($conf->{scsihw} && ($conf->{scsihw} =~ m/^lsi/)) {
6393         $maxdev = 7;
6394     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6395         $maxdev = 1;
6396     } else {
6397         $maxdev = 256;
6398     }
6399
6400     my $controller = int($drive->{index} / $maxdev);
6401     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6402
6403     return ($maxdev, $controller, $controller_prefix);
6404 }
6405
6406 # bash completion helper
6407
6408 sub complete_backup_archives {
6409     my ($cmdname, $pname, $cvalue) = @_;
6410
6411     my $cfg = PVE::Storage::config();
6412
6413     my $storeid;
6414
6415     if ($cvalue =~ m/^([^:]+):/) {
6416         $storeid = $1;
6417     }
6418
6419     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6420
6421     my $res = [];
6422     foreach my $id (keys %$data) {
6423         foreach my $item (@{$data->{$id}}) {
6424             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6425             push @$res, $item->{volid} if defined($item->{volid});
6426         }
6427     }
6428
6429     return $res;
6430 }
6431
6432 my $complete_vmid_full = sub {
6433     my ($running) = @_;
6434
6435     my $idlist = vmstatus();
6436
6437     my $res = [];
6438
6439     foreach my $id (keys %$idlist) {
6440         my $d = $idlist->{$id};
6441         if (defined($running)) {
6442             next if $d->{template};
6443             next if $running && $d->{status} ne 'running';
6444             next if !$running && $d->{status} eq 'running';
6445         }
6446         push @$res, $id;
6447
6448     }
6449     return $res;
6450 };
6451
6452 sub complete_vmid {
6453     return &$complete_vmid_full();
6454 }
6455
6456 sub complete_vmid_stopped {
6457     return &$complete_vmid_full(0);
6458 }
6459
6460 sub complete_vmid_running {
6461     return &$complete_vmid_full(1);
6462 }
6463
6464 sub complete_storage {
6465
6466     my $cfg = PVE::Storage::config();
6467     my $ids = $cfg->{ids};
6468
6469     my $res = [];
6470     foreach my $sid (keys %$ids) {
6471         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6472         push @$res, $sid;
6473     }
6474
6475     return $res;
6476 }
6477
6478 1;