PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QMPClient;
  31 use PVE::RPCEnvironment;
  32 use Time::HiRes qw(gettimeofday);
  33
  34 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  35
  36 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  37
  38 # Note about locking: we use flock on the config file protect
  39 # against concurent actions.
  40 # Aditionaly, we have a 'lock' setting in the config file. This
  41 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  42 # allowed when such lock is set. But you can ignore this kind of
  43 # lock with the --skiplock flag.
  44
  45 cfs_register_file('/qemu-server/',
  46                   \&parse_vm_config,
  47                   \&write_vm_config);
  48
  49 PVE::JSONSchema::register_standard_option('skiplock', {
  50     description => "Ignore locks - only root is allowed to use this option.",
  51     type => 'boolean',
  52     optional => 1,
  53 });
  54
  55 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  56     description => "Some command save/restore state from this location.",
  57     type => 'string',
  58     maxLength => 128,
  59     optional => 1,
  60 });
  61
  62 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  63     description => "The name of the snapshot.",
  64     type => 'string', format => 'pve-configid',
  65     maxLength => 40,
  66 });
  67
  68 #no warnings 'redefine';
  69
  70 sub cgroups_write {
  71    my ($controller, $vmid, $option, $value) = @_;
  72
  73    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  74    PVE::ProcFSTools::write_proc_entry($path, $value);
  75
  76 }
  77
  78 my $nodename = PVE::INotify::nodename();
  79
  80 mkdir "/etc/pve/nodes/$nodename";
  81 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  82 mkdir $confdir;
  83
  84 my $var_run_tmpdir = "/var/run/qemu-server";
  85 mkdir $var_run_tmpdir;
  86
  87 my $lock_dir = "/var/lock/qemu-server";
  88 mkdir $lock_dir;
  89
  90 my $pcisysfs = "/sys/bus/pci";
  91
  92 my $confdesc = {
  93     onboot => {
  94         optional => 1,
  95         type => 'boolean',
  96         description => "Specifies whether a VM will be started during system bootup.",
  97         default => 0,
  98     },
  99     autostart => {
 100         optional => 1,
 101         type => 'boolean',
 102         description => "Automatic restart after crash (currently ignored).",
 103         default => 0,
 104     },
 105     hotplug => {
 106         optional => 1,
 107         type => 'string', format => 'pve-hotplug-features',
 108         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 109         default => 'network,disk,usb',
 110     },
 111     reboot => {
 112         optional => 1,
 113         type => 'boolean',
 114         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 115         default => 1,
 116     },
 117     lock => {
 118         optional => 1,
 119         type => 'string',
 120         description => "Lock/unlock the VM.",
 121         enum => [qw(migrate backup snapshot rollback)],
 122     },
 123     cpulimit => {
 124         optional => 1,
 125         type => 'number',
 126         description => "Limit of CPU usage. Note if the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 127         minimum => 0,
 128         maximum => 128,
 129         default => 0,
 130     },
 131     cpuunits => {
 132         optional => 1,
 133         type => 'integer',
 134         description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 135         minimum => 0,
 136         maximum => 500000,
 137         default => 1000,
 138     },
 139     memory => {
 140         optional => 1,
 141         type => 'integer',
 142         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 143         minimum => 16,
 144         default => 512,
 145     },
 146     balloon => {
 147         optional => 1,
 148         type => 'integer',
 149         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 150         minimum => 0,
 151     },
 152     shares => {
 153         optional => 1,
 154         type => 'integer',
 155         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 156         minimum => 0,
 157         maximum => 50000,
 158         default => 1000,
 159     },
 160     keyboard => {
 161         optional => 1,
 162         type => 'string',
 163         description => "Keybord layout for vnc server. Default is read from the datacenter configuration file.",
 164         enum => PVE::Tools::kvmkeymaplist(),
 165         default => 'en-us',
 166     },
 167     name => {
 168         optional => 1,
 169         type => 'string', format => 'dns-name',
 170         description => "Set a name for the VM. Only used on the configuration web interface.",
 171     },
 172     scsihw => {
 173         optional => 1,
 174         type => 'string',
 175         description => "scsi controller model",
 176         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 177         default => 'lsi',
 178     },
 179     description => {
 180         optional => 1,
 181         type => 'string',
 182         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 183     },
 184     ostype => {
 185         optional => 1,
 186         type => 'string',
 187         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 l24 l26 solaris)],
 188         description => <<EODESC,
 189 Used to enable special optimization/features for specific
 190 operating systems:
 191
 192 other  => unspecified OS
 193 wxp    => Microsoft Windows XP
 194 w2k    => Microsoft Windows 2000
 195 w2k3   => Microsoft Windows 2003
 196 w2k8   => Microsoft Windows 2008
 197 wvista => Microsoft Windows Vista
 198 win7   => Microsoft Windows 7
 199 win8   => Microsoft Windows 8/2012
 200 l24    => Linux 2.4 Kernel
 201 l26    => Linux 2.6/3.X Kernel
 202 solaris => solaris/opensolaris/openindiania kernel
 203
 204 other|l24|l26|solaris                       ... no special behaviour
 205 wxp|w2k|w2k3|w2k8|wvista|win7|win8  ... use --localtime switch
 206 EODESC
 207     },
 208     boot => {
 209         optional => 1,
 210         type => 'string',
 211         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 212         pattern => '[acdn]{1,4}',
 213         default => 'cdn',
 214     },
 215     bootdisk => {
 216         optional => 1,
 217         type => 'string', format => 'pve-qm-bootdisk',
 218         description => "Enable booting from specified disk.",
 219         pattern => '(ide|sata|scsi|virtio)\d+',
 220     },
 221     smp => {
 222         optional => 1,
 223         type => 'integer',
 224         description => "The number of CPUs. Please use option -sockets instead.",
 225         minimum => 1,
 226         default => 1,
 227     },
 228     sockets => {
 229         optional => 1,
 230         type => 'integer',
 231         description => "The number of CPU sockets.",
 232         minimum => 1,
 233         default => 1,
 234     },
 235     cores => {
 236         optional => 1,
 237         type => 'integer',
 238         description => "The number of cores per socket.",
 239         minimum => 1,
 240         default => 1,
 241     },
 242     numa => {
 243         optional => 1,
 244         type => 'boolean',
 245         description => "Enable/disable Numa.",
 246         default => 0,
 247     },
 248     vcpus => {
 249         optional => 1,
 250         type => 'integer',
 251         description => "Number of hotplugged vcpus.",
 252         minimum => 1,
 253         default => 0,
 254     },
 255     acpi => {
 256         optional => 1,
 257         type => 'boolean',
 258         description => "Enable/disable ACPI.",
 259         default => 1,
 260     },
 261     agent => {
 262         optional => 1,
 263         type => 'boolean',
 264         description => "Enable/disable Qemu GuestAgent.",
 265         default => 0,
 266     },
 267     kvm => {
 268         optional => 1,
 269         type => 'boolean',
 270         description => "Enable/disable KVM hardware virtualization.",
 271         default => 1,
 272     },
 273     tdf => {
 274         optional => 1,
 275         type => 'boolean',
 276         description => "Enable/disable time drift fix.",
 277         default => 0,
 278     },
 279     localtime => {
 280         optional => 1,
 281         type => 'boolean',
 282         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 283     },
 284     freeze => {
 285         optional => 1,
 286         type => 'boolean',
 287         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 288     },
 289     vga => {
 290         optional => 1,
 291         type => 'string',
 292         description => "Select VGA type. If you want to use high resolution modes (>= 1280x1024x16) then you should use option 'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and 'cirrur' for other OS types. Option 'qxl' enables the SPICE display sever. You can also run without any graphic card using a serial devive as terminal.",
 293         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 294     },
 295     watchdog => {
 296         optional => 1,
 297         type => 'string', format => 'pve-qm-watchdog',
 298         typetext => '[[model=]i6300esb|ib700] [,[action=]reset|shutdown|poweroff|pause|debug|none]',
 299         description => "Create a virtual hardware watchdog device.  Once enabled (by a guest action), the watchdog must be periodically polled by an agent inside the guest or else the guest will be restarted (or execute the action specified)",
 300     },
 301     startdate => {
 302         optional => 1,
 303         type => 'string',
 304         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 305         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 306         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 307         default => 'now',
 308     },
 309     startup =>  get_standard_option('pve-startup-order'),
 310     template => {
 311         optional => 1,
 312         type => 'boolean',
 313         description => "Enable/disable Template.",
 314         default => 0,
 315     },
 316     args => {
 317         optional => 1,
 318         type => 'string',
 319         description => <<EODESCR,
 320 Note: this option is for experts only. It allows you to pass arbitrary arguments to kvm, for example:
 321
 322 args: -no-reboot -no-hpet
 323 EODESCR
 324     },
 325     tablet => {
 326         optional => 1,
 327         type => 'boolean',
 328         default => 1,
 329         description => "Enable/disable the usb tablet device. This device is usually needed to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with normal VNC clients. If you're running lots of console-only guests on one host, you may consider disabling this to save some context switches. This is turned of by default if you use spice (vga=qxl).",
 330     },
 331     migrate_speed => {
 332         optional => 1,
 333         type => 'integer',
 334         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 335         minimum => 0,
 336         default => 0,
 337     },
 338     migrate_downtime => {
 339         optional => 1,
 340         type => 'number',
 341         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 342         minimum => 0,
 343         default => 0.1,
 344     },
 345     cdrom => {
 346         optional => 1,
 347         type => 'string', format => 'pve-qm-drive',
 348         typetext => 'volume',
 349         description => "This is an alias for option -ide2",
 350     },
 351     cpu => {
 352         optional => 1,
 353         description => "Emulated CPU type.",
 354         type => 'string',
 355         enum => [ qw(486 athlon pentium pentium2 pentium3 coreduo core2duo kvm32 kvm64 qemu32 qemu64 phenom Conroe Penryn Nehalem Westmere SandyBridge IvyBridge Haswell Broadwell Opteron_G1 Opteron_G2 Opteron_G3 Opteron_G4 Opteron_G5 host) ],
 356         default => 'kvm64',
 357     },
 358     parent => get_standard_option('pve-snapshot-name', {
 359         optional => 1,
 360         description => "Parent snapshot name. This is used internally, and should not be modified.",
 361     }),
 362     snaptime => {
 363         optional => 1,
 364         description => "Timestamp for snapshots.",
 365         type => 'integer',
 366         minimum => 0,
 367     },
 368     vmstate => {
 369         optional => 1,
 370         type => 'string', format => 'pve-volume-id',
 371         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 372     },
 373     machine => {
 374         description => "Specific the Qemu machine type.",
 375         type => 'string',
 376         pattern => '(pc|pc(-i440fx)?-\d+\.\d+|q35|pc-q35-\d+\.\d+)',
 377         maxLength => 40,
 378         optional => 1,
 379     },
 380     smbios1 => {
 381         description => "Specify SMBIOS type 1 fields.",
 382         type => 'string', format => 'pve-qm-smbios1',
 383         typetext => "[manufacturer=str][,product=str][,version=str][,serial=str] [,uuid=uuid][,sku=str][,family=str]",
 384         maxLength => 256,
 385         optional => 1,
 386     },
 387     protection => {
 388         optional => 1,
 389         type => 'boolean',
 390         description => "Sets the protection flag of the VM. This will prevent the remove operation.",
 391         default => 0,
 392     },
 393 };
 394
 395 # what about other qemu settings ?
 396 #cpu => 'string',
 397 #machine => 'string',
 398 #fda => 'file',
 399 #fdb => 'file',
 400 #mtdblock => 'file',
 401 #sd => 'file',
 402 #pflash => 'file',
 403 #snapshot => 'bool',
 404 #bootp => 'file',
 405 ##tftp => 'dir',
 406 ##smb => 'dir',
 407 #kernel => 'file',
 408 #append => 'string',
 409 #initrd => 'file',
 410 ##soundhw => 'string',
 411
 412 while (my ($k, $v) = each %$confdesc) {
 413     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 414 }
 415
 416 my $MAX_IDE_DISKS = 4;
 417 my $MAX_SCSI_DISKS = 14;
 418 my $MAX_VIRTIO_DISKS = 16;
 419 my $MAX_SATA_DISKS = 6;
 420 my $MAX_USB_DEVICES = 5;
 421 my $MAX_NETS = 32;
 422 my $MAX_UNUSED_DISKS = 8;
 423 my $MAX_HOSTPCI_DEVICES = 4;
 424 my $MAX_SERIAL_PORTS = 4;
 425 my $MAX_PARALLEL_PORTS = 3;
 426 my $MAX_NUMA = 8;
 427 my $MAX_MEM = 4194304;
 428 my $STATICMEM = 1024;
 429
 430 my $numadesc = {
 431     optional => 1,
 432     type => 'string', format => 'pve-qm-numanode',
 433     typetext => "cpus=<id[-id],memory=<mb>[[,hostnodes=<id[-id]>] [,policy=<preferred|bind|interleave>]]",
 434     description => "numa topology",
 435 };
 436 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 437
 438 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 439     $confdesc->{"numa$i"} = $numadesc;
 440 }
 441
 442 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 443                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 444                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 445 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 446
 447 my $netdesc = {
 448     optional => 1,
 449     type => 'string', format => 'pve-qm-net',
 450     typetext => "MODEL=XX:XX:XX:XX:XX:XX [,bridge=<dev>][,queues=<nbqueues>][,rate=<mbps>] [,tag=<vlanid>][,firewall=0|1],link_down=0|1]",
 451     description => <<EODESCR,
 452 Specify network devices.
 453
 454 MODEL is one of: $nic_model_list_txt
 455
 456 XX:XX:XX:XX:XX:XX should be an unique MAC address. This is
 457 automatically generated if not specified.
 458
 459 The bridge parameter can be used to automatically add the interface to a bridge device. The Proxmox VE standard bridge is called 'vmbr0'.
 460
 461 Option 'rate' is used to limit traffic bandwidth from and to this interface. It is specified as floating point number, unit is 'Megabytes per second'.
 462
 463 If you specify no bridge, we create a kvm 'user' (NATed) network device, which provides DHCP and DNS services. The following addresses are used:
 464
 465 10.0.2.2   Gateway
 466 10.0.2.3   DNS Server
 467 10.0.2.4   SMB Server
 468
 469 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 470
 471 EODESCR
 472 };
 473 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 474
 475 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 476     $confdesc->{"net$i"} = $netdesc;
 477 }
 478
 479 my $drivename_hash;
 480
 481 my $idedesc = {
 482     optional => 1,
 483     type => 'string', format => 'pve-qm-drive',
 484     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads] [,discard=ignore|on]',
 485     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
 486 };
 487 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
 488
 489 my $scsidesc = {
 490     optional => 1,
 491     type => 'string', format => 'pve-qm-drive',
 492     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads] [,discard=ignore|on] [,iothread=on] [,queues=<nbqueues>]',
 493     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
 494 };
 495 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 496
 497 my $satadesc = {
 498     optional => 1,
 499     type => 'string', format => 'pve-qm-drive',
 500     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads]  [,discard=ignore|on]',
 501     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
 502 };
 503 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
 504
 505 my $virtiodesc = {
 506     optional => 1,
 507     type => 'string', format => 'pve-qm-drive',
 508     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads]  [,discard=ignore|on] [,iothread=on]',
 509     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
 510 };
 511 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
 512
 513 my $usbdesc = {
 514     optional => 1,
 515     type => 'string', format => 'pve-qm-usb-device',
 516     typetext => 'host=HOSTUSBDEVICE|spice',
 517     description => <<EODESCR,
 518 Configure an USB device (n is 0 to 4). This can be used to
 519 pass-through usb devices to the guest. HOSTUSBDEVICE syntax is:
 520
 521 'bus-port(.port)*' (decimal numbers) or
 522 'vendor_id:product_id' (hexadeciaml numbers)
 523
 524 You can use the 'lsusb -t' command to list existing usb devices.
 525
 526 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 527
 528 The value 'spice' can be used to add a usb redirection devices for spice.
 529
 530 EODESCR
 531 };
 532 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
 533
 534 my $hostpcidesc = {
 535         optional => 1,
 536         type => 'string', format => 'pve-qm-hostpci',
 537         typetext => "[host=]HOSTPCIDEVICE [,driver=kvm|vfio] [,rombar=on|off] [,pcie=0|1] [,x-vga=on|off]",
 538         description => <<EODESCR,
 539 Map host pci devices. HOSTPCIDEVICE syntax is:
 540
 541 'bus:dev.func' (hexadecimal numbers)
 542
 543 You can us the 'lspci' command to list existing pci devices.
 544
 545 The 'rombar' option determines whether or not the device's ROM will be visible in the guest's memory map (default is 'on').
 546
 547 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 548
 549 Experimental: user reported problems with this option.
 550 EODESCR
 551 };
 552 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
 553
 554 my $serialdesc = {
 555         optional => 1,
 556         type => 'string',
 557         pattern => '(/dev/.+|socket)',
 558         description =>  <<EODESCR,
 559 Create a serial device inside the VM (n is 0 to 3), and pass through a host serial device (i.e. /dev/ttyS0), or create a unix socket on the host side (use 'qm terminal' to open a terminal connection).
 560
 561 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 562
 563 Experimental: user reported problems with this option.
 564 EODESCR
 565 };
 566
 567 my $paralleldesc= {
 568         optional => 1,
 569         type => 'string',
 570         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
 571         description =>  <<EODESCR,
 572 Map host parallel devices (n is 0 to 2).
 573
 574 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 575
 576 Experimental: user reported problems with this option.
 577 EODESCR
 578 };
 579
 580 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
 581     $confdesc->{"parallel$i"} = $paralleldesc;
 582 }
 583
 584 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
 585     $confdesc->{"serial$i"} = $serialdesc;
 586 }
 587
 588 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
 589     $confdesc->{"hostpci$i"} = $hostpcidesc;
 590 }
 591
 592 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
 593     $drivename_hash->{"ide$i"} = 1;
 594     $confdesc->{"ide$i"} = $idedesc;
 595 }
 596
 597 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
 598     $drivename_hash->{"sata$i"} = 1;
 599     $confdesc->{"sata$i"} = $satadesc;
 600 }
 601
 602 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
 603     $drivename_hash->{"scsi$i"} = 1;
 604     $confdesc->{"scsi$i"} = $scsidesc ;
 605 }
 606
 607 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
 608     $drivename_hash->{"virtio$i"} = 1;
 609     $confdesc->{"virtio$i"} = $virtiodesc;
 610 }
 611
 612 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
 613     $confdesc->{"usb$i"} = $usbdesc;
 614 }
 615
 616 my $unuseddesc = {
 617     optional => 1,
 618     type => 'string', format => 'pve-volume-id',
 619     description => "Reference to unused volumes.",
 620 };
 621
 622 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
 623     $confdesc->{"unused$i"} = $unuseddesc;
 624 }
 625
 626 my $kvm_api_version = 0;
 627
 628 sub kvm_version {
 629
 630     return $kvm_api_version if $kvm_api_version;
 631
 632     my $fh = IO::File->new("</dev/kvm") ||
 633         return 0;
 634
 635     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
 636         $kvm_api_version = $v;
 637     }
 638
 639     $fh->close();
 640
 641     return  $kvm_api_version;
 642 }
 643
 644 my $kvm_user_version;
 645
 646 sub kvm_user_version {
 647
 648     return $kvm_user_version if $kvm_user_version;
 649
 650     $kvm_user_version = 'unknown';
 651
 652     my $tmp = `kvm -help 2>/dev/null`;
 653
 654     if ($tmp =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)[,\s]/) {
 655         $kvm_user_version = $2;
 656     }
 657
 658     return $kvm_user_version;
 659
 660 }
 661
 662 my $kernel_has_vhost_net = -c '/dev/vhost-net';
 663
 664 sub disknames {
 665     # order is important - used to autoselect boot disk
 666     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
 667             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
 668             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
 669             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))));
 670 }
 671
 672 sub valid_drivename {
 673     my $dev = shift;
 674
 675     return defined($drivename_hash->{$dev});
 676 }
 677
 678 sub option_exists {
 679     my $key = shift;
 680     return defined($confdesc->{$key});
 681 }
 682
 683 sub nic_models {
 684     return $nic_model_list;
 685 }
 686
 687 sub os_list_description {
 688
 689     return {
 690         other => 'Other',
 691         wxp => 'Windows XP',
 692         w2k => 'Windows 2000',
 693         w2k3 =>, 'Windows 2003',
 694         w2k8 => 'Windows 2008',
 695         wvista => 'Windows Vista',
 696         win7 => 'Windows 7',
 697         win8 => 'Windows 8/2012',
 698         l24 => 'Linux 2.4',
 699         l26 => 'Linux 2.6',
 700     };
 701 }
 702
 703 my $cdrom_path;
 704
 705 sub get_cdrom_path {
 706
 707     return  $cdrom_path if $cdrom_path;
 708
 709     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
 710     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
 711     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
 712 }
 713
 714 sub get_iso_path {
 715     my ($storecfg, $vmid, $cdrom) = @_;
 716
 717     if ($cdrom eq 'cdrom') {
 718         return get_cdrom_path();
 719     } elsif ($cdrom eq 'none') {
 720         return '';
 721     } elsif ($cdrom =~ m|^/|) {
 722         return $cdrom;
 723     } else {
 724         return PVE::Storage::path($storecfg, $cdrom);
 725     }
 726 }
 727
 728 # try to convert old style file names to volume IDs
 729 sub filename_to_volume_id {
 730     my ($vmid, $file, $media) = @_;
 731
 732     if (!($file eq 'none' || $file eq 'cdrom' ||
 733           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
 734
 735         return undef if $file =~ m|/|;
 736
 737         if ($media && $media eq 'cdrom') {
 738             $file = "local:iso/$file";
 739         } else {
 740             $file = "local:$vmid/$file";
 741         }
 742     }
 743
 744     return $file;
 745 }
 746
 747 sub verify_media_type {
 748     my ($opt, $vtype, $media) = @_;
 749
 750     return if !$media;
 751
 752     my $etype;
 753     if ($media eq 'disk') {
 754         $etype = 'images';
 755     } elsif ($media eq 'cdrom') {
 756         $etype = 'iso';
 757     } else {
 758         die "internal error";
 759     }
 760
 761     return if ($vtype eq $etype);
 762
 763     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
 764 }
 765
 766 sub cleanup_drive_path {
 767     my ($opt, $storecfg, $drive) = @_;
 768
 769     # try to convert filesystem paths to volume IDs
 770
 771     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
 772         ($drive->{file} !~ m|^/dev/.+|) &&
 773         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
 774         ($drive->{file} !~ m/^\d+$/)) {
 775         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
 776         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
 777         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
 778         verify_media_type($opt, $vtype, $drive->{media});
 779         $drive->{file} = $volid;
 780     }
 781
 782     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
 783 }
 784
 785 sub create_conf_nolock {
 786     my ($vmid, $settings) = @_;
 787
 788     my $filename = config_file($vmid);
 789
 790     die "configuration file '$filename' already exists\n" if -f $filename;
 791
 792     my $defaults = load_defaults();
 793
 794     $settings->{name} = "vm$vmid" if !$settings->{name};
 795     $settings->{memory} = $defaults->{memory} if !$settings->{memory};
 796
 797     my $data = '';
 798     foreach my $opt (keys %$settings) {
 799         next if !$confdesc->{$opt};
 800
 801         my $value = $settings->{$opt};
 802         next if !$value;
 803
 804         $data .= "$opt: $value\n";
 805     }
 806
 807     PVE::Tools::file_set_contents($filename, $data);
 808 }
 809
 810 sub parse_hotplug_features {
 811     my ($data) = @_;
 812
 813     my $res = {};
 814
 815     return $res if $data eq '0';
 816
 817     $data = $confdesc->{hotplug}->{default} if $data eq '1';
 818
 819     foreach my $feature (PVE::Tools::split_list($data)) {
 820         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
 821             $res->{$1} = 1;
 822         } else {
 823             warn "ignoring unknown hotplug feature '$feature'\n";
 824         }
 825     }
 826     return $res;
 827 }
 828
 829 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
 830 sub pve_verify_hotplug_features {
 831     my ($value, $noerr) = @_;
 832
 833     return $value if parse_hotplug_features($value);
 834
 835     return undef if $noerr;
 836
 837     die "unable to parse hotplug option\n";
 838 }
 839
 840 my $parse_size = sub {
 841     my ($value) = @_;
 842
 843     return undef if $value !~ m/^(\d+(\.\d+)?)([KMG])?$/;
 844     my ($size, $unit) = ($1, $3);
 845     if ($unit) {
 846         if ($unit eq 'K') {
 847             $size = $size * 1024;
 848         } elsif ($unit eq 'M') {
 849             $size = $size * 1024 * 1024;
 850         } elsif ($unit eq 'G') {
 851             $size = $size * 1024 * 1024 * 1024;
 852         }
 853     }
 854     return int($size);
 855 };
 856
 857 my $format_size = sub {
 858     my ($size) = @_;
 859
 860     $size = int($size);
 861
 862     my $kb = int($size/1024);
 863     return $size if $kb*1024 != $size;
 864
 865     my $mb = int($kb/1024);
 866     return "${kb}K" if $mb*1024 != $kb;
 867
 868     my $gb = int($mb/1024);
 869     return "${mb}M" if $gb*1024 != $mb;
 870
 871     return "${gb}G";
 872 };
 873
 874 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
 875 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
 876 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
 877 #        [,aio=native|threads][,discard=ignore|on][,iothread=on]
 878
 879 sub parse_drive {
 880     my ($key, $data) = @_;
 881
 882     my $res = {};
 883
 884     # $key may be undefined - used to verify JSON parameters
 885     if (!defined($key)) {
 886         $res->{interface} = 'unknown'; # should not harm when used to verify parameters
 887         $res->{index} = 0;
 888     } elsif ($key =~ m/^([^\d]+)(\d+)$/) {
 889         $res->{interface} = $1;
 890         $res->{index} = $2;
 891     } else {
 892         return undef;
 893     }
 894
 895     foreach my $p (split (/,/, $data)) {
 896         next if $p =~ m/^\s*$/;
 897
 898         if ($p =~ m/^(file|volume|cyls|heads|secs|trans|media|snapshot|cache|format|rerror|werror|backup|aio|bps|mbps|mbps_max|bps_rd|mbps_rd|mbps_rd_max|bps_wr|mbps_wr|mbps_wr_max|iops|iops_max|iops_rd|iops_rd_max|iops_wr|iops_wr_max|size|discard|iothread|queues)=(.+)$/) {
 899             my ($k, $v) = ($1, $2);
 900
 901             $k = 'file' if $k eq 'volume';
 902
 903             return undef if defined $res->{$k};
 904
 905             if ($k eq 'bps' || $k eq 'bps_rd' || $k eq 'bps_wr') {
 906                 return undef if !$v || $v !~ m/^\d+/;
 907                 $k = "m$k";
 908                 $v = sprintf("%.3f", $v / (1024*1024));
 909             }
 910             $res->{$k} = $v;
 911         } else {
 912             if (!$res->{file} && $p !~ m/=/) {
 913                 $res->{file} = $p;
 914             } else {
 915                 return undef;
 916             }
 917         }
 918     }
 919
 920     return undef if !$res->{file};
 921
 922     return undef if $res->{cache} &&
 923         $res->{cache} !~ m/^(off|none|writethrough|writeback|unsafe|directsync)$/;
 924     return undef if $res->{snapshot} && $res->{snapshot} !~ m/^(on|off)$/;
 925     return undef if $res->{cyls} && $res->{cyls} !~ m/^\d+$/;
 926     return undef if $res->{heads} && $res->{heads} !~ m/^\d+$/;
 927     return undef if $res->{secs} && $res->{secs} !~ m/^\d+$/;
 928     return undef if $res->{media} && $res->{media} !~ m/^(disk|cdrom)$/;
 929     return undef if $res->{trans} && $res->{trans} !~ m/^(none|lba|auto)$/;
 930     return undef if $res->{format} && $res->{format} !~ m/^(raw|cow|qcow|qed|qcow2|vmdk|cloop)$/;
 931     return undef if $res->{rerror} && $res->{rerror} !~ m/^(ignore|report|stop)$/;
 932     return undef if $res->{werror} && $res->{werror} !~ m/^(enospc|ignore|report|stop)$/;
 933     return undef if $res->{backup} && $res->{backup} !~ m/^(yes|no)$/;
 934     return undef if $res->{aio} && $res->{aio} !~ m/^(native|threads)$/;
 935     return undef if $res->{discard} && $res->{discard} !~ m/^(ignore|on)$/;
 936     return undef if $res->{iothread} && $res->{iothread} !~ m/^(on)$/;
 937     return undef if $res->{queues} && ($res->{queues} !~ m/^\d+$/ || $res->{queues} < 2);
 938
 939     return undef if $res->{mbps_rd} && $res->{mbps};
 940     return undef if $res->{mbps_wr} && $res->{mbps};
 941
 942     return undef if $res->{mbps} && $res->{mbps} !~ m/^\d+(\.\d+)?$/;
 943     return undef if $res->{mbps_max} && $res->{mbps_max} !~ m/^\d+(\.\d+)?$/;
 944     return undef if $res->{mbps_rd} && $res->{mbps_rd} !~ m/^\d+(\.\d+)?$/;
 945     return undef if $res->{mbps_rd_max} && $res->{mbps_rd_max} !~ m/^\d+(\.\d+)?$/;
 946     return undef if $res->{mbps_wr} && $res->{mbps_wr} !~ m/^\d+(\.\d+)?$/;
 947     return undef if $res->{mbps_wr_max} && $res->{mbps_wr_max} !~ m/^\d+(\.\d+)?$/;
 948
 949     return undef if $res->{iops_rd} && $res->{iops};
 950     return undef if $res->{iops_wr} && $res->{iops};
 951
 952
 953     return undef if $res->{iops} && $res->{iops} !~ m/^\d+$/;
 954     return undef if $res->{iops_max} && $res->{iops_max} !~ m/^\d+$/;
 955     return undef if $res->{iops_rd} && $res->{iops_rd} !~ m/^\d+$/;
 956     return undef if $res->{iops_rd_max} && $res->{iops_rd_max} !~ m/^\d+$/;
 957     return undef if $res->{iops_wr} && $res->{iops_wr} !~ m/^\d+$/;
 958     return undef if $res->{iops_wr_max} && $res->{iops_wr_max} !~ m/^\d+$/;
 959
 960
 961     if ($res->{size}) {
 962         return undef if !defined($res->{size} = &$parse_size($res->{size}));
 963     }
 964
 965     if ($res->{media} && ($res->{media} eq 'cdrom')) {
 966         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
 967         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
 968         return undef if $res->{interface} eq 'virtio';
 969     }
 970
 971     # rerror does not work with scsi drives
 972     if ($res->{rerror}) {
 973         return undef if $res->{interface} eq 'scsi';
 974     }
 975
 976     return $res;
 977 }
 978
 979 my @qemu_drive_options = qw(heads secs cyls trans media format cache snapshot rerror werror aio discard iops iops_rd iops_wr iops_max iops_rd_max iops_wr_max);
 980
 981 sub print_drive {
 982     my ($vmid, $drive) = @_;
 983
 984     my $opts = '';
 985     foreach my $o (@qemu_drive_options, 'mbps', 'mbps_rd', 'mbps_wr', 'mbps_max', 'mbps_rd_max', 'mbps_wr_max', 'backup', 'iothread', 'queues') {
 986         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
 987     }
 988
 989     if ($drive->{size}) {
 990         $opts .= ",size=" . &$format_size($drive->{size});
 991     }
 992
 993     return "$drive->{file}$opts";
 994 }
 995
 996 sub scsi_inquiry {
 997     my($fh, $noerr) = @_;
 998
 999     my $SG_IO = 0x2285;
1000     my $SG_GET_VERSION_NUM = 0x2282;
1001
1002     my $versionbuf = "\x00" x 8;
1003     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1004     if (!$ret) {
1005         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1006         return undef;
1007     }
1008     my $version = unpack("I", $versionbuf);
1009     if ($version < 30000) {
1010         die "scsi generic interface too old\n"  if !$noerr;
1011         return undef;
1012     }
1013
1014     my $buf = "\x00" x 36;
1015     my $sensebuf = "\x00" x 8;
1016     my $cmd = pack("C x3 C x1", 0x12, 36);
1017
1018     # see /usr/include/scsi/sg.h
1019     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1020
1021     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1022                       length($sensebuf), 0, length($buf), $buf,
1023                       $cmd, $sensebuf, 6000);
1024
1025     $ret = ioctl($fh, $SG_IO, $packet);
1026     if (!$ret) {
1027         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1028         return undef;
1029     }
1030
1031     my @res = unpack($sg_io_hdr_t, $packet);
1032     if ($res[17] || $res[18]) {
1033         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1034         return undef;
1035     }
1036
1037     my $res = {};
1038     (my $byte0, my $byte1, $res->{vendor},
1039      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1040
1041     $res->{removable} = $byte1 & 128 ? 1 : 0;
1042     $res->{type} = $byte0 & 31;
1043
1044     return $res;
1045 }
1046
1047 sub path_is_scsi {
1048     my ($path) = @_;
1049
1050     my $fh = IO::File->new("+<$path") || return undef;
1051     my $res = scsi_inquiry($fh, 1);
1052     close($fh);
1053
1054     return $res;
1055 }
1056
1057 sub machine_type_is_q35 {
1058     my ($conf) = @_;
1059
1060     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1061 }
1062
1063 sub print_tabletdevice_full {
1064     my ($conf) = @_;
1065
1066     my $q35 = machine_type_is_q35($conf);
1067
1068     # we use uhci for old VMs because tablet driver was buggy in older qemu
1069     my $usbbus = $q35 ? "ehci" : "uhci";
1070
1071     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1072 }
1073
1074 sub print_drivedevice_full {
1075     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1076
1077     my $device = '';
1078     my $maxdev = 0;
1079
1080     if ($drive->{interface} eq 'virtio') {
1081         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1082         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1083         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1084     } elsif ($drive->{interface} eq 'scsi') {
1085
1086         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1087         my $unit = $drive->{index} % $maxdev;
1088         my $devicetype = 'hd';
1089         my $path = '';
1090         if (drive_is_cdrom($drive)) {
1091               $devicetype = 'cd';
1092         } else {
1093               if ($drive->{file} =~ m|^/|) {
1094                   $path = $drive->{file};
1095               } else {
1096                   $path = PVE::Storage::path($storecfg, $drive->{file});
1097               }
1098
1099               if($path =~ m/^iscsi\:\/\//){
1100                   $devicetype = 'generic';
1101               } else {
1102                   if (my $info = path_is_scsi($path)) {
1103                       if ($info->{type} == 0) {
1104                           $devicetype = 'block';
1105                       } elsif ($info->{type} == 1) { # tape
1106                           $devicetype = 'generic';
1107                       }
1108                   }
1109               }
1110          }
1111
1112         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1113             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1114         } else {
1115             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1116         }
1117
1118     } elsif ($drive->{interface} eq 'ide'){
1119         $maxdev = 2;
1120         my $controller = int($drive->{index} / $maxdev);
1121         my $unit = $drive->{index} % $maxdev;
1122         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1123
1124         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1125     } elsif ($drive->{interface} eq 'sata'){
1126         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1127         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1128         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1129     } elsif ($drive->{interface} eq 'usb') {
1130         die "implement me";
1131         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1132     } else {
1133         die "unsupported interface type";
1134     }
1135
1136     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1137
1138     return $device;
1139 }
1140
1141 sub get_initiator_name {
1142     my $initiator;
1143
1144     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1145     while (defined(my $line = <$fh>)) {
1146         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1147         $initiator = $1;
1148         last;
1149     }
1150     $fh->close();
1151
1152     return $initiator;
1153 }
1154
1155 sub print_drive_full {
1156     my ($storecfg, $vmid, $drive) = @_;
1157
1158     my $path;
1159     my $volid = $drive->{file};
1160     my $format;
1161
1162     if (drive_is_cdrom($drive)) {
1163         $path = get_iso_path($storecfg, $vmid, $volid);
1164     } else {
1165         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1166         if ($storeid) {
1167             $path = PVE::Storage::path($storecfg, $volid);
1168             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1169             $format = qemu_img_format($scfg, $volname);
1170         } else {
1171             $path = $volid;
1172         }
1173    }
1174
1175     my $opts = '';
1176     foreach my $o (@qemu_drive_options) {
1177         next if $o eq 'bootindex';
1178         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1179     }
1180
1181     $opts .= ",format=$format" if $format && !$drive->{format};
1182
1183     foreach my $o (qw(bps bps_rd bps_wr)) {
1184         my $v = $drive->{"m$o"};
1185         $opts .= ",$o=" . int($v*1024*1024) if $v;
1186     }
1187
1188     my $cache_direct = 0;
1189
1190     if (my $cache = $drive->{cache}) {
1191         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1192     } elsif (!drive_is_cdrom($drive)) {
1193         $opts .= ",cache=none";
1194         $cache_direct = 1;
1195     }
1196
1197     # aio native works only with O_DIRECT
1198     if (!$drive->{aio}) {
1199         if($cache_direct) {
1200             $opts .= ",aio=native";
1201         } else {
1202             $opts .= ",aio=threads";
1203         }
1204     }
1205
1206     my $detectzeroes = $drive->{discard} ? "unmap" : "on";
1207     $opts .= ",detect-zeroes=$detectzeroes" if !drive_is_cdrom($drive);
1208
1209     my $pathinfo = $path ? "file=$path," : '';
1210
1211     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1212 }
1213
1214 sub print_netdevice_full {
1215     my ($vmid, $conf, $net, $netid, $bridges) = @_;
1216
1217     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1218
1219     my $device = $net->{model};
1220     if ($net->{model} eq 'virtio') {
1221          $device = 'virtio-net-pci';
1222      };
1223
1224     my $pciaddr = print_pci_addr("$netid", $bridges);
1225     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1226     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1227         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1228         my $vectors = $net->{queues} * 2 + 2;
1229         $tmpstr .= ",vectors=$vectors,mq=on";
1230     }
1231     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1232     return $tmpstr;
1233 }
1234
1235 sub print_netdev_full {
1236     my ($vmid, $conf, $net, $netid) = @_;
1237
1238     my $i = '';
1239     if ($netid =~ m/^net(\d+)$/) {
1240         $i = int($1);
1241     }
1242
1243     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1244
1245     my $ifname = "tap${vmid}i$i";
1246
1247     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1248     die "interface name '$ifname' is too long (max 15 character)\n"
1249         if length($ifname) >= 16;
1250
1251     my $vhostparam = '';
1252     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1253
1254     my $vmname = $conf->{name} || "vm$vmid";
1255
1256     my $netdev = "";
1257
1258     if ($net->{bridge}) {
1259         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/pve-bridge,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1260     } else {
1261         $netdev = "type=user,id=$netid,hostname=$vmname";
1262     }
1263
1264     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1265
1266     return $netdev;
1267 }
1268
1269 sub drive_is_cdrom {
1270     my ($drive) = @_;
1271
1272     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1273
1274 }
1275
1276 sub parse_numa {
1277     my ($data) = @_;
1278
1279     my $res = {};
1280
1281     foreach my $kvp (split(/,/, $data)) {
1282
1283         if ($kvp =~ m/^memory=(\S+)$/) {
1284             $res->{memory} = $1;
1285         } elsif ($kvp =~ m/^policy=(preferred|bind|interleave)$/) {
1286             $res->{policy} = $1;
1287         } elsif ($kvp =~ m/^cpus=(\d+)(-(\d+))?$/) {
1288             $res->{cpus}->{start} = $1;
1289             $res->{cpus}->{end} = $3;
1290         } elsif ($kvp =~ m/^hostnodes=(\d+)(-(\d+))?$/) {
1291             $res->{hostnodes}->{start} = $1;
1292             $res->{hostnodes}->{end} = $3;
1293         } else {
1294             return undef;
1295         }
1296     }
1297
1298     return $res;
1299 }
1300
1301 sub parse_hostpci {
1302     my ($value) = @_;
1303
1304     return undef if !$value;
1305
1306
1307     my @list = split(/,/, $value);
1308     my $found;
1309
1310     my $res = {};
1311     foreach my $kv (@list) {
1312
1313         if ($kv =~ m/^(host=)?([a-f0-9]{2}:[a-f0-9]{2})(\.([a-f0-9]))?$/) {
1314             $found = 1;
1315             if(defined($4)){
1316                 push @{$res->{pciid}}, { id => $2 , function => $4};
1317
1318             }else{
1319                 my $pcidevices = lspci($2);
1320                 $res->{pciid} = $pcidevices->{$2};
1321             }
1322         } elsif ($kv =~ m/^driver=(kvm|vfio)$/) {
1323             $res->{driver} = $1;
1324         } elsif ($kv =~ m/^rombar=(on|off)$/) {
1325             $res->{rombar} = $1;
1326         } elsif ($kv =~ m/^x-vga=(on|off)$/) {
1327             $res->{'x-vga'} = $1;
1328         } elsif ($kv =~ m/^pcie=(\d+)$/) {
1329             $res->{pcie} = 1 if $1 == 1;
1330         } else {
1331             warn "unknown hostpci setting '$kv'\n";
1332         }
1333     }
1334
1335     return undef if !$found;
1336
1337     return $res;
1338 }
1339
1340 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1341 sub parse_net {
1342     my ($data) = @_;
1343
1344     my $res = {};
1345
1346     foreach my $kvp (split(/,/, $data)) {
1347
1348         if ($kvp =~ m/^(ne2k_pci|e1000|e1000-82540em|e1000-82544gc|e1000-82545em|rtl8139|pcnet|virtio|ne2k_isa|i82551|i82557b|i82559er|vmxnet3)(=([0-9a-f]{2}(:[0-9a-f]{2}){5}))?$/i) {
1349             my $model = lc($1);
1350             my $mac = defined($3) ? uc($3) : PVE::Tools::random_ether_addr();
1351             $res->{model} = $model;
1352             $res->{macaddr} = $mac;
1353         } elsif ($kvp =~ m/^bridge=(\S+)$/) {
1354             $res->{bridge} = $1;
1355         } elsif ($kvp =~ m/^queues=(\d+)$/) {
1356             $res->{queues} = $1;
1357         } elsif ($kvp =~ m/^rate=(\d+(\.\d+)?)$/) {
1358             $res->{rate} = $1;
1359         } elsif ($kvp =~ m/^tag=(\d+)$/) {
1360             $res->{tag} = $1;
1361         } elsif ($kvp =~ m/^firewall=([01])$/) {
1362             $res->{firewall} = $1;
1363         } elsif ($kvp =~ m/^link_down=([01])$/) {
1364             $res->{link_down} = $1;
1365         } else {
1366             return undef;
1367         }
1368
1369     }
1370
1371     return undef if !$res->{model};
1372
1373     return $res;
1374 }
1375
1376 sub print_net {
1377     my $net = shift;
1378
1379     my $res = "$net->{model}";
1380     $res .= "=$net->{macaddr}" if $net->{macaddr};
1381     $res .= ",bridge=$net->{bridge}" if $net->{bridge};
1382     $res .= ",rate=$net->{rate}" if $net->{rate};
1383     $res .= ",tag=$net->{tag}" if $net->{tag};
1384     $res .= ",firewall=1" if $net->{firewall};
1385     $res .= ",link_down=1" if $net->{link_down};
1386     $res .= ",queues=$net->{queues}" if $net->{queues};
1387
1388     return $res;
1389 }
1390
1391 sub add_random_macs {
1392     my ($settings) = @_;
1393
1394     foreach my $opt (keys %$settings) {
1395         next if $opt !~ m/^net(\d+)$/;
1396         my $net = parse_net($settings->{$opt});
1397         next if !$net;
1398         $settings->{$opt} = print_net($net);
1399     }
1400 }
1401
1402 sub add_unused_volume {
1403     my ($config, $volid) = @_;
1404
1405     my $key;
1406     for (my $ind = $MAX_UNUSED_DISKS - 1; $ind >= 0; $ind--) {
1407         my $test = "unused$ind";
1408         if (my $vid = $config->{$test}) {
1409             return if $vid eq $volid; # do not add duplicates
1410         } else {
1411             $key = $test;
1412         }
1413     }
1414
1415     die "To many unused volume - please delete them first.\n" if !$key;
1416
1417     $config->{$key} = $volid;
1418
1419     return $key;
1420 }
1421
1422 sub vm_is_volid_owner {
1423     my ($storecfg, $vmid, $volid) = @_;
1424
1425     if ($volid !~  m|^/|) {
1426         my ($path, $owner);
1427         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1428         if ($owner && ($owner == $vmid)) {
1429             return 1;
1430         }
1431     }
1432
1433     return undef;
1434 }
1435
1436 sub split_flagged_list {
1437     my $text = shift || '';
1438     $text =~ s/[,;]/ /g;
1439     $text =~ s/^\s+//;
1440     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
1441 }
1442
1443 sub join_flagged_list {
1444     my ($how, $lst) = @_;
1445     join $how, map { $lst->{$_} . $_ } keys %$lst;
1446 }
1447
1448 sub vmconfig_delete_pending_option {
1449     my ($conf, $key, $force) = @_;
1450
1451     delete $conf->{pending}->{$key};
1452     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1453     $pending_delete_hash->{$key} = $force ? '!' : '';
1454     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1455 }
1456
1457 sub vmconfig_undelete_pending_option {
1458     my ($conf, $key) = @_;
1459
1460     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1461     delete $pending_delete_hash->{$key};
1462
1463     if (%$pending_delete_hash) {
1464         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1465     } else {
1466         delete $conf->{pending}->{delete};
1467     }
1468 }
1469
1470 sub vmconfig_register_unused_drive {
1471     my ($storecfg, $vmid, $conf, $drive) = @_;
1472
1473     if (!drive_is_cdrom($drive)) {
1474         my $volid = $drive->{file};
1475         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1476             add_unused_volume($conf, $volid, $vmid);
1477         }
1478     }
1479 }
1480
1481 sub vmconfig_cleanup_pending {
1482     my ($conf) = @_;
1483
1484     # remove pending changes when nothing changed
1485     my $changes;
1486     foreach my $opt (keys %{$conf->{pending}}) {
1487         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
1488             $changes = 1;
1489             delete $conf->{pending}->{$opt};
1490         }
1491     }
1492
1493     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
1494     my $pending_delete_hash = {};
1495     while (my ($opt, $force) = each %$current_delete_hash) {
1496         if (defined($conf->{$opt})) {
1497             $pending_delete_hash->{$opt} = $force;
1498         } else {
1499             $changes = 1;
1500         }
1501     }
1502
1503     if (%$pending_delete_hash) {
1504         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1505     } else {
1506         delete $conf->{pending}->{delete};
1507     }
1508
1509     return $changes;
1510 }
1511
1512 my $valid_smbios1_options = {
1513     manufacturer => '\S+',
1514     product => '\S+',
1515     version => '\S+',
1516     serial => '\S+',
1517     uuid => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1518     sku => '\S+',
1519     family => '\S+',
1520 };
1521
1522 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
1523 sub parse_smbios1 {
1524     my ($data) = @_;
1525
1526     my $res = {};
1527
1528     foreach my $kvp (split(/,/, $data)) {
1529         return undef if $kvp !~ m/^(\S+)=(.+)$/;
1530         my ($k, $v) = split(/=/, $kvp);
1531         return undef if !defined($k) || !defined($v);
1532         return undef if !$valid_smbios1_options->{$k};
1533         return undef if $v !~ m/^$valid_smbios1_options->{$k}$/;
1534         $res->{$k} = $v;
1535     }
1536
1537     return $res;
1538 }
1539
1540 sub print_smbios1 {
1541     my ($smbios1) = @_;
1542
1543     my $data = '';
1544     foreach my $k (keys %$smbios1) {
1545         next if !defined($smbios1->{$k});
1546         next if !$valid_smbios1_options->{$k};
1547         $data .= ',' if $data;
1548         $data .= "$k=$smbios1->{$k}";
1549     }
1550     return $data;
1551 }
1552
1553 PVE::JSONSchema::register_format('pve-qm-smbios1', \&verify_smbios1);
1554 sub verify_smbios1 {
1555     my ($value, $noerr) = @_;
1556
1557     return $value if parse_smbios1($value);
1558
1559     return undef if $noerr;
1560
1561     die "unable to parse smbios (type 1) options\n";
1562 }
1563
1564 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
1565 sub verify_bootdisk {
1566     my ($value, $noerr) = @_;
1567
1568     return $value if valid_drivename($value);
1569
1570     return undef if $noerr;
1571
1572     die "invalid boot disk '$value'\n";
1573 }
1574
1575 PVE::JSONSchema::register_format('pve-qm-numanode', \&verify_numa);
1576 sub verify_numa {
1577     my ($value, $noerr) = @_;
1578
1579     return $value if parse_numa($value);
1580
1581     return undef if $noerr;
1582
1583     die "unable to parse numa options\n";
1584 }
1585
1586 PVE::JSONSchema::register_format('pve-qm-net', \&verify_net);
1587 sub verify_net {
1588     my ($value, $noerr) = @_;
1589
1590     return $value if parse_net($value);
1591
1592     return undef if $noerr;
1593
1594     die "unable to parse network options\n";
1595 }
1596
1597 PVE::JSONSchema::register_format('pve-qm-drive', \&verify_drive);
1598 sub verify_drive {
1599     my ($value, $noerr) = @_;
1600
1601     return $value if parse_drive(undef, $value);
1602
1603     return undef if $noerr;
1604
1605     die "unable to parse drive options\n";
1606 }
1607
1608 PVE::JSONSchema::register_format('pve-qm-hostpci', \&verify_hostpci);
1609 sub verify_hostpci {
1610     my ($value, $noerr) = @_;
1611
1612     return $value if parse_hostpci($value);
1613
1614     return undef if $noerr;
1615
1616     die "unable to parse pci id\n";
1617 }
1618
1619 PVE::JSONSchema::register_format('pve-qm-watchdog', \&verify_watchdog);
1620 sub verify_watchdog {
1621     my ($value, $noerr) = @_;
1622
1623     return $value if parse_watchdog($value);
1624
1625     return undef if $noerr;
1626
1627     die "unable to parse watchdog options\n";
1628 }
1629
1630 sub parse_watchdog {
1631     my ($value) = @_;
1632
1633     return undef if !$value;
1634
1635     my $res = {};
1636
1637     foreach my $p (split(/,/, $value)) {
1638         next if $p =~ m/^\s*$/;
1639
1640         if ($p =~ m/^(model=)?(i6300esb|ib700)$/) {
1641             $res->{model} = $2;
1642         } elsif ($p =~ m/^(action=)?(reset|shutdown|poweroff|pause|debug|none)$/) {
1643             $res->{action} = $2;
1644         } else {
1645             return undef;
1646         }
1647     }
1648
1649     return $res;
1650 }
1651
1652 sub parse_usb_device {
1653     my ($value) = @_;
1654
1655     return undef if !$value;
1656
1657     my @dl = split(/,/, $value);
1658     my $found;
1659
1660     my $res = {};
1661     foreach my $v (@dl) {
1662         if ($v =~ m/^host=(0x)?([0-9A-Fa-f]{4}):(0x)?([0-9A-Fa-f]{4})$/) {
1663             $found = 1;
1664             $res->{vendorid} = $2;
1665             $res->{productid} = $4;
1666         } elsif ($v =~ m/^host=(\d+)\-(\d+(\.\d+)*)$/) {
1667             $found = 1;
1668             $res->{hostbus} = $1;
1669             $res->{hostport} = $2;
1670         } elsif ($v =~ m/^spice$/) {
1671             $found = 1;
1672             $res->{spice} = 1;
1673         } else {
1674             return undef;
1675         }
1676     }
1677     return undef if !$found;
1678
1679     return $res;
1680 }
1681
1682 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
1683 sub verify_usb_device {
1684     my ($value, $noerr) = @_;
1685
1686     return $value if parse_usb_device($value);
1687
1688     return undef if $noerr;
1689
1690     die "unable to parse usb device\n";
1691 }
1692
1693 # add JSON properties for create and set function
1694 sub json_config_properties {
1695     my $prop = shift;
1696
1697     foreach my $opt (keys %$confdesc) {
1698         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
1699         $prop->{$opt} = $confdesc->{$opt};
1700     }
1701
1702     return $prop;
1703 }
1704
1705 sub check_type {
1706     my ($key, $value) = @_;
1707
1708     die "unknown setting '$key'\n" if !$confdesc->{$key};
1709
1710     my $type = $confdesc->{$key}->{type};
1711
1712     if (!defined($value)) {
1713         die "got undefined value\n";
1714     }
1715
1716     if ($value =~ m/[\n\r]/) {
1717         die "property contains a line feed\n";
1718     }
1719
1720     if ($type eq 'boolean') {
1721         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
1722         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
1723         die "type check ('boolean') failed - got '$value'\n";
1724     } elsif ($type eq 'integer') {
1725         return int($1) if $value =~ m/^(\d+)$/;
1726         die "type check ('integer') failed - got '$value'\n";
1727     } elsif ($type eq 'number') {
1728         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
1729         die "type check ('number') failed - got '$value'\n";
1730     } elsif ($type eq 'string') {
1731         if (my $fmt = $confdesc->{$key}->{format}) {
1732             if ($fmt eq 'pve-qm-drive') {
1733                 # special case - we need to pass $key to parse_drive()
1734                 my $drive = parse_drive($key, $value);
1735                 return $value if $drive;
1736                 die "unable to parse drive options\n";
1737             }
1738             PVE::JSONSchema::check_format($fmt, $value);
1739             return $value;
1740         }
1741         $value =~ s/^\"(.*)\"$/$1/;
1742         return $value;
1743     } else {
1744         die "internal error"
1745     }
1746 }
1747
1748 sub lock_config_full {
1749     my ($vmid, $timeout, $code, @param) = @_;
1750
1751     my $filename = config_file_lock($vmid);
1752
1753     my $res = lock_file($filename, $timeout, $code, @param);
1754
1755     die $@ if $@;
1756
1757     return $res;
1758 }
1759
1760 sub lock_config_mode {
1761     my ($vmid, $timeout, $shared, $code, @param) = @_;
1762
1763     my $filename = config_file_lock($vmid);
1764
1765     my $res = lock_file_full($filename, $timeout, $shared, $code, @param);
1766
1767     die $@ if $@;
1768
1769     return $res;
1770 }
1771
1772 sub lock_config {
1773     my ($vmid, $code, @param) = @_;
1774
1775     return lock_config_full($vmid, 10, $code, @param);
1776 }
1777
1778 sub cfs_config_path {
1779     my ($vmid, $node) = @_;
1780
1781     $node = $nodename if !$node;
1782     return "nodes/$node/qemu-server/$vmid.conf";
1783 }
1784
1785 sub check_iommu_support{
1786     #fixme : need to check IOMMU support
1787     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
1788
1789     my $iommu=1;
1790     return $iommu;
1791
1792 }
1793
1794 sub config_file {
1795     my ($vmid, $node) = @_;
1796
1797     my $cfspath = cfs_config_path($vmid, $node);
1798     return "/etc/pve/$cfspath";
1799 }
1800
1801 sub config_file_lock {
1802     my ($vmid) = @_;
1803
1804     return "$lock_dir/lock-$vmid.conf";
1805 }
1806
1807 sub touch_config {
1808     my ($vmid) = @_;
1809
1810     my $conf = config_file($vmid);
1811     utime undef, undef, $conf;
1812 }
1813
1814 sub destroy_vm {
1815     my ($storecfg, $vmid, $keep_empty_config) = @_;
1816
1817     my $conffile = config_file($vmid);
1818
1819     my $conf = load_config($vmid);
1820
1821     check_lock($conf);
1822
1823     # only remove disks owned by this VM
1824     foreach_drive($conf, sub {
1825         my ($ds, $drive) = @_;
1826
1827         return if drive_is_cdrom($drive);
1828
1829         my $volid = $drive->{file};
1830
1831         return if !$volid || $volid =~ m|^/|;
1832
1833         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
1834         return if !$path || !$owner || ($owner != $vmid);
1835
1836         PVE::Storage::vdisk_free($storecfg, $volid);
1837     });
1838
1839     if ($keep_empty_config) {
1840         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
1841     } else {
1842         unlink $conffile;
1843     }
1844
1845     # also remove unused disk
1846     eval {
1847         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
1848
1849         eval {
1850             PVE::Storage::foreach_volid($dl, sub {
1851                 my ($volid, $sid, $volname, $d) = @_;
1852                 PVE::Storage::vdisk_free($storecfg, $volid);
1853             });
1854         };
1855         warn $@ if $@;
1856
1857     };
1858     warn $@ if $@;
1859 }
1860
1861 sub load_config {
1862     my ($vmid, $node) = @_;
1863
1864     my $cfspath = cfs_config_path($vmid, $node);
1865
1866     my $conf = PVE::Cluster::cfs_read_file($cfspath);
1867
1868     die "no such VM ('$vmid')\n" if !defined($conf);
1869
1870     return $conf;
1871 }
1872
1873 sub parse_vm_config {
1874     my ($filename, $raw) = @_;
1875
1876     return undef if !defined($raw);
1877
1878     my $res = {
1879         digest => Digest::SHA::sha1_hex($raw),
1880         snapshots => {},
1881         pending => {},
1882     };
1883
1884     $filename =~ m|/qemu-server/(\d+)\.conf$|
1885         || die "got strange filename '$filename'";
1886
1887     my $vmid = $1;
1888
1889     my $conf = $res;
1890     my $descr;
1891     my $section = '';
1892
1893     my @lines = split(/\n/, $raw);
1894     foreach my $line (@lines) {
1895         next if $line =~ m/^\s*$/;
1896
1897         if ($line =~ m/^\[PENDING\]\s*$/i) {
1898             $section = 'pending';
1899             if (defined($descr)) {
1900                 $descr =~ s/\s+$//;
1901                 $conf->{description} = $descr;
1902             }
1903             $descr = undef;
1904             $conf = $res->{$section} = {};
1905             next;
1906
1907         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
1908             $section = $1;
1909             if (defined($descr)) {
1910                 $descr =~ s/\s+$//;
1911                 $conf->{description} = $descr;
1912             }
1913             $descr = undef;
1914             $conf = $res->{snapshots}->{$section} = {};
1915             next;
1916         }
1917
1918         if ($line =~ m/^\#(.*)\s*$/) {
1919             $descr = '' if !defined($descr);
1920             $descr .= PVE::Tools::decode_text($1) . "\n";
1921             next;
1922         }
1923
1924         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
1925             $descr = '' if !defined($descr);
1926             $descr .= PVE::Tools::decode_text($2);
1927         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
1928             $conf->{snapstate} = $1;
1929         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
1930             my $key = $1;
1931             my $value = $2;
1932             $conf->{$key} = $value;
1933         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
1934             my $value = $1;
1935             if ($section eq 'pending') {
1936                 $conf->{delete} = $value; # we parse this later
1937             } else {
1938                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
1939             }
1940         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
1941             my $key = $1;
1942             my $value = $2;
1943             eval { $value = check_type($key, $value); };
1944             if ($@) {
1945                 warn "vm $vmid - unable to parse value of '$key' - $@";
1946             } else {
1947                 my $fmt = $confdesc->{$key}->{format};
1948                 if ($fmt && $fmt eq 'pve-qm-drive') {
1949                     my $v = parse_drive($key, $value);
1950                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
1951                         $v->{file} = $volid;
1952                         $value = print_drive($vmid, $v);
1953                     } else {
1954                         warn "vm $vmid - unable to parse value of '$key'\n";
1955                         next;
1956                     }
1957                 }
1958
1959                 if ($key eq 'cdrom') {
1960                     $conf->{ide2} = $value;
1961                 } else {
1962                     $conf->{$key} = $value;
1963                 }
1964             }
1965         }
1966     }
1967
1968     if (defined($descr)) {
1969         $descr =~ s/\s+$//;
1970         $conf->{description} = $descr;
1971     }
1972     delete $res->{snapstate}; # just to be sure
1973
1974     return $res;
1975 }
1976
1977 sub write_vm_config {
1978     my ($filename, $conf) = @_;
1979
1980     delete $conf->{snapstate}; # just to be sure
1981
1982     if ($conf->{cdrom}) {
1983         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
1984         $conf->{ide2} = $conf->{cdrom};
1985         delete $conf->{cdrom};
1986     }
1987
1988     # we do not use 'smp' any longer
1989     if ($conf->{sockets}) {
1990         delete $conf->{smp};
1991     } elsif ($conf->{smp}) {
1992         $conf->{sockets} = $conf->{smp};
1993         delete $conf->{cores};
1994         delete $conf->{smp};
1995     }
1996
1997     my $used_volids = {};
1998
1999     my $cleanup_config = sub {
2000         my ($cref, $pending, $snapname) = @_;
2001
2002         foreach my $key (keys %$cref) {
2003             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2004                 $key eq 'snapstate' || $key eq 'pending';
2005             my $value = $cref->{$key};
2006             if ($key eq 'delete') {
2007                 die "propertry 'delete' is only allowed in [PENDING]\n"
2008                     if !$pending;
2009                 # fixme: check syntax?
2010                 next;
2011             }
2012             eval { $value = check_type($key, $value); };
2013             die "unable to parse value of '$key' - $@" if $@;
2014
2015             $cref->{$key} = $value;
2016
2017             if (!$snapname && valid_drivename($key)) {
2018                 my $drive = parse_drive($key, $value);
2019                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2020             }
2021         }
2022     };
2023
2024     &$cleanup_config($conf);
2025
2026     &$cleanup_config($conf->{pending}, 1);
2027
2028     foreach my $snapname (keys %{$conf->{snapshots}}) {
2029         die "internal error" if $snapname eq 'pending';
2030         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2031     }
2032
2033     # remove 'unusedX' settings if we re-add a volume
2034     foreach my $key (keys %$conf) {
2035         my $value = $conf->{$key};
2036         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2037             delete $conf->{$key};
2038         }
2039     }
2040
2041     my $generate_raw_config = sub {
2042         my ($conf, $pending) = @_;
2043
2044         my $raw = '';
2045
2046         # add description as comment to top of file
2047         if (defined(my $descr = $conf->{description})) {
2048             if ($descr) {
2049                 foreach my $cl (split(/\n/, $descr)) {
2050                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2051                 }
2052             } else {
2053                 $raw .= "#\n" if $pending;
2054             }
2055         }
2056
2057         foreach my $key (sort keys %$conf) {
2058             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2059             $raw .= "$key: $conf->{$key}\n";
2060         }
2061         return $raw;
2062     };
2063
2064     my $raw = &$generate_raw_config($conf);
2065
2066     if (scalar(keys %{$conf->{pending}})){
2067         $raw .= "\n[PENDING]\n";
2068         $raw .= &$generate_raw_config($conf->{pending}, 1);
2069     }
2070
2071     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2072         $raw .= "\n[$snapname]\n";
2073         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2074     }
2075
2076     return $raw;
2077 }
2078
2079 sub update_config_nolock {
2080     my ($vmid, $conf, $skiplock) = @_;
2081
2082     check_lock($conf) if !$skiplock;
2083
2084     my $cfspath = cfs_config_path($vmid);
2085
2086     PVE::Cluster::cfs_write_file($cfspath, $conf);
2087 }
2088
2089 sub update_config {
2090     my ($vmid, $conf, $skiplock) = @_;
2091
2092     lock_config($vmid, &update_config_nolock, $conf, $skiplock);
2093 }
2094
2095 sub load_defaults {
2096
2097     my $res = {};
2098
2099     # we use static defaults from our JSON schema configuration
2100     foreach my $key (keys %$confdesc) {
2101         if (defined(my $default = $confdesc->{$key}->{default})) {
2102             $res->{$key} = $default;
2103         }
2104     }
2105
2106     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2107     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2108
2109     return $res;
2110 }
2111
2112 sub config_list {
2113     my $vmlist = PVE::Cluster::get_vmlist();
2114     my $res = {};
2115     return $res if !$vmlist || !$vmlist->{ids};
2116     my $ids = $vmlist->{ids};
2117
2118     foreach my $vmid (keys %$ids) {
2119         my $d = $ids->{$vmid};
2120         next if !$d->{node} || $d->{node} ne $nodename;
2121         next if !$d->{type} || $d->{type} ne 'qemu';
2122         $res->{$vmid}->{exists} = 1;
2123     }
2124     return $res;
2125 }
2126
2127 # test if VM uses local resources (to prevent migration)
2128 sub check_local_resources {
2129     my ($conf, $noerr) = @_;
2130
2131     my $loc_res = 0;
2132
2133     $loc_res = 1 if $conf->{hostusb}; # old syntax
2134     $loc_res = 1 if $conf->{hostpci}; # old syntax
2135
2136     foreach my $k (keys %$conf) {
2137         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2138         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2139     }
2140
2141     die "VM uses local resources\n" if $loc_res && !$noerr;
2142
2143     return $loc_res;
2144 }
2145
2146 # check if used storages are available on all nodes (use by migrate)
2147 sub check_storage_availability {
2148     my ($storecfg, $conf, $node) = @_;
2149
2150     foreach_drive($conf, sub {
2151         my ($ds, $drive) = @_;
2152
2153         my $volid = $drive->{file};
2154         return if !$volid;
2155
2156         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2157         return if !$sid;
2158
2159         # check if storage is available on both nodes
2160         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2161         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2162    });
2163 }
2164
2165 # list nodes where all VM images are available (used by has_feature API)
2166 sub shared_nodes {
2167     my ($conf, $storecfg) = @_;
2168
2169     my $nodelist = PVE::Cluster::get_nodelist();
2170     my $nodehash = { map { $_ => 1 } @$nodelist };
2171     my $nodename = PVE::INotify::nodename();
2172
2173     foreach_drive($conf, sub {
2174         my ($ds, $drive) = @_;
2175
2176         my $volid = $drive->{file};
2177         return if !$volid;
2178
2179         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2180         if ($storeid) {
2181             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2182             if ($scfg->{disable}) {
2183                 $nodehash = {};
2184             } elsif (my $avail = $scfg->{nodes}) {
2185                 foreach my $node (keys %$nodehash) {
2186                     delete $nodehash->{$node} if !$avail->{$node};
2187                 }
2188             } elsif (!$scfg->{shared}) {
2189                 foreach my $node (keys %$nodehash) {
2190                     delete $nodehash->{$node} if $node ne $nodename
2191                 }
2192             }
2193         }
2194     });
2195
2196     return $nodehash
2197 }
2198
2199 sub check_lock {
2200     my ($conf) = @_;
2201
2202     die "VM is locked ($conf->{lock})\n" if $conf->{lock};
2203 }
2204
2205 sub check_cmdline {
2206     my ($pidfile, $pid) = @_;
2207
2208     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2209     if (defined($fh)) {
2210         my $line = <$fh>;
2211         $fh->close;
2212         return undef if !$line;
2213         my @param = split(/\0/, $line);
2214
2215         my $cmd = $param[0];
2216         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2217
2218         for (my $i = 0; $i < scalar (@param); $i++) {
2219             my $p = $param[$i];
2220             next if !$p;
2221             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2222                 my $p = $param[$i+1];
2223                 return 1 if $p && ($p eq $pidfile);
2224                 return undef;
2225             }
2226         }
2227     }
2228     return undef;
2229 }
2230
2231 sub check_running {
2232     my ($vmid, $nocheck, $node) = @_;
2233
2234     my $filename = config_file($vmid, $node);
2235
2236     die "unable to find configuration file for VM $vmid - no such machine\n"
2237         if !$nocheck && ! -f $filename;
2238
2239     my $pidfile = pidfile_name($vmid);
2240
2241     if (my $fd = IO::File->new("<$pidfile")) {
2242         my $st = stat($fd);
2243         my $line = <$fd>;
2244         close($fd);
2245
2246         my $mtime = $st->mtime;
2247         if ($mtime > time()) {
2248             warn "file '$filename' modified in future\n";
2249         }
2250
2251         if ($line =~ m/^(\d+)$/) {
2252             my $pid = $1;
2253             if (check_cmdline($pidfile, $pid)) {
2254                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2255                     return $pid;
2256                 }
2257             }
2258         }
2259     }
2260
2261     return undef;
2262 }
2263
2264 sub vzlist {
2265
2266     my $vzlist = config_list();
2267
2268     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2269
2270     while (defined(my $de = $fd->read)) {
2271         next if $de !~ m/^(\d+)\.pid$/;
2272         my $vmid = $1;
2273         next if !defined($vzlist->{$vmid});
2274         if (my $pid = check_running($vmid)) {
2275             $vzlist->{$vmid}->{pid} = $pid;
2276         }
2277     }
2278
2279     return $vzlist;
2280 }
2281
2282 sub disksize {
2283     my ($storecfg, $conf) = @_;
2284
2285     my $bootdisk = $conf->{bootdisk};
2286     return undef if !$bootdisk;
2287     return undef if !valid_drivename($bootdisk);
2288
2289     return undef if !$conf->{$bootdisk};
2290
2291     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2292     return undef if !defined($drive);
2293
2294     return undef if drive_is_cdrom($drive);
2295
2296     my $volid = $drive->{file};
2297     return undef if !$volid;
2298
2299     return $drive->{size};
2300 }
2301
2302 my $last_proc_pid_stat;
2303
2304 # get VM status information
2305 # This must be fast and should not block ($full == false)
2306 # We only query KVM using QMP if $full == true (this can be slow)
2307 sub vmstatus {
2308     my ($opt_vmid, $full) = @_;
2309
2310     my $res = {};
2311
2312     my $storecfg = PVE::Storage::config();
2313
2314     my $list = vzlist();
2315     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2316
2317     my $cpucount = $cpuinfo->{cpus} || 1;
2318
2319     foreach my $vmid (keys %$list) {
2320         next if $opt_vmid && ($vmid ne $opt_vmid);
2321
2322         my $cfspath = cfs_config_path($vmid);
2323         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2324
2325         my $d = {};
2326         $d->{pid} = $list->{$vmid}->{pid};
2327
2328         # fixme: better status?
2329         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2330
2331         my $size = disksize($storecfg, $conf);
2332         if (defined($size)) {
2333             $d->{disk} = 0; # no info available
2334             $d->{maxdisk} = $size;
2335         } else {
2336             $d->{disk} = 0;
2337             $d->{maxdisk} = 0;
2338         }
2339
2340         $d->{cpus} = ($conf->{sockets} || 1) * ($conf->{cores} || 1);
2341         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2342         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2343
2344         $d->{name} = $conf->{name} || "VM $vmid";
2345         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024) : 0;
2346
2347         if ($conf->{balloon}) {
2348             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2349             $d->{shares} = defined($conf->{shares}) ? $conf->{shares} : 1000;
2350         }
2351
2352         $d->{uptime} = 0;
2353         $d->{cpu} = 0;
2354         $d->{mem} = 0;
2355
2356         $d->{netout} = 0;
2357         $d->{netin} = 0;
2358
2359         $d->{diskread} = 0;
2360         $d->{diskwrite} = 0;
2361
2362         $d->{template} = is_template($conf);
2363
2364         $res->{$vmid} = $d;
2365     }
2366
2367     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2368     foreach my $dev (keys %$netdev) {
2369         next if $dev !~ m/^tap([1-9]\d*)i/;
2370         my $vmid = $1;
2371         my $d = $res->{$vmid};
2372         next if !$d;
2373
2374         $d->{netout} += $netdev->{$dev}->{receive};
2375         $d->{netin} += $netdev->{$dev}->{transmit};
2376
2377         if ($full) {
2378             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2379             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2380         }
2381
2382     }
2383
2384     my $ctime = gettimeofday;
2385
2386     foreach my $vmid (keys %$list) {
2387
2388         my $d = $res->{$vmid};
2389         my $pid = $d->{pid};
2390         next if !$pid;
2391
2392         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2393         next if !$pstat; # not running
2394
2395         my $used = $pstat->{utime} + $pstat->{stime};
2396
2397         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2398
2399         if ($pstat->{vsize}) {
2400             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2401         }
2402
2403         my $old = $last_proc_pid_stat->{$pid};
2404         if (!$old) {
2405             $last_proc_pid_stat->{$pid} = {
2406                 time => $ctime,
2407                 used => $used,
2408                 cpu => 0,
2409             };
2410             next;
2411         }
2412
2413         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2414
2415         if ($dtime > 1000) {
2416             my $dutime = $used -  $old->{used};
2417
2418             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2419             $last_proc_pid_stat->{$pid} = {
2420                 time => $ctime,
2421                 used => $used,
2422                 cpu => $d->{cpu},
2423             };
2424         } else {
2425             $d->{cpu} = $old->{cpu};
2426         }
2427     }
2428
2429     return $res if !$full;
2430
2431     my $qmpclient = PVE::QMPClient->new();
2432
2433     my $ballooncb = sub {
2434         my ($vmid, $resp) = @_;
2435
2436         my $info = $resp->{'return'};
2437         return if !$info->{max_mem};
2438
2439         my $d = $res->{$vmid};
2440
2441         # use memory assigned to VM
2442         $d->{maxmem} = $info->{max_mem};
2443         $d->{balloon} = $info->{actual};
2444
2445         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2446             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2447             $d->{freemem} = $info->{free_mem};
2448         }
2449
2450         $d->{ballooninfo} = $info;
2451     };
2452
2453     my $blockstatscb = sub {
2454         my ($vmid, $resp) = @_;
2455         my $data = $resp->{'return'} || [];
2456         my $totalrdbytes = 0;
2457         my $totalwrbytes = 0;
2458
2459         for my $blockstat (@$data) {
2460             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2461             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2462
2463             $blockstat->{device} =~ s/drive-//;
2464             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2465         }
2466         $res->{$vmid}->{diskread} = $totalrdbytes;
2467         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2468     };
2469
2470     my $statuscb = sub {
2471         my ($vmid, $resp) = @_;
2472
2473         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2474         # this fails if ballon driver is not loaded, so this must be
2475         # the last commnand (following command are aborted if this fails).
2476         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2477
2478         my $status = 'unknown';
2479         if (!defined($status = $resp->{'return'}->{status})) {
2480             warn "unable to get VM status\n";
2481             return;
2482         }
2483
2484         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2485     };
2486
2487     foreach my $vmid (keys %$list) {
2488         next if $opt_vmid && ($vmid ne $opt_vmid);
2489         next if !$res->{$vmid}->{pid}; # not running
2490         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2491     }
2492
2493     $qmpclient->queue_execute(undef, 1);
2494
2495     foreach my $vmid (keys %$list) {
2496         next if $opt_vmid && ($vmid ne $opt_vmid);
2497         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2498     }
2499
2500     return $res;
2501 }
2502
2503 sub foreach_dimm {
2504     my ($conf, $vmid, $memory, $sockets, $func) = @_;
2505
2506     my $dimm_id = 0;
2507     my $current_size = 1024;
2508     my $dimm_size = 512;
2509     return if $current_size == $memory;
2510
2511     for (my $j = 0; $j < 8; $j++) {
2512         for (my $i = 0; $i < 32; $i++) {
2513             my $name = "dimm${dimm_id}";
2514             $dimm_id++;
2515             my $numanode = $i % $sockets;
2516             $current_size += $dimm_size;
2517             &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
2518             return  $current_size if $current_size >= $memory;
2519         }
2520         $dimm_size *= 2;
2521     }
2522 }
2523
2524 sub foreach_reverse_dimm {
2525     my ($conf, $vmid, $memory, $sockets, $func) = @_;
2526
2527     my $dimm_id = 253;
2528     my $current_size = 4177920;
2529     my $dimm_size = 65536;
2530     return if $current_size == $memory;
2531
2532     for (my $j = 0; $j < 8; $j++) {
2533         for (my $i = 0; $i < 32; $i++) {
2534             my $name = "dimm${dimm_id}";
2535             $dimm_id--;
2536             my $numanode = $i % $sockets;
2537             $current_size -= $dimm_size;
2538             &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
2539             return  $current_size if $current_size <= $memory;
2540         }
2541         $dimm_size /= 2;
2542     }
2543 }
2544
2545 sub foreach_drive {
2546     my ($conf, $func) = @_;
2547
2548     foreach my $ds (keys %$conf) {
2549         next if !valid_drivename($ds);
2550
2551         my $drive = parse_drive($ds, $conf->{$ds});
2552         next if !$drive;
2553
2554         &$func($ds, $drive);
2555     }
2556 }
2557
2558 sub foreach_volid {
2559     my ($conf, $func) = @_;
2560
2561     my $volhash = {};
2562
2563     my $test_volid = sub {
2564         my ($volid, $is_cdrom) = @_;
2565
2566         return if !$volid;
2567
2568         $volhash->{$volid} = $is_cdrom || 0;
2569     };
2570
2571     foreach_drive($conf, sub {
2572         my ($ds, $drive) = @_;
2573         &$test_volid($drive->{file}, drive_is_cdrom($drive));
2574     });
2575
2576     foreach my $snapname (keys %{$conf->{snapshots}}) {
2577         my $snap = $conf->{snapshots}->{$snapname};
2578         &$test_volid($snap->{vmstate}, 0);
2579         foreach_drive($snap, sub {
2580             my ($ds, $drive) = @_;
2581             &$test_volid($drive->{file}, drive_is_cdrom($drive));
2582         });
2583     }
2584
2585     foreach my $volid (keys %$volhash) {
2586         &$func($volid, $volhash->{$volid});
2587     }
2588 }
2589
2590 sub vga_conf_has_spice {
2591     my ($vga) = @_;
2592
2593     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
2594
2595     return $1 || 1;
2596 }
2597
2598 sub config_to_command {
2599     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2600
2601     my $cmd = [];
2602     my $globalFlags = [];
2603     my $machineFlags = [];
2604     my $rtcFlags = [];
2605     my $cpuFlags = [];
2606     my $devices = [];
2607     my $pciaddr = '';
2608     my $bridges = {};
2609     my $kvmver = kvm_user_version();
2610     my $vernum = 0; # unknown
2611     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
2612         $vernum = $1*1000000+$2*1000;
2613     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
2614         $vernum = $1*1000000+$2*1000+$3;
2615     }
2616
2617     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
2618
2619     my $have_ovz = -f '/proc/vz/vestat';
2620
2621     my $q35 = machine_type_is_q35($conf);
2622     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2623     my $machine_type = $forcemachine || $conf->{machine};
2624
2625     my $cpuunits = defined($conf->{cpuunits}) ?
2626             $conf->{cpuunits} : $defaults->{cpuunits};
2627
2628     push @$cmd, '/usr/bin/systemd-run';
2629     push @$cmd, '--scope';
2630     push @$cmd, '--slice', "qemu";
2631     push @$cmd, '--unit', $vmid;
2632     push @$cmd, '-p', "CPUShares=$cpuunits";
2633     if ($conf->{cpulimit}) {
2634         my $cpulimit = int($conf->{cpulimit} * 100);
2635         push @$cmd, '-p', "CPUQuota=$cpulimit\%";
2636     }
2637
2638     push @$cmd, '/usr/bin/kvm';
2639
2640     push @$cmd, '-id', $vmid;
2641
2642     my $use_virtio = 0;
2643
2644     my $qmpsocket = qmp_socket($vmid);
2645     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2646     push @$cmd, '-mon', "chardev=qmp,mode=control";
2647
2648     my $socket = vnc_socket($vmid);
2649     push @$cmd,  '-vnc', "unix:$socket,x509,password";
2650
2651     push @$cmd, '-pidfile' , pidfile_name($vmid);
2652
2653     push @$cmd, '-daemonize';
2654
2655     if ($conf->{smbios1}) {
2656         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
2657     }
2658
2659     if ($q35) {
2660         # the q35 chipset support native usb2, so we enable usb controller
2661         # by default for this machine type
2662         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
2663     } else {
2664         $pciaddr = print_pci_addr("piix3", $bridges);
2665         push @$devices, '-device', "piix3-usb-uhci,id=uhci$pciaddr.0x2";
2666
2667         my $use_usb2 = 0;
2668         for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2669             next if !$conf->{"usb$i"};
2670             $use_usb2 = 1;
2671         }
2672         # include usb device config
2673         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-usb.cfg' if $use_usb2;
2674     }
2675
2676     my $vga = $conf->{vga};
2677
2678     my $qxlnum = vga_conf_has_spice($vga);
2679     $vga = 'qxl' if $qxlnum;
2680
2681     if (!$vga) {
2682         if ($conf->{ostype} && ($conf->{ostype} eq 'win8' ||
2683                                 $conf->{ostype} eq 'win7' ||
2684                                 $conf->{ostype} eq 'w2k8')) {
2685             $vga = 'std';
2686         } else {
2687             $vga = 'cirrus';
2688         }
2689     }
2690
2691     # enable absolute mouse coordinates (needed by vnc)
2692     my $tablet;
2693     if (defined($conf->{tablet})) {
2694         $tablet = $conf->{tablet};
2695     } else {
2696         $tablet = $defaults->{tablet};
2697         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
2698         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
2699     }
2700
2701     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
2702
2703     # host pci devices
2704     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
2705         my $d = parse_hostpci($conf->{"hostpci$i"});
2706         next if !$d;
2707
2708         my $pcie = $d->{pcie};
2709         if($pcie){
2710             die "q35 machine model is not enabled" if !$q35;
2711             $pciaddr = print_pcie_addr("hostpci$i");
2712         }else{
2713             $pciaddr = print_pci_addr("hostpci$i", $bridges);
2714         }
2715
2716         my $rombar = $d->{rombar} && $d->{rombar} eq 'off' ? ",rombar=0" : "";
2717         my $driver = $d->{driver} && $d->{driver} eq 'vfio' ? "vfio-pci" : "pci-assign";
2718         my $xvga = $d->{'x-vga'} && $d->{'x-vga'} eq 'on' ? ",x-vga=on" : "";
2719         if ($xvga && $xvga ne '') {
2720             push @$cpuFlags, 'kvm=off';
2721             $vga = 'none';
2722         }
2723         $driver = "vfio-pci" if $xvga ne '';
2724         my $pcidevices = $d->{pciid};
2725         my $multifunction = 1 if @$pcidevices > 1;
2726
2727         my $j=0;
2728         foreach my $pcidevice (@$pcidevices) {
2729
2730             my $id = "hostpci$i";
2731             $id .= ".$j" if $multifunction;
2732             my $addr = $pciaddr;
2733             $addr .= ".$j" if $multifunction;
2734             my $devicestr = "$driver,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
2735
2736             if($j == 0){
2737                 $devicestr .= "$rombar$xvga";
2738                 $devicestr .= ",multifunction=on" if $multifunction;
2739             }
2740
2741             push @$devices, '-device', $devicestr;
2742             $j++;
2743         }
2744     }
2745
2746     # usb devices
2747     for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2748         my $d = parse_usb_device($conf->{"usb$i"});
2749         next if !$d;
2750         if ($d->{vendorid} && $d->{productid}) {
2751             push @$devices, '-device', "usb-host,vendorid=0x$d->{vendorid},productid=0x$d->{productid}";
2752         } elsif (defined($d->{hostbus}) && defined($d->{hostport})) {
2753             push @$devices, '-device', "usb-host,hostbus=$d->{hostbus},hostport=$d->{hostport}";
2754         } elsif ($d->{spice}) {
2755             # usb redir support for spice
2756             push @$devices, '-chardev', "spicevmc,id=usbredirchardev$i,name=usbredir";
2757             push @$devices, '-device', "usb-redir,chardev=usbredirchardev$i,id=usbredirdev$i,bus=ehci.0";
2758         }
2759     }
2760
2761     # serial devices
2762     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2763         if (my $path = $conf->{"serial$i"}) {
2764             if ($path eq 'socket') {
2765                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
2766                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
2767                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2768             } else {
2769                 die "no such serial device\n" if ! -c $path;
2770                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
2771                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2772             }
2773         }
2774     }
2775
2776     # parallel devices
2777     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
2778         if (my $path = $conf->{"parallel$i"}) {
2779             die "no such parallel device\n" if ! -c $path;
2780             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
2781             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
2782             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
2783         }
2784     }
2785
2786     my $vmname = $conf->{name} || "vm$vmid";
2787
2788     push @$cmd, '-name', $vmname;
2789
2790     my $sockets = 1;
2791     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
2792     $sockets = $conf->{sockets} if  $conf->{sockets};
2793
2794     my $cores = $conf->{cores} || 1;
2795
2796     my $maxcpus = $sockets * $cores;
2797
2798     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
2799
2800     my $allowed_vcpus = $cpuinfo->{cpus};
2801
2802     die "MAX $maxcpus vcpus allowed per VM on this node\n"
2803         if ($allowed_vcpus < $maxcpus);
2804
2805     push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
2806
2807     push @$cmd, '-nodefaults';
2808
2809     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2810
2811     my $bootindex_hash = {};
2812     my $i = 1;
2813     foreach my $o (split(//, $bootorder)) {
2814         $bootindex_hash->{$o} = $i*100;
2815         $i++;
2816     }
2817
2818     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000";
2819
2820     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
2821
2822     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
2823
2824     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
2825
2826     # time drift fix
2827     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
2828
2829     my $nokvm = defined($conf->{kvm}) && $conf->{kvm} == 0 ? 1 : 0;
2830     my $useLocaltime = $conf->{localtime};
2831
2832     if (my $ost = $conf->{ostype}) {
2833         # other, wxp, w2k, w2k3, w2k8, wvista, win7, win8, l24, l26, solaris
2834
2835         if ($ost =~ m/^w/) { # windows
2836             $useLocaltime = 1 if !defined($conf->{localtime});
2837
2838             # use time drift fix when acpi is enabled
2839             if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
2840                 $tdf = 1 if !defined($conf->{tdf});
2841             }
2842         }
2843
2844         if ($ost eq 'win7' || $ost eq 'win8' || $ost eq 'w2k8' ||
2845             $ost eq 'wvista') {
2846             push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
2847             push @$cmd, '-no-hpet';
2848             if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
2849                 push @$cpuFlags , 'hv_spinlocks=0x1fff' if !$nokvm;
2850                 push @$cpuFlags , 'hv_vapic' if !$nokvm;
2851                 push @$cpuFlags , 'hv_time' if !$nokvm;
2852
2853             } else {
2854                 push @$cpuFlags , 'hv_spinlocks=0xffff' if !$nokvm;
2855             }
2856         }
2857
2858         if ($ost eq 'win7' || $ost eq 'win8') {
2859             push @$cpuFlags , 'hv_relaxed' if !$nokvm;
2860         }
2861     }
2862
2863     push @$rtcFlags, 'driftfix=slew' if $tdf;
2864
2865     if ($nokvm) {
2866         push @$machineFlags, 'accel=tcg';
2867     } else {
2868         die "No accelerator found!\n" if !$cpuinfo->{hvm};
2869     }
2870
2871     if ($machine_type) {
2872         push @$machineFlags, "type=${machine_type}";
2873     }
2874
2875     if ($conf->{startdate}) {
2876         push @$rtcFlags, "base=$conf->{startdate}";
2877     } elsif ($useLocaltime) {
2878         push @$rtcFlags, 'base=localtime';
2879     }
2880
2881     my $cpu = $nokvm ? "qemu64" : "kvm64";
2882     $cpu = $conf->{cpu} if $conf->{cpu};
2883
2884     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
2885
2886     push @$cpuFlags , '-x2apic'
2887         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
2888
2889     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
2890
2891     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
2892
2893     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
2894
2895         push @$cpuFlags , '+kvm_pv_unhalt' if !$nokvm;
2896         push @$cpuFlags , '+kvm_pv_eoi' if !$nokvm;
2897     }
2898
2899     push @$cpuFlags, 'enforce' if $cpu ne 'host' && !$nokvm;
2900
2901     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
2902
2903     push @$cmd, '-cpu', $cpu;
2904
2905     my $memory = $conf->{memory} || $defaults->{memory};
2906     my $static_memory = 0;
2907     my $dimm_memory = 0;
2908
2909     if ($hotplug_features->{memory}) {
2910         die "Numa need to be enabled for memory hotplug\n" if !$conf->{numa};
2911         die "Total memory is bigger than ${MAX_MEM}MB\n" if $memory > $MAX_MEM;
2912         $static_memory = $STATICMEM;
2913         die "minimum memory must be ${static_memory}MB\n" if($memory < $static_memory);
2914         $dimm_memory = $memory - $static_memory;
2915         push @$cmd, '-m', "size=${static_memory},slots=255,maxmem=${MAX_MEM}M";
2916
2917     } else {
2918
2919         $static_memory = $memory;
2920         push @$cmd, '-m', $static_memory;
2921     }
2922
2923     if ($conf->{numa}) {
2924
2925         my $numa_totalmemory = undef;
2926         for (my $i = 0; $i < $MAX_NUMA; $i++) {
2927             next if !$conf->{"numa$i"};
2928             my $numa = parse_numa($conf->{"numa$i"});
2929             next if !$numa;
2930             # memory
2931             die "missing numa node$i memory value\n" if !$numa->{memory};
2932             my $numa_memory = $numa->{memory};
2933             $numa_totalmemory += $numa_memory;
2934             my $numa_object = "memory-backend-ram,id=ram-node$i,size=${numa_memory}M";
2935
2936             # cpus
2937             my $cpus_start = $numa->{cpus}->{start};
2938             die "missing numa node$i cpus\n" if !defined($cpus_start);
2939             my $cpus_end = $numa->{cpus}->{end} if defined($numa->{cpus}->{end});
2940             my $cpus = $cpus_start;
2941             if (defined($cpus_end)) {
2942                 $cpus .= "-$cpus_end";
2943                 die "numa node$i :  cpu range $cpus is incorrect\n" if $cpus_end <= $cpus_start;
2944             }
2945
2946             # hostnodes
2947             my $hostnodes_start = $numa->{hostnodes}->{start};
2948             if (defined($hostnodes_start)) {
2949                 my $hostnodes_end = $numa->{hostnodes}->{end} if defined($numa->{hostnodes}->{end});
2950                 my $hostnodes = $hostnodes_start;
2951                 if (defined($hostnodes_end)) {
2952                     $hostnodes .= "-$hostnodes_end";
2953                     die "host node $hostnodes range is incorrect\n" if $hostnodes_end <= $hostnodes_start;
2954                 }
2955
2956                 my $hostnodes_end_range = defined($hostnodes_end) ? $hostnodes_end : $hostnodes_start;
2957                 for (my $i = $hostnodes_start; $i <= $hostnodes_end_range; $i++ ) {
2958                     die "host numa node$i don't exist\n" if ! -d "/sys/devices/system/node/node$i/";
2959                 }
2960
2961                 # policy
2962                 my $policy = $numa->{policy};
2963                 die "you need to define a policy for hostnode $hostnodes\n" if !$policy;
2964                 $numa_object .= ",host-nodes=$hostnodes,policy=$policy";
2965             }
2966
2967             push @$cmd, '-object', $numa_object;
2968             push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
2969         }
2970
2971         die "total memory for NUMA nodes must be equal to vm static memory\n"
2972             if $numa_totalmemory && $numa_totalmemory != $static_memory;
2973
2974         #if no custom tology, we split memory and cores across numa nodes
2975         if(!$numa_totalmemory) {
2976
2977             my $numa_memory = ($static_memory / $sockets) . "M";
2978
2979             for (my $i = 0; $i < $sockets; $i++)  {
2980
2981                 my $cpustart = ($cores * $i);
2982                 my $cpuend = ($cpustart + $cores - 1) if $cores && $cores > 1;
2983                 my $cpus = $cpustart;
2984                 $cpus .= "-$cpuend" if $cpuend;
2985
2986                 push @$cmd, '-object', "memory-backend-ram,size=$numa_memory,id=ram-node$i";
2987                 push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
2988             }
2989         }
2990     }
2991
2992     if ($hotplug_features->{memory}) {
2993         foreach_dimm($conf, $vmid, $memory, $sockets, sub {
2994             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
2995             push @$cmd, "-object" , "memory-backend-ram,id=mem-$name,size=${dimm_size}M";
2996             push @$cmd, "-device", "pc-dimm,id=$name,memdev=mem-$name,node=$numanode";
2997
2998             #if dimm_memory is not aligned to dimm map
2999             if($current_size > $memory) {
3000                  $conf->{memory} = $current_size;
3001                  update_config_nolock($vmid, $conf, 1);
3002             }
3003         });
3004     }
3005
3006     push @$cmd, '-S' if $conf->{freeze};
3007
3008     # set keyboard layout
3009     my $kb = $conf->{keyboard} || $defaults->{keyboard};
3010     push @$cmd, '-k', $kb if $kb;
3011
3012     # enable sound
3013     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3014     #push @$cmd, '-soundhw', 'es1370';
3015     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3016
3017     if($conf->{agent}) {
3018         my $qgasocket = qmp_socket($vmid, 1);
3019         my $pciaddr = print_pci_addr("qga0", $bridges);
3020         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3021         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3022         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3023     }
3024
3025     my $spice_port;
3026
3027     if ($qxlnum) {
3028         if ($qxlnum > 1) {
3029             if ($conf->{ostype} && $conf->{ostype} =~ m/^w/){
3030                 for(my $i = 1; $i < $qxlnum; $i++){
3031                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3032                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3033                 }
3034             } else {
3035                 # assume other OS works like Linux
3036                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3037                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3038             }
3039         }
3040
3041         my $pciaddr = print_pci_addr("spice", $bridges);
3042
3043         my $nodename = PVE::INotify::nodename();
3044         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3045         $spice_port = PVE::Tools::next_spice_port($pfamily);
3046
3047         push @$devices, '-spice', "tls-port=${spice_port},addr=localhost,tls-ciphers=DES-CBC3-SHA,seamless-migration=on";
3048
3049         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3050         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3051         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3052     }
3053
3054     # enable balloon by default, unless explicitly disabled
3055     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3056         $pciaddr = print_pci_addr("balloon0", $bridges);
3057         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3058     }
3059
3060     if ($conf->{watchdog}) {
3061         my $wdopts = parse_watchdog($conf->{watchdog});
3062         $pciaddr = print_pci_addr("watchdog", $bridges);
3063         my $watchdog = $wdopts->{model} || 'i6300esb';
3064         push @$devices, '-device', "$watchdog$pciaddr";
3065         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3066     }
3067
3068     my $vollist = [];
3069     my $scsicontroller = {};
3070     my $ahcicontroller = {};
3071     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3072
3073     # Add iscsi initiator name if available
3074     if (my $initiator = get_initiator_name()) {
3075         push @$devices, '-iscsi', "initiator-name=$initiator";
3076     }
3077
3078     foreach_drive($conf, sub {
3079         my ($ds, $drive) = @_;
3080
3081         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3082             push @$vollist, $drive->{file};
3083         }
3084
3085         $use_virtio = 1 if $ds =~ m/^virtio/;
3086
3087         if (drive_is_cdrom ($drive)) {
3088             if ($bootindex_hash->{d}) {
3089                 $drive->{bootindex} = $bootindex_hash->{d};
3090                 $bootindex_hash->{d} += 1;
3091             }
3092         } else {
3093             if ($bootindex_hash->{c}) {
3094                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3095                 $bootindex_hash->{c} += 1;
3096             }
3097         }
3098
3099         if($drive->{interface} eq 'virtio'){
3100            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3101         }
3102
3103         if ($drive->{interface} eq 'scsi') {
3104
3105             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3106
3107             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3108             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3109
3110             my $iothread = '';
3111             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3112                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3113                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3114             }
3115
3116             my $queues = '';
3117             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3118                 $queues = ",num_queues=$drive->{queues}";
3119             }
3120
3121             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3122             $scsicontroller->{$controller}=1;
3123         }
3124
3125         if ($drive->{interface} eq 'sata') {
3126            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3127            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3128            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3129            $ahcicontroller->{$controller}=1;
3130         }
3131
3132         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3133         push @$devices, '-drive',$drive_cmd;
3134         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3135     });
3136
3137     for (my $i = 0; $i < $MAX_NETS; $i++) {
3138          next if !$conf->{"net$i"};
3139          my $d = parse_net($conf->{"net$i"});
3140          next if !$d;
3141
3142          $use_virtio = 1 if $d->{model} eq 'virtio';
3143
3144          if ($bootindex_hash->{n}) {
3145             $d->{bootindex} = $bootindex_hash->{n};
3146             $bootindex_hash->{n} += 1;
3147          }
3148
3149          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3150          push @$devices, '-netdev', $netdevfull;
3151
3152          my $netdevicefull = print_netdevice_full($vmid,$conf,$d,"net$i",$bridges);
3153          push @$devices, '-device', $netdevicefull;
3154     }
3155
3156     if (!$q35) {
3157         # add pci bridges
3158         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3159            $bridges->{1} = 1;
3160            $bridges->{2} = 1;
3161         }
3162
3163         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3164
3165         while (my ($k, $v) = each %$bridges) {
3166             $pciaddr = print_pci_addr("pci.$k");
3167             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3168         }
3169     }
3170
3171     # add custom args
3172     if ($conf->{args}) {
3173         my $aa = PVE::Tools::split_args($conf->{args});
3174         push @$cmd, @$aa;
3175     }
3176
3177     push @$cmd, @$devices;
3178     push @$cmd, '-rtc', join(',', @$rtcFlags)
3179         if scalar(@$rtcFlags);
3180     push @$cmd, '-machine', join(',', @$machineFlags)
3181         if scalar(@$machineFlags);
3182     push @$cmd, '-global', join(',', @$globalFlags)
3183         if scalar(@$globalFlags);
3184
3185     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3186 }
3187
3188 sub vnc_socket {
3189     my ($vmid) = @_;
3190     return "${var_run_tmpdir}/$vmid.vnc";
3191 }
3192
3193 sub spice_port {
3194     my ($vmid) = @_;
3195
3196     my $res = vm_mon_cmd($vmid, 'query-spice');
3197
3198     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3199 }
3200
3201 sub qmp_socket {
3202     my ($vmid, $qga) = @_;
3203     my $sockettype = $qga ? 'qga' : 'qmp';
3204     return "${var_run_tmpdir}/$vmid.$sockettype";
3205 }
3206
3207 sub pidfile_name {
3208     my ($vmid) = @_;
3209     return "${var_run_tmpdir}/$vmid.pid";
3210 }
3211
3212 sub vm_devices_list {
3213     my ($vmid) = @_;
3214
3215     my $res = vm_mon_cmd($vmid, 'query-pci');
3216     my $devices = {};
3217     foreach my $pcibus (@$res) {
3218         foreach my $device (@{$pcibus->{devices}}) {
3219             next if !$device->{'qdev_id'};
3220             if ($device->{'pci_bridge'}) {
3221                 $devices->{$device->{'qdev_id'}} = 1;
3222                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3223                     next if !$bridge_device->{'qdev_id'};
3224                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3225                     $devices->{$device->{'qdev_id'}}++;
3226                 }
3227             } else {
3228                 $devices->{$device->{'qdev_id'}} = 1;
3229             }
3230         }
3231     }
3232
3233     my $resblock = vm_mon_cmd($vmid, 'query-block');
3234     foreach my $block (@$resblock) {
3235         if($block->{device} =~ m/^drive-(\S+)/){
3236                 $devices->{$1} = 1;
3237         }
3238     }
3239
3240     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3241     foreach my $mice (@$resmice) {
3242         if ($mice->{name} eq 'QEMU HID Tablet') {
3243             $devices->{tablet} = 1;
3244             last;
3245         }
3246     }
3247
3248     return $devices;
3249 }
3250
3251 sub vm_deviceplug {
3252     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3253
3254     my $q35 = machine_type_is_q35($conf);
3255
3256     my $devices_list = vm_devices_list($vmid);
3257     return 1 if defined($devices_list->{$deviceid});
3258
3259     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3260
3261     if ($deviceid eq 'tablet') {
3262
3263         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3264
3265     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3266
3267         qemu_iothread_add($vmid, $deviceid, $device);
3268
3269         qemu_driveadd($storecfg, $vmid, $device);
3270         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3271
3272         qemu_deviceadd($vmid, $devicefull);
3273         eval { qemu_deviceaddverify($vmid, $deviceid); };
3274         if (my $err = $@) {
3275             eval { qemu_drivedel($vmid, $deviceid); };
3276             warn $@ if $@;
3277             die $err;
3278         }
3279
3280     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3281
3282
3283         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3284         my $pciaddr = print_pci_addr($deviceid);
3285         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3286
3287         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3288
3289         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3290             qemu_iothread_add($vmid, $deviceid, $device);
3291             $devicefull .= ",iothread=iothread-$deviceid";
3292         }
3293
3294         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3295             $devicefull .= ",num_queues=$device->{queues}";
3296         }
3297
3298         qemu_deviceadd($vmid, $devicefull);
3299         qemu_deviceaddverify($vmid, $deviceid);
3300
3301     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3302
3303         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3304         qemu_driveadd($storecfg, $vmid, $device);
3305
3306         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3307         eval { qemu_deviceadd($vmid, $devicefull); };
3308         if (my $err = $@) {
3309             eval { qemu_drivedel($vmid, $deviceid); };
3310             warn $@ if $@;
3311             die $err;
3312         }
3313
3314     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3315
3316         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3317         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid);
3318         qemu_deviceadd($vmid, $netdevicefull);
3319         eval { qemu_deviceaddverify($vmid, $deviceid); };
3320         if (my $err = $@) {
3321             eval { qemu_netdevdel($vmid, $deviceid); };
3322             warn $@ if $@;
3323             die $err;
3324         }
3325
3326     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3327
3328         my $bridgeid = $2;
3329         my $pciaddr = print_pci_addr($deviceid);
3330         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3331
3332         qemu_deviceadd($vmid, $devicefull);
3333         qemu_deviceaddverify($vmid, $deviceid);
3334
3335     } else {
3336         die "can't hotplug device '$deviceid'\n";
3337     }
3338
3339     return 1;
3340 }
3341
3342 # fixme: this should raise exceptions on error!
3343 sub vm_deviceunplug {
3344     my ($vmid, $conf, $deviceid) = @_;
3345
3346     my $devices_list = vm_devices_list($vmid);
3347     return 1 if !defined($devices_list->{$deviceid});
3348
3349     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3350
3351     if ($deviceid eq 'tablet') {
3352
3353         qemu_devicedel($vmid, $deviceid);
3354
3355     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3356
3357         qemu_devicedel($vmid, $deviceid);
3358         qemu_devicedelverify($vmid, $deviceid);
3359         qemu_drivedel($vmid, $deviceid);
3360         qemu_iothread_del($conf, $vmid, $deviceid);
3361
3362     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3363
3364         qemu_devicedel($vmid, $deviceid);
3365         qemu_devicedelverify($vmid, $deviceid);
3366         qemu_iothread_del($conf, $vmid, $deviceid);
3367
3368     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3369
3370         #qemu 2.3 segfault on drive_del with virtioscsi + iothread
3371         my $device = parse_drive($deviceid, $conf->{$deviceid});
3372         die "virtioscsi with iothread is not hot-unplugglable currently" if $device->{iothread};
3373
3374         qemu_devicedel($vmid, $deviceid);
3375         qemu_drivedel($vmid, $deviceid);
3376         qemu_deletescsihw($conf, $vmid, $deviceid);
3377
3378     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3379
3380         qemu_devicedel($vmid, $deviceid);
3381         qemu_devicedelverify($vmid, $deviceid);
3382         qemu_netdevdel($vmid, $deviceid);
3383
3384     } else {
3385         die "can't unplug device '$deviceid'\n";
3386     }
3387
3388     return 1;
3389 }
3390
3391 sub qemu_deviceadd {
3392     my ($vmid, $devicefull) = @_;
3393
3394     $devicefull = "driver=".$devicefull;
3395     my %options =  split(/[=,]/, $devicefull);
3396
3397     vm_mon_cmd($vmid, "device_add" , %options);
3398 }
3399
3400 sub qemu_devicedel {
3401     my ($vmid, $deviceid) = @_;
3402
3403     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3404 }
3405
3406 sub qemu_iothread_add {
3407     my($vmid, $deviceid, $device) = @_;
3408
3409     if ($device->{iothread}) {
3410         my $iothreads = vm_iothreads_list($vmid);
3411         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3412     }
3413 }
3414
3415 sub qemu_iothread_del {
3416     my($conf, $vmid, $deviceid) = @_;
3417
3418     my $device = parse_drive($deviceid, $conf->{$deviceid});
3419     if ($device->{iothread}) {
3420         my $iothreads = vm_iothreads_list($vmid);
3421         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3422     }
3423 }
3424
3425 sub qemu_objectadd {
3426     my($vmid, $objectid, $qomtype) = @_;
3427
3428     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3429
3430     return 1;
3431 }
3432
3433 sub qemu_objectdel {
3434     my($vmid, $objectid) = @_;
3435
3436     vm_mon_cmd($vmid, "object-del", id => $objectid);
3437
3438     return 1;
3439 }
3440
3441 sub qemu_driveadd {
3442     my ($storecfg, $vmid, $device) = @_;
3443
3444     my $drive = print_drive_full($storecfg, $vmid, $device);
3445     $drive =~ s/\\/\\\\/g;
3446     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3447
3448     # If the command succeeds qemu prints: "OK"
3449     return 1 if $ret =~ m/OK/s;
3450
3451     die "adding drive failed: $ret\n";
3452 }
3453
3454 sub qemu_drivedel {
3455     my($vmid, $deviceid) = @_;
3456
3457     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3458     $ret =~ s/^\s+//;
3459
3460     return 1 if $ret eq "";
3461
3462     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3463     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3464
3465     die "deleting drive $deviceid failed : $ret\n";
3466 }
3467
3468 sub qemu_deviceaddverify {
3469     my ($vmid, $deviceid) = @_;
3470
3471     for (my $i = 0; $i <= 5; $i++) {
3472          my $devices_list = vm_devices_list($vmid);
3473          return 1 if defined($devices_list->{$deviceid});
3474          sleep 1;
3475     }
3476
3477     die "error on hotplug device '$deviceid'\n";
3478 }
3479
3480
3481 sub qemu_devicedelverify {
3482     my ($vmid, $deviceid) = @_;
3483
3484     # need to verify that the device is correctly removed as device_del
3485     # is async and empty return is not reliable
3486
3487     for (my $i = 0; $i <= 5; $i++) {
3488          my $devices_list = vm_devices_list($vmid);
3489          return 1 if !defined($devices_list->{$deviceid});
3490          sleep 1;
3491     }
3492
3493     die "error on hot-unplugging device '$deviceid'\n";
3494 }
3495
3496 sub qemu_findorcreatescsihw {
3497     my ($storecfg, $conf, $vmid, $device) = @_;
3498
3499     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3500
3501     my $scsihwid="$controller_prefix$controller";
3502     my $devices_list = vm_devices_list($vmid);
3503
3504     if(!defined($devices_list->{$scsihwid})) {
3505         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3506     }
3507
3508     return 1;
3509 }
3510
3511 sub qemu_deletescsihw {
3512     my ($conf, $vmid, $opt) = @_;
3513
3514     my $device = parse_drive($opt, $conf->{$opt});
3515
3516     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3517         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3518         return 1;
3519     }
3520
3521     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3522
3523     my $devices_list = vm_devices_list($vmid);
3524     foreach my $opt (keys %{$devices_list}) {
3525         if (PVE::QemuServer::valid_drivename($opt)) {
3526             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3527             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3528                 return 1;
3529             }
3530         }
3531     }
3532
3533     my $scsihwid="scsihw$controller";
3534
3535     vm_deviceunplug($vmid, $conf, $scsihwid);
3536
3537     return 1;
3538 }
3539
3540 sub qemu_add_pci_bridge {
3541     my ($storecfg, $conf, $vmid, $device) = @_;
3542
3543     my $bridges = {};
3544
3545     my $bridgeid;
3546
3547     print_pci_addr($device, $bridges);
3548
3549     while (my ($k, $v) = each %$bridges) {
3550         $bridgeid = $k;
3551     }
3552     return 1 if !defined($bridgeid) || $bridgeid < 1;
3553
3554     my $bridge = "pci.$bridgeid";
3555     my $devices_list = vm_devices_list($vmid);
3556
3557     if (!defined($devices_list->{$bridge})) {
3558         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3559     }
3560
3561     return 1;
3562 }
3563
3564 sub qemu_set_link_status {
3565     my ($vmid, $device, $up) = @_;
3566
3567     vm_mon_cmd($vmid, "set_link", name => $device,
3568                up => $up ? JSON::true : JSON::false);
3569 }
3570
3571 sub qemu_netdevadd {
3572     my ($vmid, $conf, $device, $deviceid) = @_;
3573
3574     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid);
3575     my %options =  split(/[=,]/, $netdev);
3576
3577     vm_mon_cmd($vmid, "netdev_add",  %options);
3578     return 1;
3579 }
3580
3581 sub qemu_netdevdel {
3582     my ($vmid, $deviceid) = @_;
3583
3584     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3585 }
3586
3587 sub qemu_cpu_hotplug {
3588     my ($vmid, $conf, $vcpus) = @_;
3589
3590     my $sockets = 1;
3591     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3592     $sockets = $conf->{sockets} if  $conf->{sockets};
3593     my $cores = $conf->{cores} || 1;
3594     my $maxcpus = $sockets * $cores;
3595
3596     $vcpus = $maxcpus if !$vcpus;
3597
3598     die "you can't add more vcpus than maxcpus\n"
3599         if $vcpus > $maxcpus;
3600
3601     my $currentvcpus = $conf->{vcpus} || $maxcpus;
3602     die "online cpu unplug is not yet possible\n"
3603         if $vcpus < $currentvcpus;
3604
3605     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3606     die "vcpus in running vm is different than configuration\n"
3607         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
3608
3609     for (my $i = $currentvcpus; $i < $vcpus; $i++) {
3610         vm_mon_cmd($vmid, "cpu-add", id => int($i));
3611     }
3612 }
3613
3614 sub qemu_memory_hotplug {
3615     my ($vmid, $conf, $defaults, $opt, $value) = @_;
3616
3617     return $value if !check_running($vmid);
3618
3619     my $memory = $conf->{memory} || $defaults->{memory};
3620     $value = $defaults->{memory} if !$value;
3621     return $value if $value == $memory;
3622
3623     my $static_memory = $STATICMEM;
3624     my $dimm_memory = $memory - $static_memory;
3625
3626     die "memory can't be lower than $static_memory MB" if $value < $static_memory;
3627     die "you cannot add more memory than $MAX_MEM MB!\n" if $memory > $MAX_MEM;
3628
3629
3630     my $sockets = 1;
3631     $sockets = $conf->{sockets} if $conf->{sockets};
3632
3633     if($value > $memory) {
3634
3635         foreach_dimm($conf, $vmid, $value, $sockets, sub {
3636             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3637
3638                 return if $current_size <= $conf->{memory};
3639
3640                 eval { vm_mon_cmd($vmid, "object-add", 'qom-type' => "memory-backend-ram", id => "mem-$name", props => { size => int($dimm_size*1024*1024) } ) };
3641                 if (my $err = $@) {
3642                     eval { qemu_objectdel($vmid, "mem-$name"); };
3643                     die $err;
3644                 }
3645
3646                 eval { vm_mon_cmd($vmid, "device_add", driver => "pc-dimm", id => "$name", memdev => "mem-$name", node => $numanode) };
3647                 if (my $err = $@) {
3648                     eval { qemu_objectdel($vmid, "mem-$name"); };
3649                     die $err;
3650                 }
3651                 #update conf after each succesful module hotplug
3652                 $conf->{memory} = $current_size;
3653                 update_config_nolock($vmid, $conf, 1);
3654         });
3655
3656     } else {
3657
3658         foreach_reverse_dimm($conf, $vmid, $value, $sockets, sub {
3659             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3660
3661                 return if $current_size >= $conf->{memory};
3662                 print "try to unplug memory dimm $name\n";
3663
3664                 my $retry = 0;
3665                 while (1) {
3666                     eval { qemu_devicedel($vmid, $name) };
3667                     sleep 3;
3668                     my $dimm_list = qemu_dimm_list($vmid);
3669                     last if !$dimm_list->{$name};
3670                     raise_param_exc({ $name => "error unplug memory module" }) if $retry > 5;
3671                     $retry++;
3672                 }
3673
3674                 #update conf after each succesful module unplug
3675                 $conf->{memory} = $current_size;
3676
3677                 eval { qemu_objectdel($vmid, "mem-$name"); };
3678                 update_config_nolock($vmid, $conf, 1);
3679         });
3680     }
3681 }
3682
3683 sub qemu_dimm_list {
3684     my ($vmid) = @_;
3685
3686     my $dimmarray = vm_mon_cmd_nocheck($vmid, "query-memory-devices");
3687     my $dimms = {};
3688
3689     foreach my $dimm (@$dimmarray) {
3690
3691         $dimms->{$dimm->{data}->{id}}->{id} = $dimm->{data}->{id};
3692         $dimms->{$dimm->{data}->{id}}->{node} = $dimm->{data}->{node};
3693         $dimms->{$dimm->{data}->{id}}->{addr} = $dimm->{data}->{addr};
3694         $dimms->{$dimm->{data}->{id}}->{size} = $dimm->{data}->{size};
3695         $dimms->{$dimm->{data}->{id}}->{slot} = $dimm->{data}->{slot};
3696     }
3697     return $dimms;
3698 }
3699
3700 sub qemu_block_set_io_throttle {
3701     my ($vmid, $deviceid, $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr) = @_;
3702
3703     return if !check_running($vmid) ;
3704
3705     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid, bps => int($bps), bps_rd => int($bps_rd), bps_wr => int($bps_wr), iops => int($iops), iops_rd => int($iops_rd), iops_wr => int($iops_wr));
3706
3707 }
3708
3709 # old code, only used to shutdown old VM after update
3710 sub __read_avail {
3711     my ($fh, $timeout) = @_;
3712
3713     my $sel = new IO::Select;
3714     $sel->add($fh);
3715
3716     my $res = '';
3717     my $buf;
3718
3719     my @ready;
3720     while (scalar (@ready = $sel->can_read($timeout))) {
3721         my $count;
3722         if ($count = $fh->sysread($buf, 8192)) {
3723             if ($buf =~ /^(.*)\(qemu\) $/s) {
3724                 $res .= $1;
3725                 last;
3726             } else {
3727                 $res .= $buf;
3728             }
3729         } else {
3730             if (!defined($count)) {
3731                 die "$!\n";
3732             }
3733             last;
3734         }
3735     }
3736
3737     die "monitor read timeout\n" if !scalar(@ready);
3738
3739     return $res;
3740 }
3741
3742 # old code, only used to shutdown old VM after update
3743 sub vm_monitor_command {
3744     my ($vmid, $cmdstr, $nocheck) = @_;
3745
3746     my $res;
3747
3748     eval {
3749         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
3750
3751         my $sname = "${var_run_tmpdir}/$vmid.mon";
3752
3753         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
3754             die "unable to connect to VM $vmid socket - $!\n";
3755
3756         my $timeout = 3;
3757
3758         # hack: migrate sometime blocks the monitor (when migrate_downtime
3759         # is set)
3760         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3761             $timeout = 60*60; # 1 hour
3762         }
3763
3764         # read banner;
3765         my $data = __read_avail($sock, $timeout);
3766
3767         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
3768             die "got unexpected qemu monitor banner\n";
3769         }
3770
3771         my $sel = new IO::Select;
3772         $sel->add($sock);
3773
3774         if (!scalar(my @ready = $sel->can_write($timeout))) {
3775             die "monitor write error - timeout";
3776         }
3777
3778         my $fullcmd = "$cmdstr\r";
3779
3780         # syslog('info', "VM $vmid monitor command: $cmdstr");
3781
3782         my $b;
3783         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
3784             die "monitor write error - $!";
3785         }
3786
3787         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
3788
3789         $timeout = 20;
3790
3791         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3792             $timeout = 60*60; # 1 hour
3793         } elsif ($cmdstr =~ m/^(eject|change)/) {
3794             $timeout = 60; # note: cdrom mount command is slow
3795         }
3796         if ($res = __read_avail($sock, $timeout)) {
3797
3798             my @lines = split("\r?\n", $res);
3799
3800             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
3801
3802             $res = join("\n", @lines);
3803             $res .= "\n";
3804         }
3805     };
3806
3807     my $err = $@;
3808
3809     if ($err) {
3810         syslog("err", "VM $vmid monitor command failed - $err");
3811         die $err;
3812     }
3813
3814     return $res;
3815 }
3816
3817 sub qemu_block_resize {
3818     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
3819
3820     my $running = check_running($vmid);
3821
3822     return if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
3823
3824     return if !$running;
3825
3826     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
3827
3828 }
3829
3830 sub qemu_volume_snapshot {
3831     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3832
3833     my $running = check_running($vmid);
3834
3835     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
3836         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
3837     } else {
3838         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
3839     }
3840 }
3841
3842 sub qemu_volume_snapshot_delete {
3843     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3844
3845     my $running = check_running($vmid);
3846
3847     return if !PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
3848
3849     return if !$running;
3850
3851     vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
3852 }
3853
3854 sub set_migration_caps {
3855     my ($vmid) = @_;
3856
3857     my $cap_ref = [];
3858
3859     my $enabled_cap = {
3860         "auto-converge" => 1,
3861         "xbzrle" => 0,
3862         "x-rdma-pin-all" => 0,
3863         "zero-blocks" => 0,
3864     };
3865
3866     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
3867
3868     for my $supported_capability (@$supported_capabilities) {
3869         push @$cap_ref, {
3870             capability => $supported_capability->{capability},
3871             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
3872         };
3873     }
3874
3875     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
3876 }
3877
3878 my $fast_plug_option = {
3879     'lock' => 1,
3880     'name' => 1,
3881     'onboot' => 1,
3882     'shares' => 1,
3883     'startup' => 1,
3884     'description' => 1,
3885 };
3886
3887 # hotplug changes in [PENDING]
3888 # $selection hash can be used to only apply specified options, for
3889 # example: { cores => 1 } (only apply changed 'cores')
3890 # $errors ref is used to return error messages
3891 sub vmconfig_hotplug_pending {
3892     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
3893
3894     my $defaults = load_defaults();
3895
3896     # commit values which do not have any impact on running VM first
3897     # Note: those option cannot raise errors, we we do not care about
3898     # $selection and always apply them.
3899
3900     my $add_error = sub {
3901         my ($opt, $msg) = @_;
3902         $errors->{$opt} = "hotplug problem - $msg";
3903     };
3904
3905     my $changes = 0;
3906     foreach my $opt (keys %{$conf->{pending}}) { # add/change
3907         if ($fast_plug_option->{$opt}) {
3908             $conf->{$opt} = $conf->{pending}->{$opt};
3909             delete $conf->{pending}->{$opt};
3910             $changes = 1;
3911         }
3912     }
3913
3914     if ($changes) {
3915         update_config_nolock($vmid, $conf, 1);
3916         $conf = load_config($vmid); # update/reload
3917     }
3918
3919     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3920
3921     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
3922     while (my ($opt, $force) = each %$pending_delete_hash) {
3923         next if $selection && !$selection->{$opt};
3924         eval {
3925             if ($opt eq 'hotplug') {
3926                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
3927             } elsif ($opt eq 'tablet') {
3928                 die "skip\n" if !$hotplug_features->{usb};
3929                 if ($defaults->{tablet}) {
3930                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
3931                 } else {
3932                     vm_deviceunplug($vmid, $conf, $opt);
3933                 }
3934             } elsif ($opt eq 'vcpus') {
3935                 die "skip\n" if !$hotplug_features->{cpu};
3936                 qemu_cpu_hotplug($vmid, $conf, undef);
3937             } elsif ($opt eq 'balloon') {
3938                 # enable balloon device is not hotpluggable
3939                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
3940             } elsif ($fast_plug_option->{$opt}) {
3941                 # do nothing
3942             } elsif ($opt =~ m/^net(\d+)$/) {
3943                 die "skip\n" if !$hotplug_features->{network};
3944                 vm_deviceunplug($vmid, $conf, $opt);
3945             } elsif (valid_drivename($opt)) {
3946                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
3947                 vm_deviceunplug($vmid, $conf, $opt);
3948                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
3949             } elsif ($opt =~ m/^memory$/) {
3950                 die "skip\n" if !$hotplug_features->{memory};
3951                 qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
3952             } elsif ($opt eq 'cpuunits') {
3953                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
3954             } elsif ($opt eq 'cpulimit') {
3955                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
3956             } else {
3957                 die "skip\n";
3958             }
3959         };
3960         if (my $err = $@) {
3961             &$add_error($opt, $err) if $err ne "skip\n";
3962         } else {
3963             # save new config if hotplug was successful
3964             delete $conf->{$opt};
3965             vmconfig_undelete_pending_option($conf, $opt);
3966             update_config_nolock($vmid, $conf, 1);
3967             $conf = load_config($vmid); # update/reload
3968         }
3969     }
3970
3971     foreach my $opt (keys %{$conf->{pending}}) {
3972         next if $selection && !$selection->{$opt};
3973         my $value = $conf->{pending}->{$opt};
3974         eval {
3975             if ($opt eq 'hotplug') {
3976                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
3977             } elsif ($opt eq 'tablet') {
3978                 die "skip\n" if !$hotplug_features->{usb};
3979                 if ($value == 1) {
3980                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
3981                 } elsif ($value == 0) {
3982                     vm_deviceunplug($vmid, $conf, $opt);
3983                 }
3984             } elsif ($opt eq 'vcpus') {
3985                 die "skip\n" if !$hotplug_features->{cpu};
3986                 qemu_cpu_hotplug($vmid, $conf, $value);
3987             } elsif ($opt eq 'balloon') {
3988                 # enable/disable balloning device is not hotpluggable
3989                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
3990                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
3991                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
3992
3993                 # allow manual ballooning if shares is set to zero
3994                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
3995                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
3996                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
3997                 }
3998             } elsif ($opt =~ m/^net(\d+)$/) {
3999                 # some changes can be done without hotplug
4000                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4001                                     $vmid, $opt, $value);
4002             } elsif (valid_drivename($opt)) {
4003                 # some changes can be done without hotplug
4004                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4005                                      $vmid, $opt, $value, 1);
4006             } elsif ($opt =~ m/^memory$/) { #dimms
4007                 die "skip\n" if !$hotplug_features->{memory};
4008                 $value = qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4009             } elsif ($opt eq 'cpuunits') {
4010                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4011             } elsif ($opt eq 'cpulimit') {
4012                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4013                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4014             } else {
4015                 die "skip\n";  # skip non-hot-pluggable options
4016             }
4017         };
4018         if (my $err = $@) {
4019             &$add_error($opt, $err) if $err ne "skip\n";
4020         } else {
4021             # save new config if hotplug was successful
4022             $conf->{$opt} = $value;
4023             delete $conf->{pending}->{$opt};
4024             update_config_nolock($vmid, $conf, 1);
4025             $conf = load_config($vmid); # update/reload
4026         }
4027     }
4028 }
4029
4030 sub try_deallocate_drive {
4031     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4032
4033     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4034         my $volid = $drive->{file};
4035         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4036             my $sid = PVE::Storage::parse_volume_id($volid);
4037             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4038
4039             # check if the disk is really unused
4040             my $used_paths = PVE::QemuServer::get_used_paths($vmid, $storecfg, $conf, 1, $key);
4041             my $path = PVE::Storage::path($storecfg, $volid);
4042             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4043                    if $used_paths->{$path};
4044             PVE::Storage::vdisk_free($storecfg, $volid);
4045             return 1;
4046         } else {
4047             # If vm is not owner of this disk remove from config
4048             return 1;
4049         }
4050     }
4051
4052     return undef;
4053 }
4054
4055 sub vmconfig_delete_or_detach_drive {
4056     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4057
4058     my $drive = parse_drive($opt, $conf->{$opt});
4059
4060     my $rpcenv = PVE::RPCEnvironment::get();
4061     my $authuser = $rpcenv->get_user();
4062
4063     if ($force) {
4064         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4065         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4066     } else {
4067         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4068     }
4069 }
4070
4071 sub vmconfig_apply_pending {
4072     my ($vmid, $conf, $storecfg) = @_;
4073
4074     # cold plug
4075
4076     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4077     while (my ($opt, $force) = each %$pending_delete_hash) {
4078         die "internal error" if $opt =~ m/^unused/;
4079         $conf = load_config($vmid); # update/reload
4080         if (!defined($conf->{$opt})) {
4081             vmconfig_undelete_pending_option($conf, $opt);
4082             update_config_nolock($vmid, $conf, 1);
4083         } elsif (valid_drivename($opt)) {
4084             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4085             vmconfig_undelete_pending_option($conf, $opt);
4086             delete $conf->{$opt};
4087             update_config_nolock($vmid, $conf, 1);
4088         } else {
4089             vmconfig_undelete_pending_option($conf, $opt);
4090             delete $conf->{$opt};
4091             update_config_nolock($vmid, $conf, 1);
4092         }
4093     }
4094
4095     $conf = load_config($vmid); # update/reload
4096
4097     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4098         $conf = load_config($vmid); # update/reload
4099
4100         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4101             # skip if nothing changed
4102         } elsif (valid_drivename($opt)) {
4103             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4104                 if defined($conf->{$opt});
4105             $conf->{$opt} = $conf->{pending}->{$opt};
4106         } else {
4107             $conf->{$opt} = $conf->{pending}->{$opt};
4108         }
4109
4110         delete $conf->{pending}->{$opt};
4111         update_config_nolock($vmid, $conf, 1);
4112     }
4113 }
4114
4115 my $safe_num_ne = sub {
4116     my ($a, $b) = @_;
4117
4118     return 0 if !defined($a) && !defined($b);
4119     return 1 if !defined($a);
4120     return 1 if !defined($b);
4121
4122     return $a != $b;
4123 };
4124
4125 my $safe_string_ne = sub {
4126     my ($a, $b) = @_;
4127
4128     return 0 if !defined($a) && !defined($b);
4129     return 1 if !defined($a);
4130     return 1 if !defined($b);
4131
4132     return $a ne $b;
4133 };
4134
4135 sub vmconfig_update_net {
4136     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4137
4138     my $newnet = parse_net($value);
4139
4140     if ($conf->{$opt}) {
4141         my $oldnet = parse_net($conf->{$opt});
4142
4143         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4144             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4145             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4146             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4147
4148             # for non online change, we try to hot-unplug
4149             die "skip\n" if !$hotplug;
4150             vm_deviceunplug($vmid, $conf, $opt);
4151         } else {
4152
4153             die "internal error" if $opt !~ m/net(\d+)/;
4154             my $iface = "tap${vmid}i$1";
4155
4156             if (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4157                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4158             }
4159
4160             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4161                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4162                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4163                 PVE::Network::tap_unplug($iface);
4164                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall});
4165             }
4166
4167             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4168                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4169             }
4170
4171             return 1;
4172         }
4173     }
4174
4175     if ($hotplug) {
4176         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4177     } else {
4178         die "skip\n";
4179     }
4180 }
4181
4182 sub vmconfig_update_disk {
4183     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4184
4185     # fixme: do we need force?
4186
4187     my $drive = parse_drive($opt, $value);
4188
4189     if ($conf->{$opt}) {
4190
4191         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4192
4193             my $media = $drive->{media} || 'disk';
4194             my $oldmedia = $old_drive->{media} || 'disk';
4195             die "unable to change media type\n" if $media ne $oldmedia;
4196
4197             if (!drive_is_cdrom($old_drive)) {
4198
4199                 if ($drive->{file} ne $old_drive->{file}) {
4200
4201                     die "skip\n" if !$hotplug;
4202
4203                     # unplug and register as unused
4204                     vm_deviceunplug($vmid, $conf, $opt);
4205                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4206
4207                 } else {
4208                     # update existing disk
4209
4210                     # skip non hotpluggable value
4211                     if (&$safe_num_ne($drive->{discard}, $old_drive->{discard}) ||
4212                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4213                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4214                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4215                         die "skip\n";
4216                     }
4217
4218                     # apply throttle
4219                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4220                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4221                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4222                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4223                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4224                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4225                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4226                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4227                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4228                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4229                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4230                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max})) {
4231
4232                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4233                                                    ($drive->{mbps} || 0)*1024*1024,
4234                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4235                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4236                                                    $drive->{iops} || 0,
4237                                                    $drive->{iops_rd} || 0,
4238                                                    $drive->{iops_wr} || 0,
4239                                                    ($drive->{mbps_max} || 0)*1024*1024,
4240                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4241                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4242                                                    $drive->{iops_max} || 0,
4243                                                    $drive->{iops_rd_max} || 0,
4244                                                    $drive->{iops_wr_max} || 0);
4245
4246                     }
4247
4248                     return 1;
4249                 }
4250
4251             } else { # cdrom
4252
4253                 if ($drive->{file} eq 'none') {
4254                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4255                 } else {
4256                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4257                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4258                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4259                 }
4260
4261                 return 1;
4262             }
4263         }
4264     }
4265
4266     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4267     # hotplug new disks
4268     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4269 }
4270
4271 sub vm_start {
4272     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused, $forcemachine, $spice_ticket) = @_;
4273
4274     lock_config($vmid, sub {
4275         my $conf = load_config($vmid, $migratedfrom);
4276
4277         die "you can't start a vm if it's a template\n" if is_template($conf);
4278
4279         check_lock($conf) if !$skiplock;
4280
4281         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4282
4283         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4284             vmconfig_apply_pending($vmid, $conf, $storecfg);
4285             $conf = load_config($vmid); # update/reload
4286         }
4287
4288         my $defaults = load_defaults();
4289
4290         # set environment variable useful inside network script
4291         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4292
4293         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4294
4295         my $migrate_port = 0;
4296         my $migrate_uri;
4297         if ($statefile) {
4298             if ($statefile eq 'tcp') {
4299                 my $localip = "localhost";
4300                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4301                 my $nodename = PVE::INotify::nodename();
4302                 if ($datacenterconf->{migration_unsecure}) {
4303                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4304                 }
4305                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4306                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4307                 $migrate_uri = "tcp:[${localip}]:${migrate_port}";
4308                 push @$cmd, '-incoming', $migrate_uri;
4309                 push @$cmd, '-S';
4310             } else {
4311                 push @$cmd, '-loadstate', $statefile;
4312             }
4313         } elsif ($paused) {
4314             push @$cmd, '-S';
4315         }
4316
4317         # host pci devices
4318         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4319           my $d = parse_hostpci($conf->{"hostpci$i"});
4320           next if !$d;
4321           my $pcidevices = $d->{pciid};
4322           foreach my $pcidevice (@$pcidevices) {
4323                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4324
4325                 my $info = pci_device_info("0000:$pciid");
4326                 die "IOMMU not present\n" if !check_iommu_support();
4327                 die "no pci device info for device '$pciid'\n" if !$info;
4328
4329                 if ($d->{driver} && $d->{driver} eq "vfio") {
4330                     die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4331                 } else {
4332                     die "can't unbind/bind to stub pci device '$pciid'\n" if !pci_dev_bind_to_stub($info);
4333                 }
4334
4335                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4336           }
4337         }
4338
4339         PVE::Storage::activate_volumes($storecfg, $vollist);
4340
4341         eval  { run_command($cmd, timeout => $statefile ? undef : 30,
4342                     umask => 0077); };
4343         my $err = $@;
4344         die "start failed: $err" if $err;
4345
4346         print "migration listens on $migrate_uri\n" if $migrate_uri;
4347
4348         if ($statefile && $statefile ne 'tcp')  {
4349             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4350             warn $@ if $@;
4351         }
4352
4353         if ($migratedfrom) {
4354
4355             eval {
4356                 set_migration_caps($vmid);
4357             };
4358             warn $@ if $@;
4359
4360             if ($spice_port) {
4361                 print "spice listens on port $spice_port\n";
4362                 if ($spice_ticket) {
4363                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4364                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4365                 }
4366             }
4367
4368         } else {
4369
4370             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4371                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4372                     if $conf->{balloon};
4373             }
4374
4375             foreach my $opt (keys %$conf) {
4376                 next if $opt !~  m/^net\d+$/;
4377                 my $nicconf = parse_net($conf->{$opt});
4378                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4379             }
4380         }
4381
4382         vm_mon_cmd_nocheck($vmid, 'qom-set',
4383                     path => "machine/peripheral/balloon0",
4384                     property => "guest-stats-polling-interval",
4385                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4386
4387     });
4388 }
4389
4390 sub vm_mon_cmd {
4391     my ($vmid, $execute, %params) = @_;
4392
4393     my $cmd = { execute => $execute, arguments => \%params };
4394     vm_qmp_command($vmid, $cmd);
4395 }
4396
4397 sub vm_mon_cmd_nocheck {
4398     my ($vmid, $execute, %params) = @_;
4399
4400     my $cmd = { execute => $execute, arguments => \%params };
4401     vm_qmp_command($vmid, $cmd, 1);
4402 }
4403
4404 sub vm_qmp_command {
4405     my ($vmid, $cmd, $nocheck) = @_;
4406
4407     my $res;
4408
4409     my $timeout;
4410     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4411         $timeout = $cmd->{arguments}->{timeout};
4412         delete $cmd->{arguments}->{timeout};
4413     }
4414
4415     eval {
4416         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4417         my $sname = qmp_socket($vmid);
4418         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4419             my $qmpclient = PVE::QMPClient->new();
4420
4421             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4422         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
4423             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
4424                 if scalar(%{$cmd->{arguments}});
4425             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
4426         } else {
4427             die "unable to open monitor socket\n";
4428         }
4429     };
4430     if (my $err = $@) {
4431         syslog("err", "VM $vmid qmp command failed - $err");
4432         die $err;
4433     }
4434
4435     return $res;
4436 }
4437
4438 sub vm_human_monitor_command {
4439     my ($vmid, $cmdline) = @_;
4440
4441     my $res;
4442
4443     my $cmd = {
4444         execute => 'human-monitor-command',
4445         arguments => { 'command-line' => $cmdline},
4446     };
4447
4448     return vm_qmp_command($vmid, $cmd);
4449 }
4450
4451 sub vm_commandline {
4452     my ($storecfg, $vmid) = @_;
4453
4454     my $conf = load_config($vmid);
4455
4456     my $defaults = load_defaults();
4457
4458     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
4459
4460     return join(' ', @$cmd);
4461 }
4462
4463 sub vm_reset {
4464     my ($vmid, $skiplock) = @_;
4465
4466     lock_config($vmid, sub {
4467
4468         my $conf = load_config($vmid);
4469
4470         check_lock($conf) if !$skiplock;
4471
4472         vm_mon_cmd($vmid, "system_reset");
4473     });
4474 }
4475
4476 sub get_vm_volumes {
4477     my ($conf) = @_;
4478
4479     my $vollist = [];
4480     foreach_volid($conf, sub {
4481         my ($volid, $is_cdrom) = @_;
4482
4483         return if $volid =~ m|^/|;
4484
4485         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4486         return if !$sid;
4487
4488         push @$vollist, $volid;
4489     });
4490
4491     return $vollist;
4492 }
4493
4494 sub vm_stop_cleanup {
4495     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
4496
4497     eval {
4498
4499         if (!$keepActive) {
4500             my $vollist = get_vm_volumes($conf);
4501             PVE::Storage::deactivate_volumes($storecfg, $vollist);
4502         }
4503
4504         foreach my $ext (qw(mon qmp pid vnc qga)) {
4505             unlink "/var/run/qemu-server/${vmid}.$ext";
4506         }
4507
4508         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
4509     };
4510     warn $@ if $@; # avoid errors - just warn
4511 }
4512
4513 # Note: use $nockeck to skip tests if VM configuration file exists.
4514 # We need that when migration VMs to other nodes (files already moved)
4515 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
4516 sub vm_stop {
4517     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
4518
4519     $force = 1 if !defined($force) && !$shutdown;
4520
4521     if ($migratedfrom){
4522         my $pid = check_running($vmid, $nocheck, $migratedfrom);
4523         kill 15, $pid if $pid;
4524         my $conf = load_config($vmid, $migratedfrom);
4525         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
4526         return;
4527     }
4528
4529     lock_config($vmid, sub {
4530
4531         my $pid = check_running($vmid, $nocheck);
4532         return if !$pid;
4533
4534         my $conf;
4535         if (!$nocheck) {
4536             $conf = load_config($vmid);
4537             check_lock($conf) if !$skiplock;
4538             if (!defined($timeout) && $shutdown && $conf->{startup}) {
4539                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
4540                 $timeout = $opts->{down} if $opts->{down};
4541             }
4542         }
4543
4544         $timeout = 60 if !defined($timeout);
4545
4546         eval {
4547             if ($shutdown) {
4548                 if (defined($conf) && $conf->{agent}) {
4549                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
4550                 } else {
4551                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
4552                 }
4553             } else {
4554                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
4555             }
4556         };
4557         my $err = $@;
4558
4559         if (!$err) {
4560             my $count = 0;
4561             while (($count < $timeout) && check_running($vmid, $nocheck)) {
4562                 $count++;
4563                 sleep 1;
4564             }
4565
4566             if ($count >= $timeout) {
4567                 if ($force) {
4568                     warn "VM still running - terminating now with SIGTERM\n";
4569                     kill 15, $pid;
4570                 } else {
4571                     die "VM quit/powerdown failed - got timeout\n";
4572                 }
4573             } else {
4574                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4575                 return;
4576             }
4577         } else {
4578             if ($force) {
4579                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
4580                 kill 15, $pid;
4581             } else {
4582                 die "VM quit/powerdown failed\n";
4583             }
4584         }
4585
4586         # wait again
4587         $timeout = 10;
4588
4589         my $count = 0;
4590         while (($count < $timeout) && check_running($vmid, $nocheck)) {
4591             $count++;
4592             sleep 1;
4593         }
4594
4595         if ($count >= $timeout) {
4596             warn "VM still running - terminating now with SIGKILL\n";
4597             kill 9, $pid;
4598             sleep 1;
4599         }
4600
4601         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4602    });
4603 }
4604
4605 sub vm_suspend {
4606     my ($vmid, $skiplock) = @_;
4607
4608     lock_config($vmid, sub {
4609
4610         my $conf = load_config($vmid);
4611
4612         check_lock($conf) if !($skiplock || ($conf->{lock} && $conf->{lock} eq 'backup'));
4613
4614         vm_mon_cmd($vmid, "stop");
4615     });
4616 }
4617
4618 sub vm_resume {
4619     my ($vmid, $skiplock) = @_;
4620
4621     lock_config($vmid, sub {
4622
4623         my $conf = load_config($vmid);
4624
4625         check_lock($conf) if !($skiplock || ($conf->{lock} && $conf->{lock} eq 'backup'));
4626
4627         vm_mon_cmd($vmid, "cont");
4628     });
4629 }
4630
4631 sub vm_sendkey {
4632     my ($vmid, $skiplock, $key) = @_;
4633
4634     lock_config($vmid, sub {
4635
4636         my $conf = load_config($vmid);
4637
4638         # there is no qmp command, so we use the human monitor command
4639         vm_human_monitor_command($vmid, "sendkey $key");
4640     });
4641 }
4642
4643 sub vm_destroy {
4644     my ($storecfg, $vmid, $skiplock) = @_;
4645
4646     lock_config($vmid, sub {
4647
4648         my $conf = load_config($vmid);
4649
4650         check_lock($conf) if !$skiplock;
4651
4652         if (!check_running($vmid)) {
4653             destroy_vm($storecfg, $vmid);
4654         } else {
4655             die "VM $vmid is running - destroy failed\n";
4656         }
4657     });
4658 }
4659
4660 # pci helpers
4661
4662 sub file_write {
4663     my ($filename, $buf) = @_;
4664
4665     my $fh = IO::File->new($filename, "w");
4666     return undef if !$fh;
4667
4668     my $res = print $fh $buf;
4669
4670     $fh->close();
4671
4672     return $res;
4673 }
4674
4675 sub pci_device_info {
4676     my ($name) = @_;
4677
4678     my $res;
4679
4680     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
4681     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
4682
4683     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
4684     return undef if !defined($irq) || $irq !~ m/^\d+$/;
4685
4686     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
4687     return undef if !defined($vendor) || $vendor !~ s/^0x//;
4688
4689     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
4690     return undef if !defined($product) || $product !~ s/^0x//;
4691
4692     $res = {
4693         name => $name,
4694         vendor => $vendor,
4695         product => $product,
4696         domain => $domain,
4697         bus => $bus,
4698         slot => $slot,
4699         func => $func,
4700         irq => $irq,
4701         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
4702     };
4703
4704     return $res;
4705 }
4706
4707 sub pci_dev_reset {
4708     my ($dev) = @_;
4709
4710     my $name = $dev->{name};
4711
4712     my $fn = "$pcisysfs/devices/$name/reset";
4713
4714     return file_write($fn, "1");
4715 }
4716
4717 sub pci_dev_bind_to_stub {
4718     my ($dev) = @_;
4719
4720     my $name = $dev->{name};
4721
4722     my $testdir = "$pcisysfs/drivers/pci-stub/$name";
4723     return 1 if -d $testdir;
4724
4725     my $data = "$dev->{vendor} $dev->{product}";
4726     return undef if !file_write("$pcisysfs/drivers/pci-stub/new_id", $data);
4727
4728     my $fn = "$pcisysfs/devices/$name/driver/unbind";
4729     if (!file_write($fn, $name)) {
4730         return undef if -f $fn;
4731     }
4732
4733     $fn = "$pcisysfs/drivers/pci-stub/bind";
4734     if (! -d $testdir) {
4735         return undef if !file_write($fn, $name);
4736     }
4737
4738     return -d $testdir;
4739 }
4740
4741 sub pci_dev_bind_to_vfio {
4742     my ($dev) = @_;
4743
4744     my $name = $dev->{name};
4745
4746     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4747
4748     if (!-d $vfio_basedir) {
4749         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4750     }
4751     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4752
4753     my $testdir = "$vfio_basedir/$name";
4754     return 1 if -d $testdir;
4755
4756     my $data = "$dev->{vendor} $dev->{product}";
4757     return undef if !file_write("$vfio_basedir/new_id", $data);
4758
4759     my $fn = "$pcisysfs/devices/$name/driver/unbind";
4760     if (!file_write($fn, $name)) {
4761         return undef if -f $fn;
4762     }
4763
4764     $fn = "$vfio_basedir/bind";
4765     if (! -d $testdir) {
4766         return undef if !file_write($fn, $name);
4767     }
4768
4769     return -d $testdir;
4770 }
4771
4772 sub pci_dev_group_bind_to_vfio {
4773     my ($pciid) = @_;
4774
4775     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4776
4777     if (!-d $vfio_basedir) {
4778         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4779     }
4780     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4781
4782     # get IOMMU group devices
4783     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
4784       my @devs = grep /^0000:/, readdir($D);
4785     closedir($D);
4786
4787     foreach my $pciid (@devs) {
4788         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
4789
4790         # pci bridges, switches or root ports are not supported
4791         # they have a pci_bus subdirectory so skip them
4792         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
4793
4794         my $info = pci_device_info($1);
4795         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
4796     }
4797
4798     return 1;
4799 }
4800
4801 sub print_pci_addr {
4802     my ($id, $bridges) = @_;
4803
4804     my $res = '';
4805     my $devices = {
4806         piix3 => { bus => 0, addr => 1 },
4807         #addr2 : first videocard
4808         balloon0 => { bus => 0, addr => 3 },
4809         watchdog => { bus => 0, addr => 4 },
4810         scsihw0 => { bus => 0, addr => 5 },
4811         'pci.3' => { bus => 0, addr => 5 }, #can also be used for virtio-scsi-single bridge
4812         scsihw1 => { bus => 0, addr => 6 },
4813         ahci0 => { bus => 0, addr => 7 },
4814         qga0 => { bus => 0, addr => 8 },
4815         spice => { bus => 0, addr => 9 },
4816         virtio0 => { bus => 0, addr => 10 },
4817         virtio1 => { bus => 0, addr => 11 },
4818         virtio2 => { bus => 0, addr => 12 },
4819         virtio3 => { bus => 0, addr => 13 },
4820         virtio4 => { bus => 0, addr => 14 },
4821         virtio5 => { bus => 0, addr => 15 },
4822         hostpci0 => { bus => 0, addr => 16 },
4823         hostpci1 => { bus => 0, addr => 17 },
4824         net0 => { bus => 0, addr => 18 },
4825         net1 => { bus => 0, addr => 19 },
4826         net2 => { bus => 0, addr => 20 },
4827         net3 => { bus => 0, addr => 21 },
4828         net4 => { bus => 0, addr => 22 },
4829         net5 => { bus => 0, addr => 23 },
4830         vga1 => { bus => 0, addr => 24 },
4831         vga2 => { bus => 0, addr => 25 },
4832         vga3 => { bus => 0, addr => 26 },
4833         hostpci2 => { bus => 0, addr => 27 },
4834         hostpci3 => { bus => 0, addr => 28 },
4835         #addr29 : usb-host (pve-usb.cfg)
4836         'pci.1' => { bus => 0, addr => 30 },
4837         'pci.2' => { bus => 0, addr => 31 },
4838         'net6' => { bus => 1, addr => 1 },
4839         'net7' => { bus => 1, addr => 2 },
4840         'net8' => { bus => 1, addr => 3 },
4841         'net9' => { bus => 1, addr => 4 },
4842         'net10' => { bus => 1, addr => 5 },
4843         'net11' => { bus => 1, addr => 6 },
4844         'net12' => { bus => 1, addr => 7 },
4845         'net13' => { bus => 1, addr => 8 },
4846         'net14' => { bus => 1, addr => 9 },
4847         'net15' => { bus => 1, addr => 10 },
4848         'net16' => { bus => 1, addr => 11 },
4849         'net17' => { bus => 1, addr => 12 },
4850         'net18' => { bus => 1, addr => 13 },
4851         'net19' => { bus => 1, addr => 14 },
4852         'net20' => { bus => 1, addr => 15 },
4853         'net21' => { bus => 1, addr => 16 },
4854         'net22' => { bus => 1, addr => 17 },
4855         'net23' => { bus => 1, addr => 18 },
4856         'net24' => { bus => 1, addr => 19 },
4857         'net25' => { bus => 1, addr => 20 },
4858         'net26' => { bus => 1, addr => 21 },
4859         'net27' => { bus => 1, addr => 22 },
4860         'net28' => { bus => 1, addr => 23 },
4861         'net29' => { bus => 1, addr => 24 },
4862         'net30' => { bus => 1, addr => 25 },
4863         'net31' => { bus => 1, addr => 26 },
4864         'virtio6' => { bus => 2, addr => 1 },
4865         'virtio7' => { bus => 2, addr => 2 },
4866         'virtio8' => { bus => 2, addr => 3 },
4867         'virtio9' => { bus => 2, addr => 4 },
4868         'virtio10' => { bus => 2, addr => 5 },
4869         'virtio11' => { bus => 2, addr => 6 },
4870         'virtio12' => { bus => 2, addr => 7 },
4871         'virtio13' => { bus => 2, addr => 8 },
4872         'virtio14' => { bus => 2, addr => 9 },
4873         'virtio15' => { bus => 2, addr => 10 },
4874         'virtioscsi0' => { bus => 3, addr => 1 },
4875         'virtioscsi1' => { bus => 3, addr => 2 },
4876         'virtioscsi2' => { bus => 3, addr => 3 },
4877         'virtioscsi3' => { bus => 3, addr => 4 },
4878         'virtioscsi4' => { bus => 3, addr => 5 },
4879         'virtioscsi5' => { bus => 3, addr => 6 },
4880         'virtioscsi6' => { bus => 3, addr => 7 },
4881         'virtioscsi7' => { bus => 3, addr => 8 },
4882         'virtioscsi8' => { bus => 3, addr => 9 },
4883         'virtioscsi9' => { bus => 3, addr => 10 },
4884         'virtioscsi10' => { bus => 3, addr => 11 },
4885         'virtioscsi11' => { bus => 3, addr => 12 },
4886         'virtioscsi12' => { bus => 3, addr => 13 },
4887         'virtioscsi13' => { bus => 3, addr => 14 },
4888         'virtioscsi14' => { bus => 3, addr => 15 },
4889         'virtioscsi15' => { bus => 3, addr => 16 },
4890         'virtioscsi16' => { bus => 3, addr => 17 },
4891         'virtioscsi17' => { bus => 3, addr => 18 },
4892         'virtioscsi18' => { bus => 3, addr => 19 },
4893         'virtioscsi19' => { bus => 3, addr => 20 },
4894         'virtioscsi20' => { bus => 3, addr => 21 },
4895         'virtioscsi21' => { bus => 3, addr => 22 },
4896         'virtioscsi22' => { bus => 3, addr => 23 },
4897         'virtioscsi23' => { bus => 3, addr => 24 },
4898         'virtioscsi24' => { bus => 3, addr => 25 },
4899         'virtioscsi25' => { bus => 3, addr => 26 },
4900         'virtioscsi26' => { bus => 3, addr => 27 },
4901         'virtioscsi27' => { bus => 3, addr => 28 },
4902         'virtioscsi28' => { bus => 3, addr => 29 },
4903         'virtioscsi29' => { bus => 3, addr => 30 },
4904         'virtioscsi30' => { bus => 3, addr => 31 },
4905
4906     };
4907
4908     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
4909            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
4910            my $bus = $devices->{$id}->{bus};
4911            $res = ",bus=pci.$bus,addr=$addr";
4912            $bridges->{$bus} = 1 if $bridges;
4913     }
4914     return $res;
4915
4916 }
4917
4918 sub print_pcie_addr {
4919     my ($id) = @_;
4920
4921     my $res = '';
4922     my $devices = {
4923         hostpci0 => { bus => "ich9-pcie-port-1", addr => 0 },
4924         hostpci1 => { bus => "ich9-pcie-port-2", addr => 0 },
4925         hostpci2 => { bus => "ich9-pcie-port-3", addr => 0 },
4926         hostpci3 => { bus => "ich9-pcie-port-4", addr => 0 },
4927     };
4928
4929     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
4930            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
4931            my $bus = $devices->{$id}->{bus};
4932            $res = ",bus=$bus,addr=$addr";
4933     }
4934     return $res;
4935
4936 }
4937
4938 # vzdump restore implementaion
4939
4940 sub tar_archive_read_firstfile {
4941     my $archive = shift;
4942
4943     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
4944
4945     # try to detect archive type first
4946     my $pid = open (TMP, "tar tf '$archive'|") ||
4947         die "unable to open file '$archive'\n";
4948     my $firstfile = <TMP>;
4949     kill 15, $pid;
4950     close TMP;
4951
4952     die "ERROR: archive contaions no data\n" if !$firstfile;
4953     chomp $firstfile;
4954
4955     return $firstfile;
4956 }
4957
4958 sub tar_restore_cleanup {
4959     my ($storecfg, $statfile) = @_;
4960
4961     print STDERR "starting cleanup\n";
4962
4963     if (my $fd = IO::File->new($statfile, "r")) {
4964         while (defined(my $line = <$fd>)) {
4965             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
4966                 my $volid = $2;
4967                 eval {
4968                     if ($volid =~ m|^/|) {
4969                         unlink $volid || die 'unlink failed\n';
4970                     } else {
4971                         PVE::Storage::vdisk_free($storecfg, $volid);
4972                     }
4973                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
4974                 };
4975                 print STDERR "unable to cleanup '$volid' - $@" if $@;
4976             } else {
4977                 print STDERR "unable to parse line in statfile - $line";
4978             }
4979         }
4980         $fd->close();
4981     }
4982 }
4983
4984 sub restore_archive {
4985     my ($archive, $vmid, $user, $opts) = @_;
4986
4987     my $format = $opts->{format};
4988     my $comp;
4989
4990     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
4991         $format = 'tar' if !$format;
4992         $comp = 'gzip';
4993     } elsif ($archive =~ m/\.tar$/) {
4994         $format = 'tar' if !$format;
4995     } elsif ($archive =~ m/.tar.lzo$/) {
4996         $format = 'tar' if !$format;
4997         $comp = 'lzop';
4998     } elsif ($archive =~ m/\.vma$/) {
4999         $format = 'vma' if !$format;
5000     } elsif ($archive =~ m/\.vma\.gz$/) {
5001         $format = 'vma' if !$format;
5002         $comp = 'gzip';
5003     } elsif ($archive =~ m/\.vma\.lzo$/) {
5004         $format = 'vma' if !$format;
5005         $comp = 'lzop';
5006     } else {
5007         $format = 'vma' if !$format; # default
5008     }
5009
5010     # try to detect archive format
5011     if ($format eq 'tar') {
5012         return restore_tar_archive($archive, $vmid, $user, $opts);
5013     } else {
5014         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5015     }
5016 }
5017
5018 sub restore_update_config_line {
5019     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5020
5021     return if $line =~ m/^\#qmdump\#/;
5022     return if $line =~ m/^\#vzdump\#/;
5023     return if $line =~ m/^lock:/;
5024     return if $line =~ m/^unused\d+:/;
5025     return if $line =~ m/^parent:/;
5026     return if $line =~ m/^template:/; # restored VM is never a template
5027
5028     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5029         # try to convert old 1.X settings
5030         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5031         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5032             my ($model, $macaddr) = split(/\=/, $devconfig);
5033             $macaddr = PVE::Tools::random_ether_addr() if !$macaddr || $unique;
5034             my $net = {
5035                 model => $model,
5036                 bridge => "vmbr$ind",
5037                 macaddr => $macaddr,
5038             };
5039             my $netstr = print_net($net);
5040
5041             print $outfd "net$cookie->{netcount}: $netstr\n";
5042             $cookie->{netcount}++;
5043         }
5044     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5045         my ($id, $netstr) = ($1, $2);
5046         my $net = parse_net($netstr);
5047         $net->{macaddr} = PVE::Tools::random_ether_addr() if $net->{macaddr};
5048         $netstr = print_net($net);
5049         print $outfd "$id: $netstr\n";
5050     } elsif ($line =~ m/^((ide|scsi|virtio|sata)\d+):\s*(\S+)\s*$/) {
5051         my $virtdev = $1;
5052         my $value = $3;
5053         if ($line =~ m/backup=no/) {
5054             print $outfd "#$line";
5055         } elsif ($virtdev && $map->{$virtdev}) {
5056             my $di = parse_drive($virtdev, $value);
5057             delete $di->{format}; # format can change on restore
5058             $di->{file} = $map->{$virtdev};
5059             $value = print_drive($vmid, $di);
5060             print $outfd "$virtdev: $value\n";
5061         } else {
5062             print $outfd $line;
5063         }
5064     } else {
5065         print $outfd $line;
5066     }
5067 }
5068
5069 sub scan_volids {
5070     my ($cfg, $vmid) = @_;
5071
5072     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5073
5074     my $volid_hash = {};
5075     foreach my $storeid (keys %$info) {
5076         foreach my $item (@{$info->{$storeid}}) {
5077             next if !($item->{volid} && $item->{size});
5078             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5079             $volid_hash->{$item->{volid}} = $item;
5080         }
5081     }
5082
5083     return $volid_hash;
5084 }
5085
5086 sub get_used_paths {
5087     my ($vmid, $storecfg, $conf, $scan_snapshots, $skip_drive) = @_;
5088
5089     my $used_path = {};
5090
5091     my $scan_config = sub {
5092         my ($cref, $snapname) = @_;
5093
5094         foreach my $key (keys %$cref) {
5095             my $value = $cref->{$key};
5096             if (valid_drivename($key)) {
5097                 next if $skip_drive && $key eq $skip_drive;
5098                 my $drive = parse_drive($key, $value);
5099                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5100                 if ($drive->{file} =~ m!^/!) {
5101                     $used_path->{$drive->{file}}++; # = 1;
5102                 } else {
5103                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5104                     next if !$storeid;
5105                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5106                     next if !$scfg;
5107                     my $path = PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5108                     $used_path->{$path}++; # = 1;
5109                 }
5110             }
5111         }
5112     };
5113
5114     &$scan_config($conf);
5115
5116     undef $skip_drive;
5117
5118     if ($scan_snapshots) {
5119         foreach my $snapname (keys %{$conf->{snapshots}}) {
5120             &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5121         }
5122     }
5123
5124     return $used_path;
5125 }
5126
5127 sub update_disksize {
5128     my ($vmid, $conf, $volid_hash) = @_;
5129
5130     my $changes;
5131
5132     my $used = {};
5133
5134     # Note: it is allowed to define multiple storages with same path (alias), so
5135     # we need to check both 'volid' and real 'path' (two different volid can point
5136     # to the same path).
5137
5138     my $usedpath = {};
5139
5140     # update size info
5141     foreach my $opt (keys %$conf) {
5142         if (valid_drivename($opt)) {
5143             my $drive = parse_drive($opt, $conf->{$opt});
5144             my $volid = $drive->{file};
5145             next if !$volid;
5146
5147             $used->{$volid} = 1;
5148             if ($volid_hash->{$volid} &&
5149                 (my $path = $volid_hash->{$volid}->{path})) {
5150                 $usedpath->{$path} = 1;
5151             }
5152
5153             next if drive_is_cdrom($drive);
5154             next if !$volid_hash->{$volid};
5155
5156             $drive->{size} = $volid_hash->{$volid}->{size};
5157             my $new = print_drive($vmid, $drive);
5158             if ($new ne $conf->{$opt}) {
5159                 $changes = 1;
5160                 $conf->{$opt} = $new;
5161             }
5162         }
5163     }
5164
5165     # remove 'unusedX' entry if volume is used
5166     foreach my $opt (keys %$conf) {
5167         next if $opt !~ m/^unused\d+$/;
5168         my $volid = $conf->{$opt};
5169         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5170         if ($used->{$volid} || ($path && $usedpath->{$path})) {
5171             $changes = 1;
5172             delete $conf->{$opt};
5173         }
5174     }
5175
5176     foreach my $volid (sort keys %$volid_hash) {
5177         next if $volid =~ m/vm-$vmid-state-/;
5178         next if $used->{$volid};
5179         my $path = $volid_hash->{$volid}->{path};
5180         next if !$path; # just to be sure
5181         next if $usedpath->{$path};
5182         $changes = 1;
5183         add_unused_volume($conf, $volid);
5184         $usedpath->{$path} = 1; # avoid to add more than once (aliases)
5185     }
5186
5187     return $changes;
5188 }
5189
5190 sub rescan {
5191     my ($vmid, $nolock) = @_;
5192
5193     my $cfg = PVE::Cluster::cfs_read_file("storage.cfg");
5194
5195     my $volid_hash = scan_volids($cfg, $vmid);
5196
5197     my $updatefn =  sub {
5198         my ($vmid) = @_;
5199
5200         my $conf = load_config($vmid);
5201
5202         check_lock($conf);
5203
5204         my $vm_volids = {};
5205         foreach my $volid (keys %$volid_hash) {
5206             my $info = $volid_hash->{$volid};
5207             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5208         }
5209
5210         my $changes = update_disksize($vmid, $conf, $vm_volids);
5211
5212         update_config_nolock($vmid, $conf, 1) if $changes;
5213     };
5214
5215     if (defined($vmid)) {
5216         if ($nolock) {
5217             &$updatefn($vmid);
5218         } else {
5219             lock_config($vmid, $updatefn, $vmid);
5220         }
5221     } else {
5222         my $vmlist = config_list();
5223         foreach my $vmid (keys %$vmlist) {
5224             if ($nolock) {
5225                 &$updatefn($vmid);
5226             } else {
5227                 lock_config($vmid, $updatefn, $vmid);
5228             }
5229         }
5230     }
5231 }
5232
5233 sub restore_vma_archive {
5234     my ($archive, $vmid, $user, $opts, $comp) = @_;
5235
5236     my $input = $archive eq '-' ? "<&STDIN" : undef;
5237     my $readfrom = $archive;
5238
5239     my $uncomp = '';
5240     if ($comp) {
5241         $readfrom = '-';
5242         my $qarchive = PVE::Tools::shellquote($archive);
5243         if ($comp eq 'gzip') {
5244             $uncomp = "zcat $qarchive|";
5245         } elsif ($comp eq 'lzop') {
5246             $uncomp = "lzop -d -c $qarchive|";
5247         } else {
5248             die "unknown compression method '$comp'\n";
5249         }
5250
5251     }
5252
5253     my $tmpdir = "/var/tmp/vzdumptmp$$";
5254     rmtree $tmpdir;
5255
5256     # disable interrupts (always do cleanups)
5257     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5258         warn "got interrupt - ignored\n";
5259     };
5260
5261     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5262     POSIX::mkfifo($mapfifo, 0600);
5263     my $fifofh;
5264
5265     my $openfifo = sub {
5266         open($fifofh, '>', $mapfifo) || die $!;
5267     };
5268
5269     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5270
5271     my $oldtimeout;
5272     my $timeout = 5;
5273
5274     my $devinfo = {};
5275
5276     my $rpcenv = PVE::RPCEnvironment::get();
5277
5278     my $conffile = config_file($vmid);
5279     my $tmpfn = "$conffile.$$.tmp";
5280
5281     # Note: $oldconf is undef if VM does not exists
5282     my $oldconf = PVE::Cluster::cfs_read_file(cfs_config_path($vmid));
5283
5284     my $print_devmap = sub {
5285         my $virtdev_hash = {};
5286
5287         my $cfgfn = "$tmpdir/qemu-server.conf";
5288
5289         # we can read the config - that is already extracted
5290         my $fh = IO::File->new($cfgfn, "r") ||
5291             "unable to read qemu-server.conf - $!\n";
5292
5293         while (defined(my $line = <$fh>)) {
5294             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5295                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5296                 die "archive does not contain data for drive '$virtdev'\n"
5297                     if !$devinfo->{$devname};
5298                 if (defined($opts->{storage})) {
5299                     $storeid = $opts->{storage} || 'local';
5300                 } elsif (!$storeid) {
5301                     $storeid = 'local';
5302                 }
5303                 $format = 'raw' if !$format;
5304                 $devinfo->{$devname}->{devname} = $devname;
5305                 $devinfo->{$devname}->{virtdev} = $virtdev;
5306                 $devinfo->{$devname}->{format} = $format;
5307                 $devinfo->{$devname}->{storeid} = $storeid;
5308
5309                 # check permission on storage
5310                 my $pool = $opts->{pool}; # todo: do we need that?
5311                 if ($user ne 'root@pam') {
5312                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5313                 }
5314
5315                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5316             }
5317         }
5318
5319         foreach my $devname (keys %$devinfo) {
5320             die "found no device mapping information for device '$devname'\n"
5321                 if !$devinfo->{$devname}->{virtdev};
5322         }
5323
5324         my $cfg = cfs_read_file('storage.cfg');
5325
5326         # create empty/temp config
5327         if ($oldconf) {
5328             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5329             foreach_drive($oldconf, sub {
5330                 my ($ds, $drive) = @_;
5331
5332                 return if drive_is_cdrom($drive);
5333
5334                 my $volid = $drive->{file};
5335
5336                 return if !$volid || $volid =~ m|^/|;
5337
5338                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5339                 return if !$path || !$owner || ($owner != $vmid);
5340
5341                 # Note: only delete disk we want to restore
5342                 # other volumes will become unused
5343                 if ($virtdev_hash->{$ds}) {
5344                     PVE::Storage::vdisk_free($cfg, $volid);
5345                 }
5346             });
5347         }
5348
5349         my $map = {};
5350         foreach my $virtdev (sort keys %$virtdev_hash) {
5351             my $d = $virtdev_hash->{$virtdev};
5352             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5353             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5354
5355             # test if requested format is supported
5356             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5357             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5358             $d->{format} = $defFormat if !$supported;
5359
5360             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5361                                                   $d->{format}, undef, $alloc_size);
5362             print STDERR "new volume ID is '$volid'\n";
5363             $d->{volid} = $volid;
5364             my $path = PVE::Storage::path($cfg, $volid);
5365
5366             PVE::Storage::activate_volumes($cfg,[$volid]);
5367
5368             my $write_zeros = 1;
5369             # fixme: what other storages types initialize volumes with zero?
5370             if ($scfg->{type} eq 'dir' || $scfg->{type} eq 'nfs' || $scfg->{type} eq 'glusterfs' ||
5371                 $scfg->{type} eq 'sheepdog' || $scfg->{type} eq 'rbd') {
5372                 $write_zeros = 0;
5373             }
5374
5375             print $fifofh "${write_zeros}:$d->{devname}=$path\n";
5376
5377             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5378             $map->{$virtdev} = $volid;
5379         }
5380
5381         $fh->seek(0, 0) || die "seek failed - $!\n";
5382
5383         my $outfd = new IO::File ($tmpfn, "w") ||
5384             die "unable to write config for VM $vmid\n";
5385
5386         my $cookie = { netcount => 0 };
5387         while (defined(my $line = <$fh>)) {
5388             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5389         }
5390
5391         $fh->close();
5392         $outfd->close();
5393     };
5394
5395     eval {
5396         # enable interrupts
5397         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5398             die "interrupted by signal\n";
5399         };
5400         local $SIG{ALRM} = sub { die "got timeout\n"; };
5401
5402         $oldtimeout = alarm($timeout);
5403
5404         my $parser = sub {
5405             my $line = shift;
5406
5407             print "$line\n";
5408
5409             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5410                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5411                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5412             } elsif ($line =~ m/^CTIME: /) {
5413                 # we correctly received the vma config, so we can disable
5414                 # the timeout now for disk allocation (set to 10 minutes, so
5415                 # that we always timeout if something goes wrong)
5416                 alarm(600);
5417                 &$print_devmap();
5418                 print $fifofh "done\n";
5419                 my $tmp = $oldtimeout || 0;
5420                 $oldtimeout = undef;
5421                 alarm($tmp);
5422                 close($fifofh);
5423             }
5424         };
5425
5426         print "restore vma archive: $cmd\n";
5427         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5428     };
5429     my $err = $@;
5430
5431     alarm($oldtimeout) if $oldtimeout;
5432
5433     my $vollist = [];
5434     foreach my $devname (keys %$devinfo) {
5435         my $volid = $devinfo->{$devname}->{volid};
5436         push @$vollist, $volid if $volid;
5437     }
5438
5439     my $cfg = cfs_read_file('storage.cfg');
5440     PVE::Storage::deactivate_volumes($cfg, $vollist);
5441
5442     unlink $mapfifo;
5443
5444     if ($err) {
5445         rmtree $tmpdir;
5446         unlink $tmpfn;
5447
5448         foreach my $devname (keys %$devinfo) {
5449             my $volid = $devinfo->{$devname}->{volid};
5450             next if !$volid;
5451             eval {
5452                 if ($volid =~ m|^/|) {
5453                     unlink $volid || die 'unlink failed\n';
5454                 } else {
5455                     PVE::Storage::vdisk_free($cfg, $volid);
5456                 }
5457                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5458             };
5459             print STDERR "unable to cleanup '$volid' - $@" if $@;
5460         }
5461         die $err;
5462     }
5463
5464     rmtree $tmpdir;
5465
5466     rename($tmpfn, $conffile) ||
5467         die "unable to commit configuration file '$conffile'\n";
5468
5469     PVE::Cluster::cfs_update(); # make sure we read new file
5470
5471     eval { rescan($vmid, 1); };
5472     warn $@ if $@;
5473 }
5474
5475 sub restore_tar_archive {
5476     my ($archive, $vmid, $user, $opts) = @_;
5477
5478     if ($archive ne '-') {
5479         my $firstfile = tar_archive_read_firstfile($archive);
5480         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5481             if $firstfile ne 'qemu-server.conf';
5482     }
5483
5484     my $storecfg = cfs_read_file('storage.cfg');
5485
5486     # destroy existing data - keep empty config
5487     my $vmcfgfn = config_file($vmid);
5488     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5489
5490     my $tocmd = "/usr/lib/qemu-server/qmextract";
5491
5492     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5493     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5494     $tocmd .= ' --prealloc' if $opts->{prealloc};
5495     $tocmd .= ' --info' if $opts->{info};
5496
5497     # tar option "xf" does not autodetect compression when read from STDIN,
5498     # so we pipe to zcat
5499     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5500         PVE::Tools::shellquote("--to-command=$tocmd");
5501
5502     my $tmpdir = "/var/tmp/vzdumptmp$$";
5503     mkpath $tmpdir;
5504
5505     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5506     local $ENV{VZDUMP_VMID} = $vmid;
5507     local $ENV{VZDUMP_USER} = $user;
5508
5509     my $conffile = config_file($vmid);
5510     my $tmpfn = "$conffile.$$.tmp";
5511
5512     # disable interrupts (always do cleanups)
5513     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5514         print STDERR "got interrupt - ignored\n";
5515     };
5516
5517     eval {
5518         # enable interrupts
5519         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5520             die "interrupted by signal\n";
5521         };
5522
5523         if ($archive eq '-') {
5524             print "extracting archive from STDIN\n";
5525             run_command($cmd, input => "<&STDIN");
5526         } else {
5527             print "extracting archive '$archive'\n";
5528             run_command($cmd);
5529         }
5530
5531         return if $opts->{info};
5532
5533         # read new mapping
5534         my $map = {};
5535         my $statfile = "$tmpdir/qmrestore.stat";
5536         if (my $fd = IO::File->new($statfile, "r")) {
5537             while (defined (my $line = <$fd>)) {
5538                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5539                     $map->{$1} = $2 if $1;
5540                 } else {
5541                     print STDERR "unable to parse line in statfile - $line\n";
5542                 }
5543             }
5544             $fd->close();
5545         }
5546
5547         my $confsrc = "$tmpdir/qemu-server.conf";
5548
5549         my $srcfd = new IO::File($confsrc, "r") ||
5550             die "unable to open file '$confsrc'\n";
5551
5552         my $outfd = new IO::File ($tmpfn, "w") ||
5553             die "unable to write config for VM $vmid\n";
5554
5555         my $cookie = { netcount => 0 };
5556         while (defined (my $line = <$srcfd>)) {
5557             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5558         }
5559
5560         $srcfd->close();
5561         $outfd->close();
5562     };
5563     my $err = $@;
5564
5565     if ($err) {
5566
5567         unlink $tmpfn;
5568
5569         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
5570
5571         die $err;
5572     }
5573
5574     rmtree $tmpdir;
5575
5576     rename $tmpfn, $conffile ||
5577         die "unable to commit configuration file '$conffile'\n";
5578
5579     PVE::Cluster::cfs_update(); # make sure we read new file
5580
5581     eval { rescan($vmid, 1); };
5582     warn $@ if $@;
5583 };
5584
5585
5586 # Internal snapshots
5587
5588 # NOTE: Snapshot create/delete involves several non-atomic
5589 # action, and can take a long time.
5590 # So we try to avoid locking the file and use 'lock' variable
5591 # inside the config file instead.
5592
5593 my $snapshot_copy_config = sub {
5594     my ($source, $dest) = @_;
5595
5596     foreach my $k (keys %$source) {
5597         next if $k eq 'snapshots';
5598         next if $k eq 'snapstate';
5599         next if $k eq 'snaptime';
5600         next if $k eq 'vmstate';
5601         next if $k eq 'lock';
5602         next if $k eq 'digest';
5603         next if $k eq 'description';
5604         next if $k =~ m/^unused\d+$/;
5605
5606         $dest->{$k} = $source->{$k};
5607     }
5608 };
5609
5610 my $snapshot_apply_config = sub {
5611     my ($conf, $snap) = @_;
5612
5613     # copy snapshot list
5614     my $newconf = {
5615         snapshots => $conf->{snapshots},
5616     };
5617
5618     # keep description and list of unused disks
5619     foreach my $k (keys %$conf) {
5620         next if !($k =~ m/^unused\d+$/ || $k eq 'description');
5621         $newconf->{$k} = $conf->{$k};
5622     }
5623
5624     &$snapshot_copy_config($snap, $newconf);
5625
5626     return $newconf;
5627 };
5628
5629 sub foreach_writable_storage {
5630     my ($conf, $func) = @_;
5631
5632     my $sidhash = {};
5633
5634     foreach my $ds (keys %$conf) {
5635         next if !valid_drivename($ds);
5636
5637         my $drive = parse_drive($ds, $conf->{$ds});
5638         next if !$drive;
5639         next if drive_is_cdrom($drive);
5640
5641         my $volid = $drive->{file};
5642
5643         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5644         $sidhash->{$sid} = $sid if $sid;
5645     }
5646
5647     foreach my $sid (sort keys %$sidhash) {
5648         &$func($sid);
5649     }
5650 }
5651
5652 my $alloc_vmstate_volid = sub {
5653     my ($storecfg, $vmid, $conf, $snapname) = @_;
5654
5655     # Note: we try to be smart when selecting a $target storage
5656
5657     my $target;
5658
5659     # search shared storage first
5660     foreach_writable_storage($conf, sub {
5661         my ($sid) = @_;
5662         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
5663         return if !$scfg->{shared};
5664
5665         $target = $sid if !$target || $scfg->{path}; # prefer file based storage
5666     });
5667
5668     if (!$target) {
5669         # now search local storage
5670         foreach_writable_storage($conf, sub {
5671             my ($sid) = @_;
5672             my $scfg = PVE::Storage::storage_config($storecfg, $sid);
5673             return if $scfg->{shared};
5674
5675             $target = $sid if !$target || $scfg->{path}; # prefer file based storage;
5676         });
5677     }
5678
5679     $target = 'local' if !$target;
5680
5681     my $driver_state_size = 500; # assume 32MB is enough to safe all driver state;
5682     # we abort live save after $conf->{memory}, so we need at max twice that space
5683     my $size = $conf->{memory}*2 + $driver_state_size;
5684
5685     my $name = "vm-$vmid-state-$snapname";
5686     my $scfg = PVE::Storage::storage_config($storecfg, $target);
5687     $name .= ".raw" if $scfg->{path}; # add filename extension for file base storage
5688     my $volid = PVE::Storage::vdisk_alloc($storecfg, $target, $vmid, 'raw', $name, $size*1024);
5689
5690     return $volid;
5691 };
5692
5693 my $snapshot_prepare = sub {
5694     my ($vmid, $snapname, $save_vmstate, $comment) = @_;
5695
5696     my $snap;
5697
5698     my $updatefn =  sub {
5699
5700         my $conf = load_config($vmid);
5701
5702         die "you can't take a snapshot if it's a template\n"
5703             if is_template($conf);
5704
5705         check_lock($conf);
5706
5707         $conf->{lock} = 'snapshot';
5708
5709         die "snapshot name '$snapname' already used\n"
5710             if defined($conf->{snapshots}->{$snapname});
5711
5712         my $storecfg = PVE::Storage::config();
5713         die "snapshot feature is not available" if !has_feature('snapshot', $conf, $storecfg);
5714
5715         $snap = $conf->{snapshots}->{$snapname} = {};
5716
5717         if ($save_vmstate && check_running($vmid)) {
5718             $snap->{vmstate} = &$alloc_vmstate_volid($storecfg, $vmid, $conf, $snapname);
5719         }
5720
5721         &$snapshot_copy_config($conf, $snap);
5722
5723         $snap->{snapstate} = "prepare";
5724         $snap->{snaptime} = time();
5725         $snap->{description} = $comment if $comment;
5726
5727         # always overwrite machine if we save vmstate. This makes sure we
5728         # can restore it later using correct machine type
5729         $snap->{machine} = get_current_qemu_machine($vmid) if $snap->{vmstate};
5730
5731         update_config_nolock($vmid, $conf, 1);
5732     };
5733
5734     lock_config($vmid, $updatefn);
5735
5736     return $snap;
5737 };
5738
5739 my $snapshot_commit = sub {
5740     my ($vmid, $snapname) = @_;
5741
5742     my $updatefn = sub {
5743
5744         my $conf = load_config($vmid);
5745
5746         die "missing snapshot lock\n"
5747             if !($conf->{lock} && $conf->{lock} eq 'snapshot');
5748
5749         my $has_machine_config = defined($conf->{machine});
5750
5751         my $snap = $conf->{snapshots}->{$snapname};
5752
5753         die "snapshot '$snapname' does not exist\n" if !defined($snap);
5754
5755         die "wrong snapshot state\n"
5756             if !($snap->{snapstate} && $snap->{snapstate} eq "prepare");
5757
5758         delete $snap->{snapstate};
5759         delete $conf->{lock};
5760
5761         my $newconf = &$snapshot_apply_config($conf, $snap);
5762
5763         delete $newconf->{machine} if !$has_machine_config;
5764
5765         $newconf->{parent} = $snapname;
5766
5767         update_config_nolock($vmid, $newconf, 1);
5768     };
5769
5770     lock_config($vmid, $updatefn);
5771 };
5772
5773 sub snapshot_rollback {
5774     my ($vmid, $snapname) = @_;
5775
5776     my $prepare = 1;
5777
5778     my $storecfg = PVE::Storage::config();
5779
5780     my $conf = load_config($vmid);
5781
5782     my $get_snapshot_config = sub {
5783
5784         die "you can't rollback if vm is a template\n" if is_template($conf);
5785
5786         my $res = $conf->{snapshots}->{$snapname};
5787
5788         die "snapshot '$snapname' does not exist\n" if !defined($res);
5789
5790         return $res;
5791     };
5792
5793     my $snap = &$get_snapshot_config();
5794
5795     foreach_drive($snap, sub {
5796         my ($ds, $drive) = @_;
5797
5798         return if drive_is_cdrom($drive);
5799
5800         my $volid = $drive->{file};
5801
5802         PVE::Storage::volume_rollback_is_possible($storecfg, $volid, $snapname);
5803     });
5804
5805     my $updatefn = sub {
5806
5807         $conf = load_config($vmid);
5808
5809         $snap = &$get_snapshot_config();
5810
5811         die "unable to rollback to incomplete snapshot (snapstate = $snap->{snapstate})\n"
5812             if $snap->{snapstate};
5813
5814         if ($prepare) {
5815             check_lock($conf);
5816             vm_stop($storecfg, $vmid, undef, undef, 5, undef, undef);
5817         }
5818
5819         die "unable to rollback vm $vmid: vm is running\n"
5820             if check_running($vmid);
5821
5822         if ($prepare) {
5823             $conf->{lock} = 'rollback';
5824         } else {
5825             die "got wrong lock\n" if !($conf->{lock} && $conf->{lock} eq 'rollback');
5826             delete $conf->{lock};
5827         }
5828
5829         my $forcemachine;
5830
5831         if (!$prepare) {
5832             my $has_machine_config = defined($conf->{machine});
5833
5834             # copy snapshot config to current config
5835             $conf = &$snapshot_apply_config($conf, $snap);
5836             $conf->{parent} = $snapname;
5837
5838             # Note: old code did not store 'machine', so we try to be smart
5839             # and guess the snapshot was generated with kvm 1.4 (pc-i440fx-1.4).
5840             $forcemachine = $conf->{machine} || 'pc-i440fx-1.4';
5841             # we remove the 'machine' configuration if not explicitly specified
5842             # in the original config.
5843             delete $conf->{machine} if $snap->{vmstate} && !$has_machine_config;
5844         }
5845
5846         update_config_nolock($vmid, $conf, 1);
5847
5848         if (!$prepare && $snap->{vmstate}) {
5849             my $statefile = PVE::Storage::path($storecfg, $snap->{vmstate});
5850             vm_start($storecfg, $vmid, $statefile, undef, undef, undef, $forcemachine);
5851         }
5852     };
5853
5854     lock_config($vmid, $updatefn);
5855
5856     foreach_drive($snap, sub {
5857         my ($ds, $drive) = @_;
5858
5859         return if drive_is_cdrom($drive);
5860
5861         my $volid = $drive->{file};
5862         my $device = "drive-$ds";
5863
5864         PVE::Storage::volume_snapshot_rollback($storecfg, $volid, $snapname);
5865     });
5866
5867     $prepare = 0;
5868     lock_config($vmid, $updatefn);
5869 }
5870
5871 my $savevm_wait = sub {
5872     my ($vmid) = @_;
5873
5874     for(;;) {
5875         my $stat = vm_mon_cmd_nocheck($vmid, "query-savevm");
5876         if (!$stat->{status}) {
5877             die "savevm not active\n";
5878         } elsif ($stat->{status} eq 'active') {
5879             sleep(1);
5880             next;
5881         } elsif ($stat->{status} eq 'completed') {
5882             last;
5883         } else {
5884             die "query-savevm returned status '$stat->{status}'\n";
5885         }
5886     }
5887 };
5888
5889 sub do_snapshots_with_qemu {
5890     my ($storecfg, $volid) = @_;
5891
5892     my $storage_name = PVE::Storage::parse_volume_id($volid);
5893
5894     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
5895         && !$storecfg->{ids}->{$storage_name}->{krbd}){
5896         return 1;
5897     }
5898
5899     if ($volid =~ m/\.(qcow2|qed)$/){
5900         return 1;
5901     }
5902
5903     return undef;
5904 }
5905
5906 sub snapshot_create {
5907     my ($vmid, $snapname, $save_vmstate, $comment) = @_;
5908
5909     my $snap = &$snapshot_prepare($vmid, $snapname, $save_vmstate, $comment);
5910
5911     $save_vmstate = 0 if !$snap->{vmstate}; # vm is not running
5912
5913     my $config = load_config($vmid);
5914
5915     my $running = check_running($vmid);
5916
5917     my $freezefs = $running && $config->{agent};
5918     $freezefs = 0 if $snap->{vmstate}; # not needed if we save RAM
5919
5920     my $drivehash = {};
5921
5922     if ($freezefs) {
5923         eval { vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
5924         warn "guest-fsfreeze-freeze problems - $@" if $@;
5925     }
5926
5927     eval {
5928         # create internal snapshots of all drives
5929
5930         my $storecfg = PVE::Storage::config();
5931
5932         if ($running) {
5933             if ($snap->{vmstate}) {
5934                 my $path = PVE::Storage::path($storecfg, $snap->{vmstate});
5935                 vm_mon_cmd($vmid, "savevm-start", statefile => $path);
5936                 &$savevm_wait($vmid);
5937             } else {
5938                 vm_mon_cmd($vmid, "savevm-start");
5939             }
5940         };
5941
5942         foreach_drive($snap, sub {
5943             my ($ds, $drive) = @_;
5944
5945             return if drive_is_cdrom($drive);
5946
5947             my $volid = $drive->{file};
5948             my $device = "drive-$ds";
5949
5950             qemu_volume_snapshot($vmid, $device, $storecfg, $volid, $snapname);
5951             $drivehash->{$ds} = 1;
5952        });
5953     };
5954     my $err = $@;
5955
5956     if ($running) {
5957         eval { vm_mon_cmd($vmid, "savevm-end")  };
5958         warn $@ if $@;
5959
5960         if ($freezefs) {
5961             eval { vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
5962             warn "guest-fsfreeze-thaw problems - $@" if $@;
5963         }
5964
5965         # savevm-end is async, we need to wait
5966         for (;;) {
5967             my $stat = vm_mon_cmd_nocheck($vmid, "query-savevm");
5968             if (!$stat->{bytes}) {
5969                 last;
5970             } else {
5971                 print "savevm not yet finished\n";
5972                 sleep(1);
5973                 next;
5974             }
5975         }
5976     }
5977
5978     if ($err) {
5979         warn "snapshot create failed: starting cleanup\n";
5980         eval { snapshot_delete($vmid, $snapname, 0, $drivehash); };
5981         warn $@ if $@;
5982         die $err;
5983     }
5984
5985     &$snapshot_commit($vmid, $snapname);
5986 }
5987
5988 # Note: $drivehash is only set when called from snapshot_create.
5989 sub snapshot_delete {
5990     my ($vmid, $snapname, $force, $drivehash) = @_;
5991
5992     my $prepare = 1;
5993
5994     my $snap;
5995     my $unused = [];
5996
5997     my $unlink_parent = sub {
5998         my ($confref, $new_parent) = @_;
5999
6000         if ($confref->{parent} && $confref->{parent} eq $snapname) {
6001             if ($new_parent) {
6002                 $confref->{parent} = $new_parent;
6003             } else {
6004                 delete $confref->{parent};
6005             }
6006         }
6007     };
6008
6009     my $updatefn =  sub {
6010         my ($remove_drive) = @_;
6011
6012         my $conf = load_config($vmid);
6013
6014         if (!$drivehash) {
6015             check_lock($conf);
6016             die "you can't delete a snapshot if vm is a template\n"
6017                 if is_template($conf);
6018         }
6019
6020         $snap = $conf->{snapshots}->{$snapname};
6021
6022         die "snapshot '$snapname' does not exist\n" if !defined($snap);
6023
6024         # remove parent refs
6025         if (!$prepare) {
6026             &$unlink_parent($conf, $snap->{parent});
6027             foreach my $sn (keys %{$conf->{snapshots}}) {
6028                 next if $sn eq $snapname;
6029                 &$unlink_parent($conf->{snapshots}->{$sn}, $snap->{parent});
6030             }
6031         }
6032
6033         if ($remove_drive) {
6034             if ($remove_drive eq 'vmstate') {
6035                 delete $snap->{$remove_drive};
6036             } else {
6037                 my $drive = parse_drive($remove_drive, $snap->{$remove_drive});
6038                 my $volid = $drive->{file};
6039                 delete $snap->{$remove_drive};
6040                 add_unused_volume($conf, $volid);
6041             }
6042         }
6043
6044         if ($prepare) {
6045             $snap->{snapstate} = 'delete';
6046         } else {
6047             delete $conf->{snapshots}->{$snapname};
6048             delete $conf->{lock} if $drivehash;
6049             foreach my $volid (@$unused) {
6050                 add_unused_volume($conf, $volid);
6051             }
6052         }
6053
6054         update_config_nolock($vmid, $conf, 1);
6055     };
6056
6057     lock_config($vmid, $updatefn);
6058
6059     # now remove vmstate file
6060
6061     my $storecfg = PVE::Storage::config();
6062
6063     if ($snap->{vmstate}) {
6064         eval {  PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
6065         if (my $err = $@) {
6066             die $err if !$force;
6067             warn $err;
6068         }
6069         # save changes (remove vmstate from snapshot)
6070         lock_config($vmid, $updatefn, 'vmstate') if !$force;
6071     };
6072
6073     # now remove all internal snapshots
6074     foreach_drive($snap, sub {
6075         my ($ds, $drive) = @_;
6076
6077         return if drive_is_cdrom($drive);
6078
6079         my $volid = $drive->{file};
6080         my $device = "drive-$ds";
6081
6082         if (!$drivehash || $drivehash->{$ds}) {
6083             eval { qemu_volume_snapshot_delete($vmid, $device, $storecfg, $volid, $snapname); };
6084             if (my $err = $@) {
6085                 die $err if !$force;
6086                 warn $err;
6087             }
6088         }
6089
6090         # save changes (remove drive fron snapshot)
6091         lock_config($vmid, $updatefn, $ds) if !$force;
6092         push @$unused, $volid;
6093     });
6094
6095     # now cleanup config
6096     $prepare = 0;
6097     lock_config($vmid, $updatefn);
6098 }
6099
6100 sub has_feature {
6101     my ($feature, $conf, $storecfg, $snapname, $running) = @_;
6102
6103     my $err;
6104     foreach_drive($conf, sub {
6105         my ($ds, $drive) = @_;
6106
6107         return if drive_is_cdrom($drive);
6108         my $volid = $drive->{file};
6109         $err = 1 if !PVE::Storage::volume_has_feature($storecfg, $feature, $volid, $snapname, $running);
6110     });
6111
6112     return $err ? 0 : 1;
6113 }
6114
6115 sub template_create {
6116     my ($vmid, $conf, $disk) = @_;
6117
6118     my $storecfg = PVE::Storage::config();
6119
6120     foreach_drive($conf, sub {
6121         my ($ds, $drive) = @_;
6122
6123         return if drive_is_cdrom($drive);
6124         return if $disk && $ds ne $disk;
6125
6126         my $volid = $drive->{file};
6127         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6128
6129         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6130         $drive->{file} = $voliddst;
6131         $conf->{$ds} = print_drive($vmid, $drive);
6132         update_config_nolock($vmid, $conf, 1);
6133     });
6134 }
6135
6136 sub is_template {
6137     my ($conf) = @_;
6138
6139     return 1 if defined $conf->{template} && $conf->{template} == 1;
6140 }
6141
6142 sub qemu_img_convert {
6143     my ($src_volid, $dst_volid, $size, $snapname) = @_;
6144
6145     my $storecfg = PVE::Storage::config();
6146     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6147     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6148
6149     if ($src_storeid && $dst_storeid) {
6150         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6151         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6152
6153         my $src_format = qemu_img_format($src_scfg, $src_volname);
6154         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6155
6156         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6157         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6158
6159         my $cmd = [];
6160         push @$cmd, '/usr/bin/qemu-img', 'convert', '-t', 'writeback', '-p', '-n';
6161         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
6162         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path, $dst_path;
6163
6164         my $parser = sub {
6165             my $line = shift;
6166             if($line =~ m/\((\S+)\/100\%\)/){
6167                 my $percent = $1;
6168                 my $transferred = int($size * $percent / 100);
6169                 my $remaining = $size - $transferred;
6170
6171                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6172             }
6173
6174         };
6175
6176         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6177         my $err = $@;
6178         die "copy failed: $err" if $err;
6179     }
6180 }
6181
6182 sub qemu_img_format {
6183     my ($scfg, $volname) = @_;
6184
6185     if ($scfg->{path} && $volname =~ m/\.(raw|cow|qcow|qcow2|qed|vmdk|cloop)$/) {
6186         return $1;
6187     } else {
6188         return "raw";
6189     }
6190 }
6191
6192 sub qemu_drive_mirror {
6193     my ($vmid, $drive, $dst_volid, $vmiddst) = @_;
6194
6195     my $storecfg = PVE::Storage::config();
6196     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6197
6198     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6199
6200     my $format = qemu_img_format($dst_scfg, $dst_volname);
6201
6202     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6203
6204     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $dst_path };
6205     $opts->{format} = $format if $format;
6206
6207     print "drive mirror is starting (scanning bitmap) : this step can take some minutes/hours, depend of disk size and storage speed\n";
6208
6209     eval {
6210     vm_mon_cmd($vmid, "drive-mirror", %$opts);
6211         while (1) {
6212             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6213             my $stat = @$stats[0];
6214             die "mirroring job seem to have die. Maybe do you have bad sectors?" if !$stat;
6215             die "error job is not mirroring" if $stat->{type} ne "mirror";
6216
6217             my $busy = $stat->{busy};
6218             my $ready = $stat->{ready};
6219
6220             if (my $total = $stat->{len}) {
6221                 my $transferred = $stat->{offset} || 0;
6222                 my $remaining = $total - $transferred;
6223                 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6224
6225                 print "transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6226             }
6227
6228
6229             if ($stat->{ready} eq 'true') {
6230
6231                 last if $vmiddst != $vmid;
6232
6233                 # try to switch the disk if source and destination are on the same guest
6234                 eval { vm_mon_cmd($vmid, "block-job-complete", device => "drive-$drive") };
6235                 last if !$@;
6236                 die $@ if $@ !~ m/cannot be completed/;
6237             }
6238             sleep 1;
6239         }
6240
6241
6242     };
6243     my $err = $@;
6244
6245     my $cancel_job = sub {
6246         vm_mon_cmd($vmid, "block-job-cancel", device => "drive-$drive");
6247         while (1) {
6248             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6249             my $stat = @$stats[0];
6250             last if !$stat;
6251             sleep 1;
6252         }
6253     };
6254
6255     if ($err) {
6256         eval { &$cancel_job(); };
6257         die "mirroring error: $err";
6258     }
6259
6260     if ($vmiddst != $vmid) {
6261         # if we clone a disk for a new target vm, we don't switch the disk
6262         &$cancel_job(); # so we call block-job-cancel
6263     }
6264 }
6265
6266 sub clone_disk {
6267     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6268         $newvmid, $storage, $format, $full, $newvollist) = @_;
6269
6270     my $newvolid;
6271
6272     if (!$full) {
6273         print "create linked clone of drive $drivename ($drive->{file})\n";
6274         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6275         push @$newvollist, $newvolid;
6276     } else {
6277         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6278         $storeid = $storage if $storage;
6279
6280         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6281         if (!$format) {
6282             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6283             $format = qemu_img_format($scfg, $volname);
6284         }
6285
6286         # test if requested format is supported - else use default
6287         my $supported = grep { $_ eq $format } @$validFormats;
6288         $format = $defFormat if !$supported;
6289
6290         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6291
6292         print "create full clone of drive $drivename ($drive->{file})\n";
6293         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $format, undef, ($size/1024));
6294         push @$newvollist, $newvolid;
6295
6296         PVE::Storage::activate_volumes($storecfg, $newvollist);
6297
6298         if (!$running || $snapname) {
6299             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname);
6300         } else {
6301             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid);
6302         }
6303     }
6304
6305     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6306
6307     my $disk = $drive;
6308     $disk->{format} = undef;
6309     $disk->{file} = $newvolid;
6310     $disk->{size} = $size;
6311
6312     return $disk;
6313 }
6314
6315 # this only works if VM is running
6316 sub get_current_qemu_machine {
6317     my ($vmid) = @_;
6318
6319     my $cmd = { execute => 'query-machines', arguments => {} };
6320     my $res = vm_qmp_command($vmid, $cmd);
6321
6322     my ($current, $default);
6323     foreach my $e (@$res) {
6324         $default = $e->{name} if $e->{'is-default'};
6325         $current = $e->{name} if $e->{'is-current'};
6326     }
6327
6328     # fallback to the default machine if current is not supported by qemu
6329     return $current || $default || 'pc';
6330 }
6331
6332 sub qemu_machine_feature_enabled {
6333     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6334
6335     my $current_major;
6336     my $current_minor;
6337
6338     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6339
6340         $current_major = $3;
6341         $current_minor = $4;
6342
6343     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6344
6345         $current_major = $1;
6346         $current_minor = $2;
6347     }
6348
6349     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
6350
6351
6352 }
6353
6354 sub lspci {
6355
6356     my $devices = {};
6357
6358     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6359             my (undef, $id, $function) = @_;
6360             my $res = { id => $id, function => $function};
6361             push @{$devices->{$id}}, $res;
6362     });
6363
6364     return $devices;
6365 }
6366
6367 sub vm_iothreads_list {
6368     my ($vmid) = @_;
6369
6370     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6371
6372     my $iothreads = {};
6373     foreach my $iothread (@$res) {
6374         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6375     }
6376
6377     return $iothreads;
6378 }
6379
6380 sub scsihw_infos {
6381     my ($conf, $drive) = @_;
6382
6383     my $maxdev = 0;
6384
6385     if ($conf->{scsihw} && ($conf->{scsihw} =~ m/^lsi/)) {
6386         $maxdev = 7;
6387     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6388         $maxdev = 1;
6389     } else {
6390         $maxdev = 256;
6391     }
6392
6393     my $controller = int($drive->{index} / $maxdev);
6394     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6395
6396     return ($maxdev, $controller, $controller_prefix);
6397 }
6398
6399 # bash completion helper
6400
6401 sub complete_backup_archives {
6402     my ($cmdname, $pname, $cvalue) = @_;
6403
6404     my $cfg = PVE::Storage::config();
6405
6406     my $storeid;
6407
6408     if ($cvalue =~ m/^([^:]+):/) {
6409         $storeid = $1;
6410     }
6411
6412     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6413
6414     my $res = [];
6415     foreach my $id (keys %$data) {
6416         foreach my $item (@{$data->{$id}}) {
6417             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6418             push @$res, $item->{volid} if defined($item->{volid});
6419         }
6420     }
6421
6422     return $res;
6423 }
6424
6425 my $complete_vmid_full = sub {
6426     my ($running) = @_;
6427
6428     my $idlist = vmstatus();
6429
6430     my $res = [];
6431
6432     foreach my $id (keys %$idlist) {
6433         my $d = $idlist->{$id};
6434         if (defined($running)) {
6435             next if $d->{template};
6436             next if $running && $d->{status} ne 'running';
6437             next if !$running && $d->{status} eq 'running';
6438         }
6439         push @$res, $id;
6440
6441     }
6442     return $res;
6443 };
6444
6445 sub complete_vmid {
6446     return &$complete_vmid_full();
6447 }
6448
6449 sub complete_vmid_stopped {
6450     return &$complete_vmid_full(0);
6451 }
6452
6453 sub complete_vmid_running {
6454     return &$complete_vmid_full(1);
6455 }
6456
6457 sub complete_storage {
6458
6459     my $cfg = PVE::Storage::config();
6460     my $ids = $cfg->{ids};
6461
6462     my $res = [];
6463     foreach my $sid (keys %$ids) {
6464         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6465         push @$res, $sid;
6466     }
6467
6468     return $res;
6469 }
6470
6471 1;