PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QMPClient;
  31 use PVE::RPCEnvironment;
  32 use Time::HiRes qw(gettimeofday);
  33
  34 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  35
  36 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  37
  38 # Note about locking: we use flock on the config file protect
  39 # against concurent actions.
  40 # Aditionaly, we have a 'lock' setting in the config file. This
  41 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  42 # allowed when such lock is set. But you can ignore this kind of
  43 # lock with the --skiplock flag.
  44
  45 cfs_register_file('/qemu-server/',
  46                   \&parse_vm_config,
  47                   \&write_vm_config);
  48
  49 PVE::JSONSchema::register_standard_option('skiplock', {
  50     description => "Ignore locks - only root is allowed to use this option.",
  51     type => 'boolean',
  52     optional => 1,
  53 });
  54
  55 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  56     description => "Some command save/restore state from this location.",
  57     type => 'string',
  58     maxLength => 128,
  59     optional => 1,
  60 });
  61
  62 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  63     description => "The name of the snapshot.",
  64     type => 'string', format => 'pve-configid',
  65     maxLength => 40,
  66 });
  67
  68 #no warnings 'redefine';
  69
  70 sub cgroups_write {
  71    my ($controller, $vmid, $option, $value) = @_;
  72
  73    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  74    PVE::ProcFSTools::write_proc_entry($path, $value);
  75
  76 }
  77
  78 my $nodename = PVE::INotify::nodename();
  79
  80 mkdir "/etc/pve/nodes/$nodename";
  81 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  82 mkdir $confdir;
  83
  84 my $var_run_tmpdir = "/var/run/qemu-server";
  85 mkdir $var_run_tmpdir;
  86
  87 my $lock_dir = "/var/lock/qemu-server";
  88 mkdir $lock_dir;
  89
  90 my $pcisysfs = "/sys/bus/pci";
  91
  92 my $confdesc = {
  93     onboot => {
  94         optional => 1,
  95         type => 'boolean',
  96         description => "Specifies whether a VM will be started during system bootup.",
  97         default => 0,
  98     },
  99     autostart => {
 100         optional => 1,
 101         type => 'boolean',
 102         description => "Automatic restart after crash (currently ignored).",
 103         default => 0,
 104     },
 105     hotplug => {
 106         optional => 1,
 107         type => 'string', format => 'pve-hotplug-features',
 108         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 109         default => 'network,disk,usb',
 110     },
 111     reboot => {
 112         optional => 1,
 113         type => 'boolean',
 114         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 115         default => 1,
 116     },
 117     lock => {
 118         optional => 1,
 119         type => 'string',
 120         description => "Lock/unlock the VM.",
 121         enum => [qw(migrate backup snapshot rollback)],
 122     },
 123     cpulimit => {
 124         optional => 1,
 125         type => 'number',
 126         description => "Limit of CPU usage. Note if the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 127         minimum => 0,
 128         maximum => 128,
 129         default => 0,
 130     },
 131     cpuunits => {
 132         optional => 1,
 133         type => 'integer',
 134         description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 135         minimum => 0,
 136         maximum => 500000,
 137         default => 1000,
 138     },
 139     memory => {
 140         optional => 1,
 141         type => 'integer',
 142         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 143         minimum => 16,
 144         default => 512,
 145     },
 146     balloon => {
 147         optional => 1,
 148         type => 'integer',
 149         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 150         minimum => 0,
 151     },
 152     shares => {
 153         optional => 1,
 154         type => 'integer',
 155         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 156         minimum => 0,
 157         maximum => 50000,
 158         default => 1000,
 159     },
 160     keyboard => {
 161         optional => 1,
 162         type => 'string',
 163         description => "Keybord layout for vnc server. Default is read from the datacenter configuration file.",
 164         enum => PVE::Tools::kvmkeymaplist(),
 165         default => 'en-us',
 166     },
 167     name => {
 168         optional => 1,
 169         type => 'string', format => 'dns-name',
 170         description => "Set a name for the VM. Only used on the configuration web interface.",
 171     },
 172     scsihw => {
 173         optional => 1,
 174         type => 'string',
 175         description => "scsi controller model",
 176         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 177         default => 'lsi',
 178     },
 179     description => {
 180         optional => 1,
 181         type => 'string',
 182         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 183     },
 184     ostype => {
 185         optional => 1,
 186         type => 'string',
 187         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 l24 l26 solaris)],
 188         description => <<EODESC,
 189 Used to enable special optimization/features for specific
 190 operating systems:
 191
 192 other  => unspecified OS
 193 wxp    => Microsoft Windows XP
 194 w2k    => Microsoft Windows 2000
 195 w2k3   => Microsoft Windows 2003
 196 w2k8   => Microsoft Windows 2008
 197 wvista => Microsoft Windows Vista
 198 win7   => Microsoft Windows 7
 199 win8   => Microsoft Windows 8/2012
 200 l24    => Linux 2.4 Kernel
 201 l26    => Linux 2.6/3.X Kernel
 202 solaris => solaris/opensolaris/openindiania kernel
 203
 204 other|l24|l26|solaris                       ... no special behaviour
 205 wxp|w2k|w2k3|w2k8|wvista|win7|win8  ... use --localtime switch
 206 EODESC
 207     },
 208     boot => {
 209         optional => 1,
 210         type => 'string',
 211         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 212         pattern => '[acdn]{1,4}',
 213         default => 'cdn',
 214     },
 215     bootdisk => {
 216         optional => 1,
 217         type => 'string', format => 'pve-qm-bootdisk',
 218         description => "Enable booting from specified disk.",
 219         pattern => '(ide|sata|scsi|virtio)\d+',
 220     },
 221     smp => {
 222         optional => 1,
 223         type => 'integer',
 224         description => "The number of CPUs. Please use option -sockets instead.",
 225         minimum => 1,
 226         default => 1,
 227     },
 228     sockets => {
 229         optional => 1,
 230         type => 'integer',
 231         description => "The number of CPU sockets.",
 232         minimum => 1,
 233         default => 1,
 234     },
 235     cores => {
 236         optional => 1,
 237         type => 'integer',
 238         description => "The number of cores per socket.",
 239         minimum => 1,
 240         default => 1,
 241     },
 242     numa => {
 243         optional => 1,
 244         type => 'boolean',
 245         description => "Enable/disable Numa.",
 246         default => 0,
 247     },
 248     vcpus => {
 249         optional => 1,
 250         type => 'integer',
 251         description => "Number of hotplugged vcpus.",
 252         minimum => 1,
 253         default => 0,
 254     },
 255     acpi => {
 256         optional => 1,
 257         type => 'boolean',
 258         description => "Enable/disable ACPI.",
 259         default => 1,
 260     },
 261     agent => {
 262         optional => 1,
 263         type => 'boolean',
 264         description => "Enable/disable Qemu GuestAgent.",
 265         default => 0,
 266     },
 267     kvm => {
 268         optional => 1,
 269         type => 'boolean',
 270         description => "Enable/disable KVM hardware virtualization.",
 271         default => 1,
 272     },
 273     tdf => {
 274         optional => 1,
 275         type => 'boolean',
 276         description => "Enable/disable time drift fix.",
 277         default => 0,
 278     },
 279     localtime => {
 280         optional => 1,
 281         type => 'boolean',
 282         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 283     },
 284     freeze => {
 285         optional => 1,
 286         type => 'boolean',
 287         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 288     },
 289     vga => {
 290         optional => 1,
 291         type => 'string',
 292         description => "Select VGA type. If you want to use high resolution modes (>= 1280x1024x16) then you should use option 'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and 'cirrur' for other OS types. Option 'qxl' enables the SPICE display sever. You can also run without any graphic card using a serial devive as terminal.",
 293         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 294     },
 295     watchdog => {
 296         optional => 1,
 297         type => 'string', format => 'pve-qm-watchdog',
 298         typetext => '[[model=]i6300esb|ib700] [,[action=]reset|shutdown|poweroff|pause|debug|none]',
 299         description => "Create a virtual hardware watchdog device.  Once enabled (by a guest action), the watchdog must be periodically polled by an agent inside the guest or else the guest will be restarted (or execute the action specified)",
 300     },
 301     startdate => {
 302         optional => 1,
 303         type => 'string',
 304         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 305         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 306         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 307         default => 'now',
 308     },
 309     startup =>  get_standard_option('pve-startup-order'),
 310     template => {
 311         optional => 1,
 312         type => 'boolean',
 313         description => "Enable/disable Template.",
 314         default => 0,
 315     },
 316     args => {
 317         optional => 1,
 318         type => 'string',
 319         description => <<EODESCR,
 320 Note: this option is for experts only. It allows you to pass arbitrary arguments to kvm, for example:
 321
 322 args: -no-reboot -no-hpet
 323 EODESCR
 324     },
 325     tablet => {
 326         optional => 1,
 327         type => 'boolean',
 328         default => 1,
 329         description => "Enable/disable the usb tablet device. This device is usually needed to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with normal VNC clients. If you're running lots of console-only guests on one host, you may consider disabling this to save some context switches. This is turned of by default if you use spice (vga=qxl).",
 330     },
 331     migrate_speed => {
 332         optional => 1,
 333         type => 'integer',
 334         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 335         minimum => 0,
 336         default => 0,
 337     },
 338     migrate_downtime => {
 339         optional => 1,
 340         type => 'number',
 341         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 342         minimum => 0,
 343         default => 0.1,
 344     },
 345     cdrom => {
 346         optional => 1,
 347         type => 'string', format => 'pve-qm-drive',
 348         typetext => 'volume',
 349         description => "This is an alias for option -ide2",
 350     },
 351     cpu => {
 352         optional => 1,
 353         description => "Emulated CPU type.",
 354         type => 'string',
 355         enum => [ qw(486 athlon pentium pentium2 pentium3 coreduo core2duo kvm32 kvm64 qemu32 qemu64 phenom Conroe Penryn Nehalem Westmere SandyBridge IvyBridge Haswell Broadwell Opteron_G1 Opteron_G2 Opteron_G3 Opteron_G4 Opteron_G5 host) ],
 356         default => 'kvm64',
 357     },
 358     parent => get_standard_option('pve-snapshot-name', {
 359         optional => 1,
 360         description => "Parent snapshot name. This is used internally, and should not be modified.",
 361     }),
 362     snaptime => {
 363         optional => 1,
 364         description => "Timestamp for snapshots.",
 365         type => 'integer',
 366         minimum => 0,
 367     },
 368     vmstate => {
 369         optional => 1,
 370         type => 'string', format => 'pve-volume-id',
 371         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 372     },
 373     machine => {
 374         description => "Specific the Qemu machine type.",
 375         type => 'string',
 376         pattern => '(pc|pc(-i440fx)?-\d+\.\d+|q35|pc-q35-\d+\.\d+)',
 377         maxLength => 40,
 378         optional => 1,
 379     },
 380     smbios1 => {
 381         description => "Specify SMBIOS type 1 fields.",
 382         type => 'string', format => 'pve-qm-smbios1',
 383         typetext => "[manufacturer=str][,product=str][,version=str][,serial=str] [,uuid=uuid][,sku=str][,family=str]",
 384         maxLength => 256,
 385         optional => 1,
 386     },
 387 };
 388
 389 # what about other qemu settings ?
 390 #cpu => 'string',
 391 #machine => 'string',
 392 #fda => 'file',
 393 #fdb => 'file',
 394 #mtdblock => 'file',
 395 #sd => 'file',
 396 #pflash => 'file',
 397 #snapshot => 'bool',
 398 #bootp => 'file',
 399 ##tftp => 'dir',
 400 ##smb => 'dir',
 401 #kernel => 'file',
 402 #append => 'string',
 403 #initrd => 'file',
 404 ##soundhw => 'string',
 405
 406 while (my ($k, $v) = each %$confdesc) {
 407     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 408 }
 409
 410 my $MAX_IDE_DISKS = 4;
 411 my $MAX_SCSI_DISKS = 14;
 412 my $MAX_VIRTIO_DISKS = 16;
 413 my $MAX_SATA_DISKS = 6;
 414 my $MAX_USB_DEVICES = 5;
 415 my $MAX_NETS = 32;
 416 my $MAX_UNUSED_DISKS = 8;
 417 my $MAX_HOSTPCI_DEVICES = 4;
 418 my $MAX_SERIAL_PORTS = 4;
 419 my $MAX_PARALLEL_PORTS = 3;
 420 my $MAX_NUMA = 8;
 421 my $MAX_MEM = 4194304;
 422 my $STATICMEM = 1024;
 423
 424 my $numadesc = {
 425     optional => 1,
 426     type => 'string', format => 'pve-qm-numanode',
 427     typetext => "cpus=<id[-id],memory=<mb>[[,hostnodes=<id[-id]>] [,policy=<preferred|bind|interleave>]]",
 428     description => "numa topology",
 429 };
 430 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 431
 432 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 433     $confdesc->{"numa$i"} = $numadesc;
 434 }
 435
 436 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 437                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 438                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 439 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 440
 441 my $netdesc = {
 442     optional => 1,
 443     type => 'string', format => 'pve-qm-net',
 444     typetext => "MODEL=XX:XX:XX:XX:XX:XX [,bridge=<dev>][,queues=<nbqueues>][,rate=<mbps>] [,tag=<vlanid>][,firewall=0|1],link_down=0|1]",
 445     description => <<EODESCR,
 446 Specify network devices.
 447
 448 MODEL is one of: $nic_model_list_txt
 449
 450 XX:XX:XX:XX:XX:XX should be an unique MAC address. This is
 451 automatically generated if not specified.
 452
 453 The bridge parameter can be used to automatically add the interface to a bridge device. The Proxmox VE standard bridge is called 'vmbr0'.
 454
 455 Option 'rate' is used to limit traffic bandwidth from and to this interface. It is specified as floating point number, unit is 'Megabytes per second'.
 456
 457 If you specify no bridge, we create a kvm 'user' (NATed) network device, which provides DHCP and DNS services. The following addresses are used:
 458
 459 10.0.2.2   Gateway
 460 10.0.2.3   DNS Server
 461 10.0.2.4   SMB Server
 462
 463 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 464
 465 EODESCR
 466 };
 467 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 468
 469 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 470     $confdesc->{"net$i"} = $netdesc;
 471 }
 472
 473 my $drivename_hash;
 474
 475 my $idedesc = {
 476     optional => 1,
 477     type => 'string', format => 'pve-qm-drive',
 478     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads] [,discard=ignore|on]',
 479     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
 480 };
 481 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
 482
 483 my $scsidesc = {
 484     optional => 1,
 485     type => 'string', format => 'pve-qm-drive',
 486     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads] [,discard=ignore|on] [,iothread=on] [,queues=<nbqueues>]',
 487     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
 488 };
 489 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 490
 491 my $satadesc = {
 492     optional => 1,
 493     type => 'string', format => 'pve-qm-drive',
 494     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads]  [,discard=ignore|on]',
 495     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
 496 };
 497 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
 498
 499 my $virtiodesc = {
 500     optional => 1,
 501     type => 'string', format => 'pve-qm-drive',
 502     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads]  [,discard=ignore|on] [,iothread=on]',
 503     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
 504 };
 505 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
 506
 507 my $usbdesc = {
 508     optional => 1,
 509     type => 'string', format => 'pve-qm-usb-device',
 510     typetext => 'host=HOSTUSBDEVICE|spice',
 511     description => <<EODESCR,
 512 Configure an USB device (n is 0 to 4). This can be used to
 513 pass-through usb devices to the guest. HOSTUSBDEVICE syntax is:
 514
 515 'bus-port(.port)*' (decimal numbers) or
 516 'vendor_id:product_id' (hexadeciaml numbers)
 517
 518 You can use the 'lsusb -t' command to list existing usb devices.
 519
 520 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 521
 522 The value 'spice' can be used to add a usb redirection devices for spice.
 523
 524 EODESCR
 525 };
 526 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
 527
 528 my $hostpcidesc = {
 529         optional => 1,
 530         type => 'string', format => 'pve-qm-hostpci',
 531         typetext => "[host=]HOSTPCIDEVICE [,driver=kvm|vfio] [,rombar=on|off] [,pcie=0|1] [,x-vga=on|off]",
 532         description => <<EODESCR,
 533 Map host pci devices. HOSTPCIDEVICE syntax is:
 534
 535 'bus:dev.func' (hexadecimal numbers)
 536
 537 You can us the 'lspci' command to list existing pci devices.
 538
 539 The 'rombar' option determines whether or not the device's ROM will be visible in the guest's memory map (default is 'on').
 540
 541 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 542
 543 Experimental: user reported problems with this option.
 544 EODESCR
 545 };
 546 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
 547
 548 my $serialdesc = {
 549         optional => 1,
 550         type => 'string',
 551         pattern => '(/dev/.+|socket)',
 552         description =>  <<EODESCR,
 553 Create a serial device inside the VM (n is 0 to 3), and pass through a host serial device (i.e. /dev/ttyS0), or create a unix socket on the host side (use 'qm terminal' to open a terminal connection).
 554
 555 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 556
 557 Experimental: user reported problems with this option.
 558 EODESCR
 559 };
 560
 561 my $paralleldesc= {
 562         optional => 1,
 563         type => 'string',
 564         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
 565         description =>  <<EODESCR,
 566 Map host parallel devices (n is 0 to 2).
 567
 568 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 569
 570 Experimental: user reported problems with this option.
 571 EODESCR
 572 };
 573
 574 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
 575     $confdesc->{"parallel$i"} = $paralleldesc;
 576 }
 577
 578 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
 579     $confdesc->{"serial$i"} = $serialdesc;
 580 }
 581
 582 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
 583     $confdesc->{"hostpci$i"} = $hostpcidesc;
 584 }
 585
 586 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
 587     $drivename_hash->{"ide$i"} = 1;
 588     $confdesc->{"ide$i"} = $idedesc;
 589 }
 590
 591 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
 592     $drivename_hash->{"sata$i"} = 1;
 593     $confdesc->{"sata$i"} = $satadesc;
 594 }
 595
 596 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
 597     $drivename_hash->{"scsi$i"} = 1;
 598     $confdesc->{"scsi$i"} = $scsidesc ;
 599 }
 600
 601 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
 602     $drivename_hash->{"virtio$i"} = 1;
 603     $confdesc->{"virtio$i"} = $virtiodesc;
 604 }
 605
 606 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
 607     $confdesc->{"usb$i"} = $usbdesc;
 608 }
 609
 610 my $unuseddesc = {
 611     optional => 1,
 612     type => 'string', format => 'pve-volume-id',
 613     description => "Reference to unused volumes.",
 614 };
 615
 616 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
 617     $confdesc->{"unused$i"} = $unuseddesc;
 618 }
 619
 620 my $kvm_api_version = 0;
 621
 622 sub kvm_version {
 623
 624     return $kvm_api_version if $kvm_api_version;
 625
 626     my $fh = IO::File->new("</dev/kvm") ||
 627         return 0;
 628
 629     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
 630         $kvm_api_version = $v;
 631     }
 632
 633     $fh->close();
 634
 635     return  $kvm_api_version;
 636 }
 637
 638 my $kvm_user_version;
 639
 640 sub kvm_user_version {
 641
 642     return $kvm_user_version if $kvm_user_version;
 643
 644     $kvm_user_version = 'unknown';
 645
 646     my $tmp = `kvm -help 2>/dev/null`;
 647
 648     if ($tmp =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)[,\s]/) {
 649         $kvm_user_version = $2;
 650     }
 651
 652     return $kvm_user_version;
 653
 654 }
 655
 656 my $kernel_has_vhost_net = -c '/dev/vhost-net';
 657
 658 sub disknames {
 659     # order is important - used to autoselect boot disk
 660     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
 661             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
 662             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
 663             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))));
 664 }
 665
 666 sub valid_drivename {
 667     my $dev = shift;
 668
 669     return defined($drivename_hash->{$dev});
 670 }
 671
 672 sub option_exists {
 673     my $key = shift;
 674     return defined($confdesc->{$key});
 675 }
 676
 677 sub nic_models {
 678     return $nic_model_list;
 679 }
 680
 681 sub os_list_description {
 682
 683     return {
 684         other => 'Other',
 685         wxp => 'Windows XP',
 686         w2k => 'Windows 2000',
 687         w2k3 =>, 'Windows 2003',
 688         w2k8 => 'Windows 2008',
 689         wvista => 'Windows Vista',
 690         win7 => 'Windows 7',
 691         win8 => 'Windows 8/2012',
 692         l24 => 'Linux 2.4',
 693         l26 => 'Linux 2.6',
 694     };
 695 }
 696
 697 my $cdrom_path;
 698
 699 sub get_cdrom_path {
 700
 701     return  $cdrom_path if $cdrom_path;
 702
 703     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
 704     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
 705     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
 706 }
 707
 708 sub get_iso_path {
 709     my ($storecfg, $vmid, $cdrom) = @_;
 710
 711     if ($cdrom eq 'cdrom') {
 712         return get_cdrom_path();
 713     } elsif ($cdrom eq 'none') {
 714         return '';
 715     } elsif ($cdrom =~ m|^/|) {
 716         return $cdrom;
 717     } else {
 718         return PVE::Storage::path($storecfg, $cdrom);
 719     }
 720 }
 721
 722 # try to convert old style file names to volume IDs
 723 sub filename_to_volume_id {
 724     my ($vmid, $file, $media) = @_;
 725
 726     if (!($file eq 'none' || $file eq 'cdrom' ||
 727           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
 728
 729         return undef if $file =~ m|/|;
 730
 731         if ($media && $media eq 'cdrom') {
 732             $file = "local:iso/$file";
 733         } else {
 734             $file = "local:$vmid/$file";
 735         }
 736     }
 737
 738     return $file;
 739 }
 740
 741 sub verify_media_type {
 742     my ($opt, $vtype, $media) = @_;
 743
 744     return if !$media;
 745
 746     my $etype;
 747     if ($media eq 'disk') {
 748         $etype = 'images';
 749     } elsif ($media eq 'cdrom') {
 750         $etype = 'iso';
 751     } else {
 752         die "internal error";
 753     }
 754
 755     return if ($vtype eq $etype);
 756
 757     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
 758 }
 759
 760 sub cleanup_drive_path {
 761     my ($opt, $storecfg, $drive) = @_;
 762
 763     # try to convert filesystem paths to volume IDs
 764
 765     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
 766         ($drive->{file} !~ m|^/dev/.+|) &&
 767         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
 768         ($drive->{file} !~ m/^\d+$/)) {
 769         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
 770         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
 771         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
 772         verify_media_type($opt, $vtype, $drive->{media});
 773         $drive->{file} = $volid;
 774     }
 775
 776     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
 777 }
 778
 779 sub create_conf_nolock {
 780     my ($vmid, $settings) = @_;
 781
 782     my $filename = config_file($vmid);
 783
 784     die "configuration file '$filename' already exists\n" if -f $filename;
 785
 786     my $defaults = load_defaults();
 787
 788     $settings->{name} = "vm$vmid" if !$settings->{name};
 789     $settings->{memory} = $defaults->{memory} if !$settings->{memory};
 790
 791     my $data = '';
 792     foreach my $opt (keys %$settings) {
 793         next if !$confdesc->{$opt};
 794
 795         my $value = $settings->{$opt};
 796         next if !$value;
 797
 798         $data .= "$opt: $value\n";
 799     }
 800
 801     PVE::Tools::file_set_contents($filename, $data);
 802 }
 803
 804 sub parse_hotplug_features {
 805     my ($data) = @_;
 806
 807     my $res = {};
 808
 809     return $res if $data eq '0';
 810
 811     $data = $confdesc->{hotplug}->{default} if $data eq '1';
 812
 813     foreach my $feature (PVE::Tools::split_list($data)) {
 814         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
 815             $res->{$1} = 1;
 816         } else {
 817             warn "ignoring unknown hotplug feature '$feature'\n";
 818         }
 819     }
 820     return $res;
 821 }
 822
 823 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
 824 sub pve_verify_hotplug_features {
 825     my ($value, $noerr) = @_;
 826
 827     return $value if parse_hotplug_features($value);
 828
 829     return undef if $noerr;
 830
 831     die "unable to parse hotplug option\n";
 832 }
 833
 834 my $parse_size = sub {
 835     my ($value) = @_;
 836
 837     return undef if $value !~ m/^(\d+(\.\d+)?)([KMG])?$/;
 838     my ($size, $unit) = ($1, $3);
 839     if ($unit) {
 840         if ($unit eq 'K') {
 841             $size = $size * 1024;
 842         } elsif ($unit eq 'M') {
 843             $size = $size * 1024 * 1024;
 844         } elsif ($unit eq 'G') {
 845             $size = $size * 1024 * 1024 * 1024;
 846         }
 847     }
 848     return int($size);
 849 };
 850
 851 my $format_size = sub {
 852     my ($size) = @_;
 853
 854     $size = int($size);
 855
 856     my $kb = int($size/1024);
 857     return $size if $kb*1024 != $size;
 858
 859     my $mb = int($kb/1024);
 860     return "${kb}K" if $mb*1024 != $kb;
 861
 862     my $gb = int($mb/1024);
 863     return "${mb}M" if $gb*1024 != $mb;
 864
 865     return "${gb}G";
 866 };
 867
 868 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
 869 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
 870 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
 871 #        [,aio=native|threads][,discard=ignore|on][,iothread=on]
 872
 873 sub parse_drive {
 874     my ($key, $data) = @_;
 875
 876     my $res = {};
 877
 878     # $key may be undefined - used to verify JSON parameters
 879     if (!defined($key)) {
 880         $res->{interface} = 'unknown'; # should not harm when used to verify parameters
 881         $res->{index} = 0;
 882     } elsif ($key =~ m/^([^\d]+)(\d+)$/) {
 883         $res->{interface} = $1;
 884         $res->{index} = $2;
 885     } else {
 886         return undef;
 887     }
 888
 889     foreach my $p (split (/,/, $data)) {
 890         next if $p =~ m/^\s*$/;
 891
 892         if ($p =~ m/^(file|volume|cyls|heads|secs|trans|media|snapshot|cache|format|rerror|werror|backup|aio|bps|mbps|mbps_max|bps_rd|mbps_rd|mbps_rd_max|bps_wr|mbps_wr|mbps_wr_max|iops|iops_max|iops_rd|iops_rd_max|iops_wr|iops_wr_max|size|discard|iothread|queues)=(.+)$/) {
 893             my ($k, $v) = ($1, $2);
 894
 895             $k = 'file' if $k eq 'volume';
 896
 897             return undef if defined $res->{$k};
 898
 899             if ($k eq 'bps' || $k eq 'bps_rd' || $k eq 'bps_wr') {
 900                 return undef if !$v || $v !~ m/^\d+/;
 901                 $k = "m$k";
 902                 $v = sprintf("%.3f", $v / (1024*1024));
 903             }
 904             $res->{$k} = $v;
 905         } else {
 906             if (!$res->{file} && $p !~ m/=/) {
 907                 $res->{file} = $p;
 908             } else {
 909                 return undef;
 910             }
 911         }
 912     }
 913
 914     return undef if !$res->{file};
 915
 916     return undef if $res->{cache} &&
 917         $res->{cache} !~ m/^(off|none|writethrough|writeback|unsafe|directsync)$/;
 918     return undef if $res->{snapshot} && $res->{snapshot} !~ m/^(on|off)$/;
 919     return undef if $res->{cyls} && $res->{cyls} !~ m/^\d+$/;
 920     return undef if $res->{heads} && $res->{heads} !~ m/^\d+$/;
 921     return undef if $res->{secs} && $res->{secs} !~ m/^\d+$/;
 922     return undef if $res->{media} && $res->{media} !~ m/^(disk|cdrom)$/;
 923     return undef if $res->{trans} && $res->{trans} !~ m/^(none|lba|auto)$/;
 924     return undef if $res->{format} && $res->{format} !~ m/^(raw|cow|qcow|qed|qcow2|vmdk|cloop)$/;
 925     return undef if $res->{rerror} && $res->{rerror} !~ m/^(ignore|report|stop)$/;
 926     return undef if $res->{werror} && $res->{werror} !~ m/^(enospc|ignore|report|stop)$/;
 927     return undef if $res->{backup} && $res->{backup} !~ m/^(yes|no)$/;
 928     return undef if $res->{aio} && $res->{aio} !~ m/^(native|threads)$/;
 929     return undef if $res->{discard} && $res->{discard} !~ m/^(ignore|on)$/;
 930     return undef if $res->{iothread} && $res->{iothread} !~ m/^(on)$/;
 931     return undef if $res->{queues} && ($res->{queues} !~ m/^\d+$/ || $res->{queues} < 2);
 932
 933     return undef if $res->{mbps_rd} && $res->{mbps};
 934     return undef if $res->{mbps_wr} && $res->{mbps};
 935
 936     return undef if $res->{mbps} && $res->{mbps} !~ m/^\d+(\.\d+)?$/;
 937     return undef if $res->{mbps_max} && $res->{mbps_max} !~ m/^\d+(\.\d+)?$/;
 938     return undef if $res->{mbps_rd} && $res->{mbps_rd} !~ m/^\d+(\.\d+)?$/;
 939     return undef if $res->{mbps_rd_max} && $res->{mbps_rd_max} !~ m/^\d+(\.\d+)?$/;
 940     return undef if $res->{mbps_wr} && $res->{mbps_wr} !~ m/^\d+(\.\d+)?$/;
 941     return undef if $res->{mbps_wr_max} && $res->{mbps_wr_max} !~ m/^\d+(\.\d+)?$/;
 942
 943     return undef if $res->{iops_rd} && $res->{iops};
 944     return undef if $res->{iops_wr} && $res->{iops};
 945
 946
 947     return undef if $res->{iops} && $res->{iops} !~ m/^\d+$/;
 948     return undef if $res->{iops_max} && $res->{iops_max} !~ m/^\d+$/;
 949     return undef if $res->{iops_rd} && $res->{iops_rd} !~ m/^\d+$/;
 950     return undef if $res->{iops_rd_max} && $res->{iops_rd_max} !~ m/^\d+$/;
 951     return undef if $res->{iops_wr} && $res->{iops_wr} !~ m/^\d+$/;
 952     return undef if $res->{iops_wr_max} && $res->{iops_wr_max} !~ m/^\d+$/;
 953
 954
 955     if ($res->{size}) {
 956         return undef if !defined($res->{size} = &$parse_size($res->{size}));
 957     }
 958
 959     if ($res->{media} && ($res->{media} eq 'cdrom')) {
 960         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
 961         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
 962         return undef if $res->{interface} eq 'virtio';
 963     }
 964
 965     # rerror does not work with scsi drives
 966     if ($res->{rerror}) {
 967         return undef if $res->{interface} eq 'scsi';
 968     }
 969
 970     return $res;
 971 }
 972
 973 my @qemu_drive_options = qw(heads secs cyls trans media format cache snapshot rerror werror aio discard iops iops_rd iops_wr iops_max iops_rd_max iops_wr_max);
 974
 975 sub print_drive {
 976     my ($vmid, $drive) = @_;
 977
 978     my $opts = '';
 979     foreach my $o (@qemu_drive_options, 'mbps', 'mbps_rd', 'mbps_wr', 'mbps_max', 'mbps_rd_max', 'mbps_wr_max', 'backup', 'iothread', 'queues') {
 980         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
 981     }
 982
 983     if ($drive->{size}) {
 984         $opts .= ",size=" . &$format_size($drive->{size});
 985     }
 986
 987     return "$drive->{file}$opts";
 988 }
 989
 990 sub scsi_inquiry {
 991     my($fh, $noerr) = @_;
 992
 993     my $SG_IO = 0x2285;
 994     my $SG_GET_VERSION_NUM = 0x2282;
 995
 996     my $versionbuf = "\x00" x 8;
 997     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
 998     if (!$ret) {
 999         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1000         return undef;
1001     }
1002     my $version = unpack("I", $versionbuf);
1003     if ($version < 30000) {
1004         die "scsi generic interface too old\n"  if !$noerr;
1005         return undef;
1006     }
1007
1008     my $buf = "\x00" x 36;
1009     my $sensebuf = "\x00" x 8;
1010     my $cmd = pack("C x3 C x1", 0x12, 36);
1011
1012     # see /usr/include/scsi/sg.h
1013     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1014
1015     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1016                       length($sensebuf), 0, length($buf), $buf,
1017                       $cmd, $sensebuf, 6000);
1018
1019     $ret = ioctl($fh, $SG_IO, $packet);
1020     if (!$ret) {
1021         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1022         return undef;
1023     }
1024
1025     my @res = unpack($sg_io_hdr_t, $packet);
1026     if ($res[17] || $res[18]) {
1027         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1028         return undef;
1029     }
1030
1031     my $res = {};
1032     (my $byte0, my $byte1, $res->{vendor},
1033      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1034
1035     $res->{removable} = $byte1 & 128 ? 1 : 0;
1036     $res->{type} = $byte0 & 31;
1037
1038     return $res;
1039 }
1040
1041 sub path_is_scsi {
1042     my ($path) = @_;
1043
1044     my $fh = IO::File->new("+<$path") || return undef;
1045     my $res = scsi_inquiry($fh, 1);
1046     close($fh);
1047
1048     return $res;
1049 }
1050
1051 sub machine_type_is_q35 {
1052     my ($conf) = @_;
1053
1054     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1055 }
1056
1057 sub print_tabletdevice_full {
1058     my ($conf) = @_;
1059
1060     my $q35 = machine_type_is_q35($conf);
1061
1062     # we use uhci for old VMs because tablet driver was buggy in older qemu
1063     my $usbbus = $q35 ? "ehci" : "uhci";
1064
1065     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1066 }
1067
1068 sub print_drivedevice_full {
1069     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1070
1071     my $device = '';
1072     my $maxdev = 0;
1073
1074     if ($drive->{interface} eq 'virtio') {
1075         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1076         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1077         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1078     } elsif ($drive->{interface} eq 'scsi') {
1079
1080         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1081         my $unit = $drive->{index} % $maxdev;
1082         my $devicetype = 'hd';
1083         my $path = '';
1084         if (drive_is_cdrom($drive)) {
1085               $devicetype = 'cd';
1086         } else {
1087               if ($drive->{file} =~ m|^/|) {
1088                   $path = $drive->{file};
1089               } else {
1090                   $path = PVE::Storage::path($storecfg, $drive->{file});
1091               }
1092
1093               if($path =~ m/^iscsi\:\/\//){
1094                   $devicetype = 'generic';
1095               } else {
1096                   if (my $info = path_is_scsi($path)) {
1097                       if ($info->{type} == 0) {
1098                           $devicetype = 'block';
1099                       } elsif ($info->{type} == 1) { # tape
1100                           $devicetype = 'generic';
1101                       }
1102                   }
1103               }
1104          }
1105
1106         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1107             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1108         } else {
1109             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1110         }
1111
1112     } elsif ($drive->{interface} eq 'ide'){
1113         $maxdev = 2;
1114         my $controller = int($drive->{index} / $maxdev);
1115         my $unit = $drive->{index} % $maxdev;
1116         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1117
1118         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1119     } elsif ($drive->{interface} eq 'sata'){
1120         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1121         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1122         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1123     } elsif ($drive->{interface} eq 'usb') {
1124         die "implement me";
1125         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1126     } else {
1127         die "unsupported interface type";
1128     }
1129
1130     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1131
1132     return $device;
1133 }
1134
1135 sub get_initiator_name {
1136     my $initiator;
1137
1138     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1139     while (defined(my $line = <$fh>)) {
1140         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1141         $initiator = $1;
1142         last;
1143     }
1144     $fh->close();
1145
1146     return $initiator;
1147 }
1148
1149 sub print_drive_full {
1150     my ($storecfg, $vmid, $drive) = @_;
1151
1152     my $path;
1153     my $volid = $drive->{file};
1154     my $format;
1155
1156     if (drive_is_cdrom($drive)) {
1157         $path = get_iso_path($storecfg, $vmid, $volid);
1158     } else {
1159         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1160         if ($storeid) {
1161             $path = PVE::Storage::path($storecfg, $volid);
1162             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1163             $format = qemu_img_format($scfg, $volname);
1164         } else {
1165             $path = $volid;
1166         }
1167    }
1168
1169     my $opts = '';
1170     foreach my $o (@qemu_drive_options) {
1171         next if $o eq 'bootindex';
1172         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1173     }
1174
1175     $opts .= ",format=$format" if $format && !$drive->{format};
1176
1177     foreach my $o (qw(bps bps_rd bps_wr)) {
1178         my $v = $drive->{"m$o"};
1179         $opts .= ",$o=" . int($v*1024*1024) if $v;
1180     }
1181
1182     my $cache_direct = 0;
1183
1184     if (my $cache = $drive->{cache}) {
1185         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1186     } elsif (!drive_is_cdrom($drive)) {
1187         $opts .= ",cache=none";
1188         $cache_direct = 1;
1189     }
1190
1191     # aio native works only with O_DIRECT
1192     if (!$drive->{aio}) {
1193         if($cache_direct) {
1194             $opts .= ",aio=native";
1195         } else {
1196             $opts .= ",aio=threads";
1197         }
1198     }
1199
1200     my $detectzeroes = $drive->{discard} ? "unmap" : "on";
1201     $opts .= ",detect-zeroes=$detectzeroes" if !drive_is_cdrom($drive);
1202
1203     my $pathinfo = $path ? "file=$path," : '';
1204
1205     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1206 }
1207
1208 sub print_netdevice_full {
1209     my ($vmid, $conf, $net, $netid, $bridges) = @_;
1210
1211     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1212
1213     my $device = $net->{model};
1214     if ($net->{model} eq 'virtio') {
1215          $device = 'virtio-net-pci';
1216      };
1217
1218     my $pciaddr = print_pci_addr("$netid", $bridges);
1219     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1220     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1221         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1222         my $vectors = $net->{queues} * 2 + 2;
1223         $tmpstr .= ",vectors=$vectors,mq=on";
1224     }
1225     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1226     return $tmpstr;
1227 }
1228
1229 sub print_netdev_full {
1230     my ($vmid, $conf, $net, $netid) = @_;
1231
1232     my $i = '';
1233     if ($netid =~ m/^net(\d+)$/) {
1234         $i = int($1);
1235     }
1236
1237     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1238
1239     my $ifname = "tap${vmid}i$i";
1240
1241     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1242     die "interface name '$ifname' is too long (max 15 character)\n"
1243         if length($ifname) >= 16;
1244
1245     my $vhostparam = '';
1246     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1247
1248     my $vmname = $conf->{name} || "vm$vmid";
1249
1250     my $netdev = "";
1251
1252     if ($net->{bridge}) {
1253         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/pve-bridge,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1254     } else {
1255         $netdev = "type=user,id=$netid,hostname=$vmname";
1256     }
1257
1258     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1259
1260     return $netdev;
1261 }
1262
1263 sub drive_is_cdrom {
1264     my ($drive) = @_;
1265
1266     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1267
1268 }
1269
1270 sub parse_numa {
1271     my ($data) = @_;
1272
1273     my $res = {};
1274
1275     foreach my $kvp (split(/,/, $data)) {
1276
1277         if ($kvp =~ m/^memory=(\S+)$/) {
1278             $res->{memory} = $1;
1279         } elsif ($kvp =~ m/^policy=(preferred|bind|interleave)$/) {
1280             $res->{policy} = $1;
1281         } elsif ($kvp =~ m/^cpus=(\d+)(-(\d+))?$/) {
1282             $res->{cpus}->{start} = $1;
1283             $res->{cpus}->{end} = $3;
1284         } elsif ($kvp =~ m/^hostnodes=(\d+)(-(\d+))?$/) {
1285             $res->{hostnodes}->{start} = $1;
1286             $res->{hostnodes}->{end} = $3;
1287         } else {
1288             return undef;
1289         }
1290     }
1291
1292     return $res;
1293 }
1294
1295 sub parse_hostpci {
1296     my ($value) = @_;
1297
1298     return undef if !$value;
1299
1300
1301     my @list = split(/,/, $value);
1302     my $found;
1303
1304     my $res = {};
1305     foreach my $kv (@list) {
1306
1307         if ($kv =~ m/^(host=)?([a-f0-9]{2}:[a-f0-9]{2})(\.([a-f0-9]))?$/) {
1308             $found = 1;
1309             if(defined($4)){
1310                 push @{$res->{pciid}}, { id => $2 , function => $4};
1311
1312             }else{
1313                 my $pcidevices = lspci($2);
1314                 $res->{pciid} = $pcidevices->{$2};
1315             }
1316         } elsif ($kv =~ m/^driver=(kvm|vfio)$/) {
1317             $res->{driver} = $1;
1318         } elsif ($kv =~ m/^rombar=(on|off)$/) {
1319             $res->{rombar} = $1;
1320         } elsif ($kv =~ m/^x-vga=(on|off)$/) {
1321             $res->{'x-vga'} = $1;
1322         } elsif ($kv =~ m/^pcie=(\d+)$/) {
1323             $res->{pcie} = 1 if $1 == 1;
1324         } else {
1325             warn "unknown hostpci setting '$kv'\n";
1326         }
1327     }
1328
1329     return undef if !$found;
1330
1331     return $res;
1332 }
1333
1334 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1335 sub parse_net {
1336     my ($data) = @_;
1337
1338     my $res = {};
1339
1340     foreach my $kvp (split(/,/, $data)) {
1341
1342         if ($kvp =~ m/^(ne2k_pci|e1000|e1000-82540em|e1000-82544gc|e1000-82545em|rtl8139|pcnet|virtio|ne2k_isa|i82551|i82557b|i82559er|vmxnet3)(=([0-9a-f]{2}(:[0-9a-f]{2}){5}))?$/i) {
1343             my $model = lc($1);
1344             my $mac = defined($3) ? uc($3) : PVE::Tools::random_ether_addr();
1345             $res->{model} = $model;
1346             $res->{macaddr} = $mac;
1347         } elsif ($kvp =~ m/^bridge=(\S+)$/) {
1348             $res->{bridge} = $1;
1349         } elsif ($kvp =~ m/^queues=(\d+)$/) {
1350             $res->{queues} = $1;
1351         } elsif ($kvp =~ m/^rate=(\d+(\.\d+)?)$/) {
1352             $res->{rate} = $1;
1353         } elsif ($kvp =~ m/^tag=(\d+)$/) {
1354             $res->{tag} = $1;
1355         } elsif ($kvp =~ m/^firewall=([01])$/) {
1356             $res->{firewall} = $1;
1357         } elsif ($kvp =~ m/^link_down=([01])$/) {
1358             $res->{link_down} = $1;
1359         } else {
1360             return undef;
1361         }
1362
1363     }
1364
1365     return undef if !$res->{model};
1366
1367     return $res;
1368 }
1369
1370 sub print_net {
1371     my $net = shift;
1372
1373     my $res = "$net->{model}";
1374     $res .= "=$net->{macaddr}" if $net->{macaddr};
1375     $res .= ",bridge=$net->{bridge}" if $net->{bridge};
1376     $res .= ",rate=$net->{rate}" if $net->{rate};
1377     $res .= ",tag=$net->{tag}" if $net->{tag};
1378     $res .= ",firewall=1" if $net->{firewall};
1379     $res .= ",link_down=1" if $net->{link_down};
1380     $res .= ",queues=$net->{queues}" if $net->{queues};
1381
1382     return $res;
1383 }
1384
1385 sub add_random_macs {
1386     my ($settings) = @_;
1387
1388     foreach my $opt (keys %$settings) {
1389         next if $opt !~ m/^net(\d+)$/;
1390         my $net = parse_net($settings->{$opt});
1391         next if !$net;
1392         $settings->{$opt} = print_net($net);
1393     }
1394 }
1395
1396 sub add_unused_volume {
1397     my ($config, $volid) = @_;
1398
1399     my $key;
1400     for (my $ind = $MAX_UNUSED_DISKS - 1; $ind >= 0; $ind--) {
1401         my $test = "unused$ind";
1402         if (my $vid = $config->{$test}) {
1403             return if $vid eq $volid; # do not add duplicates
1404         } else {
1405             $key = $test;
1406         }
1407     }
1408
1409     die "To many unused volume - please delete them first.\n" if !$key;
1410
1411     $config->{$key} = $volid;
1412
1413     return $key;
1414 }
1415
1416 sub vm_is_volid_owner {
1417     my ($storecfg, $vmid, $volid) = @_;
1418
1419     if ($volid !~  m|^/|) {
1420         my ($path, $owner);
1421         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1422         if ($owner && ($owner == $vmid)) {
1423             return 1;
1424         }
1425     }
1426
1427     return undef;
1428 }
1429
1430 sub vmconfig_delete_pending_option {
1431     my ($conf, $key) = @_;
1432
1433     delete $conf->{pending}->{$key};
1434     my $pending_delete_hash = { $key => 1 };
1435     foreach my $opt (PVE::Tools::split_list($conf->{pending}->{delete})) {
1436         $pending_delete_hash->{$opt} = 1;
1437     }
1438     $conf->{pending}->{delete} = join(',', keys %$pending_delete_hash);
1439 }
1440
1441 sub vmconfig_undelete_pending_option {
1442     my ($conf, $key) = @_;
1443
1444     my $pending_delete_hash = {};
1445     foreach my $opt (PVE::Tools::split_list($conf->{pending}->{delete})) {
1446         $pending_delete_hash->{$opt} = 1;
1447     }
1448     delete $pending_delete_hash->{$key};
1449
1450     my @keylist = keys %$pending_delete_hash;
1451     if (scalar(@keylist)) {
1452         $conf->{pending}->{delete} = join(',', @keylist);
1453     } else {
1454         delete $conf->{pending}->{delete};
1455     }
1456 }
1457
1458 sub vmconfig_register_unused_drive {
1459     my ($storecfg, $vmid, $conf, $drive) = @_;
1460
1461     if (!drive_is_cdrom($drive)) {
1462         my $volid = $drive->{file};
1463         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1464             add_unused_volume($conf, $volid, $vmid);
1465         }
1466     }
1467 }
1468
1469 sub vmconfig_cleanup_pending {
1470     my ($conf) = @_;
1471
1472     # remove pending changes when nothing changed
1473     my $changes;
1474     foreach my $opt (keys %{$conf->{pending}}) {
1475         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
1476             $changes = 1;
1477             delete $conf->{pending}->{$opt};
1478         }
1479     }
1480
1481     # remove delete if option is not set
1482     my $pending_delete_hash = {};
1483     foreach my $opt (PVE::Tools::split_list($conf->{pending}->{delete})) {
1484         if (defined($conf->{$opt})) {
1485             $pending_delete_hash->{$opt} = 1;
1486         } else {
1487             $changes = 1;
1488         }
1489     }
1490
1491     my @keylist = keys %$pending_delete_hash;
1492     if (scalar(@keylist)) {
1493         $conf->{pending}->{delete} = join(',', @keylist);
1494     } else {
1495         delete $conf->{pending}->{delete};
1496     }
1497
1498     return $changes;
1499 }
1500
1501 my $valid_smbios1_options = {
1502     manufacturer => '\S+',
1503     product => '\S+',
1504     version => '\S+',
1505     serial => '\S+',
1506     uuid => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1507     sku => '\S+',
1508     family => '\S+',
1509 };
1510
1511 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
1512 sub parse_smbios1 {
1513     my ($data) = @_;
1514
1515     my $res = {};
1516
1517     foreach my $kvp (split(/,/, $data)) {
1518         return undef if $kvp !~ m/^(\S+)=(.+)$/;
1519         my ($k, $v) = split(/=/, $kvp);
1520         return undef if !defined($k) || !defined($v);
1521         return undef if !$valid_smbios1_options->{$k};
1522         return undef if $v !~ m/^$valid_smbios1_options->{$k}$/;
1523         $res->{$k} = $v;
1524     }
1525
1526     return $res;
1527 }
1528
1529 sub print_smbios1 {
1530     my ($smbios1) = @_;
1531
1532     my $data = '';
1533     foreach my $k (keys %$smbios1) {
1534         next if !defined($smbios1->{$k});
1535         next if !$valid_smbios1_options->{$k};
1536         $data .= ',' if $data;
1537         $data .= "$k=$smbios1->{$k}";
1538     }
1539     return $data;
1540 }
1541
1542 PVE::JSONSchema::register_format('pve-qm-smbios1', \&verify_smbios1);
1543 sub verify_smbios1 {
1544     my ($value, $noerr) = @_;
1545
1546     return $value if parse_smbios1($value);
1547
1548     return undef if $noerr;
1549
1550     die "unable to parse smbios (type 1) options\n";
1551 }
1552
1553 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
1554 sub verify_bootdisk {
1555     my ($value, $noerr) = @_;
1556
1557     return $value if valid_drivename($value);
1558
1559     return undef if $noerr;
1560
1561     die "invalid boot disk '$value'\n";
1562 }
1563
1564 PVE::JSONSchema::register_format('pve-qm-numanode', \&verify_numa);
1565 sub verify_numa {
1566     my ($value, $noerr) = @_;
1567
1568     return $value if parse_numa($value);
1569
1570     return undef if $noerr;
1571
1572     die "unable to parse numa options\n";
1573 }
1574
1575 PVE::JSONSchema::register_format('pve-qm-net', \&verify_net);
1576 sub verify_net {
1577     my ($value, $noerr) = @_;
1578
1579     return $value if parse_net($value);
1580
1581     return undef if $noerr;
1582
1583     die "unable to parse network options\n";
1584 }
1585
1586 PVE::JSONSchema::register_format('pve-qm-drive', \&verify_drive);
1587 sub verify_drive {
1588     my ($value, $noerr) = @_;
1589
1590     return $value if parse_drive(undef, $value);
1591
1592     return undef if $noerr;
1593
1594     die "unable to parse drive options\n";
1595 }
1596
1597 PVE::JSONSchema::register_format('pve-qm-hostpci', \&verify_hostpci);
1598 sub verify_hostpci {
1599     my ($value, $noerr) = @_;
1600
1601     return $value if parse_hostpci($value);
1602
1603     return undef if $noerr;
1604
1605     die "unable to parse pci id\n";
1606 }
1607
1608 PVE::JSONSchema::register_format('pve-qm-watchdog', \&verify_watchdog);
1609 sub verify_watchdog {
1610     my ($value, $noerr) = @_;
1611
1612     return $value if parse_watchdog($value);
1613
1614     return undef if $noerr;
1615
1616     die "unable to parse watchdog options\n";
1617 }
1618
1619 sub parse_watchdog {
1620     my ($value) = @_;
1621
1622     return undef if !$value;
1623
1624     my $res = {};
1625
1626     foreach my $p (split(/,/, $value)) {
1627         next if $p =~ m/^\s*$/;
1628
1629         if ($p =~ m/^(model=)?(i6300esb|ib700)$/) {
1630             $res->{model} = $2;
1631         } elsif ($p =~ m/^(action=)?(reset|shutdown|poweroff|pause|debug|none)$/) {
1632             $res->{action} = $2;
1633         } else {
1634             return undef;
1635         }
1636     }
1637
1638     return $res;
1639 }
1640
1641 sub parse_usb_device {
1642     my ($value) = @_;
1643
1644     return undef if !$value;
1645
1646     my @dl = split(/,/, $value);
1647     my $found;
1648
1649     my $res = {};
1650     foreach my $v (@dl) {
1651         if ($v =~ m/^host=(0x)?([0-9A-Fa-f]{4}):(0x)?([0-9A-Fa-f]{4})$/) {
1652             $found = 1;
1653             $res->{vendorid} = $2;
1654             $res->{productid} = $4;
1655         } elsif ($v =~ m/^host=(\d+)\-(\d+(\.\d+)*)$/) {
1656             $found = 1;
1657             $res->{hostbus} = $1;
1658             $res->{hostport} = $2;
1659         } elsif ($v =~ m/^spice$/) {
1660             $found = 1;
1661             $res->{spice} = 1;
1662         } else {
1663             return undef;
1664         }
1665     }
1666     return undef if !$found;
1667
1668     return $res;
1669 }
1670
1671 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
1672 sub verify_usb_device {
1673     my ($value, $noerr) = @_;
1674
1675     return $value if parse_usb_device($value);
1676
1677     return undef if $noerr;
1678
1679     die "unable to parse usb device\n";
1680 }
1681
1682 # add JSON properties for create and set function
1683 sub json_config_properties {
1684     my $prop = shift;
1685
1686     foreach my $opt (keys %$confdesc) {
1687         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
1688         $prop->{$opt} = $confdesc->{$opt};
1689     }
1690
1691     return $prop;
1692 }
1693
1694 sub check_type {
1695     my ($key, $value) = @_;
1696
1697     die "unknown setting '$key'\n" if !$confdesc->{$key};
1698
1699     my $type = $confdesc->{$key}->{type};
1700
1701     if (!defined($value)) {
1702         die "got undefined value\n";
1703     }
1704
1705     if ($value =~ m/[\n\r]/) {
1706         die "property contains a line feed\n";
1707     }
1708
1709     if ($type eq 'boolean') {
1710         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
1711         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
1712         die "type check ('boolean') failed - got '$value'\n";
1713     } elsif ($type eq 'integer') {
1714         return int($1) if $value =~ m/^(\d+)$/;
1715         die "type check ('integer') failed - got '$value'\n";
1716     } elsif ($type eq 'number') {
1717         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
1718         die "type check ('number') failed - got '$value'\n";
1719     } elsif ($type eq 'string') {
1720         if (my $fmt = $confdesc->{$key}->{format}) {
1721             if ($fmt eq 'pve-qm-drive') {
1722                 # special case - we need to pass $key to parse_drive()
1723                 my $drive = parse_drive($key, $value);
1724                 return $value if $drive;
1725                 die "unable to parse drive options\n";
1726             }
1727             PVE::JSONSchema::check_format($fmt, $value);
1728             return $value;
1729         }
1730         $value =~ s/^\"(.*)\"$/$1/;
1731         return $value;
1732     } else {
1733         die "internal error"
1734     }
1735 }
1736
1737 sub lock_config_full {
1738     my ($vmid, $timeout, $code, @param) = @_;
1739
1740     my $filename = config_file_lock($vmid);
1741
1742     my $res = lock_file($filename, $timeout, $code, @param);
1743
1744     die $@ if $@;
1745
1746     return $res;
1747 }
1748
1749 sub lock_config_mode {
1750     my ($vmid, $timeout, $shared, $code, @param) = @_;
1751
1752     my $filename = config_file_lock($vmid);
1753
1754     my $res = lock_file_full($filename, $timeout, $shared, $code, @param);
1755
1756     die $@ if $@;
1757
1758     return $res;
1759 }
1760
1761 sub lock_config {
1762     my ($vmid, $code, @param) = @_;
1763
1764     return lock_config_full($vmid, 10, $code, @param);
1765 }
1766
1767 sub cfs_config_path {
1768     my ($vmid, $node) = @_;
1769
1770     $node = $nodename if !$node;
1771     return "nodes/$node/qemu-server/$vmid.conf";
1772 }
1773
1774 sub check_iommu_support{
1775     #fixme : need to check IOMMU support
1776     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
1777
1778     my $iommu=1;
1779     return $iommu;
1780
1781 }
1782
1783 sub config_file {
1784     my ($vmid, $node) = @_;
1785
1786     my $cfspath = cfs_config_path($vmid, $node);
1787     return "/etc/pve/$cfspath";
1788 }
1789
1790 sub config_file_lock {
1791     my ($vmid) = @_;
1792
1793     return "$lock_dir/lock-$vmid.conf";
1794 }
1795
1796 sub touch_config {
1797     my ($vmid) = @_;
1798
1799     my $conf = config_file($vmid);
1800     utime undef, undef, $conf;
1801 }
1802
1803 sub destroy_vm {
1804     my ($storecfg, $vmid, $keep_empty_config) = @_;
1805
1806     my $conffile = config_file($vmid);
1807
1808     my $conf = load_config($vmid);
1809
1810     check_lock($conf);
1811
1812     # only remove disks owned by this VM
1813     foreach_drive($conf, sub {
1814         my ($ds, $drive) = @_;
1815
1816         return if drive_is_cdrom($drive);
1817
1818         my $volid = $drive->{file};
1819
1820         return if !$volid || $volid =~ m|^/|;
1821
1822         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
1823         return if !$path || !$owner || ($owner != $vmid);
1824
1825         PVE::Storage::vdisk_free($storecfg, $volid);
1826     });
1827
1828     if ($keep_empty_config) {
1829         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
1830     } else {
1831         unlink $conffile;
1832     }
1833
1834     # also remove unused disk
1835     eval {
1836         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
1837
1838         eval {
1839             PVE::Storage::foreach_volid($dl, sub {
1840                 my ($volid, $sid, $volname, $d) = @_;
1841                 PVE::Storage::vdisk_free($storecfg, $volid);
1842             });
1843         };
1844         warn $@ if $@;
1845
1846     };
1847     warn $@ if $@;
1848 }
1849
1850 sub load_config {
1851     my ($vmid, $node) = @_;
1852
1853     my $cfspath = cfs_config_path($vmid, $node);
1854
1855     my $conf = PVE::Cluster::cfs_read_file($cfspath);
1856
1857     die "no such VM ('$vmid')\n" if !defined($conf);
1858
1859     return $conf;
1860 }
1861
1862 sub parse_vm_config {
1863     my ($filename, $raw) = @_;
1864
1865     return undef if !defined($raw);
1866
1867     my $res = {
1868         digest => Digest::SHA::sha1_hex($raw),
1869         snapshots => {},
1870         pending => {},
1871     };
1872
1873     $filename =~ m|/qemu-server/(\d+)\.conf$|
1874         || die "got strange filename '$filename'";
1875
1876     my $vmid = $1;
1877
1878     my $conf = $res;
1879     my $descr = '';
1880     my $section = '';
1881
1882     my @lines = split(/\n/, $raw);
1883     foreach my $line (@lines) {
1884         next if $line =~ m/^\s*$/;
1885
1886         if ($line =~ m/^\[PENDING\]\s*$/i) {
1887             $section = 'pending';
1888             $conf->{description} = $descr if $descr;
1889             $descr = '';
1890             $conf = $res->{$section} = {};
1891             next;
1892
1893         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
1894             $section = $1;
1895             $conf->{description} = $descr if $descr;
1896             $descr = '';
1897             $conf = $res->{snapshots}->{$section} = {};
1898             next;
1899         }
1900
1901         if ($line =~ m/^\#(.*)\s*$/) {
1902             $descr .= PVE::Tools::decode_text($1) . "\n";
1903             next;
1904         }
1905
1906         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
1907             $descr .= PVE::Tools::decode_text($2);
1908         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
1909             $conf->{snapstate} = $1;
1910         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
1911             my $key = $1;
1912             my $value = $2;
1913             $conf->{$key} = $value;
1914         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
1915             my $value = $1;
1916             if ($section eq 'pending') {
1917                 $conf->{delete} = $value; # we parse this later
1918             } else {
1919                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
1920             }
1921         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
1922             my $key = $1;
1923             my $value = $2;
1924             eval { $value = check_type($key, $value); };
1925             if ($@) {
1926                 warn "vm $vmid - unable to parse value of '$key' - $@";
1927             } else {
1928                 my $fmt = $confdesc->{$key}->{format};
1929                 if ($fmt && $fmt eq 'pve-qm-drive') {
1930                     my $v = parse_drive($key, $value);
1931                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
1932                         $v->{file} = $volid;
1933                         $value = print_drive($vmid, $v);
1934                     } else {
1935                         warn "vm $vmid - unable to parse value of '$key'\n";
1936                         next;
1937                     }
1938                 }
1939
1940                 if ($key eq 'cdrom') {
1941                     $conf->{ide2} = $value;
1942                 } else {
1943                     $conf->{$key} = $value;
1944                 }
1945             }
1946         }
1947     }
1948
1949     $conf->{description} = $descr if $descr;
1950
1951     delete $res->{snapstate}; # just to be sure
1952
1953     return $res;
1954 }
1955
1956 sub write_vm_config {
1957     my ($filename, $conf) = @_;
1958
1959     delete $conf->{snapstate}; # just to be sure
1960
1961     if ($conf->{cdrom}) {
1962         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
1963         $conf->{ide2} = $conf->{cdrom};
1964         delete $conf->{cdrom};
1965     }
1966
1967     # we do not use 'smp' any longer
1968     if ($conf->{sockets}) {
1969         delete $conf->{smp};
1970     } elsif ($conf->{smp}) {
1971         $conf->{sockets} = $conf->{smp};
1972         delete $conf->{cores};
1973         delete $conf->{smp};
1974     }
1975
1976     my $used_volids = {};
1977
1978     my $cleanup_config = sub {
1979         my ($cref, $pending, $snapname) = @_;
1980
1981         foreach my $key (keys %$cref) {
1982             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
1983                 $key eq 'snapstate' || $key eq 'pending';
1984             my $value = $cref->{$key};
1985             if ($key eq 'delete') {
1986                 die "propertry 'delete' is only allowed in [PENDING]\n"
1987                     if !$pending;
1988                 # fixme: check syntax?
1989                 next;
1990             }
1991             eval { $value = check_type($key, $value); };
1992             die "unable to parse value of '$key' - $@" if $@;
1993
1994             $cref->{$key} = $value;
1995
1996             if (!$snapname && valid_drivename($key)) {
1997                 my $drive = parse_drive($key, $value);
1998                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
1999             }
2000         }
2001     };
2002
2003     &$cleanup_config($conf);
2004
2005     &$cleanup_config($conf->{pending}, 1);
2006
2007     foreach my $snapname (keys %{$conf->{snapshots}}) {
2008         die "internal error" if $snapname eq 'pending';
2009         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2010     }
2011
2012     # remove 'unusedX' settings if we re-add a volume
2013     foreach my $key (keys %$conf) {
2014         my $value = $conf->{$key};
2015         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2016             delete $conf->{$key};
2017         }
2018     }
2019
2020     my $generate_raw_config = sub {
2021         my ($conf) = @_;
2022
2023         my $raw = '';
2024
2025         # add description as comment to top of file
2026         my $descr = $conf->{description} || '';
2027         foreach my $cl (split(/\n/, $descr)) {
2028             $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2029         }
2030
2031         foreach my $key (sort keys %$conf) {
2032             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2033             $raw .= "$key: $conf->{$key}\n";
2034         }
2035         return $raw;
2036     };
2037
2038     my $raw = &$generate_raw_config($conf);
2039
2040     if (scalar(keys %{$conf->{pending}})){
2041         $raw .= "\n[PENDING]\n";
2042         $raw .= &$generate_raw_config($conf->{pending});
2043     }
2044
2045     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2046         $raw .= "\n[$snapname]\n";
2047         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2048     }
2049
2050     return $raw;
2051 }
2052
2053 sub update_config_nolock {
2054     my ($vmid, $conf, $skiplock) = @_;
2055
2056     check_lock($conf) if !$skiplock;
2057
2058     my $cfspath = cfs_config_path($vmid);
2059
2060     PVE::Cluster::cfs_write_file($cfspath, $conf);
2061 }
2062
2063 sub update_config {
2064     my ($vmid, $conf, $skiplock) = @_;
2065
2066     lock_config($vmid, &update_config_nolock, $conf, $skiplock);
2067 }
2068
2069 sub load_defaults {
2070
2071     my $res = {};
2072
2073     # we use static defaults from our JSON schema configuration
2074     foreach my $key (keys %$confdesc) {
2075         if (defined(my $default = $confdesc->{$key}->{default})) {
2076             $res->{$key} = $default;
2077         }
2078     }
2079
2080     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2081     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2082
2083     return $res;
2084 }
2085
2086 sub config_list {
2087     my $vmlist = PVE::Cluster::get_vmlist();
2088     my $res = {};
2089     return $res if !$vmlist || !$vmlist->{ids};
2090     my $ids = $vmlist->{ids};
2091
2092     foreach my $vmid (keys %$ids) {
2093         my $d = $ids->{$vmid};
2094         next if !$d->{node} || $d->{node} ne $nodename;
2095         next if !$d->{type} || $d->{type} ne 'qemu';
2096         $res->{$vmid}->{exists} = 1;
2097     }
2098     return $res;
2099 }
2100
2101 # test if VM uses local resources (to prevent migration)
2102 sub check_local_resources {
2103     my ($conf, $noerr) = @_;
2104
2105     my $loc_res = 0;
2106
2107     $loc_res = 1 if $conf->{hostusb}; # old syntax
2108     $loc_res = 1 if $conf->{hostpci}; # old syntax
2109
2110     foreach my $k (keys %$conf) {
2111         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2112         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2113     }
2114
2115     die "VM uses local resources\n" if $loc_res && !$noerr;
2116
2117     return $loc_res;
2118 }
2119
2120 # check if used storages are available on all nodes (use by migrate)
2121 sub check_storage_availability {
2122     my ($storecfg, $conf, $node) = @_;
2123
2124     foreach_drive($conf, sub {
2125         my ($ds, $drive) = @_;
2126
2127         my $volid = $drive->{file};
2128         return if !$volid;
2129
2130         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2131         return if !$sid;
2132
2133         # check if storage is available on both nodes
2134         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2135         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2136    });
2137 }
2138
2139 # list nodes where all VM images are available (used by has_feature API)
2140 sub shared_nodes {
2141     my ($conf, $storecfg) = @_;
2142
2143     my $nodelist = PVE::Cluster::get_nodelist();
2144     my $nodehash = { map { $_ => 1 } @$nodelist };
2145     my $nodename = PVE::INotify::nodename();
2146
2147     foreach_drive($conf, sub {
2148         my ($ds, $drive) = @_;
2149
2150         my $volid = $drive->{file};
2151         return if !$volid;
2152
2153         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2154         if ($storeid) {
2155             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2156             if ($scfg->{disable}) {
2157                 $nodehash = {};
2158             } elsif (my $avail = $scfg->{nodes}) {
2159                 foreach my $node (keys %$nodehash) {
2160                     delete $nodehash->{$node} if !$avail->{$node};
2161                 }
2162             } elsif (!$scfg->{shared}) {
2163                 foreach my $node (keys %$nodehash) {
2164                     delete $nodehash->{$node} if $node ne $nodename
2165                 }
2166             }
2167         }
2168     });
2169
2170     return $nodehash
2171 }
2172
2173 sub check_lock {
2174     my ($conf) = @_;
2175
2176     die "VM is locked ($conf->{lock})\n" if $conf->{lock};
2177 }
2178
2179 sub check_cmdline {
2180     my ($pidfile, $pid) = @_;
2181
2182     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2183     if (defined($fh)) {
2184         my $line = <$fh>;
2185         $fh->close;
2186         return undef if !$line;
2187         my @param = split(/\0/, $line);
2188
2189         my $cmd = $param[0];
2190         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2191
2192         for (my $i = 0; $i < scalar (@param); $i++) {
2193             my $p = $param[$i];
2194             next if !$p;
2195             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2196                 my $p = $param[$i+1];
2197                 return 1 if $p && ($p eq $pidfile);
2198                 return undef;
2199             }
2200         }
2201     }
2202     return undef;
2203 }
2204
2205 sub check_running {
2206     my ($vmid, $nocheck, $node) = @_;
2207
2208     my $filename = config_file($vmid, $node);
2209
2210     die "unable to find configuration file for VM $vmid - no such machine\n"
2211         if !$nocheck && ! -f $filename;
2212
2213     my $pidfile = pidfile_name($vmid);
2214
2215     if (my $fd = IO::File->new("<$pidfile")) {
2216         my $st = stat($fd);
2217         my $line = <$fd>;
2218         close($fd);
2219
2220         my $mtime = $st->mtime;
2221         if ($mtime > time()) {
2222             warn "file '$filename' modified in future\n";
2223         }
2224
2225         if ($line =~ m/^(\d+)$/) {
2226             my $pid = $1;
2227             if (check_cmdline($pidfile, $pid)) {
2228                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2229                     return $pid;
2230                 }
2231             }
2232         }
2233     }
2234
2235     return undef;
2236 }
2237
2238 sub vzlist {
2239
2240     my $vzlist = config_list();
2241
2242     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2243
2244     while (defined(my $de = $fd->read)) {
2245         next if $de !~ m/^(\d+)\.pid$/;
2246         my $vmid = $1;
2247         next if !defined($vzlist->{$vmid});
2248         if (my $pid = check_running($vmid)) {
2249             $vzlist->{$vmid}->{pid} = $pid;
2250         }
2251     }
2252
2253     return $vzlist;
2254 }
2255
2256 sub disksize {
2257     my ($storecfg, $conf) = @_;
2258
2259     my $bootdisk = $conf->{bootdisk};
2260     return undef if !$bootdisk;
2261     return undef if !valid_drivename($bootdisk);
2262
2263     return undef if !$conf->{$bootdisk};
2264
2265     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2266     return undef if !defined($drive);
2267
2268     return undef if drive_is_cdrom($drive);
2269
2270     my $volid = $drive->{file};
2271     return undef if !$volid;
2272
2273     return $drive->{size};
2274 }
2275
2276 my $last_proc_pid_stat;
2277
2278 # get VM status information
2279 # This must be fast and should not block ($full == false)
2280 # We only query KVM using QMP if $full == true (this can be slow)
2281 sub vmstatus {
2282     my ($opt_vmid, $full) = @_;
2283
2284     my $res = {};
2285
2286     my $storecfg = PVE::Storage::config();
2287
2288     my $list = vzlist();
2289     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2290
2291     my $cpucount = $cpuinfo->{cpus} || 1;
2292
2293     foreach my $vmid (keys %$list) {
2294         next if $opt_vmid && ($vmid ne $opt_vmid);
2295
2296         my $cfspath = cfs_config_path($vmid);
2297         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2298
2299         my $d = {};
2300         $d->{pid} = $list->{$vmid}->{pid};
2301
2302         # fixme: better status?
2303         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2304
2305         my $size = disksize($storecfg, $conf);
2306         if (defined($size)) {
2307             $d->{disk} = 0; # no info available
2308             $d->{maxdisk} = $size;
2309         } else {
2310             $d->{disk} = 0;
2311             $d->{maxdisk} = 0;
2312         }
2313
2314         $d->{cpus} = ($conf->{sockets} || 1) * ($conf->{cores} || 1);
2315         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2316         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2317
2318         $d->{name} = $conf->{name} || "VM $vmid";
2319         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024) : 0;
2320
2321         if ($conf->{balloon}) {
2322             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2323             $d->{shares} = defined($conf->{shares}) ? $conf->{shares} : 1000;
2324         }
2325
2326         $d->{uptime} = 0;
2327         $d->{cpu} = 0;
2328         $d->{mem} = 0;
2329
2330         $d->{netout} = 0;
2331         $d->{netin} = 0;
2332
2333         $d->{diskread} = 0;
2334         $d->{diskwrite} = 0;
2335
2336         $d->{template} = is_template($conf);
2337
2338         $res->{$vmid} = $d;
2339     }
2340
2341     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2342     foreach my $dev (keys %$netdev) {
2343         next if $dev !~ m/^tap([1-9]\d*)i/;
2344         my $vmid = $1;
2345         my $d = $res->{$vmid};
2346         next if !$d;
2347
2348         $d->{netout} += $netdev->{$dev}->{receive};
2349         $d->{netin} += $netdev->{$dev}->{transmit};
2350     }
2351
2352     my $ctime = gettimeofday;
2353
2354     foreach my $vmid (keys %$list) {
2355
2356         my $d = $res->{$vmid};
2357         my $pid = $d->{pid};
2358         next if !$pid;
2359
2360         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2361         next if !$pstat; # not running
2362
2363         my $used = $pstat->{utime} + $pstat->{stime};
2364
2365         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2366
2367         if ($pstat->{vsize}) {
2368             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2369         }
2370
2371         my $old = $last_proc_pid_stat->{$pid};
2372         if (!$old) {
2373             $last_proc_pid_stat->{$pid} = {
2374                 time => $ctime,
2375                 used => $used,
2376                 cpu => 0,
2377             };
2378             next;
2379         }
2380
2381         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2382
2383         if ($dtime > 1000) {
2384             my $dutime = $used -  $old->{used};
2385
2386             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2387             $last_proc_pid_stat->{$pid} = {
2388                 time => $ctime,
2389                 used => $used,
2390                 cpu => $d->{cpu},
2391             };
2392         } else {
2393             $d->{cpu} = $old->{cpu};
2394         }
2395     }
2396
2397     return $res if !$full;
2398
2399     my $qmpclient = PVE::QMPClient->new();
2400
2401     my $ballooncb = sub {
2402         my ($vmid, $resp) = @_;
2403
2404         my $info = $resp->{'return'};
2405         return if !$info->{max_mem};
2406
2407         my $d = $res->{$vmid};
2408
2409         # use memory assigned to VM
2410         $d->{maxmem} = $info->{max_mem};
2411         $d->{balloon} = $info->{actual};
2412
2413         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2414             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2415             $d->{freemem} = $info->{free_mem};
2416         }
2417
2418     };
2419
2420     my $blockstatscb = sub {
2421         my ($vmid, $resp) = @_;
2422         my $data = $resp->{'return'} || [];
2423         my $totalrdbytes = 0;
2424         my $totalwrbytes = 0;
2425         for my $blockstat (@$data) {
2426             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2427             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2428         }
2429         $res->{$vmid}->{diskread} = $totalrdbytes;
2430         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2431     };
2432
2433     my $statuscb = sub {
2434         my ($vmid, $resp) = @_;
2435
2436         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2437         # this fails if ballon driver is not loaded, so this must be
2438         # the last commnand (following command are aborted if this fails).
2439         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2440
2441         my $status = 'unknown';
2442         if (!defined($status = $resp->{'return'}->{status})) {
2443             warn "unable to get VM status\n";
2444             return;
2445         }
2446
2447         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2448     };
2449
2450     foreach my $vmid (keys %$list) {
2451         next if $opt_vmid && ($vmid ne $opt_vmid);
2452         next if !$res->{$vmid}->{pid}; # not running
2453         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2454     }
2455
2456     $qmpclient->queue_execute(undef, 1);
2457
2458     foreach my $vmid (keys %$list) {
2459         next if $opt_vmid && ($vmid ne $opt_vmid);
2460         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2461     }
2462
2463     return $res;
2464 }
2465
2466 sub foreach_dimm {
2467     my ($conf, $vmid, $memory, $sockets, $func) = @_;
2468
2469     my $dimm_id = 0;
2470     my $current_size = 1024;
2471     my $dimm_size = 512;
2472     return if $current_size == $memory;
2473
2474     for (my $j = 0; $j < 8; $j++) {
2475         for (my $i = 0; $i < 32; $i++) {
2476             my $name = "dimm${dimm_id}";
2477             $dimm_id++;
2478             my $numanode = $i % $sockets;
2479             $current_size += $dimm_size;
2480             &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
2481             return  $current_size if $current_size >= $memory;
2482         }
2483         $dimm_size *= 2;
2484     }
2485 }
2486
2487 sub foreach_drive {
2488     my ($conf, $func) = @_;
2489
2490     foreach my $ds (keys %$conf) {
2491         next if !valid_drivename($ds);
2492
2493         my $drive = parse_drive($ds, $conf->{$ds});
2494         next if !$drive;
2495
2496         &$func($ds, $drive);
2497     }
2498 }
2499
2500 sub foreach_volid {
2501     my ($conf, $func) = @_;
2502
2503     my $volhash = {};
2504
2505     my $test_volid = sub {
2506         my ($volid, $is_cdrom) = @_;
2507
2508         return if !$volid;
2509
2510         $volhash->{$volid} = $is_cdrom || 0;
2511     };
2512
2513     foreach_drive($conf, sub {
2514         my ($ds, $drive) = @_;
2515         &$test_volid($drive->{file}, drive_is_cdrom($drive));
2516     });
2517
2518     foreach my $snapname (keys %{$conf->{snapshots}}) {
2519         my $snap = $conf->{snapshots}->{$snapname};
2520         &$test_volid($snap->{vmstate}, 0);
2521         foreach_drive($snap, sub {
2522             my ($ds, $drive) = @_;
2523             &$test_volid($drive->{file}, drive_is_cdrom($drive));
2524         });
2525     }
2526
2527     foreach my $volid (keys %$volhash) {
2528         &$func($volid, $volhash->{$volid});
2529     }
2530 }
2531
2532 sub vga_conf_has_spice {
2533     my ($vga) = @_;
2534
2535     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
2536
2537     return $1 || 1;
2538 }
2539
2540 sub config_to_command {
2541     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2542
2543     my $cmd = [];
2544     my $globalFlags = [];
2545     my $machineFlags = [];
2546     my $rtcFlags = [];
2547     my $cpuFlags = [];
2548     my $devices = [];
2549     my $pciaddr = '';
2550     my $bridges = {};
2551     my $kvmver = kvm_user_version();
2552     my $vernum = 0; # unknown
2553     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
2554         $vernum = $1*1000000+$2*1000;
2555     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
2556         $vernum = $1*1000000+$2*1000+$3;
2557     }
2558
2559     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
2560
2561     my $have_ovz = -f '/proc/vz/vestat';
2562
2563     my $q35 = machine_type_is_q35($conf);
2564     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2565     my $machine_type = $forcemachine || $conf->{machine};
2566
2567     my $cpuunits = defined($conf->{cpuunits}) ?
2568             $conf->{cpuunits} : $defaults->{cpuunits};
2569
2570     push @$cmd, '/usr/bin/systemd-run';
2571     push @$cmd, '--scope';
2572     push @$cmd, '--slice', "qemu";
2573     push @$cmd, '--unit', $vmid;
2574     push @$cmd, '-p', "CPUShares=$cpuunits";
2575     if ($conf->{cpulimit}) {
2576         my $cpulimit = int($conf->{cpulimit} * 100);
2577         push @$cmd, '-p', "CPUQuota=$cpulimit\%";
2578     }
2579
2580     push @$cmd, '/usr/bin/kvm';
2581
2582     push @$cmd, '-id', $vmid;
2583
2584     my $use_virtio = 0;
2585
2586     my $qmpsocket = qmp_socket($vmid);
2587     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2588     push @$cmd, '-mon', "chardev=qmp,mode=control";
2589
2590     my $socket = vnc_socket($vmid);
2591     push @$cmd,  '-vnc', "unix:$socket,x509,password";
2592
2593     push @$cmd, '-pidfile' , pidfile_name($vmid);
2594
2595     push @$cmd, '-daemonize';
2596
2597     if ($conf->{smbios1}) {
2598         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
2599     }
2600
2601     if ($q35) {
2602         # the q35 chipset support native usb2, so we enable usb controller
2603         # by default for this machine type
2604         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
2605     } else {
2606         $pciaddr = print_pci_addr("piix3", $bridges);
2607         push @$devices, '-device', "piix3-usb-uhci,id=uhci$pciaddr.0x2";
2608
2609         my $use_usb2 = 0;
2610         for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2611             next if !$conf->{"usb$i"};
2612             $use_usb2 = 1;
2613         }
2614         # include usb device config
2615         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-usb.cfg' if $use_usb2;
2616     }
2617
2618     my $vga = $conf->{vga};
2619
2620     my $qxlnum = vga_conf_has_spice($vga);
2621     $vga = 'qxl' if $qxlnum;
2622
2623     if (!$vga) {
2624         if ($conf->{ostype} && ($conf->{ostype} eq 'win8' ||
2625                                 $conf->{ostype} eq 'win7' ||
2626                                 $conf->{ostype} eq 'w2k8')) {
2627             $vga = 'std';
2628         } else {
2629             $vga = 'cirrus';
2630         }
2631     }
2632
2633     # enable absolute mouse coordinates (needed by vnc)
2634     my $tablet;
2635     if (defined($conf->{tablet})) {
2636         $tablet = $conf->{tablet};
2637     } else {
2638         $tablet = $defaults->{tablet};
2639         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
2640         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
2641     }
2642
2643     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
2644
2645     # host pci devices
2646     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
2647         my $d = parse_hostpci($conf->{"hostpci$i"});
2648         next if !$d;
2649
2650         my $pcie = $d->{pcie};
2651         if($pcie){
2652             die "q35 machine model is not enabled" if !$q35;
2653             $pciaddr = print_pcie_addr("hostpci$i");
2654         }else{
2655             $pciaddr = print_pci_addr("hostpci$i", $bridges);
2656         }
2657
2658         my $rombar = $d->{rombar} && $d->{rombar} eq 'off' ? ",rombar=0" : "";
2659         my $driver = $d->{driver} && $d->{driver} eq 'vfio' ? "vfio-pci" : "pci-assign";
2660         my $xvga = $d->{'x-vga'} && $d->{'x-vga'} eq 'on' ? ",x-vga=on" : "";
2661         if ($xvga && $xvga ne '') {
2662             push @$cpuFlags, 'kvm=off';
2663             $vga = 'none';
2664         }
2665         $driver = "vfio-pci" if $xvga ne '';
2666         my $pcidevices = $d->{pciid};
2667         my $multifunction = 1 if @$pcidevices > 1;
2668
2669         my $j=0;
2670         foreach my $pcidevice (@$pcidevices) {
2671
2672             my $id = "hostpci$i";
2673             $id .= ".$j" if $multifunction;
2674             my $addr = $pciaddr;
2675             $addr .= ".$j" if $multifunction;
2676             my $devicestr = "$driver,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
2677
2678             if($j == 0){
2679                 $devicestr .= "$rombar$xvga";
2680                 $devicestr .= ",multifunction=on" if $multifunction;
2681             }
2682
2683             push @$devices, '-device', $devicestr;
2684             $j++;
2685         }
2686     }
2687
2688     # usb devices
2689     for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2690         my $d = parse_usb_device($conf->{"usb$i"});
2691         next if !$d;
2692         if ($d->{vendorid} && $d->{productid}) {
2693             push @$devices, '-device', "usb-host,vendorid=0x$d->{vendorid},productid=0x$d->{productid}";
2694         } elsif (defined($d->{hostbus}) && defined($d->{hostport})) {
2695             push @$devices, '-device', "usb-host,hostbus=$d->{hostbus},hostport=$d->{hostport}";
2696         } elsif ($d->{spice}) {
2697             # usb redir support for spice
2698             push @$devices, '-chardev', "spicevmc,id=usbredirchardev$i,name=usbredir";
2699             push @$devices, '-device', "usb-redir,chardev=usbredirchardev$i,id=usbredirdev$i,bus=ehci.0";
2700         }
2701     }
2702
2703     # serial devices
2704     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2705         if (my $path = $conf->{"serial$i"}) {
2706             if ($path eq 'socket') {
2707                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
2708                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
2709                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2710             } else {
2711                 die "no such serial device\n" if ! -c $path;
2712                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
2713                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2714             }
2715         }
2716     }
2717
2718     # parallel devices
2719     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
2720         if (my $path = $conf->{"parallel$i"}) {
2721             die "no such parallel device\n" if ! -c $path;
2722             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
2723             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
2724             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
2725         }
2726     }
2727
2728     my $vmname = $conf->{name} || "vm$vmid";
2729
2730     push @$cmd, '-name', $vmname;
2731
2732     my $sockets = 1;
2733     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
2734     $sockets = $conf->{sockets} if  $conf->{sockets};
2735
2736     my $cores = $conf->{cores} || 1;
2737
2738     my $maxcpus = $sockets * $cores;
2739
2740     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
2741
2742     my $allowed_vcpus = $cpuinfo->{cpus};
2743
2744     die "MAX $maxcpus vcpus allowed per VM on this node\n"
2745         if ($allowed_vcpus < $maxcpus);
2746
2747     push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
2748
2749     push @$cmd, '-nodefaults';
2750
2751     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2752
2753     my $bootindex_hash = {};
2754     my $i = 1;
2755     foreach my $o (split(//, $bootorder)) {
2756         $bootindex_hash->{$o} = $i*100;
2757         $i++;
2758     }
2759
2760     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000";
2761
2762     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
2763
2764     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
2765
2766     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
2767
2768     # time drift fix
2769     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
2770
2771     my $nokvm = defined($conf->{kvm}) && $conf->{kvm} == 0 ? 1 : 0;
2772     my $useLocaltime = $conf->{localtime};
2773
2774     if (my $ost = $conf->{ostype}) {
2775         # other, wxp, w2k, w2k3, w2k8, wvista, win7, win8, l24, l26, solaris
2776
2777         if ($ost =~ m/^w/) { # windows
2778             $useLocaltime = 1 if !defined($conf->{localtime});
2779
2780             # use time drift fix when acpi is enabled
2781             if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
2782                 $tdf = 1 if !defined($conf->{tdf});
2783             }
2784         }
2785
2786         if ($ost eq 'win7' || $ost eq 'win8' || $ost eq 'w2k8' ||
2787             $ost eq 'wvista') {
2788             push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
2789             push @$cmd, '-no-hpet';
2790             if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
2791                 push @$cpuFlags , 'hv_spinlocks=0x1fff' if !$nokvm;
2792                 push @$cpuFlags , 'hv_vapic' if !$nokvm;
2793                 push @$cpuFlags , 'hv_time' if !$nokvm;
2794
2795             } else {
2796                 push @$cpuFlags , 'hv_spinlocks=0xffff' if !$nokvm;
2797             }
2798         }
2799
2800         if ($ost eq 'win7' || $ost eq 'win8') {
2801             push @$cpuFlags , 'hv_relaxed' if !$nokvm;
2802         }
2803     }
2804
2805     push @$rtcFlags, 'driftfix=slew' if $tdf;
2806
2807     if ($nokvm) {
2808         push @$machineFlags, 'accel=tcg';
2809     } else {
2810         die "No accelerator found!\n" if !$cpuinfo->{hvm};
2811     }
2812
2813     if ($machine_type) {
2814         push @$machineFlags, "type=${machine_type}";
2815     }
2816
2817     if ($conf->{startdate}) {
2818         push @$rtcFlags, "base=$conf->{startdate}";
2819     } elsif ($useLocaltime) {
2820         push @$rtcFlags, 'base=localtime';
2821     }
2822
2823     my $cpu = $nokvm ? "qemu64" : "kvm64";
2824     $cpu = $conf->{cpu} if $conf->{cpu};
2825
2826     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
2827
2828     push @$cpuFlags , '+x2apic' if !$nokvm && $conf->{ostype} ne 'solaris';
2829
2830     push @$cpuFlags , '-x2apic' if $conf->{ostype} eq 'solaris';
2831
2832     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
2833
2834     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
2835
2836         push @$cpuFlags , '+kvm_pv_unhalt' if !$nokvm;
2837         push @$cpuFlags , '+kvm_pv_eoi' if !$nokvm;
2838     }
2839
2840     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
2841
2842     push @$cmd, '-cpu', "$cpu,enforce";
2843
2844     my $memory = $conf->{memory} || $defaults->{memory};
2845     my $static_memory = 0;
2846     my $dimm_memory = 0;
2847
2848     if ($hotplug_features->{memory}) {
2849         die "Numa need to be enabled for memory hotplug\n" if !$conf->{numa};
2850         die "Total memory is bigger than ${MAX_MEM}MB\n" if $memory > $MAX_MEM;
2851         $static_memory = $STATICMEM;
2852         die "minimum memory must be ${static_memory}MB\n" if($memory < $static_memory);
2853         $dimm_memory = $memory - $static_memory;
2854         push @$cmd, '-m', "size=${static_memory},slots=255,maxmem=${MAX_MEM}M";
2855
2856     } else {
2857
2858         $static_memory = $memory;
2859         push @$cmd, '-m', $static_memory;
2860     }
2861
2862     if ($conf->{numa}) {
2863
2864         my $numa_totalmemory = undef;
2865         for (my $i = 0; $i < $MAX_NUMA; $i++) {
2866             next if !$conf->{"numa$i"};
2867             my $numa = parse_numa($conf->{"numa$i"});
2868             next if !$numa;
2869             # memory
2870             die "missing numa node$i memory value\n" if !$numa->{memory};
2871             my $numa_memory = $numa->{memory};
2872             $numa_totalmemory += $numa_memory;
2873             my $numa_object = "memory-backend-ram,id=ram-node$i,size=${numa_memory}M";
2874
2875             # cpus
2876             my $cpus_start = $numa->{cpus}->{start};
2877             die "missing numa node$i cpus\n" if !defined($cpus_start);
2878             my $cpus_end = $numa->{cpus}->{end} if defined($numa->{cpus}->{end});
2879             my $cpus = $cpus_start;
2880             if (defined($cpus_end)) {
2881                 $cpus .= "-$cpus_end";
2882                 die "numa node$i :  cpu range $cpus is incorrect\n" if $cpus_end <= $cpus_start;
2883             }
2884
2885             # hostnodes
2886             my $hostnodes_start = $numa->{hostnodes}->{start};
2887             if (defined($hostnodes_start)) {
2888                 my $hostnodes_end = $numa->{hostnodes}->{end} if defined($numa->{hostnodes}->{end});
2889                 my $hostnodes = $hostnodes_start;
2890                 if (defined($hostnodes_end)) {
2891                     $hostnodes .= "-$hostnodes_end";
2892                     die "host node $hostnodes range is incorrect\n" if $hostnodes_end <= $hostnodes_start;
2893                 }
2894
2895                 my $hostnodes_end_range = defined($hostnodes_end) ? $hostnodes_end : $hostnodes_start;
2896                 for (my $i = $hostnodes_start; $i <= $hostnodes_end_range; $i++ ) {
2897                     die "host numa node$i don't exist\n" if ! -d "/sys/devices/system/node/node$i/";
2898                 }
2899
2900                 # policy
2901                 my $policy = $numa->{policy};
2902                 die "you need to define a policy for hostnode $hostnodes\n" if !$policy;
2903                 $numa_object .= ",host-nodes=$hostnodes,policy=$policy";
2904             }
2905
2906             push @$cmd, '-object', $numa_object;
2907             push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
2908         }
2909
2910         die "total memory for NUMA nodes must be equal to vm static memory\n"
2911             if $numa_totalmemory && $numa_totalmemory != $static_memory;
2912
2913         #if no custom tology, we split memory and cores across numa nodes
2914         if(!$numa_totalmemory) {
2915
2916             my $numa_memory = ($static_memory / $sockets) . "M";
2917
2918             for (my $i = 0; $i < $sockets; $i++)  {
2919
2920                 my $cpustart = ($cores * $i);
2921                 my $cpuend = ($cpustart + $cores - 1) if $cores && $cores > 1;
2922                 my $cpus = $cpustart;
2923                 $cpus .= "-$cpuend" if $cpuend;
2924
2925                 push @$cmd, '-object', "memory-backend-ram,size=$numa_memory,id=ram-node$i";
2926                 push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
2927             }
2928         }
2929     }
2930
2931     if ($hotplug_features->{memory}) {
2932         foreach_dimm($conf, $vmid, $memory, $sockets, sub {
2933             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
2934             push @$cmd, "-object" , "memory-backend-ram,id=mem-$name,size=${dimm_size}M";
2935             push @$cmd, "-device", "pc-dimm,id=$name,memdev=mem-$name,node=$numanode";
2936
2937             #if dimm_memory is not aligned to dimm map
2938             if($current_size > $memory) {
2939                  $conf->{memory} = $current_size;
2940                  update_config_nolock($vmid, $conf, 1);
2941             }
2942         });
2943     }
2944
2945     push @$cmd, '-S' if $conf->{freeze};
2946
2947     # set keyboard layout
2948     my $kb = $conf->{keyboard} || $defaults->{keyboard};
2949     push @$cmd, '-k', $kb if $kb;
2950
2951     # enable sound
2952     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
2953     #push @$cmd, '-soundhw', 'es1370';
2954     #push @$cmd, '-soundhw', $soundhw if $soundhw;
2955
2956     if($conf->{agent}) {
2957         my $qgasocket = qmp_socket($vmid, 1);
2958         my $pciaddr = print_pci_addr("qga0", $bridges);
2959         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
2960         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
2961         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
2962     }
2963
2964     my $spice_port;
2965
2966     if ($qxlnum) {
2967         if ($qxlnum > 1) {
2968             if ($conf->{ostype} && $conf->{ostype} =~ m/^w/){
2969                 for(my $i = 1; $i < $qxlnum; $i++){
2970                     my $pciaddr = print_pci_addr("vga$i", $bridges);
2971                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
2972                 }
2973             } else {
2974                 # assume other OS works like Linux
2975                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
2976                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
2977             }
2978         }
2979
2980         my $pciaddr = print_pci_addr("spice", $bridges);
2981
2982         my $nodename = PVE::INotify::nodename();
2983         my $pfamily = PVE::Tools::get_host_address_family($nodename);
2984         $spice_port = PVE::Tools::next_spice_port($pfamily);
2985
2986         push @$devices, '-spice', "tls-port=${spice_port},addr=localhost,tls-ciphers=DES-CBC3-SHA,seamless-migration=on";
2987
2988         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
2989         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
2990         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
2991     }
2992
2993     # enable balloon by default, unless explicitly disabled
2994     if (!defined($conf->{balloon}) || $conf->{balloon}) {
2995         $pciaddr = print_pci_addr("balloon0", $bridges);
2996         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
2997     }
2998
2999     if ($conf->{watchdog}) {
3000         my $wdopts = parse_watchdog($conf->{watchdog});
3001         $pciaddr = print_pci_addr("watchdog", $bridges);
3002         my $watchdog = $wdopts->{model} || 'i6300esb';
3003         push @$devices, '-device', "$watchdog$pciaddr";
3004         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3005     }
3006
3007     my $vollist = [];
3008     my $scsicontroller = {};
3009     my $ahcicontroller = {};
3010     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3011
3012     # Add iscsi initiator name if available
3013     if (my $initiator = get_initiator_name()) {
3014         push @$devices, '-iscsi', "initiator-name=$initiator";
3015     }
3016
3017     foreach_drive($conf, sub {
3018         my ($ds, $drive) = @_;
3019
3020         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3021             push @$vollist, $drive->{file};
3022         }
3023
3024         $use_virtio = 1 if $ds =~ m/^virtio/;
3025
3026         if (drive_is_cdrom ($drive)) {
3027             if ($bootindex_hash->{d}) {
3028                 $drive->{bootindex} = $bootindex_hash->{d};
3029                 $bootindex_hash->{d} += 1;
3030             }
3031         } else {
3032             if ($bootindex_hash->{c}) {
3033                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3034                 $bootindex_hash->{c} += 1;
3035             }
3036         }
3037
3038         if($drive->{interface} eq 'virtio'){
3039            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3040         }
3041
3042         if ($drive->{interface} eq 'scsi') {
3043
3044             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3045
3046             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3047             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3048
3049             my $iothread = '';
3050             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3051                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3052                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3053             }
3054
3055             my $queues = '';
3056             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3057                 $queues = ",num_queues=$drive->{queues}";
3058             }
3059
3060             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3061             $scsicontroller->{$controller}=1;
3062         }
3063
3064         if ($drive->{interface} eq 'sata') {
3065            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3066            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3067            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3068            $ahcicontroller->{$controller}=1;
3069         }
3070
3071         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3072         push @$devices, '-drive',$drive_cmd;
3073         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3074     });
3075
3076     for (my $i = 0; $i < $MAX_NETS; $i++) {
3077          next if !$conf->{"net$i"};
3078          my $d = parse_net($conf->{"net$i"});
3079          next if !$d;
3080
3081          $use_virtio = 1 if $d->{model} eq 'virtio';
3082
3083          if ($bootindex_hash->{n}) {
3084             $d->{bootindex} = $bootindex_hash->{n};
3085             $bootindex_hash->{n} += 1;
3086          }
3087
3088          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3089          push @$devices, '-netdev', $netdevfull;
3090
3091          my $netdevicefull = print_netdevice_full($vmid,$conf,$d,"net$i",$bridges);
3092          push @$devices, '-device', $netdevicefull;
3093     }
3094
3095     if (!$q35) {
3096         # add pci bridges
3097         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3098            $bridges->{1} = 1;
3099            $bridges->{2} = 1;
3100         }
3101
3102         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3103
3104         while (my ($k, $v) = each %$bridges) {
3105             $pciaddr = print_pci_addr("pci.$k");
3106             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3107         }
3108     }
3109
3110     # add custom args
3111     if ($conf->{args}) {
3112         my $aa = PVE::Tools::split_args($conf->{args});
3113         push @$cmd, @$aa;
3114     }
3115
3116     push @$cmd, @$devices;
3117     push @$cmd, '-rtc', join(',', @$rtcFlags)
3118         if scalar(@$rtcFlags);
3119     push @$cmd, '-machine', join(',', @$machineFlags)
3120         if scalar(@$machineFlags);
3121     push @$cmd, '-global', join(',', @$globalFlags)
3122         if scalar(@$globalFlags);
3123
3124     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3125 }
3126
3127 sub vnc_socket {
3128     my ($vmid) = @_;
3129     return "${var_run_tmpdir}/$vmid.vnc";
3130 }
3131
3132 sub spice_port {
3133     my ($vmid) = @_;
3134
3135     my $res = vm_mon_cmd($vmid, 'query-spice');
3136
3137     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3138 }
3139
3140 sub qmp_socket {
3141     my ($vmid, $qga) = @_;
3142     my $sockettype = $qga ? 'qga' : 'qmp';
3143     return "${var_run_tmpdir}/$vmid.$sockettype";
3144 }
3145
3146 sub pidfile_name {
3147     my ($vmid) = @_;
3148     return "${var_run_tmpdir}/$vmid.pid";
3149 }
3150
3151 sub vm_devices_list {
3152     my ($vmid) = @_;
3153
3154     my $res = vm_mon_cmd($vmid, 'query-pci');
3155     my $devices = {};
3156     foreach my $pcibus (@$res) {
3157         foreach my $device (@{$pcibus->{devices}}) {
3158             next if !$device->{'qdev_id'};
3159             if ($device->{'pci_bridge'}) {
3160                 $devices->{$device->{'qdev_id'}} = 1;
3161                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3162                     next if !$bridge_device->{'qdev_id'};
3163                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3164                     $devices->{$device->{'qdev_id'}}++;
3165                 }
3166             } else {
3167                 $devices->{$device->{'qdev_id'}} = 1;
3168             }
3169         }
3170     }
3171
3172     my $resblock = vm_mon_cmd($vmid, 'query-block');
3173     foreach my $block (@$resblock) {
3174         if($block->{device} =~ m/^drive-(\S+)/){
3175                 $devices->{$1} = 1;
3176         }
3177     }
3178
3179     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3180     foreach my $mice (@$resmice) {
3181         if ($mice->{name} eq 'QEMU HID Tablet') {
3182             $devices->{tablet} = 1;
3183             last;
3184         }
3185     }
3186
3187     return $devices;
3188 }
3189
3190 sub vm_deviceplug {
3191     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3192
3193     my $q35 = machine_type_is_q35($conf);
3194
3195     my $devices_list = vm_devices_list($vmid);
3196     return 1 if defined($devices_list->{$deviceid});
3197
3198     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3199
3200     if ($deviceid eq 'tablet') {
3201
3202         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3203
3204     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3205
3206         qemu_iothread_add($vmid, $deviceid, $device);
3207
3208         qemu_driveadd($storecfg, $vmid, $device);
3209         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3210
3211         qemu_deviceadd($vmid, $devicefull);
3212         eval { qemu_deviceaddverify($vmid, $deviceid); };
3213         if (my $err = $@) {
3214             eval { qemu_drivedel($vmid, $deviceid); };
3215             warn $@ if $@;
3216             die $err;
3217         }
3218
3219     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3220
3221
3222         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3223         my $pciaddr = print_pci_addr($deviceid);
3224         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3225
3226         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3227
3228         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3229             qemu_iothread_add($vmid, $deviceid, $device);
3230             $devicefull .= ",iothread=iothread-$deviceid";
3231         }
3232
3233         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3234             $devicefull .= ",num_queues=$device->{queues}";
3235         }
3236
3237         qemu_deviceadd($vmid, $devicefull);
3238         qemu_deviceaddverify($vmid, $deviceid);
3239
3240     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3241
3242         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3243         qemu_driveadd($storecfg, $vmid, $device);
3244
3245         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3246         eval { qemu_deviceadd($vmid, $devicefull); };
3247         if (my $err = $@) {
3248             eval { qemu_drivedel($vmid, $deviceid); };
3249             warn $@ if $@;
3250             die $err;
3251         }
3252
3253     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3254
3255         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3256         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid);
3257         qemu_deviceadd($vmid, $netdevicefull);
3258         eval { qemu_deviceaddverify($vmid, $deviceid); };
3259         if (my $err = $@) {
3260             eval { qemu_netdevdel($vmid, $deviceid); };
3261             warn $@ if $@;
3262             die $err;
3263         }
3264
3265     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3266
3267         my $bridgeid = $2;
3268         my $pciaddr = print_pci_addr($deviceid);
3269         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3270
3271         qemu_deviceadd($vmid, $devicefull);
3272         qemu_deviceaddverify($vmid, $deviceid);
3273
3274     } else {
3275         die "can't hotplug device '$deviceid'\n";
3276     }
3277
3278     return 1;
3279 }
3280
3281 # fixme: this should raise exceptions on error!
3282 sub vm_deviceunplug {
3283     my ($vmid, $conf, $deviceid) = @_;
3284
3285     my $devices_list = vm_devices_list($vmid);
3286     return 1 if !defined($devices_list->{$deviceid});
3287
3288     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3289
3290     if ($deviceid eq 'tablet') {
3291
3292         qemu_devicedel($vmid, $deviceid);
3293
3294     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3295
3296         qemu_devicedel($vmid, $deviceid);
3297         qemu_devicedelverify($vmid, $deviceid);
3298         qemu_drivedel($vmid, $deviceid);
3299         qemu_iothread_del($conf, $vmid, $deviceid);
3300
3301     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3302
3303         qemu_devicedel($vmid, $deviceid);
3304         qemu_devicedelverify($vmid, $deviceid);
3305         qemu_iothread_del($conf, $vmid, $deviceid);
3306
3307     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3308
3309         #qemu 2.3 segfault on drive_del with virtioscsi + iothread
3310         my $device = parse_drive($deviceid, $conf->{$deviceid});
3311         die "virtioscsi with iothread is not hot-unplugglable currently" if $device->{iothread};
3312
3313         qemu_devicedel($vmid, $deviceid);
3314         qemu_drivedel($vmid, $deviceid);
3315         qemu_deletescsihw($conf, $vmid, $deviceid);
3316
3317     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3318
3319         qemu_devicedel($vmid, $deviceid);
3320         qemu_devicedelverify($vmid, $deviceid);
3321         qemu_netdevdel($vmid, $deviceid);
3322
3323     } else {
3324         die "can't unplug device '$deviceid'\n";
3325     }
3326
3327     return 1;
3328 }
3329
3330 sub qemu_deviceadd {
3331     my ($vmid, $devicefull) = @_;
3332
3333     $devicefull = "driver=".$devicefull;
3334     my %options =  split(/[=,]/, $devicefull);
3335
3336     vm_mon_cmd($vmid, "device_add" , %options);
3337 }
3338
3339 sub qemu_devicedel {
3340     my ($vmid, $deviceid) = @_;
3341
3342     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3343 }
3344
3345 sub qemu_iothread_add {
3346     my($vmid, $deviceid, $device) = @_;
3347
3348     if ($device->{iothread}) {
3349         my $iothreads = vm_iothreads_list($vmid);
3350         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3351     }
3352 }
3353
3354 sub qemu_iothread_del {
3355     my($conf, $vmid, $deviceid) = @_;
3356
3357     my $device = parse_drive($deviceid, $conf->{$deviceid});
3358     if ($device->{iothread}) {
3359         my $iothreads = vm_iothreads_list($vmid);
3360         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3361     }
3362 }
3363
3364 sub qemu_objectadd {
3365     my($vmid, $objectid, $qomtype) = @_;
3366
3367     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3368
3369     return 1;
3370 }
3371
3372 sub qemu_objectdel {
3373     my($vmid, $objectid) = @_;
3374
3375     vm_mon_cmd($vmid, "object-del", id => $objectid);
3376
3377     return 1;
3378 }
3379
3380 sub qemu_driveadd {
3381     my ($storecfg, $vmid, $device) = @_;
3382
3383     my $drive = print_drive_full($storecfg, $vmid, $device);
3384     $drive =~ s/\\/\\\\/g;
3385     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3386
3387     # If the command succeeds qemu prints: "OK"
3388     return 1 if $ret =~ m/OK/s;
3389
3390     die "adding drive failed: $ret\n";
3391 }
3392
3393 sub qemu_drivedel {
3394     my($vmid, $deviceid) = @_;
3395
3396     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3397     $ret =~ s/^\s+//;
3398
3399     return 1 if $ret eq "";
3400
3401     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3402     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3403
3404     die "deleting drive $deviceid failed : $ret\n";
3405 }
3406
3407 sub qemu_deviceaddverify {
3408     my ($vmid, $deviceid) = @_;
3409
3410     for (my $i = 0; $i <= 5; $i++) {
3411          my $devices_list = vm_devices_list($vmid);
3412          return 1 if defined($devices_list->{$deviceid});
3413          sleep 1;
3414     }
3415
3416     die "error on hotplug device '$deviceid'\n";
3417 }
3418
3419
3420 sub qemu_devicedelverify {
3421     my ($vmid, $deviceid) = @_;
3422
3423     # need to verify that the device is correctly removed as device_del
3424     # is async and empty return is not reliable
3425
3426     for (my $i = 0; $i <= 5; $i++) {
3427          my $devices_list = vm_devices_list($vmid);
3428          return 1 if !defined($devices_list->{$deviceid});
3429          sleep 1;
3430     }
3431
3432     die "error on hot-unplugging device '$deviceid'\n";
3433 }
3434
3435 sub qemu_findorcreatescsihw {
3436     my ($storecfg, $conf, $vmid, $device) = @_;
3437
3438     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3439
3440     my $scsihwid="$controller_prefix$controller";
3441     my $devices_list = vm_devices_list($vmid);
3442
3443     if(!defined($devices_list->{$scsihwid})) {
3444         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3445     }
3446
3447     return 1;
3448 }
3449
3450 sub qemu_deletescsihw {
3451     my ($conf, $vmid, $opt) = @_;
3452
3453     my $device = parse_drive($opt, $conf->{$opt});
3454
3455     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3456         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3457         return 1;
3458     }
3459
3460     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3461
3462     my $devices_list = vm_devices_list($vmid);
3463     foreach my $opt (keys %{$devices_list}) {
3464         if (PVE::QemuServer::valid_drivename($opt)) {
3465             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3466             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3467                 return 1;
3468             }
3469         }
3470     }
3471
3472     my $scsihwid="scsihw$controller";
3473
3474     vm_deviceunplug($vmid, $conf, $scsihwid);
3475
3476     return 1;
3477 }
3478
3479 sub qemu_add_pci_bridge {
3480     my ($storecfg, $conf, $vmid, $device) = @_;
3481
3482     my $bridges = {};
3483
3484     my $bridgeid;
3485
3486     print_pci_addr($device, $bridges);
3487
3488     while (my ($k, $v) = each %$bridges) {
3489         $bridgeid = $k;
3490     }
3491     return 1 if !defined($bridgeid) || $bridgeid < 1;
3492
3493     my $bridge = "pci.$bridgeid";
3494     my $devices_list = vm_devices_list($vmid);
3495
3496     if (!defined($devices_list->{$bridge})) {
3497         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3498     }
3499
3500     return 1;
3501 }
3502
3503 sub qemu_set_link_status {
3504     my ($vmid, $device, $up) = @_;
3505
3506     vm_mon_cmd($vmid, "set_link", name => $device,
3507                up => $up ? JSON::true : JSON::false);
3508 }
3509
3510 sub qemu_netdevadd {
3511     my ($vmid, $conf, $device, $deviceid) = @_;
3512
3513     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid);
3514     my %options =  split(/[=,]/, $netdev);
3515
3516     vm_mon_cmd($vmid, "netdev_add",  %options);
3517     return 1;
3518 }
3519
3520 sub qemu_netdevdel {
3521     my ($vmid, $deviceid) = @_;
3522
3523     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3524 }
3525
3526 sub qemu_cpu_hotplug {
3527     my ($vmid, $conf, $vcpus) = @_;
3528
3529     my $sockets = 1;
3530     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3531     $sockets = $conf->{sockets} if  $conf->{sockets};
3532     my $cores = $conf->{cores} || 1;
3533     my $maxcpus = $sockets * $cores;
3534
3535     $vcpus = $maxcpus if !$vcpus;
3536
3537     die "you can't add more vcpus than maxcpus\n"
3538         if $vcpus > $maxcpus;
3539
3540     my $currentvcpus = $conf->{vcpus} || $maxcpus;
3541     die "online cpu unplug is not yet possible\n"
3542         if $vcpus < $currentvcpus;
3543
3544     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3545     die "vcpus in running vm is different than configuration\n"
3546         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
3547
3548     for (my $i = $currentvcpus; $i < $vcpus; $i++) {
3549         vm_mon_cmd($vmid, "cpu-add", id => int($i));
3550     }
3551 }
3552
3553 sub qemu_memory_hotplug {
3554     my ($vmid, $conf, $defaults, $opt, $value) = @_;
3555
3556     return $value if !check_running($vmid);
3557
3558     my $memory = $conf->{memory} || $defaults->{memory};
3559     $value = $defaults->{memory} if !$value;
3560     return $value if $value == $memory;
3561
3562     my $static_memory = $STATICMEM;
3563     my $dimm_memory = $memory - $static_memory;
3564
3565     die "memory can't be lower than $static_memory MB" if $value < $static_memory;
3566     die "memory unplug is not yet available" if $value < $memory;
3567     die "you cannot add more memory than $MAX_MEM MB!\n" if $memory > $MAX_MEM;
3568
3569
3570     my $sockets = 1;
3571     $sockets = $conf->{sockets} if $conf->{sockets};
3572
3573     foreach_dimm($conf, $vmid, $value, $sockets, sub {
3574         my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3575
3576             return if $current_size <= $conf->{memory};
3577
3578             eval { vm_mon_cmd($vmid, "object-add", 'qom-type' => "memory-backend-ram", id => "mem-$name", props => { size => int($dimm_size*1024*1024) } ) };
3579             if (my $err = $@) {
3580                 eval { qemu_objectdel($vmid, "mem-$name"); };
3581                 die $err;
3582             }
3583
3584             eval { vm_mon_cmd($vmid, "device_add", driver => "pc-dimm", id => "$name", memdev => "mem-$name", node => $numanode) };
3585             if (my $err = $@) {
3586                 eval { qemu_objectdel($vmid, "mem-$name"); };
3587                 die $err;
3588             }
3589             #update conf after each succesful module hotplug
3590             $conf->{memory} = $current_size;
3591             update_config_nolock($vmid, $conf, 1);
3592     });
3593 }
3594
3595 sub qemu_block_set_io_throttle {
3596     my ($vmid, $deviceid, $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr) = @_;
3597
3598     return if !check_running($vmid) ;
3599
3600     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid, bps => int($bps), bps_rd => int($bps_rd), bps_wr => int($bps_wr), iops => int($iops), iops_rd => int($iops_rd), iops_wr => int($iops_wr));
3601
3602 }
3603
3604 # old code, only used to shutdown old VM after update
3605 sub __read_avail {
3606     my ($fh, $timeout) = @_;
3607
3608     my $sel = new IO::Select;
3609     $sel->add($fh);
3610
3611     my $res = '';
3612     my $buf;
3613
3614     my @ready;
3615     while (scalar (@ready = $sel->can_read($timeout))) {
3616         my $count;
3617         if ($count = $fh->sysread($buf, 8192)) {
3618             if ($buf =~ /^(.*)\(qemu\) $/s) {
3619                 $res .= $1;
3620                 last;
3621             } else {
3622                 $res .= $buf;
3623             }
3624         } else {
3625             if (!defined($count)) {
3626                 die "$!\n";
3627             }
3628             last;
3629         }
3630     }
3631
3632     die "monitor read timeout\n" if !scalar(@ready);
3633
3634     return $res;
3635 }
3636
3637 # old code, only used to shutdown old VM after update
3638 sub vm_monitor_command {
3639     my ($vmid, $cmdstr, $nocheck) = @_;
3640
3641     my $res;
3642
3643     eval {
3644         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
3645
3646         my $sname = "${var_run_tmpdir}/$vmid.mon";
3647
3648         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
3649             die "unable to connect to VM $vmid socket - $!\n";
3650
3651         my $timeout = 3;
3652
3653         # hack: migrate sometime blocks the monitor (when migrate_downtime
3654         # is set)
3655         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3656             $timeout = 60*60; # 1 hour
3657         }
3658
3659         # read banner;
3660         my $data = __read_avail($sock, $timeout);
3661
3662         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
3663             die "got unexpected qemu monitor banner\n";
3664         }
3665
3666         my $sel = new IO::Select;
3667         $sel->add($sock);
3668
3669         if (!scalar(my @ready = $sel->can_write($timeout))) {
3670             die "monitor write error - timeout";
3671         }
3672
3673         my $fullcmd = "$cmdstr\r";
3674
3675         # syslog('info', "VM $vmid monitor command: $cmdstr");
3676
3677         my $b;
3678         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
3679             die "monitor write error - $!";
3680         }
3681
3682         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
3683
3684         $timeout = 20;
3685
3686         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3687             $timeout = 60*60; # 1 hour
3688         } elsif ($cmdstr =~ m/^(eject|change)/) {
3689             $timeout = 60; # note: cdrom mount command is slow
3690         }
3691         if ($res = __read_avail($sock, $timeout)) {
3692
3693             my @lines = split("\r?\n", $res);
3694
3695             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
3696
3697             $res = join("\n", @lines);
3698             $res .= "\n";
3699         }
3700     };
3701
3702     my $err = $@;
3703
3704     if ($err) {
3705         syslog("err", "VM $vmid monitor command failed - $err");
3706         die $err;
3707     }
3708
3709     return $res;
3710 }
3711
3712 sub qemu_block_resize {
3713     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
3714
3715     my $running = check_running($vmid);
3716
3717     return if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
3718
3719     return if !$running;
3720
3721     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
3722
3723 }
3724
3725 sub qemu_volume_snapshot {
3726     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3727
3728     my $running = check_running($vmid);
3729
3730     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
3731         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
3732     } else {
3733         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
3734     }
3735 }
3736
3737 sub qemu_volume_snapshot_delete {
3738     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3739
3740     my $running = check_running($vmid);
3741
3742     return if !PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
3743
3744     return if !$running;
3745
3746     vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
3747 }
3748
3749 sub set_migration_caps {
3750     my ($vmid) = @_;
3751
3752     my $cap_ref = [];
3753
3754     my $enabled_cap = {
3755         "auto-converge" => 1,
3756         "xbzrle" => 0,
3757         "x-rdma-pin-all" => 0,
3758         "zero-blocks" => 0,
3759     };
3760
3761     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
3762
3763     for my $supported_capability (@$supported_capabilities) {
3764         push @$cap_ref, {
3765             capability => $supported_capability->{capability},
3766             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
3767         };
3768     }
3769
3770     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
3771 }
3772
3773 my $fast_plug_option = {
3774     'lock' => 1,
3775     'name' => 1,
3776     'onboot' => 1,
3777     'shares' => 1,
3778     'startup' => 1,
3779 };
3780
3781 # hotplug changes in [PENDING]
3782 # $selection hash can be used to only apply specified options, for
3783 # example: { cores => 1 } (only apply changed 'cores')
3784 # $errors ref is used to return error messages
3785 sub vmconfig_hotplug_pending {
3786     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
3787
3788     my $defaults = load_defaults();
3789
3790     # commit values which do not have any impact on running VM first
3791     # Note: those option cannot raise errors, we we do not care about
3792     # $selection and always apply them.
3793
3794     my $add_error = sub {
3795         my ($opt, $msg) = @_;
3796         $errors->{$opt} = "hotplug problem - $msg";
3797     };
3798
3799     my $changes = 0;
3800     foreach my $opt (keys %{$conf->{pending}}) { # add/change
3801         if ($fast_plug_option->{$opt}) {
3802             $conf->{$opt} = $conf->{pending}->{$opt};
3803             delete $conf->{pending}->{$opt};
3804             $changes = 1;
3805         }
3806     }
3807
3808     if ($changes) {
3809         update_config_nolock($vmid, $conf, 1);
3810         $conf = load_config($vmid); # update/reload
3811     }
3812
3813     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3814
3815     my @delete = PVE::Tools::split_list($conf->{pending}->{delete});
3816     foreach my $opt (@delete) {
3817         next if $selection && !$selection->{$opt};
3818         eval {
3819             if ($opt eq 'hotplug') {
3820                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
3821             } elsif ($opt eq 'tablet') {
3822                 die "skip\n" if !$hotplug_features->{usb};
3823                 if ($defaults->{tablet}) {
3824                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
3825                 } else {
3826                     vm_deviceunplug($vmid, $conf, $opt);
3827                 }
3828             } elsif ($opt eq 'vcpus') {
3829                 die "skip\n" if !$hotplug_features->{cpu};
3830                 qemu_cpu_hotplug($vmid, $conf, undef);
3831             } elsif ($opt eq 'balloon') {
3832                 # enable balloon device is not hotpluggable
3833                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
3834             } elsif ($fast_plug_option->{$opt}) {
3835                 # do nothing
3836             } elsif ($opt =~ m/^net(\d+)$/) {
3837                 die "skip\n" if !$hotplug_features->{network};
3838                 vm_deviceunplug($vmid, $conf, $opt);
3839             } elsif (valid_drivename($opt)) {
3840                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
3841                 vm_deviceunplug($vmid, $conf, $opt);
3842                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}));
3843             } elsif ($opt =~ m/^memory$/) {
3844                 die "skip\n" if !$hotplug_features->{memory};
3845                 qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
3846             } elsif ($opt eq 'cpuunits') {
3847                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
3848             } elsif ($opt eq 'cpulimit') {
3849                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
3850             } else {
3851                 die "skip\n";
3852             }
3853         };
3854         if (my $err = $@) {
3855             &$add_error($opt, $err) if $err ne "skip\n";
3856         } else {
3857             # save new config if hotplug was successful
3858             delete $conf->{$opt};
3859             vmconfig_undelete_pending_option($conf, $opt);
3860             update_config_nolock($vmid, $conf, 1);
3861             $conf = load_config($vmid); # update/reload
3862         }
3863     }
3864
3865     foreach my $opt (keys %{$conf->{pending}}) {
3866         next if $selection && !$selection->{$opt};
3867         my $value = $conf->{pending}->{$opt};
3868         eval {
3869             if ($opt eq 'hotplug') {
3870                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
3871             } elsif ($opt eq 'tablet') {
3872                 die "skip\n" if !$hotplug_features->{usb};
3873                 if ($value == 1) {
3874                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
3875                 } elsif ($value == 0) {
3876                     vm_deviceunplug($vmid, $conf, $opt);
3877                 }
3878             } elsif ($opt eq 'vcpus') {
3879                 die "skip\n" if !$hotplug_features->{cpu};
3880                 qemu_cpu_hotplug($vmid, $conf, $value);
3881             } elsif ($opt eq 'balloon') {
3882                 # enable/disable balloning device is not hotpluggable
3883                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
3884                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
3885                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
3886
3887                 # allow manual ballooning if shares is set to zero
3888                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
3889                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
3890                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
3891                 }
3892             } elsif ($opt =~ m/^net(\d+)$/) {
3893                 # some changes can be done without hotplug
3894                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
3895                                     $vmid, $opt, $value);
3896             } elsif (valid_drivename($opt)) {
3897                 # some changes can be done without hotplug
3898                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
3899                                      $vmid, $opt, $value, 1);
3900             } elsif ($opt =~ m/^memory$/) { #dimms
3901                 die "skip\n" if !$hotplug_features->{memory};
3902                 $value = qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
3903             } elsif ($opt eq 'cpuunits') {
3904                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
3905             } elsif ($opt eq 'cpulimit') {
3906                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
3907                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
3908             } else {
3909                 die "skip\n";  # skip non-hot-pluggable options
3910             }
3911         };
3912         if (my $err = $@) {
3913             &$add_error($opt, $err) if $err ne "skip\n";
3914         } else {
3915             # save new config if hotplug was successful
3916             $conf->{$opt} = $value;
3917             delete $conf->{pending}->{$opt};
3918             update_config_nolock($vmid, $conf, 1);
3919             $conf = load_config($vmid); # update/reload
3920         }
3921     }
3922 }
3923
3924 sub vmconfig_apply_pending {
3925     my ($vmid, $conf, $storecfg) = @_;
3926
3927     # cold plug
3928
3929     my @delete = PVE::Tools::split_list($conf->{pending}->{delete});
3930     foreach my $opt (@delete) { # delete
3931         die "internal error" if $opt =~ m/^unused/;
3932         $conf = load_config($vmid); # update/reload
3933         if (!defined($conf->{$opt})) {
3934             vmconfig_undelete_pending_option($conf, $opt);
3935             update_config_nolock($vmid, $conf, 1);
3936         } elsif (valid_drivename($opt)) {
3937             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}));
3938             vmconfig_undelete_pending_option($conf, $opt);
3939             delete $conf->{$opt};
3940             update_config_nolock($vmid, $conf, 1);
3941         } else {
3942             vmconfig_undelete_pending_option($conf, $opt);
3943             delete $conf->{$opt};
3944             update_config_nolock($vmid, $conf, 1);
3945         }
3946     }
3947
3948     $conf = load_config($vmid); # update/reload
3949
3950     foreach my $opt (keys %{$conf->{pending}}) { # add/change
3951         $conf = load_config($vmid); # update/reload
3952
3953         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
3954             # skip if nothing changed
3955         } elsif (valid_drivename($opt)) {
3956             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
3957                 if defined($conf->{$opt});
3958             $conf->{$opt} = $conf->{pending}->{$opt};
3959         } else {
3960             $conf->{$opt} = $conf->{pending}->{$opt};
3961         }
3962
3963         delete $conf->{pending}->{$opt};
3964         update_config_nolock($vmid, $conf, 1);
3965     }
3966 }
3967
3968 my $safe_num_ne = sub {
3969     my ($a, $b) = @_;
3970
3971     return 0 if !defined($a) && !defined($b);
3972     return 1 if !defined($a);
3973     return 1 if !defined($b);
3974
3975     return $a != $b;
3976 };
3977
3978 my $safe_string_ne = sub {
3979     my ($a, $b) = @_;
3980
3981     return 0 if !defined($a) && !defined($b);
3982     return 1 if !defined($a);
3983     return 1 if !defined($b);
3984
3985     return $a ne $b;
3986 };
3987
3988 sub vmconfig_update_net {
3989     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
3990
3991     my $newnet = parse_net($value);
3992
3993     if ($conf->{$opt}) {
3994         my $oldnet = parse_net($conf->{$opt});
3995
3996         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
3997             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
3998             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
3999             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4000
4001             # for non online change, we try to hot-unplug
4002             die "skip\n" if !$hotplug;
4003             vm_deviceunplug($vmid, $conf, $opt);
4004         } else {
4005
4006             die "internal error" if $opt !~ m/net(\d+)/;
4007             my $iface = "tap${vmid}i$1";
4008
4009             if (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4010                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4011             }
4012
4013             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4014                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4015                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4016                 PVE::Network::tap_unplug($iface);
4017                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall});
4018             }
4019
4020             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4021                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4022             }
4023
4024             return 1;
4025         }
4026     }
4027
4028     if ($hotplug) {
4029         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4030     } else {
4031         die "skip\n";
4032     }
4033 }
4034
4035 sub vmconfig_update_disk {
4036     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4037
4038     # fixme: do we need force?
4039
4040     my $drive = parse_drive($opt, $value);
4041
4042     if ($conf->{$opt}) {
4043
4044         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4045
4046             my $media = $drive->{media} || 'disk';
4047             my $oldmedia = $old_drive->{media} || 'disk';
4048             die "unable to change media type\n" if $media ne $oldmedia;
4049
4050             if (!drive_is_cdrom($old_drive)) {
4051
4052                 if ($drive->{file} ne $old_drive->{file}) {
4053
4054                     die "skip\n" if !$hotplug;
4055
4056                     # unplug and register as unused
4057                     vm_deviceunplug($vmid, $conf, $opt);
4058                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4059
4060                 } else {
4061                     # update existing disk
4062
4063                     # skip non hotpluggable value
4064                     if (&$safe_num_ne($drive->{discard}, $old_drive->{discard}) ||
4065                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4066                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4067                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4068                         die "skip\n";
4069                     }
4070
4071                     # apply throttle
4072                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4073                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4074                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4075                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4076                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4077                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4078                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4079                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4080                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4081                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4082                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4083                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max})) {
4084
4085                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4086                                                    ($drive->{mbps} || 0)*1024*1024,
4087                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4088                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4089                                                    $drive->{iops} || 0,
4090                                                    $drive->{iops_rd} || 0,
4091                                                    $drive->{iops_wr} || 0,
4092                                                    ($drive->{mbps_max} || 0)*1024*1024,
4093                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4094                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4095                                                    $drive->{iops_max} || 0,
4096                                                    $drive->{iops_rd_max} || 0,
4097                                                    $drive->{iops_wr_max} || 0);
4098
4099                     }
4100
4101                     return 1;
4102                 }
4103
4104             } else { # cdrom
4105
4106                 if ($drive->{file} eq 'none') {
4107                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4108                 } else {
4109                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4110                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4111                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4112                 }
4113
4114                 return 1;
4115             }
4116         }
4117     }
4118
4119     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4120     # hotplug new disks
4121     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4122 }
4123
4124 sub vm_start {
4125     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused, $forcemachine, $spice_ticket) = @_;
4126
4127     lock_config($vmid, sub {
4128         my $conf = load_config($vmid, $migratedfrom);
4129
4130         die "you can't start a vm if it's a template\n" if is_template($conf);
4131
4132         check_lock($conf) if !$skiplock;
4133
4134         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4135
4136         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4137             vmconfig_apply_pending($vmid, $conf, $storecfg);
4138             $conf = load_config($vmid); # update/reload
4139         }
4140
4141         my $defaults = load_defaults();
4142
4143         # set environment variable useful inside network script
4144         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4145
4146         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4147
4148         my $migrate_port = 0;
4149         my $migrate_uri;
4150         if ($statefile) {
4151             if ($statefile eq 'tcp') {
4152                 my $localip = "localhost";
4153                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4154                 my $nodename = PVE::INotify::nodename();
4155                 if ($datacenterconf->{migration_unsecure}) {
4156                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4157                 }
4158                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4159                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4160                 $migrate_uri = "tcp:[${localip}]:${migrate_port}";
4161                 push @$cmd, '-incoming', $migrate_uri;
4162                 push @$cmd, '-S';
4163             } else {
4164                 push @$cmd, '-loadstate', $statefile;
4165             }
4166         } elsif ($paused) {
4167             push @$cmd, '-S';
4168         }
4169
4170         # host pci devices
4171         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4172           my $d = parse_hostpci($conf->{"hostpci$i"});
4173           next if !$d;
4174           my $pcidevices = $d->{pciid};
4175           foreach my $pcidevice (@$pcidevices) {
4176                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4177
4178                 my $info = pci_device_info("0000:$pciid");
4179                 die "IOMMU not present\n" if !check_iommu_support();
4180                 die "no pci device info for device '$pciid'\n" if !$info;
4181
4182                 if ($d->{driver} && $d->{driver} eq "vfio") {
4183                     die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4184                 } else {
4185                     die "can't unbind/bind to stub pci device '$pciid'\n" if !pci_dev_bind_to_stub($info);
4186                 }
4187
4188                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4189           }
4190         }
4191
4192         PVE::Storage::activate_volumes($storecfg, $vollist);
4193
4194         eval  { run_command($cmd, timeout => $statefile ? undef : 30,
4195                     umask => 0077); };
4196         my $err = $@;
4197         die "start failed: $err" if $err;
4198
4199         print "migration listens on $migrate_uri\n" if $migrate_uri;
4200
4201         if ($statefile && $statefile ne 'tcp')  {
4202             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4203             warn $@ if $@;
4204         }
4205
4206         if ($migratedfrom) {
4207
4208             eval {
4209                 set_migration_caps($vmid);
4210             };
4211             warn $@ if $@;
4212
4213             if ($spice_port) {
4214                 print "spice listens on port $spice_port\n";
4215                 if ($spice_ticket) {
4216                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4217                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4218                 }
4219             }
4220
4221         } else {
4222
4223             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4224                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4225                     if $conf->{balloon};
4226             }
4227
4228             foreach my $opt (keys %$conf) {
4229                 next if $opt !~  m/^net\d+$/;
4230                 my $nicconf = parse_net($conf->{$opt});
4231                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4232             }
4233         }
4234
4235         vm_mon_cmd_nocheck($vmid, 'qom-set',
4236                     path => "machine/peripheral/balloon0",
4237                     property => "guest-stats-polling-interval",
4238                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4239
4240     });
4241 }
4242
4243 sub vm_mon_cmd {
4244     my ($vmid, $execute, %params) = @_;
4245
4246     my $cmd = { execute => $execute, arguments => \%params };
4247     vm_qmp_command($vmid, $cmd);
4248 }
4249
4250 sub vm_mon_cmd_nocheck {
4251     my ($vmid, $execute, %params) = @_;
4252
4253     my $cmd = { execute => $execute, arguments => \%params };
4254     vm_qmp_command($vmid, $cmd, 1);
4255 }
4256
4257 sub vm_qmp_command {
4258     my ($vmid, $cmd, $nocheck) = @_;
4259
4260     my $res;
4261
4262     my $timeout;
4263     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4264         $timeout = $cmd->{arguments}->{timeout};
4265         delete $cmd->{arguments}->{timeout};
4266     }
4267
4268     eval {
4269         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4270         my $sname = qmp_socket($vmid);
4271         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4272             my $qmpclient = PVE::QMPClient->new();
4273
4274             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4275         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
4276             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
4277                 if scalar(%{$cmd->{arguments}});
4278             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
4279         } else {
4280             die "unable to open monitor socket\n";
4281         }
4282     };
4283     if (my $err = $@) {
4284         syslog("err", "VM $vmid qmp command failed - $err");
4285         die $err;
4286     }
4287
4288     return $res;
4289 }
4290
4291 sub vm_human_monitor_command {
4292     my ($vmid, $cmdline) = @_;
4293
4294     my $res;
4295
4296     my $cmd = {
4297         execute => 'human-monitor-command',
4298         arguments => { 'command-line' => $cmdline},
4299     };
4300
4301     return vm_qmp_command($vmid, $cmd);
4302 }
4303
4304 sub vm_commandline {
4305     my ($storecfg, $vmid) = @_;
4306
4307     my $conf = load_config($vmid);
4308
4309     my $defaults = load_defaults();
4310
4311     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
4312
4313     return join(' ', @$cmd);
4314 }
4315
4316 sub vm_reset {
4317     my ($vmid, $skiplock) = @_;
4318
4319     lock_config($vmid, sub {
4320
4321         my $conf = load_config($vmid);
4322
4323         check_lock($conf) if !$skiplock;
4324
4325         vm_mon_cmd($vmid, "system_reset");
4326     });
4327 }
4328
4329 sub get_vm_volumes {
4330     my ($conf) = @_;
4331
4332     my $vollist = [];
4333     foreach_volid($conf, sub {
4334         my ($volid, $is_cdrom) = @_;
4335
4336         return if $volid =~ m|^/|;
4337
4338         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4339         return if !$sid;
4340
4341         push @$vollist, $volid;
4342     });
4343
4344     return $vollist;
4345 }
4346
4347 sub vm_stop_cleanup {
4348     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
4349
4350     eval {
4351
4352         if (!$keepActive) {
4353             my $vollist = get_vm_volumes($conf);
4354             PVE::Storage::deactivate_volumes($storecfg, $vollist);
4355         }
4356
4357         foreach my $ext (qw(mon qmp pid vnc qga)) {
4358             unlink "/var/run/qemu-server/${vmid}.$ext";
4359         }
4360
4361         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
4362     };
4363     warn $@ if $@; # avoid errors - just warn
4364 }
4365
4366 # Note: use $nockeck to skip tests if VM configuration file exists.
4367 # We need that when migration VMs to other nodes (files already moved)
4368 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
4369 sub vm_stop {
4370     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
4371
4372     $force = 1 if !defined($force) && !$shutdown;
4373
4374     if ($migratedfrom){
4375         my $pid = check_running($vmid, $nocheck, $migratedfrom);
4376         kill 15, $pid if $pid;
4377         my $conf = load_config($vmid, $migratedfrom);
4378         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
4379         return;
4380     }
4381
4382     lock_config($vmid, sub {
4383
4384         my $pid = check_running($vmid, $nocheck);
4385         return if !$pid;
4386
4387         my $conf;
4388         if (!$nocheck) {
4389             $conf = load_config($vmid);
4390             check_lock($conf) if !$skiplock;
4391             if (!defined($timeout) && $shutdown && $conf->{startup}) {
4392                 my $opts = parse_startup($conf->{startup});
4393                 $timeout = $opts->{down} if $opts->{down};
4394             }
4395         }
4396
4397         $timeout = 60 if !defined($timeout);
4398
4399         eval {
4400             if ($shutdown) {
4401                 if (defined($conf) && $conf->{agent}) {
4402                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
4403                 } else {
4404                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
4405                 }
4406             } else {
4407                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
4408             }
4409         };
4410         my $err = $@;
4411
4412         if (!$err) {
4413             my $count = 0;
4414             while (($count < $timeout) && check_running($vmid, $nocheck)) {
4415                 $count++;
4416                 sleep 1;
4417             }
4418
4419             if ($count >= $timeout) {
4420                 if ($force) {
4421                     warn "VM still running - terminating now with SIGTERM\n";
4422                     kill 15, $pid;
4423                 } else {
4424                     die "VM quit/powerdown failed - got timeout\n";
4425                 }
4426             } else {
4427                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4428                 return;
4429             }
4430         } else {
4431             if ($force) {
4432                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
4433                 kill 15, $pid;
4434             } else {
4435                 die "VM quit/powerdown failed\n";
4436             }
4437         }
4438
4439         # wait again
4440         $timeout = 10;
4441
4442         my $count = 0;
4443         while (($count < $timeout) && check_running($vmid, $nocheck)) {
4444             $count++;
4445             sleep 1;
4446         }
4447
4448         if ($count >= $timeout) {
4449             warn "VM still running - terminating now with SIGKILL\n";
4450             kill 9, $pid;
4451             sleep 1;
4452         }
4453
4454         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4455    });
4456 }
4457
4458 sub vm_suspend {
4459     my ($vmid, $skiplock) = @_;
4460
4461     lock_config($vmid, sub {
4462
4463         my $conf = load_config($vmid);
4464
4465         check_lock($conf) if !($skiplock || ($conf->{lock} && $conf->{lock} eq 'backup'));
4466
4467         vm_mon_cmd($vmid, "stop");
4468     });
4469 }
4470
4471 sub vm_resume {
4472     my ($vmid, $skiplock) = @_;
4473
4474     lock_config($vmid, sub {
4475
4476         my $conf = load_config($vmid);
4477
4478         check_lock($conf) if !($skiplock || ($conf->{lock} && $conf->{lock} eq 'backup'));
4479
4480         vm_mon_cmd($vmid, "cont");
4481     });
4482 }
4483
4484 sub vm_sendkey {
4485     my ($vmid, $skiplock, $key) = @_;
4486
4487     lock_config($vmid, sub {
4488
4489         my $conf = load_config($vmid);
4490
4491         # there is no qmp command, so we use the human monitor command
4492         vm_human_monitor_command($vmid, "sendkey $key");
4493     });
4494 }
4495
4496 sub vm_destroy {
4497     my ($storecfg, $vmid, $skiplock) = @_;
4498
4499     lock_config($vmid, sub {
4500
4501         my $conf = load_config($vmid);
4502
4503         check_lock($conf) if !$skiplock;
4504
4505         if (!check_running($vmid)) {
4506             destroy_vm($storecfg, $vmid);
4507         } else {
4508             die "VM $vmid is running - destroy failed\n";
4509         }
4510     });
4511 }
4512
4513 # pci helpers
4514
4515 sub file_write {
4516     my ($filename, $buf) = @_;
4517
4518     my $fh = IO::File->new($filename, "w");
4519     return undef if !$fh;
4520
4521     my $res = print $fh $buf;
4522
4523     $fh->close();
4524
4525     return $res;
4526 }
4527
4528 sub pci_device_info {
4529     my ($name) = @_;
4530
4531     my $res;
4532
4533     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
4534     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
4535
4536     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
4537     return undef if !defined($irq) || $irq !~ m/^\d+$/;
4538
4539     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
4540     return undef if !defined($vendor) || $vendor !~ s/^0x//;
4541
4542     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
4543     return undef if !defined($product) || $product !~ s/^0x//;
4544
4545     $res = {
4546         name => $name,
4547         vendor => $vendor,
4548         product => $product,
4549         domain => $domain,
4550         bus => $bus,
4551         slot => $slot,
4552         func => $func,
4553         irq => $irq,
4554         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
4555     };
4556
4557     return $res;
4558 }
4559
4560 sub pci_dev_reset {
4561     my ($dev) = @_;
4562
4563     my $name = $dev->{name};
4564
4565     my $fn = "$pcisysfs/devices/$name/reset";
4566
4567     return file_write($fn, "1");
4568 }
4569
4570 sub pci_dev_bind_to_stub {
4571     my ($dev) = @_;
4572
4573     my $name = $dev->{name};
4574
4575     my $testdir = "$pcisysfs/drivers/pci-stub/$name";
4576     return 1 if -d $testdir;
4577
4578     my $data = "$dev->{vendor} $dev->{product}";
4579     return undef if !file_write("$pcisysfs/drivers/pci-stub/new_id", $data);
4580
4581     my $fn = "$pcisysfs/devices/$name/driver/unbind";
4582     if (!file_write($fn, $name)) {
4583         return undef if -f $fn;
4584     }
4585
4586     $fn = "$pcisysfs/drivers/pci-stub/bind";
4587     if (! -d $testdir) {
4588         return undef if !file_write($fn, $name);
4589     }
4590
4591     return -d $testdir;
4592 }
4593
4594 sub pci_dev_bind_to_vfio {
4595     my ($dev) = @_;
4596
4597     my $name = $dev->{name};
4598
4599     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4600
4601     if (!-d $vfio_basedir) {
4602         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4603     }
4604     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4605
4606     my $testdir = "$vfio_basedir/$name";
4607     return 1 if -d $testdir;
4608
4609     my $data = "$dev->{vendor} $dev->{product}";
4610     return undef if !file_write("$vfio_basedir/new_id", $data);
4611
4612     my $fn = "$pcisysfs/devices/$name/driver/unbind";
4613     if (!file_write($fn, $name)) {
4614         return undef if -f $fn;
4615     }
4616
4617     $fn = "$vfio_basedir/bind";
4618     if (! -d $testdir) {
4619         return undef if !file_write($fn, $name);
4620     }
4621
4622     return -d $testdir;
4623 }
4624
4625 sub pci_dev_group_bind_to_vfio {
4626     my ($pciid) = @_;
4627
4628     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4629
4630     if (!-d $vfio_basedir) {
4631         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4632     }
4633     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4634
4635     # get IOMMU group devices
4636     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
4637       my @devs = grep /^0000:/, readdir($D);
4638     closedir($D);
4639
4640     foreach my $pciid (@devs) {
4641         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
4642
4643         # pci bridges, switches or root ports are not supported
4644         # they have a pci_bus subdirectory so skip them
4645         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
4646
4647         my $info = pci_device_info($1);
4648         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
4649     }
4650
4651     return 1;
4652 }
4653
4654 sub print_pci_addr {
4655     my ($id, $bridges) = @_;
4656
4657     my $res = '';
4658     my $devices = {
4659         piix3 => { bus => 0, addr => 1 },
4660         #addr2 : first videocard
4661         balloon0 => { bus => 0, addr => 3 },
4662         watchdog => { bus => 0, addr => 4 },
4663         scsihw0 => { bus => 0, addr => 5 },
4664         'pci.3' => { bus => 0, addr => 5 }, #can also be used for virtio-scsi-single bridge
4665         scsihw1 => { bus => 0, addr => 6 },
4666         ahci0 => { bus => 0, addr => 7 },
4667         qga0 => { bus => 0, addr => 8 },
4668         spice => { bus => 0, addr => 9 },
4669         virtio0 => { bus => 0, addr => 10 },
4670         virtio1 => { bus => 0, addr => 11 },
4671         virtio2 => { bus => 0, addr => 12 },
4672         virtio3 => { bus => 0, addr => 13 },
4673         virtio4 => { bus => 0, addr => 14 },
4674         virtio5 => { bus => 0, addr => 15 },
4675         hostpci0 => { bus => 0, addr => 16 },
4676         hostpci1 => { bus => 0, addr => 17 },
4677         net0 => { bus => 0, addr => 18 },
4678         net1 => { bus => 0, addr => 19 },
4679         net2 => { bus => 0, addr => 20 },
4680         net3 => { bus => 0, addr => 21 },
4681         net4 => { bus => 0, addr => 22 },
4682         net5 => { bus => 0, addr => 23 },
4683         vga1 => { bus => 0, addr => 24 },
4684         vga2 => { bus => 0, addr => 25 },
4685         vga3 => { bus => 0, addr => 26 },
4686         hostpci2 => { bus => 0, addr => 27 },
4687         hostpci3 => { bus => 0, addr => 28 },
4688         #addr29 : usb-host (pve-usb.cfg)
4689         'pci.1' => { bus => 0, addr => 30 },
4690         'pci.2' => { bus => 0, addr => 31 },
4691         'net6' => { bus => 1, addr => 1 },
4692         'net7' => { bus => 1, addr => 2 },
4693         'net8' => { bus => 1, addr => 3 },
4694         'net9' => { bus => 1, addr => 4 },
4695         'net10' => { bus => 1, addr => 5 },
4696         'net11' => { bus => 1, addr => 6 },
4697         'net12' => { bus => 1, addr => 7 },
4698         'net13' => { bus => 1, addr => 8 },
4699         'net14' => { bus => 1, addr => 9 },
4700         'net15' => { bus => 1, addr => 10 },
4701         'net16' => { bus => 1, addr => 11 },
4702         'net17' => { bus => 1, addr => 12 },
4703         'net18' => { bus => 1, addr => 13 },
4704         'net19' => { bus => 1, addr => 14 },
4705         'net20' => { bus => 1, addr => 15 },
4706         'net21' => { bus => 1, addr => 16 },
4707         'net22' => { bus => 1, addr => 17 },
4708         'net23' => { bus => 1, addr => 18 },
4709         'net24' => { bus => 1, addr => 19 },
4710         'net25' => { bus => 1, addr => 20 },
4711         'net26' => { bus => 1, addr => 21 },
4712         'net27' => { bus => 1, addr => 22 },
4713         'net28' => { bus => 1, addr => 23 },
4714         'net29' => { bus => 1, addr => 24 },
4715         'net30' => { bus => 1, addr => 25 },
4716         'net31' => { bus => 1, addr => 26 },
4717         'virtio6' => { bus => 2, addr => 1 },
4718         'virtio7' => { bus => 2, addr => 2 },
4719         'virtio8' => { bus => 2, addr => 3 },
4720         'virtio9' => { bus => 2, addr => 4 },
4721         'virtio10' => { bus => 2, addr => 5 },
4722         'virtio11' => { bus => 2, addr => 6 },
4723         'virtio12' => { bus => 2, addr => 7 },
4724         'virtio13' => { bus => 2, addr => 8 },
4725         'virtio14' => { bus => 2, addr => 9 },
4726         'virtio15' => { bus => 2, addr => 10 },
4727         'virtioscsi0' => { bus => 3, addr => 1 },
4728         'virtioscsi1' => { bus => 3, addr => 2 },
4729         'virtioscsi2' => { bus => 3, addr => 3 },
4730         'virtioscsi3' => { bus => 3, addr => 4 },
4731         'virtioscsi4' => { bus => 3, addr => 5 },
4732         'virtioscsi5' => { bus => 3, addr => 6 },
4733         'virtioscsi6' => { bus => 3, addr => 7 },
4734         'virtioscsi7' => { bus => 3, addr => 8 },
4735         'virtioscsi8' => { bus => 3, addr => 9 },
4736         'virtioscsi9' => { bus => 3, addr => 10 },
4737         'virtioscsi10' => { bus => 3, addr => 11 },
4738         'virtioscsi11' => { bus => 3, addr => 12 },
4739         'virtioscsi12' => { bus => 3, addr => 13 },
4740         'virtioscsi13' => { bus => 3, addr => 14 },
4741         'virtioscsi14' => { bus => 3, addr => 15 },
4742         'virtioscsi15' => { bus => 3, addr => 16 },
4743         'virtioscsi16' => { bus => 3, addr => 17 },
4744         'virtioscsi17' => { bus => 3, addr => 18 },
4745         'virtioscsi18' => { bus => 3, addr => 19 },
4746         'virtioscsi19' => { bus => 3, addr => 20 },
4747         'virtioscsi20' => { bus => 3, addr => 21 },
4748         'virtioscsi21' => { bus => 3, addr => 22 },
4749         'virtioscsi22' => { bus => 3, addr => 23 },
4750         'virtioscsi23' => { bus => 3, addr => 24 },
4751         'virtioscsi24' => { bus => 3, addr => 25 },
4752         'virtioscsi25' => { bus => 3, addr => 26 },
4753         'virtioscsi26' => { bus => 3, addr => 27 },
4754         'virtioscsi27' => { bus => 3, addr => 28 },
4755         'virtioscsi28' => { bus => 3, addr => 29 },
4756         'virtioscsi29' => { bus => 3, addr => 30 },
4757         'virtioscsi30' => { bus => 3, addr => 31 },
4758
4759     };
4760
4761     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
4762            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
4763            my $bus = $devices->{$id}->{bus};
4764            $res = ",bus=pci.$bus,addr=$addr";
4765            $bridges->{$bus} = 1 if $bridges;
4766     }
4767     return $res;
4768
4769 }
4770
4771 sub print_pcie_addr {
4772     my ($id) = @_;
4773
4774     my $res = '';
4775     my $devices = {
4776         hostpci0 => { bus => "ich9-pcie-port-1", addr => 0 },
4777         hostpci1 => { bus => "ich9-pcie-port-2", addr => 0 },
4778         hostpci2 => { bus => "ich9-pcie-port-3", addr => 0 },
4779         hostpci3 => { bus => "ich9-pcie-port-4", addr => 0 },
4780     };
4781
4782     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
4783            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
4784            my $bus = $devices->{$id}->{bus};
4785            $res = ",bus=$bus,addr=$addr";
4786     }
4787     return $res;
4788
4789 }
4790
4791 # vzdump restore implementaion
4792
4793 sub tar_archive_read_firstfile {
4794     my $archive = shift;
4795
4796     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
4797
4798     # try to detect archive type first
4799     my $pid = open (TMP, "tar tf '$archive'|") ||
4800         die "unable to open file '$archive'\n";
4801     my $firstfile = <TMP>;
4802     kill 15, $pid;
4803     close TMP;
4804
4805     die "ERROR: archive contaions no data\n" if !$firstfile;
4806     chomp $firstfile;
4807
4808     return $firstfile;
4809 }
4810
4811 sub tar_restore_cleanup {
4812     my ($storecfg, $statfile) = @_;
4813
4814     print STDERR "starting cleanup\n";
4815
4816     if (my $fd = IO::File->new($statfile, "r")) {
4817         while (defined(my $line = <$fd>)) {
4818             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
4819                 my $volid = $2;
4820                 eval {
4821                     if ($volid =~ m|^/|) {
4822                         unlink $volid || die 'unlink failed\n';
4823                     } else {
4824                         PVE::Storage::vdisk_free($storecfg, $volid);
4825                     }
4826                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
4827                 };
4828                 print STDERR "unable to cleanup '$volid' - $@" if $@;
4829             } else {
4830                 print STDERR "unable to parse line in statfile - $line";
4831             }
4832         }
4833         $fd->close();
4834     }
4835 }
4836
4837 sub restore_archive {
4838     my ($archive, $vmid, $user, $opts) = @_;
4839
4840     my $format = $opts->{format};
4841     my $comp;
4842
4843     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
4844         $format = 'tar' if !$format;
4845         $comp = 'gzip';
4846     } elsif ($archive =~ m/\.tar$/) {
4847         $format = 'tar' if !$format;
4848     } elsif ($archive =~ m/.tar.lzo$/) {
4849         $format = 'tar' if !$format;
4850         $comp = 'lzop';
4851     } elsif ($archive =~ m/\.vma$/) {
4852         $format = 'vma' if !$format;
4853     } elsif ($archive =~ m/\.vma\.gz$/) {
4854         $format = 'vma' if !$format;
4855         $comp = 'gzip';
4856     } elsif ($archive =~ m/\.vma\.lzo$/) {
4857         $format = 'vma' if !$format;
4858         $comp = 'lzop';
4859     } else {
4860         $format = 'vma' if !$format; # default
4861     }
4862
4863     # try to detect archive format
4864     if ($format eq 'tar') {
4865         return restore_tar_archive($archive, $vmid, $user, $opts);
4866     } else {
4867         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
4868     }
4869 }
4870
4871 sub restore_update_config_line {
4872     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
4873
4874     return if $line =~ m/^\#qmdump\#/;
4875     return if $line =~ m/^\#vzdump\#/;
4876     return if $line =~ m/^lock:/;
4877     return if $line =~ m/^unused\d+:/;
4878     return if $line =~ m/^parent:/;
4879     return if $line =~ m/^template:/; # restored VM is never a template
4880
4881     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
4882         # try to convert old 1.X settings
4883         my ($id, $ind, $ethcfg) = ($1, $2, $3);
4884         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
4885             my ($model, $macaddr) = split(/\=/, $devconfig);
4886             $macaddr = PVE::Tools::random_ether_addr() if !$macaddr || $unique;
4887             my $net = {
4888                 model => $model,
4889                 bridge => "vmbr$ind",
4890                 macaddr => $macaddr,
4891             };
4892             my $netstr = print_net($net);
4893
4894             print $outfd "net$cookie->{netcount}: $netstr\n";
4895             $cookie->{netcount}++;
4896         }
4897     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
4898         my ($id, $netstr) = ($1, $2);
4899         my $net = parse_net($netstr);
4900         $net->{macaddr} = PVE::Tools::random_ether_addr() if $net->{macaddr};
4901         $netstr = print_net($net);
4902         print $outfd "$id: $netstr\n";
4903     } elsif ($line =~ m/^((ide|scsi|virtio|sata)\d+):\s*(\S+)\s*$/) {
4904         my $virtdev = $1;
4905         my $value = $3;
4906         if ($line =~ m/backup=no/) {
4907             print $outfd "#$line";
4908         } elsif ($virtdev && $map->{$virtdev}) {
4909             my $di = parse_drive($virtdev, $value);
4910             delete $di->{format}; # format can change on restore
4911             $di->{file} = $map->{$virtdev};
4912             $value = print_drive($vmid, $di);
4913             print $outfd "$virtdev: $value\n";
4914         } else {
4915             print $outfd $line;
4916         }
4917     } else {
4918         print $outfd $line;
4919     }
4920 }
4921
4922 sub scan_volids {
4923     my ($cfg, $vmid) = @_;
4924
4925     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
4926
4927     my $volid_hash = {};
4928     foreach my $storeid (keys %$info) {
4929         foreach my $item (@{$info->{$storeid}}) {
4930             next if !($item->{volid} && $item->{size});
4931             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
4932             $volid_hash->{$item->{volid}} = $item;
4933         }
4934     }
4935
4936     return $volid_hash;
4937 }
4938
4939 sub get_used_paths {
4940     my ($vmid, $storecfg, $conf, $scan_snapshots, $skip_drive) = @_;
4941
4942     my $used_path = {};
4943
4944     my $scan_config = sub {
4945         my ($cref, $snapname) = @_;
4946
4947         foreach my $key (keys %$cref) {
4948             my $value = $cref->{$key};
4949             if (valid_drivename($key)) {
4950                 next if $skip_drive && $key eq $skip_drive;
4951                 my $drive = parse_drive($key, $value);
4952                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
4953                 if ($drive->{file} =~ m!^/!) {
4954                     $used_path->{$drive->{file}}++; # = 1;
4955                 } else {
4956                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
4957                     next if !$storeid;
4958                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
4959                     next if !$scfg;
4960                     my $path = PVE::Storage::path($storecfg, $drive->{file}, $snapname);
4961                     $used_path->{$path}++; # = 1;
4962                 }
4963             }
4964         }
4965     };
4966
4967     &$scan_config($conf);
4968
4969     undef $skip_drive;
4970
4971     if ($scan_snapshots) {
4972         foreach my $snapname (keys %{$conf->{snapshots}}) {
4973             &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
4974         }
4975     }
4976
4977     return $used_path;
4978 }
4979
4980 sub update_disksize {
4981     my ($vmid, $conf, $volid_hash) = @_;
4982
4983     my $changes;
4984
4985     my $used = {};
4986
4987     # Note: it is allowed to define multiple storages with same path (alias), so
4988     # we need to check both 'volid' and real 'path' (two different volid can point
4989     # to the same path).
4990
4991     my $usedpath = {};
4992
4993     # update size info
4994     foreach my $opt (keys %$conf) {
4995         if (valid_drivename($opt)) {
4996             my $drive = parse_drive($opt, $conf->{$opt});
4997             my $volid = $drive->{file};
4998             next if !$volid;
4999
5000             $used->{$volid} = 1;
5001             if ($volid_hash->{$volid} &&
5002                 (my $path = $volid_hash->{$volid}->{path})) {
5003                 $usedpath->{$path} = 1;
5004             }
5005
5006             next if drive_is_cdrom($drive);
5007             next if !$volid_hash->{$volid};
5008
5009             $drive->{size} = $volid_hash->{$volid}->{size};
5010             my $new = print_drive($vmid, $drive);
5011             if ($new ne $conf->{$opt}) {
5012                 $changes = 1;
5013                 $conf->{$opt} = $new;
5014             }
5015         }
5016     }
5017
5018     # remove 'unusedX' entry if volume is used
5019     foreach my $opt (keys %$conf) {
5020         next if $opt !~ m/^unused\d+$/;
5021         my $volid = $conf->{$opt};
5022         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5023         if ($used->{$volid} || ($path && $usedpath->{$path})) {
5024             $changes = 1;
5025             delete $conf->{$opt};
5026         }
5027     }
5028
5029     foreach my $volid (sort keys %$volid_hash) {
5030         next if $volid =~ m/vm-$vmid-state-/;
5031         next if $used->{$volid};
5032         my $path = $volid_hash->{$volid}->{path};
5033         next if !$path; # just to be sure
5034         next if $usedpath->{$path};
5035         $changes = 1;
5036         add_unused_volume($conf, $volid);
5037         $usedpath->{$path} = 1; # avoid to add more than once (aliases)
5038     }
5039
5040     return $changes;
5041 }
5042
5043 sub rescan {
5044     my ($vmid, $nolock) = @_;
5045
5046     my $cfg = PVE::Cluster::cfs_read_file("storage.cfg");
5047
5048     my $volid_hash = scan_volids($cfg, $vmid);
5049
5050     my $updatefn =  sub {
5051         my ($vmid) = @_;
5052
5053         my $conf = load_config($vmid);
5054
5055         check_lock($conf);
5056
5057         my $vm_volids = {};
5058         foreach my $volid (keys %$volid_hash) {
5059             my $info = $volid_hash->{$volid};
5060             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5061         }
5062
5063         my $changes = update_disksize($vmid, $conf, $vm_volids);
5064
5065         update_config_nolock($vmid, $conf, 1) if $changes;
5066     };
5067
5068     if (defined($vmid)) {
5069         if ($nolock) {
5070             &$updatefn($vmid);
5071         } else {
5072             lock_config($vmid, $updatefn, $vmid);
5073         }
5074     } else {
5075         my $vmlist = config_list();
5076         foreach my $vmid (keys %$vmlist) {
5077             if ($nolock) {
5078                 &$updatefn($vmid);
5079             } else {
5080                 lock_config($vmid, $updatefn, $vmid);
5081             }
5082         }
5083     }
5084 }
5085
5086 sub restore_vma_archive {
5087     my ($archive, $vmid, $user, $opts, $comp) = @_;
5088
5089     my $input = $archive eq '-' ? "<&STDIN" : undef;
5090     my $readfrom = $archive;
5091
5092     my $uncomp = '';
5093     if ($comp) {
5094         $readfrom = '-';
5095         my $qarchive = PVE::Tools::shellquote($archive);
5096         if ($comp eq 'gzip') {
5097             $uncomp = "zcat $qarchive|";
5098         } elsif ($comp eq 'lzop') {
5099             $uncomp = "lzop -d -c $qarchive|";
5100         } else {
5101             die "unknown compression method '$comp'\n";
5102         }
5103
5104     }
5105
5106     my $tmpdir = "/var/tmp/vzdumptmp$$";
5107     rmtree $tmpdir;
5108
5109     # disable interrupts (always do cleanups)
5110     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5111         warn "got interrupt - ignored\n";
5112     };
5113
5114     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5115     POSIX::mkfifo($mapfifo, 0600);
5116     my $fifofh;
5117
5118     my $openfifo = sub {
5119         open($fifofh, '>', $mapfifo) || die $!;
5120     };
5121
5122     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5123
5124     my $oldtimeout;
5125     my $timeout = 5;
5126
5127     my $devinfo = {};
5128
5129     my $rpcenv = PVE::RPCEnvironment::get();
5130
5131     my $conffile = config_file($vmid);
5132     my $tmpfn = "$conffile.$$.tmp";
5133
5134     # Note: $oldconf is undef if VM does not exists
5135     my $oldconf = PVE::Cluster::cfs_read_file(cfs_config_path($vmid));
5136
5137     my $print_devmap = sub {
5138         my $virtdev_hash = {};
5139
5140         my $cfgfn = "$tmpdir/qemu-server.conf";
5141
5142         # we can read the config - that is already extracted
5143         my $fh = IO::File->new($cfgfn, "r") ||
5144             "unable to read qemu-server.conf - $!\n";
5145
5146         while (defined(my $line = <$fh>)) {
5147             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5148                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5149                 die "archive does not contain data for drive '$virtdev'\n"
5150                     if !$devinfo->{$devname};
5151                 if (defined($opts->{storage})) {
5152                     $storeid = $opts->{storage} || 'local';
5153                 } elsif (!$storeid) {
5154                     $storeid = 'local';
5155                 }
5156                 $format = 'raw' if !$format;
5157                 $devinfo->{$devname}->{devname} = $devname;
5158                 $devinfo->{$devname}->{virtdev} = $virtdev;
5159                 $devinfo->{$devname}->{format} = $format;
5160                 $devinfo->{$devname}->{storeid} = $storeid;
5161
5162                 # check permission on storage
5163                 my $pool = $opts->{pool}; # todo: do we need that?
5164                 if ($user ne 'root@pam') {
5165                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5166                 }
5167
5168                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5169             }
5170         }
5171
5172         foreach my $devname (keys %$devinfo) {
5173             die "found no device mapping information for device '$devname'\n"
5174                 if !$devinfo->{$devname}->{virtdev};
5175         }
5176
5177         my $cfg = cfs_read_file('storage.cfg');
5178
5179         # create empty/temp config
5180         if ($oldconf) {
5181             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5182             foreach_drive($oldconf, sub {
5183                 my ($ds, $drive) = @_;
5184
5185                 return if drive_is_cdrom($drive);
5186
5187                 my $volid = $drive->{file};
5188
5189                 return if !$volid || $volid =~ m|^/|;
5190
5191                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5192                 return if !$path || !$owner || ($owner != $vmid);
5193
5194                 # Note: only delete disk we want to restore
5195                 # other volumes will become unused
5196                 if ($virtdev_hash->{$ds}) {
5197                     PVE::Storage::vdisk_free($cfg, $volid);
5198                 }
5199             });
5200         }
5201
5202         my $map = {};
5203         foreach my $virtdev (sort keys %$virtdev_hash) {
5204             my $d = $virtdev_hash->{$virtdev};
5205             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5206             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5207
5208             # test if requested format is supported
5209             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5210             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5211             $d->{format} = $defFormat if !$supported;
5212
5213             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5214                                                   $d->{format}, undef, $alloc_size);
5215             print STDERR "new volume ID is '$volid'\n";
5216             $d->{volid} = $volid;
5217             my $path = PVE::Storage::path($cfg, $volid);
5218
5219             my $write_zeros = 1;
5220             # fixme: what other storages types initialize volumes with zero?
5221             if ($scfg->{type} eq 'dir' || $scfg->{type} eq 'nfs' || $scfg->{type} eq 'glusterfs' ||
5222                 $scfg->{type} eq 'sheepdog' || $scfg->{type} eq 'rbd') {
5223                 $write_zeros = 0;
5224             }
5225
5226             print $fifofh "${write_zeros}:$d->{devname}=$path\n";
5227
5228             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5229             $map->{$virtdev} = $volid;
5230         }
5231
5232         $fh->seek(0, 0) || die "seek failed - $!\n";
5233
5234         my $outfd = new IO::File ($tmpfn, "w") ||
5235             die "unable to write config for VM $vmid\n";
5236
5237         my $cookie = { netcount => 0 };
5238         while (defined(my $line = <$fh>)) {
5239             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5240         }
5241
5242         $fh->close();
5243         $outfd->close();
5244     };
5245
5246     eval {
5247         # enable interrupts
5248         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5249             die "interrupted by signal\n";
5250         };
5251         local $SIG{ALRM} = sub { die "got timeout\n"; };
5252
5253         $oldtimeout = alarm($timeout);
5254
5255         my $parser = sub {
5256             my $line = shift;
5257
5258             print "$line\n";
5259
5260             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5261                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5262                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5263             } elsif ($line =~ m/^CTIME: /) {
5264                 # we correctly received the vma config, so we can disable
5265                 # the timeout now for disk allocation (set to 10 minutes, so
5266                 # that we always timeout if something goes wrong)
5267                 alarm(600);
5268                 &$print_devmap();
5269                 print $fifofh "done\n";
5270                 my $tmp = $oldtimeout || 0;
5271                 $oldtimeout = undef;
5272                 alarm($tmp);
5273                 close($fifofh);
5274             }
5275         };
5276
5277         print "restore vma archive: $cmd\n";
5278         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5279     };
5280     my $err = $@;
5281
5282     alarm($oldtimeout) if $oldtimeout;
5283
5284     unlink $mapfifo;
5285
5286     if ($err) {
5287         rmtree $tmpdir;
5288         unlink $tmpfn;
5289
5290         my $cfg = cfs_read_file('storage.cfg');
5291         foreach my $devname (keys %$devinfo) {
5292             my $volid = $devinfo->{$devname}->{volid};
5293             next if !$volid;
5294             eval {
5295                 if ($volid =~ m|^/|) {
5296                     unlink $volid || die 'unlink failed\n';
5297                 } else {
5298                     PVE::Storage::vdisk_free($cfg, $volid);
5299                 }
5300                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5301             };
5302             print STDERR "unable to cleanup '$volid' - $@" if $@;
5303         }
5304         die $err;
5305     }
5306
5307     rmtree $tmpdir;
5308
5309     rename($tmpfn, $conffile) ||
5310         die "unable to commit configuration file '$conffile'\n";
5311
5312     PVE::Cluster::cfs_update(); # make sure we read new file
5313
5314     eval { rescan($vmid, 1); };
5315     warn $@ if $@;
5316 }
5317
5318 sub restore_tar_archive {
5319     my ($archive, $vmid, $user, $opts) = @_;
5320
5321     if ($archive ne '-') {
5322         my $firstfile = tar_archive_read_firstfile($archive);
5323         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5324             if $firstfile ne 'qemu-server.conf';
5325     }
5326
5327     my $storecfg = cfs_read_file('storage.cfg');
5328
5329     # destroy existing data - keep empty config
5330     my $vmcfgfn = config_file($vmid);
5331     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5332
5333     my $tocmd = "/usr/lib/qemu-server/qmextract";
5334
5335     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5336     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5337     $tocmd .= ' --prealloc' if $opts->{prealloc};
5338     $tocmd .= ' --info' if $opts->{info};
5339
5340     # tar option "xf" does not autodetect compression when read from STDIN,
5341     # so we pipe to zcat
5342     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5343         PVE::Tools::shellquote("--to-command=$tocmd");
5344
5345     my $tmpdir = "/var/tmp/vzdumptmp$$";
5346     mkpath $tmpdir;
5347
5348     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5349     local $ENV{VZDUMP_VMID} = $vmid;
5350     local $ENV{VZDUMP_USER} = $user;
5351
5352     my $conffile = config_file($vmid);
5353     my $tmpfn = "$conffile.$$.tmp";
5354
5355     # disable interrupts (always do cleanups)
5356     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5357         print STDERR "got interrupt - ignored\n";
5358     };
5359
5360     eval {
5361         # enable interrupts
5362         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5363             die "interrupted by signal\n";
5364         };
5365
5366         if ($archive eq '-') {
5367             print "extracting archive from STDIN\n";
5368             run_command($cmd, input => "<&STDIN");
5369         } else {
5370             print "extracting archive '$archive'\n";
5371             run_command($cmd);
5372         }
5373
5374         return if $opts->{info};
5375
5376         # read new mapping
5377         my $map = {};
5378         my $statfile = "$tmpdir/qmrestore.stat";
5379         if (my $fd = IO::File->new($statfile, "r")) {
5380             while (defined (my $line = <$fd>)) {
5381                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5382                     $map->{$1} = $2 if $1;
5383                 } else {
5384                     print STDERR "unable to parse line in statfile - $line\n";
5385                 }
5386             }
5387             $fd->close();
5388         }
5389
5390         my $confsrc = "$tmpdir/qemu-server.conf";
5391
5392         my $srcfd = new IO::File($confsrc, "r") ||
5393             die "unable to open file '$confsrc'\n";
5394
5395         my $outfd = new IO::File ($tmpfn, "w") ||
5396             die "unable to write config for VM $vmid\n";
5397
5398         my $cookie = { netcount => 0 };
5399         while (defined (my $line = <$srcfd>)) {
5400             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5401         }
5402
5403         $srcfd->close();
5404         $outfd->close();
5405     };
5406     my $err = $@;
5407
5408     if ($err) {
5409
5410         unlink $tmpfn;
5411
5412         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
5413
5414         die $err;
5415     }
5416
5417     rmtree $tmpdir;
5418
5419     rename $tmpfn, $conffile ||
5420         die "unable to commit configuration file '$conffile'\n";
5421
5422     PVE::Cluster::cfs_update(); # make sure we read new file
5423
5424     eval { rescan($vmid, 1); };
5425     warn $@ if $@;
5426 };
5427
5428
5429 # Internal snapshots
5430
5431 # NOTE: Snapshot create/delete involves several non-atomic
5432 # action, and can take a long time.
5433 # So we try to avoid locking the file and use 'lock' variable
5434 # inside the config file instead.
5435
5436 my $snapshot_copy_config = sub {
5437     my ($source, $dest) = @_;
5438
5439     foreach my $k (keys %$source) {
5440         next if $k eq 'snapshots';
5441         next if $k eq 'snapstate';
5442         next if $k eq 'snaptime';
5443         next if $k eq 'vmstate';
5444         next if $k eq 'lock';
5445         next if $k eq 'digest';
5446         next if $k eq 'description';
5447         next if $k =~ m/^unused\d+$/;
5448
5449         $dest->{$k} = $source->{$k};
5450     }
5451 };
5452
5453 my $snapshot_apply_config = sub {
5454     my ($conf, $snap) = @_;
5455
5456     # copy snapshot list
5457     my $newconf = {
5458         snapshots => $conf->{snapshots},
5459     };
5460
5461     # keep description and list of unused disks
5462     foreach my $k (keys %$conf) {
5463         next if !($k =~ m/^unused\d+$/ || $k eq 'description');
5464         $newconf->{$k} = $conf->{$k};
5465     }
5466
5467     &$snapshot_copy_config($snap, $newconf);
5468
5469     return $newconf;
5470 };
5471
5472 sub foreach_writable_storage {
5473     my ($conf, $func) = @_;
5474
5475     my $sidhash = {};
5476
5477     foreach my $ds (keys %$conf) {
5478         next if !valid_drivename($ds);
5479
5480         my $drive = parse_drive($ds, $conf->{$ds});
5481         next if !$drive;
5482         next if drive_is_cdrom($drive);
5483
5484         my $volid = $drive->{file};
5485
5486         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5487         $sidhash->{$sid} = $sid if $sid;
5488     }
5489
5490     foreach my $sid (sort keys %$sidhash) {
5491         &$func($sid);
5492     }
5493 }
5494
5495 my $alloc_vmstate_volid = sub {
5496     my ($storecfg, $vmid, $conf, $snapname) = @_;
5497
5498     # Note: we try to be smart when selecting a $target storage
5499
5500     my $target;
5501
5502     # search shared storage first
5503     foreach_writable_storage($conf, sub {
5504         my ($sid) = @_;
5505         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
5506         return if !$scfg->{shared};
5507
5508         $target = $sid if !$target || $scfg->{path}; # prefer file based storage
5509     });
5510
5511     if (!$target) {
5512         # now search local storage
5513         foreach_writable_storage($conf, sub {
5514             my ($sid) = @_;
5515             my $scfg = PVE::Storage::storage_config($storecfg, $sid);
5516             return if $scfg->{shared};
5517
5518             $target = $sid if !$target || $scfg->{path}; # prefer file based storage;
5519         });
5520     }
5521
5522     $target = 'local' if !$target;
5523
5524     my $driver_state_size = 500; # assume 32MB is enough to safe all driver state;
5525     # we abort live save after $conf->{memory}, so we need at max twice that space
5526     my $size = $conf->{memory}*2 + $driver_state_size;
5527
5528     my $name = "vm-$vmid-state-$snapname";
5529     my $scfg = PVE::Storage::storage_config($storecfg, $target);
5530     $name .= ".raw" if $scfg->{path}; # add filename extension for file base storage
5531     my $volid = PVE::Storage::vdisk_alloc($storecfg, $target, $vmid, 'raw', $name, $size*1024);
5532
5533     return $volid;
5534 };
5535
5536 my $snapshot_prepare = sub {
5537     my ($vmid, $snapname, $save_vmstate, $comment) = @_;
5538
5539     my $snap;
5540
5541     my $updatefn =  sub {
5542
5543         my $conf = load_config($vmid);
5544
5545         die "you can't take a snapshot if it's a template\n"
5546             if is_template($conf);
5547
5548         check_lock($conf);
5549
5550         $conf->{lock} = 'snapshot';
5551
5552         die "snapshot name '$snapname' already used\n"
5553             if defined($conf->{snapshots}->{$snapname});
5554
5555         my $storecfg = PVE::Storage::config();
5556         die "snapshot feature is not available" if !has_feature('snapshot', $conf, $storecfg);
5557
5558         $snap = $conf->{snapshots}->{$snapname} = {};
5559
5560         if ($save_vmstate && check_running($vmid)) {
5561             $snap->{vmstate} = &$alloc_vmstate_volid($storecfg, $vmid, $conf, $snapname);
5562         }
5563
5564         &$snapshot_copy_config($conf, $snap);
5565
5566         $snap->{snapstate} = "prepare";
5567         $snap->{snaptime} = time();
5568         $snap->{description} = $comment if $comment;
5569
5570         # always overwrite machine if we save vmstate. This makes sure we
5571         # can restore it later using correct machine type
5572         $snap->{machine} = get_current_qemu_machine($vmid) if $snap->{vmstate};
5573
5574         update_config_nolock($vmid, $conf, 1);
5575     };
5576
5577     lock_config($vmid, $updatefn);
5578
5579     return $snap;
5580 };
5581
5582 my $snapshot_commit = sub {
5583     my ($vmid, $snapname) = @_;
5584
5585     my $updatefn = sub {
5586
5587         my $conf = load_config($vmid);
5588
5589         die "missing snapshot lock\n"
5590             if !($conf->{lock} && $conf->{lock} eq 'snapshot');
5591
5592         my $has_machine_config = defined($conf->{machine});
5593
5594         my $snap = $conf->{snapshots}->{$snapname};
5595
5596         die "snapshot '$snapname' does not exist\n" if !defined($snap);
5597
5598         die "wrong snapshot state\n"
5599             if !($snap->{snapstate} && $snap->{snapstate} eq "prepare");
5600
5601         delete $snap->{snapstate};
5602         delete $conf->{lock};
5603
5604         my $newconf = &$snapshot_apply_config($conf, $snap);
5605
5606         delete $newconf->{machine} if !$has_machine_config;
5607
5608         $newconf->{parent} = $snapname;
5609
5610         update_config_nolock($vmid, $newconf, 1);
5611     };
5612
5613     lock_config($vmid, $updatefn);
5614 };
5615
5616 sub snapshot_rollback {
5617     my ($vmid, $snapname) = @_;
5618
5619     my $prepare = 1;
5620
5621     my $storecfg = PVE::Storage::config();
5622
5623     my $conf = load_config($vmid);
5624
5625     my $get_snapshot_config = sub {
5626
5627         die "you can't rollback if vm is a template\n" if is_template($conf);
5628
5629         my $res = $conf->{snapshots}->{$snapname};
5630
5631         die "snapshot '$snapname' does not exist\n" if !defined($res);
5632
5633         return $res;
5634     };
5635
5636     my $snap = &$get_snapshot_config();
5637
5638     foreach_drive($snap, sub {
5639         my ($ds, $drive) = @_;
5640
5641         return if drive_is_cdrom($drive);
5642
5643         my $volid = $drive->{file};
5644
5645         PVE::Storage::volume_rollback_is_possible($storecfg, $volid, $snapname);
5646     });
5647
5648     my $updatefn = sub {
5649
5650         $conf = load_config($vmid);
5651
5652         $snap = &$get_snapshot_config();
5653
5654         die "unable to rollback to incomplete snapshot (snapstate = $snap->{snapstate})\n"
5655             if $snap->{snapstate};
5656
5657         if ($prepare) {
5658             check_lock($conf);
5659             vm_stop($storecfg, $vmid, undef, undef, 5, undef, undef);
5660         }
5661
5662         die "unable to rollback vm $vmid: vm is running\n"
5663             if check_running($vmid);
5664
5665         if ($prepare) {
5666             $conf->{lock} = 'rollback';
5667         } else {
5668             die "got wrong lock\n" if !($conf->{lock} && $conf->{lock} eq 'rollback');
5669             delete $conf->{lock};
5670         }
5671
5672         my $forcemachine;
5673
5674         if (!$prepare) {
5675             my $has_machine_config = defined($conf->{machine});
5676
5677             # copy snapshot config to current config
5678             $conf = &$snapshot_apply_config($conf, $snap);
5679             $conf->{parent} = $snapname;
5680
5681             # Note: old code did not store 'machine', so we try to be smart
5682             # and guess the snapshot was generated with kvm 1.4 (pc-i440fx-1.4).
5683             $forcemachine = $conf->{machine} || 'pc-i440fx-1.4';
5684             # we remove the 'machine' configuration if not explicitly specified
5685             # in the original config.
5686             delete $conf->{machine} if $snap->{vmstate} && !$has_machine_config;
5687         }
5688
5689         update_config_nolock($vmid, $conf, 1);
5690
5691         if (!$prepare && $snap->{vmstate}) {
5692             my $statefile = PVE::Storage::path($storecfg, $snap->{vmstate});
5693             vm_start($storecfg, $vmid, $statefile, undef, undef, undef, $forcemachine);
5694         }
5695     };
5696
5697     lock_config($vmid, $updatefn);
5698
5699     foreach_drive($snap, sub {
5700         my ($ds, $drive) = @_;
5701
5702         return if drive_is_cdrom($drive);
5703
5704         my $volid = $drive->{file};
5705         my $device = "drive-$ds";
5706
5707         PVE::Storage::volume_snapshot_rollback($storecfg, $volid, $snapname);
5708     });
5709
5710     $prepare = 0;
5711     lock_config($vmid, $updatefn);
5712 }
5713
5714 my $savevm_wait = sub {
5715     my ($vmid) = @_;
5716
5717     for(;;) {
5718         my $stat = vm_mon_cmd_nocheck($vmid, "query-savevm");
5719         if (!$stat->{status}) {
5720             die "savevm not active\n";
5721         } elsif ($stat->{status} eq 'active') {
5722             sleep(1);
5723             next;
5724         } elsif ($stat->{status} eq 'completed') {
5725             last;
5726         } else {
5727             die "query-savevm returned status '$stat->{status}'\n";
5728         }
5729     }
5730 };
5731
5732 sub do_snapshots_with_qemu {
5733     my ($storecfg, $volid) = @_;
5734
5735     my $storage_name = PVE::Storage::parse_volume_id($volid);
5736
5737     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}} ){
5738         return 1;
5739     }
5740
5741     if ($volid =~ m/\.(qcow2|qed)$/){
5742         return 1;
5743     }
5744
5745     return undef;
5746 }
5747
5748 sub snapshot_create {
5749     my ($vmid, $snapname, $save_vmstate, $comment) = @_;
5750
5751     my $snap = &$snapshot_prepare($vmid, $snapname, $save_vmstate, $comment);
5752
5753     $save_vmstate = 0 if !$snap->{vmstate}; # vm is not running
5754
5755     my $config = load_config($vmid);
5756
5757     my $running = check_running($vmid);
5758
5759     my $freezefs = $running && $config->{agent};
5760     $freezefs = 0 if $snap->{vmstate}; # not needed if we save RAM
5761
5762     my $drivehash = {};
5763
5764     if ($freezefs) {
5765         eval { vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
5766         warn "guest-fsfreeze-freeze problems - $@" if $@;
5767     }
5768
5769     eval {
5770         # create internal snapshots of all drives
5771
5772         my $storecfg = PVE::Storage::config();
5773
5774         if ($running) {
5775             if ($snap->{vmstate}) {
5776                 my $path = PVE::Storage::path($storecfg, $snap->{vmstate});
5777                 vm_mon_cmd($vmid, "savevm-start", statefile => $path);
5778                 &$savevm_wait($vmid);
5779             } else {
5780                 vm_mon_cmd($vmid, "savevm-start");
5781             }
5782         };
5783
5784         foreach_drive($snap, sub {
5785             my ($ds, $drive) = @_;
5786
5787             return if drive_is_cdrom($drive);
5788
5789             my $volid = $drive->{file};
5790             my $device = "drive-$ds";
5791
5792             qemu_volume_snapshot($vmid, $device, $storecfg, $volid, $snapname);
5793             $drivehash->{$ds} = 1;
5794        });
5795     };
5796     my $err = $@;
5797
5798     if ($running) {
5799         eval { vm_mon_cmd($vmid, "savevm-end")  };
5800         warn $@ if $@;
5801
5802         if ($freezefs) {
5803             eval { vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
5804             warn "guest-fsfreeze-thaw problems - $@" if $@;
5805         }
5806
5807         # savevm-end is async, we need to wait
5808         for (;;) {
5809             my $stat = vm_mon_cmd_nocheck($vmid, "query-savevm");
5810             if (!$stat->{bytes}) {
5811                 last;
5812             } else {
5813                 print "savevm not yet finished\n";
5814                 sleep(1);
5815                 next;
5816             }
5817         }
5818     }
5819
5820     if ($err) {
5821         warn "snapshot create failed: starting cleanup\n";
5822         eval { snapshot_delete($vmid, $snapname, 0, $drivehash); };
5823         warn $@ if $@;
5824         die $err;
5825     }
5826
5827     &$snapshot_commit($vmid, $snapname);
5828 }
5829
5830 # Note: $drivehash is only set when called from snapshot_create.
5831 sub snapshot_delete {
5832     my ($vmid, $snapname, $force, $drivehash) = @_;
5833
5834     my $prepare = 1;
5835
5836     my $snap;
5837     my $unused = [];
5838
5839     my $unlink_parent = sub {
5840         my ($confref, $new_parent) = @_;
5841
5842         if ($confref->{parent} && $confref->{parent} eq $snapname) {
5843             if ($new_parent) {
5844                 $confref->{parent} = $new_parent;
5845             } else {
5846                 delete $confref->{parent};
5847             }
5848         }
5849     };
5850
5851     my $updatefn =  sub {
5852         my ($remove_drive) = @_;
5853
5854         my $conf = load_config($vmid);
5855
5856         if (!$drivehash) {
5857             check_lock($conf);
5858             die "you can't delete a snapshot if vm is a template\n"
5859                 if is_template($conf);
5860         }
5861
5862         $snap = $conf->{snapshots}->{$snapname};
5863
5864         die "snapshot '$snapname' does not exist\n" if !defined($snap);
5865
5866         # remove parent refs
5867         if (!$prepare) {
5868             &$unlink_parent($conf, $snap->{parent});
5869             foreach my $sn (keys %{$conf->{snapshots}}) {
5870                 next if $sn eq $snapname;
5871                 &$unlink_parent($conf->{snapshots}->{$sn}, $snap->{parent});
5872             }
5873         }
5874
5875         if ($remove_drive) {
5876             if ($remove_drive eq 'vmstate') {
5877                 delete $snap->{$remove_drive};
5878             } else {
5879                 my $drive = parse_drive($remove_drive, $snap->{$remove_drive});
5880                 my $volid = $drive->{file};
5881                 delete $snap->{$remove_drive};
5882                 add_unused_volume($conf, $volid);
5883             }
5884         }
5885
5886         if ($prepare) {
5887             $snap->{snapstate} = 'delete';
5888         } else {
5889             delete $conf->{snapshots}->{$snapname};
5890             delete $conf->{lock} if $drivehash;
5891             foreach my $volid (@$unused) {
5892                 add_unused_volume($conf, $volid);
5893             }
5894         }
5895
5896         update_config_nolock($vmid, $conf, 1);
5897     };
5898
5899     lock_config($vmid, $updatefn);
5900
5901     # now remove vmstate file
5902
5903     my $storecfg = PVE::Storage::config();
5904
5905     if ($snap->{vmstate}) {
5906         eval {  PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
5907         if (my $err = $@) {
5908             die $err if !$force;
5909             warn $err;
5910         }
5911         # save changes (remove vmstate from snapshot)
5912         lock_config($vmid, $updatefn, 'vmstate') if !$force;
5913     };
5914
5915     # now remove all internal snapshots
5916     foreach_drive($snap, sub {
5917         my ($ds, $drive) = @_;
5918
5919         return if drive_is_cdrom($drive);
5920
5921         my $volid = $drive->{file};
5922         my $device = "drive-$ds";
5923
5924         if (!$drivehash || $drivehash->{$ds}) {
5925             eval { qemu_volume_snapshot_delete($vmid, $device, $storecfg, $volid, $snapname); };
5926             if (my $err = $@) {
5927                 die $err if !$force;
5928                 warn $err;
5929             }
5930         }
5931
5932         # save changes (remove drive fron snapshot)
5933         lock_config($vmid, $updatefn, $ds) if !$force;
5934         push @$unused, $volid;
5935     });
5936
5937     # now cleanup config
5938     $prepare = 0;
5939     lock_config($vmid, $updatefn);
5940 }
5941
5942 sub has_feature {
5943     my ($feature, $conf, $storecfg, $snapname, $running) = @_;
5944
5945     my $err;
5946     foreach_drive($conf, sub {
5947         my ($ds, $drive) = @_;
5948
5949         return if drive_is_cdrom($drive);
5950         my $volid = $drive->{file};
5951         $err = 1 if !PVE::Storage::volume_has_feature($storecfg, $feature, $volid, $snapname, $running);
5952     });
5953
5954     return $err ? 0 : 1;
5955 }
5956
5957 sub template_create {
5958     my ($vmid, $conf, $disk) = @_;
5959
5960     my $storecfg = PVE::Storage::config();
5961
5962     foreach_drive($conf, sub {
5963         my ($ds, $drive) = @_;
5964
5965         return if drive_is_cdrom($drive);
5966         return if $disk && $ds ne $disk;
5967
5968         my $volid = $drive->{file};
5969         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
5970
5971         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
5972         $drive->{file} = $voliddst;
5973         $conf->{$ds} = print_drive($vmid, $drive);
5974         update_config_nolock($vmid, $conf, 1);
5975     });
5976 }
5977
5978 sub is_template {
5979     my ($conf) = @_;
5980
5981     return 1 if defined $conf->{template} && $conf->{template} == 1;
5982 }
5983
5984 sub qemu_img_convert {
5985     my ($src_volid, $dst_volid, $size, $snapname) = @_;
5986
5987     my $storecfg = PVE::Storage::config();
5988     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
5989     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
5990
5991     if ($src_storeid && $dst_storeid) {
5992         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
5993         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5994
5995         my $src_format = qemu_img_format($src_scfg, $src_volname);
5996         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
5997
5998         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
5999         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6000
6001         my $cmd = [];
6002         push @$cmd, '/usr/bin/qemu-img', 'convert', '-t', 'writeback', '-p', '-n';
6003         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
6004         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path, $dst_path;
6005
6006         my $parser = sub {
6007             my $line = shift;
6008             if($line =~ m/\((\S+)\/100\%\)/){
6009                 my $percent = $1;
6010                 my $transferred = int($size * $percent / 100);
6011                 my $remaining = $size - $transferred;
6012
6013                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6014             }
6015
6016         };
6017
6018         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6019         my $err = $@;
6020         die "copy failed: $err" if $err;
6021     }
6022 }
6023
6024 sub qemu_img_format {
6025     my ($scfg, $volname) = @_;
6026
6027     if ($scfg->{path} && $volname =~ m/\.(raw|cow|qcow|qcow2|qed|vmdk|cloop)$/) {
6028         return $1;
6029     } else {
6030         return "raw";
6031     }
6032 }
6033
6034 sub qemu_drive_mirror {
6035     my ($vmid, $drive, $dst_volid, $vmiddst) = @_;
6036
6037     my $storecfg = PVE::Storage::config();
6038     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6039
6040     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6041
6042     my $format = qemu_img_format($dst_scfg, $dst_volname);
6043
6044     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6045
6046     #drive-mirror is doing lseek on source image before starting, and this can take a lot of time for big nfs volume
6047     #during this time, qmp socket is hanging
6048     #http://lists.nongnu.org/archive/html/qemu-devel/2015-05/msg01838.html
6049     #so we need to setup a big timeout
6050     my $opts = { timeout => 14400, device => "drive-$drive", mode => "existing", sync => "full", target => $dst_path };
6051     $opts->{format} = $format if $format;
6052
6053     print "drive mirror is starting : this step can take some minutes/hours, depend of disk size and storage speed\n";
6054
6055     vm_mon_cmd($vmid, "drive-mirror", %$opts);
6056     eval {
6057         while (1) {
6058             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6059             my $stat = @$stats[0];
6060             die "mirroring job seem to have die. Maybe do you have bad sectors?" if !$stat;
6061             die "error job is not mirroring" if $stat->{type} ne "mirror";
6062
6063             my $busy = $stat->{busy};
6064             my $ready = $stat->{ready};
6065
6066             if (my $total = $stat->{len}) {
6067                 my $transferred = $stat->{offset} || 0;
6068                 my $remaining = $total - $transferred;
6069                 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6070
6071                 print "transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6072             }
6073
6074
6075             if ($stat->{ready} eq 'true') {
6076
6077                 last if $vmiddst != $vmid;
6078
6079                 # try to switch the disk if source and destination are on the same guest
6080                 eval { vm_mon_cmd($vmid, "block-job-complete", device => "drive-$drive") };
6081                 last if !$@;
6082                 die $@ if $@ !~ m/cannot be completed/;
6083             }
6084             sleep 1;
6085         }
6086
6087
6088     };
6089     my $err = $@;
6090
6091     my $cancel_job = sub {
6092         vm_mon_cmd($vmid, "block-job-cancel", device => "drive-$drive");
6093         while (1) {
6094             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6095             my $stat = @$stats[0];
6096             last if !$stat;
6097             sleep 1;
6098         }
6099     };
6100
6101     if ($err) {
6102         eval { &$cancel_job(); };
6103         die "mirroring error: $err";
6104     }
6105
6106     if ($vmiddst != $vmid) {
6107         # if we clone a disk for a new target vm, we don't switch the disk
6108         &$cancel_job(); # so we call block-job-cancel
6109     }
6110 }
6111
6112 sub clone_disk {
6113     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6114         $newvmid, $storage, $format, $full, $newvollist) = @_;
6115
6116     my $newvolid;
6117
6118     if (!$full) {
6119         print "create linked clone of drive $drivename ($drive->{file})\n";
6120         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6121         push @$newvollist, $newvolid;
6122     } else {
6123         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6124         $storeid = $storage if $storage;
6125
6126         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6127         if (!$format) {
6128             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6129             $format = qemu_img_format($scfg, $volname);
6130         }
6131
6132         # test if requested format is supported - else use default
6133         my $supported = grep { $_ eq $format } @$validFormats;
6134         $format = $defFormat if !$supported;
6135
6136         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6137
6138         print "create full clone of drive $drivename ($drive->{file})\n";
6139         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $format, undef, ($size/1024));
6140         push @$newvollist, $newvolid;
6141
6142         if (!$running || $snapname) {
6143             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname);
6144         } else {
6145             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid);
6146         }
6147     }
6148
6149     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6150
6151     my $disk = $drive;
6152     $disk->{format} = undef;
6153     $disk->{file} = $newvolid;
6154     $disk->{size} = $size;
6155
6156     return $disk;
6157 }
6158
6159 # this only works if VM is running
6160 sub get_current_qemu_machine {
6161     my ($vmid) = @_;
6162
6163     my $cmd = { execute => 'query-machines', arguments => {} };
6164     my $res = vm_qmp_command($vmid, $cmd);
6165
6166     my ($current, $default);
6167     foreach my $e (@$res) {
6168         $default = $e->{name} if $e->{'is-default'};
6169         $current = $e->{name} if $e->{'is-current'};
6170     }
6171
6172     # fallback to the default machine if current is not supported by qemu
6173     return $current || $default || 'pc';
6174 }
6175
6176 sub qemu_machine_feature_enabled {
6177     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6178
6179     my $current_major;
6180     my $current_minor;
6181
6182     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6183
6184         $current_major = $3;
6185         $current_minor = $4;
6186
6187     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6188
6189         $current_major = $1;
6190         $current_minor = $2;
6191     }
6192
6193     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
6194
6195
6196 }
6197
6198 sub lspci {
6199
6200     my $devices = {};
6201
6202     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6203             my (undef, $id, $function) = @_;
6204             my $res = { id => $id, function => $function};
6205             push @{$devices->{$id}}, $res;
6206     });
6207
6208     return $devices;
6209 }
6210
6211 sub vm_iothreads_list {
6212     my ($vmid) = @_;
6213
6214     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6215
6216     my $iothreads = {};
6217     foreach my $iothread (@$res) {
6218         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6219     }
6220
6221     return $iothreads;
6222 }
6223
6224 sub scsihw_infos {
6225     my ($conf, $drive) = @_;
6226
6227     my $maxdev = 0;
6228
6229     if ($conf->{scsihw} && ($conf->{scsihw} =~ m/^lsi/)) {
6230         $maxdev = 7;
6231     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6232         $maxdev = 1;
6233     } else {
6234         $maxdev = 256;
6235     }
6236
6237     my $controller = int($drive->{index} / $maxdev);
6238     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6239
6240     return ($maxdev, $controller, $controller_prefix);
6241 }
6242
6243 1;