PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QMPClient;
  31 use PVE::RPCEnvironment;
  32 use Time::HiRes qw(gettimeofday);
  33
  34 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  35
  36 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  37
  38 # Note about locking: we use flock on the config file protect
  39 # against concurent actions.
  40 # Aditionaly, we have a 'lock' setting in the config file. This
  41 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  42 # allowed when such lock is set. But you can ignore this kind of
  43 # lock with the --skiplock flag.
  44
  45 cfs_register_file('/qemu-server/',
  46                   \&parse_vm_config,
  47                   \&write_vm_config);
  48
  49 PVE::JSONSchema::register_standard_option('skiplock', {
  50     description => "Ignore locks - only root is allowed to use this option.",
  51     type => 'boolean',
  52     optional => 1,
  53 });
  54
  55 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  56     description => "Some command save/restore state from this location.",
  57     type => 'string',
  58     maxLength => 128,
  59     optional => 1,
  60 });
  61
  62 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  63     description => "The name of the snapshot.",
  64     type => 'string', format => 'pve-configid',
  65     maxLength => 40,
  66 });
  67
  68 #no warnings 'redefine';
  69
  70 sub cgroups_write {
  71    my ($controller, $vmid, $option, $value) = @_;
  72
  73    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  74    PVE::ProcFSTools::write_proc_entry($path, $value);
  75
  76 }
  77
  78 my $nodename = PVE::INotify::nodename();
  79
  80 mkdir "/etc/pve/nodes/$nodename";
  81 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  82 mkdir $confdir;
  83
  84 my $var_run_tmpdir = "/var/run/qemu-server";
  85 mkdir $var_run_tmpdir;
  86
  87 my $lock_dir = "/var/lock/qemu-server";
  88 mkdir $lock_dir;
  89
  90 my $pcisysfs = "/sys/bus/pci";
  91
  92 my $confdesc = {
  93     onboot => {
  94         optional => 1,
  95         type => 'boolean',
  96         description => "Specifies whether a VM will be started during system bootup.",
  97         default => 0,
  98     },
  99     autostart => {
 100         optional => 1,
 101         type => 'boolean',
 102         description => "Automatic restart after crash (currently ignored).",
 103         default => 0,
 104     },
 105     hotplug => {
 106         optional => 1,
 107         type => 'string', format => 'pve-hotplug-features',
 108         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 109         default => 'network,disk,usb',
 110     },
 111     reboot => {
 112         optional => 1,
 113         type => 'boolean',
 114         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 115         default => 1,
 116     },
 117     lock => {
 118         optional => 1,
 119         type => 'string',
 120         description => "Lock/unlock the VM.",
 121         enum => [qw(migrate backup snapshot rollback)],
 122     },
 123     cpulimit => {
 124         optional => 1,
 125         type => 'integer',
 126         description => "Limit of CPU usage in per cent. Note if the computer has 2 CPUs, it has total of 200% CPU time. Value '0' indicates no CPU limit.\n\n.",
 127         minimum => 0,
 128         default => 0,
 129     },
 130     cpuunits => {
 131         optional => 1,
 132         type => 'integer',
 133         description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 134         minimum => 0,
 135         maximum => 500000,
 136         default => 1000,
 137     },
 138     memory => {
 139         optional => 1,
 140         type => 'integer',
 141         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 142         minimum => 16,
 143         default => 512,
 144     },
 145     balloon => {
 146         optional => 1,
 147         type => 'integer',
 148         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 149         minimum => 0,
 150     },
 151     shares => {
 152         optional => 1,
 153         type => 'integer',
 154         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 155         minimum => 0,
 156         maximum => 50000,
 157         default => 1000,
 158     },
 159     keyboard => {
 160         optional => 1,
 161         type => 'string',
 162         description => "Keybord layout for vnc server. Default is read from the datacenter configuration file.",
 163         enum => PVE::Tools::kvmkeymaplist(),
 164         default => 'en-us',
 165     },
 166     name => {
 167         optional => 1,
 168         type => 'string', format => 'dns-name',
 169         description => "Set a name for the VM. Only used on the configuration web interface.",
 170     },
 171     scsihw => {
 172         optional => 1,
 173         type => 'string',
 174         description => "scsi controller model",
 175         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 176         default => 'lsi',
 177     },
 178     description => {
 179         optional => 1,
 180         type => 'string',
 181         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 182     },
 183     ostype => {
 184         optional => 1,
 185         type => 'string',
 186         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 l24 l26 solaris)],
 187         description => <<EODESC,
 188 Used to enable special optimization/features for specific
 189 operating systems:
 190
 191 other  => unspecified OS
 192 wxp    => Microsoft Windows XP
 193 w2k    => Microsoft Windows 2000
 194 w2k3   => Microsoft Windows 2003
 195 w2k8   => Microsoft Windows 2008
 196 wvista => Microsoft Windows Vista
 197 win7   => Microsoft Windows 7
 198 win8   => Microsoft Windows 8/2012
 199 l24    => Linux 2.4 Kernel
 200 l26    => Linux 2.6/3.X Kernel
 201 solaris => solaris/opensolaris/openindiania kernel
 202
 203 other|l24|l26|solaris                       ... no special behaviour
 204 wxp|w2k|w2k3|w2k8|wvista|win7|win8  ... use --localtime switch
 205 EODESC
 206     },
 207     boot => {
 208         optional => 1,
 209         type => 'string',
 210         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 211         pattern => '[acdn]{1,4}',
 212         default => 'cdn',
 213     },
 214     bootdisk => {
 215         optional => 1,
 216         type => 'string', format => 'pve-qm-bootdisk',
 217         description => "Enable booting from specified disk.",
 218         pattern => '(ide|sata|scsi|virtio)\d+',
 219     },
 220     smp => {
 221         optional => 1,
 222         type => 'integer',
 223         description => "The number of CPUs. Please use option -sockets instead.",
 224         minimum => 1,
 225         default => 1,
 226     },
 227     sockets => {
 228         optional => 1,
 229         type => 'integer',
 230         description => "The number of CPU sockets.",
 231         minimum => 1,
 232         default => 1,
 233     },
 234     cores => {
 235         optional => 1,
 236         type => 'integer',
 237         description => "The number of cores per socket.",
 238         minimum => 1,
 239         default => 1,
 240     },
 241     numa => {
 242         optional => 1,
 243         type => 'boolean',
 244         description => "Enable/disable Numa.",
 245         default => 0,
 246     },
 247     vcpus => {
 248         optional => 1,
 249         type => 'integer',
 250         description => "Number of hotplugged vcpus.",
 251         minimum => 1,
 252         default => 0,
 253     },
 254     acpi => {
 255         optional => 1,
 256         type => 'boolean',
 257         description => "Enable/disable ACPI.",
 258         default => 1,
 259     },
 260     agent => {
 261         optional => 1,
 262         type => 'boolean',
 263         description => "Enable/disable Qemu GuestAgent.",
 264         default => 0,
 265     },
 266     kvm => {
 267         optional => 1,
 268         type => 'boolean',
 269         description => "Enable/disable KVM hardware virtualization.",
 270         default => 1,
 271     },
 272     tdf => {
 273         optional => 1,
 274         type => 'boolean',
 275         description => "Enable/disable time drift fix.",
 276         default => 0,
 277     },
 278     localtime => {
 279         optional => 1,
 280         type => 'boolean',
 281         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 282     },
 283     freeze => {
 284         optional => 1,
 285         type => 'boolean',
 286         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 287     },
 288     vga => {
 289         optional => 1,
 290         type => 'string',
 291         description => "Select VGA type. If you want to use high resolution modes (>= 1280x1024x16) then you should use option 'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and 'cirrur' for other OS types. Option 'qxl' enables the SPICE display sever. You can also run without any graphic card using a serial devive as terminal.",
 292         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 293     },
 294     watchdog => {
 295         optional => 1,
 296         type => 'string', format => 'pve-qm-watchdog',
 297         typetext => '[[model=]i6300esb|ib700] [,[action=]reset|shutdown|poweroff|pause|debug|none]',
 298         description => "Create a virtual hardware watchdog device.  Once enabled (by a guest action), the watchdog must be periodically polled by an agent inside the guest or else the guest will be restarted (or execute the action specified)",
 299     },
 300     startdate => {
 301         optional => 1,
 302         type => 'string',
 303         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 304         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 305         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 306         default => 'now',
 307     },
 308     startup =>  get_standard_option('pve-startup-order'),
 309     template => {
 310         optional => 1,
 311         type => 'boolean',
 312         description => "Enable/disable Template.",
 313         default => 0,
 314     },
 315     args => {
 316         optional => 1,
 317         type => 'string',
 318         description => <<EODESCR,
 319 Note: this option is for experts only. It allows you to pass arbitrary arguments to kvm, for example:
 320
 321 args: -no-reboot -no-hpet
 322 EODESCR
 323     },
 324     tablet => {
 325         optional => 1,
 326         type => 'boolean',
 327         default => 1,
 328         description => "Enable/disable the usb tablet device. This device is usually needed to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with normal VNC clients. If you're running lots of console-only guests on one host, you may consider disabling this to save some context switches. This is turned of by default if you use spice (vga=qxl).",
 329     },
 330     migrate_speed => {
 331         optional => 1,
 332         type => 'integer',
 333         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 334         minimum => 0,
 335         default => 0,
 336     },
 337     migrate_downtime => {
 338         optional => 1,
 339         type => 'number',
 340         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 341         minimum => 0,
 342         default => 0.1,
 343     },
 344     cdrom => {
 345         optional => 1,
 346         type => 'string', format => 'pve-qm-drive',
 347         typetext => 'volume',
 348         description => "This is an alias for option -ide2",
 349     },
 350     cpu => {
 351         optional => 1,
 352         description => "Emulated CPU type.",
 353         type => 'string',
 354         enum => [ qw(486 athlon pentium pentium2 pentium3 coreduo core2duo kvm32 kvm64 qemu32 qemu64 phenom Conroe Penryn Nehalem Westmere SandyBridge IvyBridge Haswell Broadwell Opteron_G1 Opteron_G2 Opteron_G3 Opteron_G4 Opteron_G5 host) ],
 355         default => 'kvm64',
 356     },
 357     parent => get_standard_option('pve-snapshot-name', {
 358         optional => 1,
 359         description => "Parent snapshot name. This is used internally, and should not be modified.",
 360     }),
 361     snaptime => {
 362         optional => 1,
 363         description => "Timestamp for snapshots.",
 364         type => 'integer',
 365         minimum => 0,
 366     },
 367     vmstate => {
 368         optional => 1,
 369         type => 'string', format => 'pve-volume-id',
 370         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 371     },
 372     machine => {
 373         description => "Specific the Qemu machine type.",
 374         type => 'string',
 375         pattern => '(pc|pc(-i440fx)?-\d+\.\d+|q35|pc-q35-\d+\.\d+)',
 376         maxLength => 40,
 377         optional => 1,
 378     },
 379     smbios1 => {
 380         description => "Specify SMBIOS type 1 fields.",
 381         type => 'string', format => 'pve-qm-smbios1',
 382         typetext => "[manufacturer=str][,product=str][,version=str][,serial=str] [,uuid=uuid][,sku=str][,family=str]",
 383         maxLength => 256,
 384         optional => 1,
 385     },
 386 };
 387
 388 # what about other qemu settings ?
 389 #cpu => 'string',
 390 #machine => 'string',
 391 #fda => 'file',
 392 #fdb => 'file',
 393 #mtdblock => 'file',
 394 #sd => 'file',
 395 #pflash => 'file',
 396 #snapshot => 'bool',
 397 #bootp => 'file',
 398 ##tftp => 'dir',
 399 ##smb => 'dir',
 400 #kernel => 'file',
 401 #append => 'string',
 402 #initrd => 'file',
 403 ##soundhw => 'string',
 404
 405 while (my ($k, $v) = each %$confdesc) {
 406     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 407 }
 408
 409 my $MAX_IDE_DISKS = 4;
 410 my $MAX_SCSI_DISKS = 14;
 411 my $MAX_VIRTIO_DISKS = 16;
 412 my $MAX_SATA_DISKS = 6;
 413 my $MAX_USB_DEVICES = 5;
 414 my $MAX_NETS = 32;
 415 my $MAX_UNUSED_DISKS = 8;
 416 my $MAX_HOSTPCI_DEVICES = 4;
 417 my $MAX_SERIAL_PORTS = 4;
 418 my $MAX_PARALLEL_PORTS = 3;
 419 my $MAX_NUMA = 8;
 420 my $MAX_MEM = 4194304;
 421 my $STATICMEM = 1024;
 422
 423 my $numadesc = {
 424     optional => 1,
 425     type => 'string', format => 'pve-qm-numanode',
 426     typetext => "cpus=<id[-id],memory=<mb>[[,hostnodes=<id[-id]>] [,policy=<preferred|bind|interleave>]]",
 427     description => "numa topology",
 428 };
 429 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 430
 431 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 432     $confdesc->{"numa$i"} = $numadesc;
 433 }
 434
 435 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 436                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 437                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 438 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 439
 440 my $netdesc = {
 441     optional => 1,
 442     type => 'string', format => 'pve-qm-net',
 443     typetext => "MODEL=XX:XX:XX:XX:XX:XX [,bridge=<dev>][,queues=<nbqueues>][,rate=<mbps>] [,tag=<vlanid>][,firewall=0|1],link_down=0|1]",
 444     description => <<EODESCR,
 445 Specify network devices.
 446
 447 MODEL is one of: $nic_model_list_txt
 448
 449 XX:XX:XX:XX:XX:XX should be an unique MAC address. This is
 450 automatically generated if not specified.
 451
 452 The bridge parameter can be used to automatically add the interface to a bridge device. The Proxmox VE standard bridge is called 'vmbr0'.
 453
 454 Option 'rate' is used to limit traffic bandwidth from and to this interface. It is specified as floating point number, unit is 'Megabytes per second'.
 455
 456 If you specify no bridge, we create a kvm 'user' (NATed) network device, which provides DHCP and DNS services. The following addresses are used:
 457
 458 10.0.2.2   Gateway
 459 10.0.2.3   DNS Server
 460 10.0.2.4   SMB Server
 461
 462 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 463
 464 EODESCR
 465 };
 466 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 467
 468 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 469     $confdesc->{"net$i"} = $netdesc;
 470 }
 471
 472 my $drivename_hash;
 473
 474 my $idedesc = {
 475     optional => 1,
 476     type => 'string', format => 'pve-qm-drive',
 477     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads] [,discard=ignore|on]',
 478     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
 479 };
 480 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
 481
 482 my $scsidesc = {
 483     optional => 1,
 484     type => 'string', format => 'pve-qm-drive',
 485     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads] [,discard=ignore|on] [,iothread=on] [,queues=<nbqueues>]',
 486     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
 487 };
 488 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 489
 490 my $satadesc = {
 491     optional => 1,
 492     type => 'string', format => 'pve-qm-drive',
 493     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads]  [,discard=ignore|on]',
 494     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
 495 };
 496 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
 497
 498 my $virtiodesc = {
 499     optional => 1,
 500     type => 'string', format => 'pve-qm-drive',
 501     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads]  [,discard=ignore|on] [,iothread=on]',
 502     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
 503 };
 504 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
 505
 506 my $usbdesc = {
 507     optional => 1,
 508     type => 'string', format => 'pve-qm-usb-device',
 509     typetext => 'host=HOSTUSBDEVICE|spice',
 510     description => <<EODESCR,
 511 Configure an USB device (n is 0 to 4). This can be used to
 512 pass-through usb devices to the guest. HOSTUSBDEVICE syntax is:
 513
 514 'bus-port(.port)*' (decimal numbers) or
 515 'vendor_id:product_id' (hexadeciaml numbers)
 516
 517 You can use the 'lsusb -t' command to list existing usb devices.
 518
 519 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 520
 521 The value 'spice' can be used to add a usb redirection devices for spice.
 522
 523 EODESCR
 524 };
 525 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
 526
 527 my $hostpcidesc = {
 528         optional => 1,
 529         type => 'string', format => 'pve-qm-hostpci',
 530         typetext => "[host=]HOSTPCIDEVICE [,driver=kvm|vfio] [,rombar=on|off] [,pcie=0|1] [,x-vga=on|off]",
 531         description => <<EODESCR,
 532 Map host pci devices. HOSTPCIDEVICE syntax is:
 533
 534 'bus:dev.func' (hexadecimal numbers)
 535
 536 You can us the 'lspci' command to list existing pci devices.
 537
 538 The 'rombar' option determines whether or not the device's ROM will be visible in the guest's memory map (default is 'on').
 539
 540 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 541
 542 Experimental: user reported problems with this option.
 543 EODESCR
 544 };
 545 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
 546
 547 my $serialdesc = {
 548         optional => 1,
 549         type => 'string',
 550         pattern => '(/dev/.+|socket)',
 551         description =>  <<EODESCR,
 552 Create a serial device inside the VM (n is 0 to 3), and pass through a host serial device (i.e. /dev/ttyS0), or create a unix socket on the host side (use 'qm terminal' to open a terminal connection).
 553
 554 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 555
 556 Experimental: user reported problems with this option.
 557 EODESCR
 558 };
 559
 560 my $paralleldesc= {
 561         optional => 1,
 562         type => 'string',
 563         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
 564         description =>  <<EODESCR,
 565 Map host parallel devices (n is 0 to 2).
 566
 567 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 568
 569 Experimental: user reported problems with this option.
 570 EODESCR
 571 };
 572
 573 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
 574     $confdesc->{"parallel$i"} = $paralleldesc;
 575 }
 576
 577 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
 578     $confdesc->{"serial$i"} = $serialdesc;
 579 }
 580
 581 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
 582     $confdesc->{"hostpci$i"} = $hostpcidesc;
 583 }
 584
 585 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
 586     $drivename_hash->{"ide$i"} = 1;
 587     $confdesc->{"ide$i"} = $idedesc;
 588 }
 589
 590 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
 591     $drivename_hash->{"sata$i"} = 1;
 592     $confdesc->{"sata$i"} = $satadesc;
 593 }
 594
 595 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
 596     $drivename_hash->{"scsi$i"} = 1;
 597     $confdesc->{"scsi$i"} = $scsidesc ;
 598 }
 599
 600 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
 601     $drivename_hash->{"virtio$i"} = 1;
 602     $confdesc->{"virtio$i"} = $virtiodesc;
 603 }
 604
 605 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
 606     $confdesc->{"usb$i"} = $usbdesc;
 607 }
 608
 609 my $unuseddesc = {
 610     optional => 1,
 611     type => 'string', format => 'pve-volume-id',
 612     description => "Reference to unused volumes.",
 613 };
 614
 615 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
 616     $confdesc->{"unused$i"} = $unuseddesc;
 617 }
 618
 619 my $kvm_api_version = 0;
 620
 621 sub kvm_version {
 622
 623     return $kvm_api_version if $kvm_api_version;
 624
 625     my $fh = IO::File->new("</dev/kvm") ||
 626         return 0;
 627
 628     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
 629         $kvm_api_version = $v;
 630     }
 631
 632     $fh->close();
 633
 634     return  $kvm_api_version;
 635 }
 636
 637 my $kvm_user_version;
 638
 639 sub kvm_user_version {
 640
 641     return $kvm_user_version if $kvm_user_version;
 642
 643     $kvm_user_version = 'unknown';
 644
 645     my $tmp = `kvm -help 2>/dev/null`;
 646
 647     if ($tmp =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)[,\s]/) {
 648         $kvm_user_version = $2;
 649     }
 650
 651     return $kvm_user_version;
 652
 653 }
 654
 655 my $kernel_has_vhost_net = -c '/dev/vhost-net';
 656
 657 sub disknames {
 658     # order is important - used to autoselect boot disk
 659     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
 660             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
 661             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
 662             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))));
 663 }
 664
 665 sub valid_drivename {
 666     my $dev = shift;
 667
 668     return defined($drivename_hash->{$dev});
 669 }
 670
 671 sub option_exists {
 672     my $key = shift;
 673     return defined($confdesc->{$key});
 674 }
 675
 676 sub nic_models {
 677     return $nic_model_list;
 678 }
 679
 680 sub os_list_description {
 681
 682     return {
 683         other => 'Other',
 684         wxp => 'Windows XP',
 685         w2k => 'Windows 2000',
 686         w2k3 =>, 'Windows 2003',
 687         w2k8 => 'Windows 2008',
 688         wvista => 'Windows Vista',
 689         win7 => 'Windows 7',
 690         win8 => 'Windows 8/2012',
 691         l24 => 'Linux 2.4',
 692         l26 => 'Linux 2.6',
 693     };
 694 }
 695
 696 my $cdrom_path;
 697
 698 sub get_cdrom_path {
 699
 700     return  $cdrom_path if $cdrom_path;
 701
 702     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
 703     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
 704     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
 705 }
 706
 707 sub get_iso_path {
 708     my ($storecfg, $vmid, $cdrom) = @_;
 709
 710     if ($cdrom eq 'cdrom') {
 711         return get_cdrom_path();
 712     } elsif ($cdrom eq 'none') {
 713         return '';
 714     } elsif ($cdrom =~ m|^/|) {
 715         return $cdrom;
 716     } else {
 717         return PVE::Storage::path($storecfg, $cdrom);
 718     }
 719 }
 720
 721 # try to convert old style file names to volume IDs
 722 sub filename_to_volume_id {
 723     my ($vmid, $file, $media) = @_;
 724
 725     if (!($file eq 'none' || $file eq 'cdrom' ||
 726           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
 727
 728         return undef if $file =~ m|/|;
 729
 730         if ($media && $media eq 'cdrom') {
 731             $file = "local:iso/$file";
 732         } else {
 733             $file = "local:$vmid/$file";
 734         }
 735     }
 736
 737     return $file;
 738 }
 739
 740 sub verify_media_type {
 741     my ($opt, $vtype, $media) = @_;
 742
 743     return if !$media;
 744
 745     my $etype;
 746     if ($media eq 'disk') {
 747         $etype = 'images';
 748     } elsif ($media eq 'cdrom') {
 749         $etype = 'iso';
 750     } else {
 751         die "internal error";
 752     }
 753
 754     return if ($vtype eq $etype);
 755
 756     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
 757 }
 758
 759 sub cleanup_drive_path {
 760     my ($opt, $storecfg, $drive) = @_;
 761
 762     # try to convert filesystem paths to volume IDs
 763
 764     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
 765         ($drive->{file} !~ m|^/dev/.+|) &&
 766         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
 767         ($drive->{file} !~ m/^\d+$/)) {
 768         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
 769         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
 770         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
 771         verify_media_type($opt, $vtype, $drive->{media});
 772         $drive->{file} = $volid;
 773     }
 774
 775     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
 776 }
 777
 778 sub create_conf_nolock {
 779     my ($vmid, $settings) = @_;
 780
 781     my $filename = config_file($vmid);
 782
 783     die "configuration file '$filename' already exists\n" if -f $filename;
 784
 785     my $defaults = load_defaults();
 786
 787     $settings->{name} = "vm$vmid" if !$settings->{name};
 788     $settings->{memory} = $defaults->{memory} if !$settings->{memory};
 789
 790     my $data = '';
 791     foreach my $opt (keys %$settings) {
 792         next if !$confdesc->{$opt};
 793
 794         my $value = $settings->{$opt};
 795         next if !$value;
 796
 797         $data .= "$opt: $value\n";
 798     }
 799
 800     PVE::Tools::file_set_contents($filename, $data);
 801 }
 802
 803 sub parse_hotplug_features {
 804     my ($data) = @_;
 805
 806     my $res = {};
 807
 808     return $res if $data eq '0';
 809
 810     $data = $confdesc->{hotplug}->{default} if $data eq '1';
 811
 812     foreach my $feature (PVE::Tools::split_list($data)) {
 813         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
 814             $res->{$1} = 1;
 815         } else {
 816             warn "ignoring unknown hotplug feature '$feature'\n";
 817         }
 818     }
 819     return $res;
 820 }
 821
 822 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
 823 sub pve_verify_hotplug_features {
 824     my ($value, $noerr) = @_;
 825
 826     return $value if parse_hotplug_features($value);
 827
 828     return undef if $noerr;
 829
 830     die "unable to parse hotplug option\n";
 831 }
 832
 833 my $parse_size = sub {
 834     my ($value) = @_;
 835
 836     return undef if $value !~ m/^(\d+(\.\d+)?)([KMG])?$/;
 837     my ($size, $unit) = ($1, $3);
 838     if ($unit) {
 839         if ($unit eq 'K') {
 840             $size = $size * 1024;
 841         } elsif ($unit eq 'M') {
 842             $size = $size * 1024 * 1024;
 843         } elsif ($unit eq 'G') {
 844             $size = $size * 1024 * 1024 * 1024;
 845         }
 846     }
 847     return int($size);
 848 };
 849
 850 my $format_size = sub {
 851     my ($size) = @_;
 852
 853     $size = int($size);
 854
 855     my $kb = int($size/1024);
 856     return $size if $kb*1024 != $size;
 857
 858     my $mb = int($kb/1024);
 859     return "${kb}K" if $mb*1024 != $kb;
 860
 861     my $gb = int($mb/1024);
 862     return "${mb}M" if $gb*1024 != $mb;
 863
 864     return "${gb}G";
 865 };
 866
 867 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
 868 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
 869 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
 870 #        [,aio=native|threads][,discard=ignore|on][,iothread=on]
 871
 872 sub parse_drive {
 873     my ($key, $data) = @_;
 874
 875     my $res = {};
 876
 877     # $key may be undefined - used to verify JSON parameters
 878     if (!defined($key)) {
 879         $res->{interface} = 'unknown'; # should not harm when used to verify parameters
 880         $res->{index} = 0;
 881     } elsif ($key =~ m/^([^\d]+)(\d+)$/) {
 882         $res->{interface} = $1;
 883         $res->{index} = $2;
 884     } else {
 885         return undef;
 886     }
 887
 888     foreach my $p (split (/,/, $data)) {
 889         next if $p =~ m/^\s*$/;
 890
 891         if ($p =~ m/^(file|volume|cyls|heads|secs|trans|media|snapshot|cache|format|rerror|werror|backup|aio|bps|mbps|mbps_max|bps_rd|mbps_rd|mbps_rd_max|bps_wr|mbps_wr|mbps_wr_max|iops|iops_max|iops_rd|iops_rd_max|iops_wr|iops_wr_max|size|discard|iothread|queues)=(.+)$/) {
 892             my ($k, $v) = ($1, $2);
 893
 894             $k = 'file' if $k eq 'volume';
 895
 896             return undef if defined $res->{$k};
 897
 898             if ($k eq 'bps' || $k eq 'bps_rd' || $k eq 'bps_wr') {
 899                 return undef if !$v || $v !~ m/^\d+/;
 900                 $k = "m$k";
 901                 $v = sprintf("%.3f", $v / (1024*1024));
 902             }
 903             $res->{$k} = $v;
 904         } else {
 905             if (!$res->{file} && $p !~ m/=/) {
 906                 $res->{file} = $p;
 907             } else {
 908                 return undef;
 909             }
 910         }
 911     }
 912
 913     return undef if !$res->{file};
 914
 915     if($res->{file} =~ m/\.(raw|cow|qcow|qcow2|vmdk|cloop)$/){
 916         $res->{format} = $1;
 917     }
 918
 919     return undef if $res->{cache} &&
 920         $res->{cache} !~ m/^(off|none|writethrough|writeback|unsafe|directsync)$/;
 921     return undef if $res->{snapshot} && $res->{snapshot} !~ m/^(on|off)$/;
 922     return undef if $res->{cyls} && $res->{cyls} !~ m/^\d+$/;
 923     return undef if $res->{heads} && $res->{heads} !~ m/^\d+$/;
 924     return undef if $res->{secs} && $res->{secs} !~ m/^\d+$/;
 925     return undef if $res->{media} && $res->{media} !~ m/^(disk|cdrom)$/;
 926     return undef if $res->{trans} && $res->{trans} !~ m/^(none|lba|auto)$/;
 927     return undef if $res->{format} && $res->{format} !~ m/^(raw|cow|qcow|qcow2|vmdk|cloop)$/;
 928     return undef if $res->{rerror} && $res->{rerror} !~ m/^(ignore|report|stop)$/;
 929     return undef if $res->{werror} && $res->{werror} !~ m/^(enospc|ignore|report|stop)$/;
 930     return undef if $res->{backup} && $res->{backup} !~ m/^(yes|no)$/;
 931     return undef if $res->{aio} && $res->{aio} !~ m/^(native|threads)$/;
 932     return undef if $res->{discard} && $res->{discard} !~ m/^(ignore|on)$/;
 933     return undef if $res->{iothread} && $res->{iothread} !~ m/^(on)$/;
 934     return undef if $res->{queues} && ($res->{queues} !~ m/^\d+$/ || $res->{queues} < 2);
 935
 936     return undef if $res->{mbps_rd} && $res->{mbps};
 937     return undef if $res->{mbps_wr} && $res->{mbps};
 938
 939     return undef if $res->{mbps} && $res->{mbps} !~ m/^\d+(\.\d+)?$/;
 940     return undef if $res->{mbps_max} && $res->{mbps_max} !~ m/^\d+(\.\d+)?$/;
 941     return undef if $res->{mbps_rd} && $res->{mbps_rd} !~ m/^\d+(\.\d+)?$/;
 942     return undef if $res->{mbps_rd_max} && $res->{mbps_rd_max} !~ m/^\d+(\.\d+)?$/;
 943     return undef if $res->{mbps_wr} && $res->{mbps_wr} !~ m/^\d+(\.\d+)?$/;
 944     return undef if $res->{mbps_wr_max} && $res->{mbps_wr_max} !~ m/^\d+(\.\d+)?$/;
 945
 946     return undef if $res->{iops_rd} && $res->{iops};
 947     return undef if $res->{iops_wr} && $res->{iops};
 948
 949
 950     return undef if $res->{iops} && $res->{iops} !~ m/^\d+$/;
 951     return undef if $res->{iops_max} && $res->{iops_max} !~ m/^\d+$/;
 952     return undef if $res->{iops_rd} && $res->{iops_rd} !~ m/^\d+$/;
 953     return undef if $res->{iops_rd_max} && $res->{iops_rd_max} !~ m/^\d+$/;
 954     return undef if $res->{iops_wr} && $res->{iops_wr} !~ m/^\d+$/;
 955     return undef if $res->{iops_wr_max} && $res->{iops_wr_max} !~ m/^\d+$/;
 956
 957
 958     if ($res->{size}) {
 959         return undef if !defined($res->{size} = &$parse_size($res->{size}));
 960     }
 961
 962     if ($res->{media} && ($res->{media} eq 'cdrom')) {
 963         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
 964         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
 965         return undef if $res->{interface} eq 'virtio';
 966     }
 967
 968     # rerror does not work with scsi drives
 969     if ($res->{rerror}) {
 970         return undef if $res->{interface} eq 'scsi';
 971     }
 972
 973     return $res;
 974 }
 975
 976 my @qemu_drive_options = qw(heads secs cyls trans media format cache snapshot rerror werror aio discard iops iops_rd iops_wr iops_max iops_rd_max iops_wr_max);
 977
 978 sub print_drive {
 979     my ($vmid, $drive) = @_;
 980
 981     my $opts = '';
 982     foreach my $o (@qemu_drive_options, 'mbps', 'mbps_rd', 'mbps_wr', 'mbps_max', 'mbps_rd_max', 'mbps_wr_max', 'backup', 'iothread', 'queues') {
 983         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
 984     }
 985
 986     if ($drive->{size}) {
 987         $opts .= ",size=" . &$format_size($drive->{size});
 988     }
 989
 990     return "$drive->{file}$opts";
 991 }
 992
 993 sub scsi_inquiry {
 994     my($fh, $noerr) = @_;
 995
 996     my $SG_IO = 0x2285;
 997     my $SG_GET_VERSION_NUM = 0x2282;
 998
 999     my $versionbuf = "\x00" x 8;
1000     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1001     if (!$ret) {
1002         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1003         return undef;
1004     }
1005     my $version = unpack("I", $versionbuf);
1006     if ($version < 30000) {
1007         die "scsi generic interface too old\n"  if !$noerr;
1008         return undef;
1009     }
1010
1011     my $buf = "\x00" x 36;
1012     my $sensebuf = "\x00" x 8;
1013     my $cmd = pack("C x3 C x1", 0x12, 36);
1014
1015     # see /usr/include/scsi/sg.h
1016     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1017
1018     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1019                       length($sensebuf), 0, length($buf), $buf,
1020                       $cmd, $sensebuf, 6000);
1021
1022     $ret = ioctl($fh, $SG_IO, $packet);
1023     if (!$ret) {
1024         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1025         return undef;
1026     }
1027
1028     my @res = unpack($sg_io_hdr_t, $packet);
1029     if ($res[17] || $res[18]) {
1030         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1031         return undef;
1032     }
1033
1034     my $res = {};
1035     (my $byte0, my $byte1, $res->{vendor},
1036      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1037
1038     $res->{removable} = $byte1 & 128 ? 1 : 0;
1039     $res->{type} = $byte0 & 31;
1040
1041     return $res;
1042 }
1043
1044 sub path_is_scsi {
1045     my ($path) = @_;
1046
1047     my $fh = IO::File->new("+<$path") || return undef;
1048     my $res = scsi_inquiry($fh, 1);
1049     close($fh);
1050
1051     return $res;
1052 }
1053
1054 sub machine_type_is_q35 {
1055     my ($conf) = @_;
1056
1057     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1058 }
1059
1060 sub print_tabletdevice_full {
1061     my ($conf) = @_;
1062
1063     my $q35 = machine_type_is_q35($conf);
1064
1065     # we use uhci for old VMs because tablet driver was buggy in older qemu
1066     my $usbbus = $q35 ? "ehci" : "uhci";
1067
1068     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1069 }
1070
1071 sub print_drivedevice_full {
1072     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1073
1074     my $device = '';
1075     my $maxdev = 0;
1076
1077     if ($drive->{interface} eq 'virtio') {
1078         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1079         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1080         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1081     } elsif ($drive->{interface} eq 'scsi') {
1082
1083         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1084         my $unit = $drive->{index} % $maxdev;
1085         my $devicetype = 'hd';
1086         my $path = '';
1087         if (drive_is_cdrom($drive)) {
1088               $devicetype = 'cd';
1089         } else {
1090               if ($drive->{file} =~ m|^/|) {
1091                   $path = $drive->{file};
1092               } else {
1093                   $path = PVE::Storage::path($storecfg, $drive->{file});
1094               }
1095
1096               if($path =~ m/^iscsi\:\/\//){
1097                   $devicetype = 'generic';
1098               } else {
1099                   if (my $info = path_is_scsi($path)) {
1100                       if ($info->{type} == 0) {
1101                           $devicetype = 'block';
1102                       } elsif ($info->{type} == 1) { # tape
1103                           $devicetype = 'generic';
1104                       }
1105                   }
1106               }
1107          }
1108
1109         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1110             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1111         } else {
1112             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1113         }
1114
1115     } elsif ($drive->{interface} eq 'ide'){
1116         $maxdev = 2;
1117         my $controller = int($drive->{index} / $maxdev);
1118         my $unit = $drive->{index} % $maxdev;
1119         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1120
1121         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1122     } elsif ($drive->{interface} eq 'sata'){
1123         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1124         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1125         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1126     } elsif ($drive->{interface} eq 'usb') {
1127         die "implement me";
1128         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1129     } else {
1130         die "unsupported interface type";
1131     }
1132
1133     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1134
1135     return $device;
1136 }
1137
1138 sub get_initiator_name {
1139     my $initiator;
1140
1141     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1142     while (defined(my $line = <$fh>)) {
1143         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1144         $initiator = $1;
1145         last;
1146     }
1147     $fh->close();
1148
1149     return $initiator;
1150 }
1151
1152 sub print_drive_full {
1153     my ($storecfg, $vmid, $drive) = @_;
1154
1155     my $opts = '';
1156     foreach my $o (@qemu_drive_options) {
1157         next if $o eq 'bootindex';
1158         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1159     }
1160
1161     foreach my $o (qw(bps bps_rd bps_wr)) {
1162         my $v = $drive->{"m$o"};
1163         $opts .= ",$o=" . int($v*1024*1024) if $v;
1164     }
1165
1166     # aio native works only with O_DIRECT
1167     if (!$drive->{aio}) {
1168         if(!$drive->{cache} || $drive->{cache} eq 'none' || $drive->{cache} eq 'directsync') {
1169             $opts .= ",aio=native";
1170         } else {
1171             $opts .= ",aio=threads";
1172         }
1173     }
1174
1175
1176     my $path;
1177     my $volid = $drive->{file};
1178     if (drive_is_cdrom($drive)) {
1179         $path = get_iso_path($storecfg, $vmid, $volid);
1180     } else {
1181         if ($volid =~ m|^/|) {
1182             $path = $volid;
1183         } else {
1184             $path = PVE::Storage::path($storecfg, $volid);
1185         }
1186     }
1187
1188     $opts .= ",cache=none" if !$drive->{cache} && !drive_is_cdrom($drive);
1189
1190     my $detectzeroes = $drive->{discard} ? "unmap" : "on";
1191     $opts .= ",detect-zeroes=$detectzeroes" if !drive_is_cdrom($drive);
1192
1193     my $pathinfo = $path ? "file=$path," : '';
1194
1195     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1196 }
1197
1198 sub print_netdevice_full {
1199     my ($vmid, $conf, $net, $netid, $bridges) = @_;
1200
1201     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1202
1203     my $device = $net->{model};
1204     if ($net->{model} eq 'virtio') {
1205          $device = 'virtio-net-pci';
1206      };
1207
1208     my $pciaddr = print_pci_addr("$netid", $bridges);
1209     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1210     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1211         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1212         my $vectors = $net->{queues} * 2 + 2;
1213         $tmpstr .= ",vectors=$vectors,mq=on";
1214     }
1215     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1216     return $tmpstr;
1217 }
1218
1219 sub print_netdev_full {
1220     my ($vmid, $conf, $net, $netid) = @_;
1221
1222     my $i = '';
1223     if ($netid =~ m/^net(\d+)$/) {
1224         $i = int($1);
1225     }
1226
1227     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1228
1229     my $ifname = "tap${vmid}i$i";
1230
1231     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1232     die "interface name '$ifname' is too long (max 15 character)\n"
1233         if length($ifname) >= 16;
1234
1235     my $vhostparam = '';
1236     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1237
1238     my $vmname = $conf->{name} || "vm$vmid";
1239
1240     my $netdev = "";
1241
1242     if ($net->{bridge}) {
1243         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/pve-bridge,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1244     } else {
1245         $netdev = "type=user,id=$netid,hostname=$vmname";
1246     }
1247
1248     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1249
1250     return $netdev;
1251 }
1252
1253 sub drive_is_cdrom {
1254     my ($drive) = @_;
1255
1256     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1257
1258 }
1259
1260 sub parse_numa {
1261     my ($data) = @_;
1262
1263     my $res = {};
1264
1265     foreach my $kvp (split(/,/, $data)) {
1266
1267         if ($kvp =~ m/^memory=(\S+)$/) {
1268             $res->{memory} = $1;
1269         } elsif ($kvp =~ m/^policy=(preferred|bind|interleave)$/) {
1270             $res->{policy} = $1;
1271         } elsif ($kvp =~ m/^cpus=(\d+)(-(\d+))?$/) {
1272             $res->{cpus}->{start} = $1;
1273             $res->{cpus}->{end} = $3;
1274         } elsif ($kvp =~ m/^hostnodes=(\d+)(-(\d+))?$/) {
1275             $res->{hostnodes}->{start} = $1;
1276             $res->{hostnodes}->{end} = $3;
1277         } else {
1278             return undef;
1279         }
1280     }
1281
1282     return $res;
1283 }
1284
1285 sub parse_hostpci {
1286     my ($value) = @_;
1287
1288     return undef if !$value;
1289
1290
1291     my @list = split(/,/, $value);
1292     my $found;
1293
1294     my $res = {};
1295     foreach my $kv (@list) {
1296
1297         if ($kv =~ m/^(host=)?([a-f0-9]{2}:[a-f0-9]{2})(\.([a-f0-9]))?$/) {
1298             $found = 1;
1299             if(defined($4)){
1300                 push @{$res->{pciid}}, { id => $2 , function => $4};
1301
1302             }else{
1303                 my $pcidevices = lspci($2);
1304                 $res->{pciid} = $pcidevices->{$2};
1305             }
1306         } elsif ($kv =~ m/^driver=(kvm|vfio)$/) {
1307             $res->{driver} = $1;
1308         } elsif ($kv =~ m/^rombar=(on|off)$/) {
1309             $res->{rombar} = $1;
1310         } elsif ($kv =~ m/^x-vga=(on|off)$/) {
1311             $res->{'x-vga'} = $1;
1312         } elsif ($kv =~ m/^pcie=(\d+)$/) {
1313             $res->{pcie} = 1 if $1 == 1;
1314         } else {
1315             warn "unknown hostpci setting '$kv'\n";
1316         }
1317     }
1318
1319     return undef if !$found;
1320
1321     return $res;
1322 }
1323
1324 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1325 sub parse_net {
1326     my ($data) = @_;
1327
1328     my $res = {};
1329
1330     foreach my $kvp (split(/,/, $data)) {
1331
1332         if ($kvp =~ m/^(ne2k_pci|e1000|e1000-82540em|e1000-82544gc|e1000-82545em|rtl8139|pcnet|virtio|ne2k_isa|i82551|i82557b|i82559er|vmxnet3)(=([0-9a-f]{2}(:[0-9a-f]{2}){5}))?$/i) {
1333             my $model = lc($1);
1334             my $mac = defined($3) ? uc($3) : PVE::Tools::random_ether_addr();
1335             $res->{model} = $model;
1336             $res->{macaddr} = $mac;
1337         } elsif ($kvp =~ m/^bridge=(\S+)$/) {
1338             $res->{bridge} = $1;
1339         } elsif ($kvp =~ m/^queues=(\d+)$/) {
1340             $res->{queues} = $1;
1341         } elsif ($kvp =~ m/^rate=(\d+(\.\d+)?)$/) {
1342             $res->{rate} = $1;
1343         } elsif ($kvp =~ m/^tag=(\d+)$/) {
1344             $res->{tag} = $1;
1345         } elsif ($kvp =~ m/^firewall=([01])$/) {
1346             $res->{firewall} = $1;
1347         } elsif ($kvp =~ m/^link_down=([01])$/) {
1348             $res->{link_down} = $1;
1349         } else {
1350             return undef;
1351         }
1352
1353     }
1354
1355     return undef if !$res->{model};
1356
1357     return $res;
1358 }
1359
1360 sub print_net {
1361     my $net = shift;
1362
1363     my $res = "$net->{model}";
1364     $res .= "=$net->{macaddr}" if $net->{macaddr};
1365     $res .= ",bridge=$net->{bridge}" if $net->{bridge};
1366     $res .= ",rate=$net->{rate}" if $net->{rate};
1367     $res .= ",tag=$net->{tag}" if $net->{tag};
1368     $res .= ",firewall=1" if $net->{firewall};
1369     $res .= ",link_down=1" if $net->{link_down};
1370     $res .= ",queues=$net->{queues}" if $net->{queues};
1371
1372     return $res;
1373 }
1374
1375 sub add_random_macs {
1376     my ($settings) = @_;
1377
1378     foreach my $opt (keys %$settings) {
1379         next if $opt !~ m/^net(\d+)$/;
1380         my $net = parse_net($settings->{$opt});
1381         next if !$net;
1382         $settings->{$opt} = print_net($net);
1383     }
1384 }
1385
1386 sub add_unused_volume {
1387     my ($config, $volid) = @_;
1388
1389     my $key;
1390     for (my $ind = $MAX_UNUSED_DISKS - 1; $ind >= 0; $ind--) {
1391         my $test = "unused$ind";
1392         if (my $vid = $config->{$test}) {
1393             return if $vid eq $volid; # do not add duplicates
1394         } else {
1395             $key = $test;
1396         }
1397     }
1398
1399     die "To many unused volume - please delete them first.\n" if !$key;
1400
1401     $config->{$key} = $volid;
1402
1403     return $key;
1404 }
1405
1406 sub vm_is_volid_owner {
1407     my ($storecfg, $vmid, $volid) = @_;
1408
1409     if ($volid !~  m|^/|) {
1410         my ($path, $owner);
1411         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1412         if ($owner && ($owner == $vmid)) {
1413             return 1;
1414         }
1415     }
1416
1417     return undef;
1418 }
1419
1420 sub vmconfig_delete_pending_option {
1421     my ($conf, $key) = @_;
1422
1423     delete $conf->{pending}->{$key};
1424     my $pending_delete_hash = { $key => 1 };
1425     foreach my $opt (PVE::Tools::split_list($conf->{pending}->{delete})) {
1426         $pending_delete_hash->{$opt} = 1;
1427     }
1428     $conf->{pending}->{delete} = join(',', keys %$pending_delete_hash);
1429 }
1430
1431 sub vmconfig_undelete_pending_option {
1432     my ($conf, $key) = @_;
1433
1434     my $pending_delete_hash = {};
1435     foreach my $opt (PVE::Tools::split_list($conf->{pending}->{delete})) {
1436         $pending_delete_hash->{$opt} = 1;
1437     }
1438     delete $pending_delete_hash->{$key};
1439
1440     my @keylist = keys %$pending_delete_hash;
1441     if (scalar(@keylist)) {
1442         $conf->{pending}->{delete} = join(',', @keylist);
1443     } else {
1444         delete $conf->{pending}->{delete};
1445     }
1446 }
1447
1448 sub vmconfig_register_unused_drive {
1449     my ($storecfg, $vmid, $conf, $drive) = @_;
1450
1451     if (!drive_is_cdrom($drive)) {
1452         my $volid = $drive->{file};
1453         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1454             add_unused_volume($conf, $volid, $vmid);
1455         }
1456     }
1457 }
1458
1459 sub vmconfig_cleanup_pending {
1460     my ($conf) = @_;
1461
1462     # remove pending changes when nothing changed
1463     my $changes;
1464     foreach my $opt (keys %{$conf->{pending}}) {
1465         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
1466             $changes = 1;
1467             delete $conf->{pending}->{$opt};
1468         }
1469     }
1470
1471     # remove delete if option is not set
1472     my $pending_delete_hash = {};
1473     foreach my $opt (PVE::Tools::split_list($conf->{pending}->{delete})) {
1474         if (defined($conf->{$opt})) {
1475             $pending_delete_hash->{$opt} = 1;
1476         } else {
1477             $changes = 1;
1478         }
1479     }
1480
1481     my @keylist = keys %$pending_delete_hash;
1482     if (scalar(@keylist)) {
1483         $conf->{pending}->{delete} = join(',', @keylist);
1484     } else {
1485         delete $conf->{pending}->{delete};
1486     }
1487
1488     return $changes;
1489 }
1490
1491 my $valid_smbios1_options = {
1492     manufacturer => '\S+',
1493     product => '\S+',
1494     version => '\S+',
1495     serial => '\S+',
1496     uuid => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1497     sku => '\S+',
1498     family => '\S+',
1499 };
1500
1501 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
1502 sub parse_smbios1 {
1503     my ($data) = @_;
1504
1505     my $res = {};
1506
1507     foreach my $kvp (split(/,/, $data)) {
1508         return undef if $kvp !~ m/^(\S+)=(.+)$/;
1509         my ($k, $v) = split(/=/, $kvp);
1510         return undef if !defined($k) || !defined($v);
1511         return undef if !$valid_smbios1_options->{$k};
1512         return undef if $v !~ m/^$valid_smbios1_options->{$k}$/;
1513         $res->{$k} = $v;
1514     }
1515
1516     return $res;
1517 }
1518
1519 sub print_smbios1 {
1520     my ($smbios1) = @_;
1521
1522     my $data = '';
1523     foreach my $k (keys %$smbios1) {
1524         next if !defined($smbios1->{$k});
1525         next if !$valid_smbios1_options->{$k};
1526         $data .= ',' if $data;
1527         $data .= "$k=$smbios1->{$k}";
1528     }
1529     return $data;
1530 }
1531
1532 PVE::JSONSchema::register_format('pve-qm-smbios1', \&verify_smbios1);
1533 sub verify_smbios1 {
1534     my ($value, $noerr) = @_;
1535
1536     return $value if parse_smbios1($value);
1537
1538     return undef if $noerr;
1539
1540     die "unable to parse smbios (type 1) options\n";
1541 }
1542
1543 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
1544 sub verify_bootdisk {
1545     my ($value, $noerr) = @_;
1546
1547     return $value if valid_drivename($value);
1548
1549     return undef if $noerr;
1550
1551     die "invalid boot disk '$value'\n";
1552 }
1553
1554 PVE::JSONSchema::register_format('pve-qm-numanode', \&verify_numa);
1555 sub verify_numa {
1556     my ($value, $noerr) = @_;
1557
1558     return $value if parse_numa($value);
1559
1560     return undef if $noerr;
1561
1562     die "unable to parse numa options\n";
1563 }
1564
1565 PVE::JSONSchema::register_format('pve-qm-net', \&verify_net);
1566 sub verify_net {
1567     my ($value, $noerr) = @_;
1568
1569     return $value if parse_net($value);
1570
1571     return undef if $noerr;
1572
1573     die "unable to parse network options\n";
1574 }
1575
1576 PVE::JSONSchema::register_format('pve-qm-drive', \&verify_drive);
1577 sub verify_drive {
1578     my ($value, $noerr) = @_;
1579
1580     return $value if parse_drive(undef, $value);
1581
1582     return undef if $noerr;
1583
1584     die "unable to parse drive options\n";
1585 }
1586
1587 PVE::JSONSchema::register_format('pve-qm-hostpci', \&verify_hostpci);
1588 sub verify_hostpci {
1589     my ($value, $noerr) = @_;
1590
1591     return $value if parse_hostpci($value);
1592
1593     return undef if $noerr;
1594
1595     die "unable to parse pci id\n";
1596 }
1597
1598 PVE::JSONSchema::register_format('pve-qm-watchdog', \&verify_watchdog);
1599 sub verify_watchdog {
1600     my ($value, $noerr) = @_;
1601
1602     return $value if parse_watchdog($value);
1603
1604     return undef if $noerr;
1605
1606     die "unable to parse watchdog options\n";
1607 }
1608
1609 sub parse_watchdog {
1610     my ($value) = @_;
1611
1612     return undef if !$value;
1613
1614     my $res = {};
1615
1616     foreach my $p (split(/,/, $value)) {
1617         next if $p =~ m/^\s*$/;
1618
1619         if ($p =~ m/^(model=)?(i6300esb|ib700)$/) {
1620             $res->{model} = $2;
1621         } elsif ($p =~ m/^(action=)?(reset|shutdown|poweroff|pause|debug|none)$/) {
1622             $res->{action} = $2;
1623         } else {
1624             return undef;
1625         }
1626     }
1627
1628     return $res;
1629 }
1630
1631 sub parse_usb_device {
1632     my ($value) = @_;
1633
1634     return undef if !$value;
1635
1636     my @dl = split(/,/, $value);
1637     my $found;
1638
1639     my $res = {};
1640     foreach my $v (@dl) {
1641         if ($v =~ m/^host=(0x)?([0-9A-Fa-f]{4}):(0x)?([0-9A-Fa-f]{4})$/) {
1642             $found = 1;
1643             $res->{vendorid} = $2;
1644             $res->{productid} = $4;
1645         } elsif ($v =~ m/^host=(\d+)\-(\d+(\.\d+)*)$/) {
1646             $found = 1;
1647             $res->{hostbus} = $1;
1648             $res->{hostport} = $2;
1649         } elsif ($v =~ m/^spice$/) {
1650             $found = 1;
1651             $res->{spice} = 1;
1652         } else {
1653             return undef;
1654         }
1655     }
1656     return undef if !$found;
1657
1658     return $res;
1659 }
1660
1661 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
1662 sub verify_usb_device {
1663     my ($value, $noerr) = @_;
1664
1665     return $value if parse_usb_device($value);
1666
1667     return undef if $noerr;
1668
1669     die "unable to parse usb device\n";
1670 }
1671
1672 # add JSON properties for create and set function
1673 sub json_config_properties {
1674     my $prop = shift;
1675
1676     foreach my $opt (keys %$confdesc) {
1677         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
1678         $prop->{$opt} = $confdesc->{$opt};
1679     }
1680
1681     return $prop;
1682 }
1683
1684 sub check_type {
1685     my ($key, $value) = @_;
1686
1687     die "unknown setting '$key'\n" if !$confdesc->{$key};
1688
1689     my $type = $confdesc->{$key}->{type};
1690
1691     if (!defined($value)) {
1692         die "got undefined value\n";
1693     }
1694
1695     if ($value =~ m/[\n\r]/) {
1696         die "property contains a line feed\n";
1697     }
1698
1699     if ($type eq 'boolean') {
1700         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
1701         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
1702         die "type check ('boolean') failed - got '$value'\n";
1703     } elsif ($type eq 'integer') {
1704         return int($1) if $value =~ m/^(\d+)$/;
1705         die "type check ('integer') failed - got '$value'\n";
1706     } elsif ($type eq 'number') {
1707         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
1708         die "type check ('number') failed - got '$value'\n";
1709     } elsif ($type eq 'string') {
1710         if (my $fmt = $confdesc->{$key}->{format}) {
1711             if ($fmt eq 'pve-qm-drive') {
1712                 # special case - we need to pass $key to parse_drive()
1713                 my $drive = parse_drive($key, $value);
1714                 return $value if $drive;
1715                 die "unable to parse drive options\n";
1716             }
1717             PVE::JSONSchema::check_format($fmt, $value);
1718             return $value;
1719         }
1720         $value =~ s/^\"(.*)\"$/$1/;
1721         return $value;
1722     } else {
1723         die "internal error"
1724     }
1725 }
1726
1727 sub lock_config_full {
1728     my ($vmid, $timeout, $code, @param) = @_;
1729
1730     my $filename = config_file_lock($vmid);
1731
1732     my $res = lock_file($filename, $timeout, $code, @param);
1733
1734     die $@ if $@;
1735
1736     return $res;
1737 }
1738
1739 sub lock_config_mode {
1740     my ($vmid, $timeout, $shared, $code, @param) = @_;
1741
1742     my $filename = config_file_lock($vmid);
1743
1744     my $res = lock_file_full($filename, $timeout, $shared, $code, @param);
1745
1746     die $@ if $@;
1747
1748     return $res;
1749 }
1750
1751 sub lock_config {
1752     my ($vmid, $code, @param) = @_;
1753
1754     return lock_config_full($vmid, 10, $code, @param);
1755 }
1756
1757 sub cfs_config_path {
1758     my ($vmid, $node) = @_;
1759
1760     $node = $nodename if !$node;
1761     return "nodes/$node/qemu-server/$vmid.conf";
1762 }
1763
1764 sub check_iommu_support{
1765     #fixme : need to check IOMMU support
1766     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
1767
1768     my $iommu=1;
1769     return $iommu;
1770
1771 }
1772
1773 sub config_file {
1774     my ($vmid, $node) = @_;
1775
1776     my $cfspath = cfs_config_path($vmid, $node);
1777     return "/etc/pve/$cfspath";
1778 }
1779
1780 sub config_file_lock {
1781     my ($vmid) = @_;
1782
1783     return "$lock_dir/lock-$vmid.conf";
1784 }
1785
1786 sub touch_config {
1787     my ($vmid) = @_;
1788
1789     my $conf = config_file($vmid);
1790     utime undef, undef, $conf;
1791 }
1792
1793 sub destroy_vm {
1794     my ($storecfg, $vmid, $keep_empty_config) = @_;
1795
1796     my $conffile = config_file($vmid);
1797
1798     my $conf = load_config($vmid);
1799
1800     check_lock($conf);
1801
1802     # only remove disks owned by this VM
1803     foreach_drive($conf, sub {
1804         my ($ds, $drive) = @_;
1805
1806         return if drive_is_cdrom($drive);
1807
1808         my $volid = $drive->{file};
1809
1810         return if !$volid || $volid =~ m|^/|;
1811
1812         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
1813         return if !$path || !$owner || ($owner != $vmid);
1814
1815         PVE::Storage::vdisk_free($storecfg, $volid);
1816     });
1817
1818     if ($keep_empty_config) {
1819         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
1820     } else {
1821         unlink $conffile;
1822     }
1823
1824     # also remove unused disk
1825     eval {
1826         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
1827
1828         eval {
1829             PVE::Storage::foreach_volid($dl, sub {
1830                 my ($volid, $sid, $volname, $d) = @_;
1831                 PVE::Storage::vdisk_free($storecfg, $volid);
1832             });
1833         };
1834         warn $@ if $@;
1835
1836     };
1837     warn $@ if $@;
1838 }
1839
1840 sub load_config {
1841     my ($vmid, $node) = @_;
1842
1843     my $cfspath = cfs_config_path($vmid, $node);
1844
1845     my $conf = PVE::Cluster::cfs_read_file($cfspath);
1846
1847     die "no such VM ('$vmid')\n" if !defined($conf);
1848
1849     return $conf;
1850 }
1851
1852 sub parse_vm_config {
1853     my ($filename, $raw) = @_;
1854
1855     return undef if !defined($raw);
1856
1857     my $res = {
1858         digest => Digest::SHA::sha1_hex($raw),
1859         snapshots => {},
1860         pending => {},
1861     };
1862
1863     $filename =~ m|/qemu-server/(\d+)\.conf$|
1864         || die "got strange filename '$filename'";
1865
1866     my $vmid = $1;
1867
1868     my $conf = $res;
1869     my $descr = '';
1870     my $section = '';
1871
1872     my @lines = split(/\n/, $raw);
1873     foreach my $line (@lines) {
1874         next if $line =~ m/^\s*$/;
1875
1876         if ($line =~ m/^\[PENDING\]\s*$/i) {
1877             $section = 'pending';
1878             $conf->{description} = $descr if $descr;
1879             $descr = '';
1880             $conf = $res->{$section} = {};
1881             next;
1882
1883         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
1884             $section = $1;
1885             $conf->{description} = $descr if $descr;
1886             $descr = '';
1887             $conf = $res->{snapshots}->{$section} = {};
1888             next;
1889         }
1890
1891         if ($line =~ m/^\#(.*)\s*$/) {
1892             $descr .= PVE::Tools::decode_text($1) . "\n";
1893             next;
1894         }
1895
1896         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
1897             $descr .= PVE::Tools::decode_text($2);
1898         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
1899             $conf->{snapstate} = $1;
1900         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
1901             my $key = $1;
1902             my $value = $2;
1903             $conf->{$key} = $value;
1904         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
1905             my $value = $1;
1906             if ($section eq 'pending') {
1907                 $conf->{delete} = $value; # we parse this later
1908             } else {
1909                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
1910             }
1911         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
1912             my $key = $1;
1913             my $value = $2;
1914             eval { $value = check_type($key, $value); };
1915             if ($@) {
1916                 warn "vm $vmid - unable to parse value of '$key' - $@";
1917             } else {
1918                 my $fmt = $confdesc->{$key}->{format};
1919                 if ($fmt && $fmt eq 'pve-qm-drive') {
1920                     my $v = parse_drive($key, $value);
1921                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
1922                         $v->{file} = $volid;
1923                         $value = print_drive($vmid, $v);
1924                     } else {
1925                         warn "vm $vmid - unable to parse value of '$key'\n";
1926                         next;
1927                     }
1928                 }
1929
1930                 if ($key eq 'cdrom') {
1931                     $conf->{ide2} = $value;
1932                 } else {
1933                     $conf->{$key} = $value;
1934                 }
1935             }
1936         }
1937     }
1938
1939     $conf->{description} = $descr if $descr;
1940
1941     delete $res->{snapstate}; # just to be sure
1942
1943     return $res;
1944 }
1945
1946 sub write_vm_config {
1947     my ($filename, $conf) = @_;
1948
1949     delete $conf->{snapstate}; # just to be sure
1950
1951     if ($conf->{cdrom}) {
1952         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
1953         $conf->{ide2} = $conf->{cdrom};
1954         delete $conf->{cdrom};
1955     }
1956
1957     # we do not use 'smp' any longer
1958     if ($conf->{sockets}) {
1959         delete $conf->{smp};
1960     } elsif ($conf->{smp}) {
1961         $conf->{sockets} = $conf->{smp};
1962         delete $conf->{cores};
1963         delete $conf->{smp};
1964     }
1965
1966     my $used_volids = {};
1967
1968     my $cleanup_config = sub {
1969         my ($cref, $pending, $snapname) = @_;
1970
1971         foreach my $key (keys %$cref) {
1972             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
1973                 $key eq 'snapstate' || $key eq 'pending';
1974             my $value = $cref->{$key};
1975             if ($key eq 'delete') {
1976                 die "propertry 'delete' is only allowed in [PENDING]\n"
1977                     if !$pending;
1978                 # fixme: check syntax?
1979                 next;
1980             }
1981             eval { $value = check_type($key, $value); };
1982             die "unable to parse value of '$key' - $@" if $@;
1983
1984             $cref->{$key} = $value;
1985
1986             if (!$snapname && valid_drivename($key)) {
1987                 my $drive = parse_drive($key, $value);
1988                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
1989             }
1990         }
1991     };
1992
1993     &$cleanup_config($conf);
1994
1995     &$cleanup_config($conf->{pending}, 1);
1996
1997     foreach my $snapname (keys %{$conf->{snapshots}}) {
1998         die "internal error" if $snapname eq 'pending';
1999         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2000     }
2001
2002     # remove 'unusedX' settings if we re-add a volume
2003     foreach my $key (keys %$conf) {
2004         my $value = $conf->{$key};
2005         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2006             delete $conf->{$key};
2007         }
2008     }
2009
2010     my $generate_raw_config = sub {
2011         my ($conf) = @_;
2012
2013         my $raw = '';
2014
2015         # add description as comment to top of file
2016         my $descr = $conf->{description} || '';
2017         foreach my $cl (split(/\n/, $descr)) {
2018             $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2019         }
2020
2021         foreach my $key (sort keys %$conf) {
2022             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2023             $raw .= "$key: $conf->{$key}\n";
2024         }
2025         return $raw;
2026     };
2027
2028     my $raw = &$generate_raw_config($conf);
2029
2030     if (scalar(keys %{$conf->{pending}})){
2031         $raw .= "\n[PENDING]\n";
2032         $raw .= &$generate_raw_config($conf->{pending});
2033     }
2034
2035     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2036         $raw .= "\n[$snapname]\n";
2037         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2038     }
2039
2040     return $raw;
2041 }
2042
2043 sub update_config_nolock {
2044     my ($vmid, $conf, $skiplock) = @_;
2045
2046     check_lock($conf) if !$skiplock;
2047
2048     my $cfspath = cfs_config_path($vmid);
2049
2050     PVE::Cluster::cfs_write_file($cfspath, $conf);
2051 }
2052
2053 sub update_config {
2054     my ($vmid, $conf, $skiplock) = @_;
2055
2056     lock_config($vmid, &update_config_nolock, $conf, $skiplock);
2057 }
2058
2059 sub load_defaults {
2060
2061     my $res = {};
2062
2063     # we use static defaults from our JSON schema configuration
2064     foreach my $key (keys %$confdesc) {
2065         if (defined(my $default = $confdesc->{$key}->{default})) {
2066             $res->{$key} = $default;
2067         }
2068     }
2069
2070     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2071     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2072
2073     return $res;
2074 }
2075
2076 sub config_list {
2077     my $vmlist = PVE::Cluster::get_vmlist();
2078     my $res = {};
2079     return $res if !$vmlist || !$vmlist->{ids};
2080     my $ids = $vmlist->{ids};
2081
2082     foreach my $vmid (keys %$ids) {
2083         my $d = $ids->{$vmid};
2084         next if !$d->{node} || $d->{node} ne $nodename;
2085         next if !$d->{type} || $d->{type} ne 'qemu';
2086         $res->{$vmid}->{exists} = 1;
2087     }
2088     return $res;
2089 }
2090
2091 # test if VM uses local resources (to prevent migration)
2092 sub check_local_resources {
2093     my ($conf, $noerr) = @_;
2094
2095     my $loc_res = 0;
2096
2097     $loc_res = 1 if $conf->{hostusb}; # old syntax
2098     $loc_res = 1 if $conf->{hostpci}; # old syntax
2099
2100     foreach my $k (keys %$conf) {
2101         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2102         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2103     }
2104
2105     die "VM uses local resources\n" if $loc_res && !$noerr;
2106
2107     return $loc_res;
2108 }
2109
2110 # check if used storages are available on all nodes (use by migrate)
2111 sub check_storage_availability {
2112     my ($storecfg, $conf, $node) = @_;
2113
2114     foreach_drive($conf, sub {
2115         my ($ds, $drive) = @_;
2116
2117         my $volid = $drive->{file};
2118         return if !$volid;
2119
2120         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2121         return if !$sid;
2122
2123         # check if storage is available on both nodes
2124         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2125         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2126    });
2127 }
2128
2129 # list nodes where all VM images are available (used by has_feature API)
2130 sub shared_nodes {
2131     my ($conf, $storecfg) = @_;
2132
2133     my $nodelist = PVE::Cluster::get_nodelist();
2134     my $nodehash = { map { $_ => 1 } @$nodelist };
2135     my $nodename = PVE::INotify::nodename();
2136
2137     foreach_drive($conf, sub {
2138         my ($ds, $drive) = @_;
2139
2140         my $volid = $drive->{file};
2141         return if !$volid;
2142
2143         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2144         if ($storeid) {
2145             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2146             if ($scfg->{disable}) {
2147                 $nodehash = {};
2148             } elsif (my $avail = $scfg->{nodes}) {
2149                 foreach my $node (keys %$nodehash) {
2150                     delete $nodehash->{$node} if !$avail->{$node};
2151                 }
2152             } elsif (!$scfg->{shared}) {
2153                 foreach my $node (keys %$nodehash) {
2154                     delete $nodehash->{$node} if $node ne $nodename
2155                 }
2156             }
2157         }
2158     });
2159
2160     return $nodehash
2161 }
2162
2163 sub check_lock {
2164     my ($conf) = @_;
2165
2166     die "VM is locked ($conf->{lock})\n" if $conf->{lock};
2167 }
2168
2169 sub check_cmdline {
2170     my ($pidfile, $pid) = @_;
2171
2172     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2173     if (defined($fh)) {
2174         my $line = <$fh>;
2175         $fh->close;
2176         return undef if !$line;
2177         my @param = split(/\0/, $line);
2178
2179         my $cmd = $param[0];
2180         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2181
2182         for (my $i = 0; $i < scalar (@param); $i++) {
2183             my $p = $param[$i];
2184             next if !$p;
2185             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2186                 my $p = $param[$i+1];
2187                 return 1 if $p && ($p eq $pidfile);
2188                 return undef;
2189             }
2190         }
2191     }
2192     return undef;
2193 }
2194
2195 sub check_running {
2196     my ($vmid, $nocheck, $node) = @_;
2197
2198     my $filename = config_file($vmid, $node);
2199
2200     die "unable to find configuration file for VM $vmid - no such machine\n"
2201         if !$nocheck && ! -f $filename;
2202
2203     my $pidfile = pidfile_name($vmid);
2204
2205     if (my $fd = IO::File->new("<$pidfile")) {
2206         my $st = stat($fd);
2207         my $line = <$fd>;
2208         close($fd);
2209
2210         my $mtime = $st->mtime;
2211         if ($mtime > time()) {
2212             warn "file '$filename' modified in future\n";
2213         }
2214
2215         if ($line =~ m/^(\d+)$/) {
2216             my $pid = $1;
2217             if (check_cmdline($pidfile, $pid)) {
2218                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2219                     return $pid;
2220                 }
2221             }
2222         }
2223     }
2224
2225     return undef;
2226 }
2227
2228 sub vzlist {
2229
2230     my $vzlist = config_list();
2231
2232     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2233
2234     while (defined(my $de = $fd->read)) {
2235         next if $de !~ m/^(\d+)\.pid$/;
2236         my $vmid = $1;
2237         next if !defined($vzlist->{$vmid});
2238         if (my $pid = check_running($vmid)) {
2239             $vzlist->{$vmid}->{pid} = $pid;
2240         }
2241     }
2242
2243     return $vzlist;
2244 }
2245
2246 sub disksize {
2247     my ($storecfg, $conf) = @_;
2248
2249     my $bootdisk = $conf->{bootdisk};
2250     return undef if !$bootdisk;
2251     return undef if !valid_drivename($bootdisk);
2252
2253     return undef if !$conf->{$bootdisk};
2254
2255     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2256     return undef if !defined($drive);
2257
2258     return undef if drive_is_cdrom($drive);
2259
2260     my $volid = $drive->{file};
2261     return undef if !$volid;
2262
2263     return $drive->{size};
2264 }
2265
2266 my $last_proc_pid_stat;
2267
2268 # get VM status information
2269 # This must be fast and should not block ($full == false)
2270 # We only query KVM using QMP if $full == true (this can be slow)
2271 sub vmstatus {
2272     my ($opt_vmid, $full) = @_;
2273
2274     my $res = {};
2275
2276     my $storecfg = PVE::Storage::config();
2277
2278     my $list = vzlist();
2279     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2280
2281     my $cpucount = $cpuinfo->{cpus} || 1;
2282
2283     foreach my $vmid (keys %$list) {
2284         next if $opt_vmid && ($vmid ne $opt_vmid);
2285
2286         my $cfspath = cfs_config_path($vmid);
2287         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2288
2289         my $d = {};
2290         $d->{pid} = $list->{$vmid}->{pid};
2291
2292         # fixme: better status?
2293         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2294
2295         my $size = disksize($storecfg, $conf);
2296         if (defined($size)) {
2297             $d->{disk} = 0; # no info available
2298             $d->{maxdisk} = $size;
2299         } else {
2300             $d->{disk} = 0;
2301             $d->{maxdisk} = 0;
2302         }
2303
2304         $d->{cpus} = ($conf->{sockets} || 1) * ($conf->{cores} || 1);
2305         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2306         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2307
2308         $d->{name} = $conf->{name} || "VM $vmid";
2309         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024) : 0;
2310
2311         if ($conf->{balloon}) {
2312             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2313             $d->{shares} = defined($conf->{shares}) ? $conf->{shares} : 1000;
2314         }
2315
2316         $d->{uptime} = 0;
2317         $d->{cpu} = 0;
2318         $d->{mem} = 0;
2319
2320         $d->{netout} = 0;
2321         $d->{netin} = 0;
2322
2323         $d->{diskread} = 0;
2324         $d->{diskwrite} = 0;
2325
2326         $d->{template} = is_template($conf);
2327
2328         $res->{$vmid} = $d;
2329     }
2330
2331     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2332     foreach my $dev (keys %$netdev) {
2333         next if $dev !~ m/^tap([1-9]\d*)i/;
2334         my $vmid = $1;
2335         my $d = $res->{$vmid};
2336         next if !$d;
2337
2338         $d->{netout} += $netdev->{$dev}->{receive};
2339         $d->{netin} += $netdev->{$dev}->{transmit};
2340     }
2341
2342     my $ctime = gettimeofday;
2343
2344     foreach my $vmid (keys %$list) {
2345
2346         my $d = $res->{$vmid};
2347         my $pid = $d->{pid};
2348         next if !$pid;
2349
2350         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2351         next if !$pstat; # not running
2352
2353         my $used = $pstat->{utime} + $pstat->{stime};
2354
2355         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2356
2357         if ($pstat->{vsize}) {
2358             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2359         }
2360
2361         my $old = $last_proc_pid_stat->{$pid};
2362         if (!$old) {
2363             $last_proc_pid_stat->{$pid} = {
2364                 time => $ctime,
2365                 used => $used,
2366                 cpu => 0,
2367             };
2368             next;
2369         }
2370
2371         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2372
2373         if ($dtime > 1000) {
2374             my $dutime = $used -  $old->{used};
2375
2376             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2377             $last_proc_pid_stat->{$pid} = {
2378                 time => $ctime,
2379                 used => $used,
2380                 cpu => $d->{cpu},
2381             };
2382         } else {
2383             $d->{cpu} = $old->{cpu};
2384         }
2385     }
2386
2387     return $res if !$full;
2388
2389     my $qmpclient = PVE::QMPClient->new();
2390
2391     my $ballooncb = sub {
2392         my ($vmid, $resp) = @_;
2393
2394         my $info = $resp->{'return'};
2395         return if !$info->{max_mem};
2396
2397         my $d = $res->{$vmid};
2398
2399         # use memory assigned to VM
2400         $d->{maxmem} = $info->{max_mem};
2401         $d->{balloon} = $info->{actual};
2402
2403         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2404             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2405             $d->{freemem} = $info->{free_mem};
2406         }
2407
2408     };
2409
2410     my $blockstatscb = sub {
2411         my ($vmid, $resp) = @_;
2412         my $data = $resp->{'return'} || [];
2413         my $totalrdbytes = 0;
2414         my $totalwrbytes = 0;
2415         for my $blockstat (@$data) {
2416             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2417             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2418         }
2419         $res->{$vmid}->{diskread} = $totalrdbytes;
2420         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2421     };
2422
2423     my $statuscb = sub {
2424         my ($vmid, $resp) = @_;
2425
2426         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2427         # this fails if ballon driver is not loaded, so this must be
2428         # the last commnand (following command are aborted if this fails).
2429         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2430
2431         my $status = 'unknown';
2432         if (!defined($status = $resp->{'return'}->{status})) {
2433             warn "unable to get VM status\n";
2434             return;
2435         }
2436
2437         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2438     };
2439
2440     foreach my $vmid (keys %$list) {
2441         next if $opt_vmid && ($vmid ne $opt_vmid);
2442         next if !$res->{$vmid}->{pid}; # not running
2443         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2444     }
2445
2446     $qmpclient->queue_execute(undef, 1);
2447
2448     foreach my $vmid (keys %$list) {
2449         next if $opt_vmid && ($vmid ne $opt_vmid);
2450         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2451     }
2452
2453     return $res;
2454 }
2455
2456 sub foreach_dimm {
2457     my ($conf, $vmid, $memory, $sockets, $func) = @_;
2458
2459     my $dimm_id = 0;
2460     my $current_size = 1024;
2461     my $dimm_size = 512;
2462     return if $current_size == $memory;
2463
2464     for (my $j = 0; $j < 8; $j++) {
2465         for (my $i = 0; $i < 32; $i++) {
2466             my $name = "dimm${dimm_id}";
2467             $dimm_id++;
2468             my $numanode = $i % $sockets;
2469             $current_size += $dimm_size;
2470             &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
2471             return  $current_size if $current_size >= $memory;
2472         }
2473         $dimm_size *= 2;
2474     }
2475 }
2476
2477 sub foreach_drive {
2478     my ($conf, $func) = @_;
2479
2480     foreach my $ds (keys %$conf) {
2481         next if !valid_drivename($ds);
2482
2483         my $drive = parse_drive($ds, $conf->{$ds});
2484         next if !$drive;
2485
2486         &$func($ds, $drive);
2487     }
2488 }
2489
2490 sub foreach_volid {
2491     my ($conf, $func) = @_;
2492
2493     my $volhash = {};
2494
2495     my $test_volid = sub {
2496         my ($volid, $is_cdrom) = @_;
2497
2498         return if !$volid;
2499
2500         $volhash->{$volid} = $is_cdrom || 0;
2501     };
2502
2503     foreach_drive($conf, sub {
2504         my ($ds, $drive) = @_;
2505         &$test_volid($drive->{file}, drive_is_cdrom($drive));
2506     });
2507
2508     foreach my $snapname (keys %{$conf->{snapshots}}) {
2509         my $snap = $conf->{snapshots}->{$snapname};
2510         &$test_volid($snap->{vmstate}, 0);
2511         foreach_drive($snap, sub {
2512             my ($ds, $drive) = @_;
2513             &$test_volid($drive->{file}, drive_is_cdrom($drive));
2514         });
2515     }
2516
2517     foreach my $volid (keys %$volhash) {
2518         &$func($volid, $volhash->{$volid});
2519     }
2520 }
2521
2522 sub vga_conf_has_spice {
2523     my ($vga) = @_;
2524
2525     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
2526
2527     return $1 || 1;
2528 }
2529
2530 sub config_to_command {
2531     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2532
2533     my $cmd = [];
2534     my $globalFlags = [];
2535     my $machineFlags = [];
2536     my $rtcFlags = [];
2537     my $cpuFlags = [];
2538     my $devices = [];
2539     my $pciaddr = '';
2540     my $bridges = {};
2541     my $kvmver = kvm_user_version();
2542     my $vernum = 0; # unknown
2543     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
2544         $vernum = $1*1000000+$2*1000;
2545     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
2546         $vernum = $1*1000000+$2*1000+$3;
2547     }
2548
2549     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
2550
2551     my $have_ovz = -f '/proc/vz/vestat';
2552
2553     my $q35 = machine_type_is_q35($conf);
2554     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2555     my $machine_type = $forcemachine || $conf->{machine};
2556
2557     my $cpuunits = defined($conf->{cpuunits}) ?
2558             $conf->{cpuunits} : $defaults->{cpuunits};
2559
2560     push @$cmd, '/usr/bin/systemd-run';
2561     push @$cmd, '--scope';
2562     push @$cmd, '--slice', "qemu";
2563     push @$cmd, '--unit', $vmid;
2564     push @$cmd, '-p', "CPUShares=$cpuunits";
2565     if ($conf->{cpulimit}) {
2566         my $cpulimit = $conf->{cpulimit} * 100;
2567         push @$cmd, '-p', "CPUQuota=$cpulimit"."\%";
2568     }
2569
2570     push @$cmd, '/usr/bin/kvm';
2571
2572     push @$cmd, '-id', $vmid;
2573
2574     my $use_virtio = 0;
2575
2576     my $qmpsocket = qmp_socket($vmid);
2577     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2578     push @$cmd, '-mon', "chardev=qmp,mode=control";
2579
2580     my $socket = vnc_socket($vmid);
2581     push @$cmd,  '-vnc', "unix:$socket,x509,password";
2582
2583     push @$cmd, '-pidfile' , pidfile_name($vmid);
2584
2585     push @$cmd, '-daemonize';
2586
2587     if ($conf->{smbios1}) {
2588         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
2589     }
2590
2591     if ($q35) {
2592         # the q35 chipset support native usb2, so we enable usb controller
2593         # by default for this machine type
2594         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
2595     } else {
2596         $pciaddr = print_pci_addr("piix3", $bridges);
2597         push @$devices, '-device', "piix3-usb-uhci,id=uhci$pciaddr.0x2";
2598
2599         my $use_usb2 = 0;
2600         for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2601             next if !$conf->{"usb$i"};
2602             $use_usb2 = 1;
2603         }
2604         # include usb device config
2605         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-usb.cfg' if $use_usb2;
2606     }
2607
2608     my $vga = $conf->{vga};
2609
2610     my $qxlnum = vga_conf_has_spice($vga);
2611     $vga = 'qxl' if $qxlnum;
2612
2613     if (!$vga) {
2614         if ($conf->{ostype} && ($conf->{ostype} eq 'win8' ||
2615                                 $conf->{ostype} eq 'win7' ||
2616                                 $conf->{ostype} eq 'w2k8')) {
2617             $vga = 'std';
2618         } else {
2619             $vga = 'cirrus';
2620         }
2621     }
2622
2623     # enable absolute mouse coordinates (needed by vnc)
2624     my $tablet;
2625     if (defined($conf->{tablet})) {
2626         $tablet = $conf->{tablet};
2627     } else {
2628         $tablet = $defaults->{tablet};
2629         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
2630         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
2631     }
2632
2633     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
2634
2635     # host pci devices
2636     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
2637         my $d = parse_hostpci($conf->{"hostpci$i"});
2638         next if !$d;
2639
2640         my $pcie = $d->{pcie};
2641         if($pcie){
2642             die "q35 machine model is not enabled" if !$q35;
2643             $pciaddr = print_pcie_addr("hostpci$i");
2644         }else{
2645             $pciaddr = print_pci_addr("hostpci$i", $bridges);
2646         }
2647
2648         my $rombar = $d->{rombar} && $d->{rombar} eq 'off' ? ",rombar=0" : "";
2649         my $driver = $d->{driver} && $d->{driver} eq 'vfio' ? "vfio-pci" : "pci-assign";
2650         my $xvga = $d->{'x-vga'} && $d->{'x-vga'} eq 'on' ? ",x-vga=on" : "";
2651         if ($xvga && $xvga ne '') {
2652             push @$cpuFlags, 'kvm=off';
2653             $vga = 'none';
2654         }
2655         $driver = "vfio-pci" if $xvga ne '';
2656         my $pcidevices = $d->{pciid};
2657         my $multifunction = 1 if @$pcidevices > 1;
2658
2659         my $j=0;
2660         foreach my $pcidevice (@$pcidevices) {
2661
2662             my $id = "hostpci$i";
2663             $id .= ".$j" if $multifunction;
2664             my $addr = $pciaddr;
2665             $addr .= ".$j" if $multifunction;
2666             my $devicestr = "$driver,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
2667
2668             if($j == 0){
2669                 $devicestr .= "$rombar$xvga";
2670                 $devicestr .= ",multifunction=on" if $multifunction;
2671             }
2672
2673             push @$devices, '-device', $devicestr;
2674             $j++;
2675         }
2676     }
2677
2678     # usb devices
2679     for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2680         my $d = parse_usb_device($conf->{"usb$i"});
2681         next if !$d;
2682         if ($d->{vendorid} && $d->{productid}) {
2683             push @$devices, '-device', "usb-host,vendorid=0x$d->{vendorid},productid=0x$d->{productid}";
2684         } elsif (defined($d->{hostbus}) && defined($d->{hostport})) {
2685             push @$devices, '-device', "usb-host,hostbus=$d->{hostbus},hostport=$d->{hostport}";
2686         } elsif ($d->{spice}) {
2687             # usb redir support for spice
2688             push @$devices, '-chardev', "spicevmc,id=usbredirchardev$i,name=usbredir";
2689             push @$devices, '-device', "usb-redir,chardev=usbredirchardev$i,id=usbredirdev$i,bus=ehci.0";
2690         }
2691     }
2692
2693     # serial devices
2694     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2695         if (my $path = $conf->{"serial$i"}) {
2696             if ($path eq 'socket') {
2697                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
2698                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
2699                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2700             } else {
2701                 die "no such serial device\n" if ! -c $path;
2702                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
2703                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2704             }
2705         }
2706     }
2707
2708     # parallel devices
2709     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
2710         if (my $path = $conf->{"parallel$i"}) {
2711             die "no such parallel device\n" if ! -c $path;
2712             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
2713             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
2714             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
2715         }
2716     }
2717
2718     my $vmname = $conf->{name} || "vm$vmid";
2719
2720     push @$cmd, '-name', $vmname;
2721
2722     my $sockets = 1;
2723     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
2724     $sockets = $conf->{sockets} if  $conf->{sockets};
2725
2726     my $cores = $conf->{cores} || 1;
2727
2728     my $maxcpus = $sockets * $cores;
2729
2730     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
2731
2732     my $allowed_vcpus = $cpuinfo->{cpus};
2733
2734     die "MAX $maxcpus vcpus allowed per VM on this node\n"
2735         if ($allowed_vcpus < $maxcpus);
2736
2737     push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
2738
2739     push @$cmd, '-nodefaults';
2740
2741     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2742
2743     my $bootindex_hash = {};
2744     my $i = 1;
2745     foreach my $o (split(//, $bootorder)) {
2746         $bootindex_hash->{$o} = $i*100;
2747         $i++;
2748     }
2749
2750     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000";
2751
2752     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
2753
2754     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
2755
2756     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
2757
2758     # time drift fix
2759     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
2760
2761     my $nokvm = defined($conf->{kvm}) && $conf->{kvm} == 0 ? 1 : 0;
2762     my $useLocaltime = $conf->{localtime};
2763
2764     if (my $ost = $conf->{ostype}) {
2765         # other, wxp, w2k, w2k3, w2k8, wvista, win7, win8, l24, l26, solaris
2766
2767         if ($ost =~ m/^w/) { # windows
2768             $useLocaltime = 1 if !defined($conf->{localtime});
2769
2770             # use time drift fix when acpi is enabled
2771             if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
2772                 $tdf = 1 if !defined($conf->{tdf});
2773             }
2774         }
2775
2776         if ($ost eq 'win7' || $ost eq 'win8' || $ost eq 'w2k8' ||
2777             $ost eq 'wvista') {
2778             push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
2779             push @$cmd, '-no-hpet';
2780             if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
2781                 push @$cpuFlags , 'hv_spinlocks=0x1fff' if !$nokvm;
2782                 push @$cpuFlags , 'hv_vapic' if !$nokvm;
2783                 push @$cpuFlags , 'hv_time' if !$nokvm;
2784
2785             } else {
2786                 push @$cpuFlags , 'hv_spinlocks=0xffff' if !$nokvm;
2787             }
2788         }
2789
2790         if ($ost eq 'win7' || $ost eq 'win8') {
2791             push @$cpuFlags , 'hv_relaxed' if !$nokvm;
2792         }
2793     }
2794
2795     push @$rtcFlags, 'driftfix=slew' if $tdf;
2796
2797     if ($nokvm) {
2798         push @$machineFlags, 'accel=tcg';
2799     } else {
2800         die "No accelerator found!\n" if !$cpuinfo->{hvm};
2801     }
2802
2803     if ($machine_type) {
2804         push @$machineFlags, "type=${machine_type}";
2805     }
2806
2807     if ($conf->{startdate}) {
2808         push @$rtcFlags, "base=$conf->{startdate}";
2809     } elsif ($useLocaltime) {
2810         push @$rtcFlags, 'base=localtime';
2811     }
2812
2813     my $cpu = $nokvm ? "qemu64" : "kvm64";
2814     $cpu = $conf->{cpu} if $conf->{cpu};
2815
2816     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
2817
2818     push @$cpuFlags , '+x2apic' if !$nokvm && $conf->{ostype} ne 'solaris';
2819
2820     push @$cpuFlags , '-x2apic' if $conf->{ostype} eq 'solaris';
2821
2822     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
2823
2824     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
2825
2826         push @$cpuFlags , '+kvm_pv_unhalt' if !$nokvm;
2827         push @$cpuFlags , '+kvm_pv_eoi' if !$nokvm;
2828     }
2829
2830     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
2831
2832     push @$cmd, '-cpu', "$cpu,enforce";
2833
2834     my $memory = $conf->{memory} || $defaults->{memory};
2835     my $static_memory = 0;
2836     my $dimm_memory = 0;
2837
2838     if ($hotplug_features->{memory}) {
2839         die "Numa need to be enabled for memory hotplug\n" if !$conf->{numa};
2840         die "Total memory is bigger than ${MAX_MEM}MB\n" if $memory > $MAX_MEM;
2841         $static_memory = $STATICMEM;
2842         die "minimum memory must be ${static_memory}MB\n" if($memory < $static_memory);
2843         $dimm_memory = $memory - $static_memory;
2844         push @$cmd, '-m', "size=${static_memory},slots=255,maxmem=${MAX_MEM}M";
2845
2846     } else {
2847
2848         $static_memory = $memory;
2849         push @$cmd, '-m', $static_memory;
2850     }
2851
2852     if ($conf->{numa}) {
2853
2854         my $numa_totalmemory = undef;
2855         for (my $i = 0; $i < $MAX_NUMA; $i++) {
2856             next if !$conf->{"numa$i"};
2857             my $numa = parse_numa($conf->{"numa$i"});
2858             next if !$numa;
2859             # memory
2860             die "missing numa node$i memory value\n" if !$numa->{memory};
2861             my $numa_memory = $numa->{memory};
2862             $numa_totalmemory += $numa_memory;
2863             my $numa_object = "memory-backend-ram,id=ram-node$i,size=${numa_memory}M";
2864
2865             # cpus
2866             my $cpus_start = $numa->{cpus}->{start};
2867             die "missing numa node$i cpus\n" if !defined($cpus_start);
2868             my $cpus_end = $numa->{cpus}->{end} if defined($numa->{cpus}->{end});
2869             my $cpus = $cpus_start;
2870             if (defined($cpus_end)) {
2871                 $cpus .= "-$cpus_end";
2872                 die "numa node$i :  cpu range $cpus is incorrect\n" if $cpus_end <= $cpus_start;
2873             }
2874
2875             # hostnodes
2876             my $hostnodes_start = $numa->{hostnodes}->{start};
2877             if (defined($hostnodes_start)) {
2878                 my $hostnodes_end = $numa->{hostnodes}->{end} if defined($numa->{hostnodes}->{end});
2879                 my $hostnodes = $hostnodes_start;
2880                 if (defined($hostnodes_end)) {
2881                     $hostnodes .= "-$hostnodes_end";
2882                     die "host node $hostnodes range is incorrect\n" if $hostnodes_end <= $hostnodes_start;
2883                 }
2884
2885                 my $hostnodes_end_range = defined($hostnodes_end) ? $hostnodes_end : $hostnodes_start;
2886                 for (my $i = $hostnodes_start; $i <= $hostnodes_end_range; $i++ ) {
2887                     die "host numa node$i don't exist\n" if ! -d "/sys/devices/system/node/node$i/";
2888                 }
2889
2890                 # policy
2891                 my $policy = $numa->{policy};
2892                 die "you need to define a policy for hostnode $hostnodes\n" if !$policy;
2893                 $numa_object .= ",host-nodes=$hostnodes,policy=$policy";
2894             }
2895
2896             push @$cmd, '-object', $numa_object;
2897             push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
2898         }
2899
2900         die "total memory for NUMA nodes must be equal to vm static memory\n"
2901             if $numa_totalmemory && $numa_totalmemory != $static_memory;
2902
2903         #if no custom tology, we split memory and cores across numa nodes
2904         if(!$numa_totalmemory) {
2905
2906             my $numa_memory = ($static_memory / $sockets) . "M";
2907
2908             for (my $i = 0; $i < $sockets; $i++)  {
2909
2910                 my $cpustart = ($cores * $i);
2911                 my $cpuend = ($cpustart + $cores - 1) if $cores && $cores > 1;
2912                 my $cpus = $cpustart;
2913                 $cpus .= "-$cpuend" if $cpuend;
2914
2915                 push @$cmd, '-object', "memory-backend-ram,size=$numa_memory,id=ram-node$i";
2916                 push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
2917             }
2918         }
2919     }
2920
2921     if ($hotplug_features->{memory}) {
2922         foreach_dimm($conf, $vmid, $memory, $sockets, sub {
2923             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
2924             push @$cmd, "-object" , "memory-backend-ram,id=mem-$name,size=${dimm_size}M";
2925             push @$cmd, "-device", "pc-dimm,id=$name,memdev=mem-$name,node=$numanode";
2926
2927             #if dimm_memory is not aligned to dimm map
2928             if($current_size > $memory) {
2929                  $conf->{memory} = $current_size;
2930                  update_config_nolock($vmid, $conf, 1);
2931             }
2932         });
2933     }
2934
2935     push @$cmd, '-S' if $conf->{freeze};
2936
2937     # set keyboard layout
2938     my $kb = $conf->{keyboard} || $defaults->{keyboard};
2939     push @$cmd, '-k', $kb if $kb;
2940
2941     # enable sound
2942     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
2943     #push @$cmd, '-soundhw', 'es1370';
2944     #push @$cmd, '-soundhw', $soundhw if $soundhw;
2945
2946     if($conf->{agent}) {
2947         my $qgasocket = qmp_socket($vmid, 1);
2948         my $pciaddr = print_pci_addr("qga0", $bridges);
2949         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
2950         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
2951         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
2952     }
2953
2954     my $spice_port;
2955
2956     if ($qxlnum) {
2957         if ($qxlnum > 1) {
2958             if ($conf->{ostype} && $conf->{ostype} =~ m/^w/){
2959                 for(my $i = 1; $i < $qxlnum; $i++){
2960                     my $pciaddr = print_pci_addr("vga$i", $bridges);
2961                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
2962                 }
2963             } else {
2964                 # assume other OS works like Linux
2965                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
2966                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
2967             }
2968         }
2969
2970         my $pciaddr = print_pci_addr("spice", $bridges);
2971
2972         my $nodename = PVE::INotify::nodename();
2973         my $pfamily = PVE::Tools::get_host_address_family($nodename);
2974         $spice_port = PVE::Tools::next_spice_port($pfamily);
2975
2976         push @$devices, '-spice', "tls-port=${spice_port},addr=localhost,tls-ciphers=DES-CBC3-SHA,seamless-migration=on";
2977
2978         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
2979         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
2980         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
2981     }
2982
2983     # enable balloon by default, unless explicitly disabled
2984     if (!defined($conf->{balloon}) || $conf->{balloon}) {
2985         $pciaddr = print_pci_addr("balloon0", $bridges);
2986         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
2987     }
2988
2989     if ($conf->{watchdog}) {
2990         my $wdopts = parse_watchdog($conf->{watchdog});
2991         $pciaddr = print_pci_addr("watchdog", $bridges);
2992         my $watchdog = $wdopts->{model} || 'i6300esb';
2993         push @$devices, '-device', "$watchdog$pciaddr";
2994         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
2995     }
2996
2997     my $vollist = [];
2998     my $scsicontroller = {};
2999     my $ahcicontroller = {};
3000     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3001
3002     # Add iscsi initiator name if available
3003     if (my $initiator = get_initiator_name()) {
3004         push @$devices, '-iscsi', "initiator-name=$initiator";
3005     }
3006
3007     foreach_drive($conf, sub {
3008         my ($ds, $drive) = @_;
3009
3010         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3011             push @$vollist, $drive->{file};
3012         }
3013
3014         $use_virtio = 1 if $ds =~ m/^virtio/;
3015
3016         if (drive_is_cdrom ($drive)) {
3017             if ($bootindex_hash->{d}) {
3018                 $drive->{bootindex} = $bootindex_hash->{d};
3019                 $bootindex_hash->{d} += 1;
3020             }
3021         } else {
3022             if ($bootindex_hash->{c}) {
3023                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3024                 $bootindex_hash->{c} += 1;
3025             }
3026         }
3027
3028         if($drive->{interface} eq 'virtio'){
3029            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3030         }
3031
3032         if ($drive->{interface} eq 'scsi') {
3033
3034             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3035
3036             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3037             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3038
3039             my $iothread = '';
3040             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3041                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3042                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3043             }
3044
3045             my $queues = '';
3046             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3047                 $queues = ",num_queues=$drive->{queues}";
3048             }
3049
3050             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3051             $scsicontroller->{$controller}=1;
3052         }
3053
3054         if ($drive->{interface} eq 'sata') {
3055            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3056            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3057            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3058            $ahcicontroller->{$controller}=1;
3059         }
3060
3061         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3062         push @$devices, '-drive',$drive_cmd;
3063         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3064     });
3065
3066     for (my $i = 0; $i < $MAX_NETS; $i++) {
3067          next if !$conf->{"net$i"};
3068          my $d = parse_net($conf->{"net$i"});
3069          next if !$d;
3070
3071          $use_virtio = 1 if $d->{model} eq 'virtio';
3072
3073          if ($bootindex_hash->{n}) {
3074             $d->{bootindex} = $bootindex_hash->{n};
3075             $bootindex_hash->{n} += 1;
3076          }
3077
3078          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3079          push @$devices, '-netdev', $netdevfull;
3080
3081          my $netdevicefull = print_netdevice_full($vmid,$conf,$d,"net$i",$bridges);
3082          push @$devices, '-device', $netdevicefull;
3083     }
3084
3085     if (!$q35) {
3086         # add pci bridges
3087         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3088            $bridges->{1} = 1;
3089            $bridges->{2} = 1;
3090         }
3091
3092         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3093
3094         while (my ($k, $v) = each %$bridges) {
3095             $pciaddr = print_pci_addr("pci.$k");
3096             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3097         }
3098     }
3099
3100     # add custom args
3101     if ($conf->{args}) {
3102         my $aa = PVE::Tools::split_args($conf->{args});
3103         push @$cmd, @$aa;
3104     }
3105
3106     push @$cmd, @$devices;
3107     push @$cmd, '-rtc', join(',', @$rtcFlags)
3108         if scalar(@$rtcFlags);
3109     push @$cmd, '-machine', join(',', @$machineFlags)
3110         if scalar(@$machineFlags);
3111     push @$cmd, '-global', join(',', @$globalFlags)
3112         if scalar(@$globalFlags);
3113
3114     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3115 }
3116
3117 sub vnc_socket {
3118     my ($vmid) = @_;
3119     return "${var_run_tmpdir}/$vmid.vnc";
3120 }
3121
3122 sub spice_port {
3123     my ($vmid) = @_;
3124
3125     my $res = vm_mon_cmd($vmid, 'query-spice');
3126
3127     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3128 }
3129
3130 sub qmp_socket {
3131     my ($vmid, $qga) = @_;
3132     my $sockettype = $qga ? 'qga' : 'qmp';
3133     return "${var_run_tmpdir}/$vmid.$sockettype";
3134 }
3135
3136 sub pidfile_name {
3137     my ($vmid) = @_;
3138     return "${var_run_tmpdir}/$vmid.pid";
3139 }
3140
3141 sub vm_devices_list {
3142     my ($vmid) = @_;
3143
3144     my $res = vm_mon_cmd($vmid, 'query-pci');
3145     my $devices = {};
3146     foreach my $pcibus (@$res) {
3147         foreach my $device (@{$pcibus->{devices}}) {
3148             next if !$device->{'qdev_id'};
3149             if ($device->{'pci_bridge'}) {
3150                 $devices->{$device->{'qdev_id'}} = 1;
3151                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3152                     next if !$bridge_device->{'qdev_id'};
3153                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3154                     $devices->{$device->{'qdev_id'}}++;
3155                 }
3156             } else {
3157                 $devices->{$device->{'qdev_id'}} = 1;
3158             }
3159         }
3160     }
3161
3162     my $resblock = vm_mon_cmd($vmid, 'query-block');
3163     foreach my $block (@$resblock) {
3164         if($block->{device} =~ m/^drive-(\S+)/){
3165                 $devices->{$1} = 1;
3166         }
3167     }
3168
3169     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3170     foreach my $mice (@$resmice) {
3171         if ($mice->{name} eq 'QEMU HID Tablet') {
3172             $devices->{tablet} = 1;
3173             last;
3174         }
3175     }
3176
3177     return $devices;
3178 }
3179
3180 sub vm_deviceplug {
3181     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3182
3183     my $q35 = machine_type_is_q35($conf);
3184
3185     my $devices_list = vm_devices_list($vmid);
3186     return 1 if defined($devices_list->{$deviceid});
3187
3188     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3189
3190     if ($deviceid eq 'tablet') {
3191
3192         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3193
3194     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3195
3196         qemu_iothread_add($vmid, $deviceid, $device);
3197
3198         qemu_driveadd($storecfg, $vmid, $device);
3199         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3200
3201         qemu_deviceadd($vmid, $devicefull);
3202         eval { qemu_deviceaddverify($vmid, $deviceid); };
3203         if (my $err = $@) {
3204             eval { qemu_drivedel($vmid, $deviceid); };
3205             warn $@ if $@;
3206             die $err;
3207         }
3208
3209     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3210
3211
3212         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3213         my $pciaddr = print_pci_addr($deviceid);
3214         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3215
3216         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3217
3218         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3219             qemu_iothread_add($vmid, $deviceid, $device);
3220             $devicefull .= ",iothread=iothread-$deviceid";
3221         }
3222
3223         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3224             $devicefull .= ",num_queues=$device->{queues}";
3225         }
3226
3227         qemu_deviceadd($vmid, $devicefull);
3228         qemu_deviceaddverify($vmid, $deviceid);
3229
3230     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3231
3232         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3233         qemu_driveadd($storecfg, $vmid, $device);
3234
3235         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3236         eval { qemu_deviceadd($vmid, $devicefull); };
3237         if (my $err = $@) {
3238             eval { qemu_drivedel($vmid, $deviceid); };
3239             warn $@ if $@;
3240             die $err;
3241         }
3242
3243     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3244
3245         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3246         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid);
3247         qemu_deviceadd($vmid, $netdevicefull);
3248         eval { qemu_deviceaddverify($vmid, $deviceid); };
3249         if (my $err = $@) {
3250             eval { qemu_netdevdel($vmid, $deviceid); };
3251             warn $@ if $@;
3252             die $err;
3253         }
3254
3255     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3256
3257         my $bridgeid = $2;
3258         my $pciaddr = print_pci_addr($deviceid);
3259         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3260
3261         qemu_deviceadd($vmid, $devicefull);
3262         qemu_deviceaddverify($vmid, $deviceid);
3263
3264     } else {
3265         die "can't hotplug device '$deviceid'\n";
3266     }
3267
3268     return 1;
3269 }
3270
3271 # fixme: this should raise exceptions on error!
3272 sub vm_deviceunplug {
3273     my ($vmid, $conf, $deviceid) = @_;
3274
3275     my $devices_list = vm_devices_list($vmid);
3276     return 1 if !defined($devices_list->{$deviceid});
3277
3278     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3279
3280     if ($deviceid eq 'tablet') {
3281
3282         qemu_devicedel($vmid, $deviceid);
3283
3284     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3285
3286         qemu_devicedel($vmid, $deviceid);
3287         qemu_devicedelverify($vmid, $deviceid);
3288         qemu_drivedel($vmid, $deviceid);
3289         qemu_iothread_del($conf, $vmid, $deviceid);
3290
3291     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3292
3293         qemu_devicedel($vmid, $deviceid);
3294         qemu_devicedelverify($vmid, $deviceid);
3295         qemu_iothread_del($conf, $vmid, $deviceid);
3296
3297     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3298
3299         #qemu 2.3 segfault on drive_del with virtioscsi + iothread
3300         my $device = parse_drive($deviceid, $conf->{$deviceid});
3301         die "virtioscsi with iothread is not hot-unplugglable currently" if $device->{iothread};
3302
3303         qemu_devicedel($vmid, $deviceid);
3304         qemu_drivedel($vmid, $deviceid);
3305         qemu_deletescsihw($conf, $vmid, $deviceid);
3306
3307     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3308
3309         qemu_devicedel($vmid, $deviceid);
3310         qemu_devicedelverify($vmid, $deviceid);
3311         qemu_netdevdel($vmid, $deviceid);
3312
3313     } else {
3314         die "can't unplug device '$deviceid'\n";
3315     }
3316
3317     return 1;
3318 }
3319
3320 sub qemu_deviceadd {
3321     my ($vmid, $devicefull) = @_;
3322
3323     $devicefull = "driver=".$devicefull;
3324     my %options =  split(/[=,]/, $devicefull);
3325
3326     vm_mon_cmd($vmid, "device_add" , %options);
3327 }
3328
3329 sub qemu_devicedel {
3330     my ($vmid, $deviceid) = @_;
3331
3332     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3333 }
3334
3335 sub qemu_iothread_add {
3336     my($vmid, $deviceid, $device) = @_;
3337
3338     if ($device->{iothread}) {
3339         my $iothreads = vm_iothreads_list($vmid);
3340         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3341     }
3342 }
3343
3344 sub qemu_iothread_del {
3345     my($conf, $vmid, $deviceid) = @_;
3346
3347     my $device = parse_drive($deviceid, $conf->{$deviceid});
3348     if ($device->{iothread}) {
3349         my $iothreads = vm_iothreads_list($vmid);
3350         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3351     }
3352 }
3353
3354 sub qemu_objectadd {
3355     my($vmid, $objectid, $qomtype) = @_;
3356
3357     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3358
3359     return 1;
3360 }
3361
3362 sub qemu_objectdel {
3363     my($vmid, $objectid) = @_;
3364
3365     vm_mon_cmd($vmid, "object-del", id => $objectid);
3366
3367     return 1;
3368 }
3369
3370 sub qemu_driveadd {
3371     my ($storecfg, $vmid, $device) = @_;
3372
3373     my $drive = print_drive_full($storecfg, $vmid, $device);
3374     $drive =~ s/\\/\\\\/g;
3375     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3376
3377     # If the command succeeds qemu prints: "OK"
3378     return 1 if $ret =~ m/OK/s;
3379
3380     die "adding drive failed: $ret\n";
3381 }
3382
3383 sub qemu_drivedel {
3384     my($vmid, $deviceid) = @_;
3385
3386     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3387     $ret =~ s/^\s+//;
3388
3389     return 1 if $ret eq "";
3390
3391     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3392     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3393
3394     die "deleting drive $deviceid failed : $ret\n";
3395 }
3396
3397 sub qemu_deviceaddverify {
3398     my ($vmid, $deviceid) = @_;
3399
3400     for (my $i = 0; $i <= 5; $i++) {
3401          my $devices_list = vm_devices_list($vmid);
3402          return 1 if defined($devices_list->{$deviceid});
3403          sleep 1;
3404     }
3405
3406     die "error on hotplug device '$deviceid'\n";
3407 }
3408
3409
3410 sub qemu_devicedelverify {
3411     my ($vmid, $deviceid) = @_;
3412
3413     # need to verify that the device is correctly removed as device_del
3414     # is async and empty return is not reliable
3415
3416     for (my $i = 0; $i <= 5; $i++) {
3417          my $devices_list = vm_devices_list($vmid);
3418          return 1 if !defined($devices_list->{$deviceid});
3419          sleep 1;
3420     }
3421
3422     die "error on hot-unplugging device '$deviceid'\n";
3423 }
3424
3425 sub qemu_findorcreatescsihw {
3426     my ($storecfg, $conf, $vmid, $device) = @_;
3427
3428     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3429
3430     my $scsihwid="$controller_prefix$controller";
3431     my $devices_list = vm_devices_list($vmid);
3432
3433     if(!defined($devices_list->{$scsihwid})) {
3434         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3435     }
3436
3437     return 1;
3438 }
3439
3440 sub qemu_deletescsihw {
3441     my ($conf, $vmid, $opt) = @_;
3442
3443     my $device = parse_drive($opt, $conf->{$opt});
3444
3445     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3446         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3447         return 1;
3448     }
3449
3450     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3451
3452     my $devices_list = vm_devices_list($vmid);
3453     foreach my $opt (keys %{$devices_list}) {
3454         if (PVE::QemuServer::valid_drivename($opt)) {
3455             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3456             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3457                 return 1;
3458             }
3459         }
3460     }
3461
3462     my $scsihwid="scsihw$controller";
3463
3464     vm_deviceunplug($vmid, $conf, $scsihwid);
3465
3466     return 1;
3467 }
3468
3469 sub qemu_add_pci_bridge {
3470     my ($storecfg, $conf, $vmid, $device) = @_;
3471
3472     my $bridges = {};
3473
3474     my $bridgeid;
3475
3476     print_pci_addr($device, $bridges);
3477
3478     while (my ($k, $v) = each %$bridges) {
3479         $bridgeid = $k;
3480     }
3481     return 1 if !defined($bridgeid) || $bridgeid < 1;
3482
3483     my $bridge = "pci.$bridgeid";
3484     my $devices_list = vm_devices_list($vmid);
3485
3486     if (!defined($devices_list->{$bridge})) {
3487         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3488     }
3489
3490     return 1;
3491 }
3492
3493 sub qemu_set_link_status {
3494     my ($vmid, $device, $up) = @_;
3495
3496     vm_mon_cmd($vmid, "set_link", name => $device,
3497                up => $up ? JSON::true : JSON::false);
3498 }
3499
3500 sub qemu_netdevadd {
3501     my ($vmid, $conf, $device, $deviceid) = @_;
3502
3503     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid);
3504     my %options =  split(/[=,]/, $netdev);
3505
3506     vm_mon_cmd($vmid, "netdev_add",  %options);
3507     return 1;
3508 }
3509
3510 sub qemu_netdevdel {
3511     my ($vmid, $deviceid) = @_;
3512
3513     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3514 }
3515
3516 sub qemu_cpu_hotplug {
3517     my ($vmid, $conf, $vcpus) = @_;
3518
3519     my $sockets = 1;
3520     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3521     $sockets = $conf->{sockets} if  $conf->{sockets};
3522     my $cores = $conf->{cores} || 1;
3523     my $maxcpus = $sockets * $cores;
3524
3525     $vcpus = $maxcpus if !$vcpus;
3526
3527     die "you can't add more vcpus than maxcpus\n"
3528         if $vcpus > $maxcpus;
3529
3530     my $currentvcpus = $conf->{vcpus} || $maxcpus;
3531     die "online cpu unplug is not yet possible\n"
3532         if $vcpus < $currentvcpus;
3533
3534     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3535     die "vcpus in running vm is different than configuration\n"
3536         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
3537
3538     for (my $i = $currentvcpus; $i < $vcpus; $i++) {
3539         vm_mon_cmd($vmid, "cpu-add", id => int($i));
3540     }
3541 }
3542
3543 sub qemu_memory_hotplug {
3544     my ($vmid, $conf, $defaults, $opt, $value) = @_;
3545
3546     return $value if !check_running($vmid);
3547
3548     my $memory = $conf->{memory} || $defaults->{memory};
3549     $value = $defaults->{memory} if !$value;
3550     return $value if $value == $memory;
3551
3552     my $static_memory = $STATICMEM;
3553     my $dimm_memory = $memory - $static_memory;
3554
3555     die "memory can't be lower than $static_memory MB" if $value < $static_memory;
3556     die "memory unplug is not yet available" if $value < $memory;
3557     die "you cannot add more memory than $MAX_MEM MB!\n" if $memory > $MAX_MEM;
3558
3559
3560     my $sockets = 1;
3561     $sockets = $conf->{sockets} if $conf->{sockets};
3562
3563     foreach_dimm($conf, $vmid, $value, $sockets, sub {
3564         my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3565
3566             return if $current_size <= $conf->{memory};
3567
3568             eval { vm_mon_cmd($vmid, "object-add", 'qom-type' => "memory-backend-ram", id => "mem-$name", props => { size => int($dimm_size*1024*1024) } ) };
3569             if (my $err = $@) {
3570                 eval { qemu_objectdel($vmid, "mem-$name"); };
3571                 die $err;
3572             }
3573
3574             eval { vm_mon_cmd($vmid, "device_add", driver => "pc-dimm", id => "$name", memdev => "mem-$name", node => $numanode) };
3575             if (my $err = $@) {
3576                 eval { qemu_objectdel($vmid, "mem-$name"); };
3577                 die $err;
3578             }
3579             #update conf after each succesful module hotplug
3580             $conf->{memory} = $current_size;
3581             update_config_nolock($vmid, $conf, 1);
3582     });
3583 }
3584
3585 sub qemu_block_set_io_throttle {
3586     my ($vmid, $deviceid, $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr) = @_;
3587
3588     return if !check_running($vmid) ;
3589
3590     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid, bps => int($bps), bps_rd => int($bps_rd), bps_wr => int($bps_wr), iops => int($iops), iops_rd => int($iops_rd), iops_wr => int($iops_wr));
3591
3592 }
3593
3594 # old code, only used to shutdown old VM after update
3595 sub __read_avail {
3596     my ($fh, $timeout) = @_;
3597
3598     my $sel = new IO::Select;
3599     $sel->add($fh);
3600
3601     my $res = '';
3602     my $buf;
3603
3604     my @ready;
3605     while (scalar (@ready = $sel->can_read($timeout))) {
3606         my $count;
3607         if ($count = $fh->sysread($buf, 8192)) {
3608             if ($buf =~ /^(.*)\(qemu\) $/s) {
3609                 $res .= $1;
3610                 last;
3611             } else {
3612                 $res .= $buf;
3613             }
3614         } else {
3615             if (!defined($count)) {
3616                 die "$!\n";
3617             }
3618             last;
3619         }
3620     }
3621
3622     die "monitor read timeout\n" if !scalar(@ready);
3623
3624     return $res;
3625 }
3626
3627 # old code, only used to shutdown old VM after update
3628 sub vm_monitor_command {
3629     my ($vmid, $cmdstr, $nocheck) = @_;
3630
3631     my $res;
3632
3633     eval {
3634         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
3635
3636         my $sname = "${var_run_tmpdir}/$vmid.mon";
3637
3638         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
3639             die "unable to connect to VM $vmid socket - $!\n";
3640
3641         my $timeout = 3;
3642
3643         # hack: migrate sometime blocks the monitor (when migrate_downtime
3644         # is set)
3645         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3646             $timeout = 60*60; # 1 hour
3647         }
3648
3649         # read banner;
3650         my $data = __read_avail($sock, $timeout);
3651
3652         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
3653             die "got unexpected qemu monitor banner\n";
3654         }
3655
3656         my $sel = new IO::Select;
3657         $sel->add($sock);
3658
3659         if (!scalar(my @ready = $sel->can_write($timeout))) {
3660             die "monitor write error - timeout";
3661         }
3662
3663         my $fullcmd = "$cmdstr\r";
3664
3665         # syslog('info', "VM $vmid monitor command: $cmdstr");
3666
3667         my $b;
3668         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
3669             die "monitor write error - $!";
3670         }
3671
3672         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
3673
3674         $timeout = 20;
3675
3676         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3677             $timeout = 60*60; # 1 hour
3678         } elsif ($cmdstr =~ m/^(eject|change)/) {
3679             $timeout = 60; # note: cdrom mount command is slow
3680         }
3681         if ($res = __read_avail($sock, $timeout)) {
3682
3683             my @lines = split("\r?\n", $res);
3684
3685             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
3686
3687             $res = join("\n", @lines);
3688             $res .= "\n";
3689         }
3690     };
3691
3692     my $err = $@;
3693
3694     if ($err) {
3695         syslog("err", "VM $vmid monitor command failed - $err");
3696         die $err;
3697     }
3698
3699     return $res;
3700 }
3701
3702 sub qemu_block_resize {
3703     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
3704
3705     my $running = check_running($vmid);
3706
3707     return if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
3708
3709     return if !$running;
3710
3711     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
3712
3713 }
3714
3715 sub qemu_volume_snapshot {
3716     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3717
3718     my $running = check_running($vmid);
3719
3720     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
3721         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
3722     } else {
3723         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
3724     }
3725 }
3726
3727 sub qemu_volume_snapshot_delete {
3728     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3729
3730     my $running = check_running($vmid);
3731
3732     return if !PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
3733
3734     return if !$running;
3735
3736     vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
3737 }
3738
3739 sub set_migration_caps {
3740     my ($vmid) = @_;
3741
3742     my $cap_ref = [];
3743
3744     my $enabled_cap = {
3745         "auto-converge" => 1,
3746         "xbzrle" => 0,
3747         "x-rdma-pin-all" => 0,
3748         "zero-blocks" => 0,
3749     };
3750
3751     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
3752
3753     for my $supported_capability (@$supported_capabilities) {
3754         push @$cap_ref, {
3755             capability => $supported_capability->{capability},
3756             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
3757         };
3758     }
3759
3760     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
3761 }
3762
3763 my $fast_plug_option = {
3764     'lock' => 1,
3765     'name' => 1,
3766     'onboot' => 1,
3767     'shares' => 1,
3768     'startup' => 1,
3769 };
3770
3771 # hotplug changes in [PENDING]
3772 # $selection hash can be used to only apply specified options, for
3773 # example: { cores => 1 } (only apply changed 'cores')
3774 # $errors ref is used to return error messages
3775 sub vmconfig_hotplug_pending {
3776     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
3777
3778     my $defaults = load_defaults();
3779
3780     # commit values which do not have any impact on running VM first
3781     # Note: those option cannot raise errors, we we do not care about
3782     # $selection and always apply them.
3783
3784     my $add_error = sub {
3785         my ($opt, $msg) = @_;
3786         $errors->{$opt} = "hotplug problem - $msg";
3787     };
3788
3789     my $changes = 0;
3790     foreach my $opt (keys %{$conf->{pending}}) { # add/change
3791         if ($fast_plug_option->{$opt}) {
3792             $conf->{$opt} = $conf->{pending}->{$opt};
3793             delete $conf->{pending}->{$opt};
3794             $changes = 1;
3795         }
3796     }
3797
3798     if ($changes) {
3799         update_config_nolock($vmid, $conf, 1);
3800         $conf = load_config($vmid); # update/reload
3801     }
3802
3803     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3804
3805     my @delete = PVE::Tools::split_list($conf->{pending}->{delete});
3806     foreach my $opt (@delete) {
3807         next if $selection && !$selection->{$opt};
3808         eval {
3809             if ($opt eq 'hotplug') {
3810                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
3811             } elsif ($opt eq 'tablet') {
3812                 die "skip\n" if !$hotplug_features->{usb};
3813                 if ($defaults->{tablet}) {
3814                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
3815                 } else {
3816                     vm_deviceunplug($vmid, $conf, $opt);
3817                 }
3818             } elsif ($opt eq 'vcpus') {
3819                 die "skip\n" if !$hotplug_features->{cpu};
3820                 qemu_cpu_hotplug($vmid, $conf, undef);
3821             } elsif ($opt eq 'balloon') {
3822                 # enable balloon device is not hotpluggable
3823                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
3824             } elsif ($fast_plug_option->{$opt}) {
3825                 # do nothing
3826             } elsif ($opt =~ m/^net(\d+)$/) {
3827                 die "skip\n" if !$hotplug_features->{network};
3828                 vm_deviceunplug($vmid, $conf, $opt);
3829             } elsif (valid_drivename($opt)) {
3830                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
3831                 vm_deviceunplug($vmid, $conf, $opt);
3832                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}));
3833             } elsif ($opt =~ m/^memory$/) {
3834                 die "skip\n" if !$hotplug_features->{memory};
3835                 qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
3836             } elsif ($opt eq 'cpuunits') {
3837                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
3838             } elsif ($opt eq 'cpulimit') {
3839                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
3840             } else {
3841                 die "skip\n";
3842             }
3843         };
3844         if (my $err = $@) {
3845             &$add_error($opt, $err) if $err ne "skip\n";
3846         } else {
3847             # save new config if hotplug was successful
3848             delete $conf->{$opt};
3849             vmconfig_undelete_pending_option($conf, $opt);
3850             update_config_nolock($vmid, $conf, 1);
3851             $conf = load_config($vmid); # update/reload
3852         }
3853     }
3854
3855     foreach my $opt (keys %{$conf->{pending}}) {
3856         next if $selection && !$selection->{$opt};
3857         my $value = $conf->{pending}->{$opt};
3858         eval {
3859             if ($opt eq 'hotplug') {
3860                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
3861             } elsif ($opt eq 'tablet') {
3862                 die "skip\n" if !$hotplug_features->{usb};
3863                 if ($value == 1) {
3864                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
3865                 } elsif ($value == 0) {
3866                     vm_deviceunplug($vmid, $conf, $opt);
3867                 }
3868             } elsif ($opt eq 'vcpus') {
3869                 die "skip\n" if !$hotplug_features->{cpu};
3870                 qemu_cpu_hotplug($vmid, $conf, $value);
3871             } elsif ($opt eq 'balloon') {
3872                 # enable/disable balloning device is not hotpluggable
3873                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
3874                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
3875                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
3876
3877                 # allow manual ballooning if shares is set to zero
3878                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
3879                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
3880                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
3881                 }
3882             } elsif ($opt =~ m/^net(\d+)$/) {
3883                 # some changes can be done without hotplug
3884                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
3885                                     $vmid, $opt, $value);
3886             } elsif (valid_drivename($opt)) {
3887                 # some changes can be done without hotplug
3888                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
3889                                      $vmid, $opt, $value, 1);
3890             } elsif ($opt =~ m/^memory$/) { #dimms
3891                 die "skip\n" if !$hotplug_features->{memory};
3892                 $value = qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
3893             } elsif ($opt eq 'cpuunits') {
3894                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
3895             } elsif ($opt eq 'cpulimit') {
3896                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : $conf->{pending}->{$opt} * 100000;
3897                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
3898             } else {
3899                 die "skip\n";  # skip non-hot-pluggable options
3900             }
3901         };
3902         if (my $err = $@) {
3903             &$add_error($opt, $err) if $err ne "skip\n";
3904         } else {
3905             # save new config if hotplug was successful
3906             $conf->{$opt} = $value;
3907             delete $conf->{pending}->{$opt};
3908             update_config_nolock($vmid, $conf, 1);
3909             $conf = load_config($vmid); # update/reload
3910         }
3911     }
3912 }
3913
3914 sub vmconfig_apply_pending {
3915     my ($vmid, $conf, $storecfg) = @_;
3916
3917     # cold plug
3918
3919     my @delete = PVE::Tools::split_list($conf->{pending}->{delete});
3920     foreach my $opt (@delete) { # delete
3921         die "internal error" if $opt =~ m/^unused/;
3922         $conf = load_config($vmid); # update/reload
3923         if (!defined($conf->{$opt})) {
3924             vmconfig_undelete_pending_option($conf, $opt);
3925             update_config_nolock($vmid, $conf, 1);
3926         } elsif (valid_drivename($opt)) {
3927             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}));
3928             vmconfig_undelete_pending_option($conf, $opt);
3929             delete $conf->{$opt};
3930             update_config_nolock($vmid, $conf, 1);
3931         } else {
3932             vmconfig_undelete_pending_option($conf, $opt);
3933             delete $conf->{$opt};
3934             update_config_nolock($vmid, $conf, 1);
3935         }
3936     }
3937
3938     $conf = load_config($vmid); # update/reload
3939
3940     foreach my $opt (keys %{$conf->{pending}}) { # add/change
3941         $conf = load_config($vmid); # update/reload
3942
3943         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
3944             # skip if nothing changed
3945         } elsif (valid_drivename($opt)) {
3946             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
3947                 if defined($conf->{$opt});
3948             $conf->{$opt} = $conf->{pending}->{$opt};
3949         } else {
3950             $conf->{$opt} = $conf->{pending}->{$opt};
3951         }
3952
3953         delete $conf->{pending}->{$opt};
3954         update_config_nolock($vmid, $conf, 1);
3955     }
3956 }
3957
3958 my $safe_num_ne = sub {
3959     my ($a, $b) = @_;
3960
3961     return 0 if !defined($a) && !defined($b);
3962     return 1 if !defined($a);
3963     return 1 if !defined($b);
3964
3965     return $a != $b;
3966 };
3967
3968 my $safe_string_ne = sub {
3969     my ($a, $b) = @_;
3970
3971     return 0 if !defined($a) && !defined($b);
3972     return 1 if !defined($a);
3973     return 1 if !defined($b);
3974
3975     return $a ne $b;
3976 };
3977
3978 sub vmconfig_update_net {
3979     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
3980
3981     my $newnet = parse_net($value);
3982
3983     if ($conf->{$opt}) {
3984         my $oldnet = parse_net($conf->{$opt});
3985
3986         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
3987             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
3988             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
3989             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
3990
3991             # for non online change, we try to hot-unplug
3992             die "skip\n" if !$hotplug;
3993             vm_deviceunplug($vmid, $conf, $opt);
3994         } else {
3995
3996             die "internal error" if $opt !~ m/net(\d+)/;
3997             my $iface = "tap${vmid}i$1";
3998
3999             if (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4000                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4001             }
4002
4003             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4004                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4005                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4006                 PVE::Network::tap_unplug($iface);
4007                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall});
4008             }
4009
4010             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4011                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4012             }
4013
4014             return 1;
4015         }
4016     }
4017
4018     if ($hotplug) {
4019         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4020     } else {
4021         die "skip\n";
4022     }
4023 }
4024
4025 sub vmconfig_update_disk {
4026     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4027
4028     # fixme: do we need force?
4029
4030     my $drive = parse_drive($opt, $value);
4031
4032     if ($conf->{$opt}) {
4033
4034         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4035
4036             my $media = $drive->{media} || 'disk';
4037             my $oldmedia = $old_drive->{media} || 'disk';
4038             die "unable to change media type\n" if $media ne $oldmedia;
4039
4040             if (!drive_is_cdrom($old_drive)) {
4041
4042                 if ($drive->{file} ne $old_drive->{file}) {
4043
4044                     die "skip\n" if !$hotplug;
4045
4046                     # unplug and register as unused
4047                     vm_deviceunplug($vmid, $conf, $opt);
4048                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4049
4050                 } else {
4051                     # update existing disk
4052
4053                     # skip non hotpluggable value
4054                     if (&$safe_num_ne($drive->{discard}, $old_drive->{discard}) ||
4055                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4056                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4057                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4058                         die "skip\n";
4059                     }
4060
4061                     # apply throttle
4062                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4063                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4064                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4065                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4066                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4067                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4068                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4069                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4070                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4071                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4072                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4073                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max})) {
4074
4075                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4076                                                    ($drive->{mbps} || 0)*1024*1024,
4077                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4078                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4079                                                    $drive->{iops} || 0,
4080                                                    $drive->{iops_rd} || 0,
4081                                                    $drive->{iops_wr} || 0,
4082                                                    ($drive->{mbps_max} || 0)*1024*1024,
4083                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4084                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4085                                                    $drive->{iops_max} || 0,
4086                                                    $drive->{iops_rd_max} || 0,
4087                                                    $drive->{iops_wr_max} || 0);
4088
4089                     }
4090
4091                     return 1;
4092                 }
4093
4094             } else { # cdrom
4095
4096                 if ($drive->{file} eq 'none') {
4097                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4098                 } else {
4099                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4100                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4101                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4102                 }
4103
4104                 return 1;
4105             }
4106         }
4107     }
4108
4109     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4110     # hotplug new disks
4111     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4112 }
4113
4114 sub vm_start {
4115     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused, $forcemachine, $spice_ticket) = @_;
4116
4117     lock_config($vmid, sub {
4118         my $conf = load_config($vmid, $migratedfrom);
4119
4120         die "you can't start a vm if it's a template\n" if is_template($conf);
4121
4122         check_lock($conf) if !$skiplock;
4123
4124         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4125
4126         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4127             vmconfig_apply_pending($vmid, $conf, $storecfg);
4128             $conf = load_config($vmid); # update/reload
4129         }
4130
4131         my $defaults = load_defaults();
4132
4133         # set environment variable useful inside network script
4134         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4135
4136         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4137
4138         my $migrate_port = 0;
4139         my $migrate_uri;
4140         if ($statefile) {
4141             if ($statefile eq 'tcp') {
4142                 my $localip = "localhost";
4143                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4144                 my $nodename = PVE::INotify::nodename();
4145                 if ($datacenterconf->{migration_unsecure}) {
4146                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4147                 }
4148                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4149                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4150                 $migrate_uri = "tcp:[${localip}]:${migrate_port}";
4151                 push @$cmd, '-incoming', $migrate_uri;
4152                 push @$cmd, '-S';
4153             } else {
4154                 push @$cmd, '-loadstate', $statefile;
4155             }
4156         } elsif ($paused) {
4157             push @$cmd, '-S';
4158         }
4159
4160         # host pci devices
4161         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4162           my $d = parse_hostpci($conf->{"hostpci$i"});
4163           next if !$d;
4164           my $pcidevices = $d->{pciid};
4165           foreach my $pcidevice (@$pcidevices) {
4166                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4167
4168                 my $info = pci_device_info("0000:$pciid");
4169                 die "IOMMU not present\n" if !check_iommu_support();
4170                 die "no pci device info for device '$pciid'\n" if !$info;
4171
4172                 if ($d->{driver} && $d->{driver} eq "vfio") {
4173                     die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4174                 } else {
4175                     die "can't unbind/bind to stub pci device '$pciid'\n" if !pci_dev_bind_to_stub($info);
4176                 }
4177
4178                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4179           }
4180         }
4181
4182         PVE::Storage::activate_volumes($storecfg, $vollist);
4183
4184         eval  { run_command($cmd, timeout => $statefile ? undef : 30,
4185                     umask => 0077); };
4186         my $err = $@;
4187         die "start failed: $err" if $err;
4188
4189         print "migration listens on $migrate_uri\n" if $migrate_uri;
4190
4191         if ($statefile && $statefile ne 'tcp')  {
4192             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4193             warn $@ if $@;
4194         }
4195
4196         if ($migratedfrom) {
4197
4198             eval {
4199                 set_migration_caps($vmid);
4200             };
4201             warn $@ if $@;
4202
4203             if ($spice_port) {
4204                 print "spice listens on port $spice_port\n";
4205                 if ($spice_ticket) {
4206                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4207                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4208                 }
4209             }
4210
4211         } else {
4212
4213             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4214                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4215                     if $conf->{balloon};
4216             }
4217
4218             foreach my $opt (keys %$conf) {
4219                 next if $opt !~  m/^net\d+$/;
4220                 my $nicconf = parse_net($conf->{$opt});
4221                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4222             }
4223         }
4224
4225         vm_mon_cmd_nocheck($vmid, 'qom-set',
4226                     path => "machine/peripheral/balloon0",
4227                     property => "guest-stats-polling-interval",
4228                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4229
4230     });
4231 }
4232
4233 sub vm_mon_cmd {
4234     my ($vmid, $execute, %params) = @_;
4235
4236     my $cmd = { execute => $execute, arguments => \%params };
4237     vm_qmp_command($vmid, $cmd);
4238 }
4239
4240 sub vm_mon_cmd_nocheck {
4241     my ($vmid, $execute, %params) = @_;
4242
4243     my $cmd = { execute => $execute, arguments => \%params };
4244     vm_qmp_command($vmid, $cmd, 1);
4245 }
4246
4247 sub vm_qmp_command {
4248     my ($vmid, $cmd, $nocheck) = @_;
4249
4250     my $res;
4251
4252     my $timeout;
4253     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4254         $timeout = $cmd->{arguments}->{timeout};
4255         delete $cmd->{arguments}->{timeout};
4256     }
4257
4258     eval {
4259         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4260         my $sname = qmp_socket($vmid);
4261         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4262             my $qmpclient = PVE::QMPClient->new();
4263
4264             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4265         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
4266             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
4267                 if scalar(%{$cmd->{arguments}});
4268             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
4269         } else {
4270             die "unable to open monitor socket\n";
4271         }
4272     };
4273     if (my $err = $@) {
4274         syslog("err", "VM $vmid qmp command failed - $err");
4275         die $err;
4276     }
4277
4278     return $res;
4279 }
4280
4281 sub vm_human_monitor_command {
4282     my ($vmid, $cmdline) = @_;
4283
4284     my $res;
4285
4286     my $cmd = {
4287         execute => 'human-monitor-command',
4288         arguments => { 'command-line' => $cmdline},
4289     };
4290
4291     return vm_qmp_command($vmid, $cmd);
4292 }
4293
4294 sub vm_commandline {
4295     my ($storecfg, $vmid) = @_;
4296
4297     my $conf = load_config($vmid);
4298
4299     my $defaults = load_defaults();
4300
4301     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
4302
4303     return join(' ', @$cmd);
4304 }
4305
4306 sub vm_reset {
4307     my ($vmid, $skiplock) = @_;
4308
4309     lock_config($vmid, sub {
4310
4311         my $conf = load_config($vmid);
4312
4313         check_lock($conf) if !$skiplock;
4314
4315         vm_mon_cmd($vmid, "system_reset");
4316     });
4317 }
4318
4319 sub get_vm_volumes {
4320     my ($conf) = @_;
4321
4322     my $vollist = [];
4323     foreach_volid($conf, sub {
4324         my ($volid, $is_cdrom) = @_;
4325
4326         return if $volid =~ m|^/|;
4327
4328         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4329         return if !$sid;
4330
4331         push @$vollist, $volid;
4332     });
4333
4334     return $vollist;
4335 }
4336
4337 sub vm_stop_cleanup {
4338     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
4339
4340     eval {
4341
4342         if (!$keepActive) {
4343             my $vollist = get_vm_volumes($conf);
4344             PVE::Storage::deactivate_volumes($storecfg, $vollist);
4345         }
4346
4347         foreach my $ext (qw(mon qmp pid vnc qga)) {
4348             unlink "/var/run/qemu-server/${vmid}.$ext";
4349         }
4350
4351         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
4352     };
4353     warn $@ if $@; # avoid errors - just warn
4354 }
4355
4356 # Note: use $nockeck to skip tests if VM configuration file exists.
4357 # We need that when migration VMs to other nodes (files already moved)
4358 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
4359 sub vm_stop {
4360     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
4361
4362     $force = 1 if !defined($force) && !$shutdown;
4363
4364     if ($migratedfrom){
4365         my $pid = check_running($vmid, $nocheck, $migratedfrom);
4366         kill 15, $pid if $pid;
4367         my $conf = load_config($vmid, $migratedfrom);
4368         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
4369         return;
4370     }
4371
4372     lock_config($vmid, sub {
4373
4374         my $pid = check_running($vmid, $nocheck);
4375         return if !$pid;
4376
4377         my $conf;
4378         if (!$nocheck) {
4379             $conf = load_config($vmid);
4380             check_lock($conf) if !$skiplock;
4381             if (!defined($timeout) && $shutdown && $conf->{startup}) {
4382                 my $opts = parse_startup($conf->{startup});
4383                 $timeout = $opts->{down} if $opts->{down};
4384             }
4385         }
4386
4387         $timeout = 60 if !defined($timeout);
4388
4389         eval {
4390             if ($shutdown) {
4391                 if (defined($conf) && $conf->{agent}) {
4392                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
4393                 } else {
4394                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
4395                 }
4396             } else {
4397                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
4398             }
4399         };
4400         my $err = $@;
4401
4402         if (!$err) {
4403             my $count = 0;
4404             while (($count < $timeout) && check_running($vmid, $nocheck)) {
4405                 $count++;
4406                 sleep 1;
4407             }
4408
4409             if ($count >= $timeout) {
4410                 if ($force) {
4411                     warn "VM still running - terminating now with SIGTERM\n";
4412                     kill 15, $pid;
4413                 } else {
4414                     die "VM quit/powerdown failed - got timeout\n";
4415                 }
4416             } else {
4417                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4418                 return;
4419             }
4420         } else {
4421             if ($force) {
4422                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
4423                 kill 15, $pid;
4424             } else {
4425                 die "VM quit/powerdown failed\n";
4426             }
4427         }
4428
4429         # wait again
4430         $timeout = 10;
4431
4432         my $count = 0;
4433         while (($count < $timeout) && check_running($vmid, $nocheck)) {
4434             $count++;
4435             sleep 1;
4436         }
4437
4438         if ($count >= $timeout) {
4439             warn "VM still running - terminating now with SIGKILL\n";
4440             kill 9, $pid;
4441             sleep 1;
4442         }
4443
4444         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4445    });
4446 }
4447
4448 sub vm_suspend {
4449     my ($vmid, $skiplock) = @_;
4450
4451     lock_config($vmid, sub {
4452
4453         my $conf = load_config($vmid);
4454
4455         check_lock($conf) if !($skiplock || ($conf->{lock} && $conf->{lock} eq 'backup'));
4456
4457         vm_mon_cmd($vmid, "stop");
4458     });
4459 }
4460
4461 sub vm_resume {
4462     my ($vmid, $skiplock) = @_;
4463
4464     lock_config($vmid, sub {
4465
4466         my $conf = load_config($vmid);
4467
4468         check_lock($conf) if !($skiplock || ($conf->{lock} && $conf->{lock} eq 'backup'));
4469
4470         vm_mon_cmd($vmid, "cont");
4471     });
4472 }
4473
4474 sub vm_sendkey {
4475     my ($vmid, $skiplock, $key) = @_;
4476
4477     lock_config($vmid, sub {
4478
4479         my $conf = load_config($vmid);
4480
4481         # there is no qmp command, so we use the human monitor command
4482         vm_human_monitor_command($vmid, "sendkey $key");
4483     });
4484 }
4485
4486 sub vm_destroy {
4487     my ($storecfg, $vmid, $skiplock) = @_;
4488
4489     lock_config($vmid, sub {
4490
4491         my $conf = load_config($vmid);
4492
4493         check_lock($conf) if !$skiplock;
4494
4495         if (!check_running($vmid)) {
4496             destroy_vm($storecfg, $vmid);
4497         } else {
4498             die "VM $vmid is running - destroy failed\n";
4499         }
4500     });
4501 }
4502
4503 # pci helpers
4504
4505 sub file_write {
4506     my ($filename, $buf) = @_;
4507
4508     my $fh = IO::File->new($filename, "w");
4509     return undef if !$fh;
4510
4511     my $res = print $fh $buf;
4512
4513     $fh->close();
4514
4515     return $res;
4516 }
4517
4518 sub pci_device_info {
4519     my ($name) = @_;
4520
4521     my $res;
4522
4523     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
4524     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
4525
4526     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
4527     return undef if !defined($irq) || $irq !~ m/^\d+$/;
4528
4529     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
4530     return undef if !defined($vendor) || $vendor !~ s/^0x//;
4531
4532     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
4533     return undef if !defined($product) || $product !~ s/^0x//;
4534
4535     $res = {
4536         name => $name,
4537         vendor => $vendor,
4538         product => $product,
4539         domain => $domain,
4540         bus => $bus,
4541         slot => $slot,
4542         func => $func,
4543         irq => $irq,
4544         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
4545     };
4546
4547     return $res;
4548 }
4549
4550 sub pci_dev_reset {
4551     my ($dev) = @_;
4552
4553     my $name = $dev->{name};
4554
4555     my $fn = "$pcisysfs/devices/$name/reset";
4556
4557     return file_write($fn, "1");
4558 }
4559
4560 sub pci_dev_bind_to_stub {
4561     my ($dev) = @_;
4562
4563     my $name = $dev->{name};
4564
4565     my $testdir = "$pcisysfs/drivers/pci-stub/$name";
4566     return 1 if -d $testdir;
4567
4568     my $data = "$dev->{vendor} $dev->{product}";
4569     return undef if !file_write("$pcisysfs/drivers/pci-stub/new_id", $data);
4570
4571     my $fn = "$pcisysfs/devices/$name/driver/unbind";
4572     if (!file_write($fn, $name)) {
4573         return undef if -f $fn;
4574     }
4575
4576     $fn = "$pcisysfs/drivers/pci-stub/bind";
4577     if (! -d $testdir) {
4578         return undef if !file_write($fn, $name);
4579     }
4580
4581     return -d $testdir;
4582 }
4583
4584 sub pci_dev_bind_to_vfio {
4585     my ($dev) = @_;
4586
4587     my $name = $dev->{name};
4588
4589     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4590
4591     if (!-d $vfio_basedir) {
4592         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4593     }
4594     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4595
4596     my $testdir = "$vfio_basedir/$name";
4597     return 1 if -d $testdir;
4598
4599     my $data = "$dev->{vendor} $dev->{product}";
4600     return undef if !file_write("$vfio_basedir/new_id", $data);
4601
4602     my $fn = "$pcisysfs/devices/$name/driver/unbind";
4603     if (!file_write($fn, $name)) {
4604         return undef if -f $fn;
4605     }
4606
4607     $fn = "$vfio_basedir/bind";
4608     if (! -d $testdir) {
4609         return undef if !file_write($fn, $name);
4610     }
4611
4612     return -d $testdir;
4613 }
4614
4615 sub pci_dev_group_bind_to_vfio {
4616     my ($pciid) = @_;
4617
4618     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4619
4620     if (!-d $vfio_basedir) {
4621         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4622     }
4623     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4624
4625     # get IOMMU group devices
4626     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
4627       my @devs = grep /^0000:/, readdir($D);
4628     closedir($D);
4629
4630     foreach my $pciid (@devs) {
4631         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
4632
4633         # pci bridges, switches or root ports are not supported
4634         # they have a pci_bus subdirectory so skip them
4635         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
4636
4637         my $info = pci_device_info($1);
4638         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
4639     }
4640
4641     return 1;
4642 }
4643
4644 sub print_pci_addr {
4645     my ($id, $bridges) = @_;
4646
4647     my $res = '';
4648     my $devices = {
4649         piix3 => { bus => 0, addr => 1 },
4650         #addr2 : first videocard
4651         balloon0 => { bus => 0, addr => 3 },
4652         watchdog => { bus => 0, addr => 4 },
4653         scsihw0 => { bus => 0, addr => 5 },
4654         'pci.3' => { bus => 0, addr => 5 }, #can also be used for virtio-scsi-single bridge
4655         scsihw1 => { bus => 0, addr => 6 },
4656         ahci0 => { bus => 0, addr => 7 },
4657         qga0 => { bus => 0, addr => 8 },
4658         spice => { bus => 0, addr => 9 },
4659         virtio0 => { bus => 0, addr => 10 },
4660         virtio1 => { bus => 0, addr => 11 },
4661         virtio2 => { bus => 0, addr => 12 },
4662         virtio3 => { bus => 0, addr => 13 },
4663         virtio4 => { bus => 0, addr => 14 },
4664         virtio5 => { bus => 0, addr => 15 },
4665         hostpci0 => { bus => 0, addr => 16 },
4666         hostpci1 => { bus => 0, addr => 17 },
4667         net0 => { bus => 0, addr => 18 },
4668         net1 => { bus => 0, addr => 19 },
4669         net2 => { bus => 0, addr => 20 },
4670         net3 => { bus => 0, addr => 21 },
4671         net4 => { bus => 0, addr => 22 },
4672         net5 => { bus => 0, addr => 23 },
4673         vga1 => { bus => 0, addr => 24 },
4674         vga2 => { bus => 0, addr => 25 },
4675         vga3 => { bus => 0, addr => 26 },
4676         hostpci2 => { bus => 0, addr => 27 },
4677         hostpci3 => { bus => 0, addr => 28 },
4678         #addr29 : usb-host (pve-usb.cfg)
4679         'pci.1' => { bus => 0, addr => 30 },
4680         'pci.2' => { bus => 0, addr => 31 },
4681         'net6' => { bus => 1, addr => 1 },
4682         'net7' => { bus => 1, addr => 2 },
4683         'net8' => { bus => 1, addr => 3 },
4684         'net9' => { bus => 1, addr => 4 },
4685         'net10' => { bus => 1, addr => 5 },
4686         'net11' => { bus => 1, addr => 6 },
4687         'net12' => { bus => 1, addr => 7 },
4688         'net13' => { bus => 1, addr => 8 },
4689         'net14' => { bus => 1, addr => 9 },
4690         'net15' => { bus => 1, addr => 10 },
4691         'net16' => { bus => 1, addr => 11 },
4692         'net17' => { bus => 1, addr => 12 },
4693         'net18' => { bus => 1, addr => 13 },
4694         'net19' => { bus => 1, addr => 14 },
4695         'net20' => { bus => 1, addr => 15 },
4696         'net21' => { bus => 1, addr => 16 },
4697         'net22' => { bus => 1, addr => 17 },
4698         'net23' => { bus => 1, addr => 18 },
4699         'net24' => { bus => 1, addr => 19 },
4700         'net25' => { bus => 1, addr => 20 },
4701         'net26' => { bus => 1, addr => 21 },
4702         'net27' => { bus => 1, addr => 22 },
4703         'net28' => { bus => 1, addr => 23 },
4704         'net29' => { bus => 1, addr => 24 },
4705         'net30' => { bus => 1, addr => 25 },
4706         'net31' => { bus => 1, addr => 26 },
4707         'virtio6' => { bus => 2, addr => 1 },
4708         'virtio7' => { bus => 2, addr => 2 },
4709         'virtio8' => { bus => 2, addr => 3 },
4710         'virtio9' => { bus => 2, addr => 4 },
4711         'virtio10' => { bus => 2, addr => 5 },
4712         'virtio11' => { bus => 2, addr => 6 },
4713         'virtio12' => { bus => 2, addr => 7 },
4714         'virtio13' => { bus => 2, addr => 8 },
4715         'virtio14' => { bus => 2, addr => 9 },
4716         'virtio15' => { bus => 2, addr => 10 },
4717         'virtioscsi0' => { bus => 3, addr => 1 },
4718         'virtioscsi1' => { bus => 3, addr => 2 },
4719         'virtioscsi2' => { bus => 3, addr => 3 },
4720         'virtioscsi3' => { bus => 3, addr => 4 },
4721         'virtioscsi4' => { bus => 3, addr => 5 },
4722         'virtioscsi5' => { bus => 3, addr => 6 },
4723         'virtioscsi6' => { bus => 3, addr => 7 },
4724         'virtioscsi7' => { bus => 3, addr => 8 },
4725         'virtioscsi8' => { bus => 3, addr => 9 },
4726         'virtioscsi9' => { bus => 3, addr => 10 },
4727         'virtioscsi10' => { bus => 3, addr => 11 },
4728         'virtioscsi11' => { bus => 3, addr => 12 },
4729         'virtioscsi12' => { bus => 3, addr => 13 },
4730         'virtioscsi13' => { bus => 3, addr => 14 },
4731         'virtioscsi14' => { bus => 3, addr => 15 },
4732         'virtioscsi15' => { bus => 3, addr => 16 },
4733         'virtioscsi16' => { bus => 3, addr => 17 },
4734         'virtioscsi17' => { bus => 3, addr => 18 },
4735         'virtioscsi18' => { bus => 3, addr => 19 },
4736         'virtioscsi19' => { bus => 3, addr => 20 },
4737         'virtioscsi20' => { bus => 3, addr => 21 },
4738         'virtioscsi21' => { bus => 3, addr => 22 },
4739         'virtioscsi22' => { bus => 3, addr => 23 },
4740         'virtioscsi23' => { bus => 3, addr => 24 },
4741         'virtioscsi24' => { bus => 3, addr => 25 },
4742         'virtioscsi25' => { bus => 3, addr => 26 },
4743         'virtioscsi26' => { bus => 3, addr => 27 },
4744         'virtioscsi27' => { bus => 3, addr => 28 },
4745         'virtioscsi28' => { bus => 3, addr => 29 },
4746         'virtioscsi29' => { bus => 3, addr => 30 },
4747         'virtioscsi30' => { bus => 3, addr => 31 },
4748
4749     };
4750
4751     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
4752            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
4753            my $bus = $devices->{$id}->{bus};
4754            $res = ",bus=pci.$bus,addr=$addr";
4755            $bridges->{$bus} = 1 if $bridges;
4756     }
4757     return $res;
4758
4759 }
4760
4761 sub print_pcie_addr {
4762     my ($id) = @_;
4763
4764     my $res = '';
4765     my $devices = {
4766         hostpci0 => { bus => "ich9-pcie-port-1", addr => 0 },
4767         hostpci1 => { bus => "ich9-pcie-port-2", addr => 0 },
4768         hostpci2 => { bus => "ich9-pcie-port-3", addr => 0 },
4769         hostpci3 => { bus => "ich9-pcie-port-4", addr => 0 },
4770     };
4771
4772     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
4773            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
4774            my $bus = $devices->{$id}->{bus};
4775            $res = ",bus=$bus,addr=$addr";
4776     }
4777     return $res;
4778
4779 }
4780
4781 # vzdump restore implementaion
4782
4783 sub tar_archive_read_firstfile {
4784     my $archive = shift;
4785
4786     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
4787
4788     # try to detect archive type first
4789     my $pid = open (TMP, "tar tf '$archive'|") ||
4790         die "unable to open file '$archive'\n";
4791     my $firstfile = <TMP>;
4792     kill 15, $pid;
4793     close TMP;
4794
4795     die "ERROR: archive contaions no data\n" if !$firstfile;
4796     chomp $firstfile;
4797
4798     return $firstfile;
4799 }
4800
4801 sub tar_restore_cleanup {
4802     my ($storecfg, $statfile) = @_;
4803
4804     print STDERR "starting cleanup\n";
4805
4806     if (my $fd = IO::File->new($statfile, "r")) {
4807         while (defined(my $line = <$fd>)) {
4808             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
4809                 my $volid = $2;
4810                 eval {
4811                     if ($volid =~ m|^/|) {
4812                         unlink $volid || die 'unlink failed\n';
4813                     } else {
4814                         PVE::Storage::vdisk_free($storecfg, $volid);
4815                     }
4816                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
4817                 };
4818                 print STDERR "unable to cleanup '$volid' - $@" if $@;
4819             } else {
4820                 print STDERR "unable to parse line in statfile - $line";
4821             }
4822         }
4823         $fd->close();
4824     }
4825 }
4826
4827 sub restore_archive {
4828     my ($archive, $vmid, $user, $opts) = @_;
4829
4830     my $format = $opts->{format};
4831     my $comp;
4832
4833     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
4834         $format = 'tar' if !$format;
4835         $comp = 'gzip';
4836     } elsif ($archive =~ m/\.tar$/) {
4837         $format = 'tar' if !$format;
4838     } elsif ($archive =~ m/.tar.lzo$/) {
4839         $format = 'tar' if !$format;
4840         $comp = 'lzop';
4841     } elsif ($archive =~ m/\.vma$/) {
4842         $format = 'vma' if !$format;
4843     } elsif ($archive =~ m/\.vma\.gz$/) {
4844         $format = 'vma' if !$format;
4845         $comp = 'gzip';
4846     } elsif ($archive =~ m/\.vma\.lzo$/) {
4847         $format = 'vma' if !$format;
4848         $comp = 'lzop';
4849     } else {
4850         $format = 'vma' if !$format; # default
4851     }
4852
4853     # try to detect archive format
4854     if ($format eq 'tar') {
4855         return restore_tar_archive($archive, $vmid, $user, $opts);
4856     } else {
4857         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
4858     }
4859 }
4860
4861 sub restore_update_config_line {
4862     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
4863
4864     return if $line =~ m/^\#qmdump\#/;
4865     return if $line =~ m/^\#vzdump\#/;
4866     return if $line =~ m/^lock:/;
4867     return if $line =~ m/^unused\d+:/;
4868     return if $line =~ m/^parent:/;
4869     return if $line =~ m/^template:/; # restored VM is never a template
4870
4871     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
4872         # try to convert old 1.X settings
4873         my ($id, $ind, $ethcfg) = ($1, $2, $3);
4874         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
4875             my ($model, $macaddr) = split(/\=/, $devconfig);
4876             $macaddr = PVE::Tools::random_ether_addr() if !$macaddr || $unique;
4877             my $net = {
4878                 model => $model,
4879                 bridge => "vmbr$ind",
4880                 macaddr => $macaddr,
4881             };
4882             my $netstr = print_net($net);
4883
4884             print $outfd "net$cookie->{netcount}: $netstr\n";
4885             $cookie->{netcount}++;
4886         }
4887     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
4888         my ($id, $netstr) = ($1, $2);
4889         my $net = parse_net($netstr);
4890         $net->{macaddr} = PVE::Tools::random_ether_addr() if $net->{macaddr};
4891         $netstr = print_net($net);
4892         print $outfd "$id: $netstr\n";
4893     } elsif ($line =~ m/^((ide|scsi|virtio|sata)\d+):\s*(\S+)\s*$/) {
4894         my $virtdev = $1;
4895         my $value = $3;
4896         if ($line =~ m/backup=no/) {
4897             print $outfd "#$line";
4898         } elsif ($virtdev && $map->{$virtdev}) {
4899             my $di = parse_drive($virtdev, $value);
4900             delete $di->{format}; # format can change on restore
4901             $di->{file} = $map->{$virtdev};
4902             $value = print_drive($vmid, $di);
4903             print $outfd "$virtdev: $value\n";
4904         } else {
4905             print $outfd $line;
4906         }
4907     } else {
4908         print $outfd $line;
4909     }
4910 }
4911
4912 sub scan_volids {
4913     my ($cfg, $vmid) = @_;
4914
4915     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
4916
4917     my $volid_hash = {};
4918     foreach my $storeid (keys %$info) {
4919         foreach my $item (@{$info->{$storeid}}) {
4920             next if !($item->{volid} && $item->{size});
4921             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
4922             $volid_hash->{$item->{volid}} = $item;
4923         }
4924     }
4925
4926     return $volid_hash;
4927 }
4928
4929 sub get_used_paths {
4930     my ($vmid, $storecfg, $conf, $scan_snapshots, $skip_drive) = @_;
4931
4932     my $used_path = {};
4933
4934     my $scan_config = sub {
4935         my ($cref, $snapname) = @_;
4936
4937         foreach my $key (keys %$cref) {
4938             my $value = $cref->{$key};
4939             if (valid_drivename($key)) {
4940                 next if $skip_drive && $key eq $skip_drive;
4941                 my $drive = parse_drive($key, $value);
4942                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
4943                 if ($drive->{file} =~ m!^/!) {
4944                     $used_path->{$drive->{file}}++; # = 1;
4945                 } else {
4946                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
4947                     next if !$storeid;
4948                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
4949                     next if !$scfg;
4950                     my $path = PVE::Storage::path($storecfg, $drive->{file}, $snapname);
4951                     $used_path->{$path}++; # = 1;
4952                 }
4953             }
4954         }
4955     };
4956
4957     &$scan_config($conf);
4958
4959     undef $skip_drive;
4960
4961     if ($scan_snapshots) {
4962         foreach my $snapname (keys %{$conf->{snapshots}}) {
4963             &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
4964         }
4965     }
4966
4967     return $used_path;
4968 }
4969
4970 sub update_disksize {
4971     my ($vmid, $conf, $volid_hash) = @_;
4972
4973     my $changes;
4974
4975     my $used = {};
4976
4977     # Note: it is allowed to define multiple storages with same path (alias), so
4978     # we need to check both 'volid' and real 'path' (two different volid can point
4979     # to the same path).
4980
4981     my $usedpath = {};
4982
4983     # update size info
4984     foreach my $opt (keys %$conf) {
4985         if (valid_drivename($opt)) {
4986             my $drive = parse_drive($opt, $conf->{$opt});
4987             my $volid = $drive->{file};
4988             next if !$volid;
4989
4990             $used->{$volid} = 1;
4991             if ($volid_hash->{$volid} &&
4992                 (my $path = $volid_hash->{$volid}->{path})) {
4993                 $usedpath->{$path} = 1;
4994             }
4995
4996             next if drive_is_cdrom($drive);
4997             next if !$volid_hash->{$volid};
4998
4999             $drive->{size} = $volid_hash->{$volid}->{size};
5000             my $new = print_drive($vmid, $drive);
5001             if ($new ne $conf->{$opt}) {
5002                 $changes = 1;
5003                 $conf->{$opt} = $new;
5004             }
5005         }
5006     }
5007
5008     # remove 'unusedX' entry if volume is used
5009     foreach my $opt (keys %$conf) {
5010         next if $opt !~ m/^unused\d+$/;
5011         my $volid = $conf->{$opt};
5012         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5013         if ($used->{$volid} || ($path && $usedpath->{$path})) {
5014             $changes = 1;
5015             delete $conf->{$opt};
5016         }
5017     }
5018
5019     foreach my $volid (sort keys %$volid_hash) {
5020         next if $volid =~ m/vm-$vmid-state-/;
5021         next if $used->{$volid};
5022         my $path = $volid_hash->{$volid}->{path};
5023         next if !$path; # just to be sure
5024         next if $usedpath->{$path};
5025         $changes = 1;
5026         add_unused_volume($conf, $volid);
5027         $usedpath->{$path} = 1; # avoid to add more than once (aliases)
5028     }
5029
5030     return $changes;
5031 }
5032
5033 sub rescan {
5034     my ($vmid, $nolock) = @_;
5035
5036     my $cfg = PVE::Cluster::cfs_read_file("storage.cfg");
5037
5038     my $volid_hash = scan_volids($cfg, $vmid);
5039
5040     my $updatefn =  sub {
5041         my ($vmid) = @_;
5042
5043         my $conf = load_config($vmid);
5044
5045         check_lock($conf);
5046
5047         my $vm_volids = {};
5048         foreach my $volid (keys %$volid_hash) {
5049             my $info = $volid_hash->{$volid};
5050             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5051         }
5052
5053         my $changes = update_disksize($vmid, $conf, $vm_volids);
5054
5055         update_config_nolock($vmid, $conf, 1) if $changes;
5056     };
5057
5058     if (defined($vmid)) {
5059         if ($nolock) {
5060             &$updatefn($vmid);
5061         } else {
5062             lock_config($vmid, $updatefn, $vmid);
5063         }
5064     } else {
5065         my $vmlist = config_list();
5066         foreach my $vmid (keys %$vmlist) {
5067             if ($nolock) {
5068                 &$updatefn($vmid);
5069             } else {
5070                 lock_config($vmid, $updatefn, $vmid);
5071             }
5072         }
5073     }
5074 }
5075
5076 sub restore_vma_archive {
5077     my ($archive, $vmid, $user, $opts, $comp) = @_;
5078
5079     my $input = $archive eq '-' ? "<&STDIN" : undef;
5080     my $readfrom = $archive;
5081
5082     my $uncomp = '';
5083     if ($comp) {
5084         $readfrom = '-';
5085         my $qarchive = PVE::Tools::shellquote($archive);
5086         if ($comp eq 'gzip') {
5087             $uncomp = "zcat $qarchive|";
5088         } elsif ($comp eq 'lzop') {
5089             $uncomp = "lzop -d -c $qarchive|";
5090         } else {
5091             die "unknown compression method '$comp'\n";
5092         }
5093
5094     }
5095
5096     my $tmpdir = "/var/tmp/vzdumptmp$$";
5097     rmtree $tmpdir;
5098
5099     # disable interrupts (always do cleanups)
5100     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5101         warn "got interrupt - ignored\n";
5102     };
5103
5104     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5105     POSIX::mkfifo($mapfifo, 0600);
5106     my $fifofh;
5107
5108     my $openfifo = sub {
5109         open($fifofh, '>', $mapfifo) || die $!;
5110     };
5111
5112     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5113
5114     my $oldtimeout;
5115     my $timeout = 5;
5116
5117     my $devinfo = {};
5118
5119     my $rpcenv = PVE::RPCEnvironment::get();
5120
5121     my $conffile = config_file($vmid);
5122     my $tmpfn = "$conffile.$$.tmp";
5123
5124     # Note: $oldconf is undef if VM does not exists
5125     my $oldconf = PVE::Cluster::cfs_read_file(cfs_config_path($vmid));
5126
5127     my $print_devmap = sub {
5128         my $virtdev_hash = {};
5129
5130         my $cfgfn = "$tmpdir/qemu-server.conf";
5131
5132         # we can read the config - that is already extracted
5133         my $fh = IO::File->new($cfgfn, "r") ||
5134             "unable to read qemu-server.conf - $!\n";
5135
5136         while (defined(my $line = <$fh>)) {
5137             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5138                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5139                 die "archive does not contain data for drive '$virtdev'\n"
5140                     if !$devinfo->{$devname};
5141                 if (defined($opts->{storage})) {
5142                     $storeid = $opts->{storage} || 'local';
5143                 } elsif (!$storeid) {
5144                     $storeid = 'local';
5145                 }
5146                 $format = 'raw' if !$format;
5147                 $devinfo->{$devname}->{devname} = $devname;
5148                 $devinfo->{$devname}->{virtdev} = $virtdev;
5149                 $devinfo->{$devname}->{format} = $format;
5150                 $devinfo->{$devname}->{storeid} = $storeid;
5151
5152                 # check permission on storage
5153                 my $pool = $opts->{pool}; # todo: do we need that?
5154                 if ($user ne 'root@pam') {
5155                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5156                 }
5157
5158                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5159             }
5160         }
5161
5162         foreach my $devname (keys %$devinfo) {
5163             die "found no device mapping information for device '$devname'\n"
5164                 if !$devinfo->{$devname}->{virtdev};
5165         }
5166
5167         my $cfg = cfs_read_file('storage.cfg');
5168
5169         # create empty/temp config
5170         if ($oldconf) {
5171             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5172             foreach_drive($oldconf, sub {
5173                 my ($ds, $drive) = @_;
5174
5175                 return if drive_is_cdrom($drive);
5176
5177                 my $volid = $drive->{file};
5178
5179                 return if !$volid || $volid =~ m|^/|;
5180
5181                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5182                 return if !$path || !$owner || ($owner != $vmid);
5183
5184                 # Note: only delete disk we want to restore
5185                 # other volumes will become unused
5186                 if ($virtdev_hash->{$ds}) {
5187                     PVE::Storage::vdisk_free($cfg, $volid);
5188                 }
5189             });
5190         }
5191
5192         my $map = {};
5193         foreach my $virtdev (sort keys %$virtdev_hash) {
5194             my $d = $virtdev_hash->{$virtdev};
5195             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5196             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5197
5198             # test if requested format is supported
5199             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5200             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5201             $d->{format} = $defFormat if !$supported;
5202
5203             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5204                                                   $d->{format}, undef, $alloc_size);
5205             print STDERR "new volume ID is '$volid'\n";
5206             $d->{volid} = $volid;
5207             my $path = PVE::Storage::path($cfg, $volid);
5208
5209             my $write_zeros = 1;
5210             # fixme: what other storages types initialize volumes with zero?
5211             if ($scfg->{type} eq 'dir' || $scfg->{type} eq 'nfs' || $scfg->{type} eq 'glusterfs' ||
5212                 $scfg->{type} eq 'sheepdog' || $scfg->{type} eq 'rbd') {
5213                 $write_zeros = 0;
5214             }
5215
5216             print $fifofh "${write_zeros}:$d->{devname}=$path\n";
5217
5218             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5219             $map->{$virtdev} = $volid;
5220         }
5221
5222         $fh->seek(0, 0) || die "seek failed - $!\n";
5223
5224         my $outfd = new IO::File ($tmpfn, "w") ||
5225             die "unable to write config for VM $vmid\n";
5226
5227         my $cookie = { netcount => 0 };
5228         while (defined(my $line = <$fh>)) {
5229             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5230         }
5231
5232         $fh->close();
5233         $outfd->close();
5234     };
5235
5236     eval {
5237         # enable interrupts
5238         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5239             die "interrupted by signal\n";
5240         };
5241         local $SIG{ALRM} = sub { die "got timeout\n"; };
5242
5243         $oldtimeout = alarm($timeout);
5244
5245         my $parser = sub {
5246             my $line = shift;
5247
5248             print "$line\n";
5249
5250             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5251                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5252                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5253             } elsif ($line =~ m/^CTIME: /) {
5254                 # we correctly received the vma config, so we can disable
5255                 # the timeout now for disk allocation (set to 10 minutes, so
5256                 # that we always timeout if something goes wrong)
5257                 alarm(600);
5258                 &$print_devmap();
5259                 print $fifofh "done\n";
5260                 my $tmp = $oldtimeout || 0;
5261                 $oldtimeout = undef;
5262                 alarm($tmp);
5263                 close($fifofh);
5264             }
5265         };
5266
5267         print "restore vma archive: $cmd\n";
5268         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5269     };
5270     my $err = $@;
5271
5272     alarm($oldtimeout) if $oldtimeout;
5273
5274     unlink $mapfifo;
5275
5276     if ($err) {
5277         rmtree $tmpdir;
5278         unlink $tmpfn;
5279
5280         my $cfg = cfs_read_file('storage.cfg');
5281         foreach my $devname (keys %$devinfo) {
5282             my $volid = $devinfo->{$devname}->{volid};
5283             next if !$volid;
5284             eval {
5285                 if ($volid =~ m|^/|) {
5286                     unlink $volid || die 'unlink failed\n';
5287                 } else {
5288                     PVE::Storage::vdisk_free($cfg, $volid);
5289                 }
5290                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5291             };
5292             print STDERR "unable to cleanup '$volid' - $@" if $@;
5293         }
5294         die $err;
5295     }
5296
5297     rmtree $tmpdir;
5298
5299     rename($tmpfn, $conffile) ||
5300         die "unable to commit configuration file '$conffile'\n";
5301
5302     PVE::Cluster::cfs_update(); # make sure we read new file
5303
5304     eval { rescan($vmid, 1); };
5305     warn $@ if $@;
5306 }
5307
5308 sub restore_tar_archive {
5309     my ($archive, $vmid, $user, $opts) = @_;
5310
5311     if ($archive ne '-') {
5312         my $firstfile = tar_archive_read_firstfile($archive);
5313         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5314             if $firstfile ne 'qemu-server.conf';
5315     }
5316
5317     my $storecfg = cfs_read_file('storage.cfg');
5318
5319     # destroy existing data - keep empty config
5320     my $vmcfgfn = config_file($vmid);
5321     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5322
5323     my $tocmd = "/usr/lib/qemu-server/qmextract";
5324
5325     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5326     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5327     $tocmd .= ' --prealloc' if $opts->{prealloc};
5328     $tocmd .= ' --info' if $opts->{info};
5329
5330     # tar option "xf" does not autodetect compression when read from STDIN,
5331     # so we pipe to zcat
5332     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5333         PVE::Tools::shellquote("--to-command=$tocmd");
5334
5335     my $tmpdir = "/var/tmp/vzdumptmp$$";
5336     mkpath $tmpdir;
5337
5338     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5339     local $ENV{VZDUMP_VMID} = $vmid;
5340     local $ENV{VZDUMP_USER} = $user;
5341
5342     my $conffile = config_file($vmid);
5343     my $tmpfn = "$conffile.$$.tmp";
5344
5345     # disable interrupts (always do cleanups)
5346     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5347         print STDERR "got interrupt - ignored\n";
5348     };
5349
5350     eval {
5351         # enable interrupts
5352         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5353             die "interrupted by signal\n";
5354         };
5355
5356         if ($archive eq '-') {
5357             print "extracting archive from STDIN\n";
5358             run_command($cmd, input => "<&STDIN");
5359         } else {
5360             print "extracting archive '$archive'\n";
5361             run_command($cmd);
5362         }
5363
5364         return if $opts->{info};
5365
5366         # read new mapping
5367         my $map = {};
5368         my $statfile = "$tmpdir/qmrestore.stat";
5369         if (my $fd = IO::File->new($statfile, "r")) {
5370             while (defined (my $line = <$fd>)) {
5371                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5372                     $map->{$1} = $2 if $1;
5373                 } else {
5374                     print STDERR "unable to parse line in statfile - $line\n";
5375                 }
5376             }
5377             $fd->close();
5378         }
5379
5380         my $confsrc = "$tmpdir/qemu-server.conf";
5381
5382         my $srcfd = new IO::File($confsrc, "r") ||
5383             die "unable to open file '$confsrc'\n";
5384
5385         my $outfd = new IO::File ($tmpfn, "w") ||
5386             die "unable to write config for VM $vmid\n";
5387
5388         my $cookie = { netcount => 0 };
5389         while (defined (my $line = <$srcfd>)) {
5390             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5391         }
5392
5393         $srcfd->close();
5394         $outfd->close();
5395     };
5396     my $err = $@;
5397
5398     if ($err) {
5399
5400         unlink $tmpfn;
5401
5402         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
5403
5404         die $err;
5405     }
5406
5407     rmtree $tmpdir;
5408
5409     rename $tmpfn, $conffile ||
5410         die "unable to commit configuration file '$conffile'\n";
5411
5412     PVE::Cluster::cfs_update(); # make sure we read new file
5413
5414     eval { rescan($vmid, 1); };
5415     warn $@ if $@;
5416 };
5417
5418
5419 # Internal snapshots
5420
5421 # NOTE: Snapshot create/delete involves several non-atomic
5422 # action, and can take a long time.
5423 # So we try to avoid locking the file and use 'lock' variable
5424 # inside the config file instead.
5425
5426 my $snapshot_copy_config = sub {
5427     my ($source, $dest) = @_;
5428
5429     foreach my $k (keys %$source) {
5430         next if $k eq 'snapshots';
5431         next if $k eq 'snapstate';
5432         next if $k eq 'snaptime';
5433         next if $k eq 'vmstate';
5434         next if $k eq 'lock';
5435         next if $k eq 'digest';
5436         next if $k eq 'description';
5437         next if $k =~ m/^unused\d+$/;
5438
5439         $dest->{$k} = $source->{$k};
5440     }
5441 };
5442
5443 my $snapshot_apply_config = sub {
5444     my ($conf, $snap) = @_;
5445
5446     # copy snapshot list
5447     my $newconf = {
5448         snapshots => $conf->{snapshots},
5449     };
5450
5451     # keep description and list of unused disks
5452     foreach my $k (keys %$conf) {
5453         next if !($k =~ m/^unused\d+$/ || $k eq 'description');
5454         $newconf->{$k} = $conf->{$k};
5455     }
5456
5457     &$snapshot_copy_config($snap, $newconf);
5458
5459     return $newconf;
5460 };
5461
5462 sub foreach_writable_storage {
5463     my ($conf, $func) = @_;
5464
5465     my $sidhash = {};
5466
5467     foreach my $ds (keys %$conf) {
5468         next if !valid_drivename($ds);
5469
5470         my $drive = parse_drive($ds, $conf->{$ds});
5471         next if !$drive;
5472         next if drive_is_cdrom($drive);
5473
5474         my $volid = $drive->{file};
5475
5476         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5477         $sidhash->{$sid} = $sid if $sid;
5478     }
5479
5480     foreach my $sid (sort keys %$sidhash) {
5481         &$func($sid);
5482     }
5483 }
5484
5485 my $alloc_vmstate_volid = sub {
5486     my ($storecfg, $vmid, $conf, $snapname) = @_;
5487
5488     # Note: we try to be smart when selecting a $target storage
5489
5490     my $target;
5491
5492     # search shared storage first
5493     foreach_writable_storage($conf, sub {
5494         my ($sid) = @_;
5495         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
5496         return if !$scfg->{shared};
5497
5498         $target = $sid if !$target || $scfg->{path}; # prefer file based storage
5499     });
5500
5501     if (!$target) {
5502         # now search local storage
5503         foreach_writable_storage($conf, sub {
5504             my ($sid) = @_;
5505             my $scfg = PVE::Storage::storage_config($storecfg, $sid);
5506             return if $scfg->{shared};
5507
5508             $target = $sid if !$target || $scfg->{path}; # prefer file based storage;
5509         });
5510     }
5511
5512     $target = 'local' if !$target;
5513
5514     my $driver_state_size = 500; # assume 32MB is enough to safe all driver state;
5515     # we abort live save after $conf->{memory}, so we need at max twice that space
5516     my $size = $conf->{memory}*2 + $driver_state_size;
5517
5518     my $name = "vm-$vmid-state-$snapname";
5519     my $scfg = PVE::Storage::storage_config($storecfg, $target);
5520     $name .= ".raw" if $scfg->{path}; # add filename extension for file base storage
5521     my $volid = PVE::Storage::vdisk_alloc($storecfg, $target, $vmid, 'raw', $name, $size*1024);
5522
5523     return $volid;
5524 };
5525
5526 my $snapshot_prepare = sub {
5527     my ($vmid, $snapname, $save_vmstate, $comment) = @_;
5528
5529     my $snap;
5530
5531     my $updatefn =  sub {
5532
5533         my $conf = load_config($vmid);
5534
5535         die "you can't take a snapshot if it's a template\n"
5536             if is_template($conf);
5537
5538         check_lock($conf);
5539
5540         $conf->{lock} = 'snapshot';
5541
5542         die "snapshot name '$snapname' already used\n"
5543             if defined($conf->{snapshots}->{$snapname});
5544
5545         my $storecfg = PVE::Storage::config();
5546         die "snapshot feature is not available" if !has_feature('snapshot', $conf, $storecfg);
5547
5548         $snap = $conf->{snapshots}->{$snapname} = {};
5549
5550         if ($save_vmstate && check_running($vmid)) {
5551             $snap->{vmstate} = &$alloc_vmstate_volid($storecfg, $vmid, $conf, $snapname);
5552         }
5553
5554         &$snapshot_copy_config($conf, $snap);
5555
5556         $snap->{snapstate} = "prepare";
5557         $snap->{snaptime} = time();
5558         $snap->{description} = $comment if $comment;
5559
5560         # always overwrite machine if we save vmstate. This makes sure we
5561         # can restore it later using correct machine type
5562         $snap->{machine} = get_current_qemu_machine($vmid) if $snap->{vmstate};
5563
5564         update_config_nolock($vmid, $conf, 1);
5565     };
5566
5567     lock_config($vmid, $updatefn);
5568
5569     return $snap;
5570 };
5571
5572 my $snapshot_commit = sub {
5573     my ($vmid, $snapname) = @_;
5574
5575     my $updatefn = sub {
5576
5577         my $conf = load_config($vmid);
5578
5579         die "missing snapshot lock\n"
5580             if !($conf->{lock} && $conf->{lock} eq 'snapshot');
5581
5582         my $has_machine_config = defined($conf->{machine});
5583
5584         my $snap = $conf->{snapshots}->{$snapname};
5585
5586         die "snapshot '$snapname' does not exist\n" if !defined($snap);
5587
5588         die "wrong snapshot state\n"
5589             if !($snap->{snapstate} && $snap->{snapstate} eq "prepare");
5590
5591         delete $snap->{snapstate};
5592         delete $conf->{lock};
5593
5594         my $newconf = &$snapshot_apply_config($conf, $snap);
5595
5596         delete $newconf->{machine} if !$has_machine_config;
5597
5598         $newconf->{parent} = $snapname;
5599
5600         update_config_nolock($vmid, $newconf, 1);
5601     };
5602
5603     lock_config($vmid, $updatefn);
5604 };
5605
5606 sub snapshot_rollback {
5607     my ($vmid, $snapname) = @_;
5608
5609     my $prepare = 1;
5610
5611     my $storecfg = PVE::Storage::config();
5612
5613     my $conf = load_config($vmid);
5614
5615     my $get_snapshot_config = sub {
5616
5617         die "you can't rollback if vm is a template\n" if is_template($conf);
5618
5619         my $res = $conf->{snapshots}->{$snapname};
5620
5621         die "snapshot '$snapname' does not exist\n" if !defined($res);
5622
5623         return $res;
5624     };
5625
5626     my $snap = &$get_snapshot_config();
5627
5628     foreach_drive($snap, sub {
5629         my ($ds, $drive) = @_;
5630
5631         return if drive_is_cdrom($drive);
5632
5633         my $volid = $drive->{file};
5634
5635         PVE::Storage::volume_rollback_is_possible($storecfg, $volid, $snapname);
5636     });
5637
5638     my $updatefn = sub {
5639
5640         $conf = load_config($vmid);
5641
5642         $snap = &$get_snapshot_config();
5643
5644         die "unable to rollback to incomplete snapshot (snapstate = $snap->{snapstate})\n"
5645             if $snap->{snapstate};
5646
5647         if ($prepare) {
5648             check_lock($conf);
5649             vm_stop($storecfg, $vmid, undef, undef, 5, undef, undef);
5650         }
5651
5652         die "unable to rollback vm $vmid: vm is running\n"
5653             if check_running($vmid);
5654
5655         if ($prepare) {
5656             $conf->{lock} = 'rollback';
5657         } else {
5658             die "got wrong lock\n" if !($conf->{lock} && $conf->{lock} eq 'rollback');
5659             delete $conf->{lock};
5660         }
5661
5662         my $forcemachine;
5663
5664         if (!$prepare) {
5665             my $has_machine_config = defined($conf->{machine});
5666
5667             # copy snapshot config to current config
5668             $conf = &$snapshot_apply_config($conf, $snap);
5669             $conf->{parent} = $snapname;
5670
5671             # Note: old code did not store 'machine', so we try to be smart
5672             # and guess the snapshot was generated with kvm 1.4 (pc-i440fx-1.4).
5673             $forcemachine = $conf->{machine} || 'pc-i440fx-1.4';
5674             # we remove the 'machine' configuration if not explicitly specified
5675             # in the original config.
5676             delete $conf->{machine} if $snap->{vmstate} && !$has_machine_config;
5677         }
5678
5679         update_config_nolock($vmid, $conf, 1);
5680
5681         if (!$prepare && $snap->{vmstate}) {
5682             my $statefile = PVE::Storage::path($storecfg, $snap->{vmstate});
5683             vm_start($storecfg, $vmid, $statefile, undef, undef, undef, $forcemachine);
5684         }
5685     };
5686
5687     lock_config($vmid, $updatefn);
5688
5689     foreach_drive($snap, sub {
5690         my ($ds, $drive) = @_;
5691
5692         return if drive_is_cdrom($drive);
5693
5694         my $volid = $drive->{file};
5695         my $device = "drive-$ds";
5696
5697         PVE::Storage::volume_snapshot_rollback($storecfg, $volid, $snapname);
5698     });
5699
5700     $prepare = 0;
5701     lock_config($vmid, $updatefn);
5702 }
5703
5704 my $savevm_wait = sub {
5705     my ($vmid) = @_;
5706
5707     for(;;) {
5708         my $stat = vm_mon_cmd_nocheck($vmid, "query-savevm");
5709         if (!$stat->{status}) {
5710             die "savevm not active\n";
5711         } elsif ($stat->{status} eq 'active') {
5712             sleep(1);
5713             next;
5714         } elsif ($stat->{status} eq 'completed') {
5715             last;
5716         } else {
5717             die "query-savevm returned status '$stat->{status}'\n";
5718         }
5719     }
5720 };
5721
5722 sub do_snapshots_with_qemu {
5723     my ($storecfg, $volid) = @_;
5724
5725     my $storage_name = PVE::Storage::parse_volume_id($volid);
5726
5727     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}} ){
5728         return 1;
5729     }
5730
5731     if ($volid =~ m/\.(qcow2|qed)$/){
5732         return 1;
5733     }
5734
5735     return undef;
5736 }
5737
5738 sub snapshot_create {
5739     my ($vmid, $snapname, $save_vmstate, $comment) = @_;
5740
5741     my $snap = &$snapshot_prepare($vmid, $snapname, $save_vmstate, $comment);
5742
5743     $save_vmstate = 0 if !$snap->{vmstate}; # vm is not running
5744
5745     my $config = load_config($vmid);
5746
5747     my $running = check_running($vmid);
5748
5749     my $freezefs = $running && $config->{agent};
5750     $freezefs = 0 if $snap->{vmstate}; # not needed if we save RAM
5751
5752     my $drivehash = {};
5753
5754     if ($freezefs) {
5755         eval { vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
5756         warn "guest-fsfreeze-freeze problems - $@" if $@;
5757     }
5758
5759     eval {
5760         # create internal snapshots of all drives
5761
5762         my $storecfg = PVE::Storage::config();
5763
5764         if ($running) {
5765             if ($snap->{vmstate}) {
5766                 my $path = PVE::Storage::path($storecfg, $snap->{vmstate});
5767                 vm_mon_cmd($vmid, "savevm-start", statefile => $path);
5768                 &$savevm_wait($vmid);
5769             } else {
5770                 vm_mon_cmd($vmid, "savevm-start");
5771             }
5772         };
5773
5774         foreach_drive($snap, sub {
5775             my ($ds, $drive) = @_;
5776
5777             return if drive_is_cdrom($drive);
5778
5779             my $volid = $drive->{file};
5780             my $device = "drive-$ds";
5781
5782             qemu_volume_snapshot($vmid, $device, $storecfg, $volid, $snapname);
5783             $drivehash->{$ds} = 1;
5784        });
5785     };
5786     my $err = $@;
5787
5788     if ($running) {
5789         eval { vm_mon_cmd($vmid, "savevm-end")  };
5790         warn $@ if $@;
5791
5792         if ($freezefs) {
5793             eval { vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
5794             warn "guest-fsfreeze-thaw problems - $@" if $@;
5795         }
5796
5797         # savevm-end is async, we need to wait
5798         for (;;) {
5799             my $stat = vm_mon_cmd_nocheck($vmid, "query-savevm");
5800             if (!$stat->{bytes}) {
5801                 last;
5802             } else {
5803                 print "savevm not yet finished\n";
5804                 sleep(1);
5805                 next;
5806             }
5807         }
5808     }
5809
5810     if ($err) {
5811         warn "snapshot create failed: starting cleanup\n";
5812         eval { snapshot_delete($vmid, $snapname, 0, $drivehash); };
5813         warn $@ if $@;
5814         die $err;
5815     }
5816
5817     &$snapshot_commit($vmid, $snapname);
5818 }
5819
5820 # Note: $drivehash is only set when called from snapshot_create.
5821 sub snapshot_delete {
5822     my ($vmid, $snapname, $force, $drivehash) = @_;
5823
5824     my $prepare = 1;
5825
5826     my $snap;
5827     my $unused = [];
5828
5829     my $unlink_parent = sub {
5830         my ($confref, $new_parent) = @_;
5831
5832         if ($confref->{parent} && $confref->{parent} eq $snapname) {
5833             if ($new_parent) {
5834                 $confref->{parent} = $new_parent;
5835             } else {
5836                 delete $confref->{parent};
5837             }
5838         }
5839     };
5840
5841     my $updatefn =  sub {
5842         my ($remove_drive) = @_;
5843
5844         my $conf = load_config($vmid);
5845
5846         if (!$drivehash) {
5847             check_lock($conf);
5848             die "you can't delete a snapshot if vm is a template\n"
5849                 if is_template($conf);
5850         }
5851
5852         $snap = $conf->{snapshots}->{$snapname};
5853
5854         die "snapshot '$snapname' does not exist\n" if !defined($snap);
5855
5856         # remove parent refs
5857         if (!$prepare) {
5858             &$unlink_parent($conf, $snap->{parent});
5859             foreach my $sn (keys %{$conf->{snapshots}}) {
5860                 next if $sn eq $snapname;
5861                 &$unlink_parent($conf->{snapshots}->{$sn}, $snap->{parent});
5862             }
5863         }
5864
5865         if ($remove_drive) {
5866             if ($remove_drive eq 'vmstate') {
5867                 delete $snap->{$remove_drive};
5868             } else {
5869                 my $drive = parse_drive($remove_drive, $snap->{$remove_drive});
5870                 my $volid = $drive->{file};
5871                 delete $snap->{$remove_drive};
5872                 add_unused_volume($conf, $volid);
5873             }
5874         }
5875
5876         if ($prepare) {
5877             $snap->{snapstate} = 'delete';
5878         } else {
5879             delete $conf->{snapshots}->{$snapname};
5880             delete $conf->{lock} if $drivehash;
5881             foreach my $volid (@$unused) {
5882                 add_unused_volume($conf, $volid);
5883             }
5884         }
5885
5886         update_config_nolock($vmid, $conf, 1);
5887     };
5888
5889     lock_config($vmid, $updatefn);
5890
5891     # now remove vmstate file
5892
5893     my $storecfg = PVE::Storage::config();
5894
5895     if ($snap->{vmstate}) {
5896         eval {  PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
5897         if (my $err = $@) {
5898             die $err if !$force;
5899             warn $err;
5900         }
5901         # save changes (remove vmstate from snapshot)
5902         lock_config($vmid, $updatefn, 'vmstate') if !$force;
5903     };
5904
5905     # now remove all internal snapshots
5906     foreach_drive($snap, sub {
5907         my ($ds, $drive) = @_;
5908
5909         return if drive_is_cdrom($drive);
5910
5911         my $volid = $drive->{file};
5912         my $device = "drive-$ds";
5913
5914         if (!$drivehash || $drivehash->{$ds}) {
5915             eval { qemu_volume_snapshot_delete($vmid, $device, $storecfg, $volid, $snapname); };
5916             if (my $err = $@) {
5917                 die $err if !$force;
5918                 warn $err;
5919             }
5920         }
5921
5922         # save changes (remove drive fron snapshot)
5923         lock_config($vmid, $updatefn, $ds) if !$force;
5924         push @$unused, $volid;
5925     });
5926
5927     # now cleanup config
5928     $prepare = 0;
5929     lock_config($vmid, $updatefn);
5930 }
5931
5932 sub has_feature {
5933     my ($feature, $conf, $storecfg, $snapname, $running) = @_;
5934
5935     my $err;
5936     foreach_drive($conf, sub {
5937         my ($ds, $drive) = @_;
5938
5939         return if drive_is_cdrom($drive);
5940         my $volid = $drive->{file};
5941         $err = 1 if !PVE::Storage::volume_has_feature($storecfg, $feature, $volid, $snapname, $running);
5942     });
5943
5944     return $err ? 0 : 1;
5945 }
5946
5947 sub template_create {
5948     my ($vmid, $conf, $disk) = @_;
5949
5950     my $storecfg = PVE::Storage::config();
5951
5952     foreach_drive($conf, sub {
5953         my ($ds, $drive) = @_;
5954
5955         return if drive_is_cdrom($drive);
5956         return if $disk && $ds ne $disk;
5957
5958         my $volid = $drive->{file};
5959         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
5960
5961         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
5962         $drive->{file} = $voliddst;
5963         $conf->{$ds} = print_drive($vmid, $drive);
5964         update_config_nolock($vmid, $conf, 1);
5965     });
5966 }
5967
5968 sub is_template {
5969     my ($conf) = @_;
5970
5971     return 1 if defined $conf->{template} && $conf->{template} == 1;
5972 }
5973
5974 sub qemu_img_convert {
5975     my ($src_volid, $dst_volid, $size, $snapname) = @_;
5976
5977     my $storecfg = PVE::Storage::config();
5978     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
5979     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
5980
5981     if ($src_storeid && $dst_storeid) {
5982         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
5983         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5984
5985         my $src_format = qemu_img_format($src_scfg, $src_volname);
5986         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
5987
5988         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
5989         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5990
5991         my $cmd = [];
5992         push @$cmd, '/usr/bin/qemu-img', 'convert', '-t', 'writeback', '-p', '-n';
5993         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
5994         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path, $dst_path;
5995
5996         my $parser = sub {
5997             my $line = shift;
5998             if($line =~ m/\((\S+)\/100\%\)/){
5999                 my $percent = $1;
6000                 my $transferred = int($size * $percent / 100);
6001                 my $remaining = $size - $transferred;
6002
6003                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6004             }
6005
6006         };
6007
6008         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6009         my $err = $@;
6010         die "copy failed: $err" if $err;
6011     }
6012 }
6013
6014 sub qemu_img_format {
6015     my ($scfg, $volname) = @_;
6016
6017     if ($scfg->{path} && $volname =~ m/\.(raw|qcow2|qed|vmdk)$/) {
6018         return $1;
6019     } elsif ($scfg->{type} eq 'iscsi') {
6020         return "host_device";
6021     } else {
6022         return "raw";
6023     }
6024 }
6025
6026 sub qemu_drive_mirror {
6027     my ($vmid, $drive, $dst_volid, $vmiddst) = @_;
6028
6029     my $storecfg = PVE::Storage::config();
6030     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6031
6032     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6033
6034     my $format;
6035     if ($dst_volname =~ m/\.(raw|qcow2)$/){
6036         $format = $1;
6037     }
6038
6039     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6040
6041     #drive-mirror is doing lseek on source image before starting, and this can take a lot of time for big nfs volume
6042     #during this time, qmp socket is hanging
6043     #http://lists.nongnu.org/archive/html/qemu-devel/2015-05/msg01838.html
6044     #so we need to setup a big timeout
6045     my $opts = { timeout => 14400, device => "drive-$drive", mode => "existing", sync => "full", target => $dst_path };
6046     $opts->{format} = $format if $format;
6047
6048     print "drive mirror is starting : this step can take some minutes/hours, depend of disk size and storage speed\n";
6049
6050     vm_mon_cmd($vmid, "drive-mirror", %$opts);
6051     eval {
6052         while (1) {
6053             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6054             my $stat = @$stats[0];
6055             die "mirroring job seem to have die. Maybe do you have bad sectors?" if !$stat;
6056             die "error job is not mirroring" if $stat->{type} ne "mirror";
6057
6058             my $busy = $stat->{busy};
6059             my $ready = $stat->{ready};
6060
6061             if (my $total = $stat->{len}) {
6062                 my $transferred = $stat->{offset} || 0;
6063                 my $remaining = $total - $transferred;
6064                 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6065
6066                 print "transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6067             }
6068
6069
6070             if ($stat->{ready} eq 'true') {
6071
6072                 last if $vmiddst != $vmid;
6073
6074                 # try to switch the disk if source and destination are on the same guest
6075                 eval { vm_mon_cmd($vmid, "block-job-complete", device => "drive-$drive") };
6076                 last if !$@;
6077                 die $@ if $@ !~ m/cannot be completed/;
6078             }
6079             sleep 1;
6080         }
6081
6082
6083     };
6084     my $err = $@;
6085
6086     my $cancel_job = sub {
6087         vm_mon_cmd($vmid, "block-job-cancel", device => "drive-$drive");
6088         while (1) {
6089             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6090             my $stat = @$stats[0];
6091             last if !$stat;
6092             sleep 1;
6093         }
6094     };
6095
6096     if ($err) {
6097         eval { &$cancel_job(); };
6098         die "mirroring error: $err";
6099     }
6100
6101     if ($vmiddst != $vmid) {
6102         # if we clone a disk for a new target vm, we don't switch the disk
6103         &$cancel_job(); # so we call block-job-cancel
6104     }
6105 }
6106
6107 sub clone_disk {
6108     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6109         $newvmid, $storage, $format, $full, $newvollist) = @_;
6110
6111     my $newvolid;
6112
6113     if (!$full) {
6114         print "create linked clone of drive $drivename ($drive->{file})\n";
6115         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6116         push @$newvollist, $newvolid;
6117     } else {
6118         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6119         $storeid = $storage if $storage;
6120
6121         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6122         if (!$format) {
6123             $format = $drive->{format} || $defFormat;
6124         }
6125
6126         # test if requested format is supported - else use default
6127         my $supported = grep { $_ eq $format } @$validFormats;
6128         $format = $defFormat if !$supported;
6129
6130         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6131
6132         print "create full clone of drive $drivename ($drive->{file})\n";
6133         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $format, undef, ($size/1024));
6134         push @$newvollist, $newvolid;
6135
6136         if (!$running || $snapname) {
6137             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname);
6138         } else {
6139             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid);
6140         }
6141     }
6142
6143     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6144
6145     my $disk = $drive;
6146     $disk->{format} = undef;
6147     $disk->{file} = $newvolid;
6148     $disk->{size} = $size;
6149
6150     return $disk;
6151 }
6152
6153 # this only works if VM is running
6154 sub get_current_qemu_machine {
6155     my ($vmid) = @_;
6156
6157     my $cmd = { execute => 'query-machines', arguments => {} };
6158     my $res = vm_qmp_command($vmid, $cmd);
6159
6160     my ($current, $default);
6161     foreach my $e (@$res) {
6162         $default = $e->{name} if $e->{'is-default'};
6163         $current = $e->{name} if $e->{'is-current'};
6164     }
6165
6166     # fallback to the default machine if current is not supported by qemu
6167     return $current || $default || 'pc';
6168 }
6169
6170 sub qemu_machine_feature_enabled {
6171     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6172
6173     my $current_major;
6174     my $current_minor;
6175
6176     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6177
6178         $current_major = $3;
6179         $current_minor = $4;
6180
6181     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6182
6183         $current_major = $1;
6184         $current_minor = $2;
6185     }
6186
6187     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
6188
6189
6190 }
6191
6192 sub lspci {
6193
6194     my $devices = {};
6195
6196     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6197             my (undef, $id, $function) = @_;
6198             my $res = { id => $id, function => $function};
6199             push @{$devices->{$id}}, $res;
6200     });
6201
6202     return $devices;
6203 }
6204
6205 sub vm_iothreads_list {
6206     my ($vmid) = @_;
6207
6208     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6209
6210     my $iothreads = {};
6211     foreach my $iothread (@$res) {
6212         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6213     }
6214
6215     return $iothreads;
6216 }
6217
6218 sub scsihw_infos {
6219     my ($conf, $drive) = @_;
6220
6221     my $maxdev = 0;
6222
6223     if ($conf->{scsihw} && ($conf->{scsihw} =~ m/^lsi/)) {
6224         $maxdev = 7;
6225     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6226         $maxdev = 1;
6227     } else {
6228         $maxdev = 256;
6229     }
6230
6231     my $controller = int($drive->{index} / $maxdev);
6232     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6233
6234     return ($maxdev, $controller, $controller_prefix);
6235 }
6236
6237 1;