PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QMPClient;
  31 use PVE::RPCEnvironment;
  32 use Time::HiRes qw(gettimeofday);
  33
  34 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  35
  36 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  37
  38 # Note about locking: we use flock on the config file protect
  39 # against concurent actions.
  40 # Aditionaly, we have a 'lock' setting in the config file. This
  41 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  42 # allowed when such lock is set. But you can ignore this kind of
  43 # lock with the --skiplock flag.
  44
  45 cfs_register_file('/qemu-server/',
  46                   \&parse_vm_config,
  47                   \&write_vm_config);
  48
  49 PVE::JSONSchema::register_standard_option('skiplock', {
  50     description => "Ignore locks - only root is allowed to use this option.",
  51     type => 'boolean',
  52     optional => 1,
  53 });
  54
  55 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  56     description => "Some command save/restore state from this location.",
  57     type => 'string',
  58     maxLength => 128,
  59     optional => 1,
  60 });
  61
  62 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  63     description => "The name of the snapshot.",
  64     type => 'string', format => 'pve-configid',
  65     maxLength => 40,
  66 });
  67
  68 #no warnings 'redefine';
  69
  70 sub cgroups_write {
  71    my ($controller, $vmid, $option, $value) = @_;
  72
  73    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  74    PVE::ProcFSTools::write_proc_entry($path, $value);
  75
  76 }
  77
  78 my $nodename = PVE::INotify::nodename();
  79
  80 mkdir "/etc/pve/nodes/$nodename";
  81 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  82 mkdir $confdir;
  83
  84 my $var_run_tmpdir = "/var/run/qemu-server";
  85 mkdir $var_run_tmpdir;
  86
  87 my $lock_dir = "/var/lock/qemu-server";
  88 mkdir $lock_dir;
  89
  90 my $pcisysfs = "/sys/bus/pci";
  91
  92 my $confdesc = {
  93     onboot => {
  94         optional => 1,
  95         type => 'boolean',
  96         description => "Specifies whether a VM will be started during system bootup.",
  97         default => 0,
  98     },
  99     autostart => {
 100         optional => 1,
 101         type => 'boolean',
 102         description => "Automatic restart after crash (currently ignored).",
 103         default => 0,
 104     },
 105     hotplug => {
 106         optional => 1,
 107         type => 'string', format => 'pve-hotplug-features',
 108         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 109         default => 'network,disk,usb',
 110     },
 111     reboot => {
 112         optional => 1,
 113         type => 'boolean',
 114         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 115         default => 1,
 116     },
 117     lock => {
 118         optional => 1,
 119         type => 'string',
 120         description => "Lock/unlock the VM.",
 121         enum => [qw(migrate backup snapshot rollback)],
 122     },
 123     cpulimit => {
 124         optional => 1,
 125         type => 'number',
 126         description => "Limit of CPU usage. Note if the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 127         minimum => 0,
 128         maximum => 128,
 129         default => 0,
 130     },
 131     cpuunits => {
 132         optional => 1,
 133         type => 'integer',
 134         description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 135         minimum => 0,
 136         maximum => 500000,
 137         default => 1000,
 138     },
 139     memory => {
 140         optional => 1,
 141         type => 'integer',
 142         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 143         minimum => 16,
 144         default => 512,
 145     },
 146     balloon => {
 147         optional => 1,
 148         type => 'integer',
 149         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 150         minimum => 0,
 151     },
 152     shares => {
 153         optional => 1,
 154         type => 'integer',
 155         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 156         minimum => 0,
 157         maximum => 50000,
 158         default => 1000,
 159     },
 160     keyboard => {
 161         optional => 1,
 162         type => 'string',
 163         description => "Keybord layout for vnc server. Default is read from the datacenter configuration file.",
 164         enum => PVE::Tools::kvmkeymaplist(),
 165         default => 'en-us',
 166     },
 167     name => {
 168         optional => 1,
 169         type => 'string', format => 'dns-name',
 170         description => "Set a name for the VM. Only used on the configuration web interface.",
 171     },
 172     scsihw => {
 173         optional => 1,
 174         type => 'string',
 175         description => "scsi controller model",
 176         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 177         default => 'lsi',
 178     },
 179     description => {
 180         optional => 1,
 181         type => 'string',
 182         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 183     },
 184     ostype => {
 185         optional => 1,
 186         type => 'string',
 187         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 l24 l26 solaris)],
 188         description => <<EODESC,
 189 Used to enable special optimization/features for specific
 190 operating systems:
 191
 192 other  => unspecified OS
 193 wxp    => Microsoft Windows XP
 194 w2k    => Microsoft Windows 2000
 195 w2k3   => Microsoft Windows 2003
 196 w2k8   => Microsoft Windows 2008
 197 wvista => Microsoft Windows Vista
 198 win7   => Microsoft Windows 7
 199 win8   => Microsoft Windows 8/2012
 200 l24    => Linux 2.4 Kernel
 201 l26    => Linux 2.6/3.X Kernel
 202 solaris => solaris/opensolaris/openindiania kernel
 203
 204 other|l24|l26|solaris                       ... no special behaviour
 205 wxp|w2k|w2k3|w2k8|wvista|win7|win8  ... use --localtime switch
 206 EODESC
 207     },
 208     boot => {
 209         optional => 1,
 210         type => 'string',
 211         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 212         pattern => '[acdn]{1,4}',
 213         default => 'cdn',
 214     },
 215     bootdisk => {
 216         optional => 1,
 217         type => 'string', format => 'pve-qm-bootdisk',
 218         description => "Enable booting from specified disk.",
 219         pattern => '(ide|sata|scsi|virtio)\d+',
 220     },
 221     smp => {
 222         optional => 1,
 223         type => 'integer',
 224         description => "The number of CPUs. Please use option -sockets instead.",
 225         minimum => 1,
 226         default => 1,
 227     },
 228     sockets => {
 229         optional => 1,
 230         type => 'integer',
 231         description => "The number of CPU sockets.",
 232         minimum => 1,
 233         default => 1,
 234     },
 235     cores => {
 236         optional => 1,
 237         type => 'integer',
 238         description => "The number of cores per socket.",
 239         minimum => 1,
 240         default => 1,
 241     },
 242     numa => {
 243         optional => 1,
 244         type => 'boolean',
 245         description => "Enable/disable Numa.",
 246         default => 0,
 247     },
 248     vcpus => {
 249         optional => 1,
 250         type => 'integer',
 251         description => "Number of hotplugged vcpus.",
 252         minimum => 1,
 253         default => 0,
 254     },
 255     acpi => {
 256         optional => 1,
 257         type => 'boolean',
 258         description => "Enable/disable ACPI.",
 259         default => 1,
 260     },
 261     agent => {
 262         optional => 1,
 263         type => 'boolean',
 264         description => "Enable/disable Qemu GuestAgent.",
 265         default => 0,
 266     },
 267     kvm => {
 268         optional => 1,
 269         type => 'boolean',
 270         description => "Enable/disable KVM hardware virtualization.",
 271         default => 1,
 272     },
 273     tdf => {
 274         optional => 1,
 275         type => 'boolean',
 276         description => "Enable/disable time drift fix.",
 277         default => 0,
 278     },
 279     localtime => {
 280         optional => 1,
 281         type => 'boolean',
 282         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 283     },
 284     freeze => {
 285         optional => 1,
 286         type => 'boolean',
 287         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 288     },
 289     vga => {
 290         optional => 1,
 291         type => 'string',
 292         description => "Select VGA type. If you want to use high resolution modes (>= 1280x1024x16) then you should use option 'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and 'cirrur' for other OS types. Option 'qxl' enables the SPICE display sever. You can also run without any graphic card using a serial devive as terminal.",
 293         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 294     },
 295     watchdog => {
 296         optional => 1,
 297         type => 'string', format => 'pve-qm-watchdog',
 298         typetext => '[[model=]i6300esb|ib700] [,[action=]reset|shutdown|poweroff|pause|debug|none]',
 299         description => "Create a virtual hardware watchdog device.  Once enabled (by a guest action), the watchdog must be periodically polled by an agent inside the guest or else the guest will be restarted (or execute the action specified)",
 300     },
 301     startdate => {
 302         optional => 1,
 303         type => 'string',
 304         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 305         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 306         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 307         default => 'now',
 308     },
 309     startup =>  get_standard_option('pve-startup-order'),
 310     template => {
 311         optional => 1,
 312         type => 'boolean',
 313         description => "Enable/disable Template.",
 314         default => 0,
 315     },
 316     args => {
 317         optional => 1,
 318         type => 'string',
 319         description => <<EODESCR,
 320 Note: this option is for experts only. It allows you to pass arbitrary arguments to kvm, for example:
 321
 322 args: -no-reboot -no-hpet
 323 EODESCR
 324     },
 325     tablet => {
 326         optional => 1,
 327         type => 'boolean',
 328         default => 1,
 329         description => "Enable/disable the usb tablet device. This device is usually needed to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with normal VNC clients. If you're running lots of console-only guests on one host, you may consider disabling this to save some context switches. This is turned of by default if you use spice (vga=qxl).",
 330     },
 331     migrate_speed => {
 332         optional => 1,
 333         type => 'integer',
 334         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 335         minimum => 0,
 336         default => 0,
 337     },
 338     migrate_downtime => {
 339         optional => 1,
 340         type => 'number',
 341         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 342         minimum => 0,
 343         default => 0.1,
 344     },
 345     cdrom => {
 346         optional => 1,
 347         type => 'string', format => 'pve-qm-drive',
 348         typetext => 'volume',
 349         description => "This is an alias for option -ide2",
 350     },
 351     cpu => {
 352         optional => 1,
 353         description => "Emulated CPU type.",
 354         type => 'string',
 355         enum => [ qw(486 athlon pentium pentium2 pentium3 coreduo core2duo kvm32 kvm64 qemu32 qemu64 phenom Conroe Penryn Nehalem Westmere SandyBridge IvyBridge Haswell Broadwell Opteron_G1 Opteron_G2 Opteron_G3 Opteron_G4 Opteron_G5 host) ],
 356         default => 'kvm64',
 357     },
 358     parent => get_standard_option('pve-snapshot-name', {
 359         optional => 1,
 360         description => "Parent snapshot name. This is used internally, and should not be modified.",
 361     }),
 362     snaptime => {
 363         optional => 1,
 364         description => "Timestamp for snapshots.",
 365         type => 'integer',
 366         minimum => 0,
 367     },
 368     vmstate => {
 369         optional => 1,
 370         type => 'string', format => 'pve-volume-id',
 371         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 372     },
 373     machine => {
 374         description => "Specific the Qemu machine type.",
 375         type => 'string',
 376         pattern => '(pc|pc(-i440fx)?-\d+\.\d+|q35|pc-q35-\d+\.\d+)',
 377         maxLength => 40,
 378         optional => 1,
 379     },
 380     smbios1 => {
 381         description => "Specify SMBIOS type 1 fields.",
 382         type => 'string', format => 'pve-qm-smbios1',
 383         typetext => "[manufacturer=str][,product=str][,version=str][,serial=str] [,uuid=uuid][,sku=str][,family=str]",
 384         maxLength => 256,
 385         optional => 1,
 386     },
 387 };
 388
 389 # what about other qemu settings ?
 390 #cpu => 'string',
 391 #machine => 'string',
 392 #fda => 'file',
 393 #fdb => 'file',
 394 #mtdblock => 'file',
 395 #sd => 'file',
 396 #pflash => 'file',
 397 #snapshot => 'bool',
 398 #bootp => 'file',
 399 ##tftp => 'dir',
 400 ##smb => 'dir',
 401 #kernel => 'file',
 402 #append => 'string',
 403 #initrd => 'file',
 404 ##soundhw => 'string',
 405
 406 while (my ($k, $v) = each %$confdesc) {
 407     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 408 }
 409
 410 my $MAX_IDE_DISKS = 4;
 411 my $MAX_SCSI_DISKS = 14;
 412 my $MAX_VIRTIO_DISKS = 16;
 413 my $MAX_SATA_DISKS = 6;
 414 my $MAX_USB_DEVICES = 5;
 415 my $MAX_NETS = 32;
 416 my $MAX_UNUSED_DISKS = 8;
 417 my $MAX_HOSTPCI_DEVICES = 4;
 418 my $MAX_SERIAL_PORTS = 4;
 419 my $MAX_PARALLEL_PORTS = 3;
 420 my $MAX_NUMA = 8;
 421 my $MAX_MEM = 4194304;
 422 my $STATICMEM = 1024;
 423
 424 my $numadesc = {
 425     optional => 1,
 426     type => 'string', format => 'pve-qm-numanode',
 427     typetext => "cpus=<id[-id],memory=<mb>[[,hostnodes=<id[-id]>] [,policy=<preferred|bind|interleave>]]",
 428     description => "numa topology",
 429 };
 430 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 431
 432 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 433     $confdesc->{"numa$i"} = $numadesc;
 434 }
 435
 436 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 437                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 438                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 439 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 440
 441 my $netdesc = {
 442     optional => 1,
 443     type => 'string', format => 'pve-qm-net',
 444     typetext => "MODEL=XX:XX:XX:XX:XX:XX [,bridge=<dev>][,queues=<nbqueues>][,rate=<mbps>] [,tag=<vlanid>][,firewall=0|1],link_down=0|1]",
 445     description => <<EODESCR,
 446 Specify network devices.
 447
 448 MODEL is one of: $nic_model_list_txt
 449
 450 XX:XX:XX:XX:XX:XX should be an unique MAC address. This is
 451 automatically generated if not specified.
 452
 453 The bridge parameter can be used to automatically add the interface to a bridge device. The Proxmox VE standard bridge is called 'vmbr0'.
 454
 455 Option 'rate' is used to limit traffic bandwidth from and to this interface. It is specified as floating point number, unit is 'Megabytes per second'.
 456
 457 If you specify no bridge, we create a kvm 'user' (NATed) network device, which provides DHCP and DNS services. The following addresses are used:
 458
 459 10.0.2.2   Gateway
 460 10.0.2.3   DNS Server
 461 10.0.2.4   SMB Server
 462
 463 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 464
 465 EODESCR
 466 };
 467 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 468
 469 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 470     $confdesc->{"net$i"} = $netdesc;
 471 }
 472
 473 my $drivename_hash;
 474
 475 my $idedesc = {
 476     optional => 1,
 477     type => 'string', format => 'pve-qm-drive',
 478     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads] [,discard=ignore|on]',
 479     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
 480 };
 481 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
 482
 483 my $scsidesc = {
 484     optional => 1,
 485     type => 'string', format => 'pve-qm-drive',
 486     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads] [,discard=ignore|on] [,iothread=on] [,queues=<nbqueues>]',
 487     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
 488 };
 489 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 490
 491 my $satadesc = {
 492     optional => 1,
 493     type => 'string', format => 'pve-qm-drive',
 494     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads]  [,discard=ignore|on]',
 495     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
 496 };
 497 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
 498
 499 my $virtiodesc = {
 500     optional => 1,
 501     type => 'string', format => 'pve-qm-drive',
 502     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads]  [,discard=ignore|on] [,iothread=on]',
 503     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
 504 };
 505 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
 506
 507 my $usbdesc = {
 508     optional => 1,
 509     type => 'string', format => 'pve-qm-usb-device',
 510     typetext => 'host=HOSTUSBDEVICE|spice',
 511     description => <<EODESCR,
 512 Configure an USB device (n is 0 to 4). This can be used to
 513 pass-through usb devices to the guest. HOSTUSBDEVICE syntax is:
 514
 515 'bus-port(.port)*' (decimal numbers) or
 516 'vendor_id:product_id' (hexadeciaml numbers)
 517
 518 You can use the 'lsusb -t' command to list existing usb devices.
 519
 520 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 521
 522 The value 'spice' can be used to add a usb redirection devices for spice.
 523
 524 EODESCR
 525 };
 526 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
 527
 528 my $hostpcidesc = {
 529         optional => 1,
 530         type => 'string', format => 'pve-qm-hostpci',
 531         typetext => "[host=]HOSTPCIDEVICE [,driver=kvm|vfio] [,rombar=on|off] [,pcie=0|1] [,x-vga=on|off]",
 532         description => <<EODESCR,
 533 Map host pci devices. HOSTPCIDEVICE syntax is:
 534
 535 'bus:dev.func' (hexadecimal numbers)
 536
 537 You can us the 'lspci' command to list existing pci devices.
 538
 539 The 'rombar' option determines whether or not the device's ROM will be visible in the guest's memory map (default is 'on').
 540
 541 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 542
 543 Experimental: user reported problems with this option.
 544 EODESCR
 545 };
 546 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
 547
 548 my $serialdesc = {
 549         optional => 1,
 550         type => 'string',
 551         pattern => '(/dev/.+|socket)',
 552         description =>  <<EODESCR,
 553 Create a serial device inside the VM (n is 0 to 3), and pass through a host serial device (i.e. /dev/ttyS0), or create a unix socket on the host side (use 'qm terminal' to open a terminal connection).
 554
 555 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 556
 557 Experimental: user reported problems with this option.
 558 EODESCR
 559 };
 560
 561 my $paralleldesc= {
 562         optional => 1,
 563         type => 'string',
 564         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
 565         description =>  <<EODESCR,
 566 Map host parallel devices (n is 0 to 2).
 567
 568 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 569
 570 Experimental: user reported problems with this option.
 571 EODESCR
 572 };
 573
 574 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
 575     $confdesc->{"parallel$i"} = $paralleldesc;
 576 }
 577
 578 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
 579     $confdesc->{"serial$i"} = $serialdesc;
 580 }
 581
 582 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
 583     $confdesc->{"hostpci$i"} = $hostpcidesc;
 584 }
 585
 586 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
 587     $drivename_hash->{"ide$i"} = 1;
 588     $confdesc->{"ide$i"} = $idedesc;
 589 }
 590
 591 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
 592     $drivename_hash->{"sata$i"} = 1;
 593     $confdesc->{"sata$i"} = $satadesc;
 594 }
 595
 596 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
 597     $drivename_hash->{"scsi$i"} = 1;
 598     $confdesc->{"scsi$i"} = $scsidesc ;
 599 }
 600
 601 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
 602     $drivename_hash->{"virtio$i"} = 1;
 603     $confdesc->{"virtio$i"} = $virtiodesc;
 604 }
 605
 606 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
 607     $confdesc->{"usb$i"} = $usbdesc;
 608 }
 609
 610 my $unuseddesc = {
 611     optional => 1,
 612     type => 'string', format => 'pve-volume-id',
 613     description => "Reference to unused volumes.",
 614 };
 615
 616 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
 617     $confdesc->{"unused$i"} = $unuseddesc;
 618 }
 619
 620 my $kvm_api_version = 0;
 621
 622 sub kvm_version {
 623
 624     return $kvm_api_version if $kvm_api_version;
 625
 626     my $fh = IO::File->new("</dev/kvm") ||
 627         return 0;
 628
 629     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
 630         $kvm_api_version = $v;
 631     }
 632
 633     $fh->close();
 634
 635     return  $kvm_api_version;
 636 }
 637
 638 my $kvm_user_version;
 639
 640 sub kvm_user_version {
 641
 642     return $kvm_user_version if $kvm_user_version;
 643
 644     $kvm_user_version = 'unknown';
 645
 646     my $tmp = `kvm -help 2>/dev/null`;
 647
 648     if ($tmp =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)[,\s]/) {
 649         $kvm_user_version = $2;
 650     }
 651
 652     return $kvm_user_version;
 653
 654 }
 655
 656 my $kernel_has_vhost_net = -c '/dev/vhost-net';
 657
 658 sub disknames {
 659     # order is important - used to autoselect boot disk
 660     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
 661             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
 662             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
 663             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))));
 664 }
 665
 666 sub valid_drivename {
 667     my $dev = shift;
 668
 669     return defined($drivename_hash->{$dev});
 670 }
 671
 672 sub option_exists {
 673     my $key = shift;
 674     return defined($confdesc->{$key});
 675 }
 676
 677 sub nic_models {
 678     return $nic_model_list;
 679 }
 680
 681 sub os_list_description {
 682
 683     return {
 684         other => 'Other',
 685         wxp => 'Windows XP',
 686         w2k => 'Windows 2000',
 687         w2k3 =>, 'Windows 2003',
 688         w2k8 => 'Windows 2008',
 689         wvista => 'Windows Vista',
 690         win7 => 'Windows 7',
 691         win8 => 'Windows 8/2012',
 692         l24 => 'Linux 2.4',
 693         l26 => 'Linux 2.6',
 694     };
 695 }
 696
 697 my $cdrom_path;
 698
 699 sub get_cdrom_path {
 700
 701     return  $cdrom_path if $cdrom_path;
 702
 703     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
 704     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
 705     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
 706 }
 707
 708 sub get_iso_path {
 709     my ($storecfg, $vmid, $cdrom) = @_;
 710
 711     if ($cdrom eq 'cdrom') {
 712         return get_cdrom_path();
 713     } elsif ($cdrom eq 'none') {
 714         return '';
 715     } elsif ($cdrom =~ m|^/|) {
 716         return $cdrom;
 717     } else {
 718         return PVE::Storage::path($storecfg, $cdrom);
 719     }
 720 }
 721
 722 # try to convert old style file names to volume IDs
 723 sub filename_to_volume_id {
 724     my ($vmid, $file, $media) = @_;
 725
 726     if (!($file eq 'none' || $file eq 'cdrom' ||
 727           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
 728
 729         return undef if $file =~ m|/|;
 730
 731         if ($media && $media eq 'cdrom') {
 732             $file = "local:iso/$file";
 733         } else {
 734             $file = "local:$vmid/$file";
 735         }
 736     }
 737
 738     return $file;
 739 }
 740
 741 sub verify_media_type {
 742     my ($opt, $vtype, $media) = @_;
 743
 744     return if !$media;
 745
 746     my $etype;
 747     if ($media eq 'disk') {
 748         $etype = 'images';
 749     } elsif ($media eq 'cdrom') {
 750         $etype = 'iso';
 751     } else {
 752         die "internal error";
 753     }
 754
 755     return if ($vtype eq $etype);
 756
 757     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
 758 }
 759
 760 sub cleanup_drive_path {
 761     my ($opt, $storecfg, $drive) = @_;
 762
 763     # try to convert filesystem paths to volume IDs
 764
 765     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
 766         ($drive->{file} !~ m|^/dev/.+|) &&
 767         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
 768         ($drive->{file} !~ m/^\d+$/)) {
 769         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
 770         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
 771         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
 772         verify_media_type($opt, $vtype, $drive->{media});
 773         $drive->{file} = $volid;
 774     }
 775
 776     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
 777 }
 778
 779 sub create_conf_nolock {
 780     my ($vmid, $settings) = @_;
 781
 782     my $filename = config_file($vmid);
 783
 784     die "configuration file '$filename' already exists\n" if -f $filename;
 785
 786     my $defaults = load_defaults();
 787
 788     $settings->{name} = "vm$vmid" if !$settings->{name};
 789     $settings->{memory} = $defaults->{memory} if !$settings->{memory};
 790
 791     my $data = '';
 792     foreach my $opt (keys %$settings) {
 793         next if !$confdesc->{$opt};
 794
 795         my $value = $settings->{$opt};
 796         next if !$value;
 797
 798         $data .= "$opt: $value\n";
 799     }
 800
 801     PVE::Tools::file_set_contents($filename, $data);
 802 }
 803
 804 sub parse_hotplug_features {
 805     my ($data) = @_;
 806
 807     my $res = {};
 808
 809     return $res if $data eq '0';
 810
 811     $data = $confdesc->{hotplug}->{default} if $data eq '1';
 812
 813     foreach my $feature (PVE::Tools::split_list($data)) {
 814         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
 815             $res->{$1} = 1;
 816         } else {
 817             warn "ignoring unknown hotplug feature '$feature'\n";
 818         }
 819     }
 820     return $res;
 821 }
 822
 823 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
 824 sub pve_verify_hotplug_features {
 825     my ($value, $noerr) = @_;
 826
 827     return $value if parse_hotplug_features($value);
 828
 829     return undef if $noerr;
 830
 831     die "unable to parse hotplug option\n";
 832 }
 833
 834 my $parse_size = sub {
 835     my ($value) = @_;
 836
 837     return undef if $value !~ m/^(\d+(\.\d+)?)([KMG])?$/;
 838     my ($size, $unit) = ($1, $3);
 839     if ($unit) {
 840         if ($unit eq 'K') {
 841             $size = $size * 1024;
 842         } elsif ($unit eq 'M') {
 843             $size = $size * 1024 * 1024;
 844         } elsif ($unit eq 'G') {
 845             $size = $size * 1024 * 1024 * 1024;
 846         }
 847     }
 848     return int($size);
 849 };
 850
 851 my $format_size = sub {
 852     my ($size) = @_;
 853
 854     $size = int($size);
 855
 856     my $kb = int($size/1024);
 857     return $size if $kb*1024 != $size;
 858
 859     my $mb = int($kb/1024);
 860     return "${kb}K" if $mb*1024 != $kb;
 861
 862     my $gb = int($mb/1024);
 863     return "${mb}M" if $gb*1024 != $mb;
 864
 865     return "${gb}G";
 866 };
 867
 868 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
 869 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
 870 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
 871 #        [,aio=native|threads][,discard=ignore|on][,iothread=on]
 872
 873 sub parse_drive {
 874     my ($key, $data) = @_;
 875
 876     my $res = {};
 877
 878     # $key may be undefined - used to verify JSON parameters
 879     if (!defined($key)) {
 880         $res->{interface} = 'unknown'; # should not harm when used to verify parameters
 881         $res->{index} = 0;
 882     } elsif ($key =~ m/^([^\d]+)(\d+)$/) {
 883         $res->{interface} = $1;
 884         $res->{index} = $2;
 885     } else {
 886         return undef;
 887     }
 888
 889     foreach my $p (split (/,/, $data)) {
 890         next if $p =~ m/^\s*$/;
 891
 892         if ($p =~ m/^(file|volume|cyls|heads|secs|trans|media|snapshot|cache|format|rerror|werror|backup|aio|bps|mbps|mbps_max|bps_rd|mbps_rd|mbps_rd_max|bps_wr|mbps_wr|mbps_wr_max|iops|iops_max|iops_rd|iops_rd_max|iops_wr|iops_wr_max|size|discard|iothread|queues)=(.+)$/) {
 893             my ($k, $v) = ($1, $2);
 894
 895             $k = 'file' if $k eq 'volume';
 896
 897             return undef if defined $res->{$k};
 898
 899             if ($k eq 'bps' || $k eq 'bps_rd' || $k eq 'bps_wr') {
 900                 return undef if !$v || $v !~ m/^\d+/;
 901                 $k = "m$k";
 902                 $v = sprintf("%.3f", $v / (1024*1024));
 903             }
 904             $res->{$k} = $v;
 905         } else {
 906             if (!$res->{file} && $p !~ m/=/) {
 907                 $res->{file} = $p;
 908             } else {
 909                 return undef;
 910             }
 911         }
 912     }
 913
 914     return undef if !$res->{file};
 915
 916     if($res->{file} =~ m/\.(raw|cow|qcow|qcow2|vmdk|cloop)$/){
 917         $res->{format} = $1;
 918     }
 919
 920     return undef if $res->{cache} &&
 921         $res->{cache} !~ m/^(off|none|writethrough|writeback|unsafe|directsync)$/;
 922     return undef if $res->{snapshot} && $res->{snapshot} !~ m/^(on|off)$/;
 923     return undef if $res->{cyls} && $res->{cyls} !~ m/^\d+$/;
 924     return undef if $res->{heads} && $res->{heads} !~ m/^\d+$/;
 925     return undef if $res->{secs} && $res->{secs} !~ m/^\d+$/;
 926     return undef if $res->{media} && $res->{media} !~ m/^(disk|cdrom)$/;
 927     return undef if $res->{trans} && $res->{trans} !~ m/^(none|lba|auto)$/;
 928     return undef if $res->{format} && $res->{format} !~ m/^(raw|cow|qcow|qcow2|vmdk|cloop)$/;
 929     return undef if $res->{rerror} && $res->{rerror} !~ m/^(ignore|report|stop)$/;
 930     return undef if $res->{werror} && $res->{werror} !~ m/^(enospc|ignore|report|stop)$/;
 931     return undef if $res->{backup} && $res->{backup} !~ m/^(yes|no)$/;
 932     return undef if $res->{aio} && $res->{aio} !~ m/^(native|threads)$/;
 933     return undef if $res->{discard} && $res->{discard} !~ m/^(ignore|on)$/;
 934     return undef if $res->{iothread} && $res->{iothread} !~ m/^(on)$/;
 935     return undef if $res->{queues} && ($res->{queues} !~ m/^\d+$/ || $res->{queues} < 2);
 936
 937     return undef if $res->{mbps_rd} && $res->{mbps};
 938     return undef if $res->{mbps_wr} && $res->{mbps};
 939
 940     return undef if $res->{mbps} && $res->{mbps} !~ m/^\d+(\.\d+)?$/;
 941     return undef if $res->{mbps_max} && $res->{mbps_max} !~ m/^\d+(\.\d+)?$/;
 942     return undef if $res->{mbps_rd} && $res->{mbps_rd} !~ m/^\d+(\.\d+)?$/;
 943     return undef if $res->{mbps_rd_max} && $res->{mbps_rd_max} !~ m/^\d+(\.\d+)?$/;
 944     return undef if $res->{mbps_wr} && $res->{mbps_wr} !~ m/^\d+(\.\d+)?$/;
 945     return undef if $res->{mbps_wr_max} && $res->{mbps_wr_max} !~ m/^\d+(\.\d+)?$/;
 946
 947     return undef if $res->{iops_rd} && $res->{iops};
 948     return undef if $res->{iops_wr} && $res->{iops};
 949
 950
 951     return undef if $res->{iops} && $res->{iops} !~ m/^\d+$/;
 952     return undef if $res->{iops_max} && $res->{iops_max} !~ m/^\d+$/;
 953     return undef if $res->{iops_rd} && $res->{iops_rd} !~ m/^\d+$/;
 954     return undef if $res->{iops_rd_max} && $res->{iops_rd_max} !~ m/^\d+$/;
 955     return undef if $res->{iops_wr} && $res->{iops_wr} !~ m/^\d+$/;
 956     return undef if $res->{iops_wr_max} && $res->{iops_wr_max} !~ m/^\d+$/;
 957
 958
 959     if ($res->{size}) {
 960         return undef if !defined($res->{size} = &$parse_size($res->{size}));
 961     }
 962
 963     if ($res->{media} && ($res->{media} eq 'cdrom')) {
 964         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
 965         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
 966         return undef if $res->{interface} eq 'virtio';
 967     }
 968
 969     # rerror does not work with scsi drives
 970     if ($res->{rerror}) {
 971         return undef if $res->{interface} eq 'scsi';
 972     }
 973
 974     return $res;
 975 }
 976
 977 my @qemu_drive_options = qw(heads secs cyls trans media format cache snapshot rerror werror aio discard iops iops_rd iops_wr iops_max iops_rd_max iops_wr_max);
 978
 979 sub print_drive {
 980     my ($vmid, $drive) = @_;
 981
 982     my $opts = '';
 983     foreach my $o (@qemu_drive_options, 'mbps', 'mbps_rd', 'mbps_wr', 'mbps_max', 'mbps_rd_max', 'mbps_wr_max', 'backup', 'iothread', 'queues') {
 984         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
 985     }
 986
 987     if ($drive->{size}) {
 988         $opts .= ",size=" . &$format_size($drive->{size});
 989     }
 990
 991     return "$drive->{file}$opts";
 992 }
 993
 994 sub scsi_inquiry {
 995     my($fh, $noerr) = @_;
 996
 997     my $SG_IO = 0x2285;
 998     my $SG_GET_VERSION_NUM = 0x2282;
 999
1000     my $versionbuf = "\x00" x 8;
1001     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1002     if (!$ret) {
1003         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1004         return undef;
1005     }
1006     my $version = unpack("I", $versionbuf);
1007     if ($version < 30000) {
1008         die "scsi generic interface too old\n"  if !$noerr;
1009         return undef;
1010     }
1011
1012     my $buf = "\x00" x 36;
1013     my $sensebuf = "\x00" x 8;
1014     my $cmd = pack("C x3 C x1", 0x12, 36);
1015
1016     # see /usr/include/scsi/sg.h
1017     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1018
1019     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1020                       length($sensebuf), 0, length($buf), $buf,
1021                       $cmd, $sensebuf, 6000);
1022
1023     $ret = ioctl($fh, $SG_IO, $packet);
1024     if (!$ret) {
1025         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1026         return undef;
1027     }
1028
1029     my @res = unpack($sg_io_hdr_t, $packet);
1030     if ($res[17] || $res[18]) {
1031         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1032         return undef;
1033     }
1034
1035     my $res = {};
1036     (my $byte0, my $byte1, $res->{vendor},
1037      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1038
1039     $res->{removable} = $byte1 & 128 ? 1 : 0;
1040     $res->{type} = $byte0 & 31;
1041
1042     return $res;
1043 }
1044
1045 sub path_is_scsi {
1046     my ($path) = @_;
1047
1048     my $fh = IO::File->new("+<$path") || return undef;
1049     my $res = scsi_inquiry($fh, 1);
1050     close($fh);
1051
1052     return $res;
1053 }
1054
1055 sub machine_type_is_q35 {
1056     my ($conf) = @_;
1057
1058     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1059 }
1060
1061 sub print_tabletdevice_full {
1062     my ($conf) = @_;
1063
1064     my $q35 = machine_type_is_q35($conf);
1065
1066     # we use uhci for old VMs because tablet driver was buggy in older qemu
1067     my $usbbus = $q35 ? "ehci" : "uhci";
1068
1069     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1070 }
1071
1072 sub print_drivedevice_full {
1073     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1074
1075     my $device = '';
1076     my $maxdev = 0;
1077
1078     if ($drive->{interface} eq 'virtio') {
1079         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1080         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1081         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1082     } elsif ($drive->{interface} eq 'scsi') {
1083
1084         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1085         my $unit = $drive->{index} % $maxdev;
1086         my $devicetype = 'hd';
1087         my $path = '';
1088         if (drive_is_cdrom($drive)) {
1089               $devicetype = 'cd';
1090         } else {
1091               if ($drive->{file} =~ m|^/|) {
1092                   $path = $drive->{file};
1093               } else {
1094                   $path = PVE::Storage::path($storecfg, $drive->{file});
1095               }
1096
1097               if($path =~ m/^iscsi\:\/\//){
1098                   $devicetype = 'generic';
1099               } else {
1100                   if (my $info = path_is_scsi($path)) {
1101                       if ($info->{type} == 0) {
1102                           $devicetype = 'block';
1103                       } elsif ($info->{type} == 1) { # tape
1104                           $devicetype = 'generic';
1105                       }
1106                   }
1107               }
1108          }
1109
1110         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1111             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1112         } else {
1113             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1114         }
1115
1116     } elsif ($drive->{interface} eq 'ide'){
1117         $maxdev = 2;
1118         my $controller = int($drive->{index} / $maxdev);
1119         my $unit = $drive->{index} % $maxdev;
1120         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1121
1122         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1123     } elsif ($drive->{interface} eq 'sata'){
1124         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1125         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1126         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1127     } elsif ($drive->{interface} eq 'usb') {
1128         die "implement me";
1129         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1130     } else {
1131         die "unsupported interface type";
1132     }
1133
1134     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1135
1136     return $device;
1137 }
1138
1139 sub get_initiator_name {
1140     my $initiator;
1141
1142     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1143     while (defined(my $line = <$fh>)) {
1144         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1145         $initiator = $1;
1146         last;
1147     }
1148     $fh->close();
1149
1150     return $initiator;
1151 }
1152
1153 sub print_drive_full {
1154     my ($storecfg, $vmid, $drive) = @_;
1155
1156     my $opts = '';
1157     foreach my $o (@qemu_drive_options) {
1158         next if $o eq 'bootindex';
1159         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1160     }
1161
1162     foreach my $o (qw(bps bps_rd bps_wr)) {
1163         my $v = $drive->{"m$o"};
1164         $opts .= ",$o=" . int($v*1024*1024) if $v;
1165     }
1166
1167     my $path;
1168     my $volid = $drive->{file};
1169     if (drive_is_cdrom($drive)) {
1170         $path = get_iso_path($storecfg, $vmid, $volid);
1171     } else {
1172         if ($volid =~ m|^/|) {
1173             $path = $volid;
1174         } else {
1175             $path = PVE::Storage::path($storecfg, $volid);
1176         }
1177     }
1178
1179     my $cache_direct = 0;
1180
1181     if (my $cache = $drive->{cache}) {
1182         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1183     } elsif (!drive_is_cdrom($drive)) {
1184         $opts .= ",cache=none";
1185         $cache_direct = 1;
1186     }
1187
1188     # aio native works only with O_DIRECT
1189     if (!$drive->{aio}) {
1190         if($cache_direct) {
1191             $opts .= ",aio=native";
1192         } else {
1193             $opts .= ",aio=threads";
1194         }
1195     }
1196
1197     my $detectzeroes = $drive->{discard} ? "unmap" : "on";
1198     $opts .= ",detect-zeroes=$detectzeroes" if !drive_is_cdrom($drive);
1199
1200     my $pathinfo = $path ? "file=$path," : '';
1201
1202     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1203 }
1204
1205 sub print_netdevice_full {
1206     my ($vmid, $conf, $net, $netid, $bridges) = @_;
1207
1208     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1209
1210     my $device = $net->{model};
1211     if ($net->{model} eq 'virtio') {
1212          $device = 'virtio-net-pci';
1213      };
1214
1215     my $pciaddr = print_pci_addr("$netid", $bridges);
1216     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1217     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1218         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1219         my $vectors = $net->{queues} * 2 + 2;
1220         $tmpstr .= ",vectors=$vectors,mq=on";
1221     }
1222     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1223     return $tmpstr;
1224 }
1225
1226 sub print_netdev_full {
1227     my ($vmid, $conf, $net, $netid) = @_;
1228
1229     my $i = '';
1230     if ($netid =~ m/^net(\d+)$/) {
1231         $i = int($1);
1232     }
1233
1234     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1235
1236     my $ifname = "tap${vmid}i$i";
1237
1238     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1239     die "interface name '$ifname' is too long (max 15 character)\n"
1240         if length($ifname) >= 16;
1241
1242     my $vhostparam = '';
1243     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1244
1245     my $vmname = $conf->{name} || "vm$vmid";
1246
1247     my $netdev = "";
1248
1249     if ($net->{bridge}) {
1250         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/pve-bridge,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1251     } else {
1252         $netdev = "type=user,id=$netid,hostname=$vmname";
1253     }
1254
1255     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1256
1257     return $netdev;
1258 }
1259
1260 sub drive_is_cdrom {
1261     my ($drive) = @_;
1262
1263     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1264
1265 }
1266
1267 sub parse_numa {
1268     my ($data) = @_;
1269
1270     my $res = {};
1271
1272     foreach my $kvp (split(/,/, $data)) {
1273
1274         if ($kvp =~ m/^memory=(\S+)$/) {
1275             $res->{memory} = $1;
1276         } elsif ($kvp =~ m/^policy=(preferred|bind|interleave)$/) {
1277             $res->{policy} = $1;
1278         } elsif ($kvp =~ m/^cpus=(\d+)(-(\d+))?$/) {
1279             $res->{cpus}->{start} = $1;
1280             $res->{cpus}->{end} = $3;
1281         } elsif ($kvp =~ m/^hostnodes=(\d+)(-(\d+))?$/) {
1282             $res->{hostnodes}->{start} = $1;
1283             $res->{hostnodes}->{end} = $3;
1284         } else {
1285             return undef;
1286         }
1287     }
1288
1289     return $res;
1290 }
1291
1292 sub parse_hostpci {
1293     my ($value) = @_;
1294
1295     return undef if !$value;
1296
1297
1298     my @list = split(/,/, $value);
1299     my $found;
1300
1301     my $res = {};
1302     foreach my $kv (@list) {
1303
1304         if ($kv =~ m/^(host=)?([a-f0-9]{2}:[a-f0-9]{2})(\.([a-f0-9]))?$/) {
1305             $found = 1;
1306             if(defined($4)){
1307                 push @{$res->{pciid}}, { id => $2 , function => $4};
1308
1309             }else{
1310                 my $pcidevices = lspci($2);
1311                 $res->{pciid} = $pcidevices->{$2};
1312             }
1313         } elsif ($kv =~ m/^driver=(kvm|vfio)$/) {
1314             $res->{driver} = $1;
1315         } elsif ($kv =~ m/^rombar=(on|off)$/) {
1316             $res->{rombar} = $1;
1317         } elsif ($kv =~ m/^x-vga=(on|off)$/) {
1318             $res->{'x-vga'} = $1;
1319         } elsif ($kv =~ m/^pcie=(\d+)$/) {
1320             $res->{pcie} = 1 if $1 == 1;
1321         } else {
1322             warn "unknown hostpci setting '$kv'\n";
1323         }
1324     }
1325
1326     return undef if !$found;
1327
1328     return $res;
1329 }
1330
1331 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1332 sub parse_net {
1333     my ($data) = @_;
1334
1335     my $res = {};
1336
1337     foreach my $kvp (split(/,/, $data)) {
1338
1339         if ($kvp =~ m/^(ne2k_pci|e1000|e1000-82540em|e1000-82544gc|e1000-82545em|rtl8139|pcnet|virtio|ne2k_isa|i82551|i82557b|i82559er|vmxnet3)(=([0-9a-f]{2}(:[0-9a-f]{2}){5}))?$/i) {
1340             my $model = lc($1);
1341             my $mac = defined($3) ? uc($3) : PVE::Tools::random_ether_addr();
1342             $res->{model} = $model;
1343             $res->{macaddr} = $mac;
1344         } elsif ($kvp =~ m/^bridge=(\S+)$/) {
1345             $res->{bridge} = $1;
1346         } elsif ($kvp =~ m/^queues=(\d+)$/) {
1347             $res->{queues} = $1;
1348         } elsif ($kvp =~ m/^rate=(\d+(\.\d+)?)$/) {
1349             $res->{rate} = $1;
1350         } elsif ($kvp =~ m/^tag=(\d+)$/) {
1351             $res->{tag} = $1;
1352         } elsif ($kvp =~ m/^firewall=([01])$/) {
1353             $res->{firewall} = $1;
1354         } elsif ($kvp =~ m/^link_down=([01])$/) {
1355             $res->{link_down} = $1;
1356         } else {
1357             return undef;
1358         }
1359
1360     }
1361
1362     return undef if !$res->{model};
1363
1364     return $res;
1365 }
1366
1367 sub print_net {
1368     my $net = shift;
1369
1370     my $res = "$net->{model}";
1371     $res .= "=$net->{macaddr}" if $net->{macaddr};
1372     $res .= ",bridge=$net->{bridge}" if $net->{bridge};
1373     $res .= ",rate=$net->{rate}" if $net->{rate};
1374     $res .= ",tag=$net->{tag}" if $net->{tag};
1375     $res .= ",firewall=1" if $net->{firewall};
1376     $res .= ",link_down=1" if $net->{link_down};
1377     $res .= ",queues=$net->{queues}" if $net->{queues};
1378
1379     return $res;
1380 }
1381
1382 sub add_random_macs {
1383     my ($settings) = @_;
1384
1385     foreach my $opt (keys %$settings) {
1386         next if $opt !~ m/^net(\d+)$/;
1387         my $net = parse_net($settings->{$opt});
1388         next if !$net;
1389         $settings->{$opt} = print_net($net);
1390     }
1391 }
1392
1393 sub add_unused_volume {
1394     my ($config, $volid) = @_;
1395
1396     my $key;
1397     for (my $ind = $MAX_UNUSED_DISKS - 1; $ind >= 0; $ind--) {
1398         my $test = "unused$ind";
1399         if (my $vid = $config->{$test}) {
1400             return if $vid eq $volid; # do not add duplicates
1401         } else {
1402             $key = $test;
1403         }
1404     }
1405
1406     die "To many unused volume - please delete them first.\n" if !$key;
1407
1408     $config->{$key} = $volid;
1409
1410     return $key;
1411 }
1412
1413 sub vm_is_volid_owner {
1414     my ($storecfg, $vmid, $volid) = @_;
1415
1416     if ($volid !~  m|^/|) {
1417         my ($path, $owner);
1418         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1419         if ($owner && ($owner == $vmid)) {
1420             return 1;
1421         }
1422     }
1423
1424     return undef;
1425 }
1426
1427 sub vmconfig_delete_pending_option {
1428     my ($conf, $key) = @_;
1429
1430     delete $conf->{pending}->{$key};
1431     my $pending_delete_hash = { $key => 1 };
1432     foreach my $opt (PVE::Tools::split_list($conf->{pending}->{delete})) {
1433         $pending_delete_hash->{$opt} = 1;
1434     }
1435     $conf->{pending}->{delete} = join(',', keys %$pending_delete_hash);
1436 }
1437
1438 sub vmconfig_undelete_pending_option {
1439     my ($conf, $key) = @_;
1440
1441     my $pending_delete_hash = {};
1442     foreach my $opt (PVE::Tools::split_list($conf->{pending}->{delete})) {
1443         $pending_delete_hash->{$opt} = 1;
1444     }
1445     delete $pending_delete_hash->{$key};
1446
1447     my @keylist = keys %$pending_delete_hash;
1448     if (scalar(@keylist)) {
1449         $conf->{pending}->{delete} = join(',', @keylist);
1450     } else {
1451         delete $conf->{pending}->{delete};
1452     }
1453 }
1454
1455 sub vmconfig_register_unused_drive {
1456     my ($storecfg, $vmid, $conf, $drive) = @_;
1457
1458     if (!drive_is_cdrom($drive)) {
1459         my $volid = $drive->{file};
1460         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1461             add_unused_volume($conf, $volid, $vmid);
1462         }
1463     }
1464 }
1465
1466 sub vmconfig_cleanup_pending {
1467     my ($conf) = @_;
1468
1469     # remove pending changes when nothing changed
1470     my $changes;
1471     foreach my $opt (keys %{$conf->{pending}}) {
1472         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
1473             $changes = 1;
1474             delete $conf->{pending}->{$opt};
1475         }
1476     }
1477
1478     # remove delete if option is not set
1479     my $pending_delete_hash = {};
1480     foreach my $opt (PVE::Tools::split_list($conf->{pending}->{delete})) {
1481         if (defined($conf->{$opt})) {
1482             $pending_delete_hash->{$opt} = 1;
1483         } else {
1484             $changes = 1;
1485         }
1486     }
1487
1488     my @keylist = keys %$pending_delete_hash;
1489     if (scalar(@keylist)) {
1490         $conf->{pending}->{delete} = join(',', @keylist);
1491     } else {
1492         delete $conf->{pending}->{delete};
1493     }
1494
1495     return $changes;
1496 }
1497
1498 my $valid_smbios1_options = {
1499     manufacturer => '\S+',
1500     product => '\S+',
1501     version => '\S+',
1502     serial => '\S+',
1503     uuid => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1504     sku => '\S+',
1505     family => '\S+',
1506 };
1507
1508 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
1509 sub parse_smbios1 {
1510     my ($data) = @_;
1511
1512     my $res = {};
1513
1514     foreach my $kvp (split(/,/, $data)) {
1515         return undef if $kvp !~ m/^(\S+)=(.+)$/;
1516         my ($k, $v) = split(/=/, $kvp);
1517         return undef if !defined($k) || !defined($v);
1518         return undef if !$valid_smbios1_options->{$k};
1519         return undef if $v !~ m/^$valid_smbios1_options->{$k}$/;
1520         $res->{$k} = $v;
1521     }
1522
1523     return $res;
1524 }
1525
1526 sub print_smbios1 {
1527     my ($smbios1) = @_;
1528
1529     my $data = '';
1530     foreach my $k (keys %$smbios1) {
1531         next if !defined($smbios1->{$k});
1532         next if !$valid_smbios1_options->{$k};
1533         $data .= ',' if $data;
1534         $data .= "$k=$smbios1->{$k}";
1535     }
1536     return $data;
1537 }
1538
1539 PVE::JSONSchema::register_format('pve-qm-smbios1', \&verify_smbios1);
1540 sub verify_smbios1 {
1541     my ($value, $noerr) = @_;
1542
1543     return $value if parse_smbios1($value);
1544
1545     return undef if $noerr;
1546
1547     die "unable to parse smbios (type 1) options\n";
1548 }
1549
1550 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
1551 sub verify_bootdisk {
1552     my ($value, $noerr) = @_;
1553
1554     return $value if valid_drivename($value);
1555
1556     return undef if $noerr;
1557
1558     die "invalid boot disk '$value'\n";
1559 }
1560
1561 PVE::JSONSchema::register_format('pve-qm-numanode', \&verify_numa);
1562 sub verify_numa {
1563     my ($value, $noerr) = @_;
1564
1565     return $value if parse_numa($value);
1566
1567     return undef if $noerr;
1568
1569     die "unable to parse numa options\n";
1570 }
1571
1572 PVE::JSONSchema::register_format('pve-qm-net', \&verify_net);
1573 sub verify_net {
1574     my ($value, $noerr) = @_;
1575
1576     return $value if parse_net($value);
1577
1578     return undef if $noerr;
1579
1580     die "unable to parse network options\n";
1581 }
1582
1583 PVE::JSONSchema::register_format('pve-qm-drive', \&verify_drive);
1584 sub verify_drive {
1585     my ($value, $noerr) = @_;
1586
1587     return $value if parse_drive(undef, $value);
1588
1589     return undef if $noerr;
1590
1591     die "unable to parse drive options\n";
1592 }
1593
1594 PVE::JSONSchema::register_format('pve-qm-hostpci', \&verify_hostpci);
1595 sub verify_hostpci {
1596     my ($value, $noerr) = @_;
1597
1598     return $value if parse_hostpci($value);
1599
1600     return undef if $noerr;
1601
1602     die "unable to parse pci id\n";
1603 }
1604
1605 PVE::JSONSchema::register_format('pve-qm-watchdog', \&verify_watchdog);
1606 sub verify_watchdog {
1607     my ($value, $noerr) = @_;
1608
1609     return $value if parse_watchdog($value);
1610
1611     return undef if $noerr;
1612
1613     die "unable to parse watchdog options\n";
1614 }
1615
1616 sub parse_watchdog {
1617     my ($value) = @_;
1618
1619     return undef if !$value;
1620
1621     my $res = {};
1622
1623     foreach my $p (split(/,/, $value)) {
1624         next if $p =~ m/^\s*$/;
1625
1626         if ($p =~ m/^(model=)?(i6300esb|ib700)$/) {
1627             $res->{model} = $2;
1628         } elsif ($p =~ m/^(action=)?(reset|shutdown|poweroff|pause|debug|none)$/) {
1629             $res->{action} = $2;
1630         } else {
1631             return undef;
1632         }
1633     }
1634
1635     return $res;
1636 }
1637
1638 sub parse_usb_device {
1639     my ($value) = @_;
1640
1641     return undef if !$value;
1642
1643     my @dl = split(/,/, $value);
1644     my $found;
1645
1646     my $res = {};
1647     foreach my $v (@dl) {
1648         if ($v =~ m/^host=(0x)?([0-9A-Fa-f]{4}):(0x)?([0-9A-Fa-f]{4})$/) {
1649             $found = 1;
1650             $res->{vendorid} = $2;
1651             $res->{productid} = $4;
1652         } elsif ($v =~ m/^host=(\d+)\-(\d+(\.\d+)*)$/) {
1653             $found = 1;
1654             $res->{hostbus} = $1;
1655             $res->{hostport} = $2;
1656         } elsif ($v =~ m/^spice$/) {
1657             $found = 1;
1658             $res->{spice} = 1;
1659         } else {
1660             return undef;
1661         }
1662     }
1663     return undef if !$found;
1664
1665     return $res;
1666 }
1667
1668 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
1669 sub verify_usb_device {
1670     my ($value, $noerr) = @_;
1671
1672     return $value if parse_usb_device($value);
1673
1674     return undef if $noerr;
1675
1676     die "unable to parse usb device\n";
1677 }
1678
1679 # add JSON properties for create and set function
1680 sub json_config_properties {
1681     my $prop = shift;
1682
1683     foreach my $opt (keys %$confdesc) {
1684         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
1685         $prop->{$opt} = $confdesc->{$opt};
1686     }
1687
1688     return $prop;
1689 }
1690
1691 sub check_type {
1692     my ($key, $value) = @_;
1693
1694     die "unknown setting '$key'\n" if !$confdesc->{$key};
1695
1696     my $type = $confdesc->{$key}->{type};
1697
1698     if (!defined($value)) {
1699         die "got undefined value\n";
1700     }
1701
1702     if ($value =~ m/[\n\r]/) {
1703         die "property contains a line feed\n";
1704     }
1705
1706     if ($type eq 'boolean') {
1707         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
1708         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
1709         die "type check ('boolean') failed - got '$value'\n";
1710     } elsif ($type eq 'integer') {
1711         return int($1) if $value =~ m/^(\d+)$/;
1712         die "type check ('integer') failed - got '$value'\n";
1713     } elsif ($type eq 'number') {
1714         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
1715         die "type check ('number') failed - got '$value'\n";
1716     } elsif ($type eq 'string') {
1717         if (my $fmt = $confdesc->{$key}->{format}) {
1718             if ($fmt eq 'pve-qm-drive') {
1719                 # special case - we need to pass $key to parse_drive()
1720                 my $drive = parse_drive($key, $value);
1721                 return $value if $drive;
1722                 die "unable to parse drive options\n";
1723             }
1724             PVE::JSONSchema::check_format($fmt, $value);
1725             return $value;
1726         }
1727         $value =~ s/^\"(.*)\"$/$1/;
1728         return $value;
1729     } else {
1730         die "internal error"
1731     }
1732 }
1733
1734 sub lock_config_full {
1735     my ($vmid, $timeout, $code, @param) = @_;
1736
1737     my $filename = config_file_lock($vmid);
1738
1739     my $res = lock_file($filename, $timeout, $code, @param);
1740
1741     die $@ if $@;
1742
1743     return $res;
1744 }
1745
1746 sub lock_config_mode {
1747     my ($vmid, $timeout, $shared, $code, @param) = @_;
1748
1749     my $filename = config_file_lock($vmid);
1750
1751     my $res = lock_file_full($filename, $timeout, $shared, $code, @param);
1752
1753     die $@ if $@;
1754
1755     return $res;
1756 }
1757
1758 sub lock_config {
1759     my ($vmid, $code, @param) = @_;
1760
1761     return lock_config_full($vmid, 10, $code, @param);
1762 }
1763
1764 sub cfs_config_path {
1765     my ($vmid, $node) = @_;
1766
1767     $node = $nodename if !$node;
1768     return "nodes/$node/qemu-server/$vmid.conf";
1769 }
1770
1771 sub check_iommu_support{
1772     #fixme : need to check IOMMU support
1773     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
1774
1775     my $iommu=1;
1776     return $iommu;
1777
1778 }
1779
1780 sub config_file {
1781     my ($vmid, $node) = @_;
1782
1783     my $cfspath = cfs_config_path($vmid, $node);
1784     return "/etc/pve/$cfspath";
1785 }
1786
1787 sub config_file_lock {
1788     my ($vmid) = @_;
1789
1790     return "$lock_dir/lock-$vmid.conf";
1791 }
1792
1793 sub touch_config {
1794     my ($vmid) = @_;
1795
1796     my $conf = config_file($vmid);
1797     utime undef, undef, $conf;
1798 }
1799
1800 sub destroy_vm {
1801     my ($storecfg, $vmid, $keep_empty_config) = @_;
1802
1803     my $conffile = config_file($vmid);
1804
1805     my $conf = load_config($vmid);
1806
1807     check_lock($conf);
1808
1809     # only remove disks owned by this VM
1810     foreach_drive($conf, sub {
1811         my ($ds, $drive) = @_;
1812
1813         return if drive_is_cdrom($drive);
1814
1815         my $volid = $drive->{file};
1816
1817         return if !$volid || $volid =~ m|^/|;
1818
1819         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
1820         return if !$path || !$owner || ($owner != $vmid);
1821
1822         PVE::Storage::vdisk_free($storecfg, $volid);
1823     });
1824
1825     if ($keep_empty_config) {
1826         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
1827     } else {
1828         unlink $conffile;
1829     }
1830
1831     # also remove unused disk
1832     eval {
1833         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
1834
1835         eval {
1836             PVE::Storage::foreach_volid($dl, sub {
1837                 my ($volid, $sid, $volname, $d) = @_;
1838                 PVE::Storage::vdisk_free($storecfg, $volid);
1839             });
1840         };
1841         warn $@ if $@;
1842
1843     };
1844     warn $@ if $@;
1845 }
1846
1847 sub load_config {
1848     my ($vmid, $node) = @_;
1849
1850     my $cfspath = cfs_config_path($vmid, $node);
1851
1852     my $conf = PVE::Cluster::cfs_read_file($cfspath);
1853
1854     die "no such VM ('$vmid')\n" if !defined($conf);
1855
1856     return $conf;
1857 }
1858
1859 sub parse_vm_config {
1860     my ($filename, $raw) = @_;
1861
1862     return undef if !defined($raw);
1863
1864     my $res = {
1865         digest => Digest::SHA::sha1_hex($raw),
1866         snapshots => {},
1867         pending => {},
1868     };
1869
1870     $filename =~ m|/qemu-server/(\d+)\.conf$|
1871         || die "got strange filename '$filename'";
1872
1873     my $vmid = $1;
1874
1875     my $conf = $res;
1876     my $descr = '';
1877     my $section = '';
1878
1879     my @lines = split(/\n/, $raw);
1880     foreach my $line (@lines) {
1881         next if $line =~ m/^\s*$/;
1882
1883         if ($line =~ m/^\[PENDING\]\s*$/i) {
1884             $section = 'pending';
1885             $conf->{description} = $descr if $descr;
1886             $descr = '';
1887             $conf = $res->{$section} = {};
1888             next;
1889
1890         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
1891             $section = $1;
1892             $conf->{description} = $descr if $descr;
1893             $descr = '';
1894             $conf = $res->{snapshots}->{$section} = {};
1895             next;
1896         }
1897
1898         if ($line =~ m/^\#(.*)\s*$/) {
1899             $descr .= PVE::Tools::decode_text($1) . "\n";
1900             next;
1901         }
1902
1903         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
1904             $descr .= PVE::Tools::decode_text($2);
1905         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
1906             $conf->{snapstate} = $1;
1907         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
1908             my $key = $1;
1909             my $value = $2;
1910             $conf->{$key} = $value;
1911         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
1912             my $value = $1;
1913             if ($section eq 'pending') {
1914                 $conf->{delete} = $value; # we parse this later
1915             } else {
1916                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
1917             }
1918         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
1919             my $key = $1;
1920             my $value = $2;
1921             eval { $value = check_type($key, $value); };
1922             if ($@) {
1923                 warn "vm $vmid - unable to parse value of '$key' - $@";
1924             } else {
1925                 my $fmt = $confdesc->{$key}->{format};
1926                 if ($fmt && $fmt eq 'pve-qm-drive') {
1927                     my $v = parse_drive($key, $value);
1928                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
1929                         $v->{file} = $volid;
1930                         $value = print_drive($vmid, $v);
1931                     } else {
1932                         warn "vm $vmid - unable to parse value of '$key'\n";
1933                         next;
1934                     }
1935                 }
1936
1937                 if ($key eq 'cdrom') {
1938                     $conf->{ide2} = $value;
1939                 } else {
1940                     $conf->{$key} = $value;
1941                 }
1942             }
1943         }
1944     }
1945
1946     $conf->{description} = $descr if $descr;
1947
1948     delete $res->{snapstate}; # just to be sure
1949
1950     return $res;
1951 }
1952
1953 sub write_vm_config {
1954     my ($filename, $conf) = @_;
1955
1956     delete $conf->{snapstate}; # just to be sure
1957
1958     if ($conf->{cdrom}) {
1959         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
1960         $conf->{ide2} = $conf->{cdrom};
1961         delete $conf->{cdrom};
1962     }
1963
1964     # we do not use 'smp' any longer
1965     if ($conf->{sockets}) {
1966         delete $conf->{smp};
1967     } elsif ($conf->{smp}) {
1968         $conf->{sockets} = $conf->{smp};
1969         delete $conf->{cores};
1970         delete $conf->{smp};
1971     }
1972
1973     my $used_volids = {};
1974
1975     my $cleanup_config = sub {
1976         my ($cref, $pending, $snapname) = @_;
1977
1978         foreach my $key (keys %$cref) {
1979             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
1980                 $key eq 'snapstate' || $key eq 'pending';
1981             my $value = $cref->{$key};
1982             if ($key eq 'delete') {
1983                 die "propertry 'delete' is only allowed in [PENDING]\n"
1984                     if !$pending;
1985                 # fixme: check syntax?
1986                 next;
1987             }
1988             eval { $value = check_type($key, $value); };
1989             die "unable to parse value of '$key' - $@" if $@;
1990
1991             $cref->{$key} = $value;
1992
1993             if (!$snapname && valid_drivename($key)) {
1994                 my $drive = parse_drive($key, $value);
1995                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
1996             }
1997         }
1998     };
1999
2000     &$cleanup_config($conf);
2001
2002     &$cleanup_config($conf->{pending}, 1);
2003
2004     foreach my $snapname (keys %{$conf->{snapshots}}) {
2005         die "internal error" if $snapname eq 'pending';
2006         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2007     }
2008
2009     # remove 'unusedX' settings if we re-add a volume
2010     foreach my $key (keys %$conf) {
2011         my $value = $conf->{$key};
2012         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2013             delete $conf->{$key};
2014         }
2015     }
2016
2017     my $generate_raw_config = sub {
2018         my ($conf) = @_;
2019
2020         my $raw = '';
2021
2022         # add description as comment to top of file
2023         my $descr = $conf->{description} || '';
2024         foreach my $cl (split(/\n/, $descr)) {
2025             $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2026         }
2027
2028         foreach my $key (sort keys %$conf) {
2029             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2030             $raw .= "$key: $conf->{$key}\n";
2031         }
2032         return $raw;
2033     };
2034
2035     my $raw = &$generate_raw_config($conf);
2036
2037     if (scalar(keys %{$conf->{pending}})){
2038         $raw .= "\n[PENDING]\n";
2039         $raw .= &$generate_raw_config($conf->{pending});
2040     }
2041
2042     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2043         $raw .= "\n[$snapname]\n";
2044         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2045     }
2046
2047     return $raw;
2048 }
2049
2050 sub update_config_nolock {
2051     my ($vmid, $conf, $skiplock) = @_;
2052
2053     check_lock($conf) if !$skiplock;
2054
2055     my $cfspath = cfs_config_path($vmid);
2056
2057     PVE::Cluster::cfs_write_file($cfspath, $conf);
2058 }
2059
2060 sub update_config {
2061     my ($vmid, $conf, $skiplock) = @_;
2062
2063     lock_config($vmid, &update_config_nolock, $conf, $skiplock);
2064 }
2065
2066 sub load_defaults {
2067
2068     my $res = {};
2069
2070     # we use static defaults from our JSON schema configuration
2071     foreach my $key (keys %$confdesc) {
2072         if (defined(my $default = $confdesc->{$key}->{default})) {
2073             $res->{$key} = $default;
2074         }
2075     }
2076
2077     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2078     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2079
2080     return $res;
2081 }
2082
2083 sub config_list {
2084     my $vmlist = PVE::Cluster::get_vmlist();
2085     my $res = {};
2086     return $res if !$vmlist || !$vmlist->{ids};
2087     my $ids = $vmlist->{ids};
2088
2089     foreach my $vmid (keys %$ids) {
2090         my $d = $ids->{$vmid};
2091         next if !$d->{node} || $d->{node} ne $nodename;
2092         next if !$d->{type} || $d->{type} ne 'qemu';
2093         $res->{$vmid}->{exists} = 1;
2094     }
2095     return $res;
2096 }
2097
2098 # test if VM uses local resources (to prevent migration)
2099 sub check_local_resources {
2100     my ($conf, $noerr) = @_;
2101
2102     my $loc_res = 0;
2103
2104     $loc_res = 1 if $conf->{hostusb}; # old syntax
2105     $loc_res = 1 if $conf->{hostpci}; # old syntax
2106
2107     foreach my $k (keys %$conf) {
2108         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2109         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2110     }
2111
2112     die "VM uses local resources\n" if $loc_res && !$noerr;
2113
2114     return $loc_res;
2115 }
2116
2117 # check if used storages are available on all nodes (use by migrate)
2118 sub check_storage_availability {
2119     my ($storecfg, $conf, $node) = @_;
2120
2121     foreach_drive($conf, sub {
2122         my ($ds, $drive) = @_;
2123
2124         my $volid = $drive->{file};
2125         return if !$volid;
2126
2127         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2128         return if !$sid;
2129
2130         # check if storage is available on both nodes
2131         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2132         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2133    });
2134 }
2135
2136 # list nodes where all VM images are available (used by has_feature API)
2137 sub shared_nodes {
2138     my ($conf, $storecfg) = @_;
2139
2140     my $nodelist = PVE::Cluster::get_nodelist();
2141     my $nodehash = { map { $_ => 1 } @$nodelist };
2142     my $nodename = PVE::INotify::nodename();
2143
2144     foreach_drive($conf, sub {
2145         my ($ds, $drive) = @_;
2146
2147         my $volid = $drive->{file};
2148         return if !$volid;
2149
2150         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2151         if ($storeid) {
2152             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2153             if ($scfg->{disable}) {
2154                 $nodehash = {};
2155             } elsif (my $avail = $scfg->{nodes}) {
2156                 foreach my $node (keys %$nodehash) {
2157                     delete $nodehash->{$node} if !$avail->{$node};
2158                 }
2159             } elsif (!$scfg->{shared}) {
2160                 foreach my $node (keys %$nodehash) {
2161                     delete $nodehash->{$node} if $node ne $nodename
2162                 }
2163             }
2164         }
2165     });
2166
2167     return $nodehash
2168 }
2169
2170 sub check_lock {
2171     my ($conf) = @_;
2172
2173     die "VM is locked ($conf->{lock})\n" if $conf->{lock};
2174 }
2175
2176 sub check_cmdline {
2177     my ($pidfile, $pid) = @_;
2178
2179     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2180     if (defined($fh)) {
2181         my $line = <$fh>;
2182         $fh->close;
2183         return undef if !$line;
2184         my @param = split(/\0/, $line);
2185
2186         my $cmd = $param[0];
2187         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2188
2189         for (my $i = 0; $i < scalar (@param); $i++) {
2190             my $p = $param[$i];
2191             next if !$p;
2192             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2193                 my $p = $param[$i+1];
2194                 return 1 if $p && ($p eq $pidfile);
2195                 return undef;
2196             }
2197         }
2198     }
2199     return undef;
2200 }
2201
2202 sub check_running {
2203     my ($vmid, $nocheck, $node) = @_;
2204
2205     my $filename = config_file($vmid, $node);
2206
2207     die "unable to find configuration file for VM $vmid - no such machine\n"
2208         if !$nocheck && ! -f $filename;
2209
2210     my $pidfile = pidfile_name($vmid);
2211
2212     if (my $fd = IO::File->new("<$pidfile")) {
2213         my $st = stat($fd);
2214         my $line = <$fd>;
2215         close($fd);
2216
2217         my $mtime = $st->mtime;
2218         if ($mtime > time()) {
2219             warn "file '$filename' modified in future\n";
2220         }
2221
2222         if ($line =~ m/^(\d+)$/) {
2223             my $pid = $1;
2224             if (check_cmdline($pidfile, $pid)) {
2225                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2226                     return $pid;
2227                 }
2228             }
2229         }
2230     }
2231
2232     return undef;
2233 }
2234
2235 sub vzlist {
2236
2237     my $vzlist = config_list();
2238
2239     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2240
2241     while (defined(my $de = $fd->read)) {
2242         next if $de !~ m/^(\d+)\.pid$/;
2243         my $vmid = $1;
2244         next if !defined($vzlist->{$vmid});
2245         if (my $pid = check_running($vmid)) {
2246             $vzlist->{$vmid}->{pid} = $pid;
2247         }
2248     }
2249
2250     return $vzlist;
2251 }
2252
2253 sub disksize {
2254     my ($storecfg, $conf) = @_;
2255
2256     my $bootdisk = $conf->{bootdisk};
2257     return undef if !$bootdisk;
2258     return undef if !valid_drivename($bootdisk);
2259
2260     return undef if !$conf->{$bootdisk};
2261
2262     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2263     return undef if !defined($drive);
2264
2265     return undef if drive_is_cdrom($drive);
2266
2267     my $volid = $drive->{file};
2268     return undef if !$volid;
2269
2270     return $drive->{size};
2271 }
2272
2273 my $last_proc_pid_stat;
2274
2275 # get VM status information
2276 # This must be fast and should not block ($full == false)
2277 # We only query KVM using QMP if $full == true (this can be slow)
2278 sub vmstatus {
2279     my ($opt_vmid, $full) = @_;
2280
2281     my $res = {};
2282
2283     my $storecfg = PVE::Storage::config();
2284
2285     my $list = vzlist();
2286     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2287
2288     my $cpucount = $cpuinfo->{cpus} || 1;
2289
2290     foreach my $vmid (keys %$list) {
2291         next if $opt_vmid && ($vmid ne $opt_vmid);
2292
2293         my $cfspath = cfs_config_path($vmid);
2294         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2295
2296         my $d = {};
2297         $d->{pid} = $list->{$vmid}->{pid};
2298
2299         # fixme: better status?
2300         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2301
2302         my $size = disksize($storecfg, $conf);
2303         if (defined($size)) {
2304             $d->{disk} = 0; # no info available
2305             $d->{maxdisk} = $size;
2306         } else {
2307             $d->{disk} = 0;
2308             $d->{maxdisk} = 0;
2309         }
2310
2311         $d->{cpus} = ($conf->{sockets} || 1) * ($conf->{cores} || 1);
2312         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2313         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2314
2315         $d->{name} = $conf->{name} || "VM $vmid";
2316         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024) : 0;
2317
2318         if ($conf->{balloon}) {
2319             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2320             $d->{shares} = defined($conf->{shares}) ? $conf->{shares} : 1000;
2321         }
2322
2323         $d->{uptime} = 0;
2324         $d->{cpu} = 0;
2325         $d->{mem} = 0;
2326
2327         $d->{netout} = 0;
2328         $d->{netin} = 0;
2329
2330         $d->{diskread} = 0;
2331         $d->{diskwrite} = 0;
2332
2333         $d->{template} = is_template($conf);
2334
2335         $res->{$vmid} = $d;
2336     }
2337
2338     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2339     foreach my $dev (keys %$netdev) {
2340         next if $dev !~ m/^tap([1-9]\d*)i/;
2341         my $vmid = $1;
2342         my $d = $res->{$vmid};
2343         next if !$d;
2344
2345         $d->{netout} += $netdev->{$dev}->{receive};
2346         $d->{netin} += $netdev->{$dev}->{transmit};
2347     }
2348
2349     my $ctime = gettimeofday;
2350
2351     foreach my $vmid (keys %$list) {
2352
2353         my $d = $res->{$vmid};
2354         my $pid = $d->{pid};
2355         next if !$pid;
2356
2357         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2358         next if !$pstat; # not running
2359
2360         my $used = $pstat->{utime} + $pstat->{stime};
2361
2362         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2363
2364         if ($pstat->{vsize}) {
2365             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2366         }
2367
2368         my $old = $last_proc_pid_stat->{$pid};
2369         if (!$old) {
2370             $last_proc_pid_stat->{$pid} = {
2371                 time => $ctime,
2372                 used => $used,
2373                 cpu => 0,
2374             };
2375             next;
2376         }
2377
2378         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2379
2380         if ($dtime > 1000) {
2381             my $dutime = $used -  $old->{used};
2382
2383             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2384             $last_proc_pid_stat->{$pid} = {
2385                 time => $ctime,
2386                 used => $used,
2387                 cpu => $d->{cpu},
2388             };
2389         } else {
2390             $d->{cpu} = $old->{cpu};
2391         }
2392     }
2393
2394     return $res if !$full;
2395
2396     my $qmpclient = PVE::QMPClient->new();
2397
2398     my $ballooncb = sub {
2399         my ($vmid, $resp) = @_;
2400
2401         my $info = $resp->{'return'};
2402         return if !$info->{max_mem};
2403
2404         my $d = $res->{$vmid};
2405
2406         # use memory assigned to VM
2407         $d->{maxmem} = $info->{max_mem};
2408         $d->{balloon} = $info->{actual};
2409
2410         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2411             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2412             $d->{freemem} = $info->{free_mem};
2413         }
2414
2415     };
2416
2417     my $blockstatscb = sub {
2418         my ($vmid, $resp) = @_;
2419         my $data = $resp->{'return'} || [];
2420         my $totalrdbytes = 0;
2421         my $totalwrbytes = 0;
2422         for my $blockstat (@$data) {
2423             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2424             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2425         }
2426         $res->{$vmid}->{diskread} = $totalrdbytes;
2427         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2428     };
2429
2430     my $statuscb = sub {
2431         my ($vmid, $resp) = @_;
2432
2433         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2434         # this fails if ballon driver is not loaded, so this must be
2435         # the last commnand (following command are aborted if this fails).
2436         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2437
2438         my $status = 'unknown';
2439         if (!defined($status = $resp->{'return'}->{status})) {
2440             warn "unable to get VM status\n";
2441             return;
2442         }
2443
2444         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2445     };
2446
2447     foreach my $vmid (keys %$list) {
2448         next if $opt_vmid && ($vmid ne $opt_vmid);
2449         next if !$res->{$vmid}->{pid}; # not running
2450         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2451     }
2452
2453     $qmpclient->queue_execute(undef, 1);
2454
2455     foreach my $vmid (keys %$list) {
2456         next if $opt_vmid && ($vmid ne $opt_vmid);
2457         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2458     }
2459
2460     return $res;
2461 }
2462
2463 sub foreach_dimm {
2464     my ($conf, $vmid, $memory, $sockets, $func) = @_;
2465
2466     my $dimm_id = 0;
2467     my $current_size = 1024;
2468     my $dimm_size = 512;
2469     return if $current_size == $memory;
2470
2471     for (my $j = 0; $j < 8; $j++) {
2472         for (my $i = 0; $i < 32; $i++) {
2473             my $name = "dimm${dimm_id}";
2474             $dimm_id++;
2475             my $numanode = $i % $sockets;
2476             $current_size += $dimm_size;
2477             &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
2478             return  $current_size if $current_size >= $memory;
2479         }
2480         $dimm_size *= 2;
2481     }
2482 }
2483
2484 sub foreach_drive {
2485     my ($conf, $func) = @_;
2486
2487     foreach my $ds (keys %$conf) {
2488         next if !valid_drivename($ds);
2489
2490         my $drive = parse_drive($ds, $conf->{$ds});
2491         next if !$drive;
2492
2493         &$func($ds, $drive);
2494     }
2495 }
2496
2497 sub foreach_volid {
2498     my ($conf, $func) = @_;
2499
2500     my $volhash = {};
2501
2502     my $test_volid = sub {
2503         my ($volid, $is_cdrom) = @_;
2504
2505         return if !$volid;
2506
2507         $volhash->{$volid} = $is_cdrom || 0;
2508     };
2509
2510     foreach_drive($conf, sub {
2511         my ($ds, $drive) = @_;
2512         &$test_volid($drive->{file}, drive_is_cdrom($drive));
2513     });
2514
2515     foreach my $snapname (keys %{$conf->{snapshots}}) {
2516         my $snap = $conf->{snapshots}->{$snapname};
2517         &$test_volid($snap->{vmstate}, 0);
2518         foreach_drive($snap, sub {
2519             my ($ds, $drive) = @_;
2520             &$test_volid($drive->{file}, drive_is_cdrom($drive));
2521         });
2522     }
2523
2524     foreach my $volid (keys %$volhash) {
2525         &$func($volid, $volhash->{$volid});
2526     }
2527 }
2528
2529 sub vga_conf_has_spice {
2530     my ($vga) = @_;
2531
2532     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
2533
2534     return $1 || 1;
2535 }
2536
2537 sub config_to_command {
2538     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2539
2540     my $cmd = [];
2541     my $globalFlags = [];
2542     my $machineFlags = [];
2543     my $rtcFlags = [];
2544     my $cpuFlags = [];
2545     my $devices = [];
2546     my $pciaddr = '';
2547     my $bridges = {};
2548     my $kvmver = kvm_user_version();
2549     my $vernum = 0; # unknown
2550     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
2551         $vernum = $1*1000000+$2*1000;
2552     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
2553         $vernum = $1*1000000+$2*1000+$3;
2554     }
2555
2556     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
2557
2558     my $have_ovz = -f '/proc/vz/vestat';
2559
2560     my $q35 = machine_type_is_q35($conf);
2561     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2562     my $machine_type = $forcemachine || $conf->{machine};
2563
2564     my $cpuunits = defined($conf->{cpuunits}) ?
2565             $conf->{cpuunits} : $defaults->{cpuunits};
2566
2567     push @$cmd, '/usr/bin/systemd-run';
2568     push @$cmd, '--scope';
2569     push @$cmd, '--slice', "qemu";
2570     push @$cmd, '--unit', $vmid;
2571     push @$cmd, '-p', "CPUShares=$cpuunits";
2572     if ($conf->{cpulimit}) {
2573         my $cpulimit = int($conf->{cpulimit} * 100);
2574         push @$cmd, '-p', "CPUQuota=$cpulimit\%";
2575     }
2576
2577     push @$cmd, '/usr/bin/kvm';
2578
2579     push @$cmd, '-id', $vmid;
2580
2581     my $use_virtio = 0;
2582
2583     my $qmpsocket = qmp_socket($vmid);
2584     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2585     push @$cmd, '-mon', "chardev=qmp,mode=control";
2586
2587     my $socket = vnc_socket($vmid);
2588     push @$cmd,  '-vnc', "unix:$socket,x509,password";
2589
2590     push @$cmd, '-pidfile' , pidfile_name($vmid);
2591
2592     push @$cmd, '-daemonize';
2593
2594     if ($conf->{smbios1}) {
2595         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
2596     }
2597
2598     if ($q35) {
2599         # the q35 chipset support native usb2, so we enable usb controller
2600         # by default for this machine type
2601         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
2602     } else {
2603         $pciaddr = print_pci_addr("piix3", $bridges);
2604         push @$devices, '-device', "piix3-usb-uhci,id=uhci$pciaddr.0x2";
2605
2606         my $use_usb2 = 0;
2607         for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2608             next if !$conf->{"usb$i"};
2609             $use_usb2 = 1;
2610         }
2611         # include usb device config
2612         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-usb.cfg' if $use_usb2;
2613     }
2614
2615     my $vga = $conf->{vga};
2616
2617     my $qxlnum = vga_conf_has_spice($vga);
2618     $vga = 'qxl' if $qxlnum;
2619
2620     if (!$vga) {
2621         if ($conf->{ostype} && ($conf->{ostype} eq 'win8' ||
2622                                 $conf->{ostype} eq 'win7' ||
2623                                 $conf->{ostype} eq 'w2k8')) {
2624             $vga = 'std';
2625         } else {
2626             $vga = 'cirrus';
2627         }
2628     }
2629
2630     # enable absolute mouse coordinates (needed by vnc)
2631     my $tablet;
2632     if (defined($conf->{tablet})) {
2633         $tablet = $conf->{tablet};
2634     } else {
2635         $tablet = $defaults->{tablet};
2636         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
2637         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
2638     }
2639
2640     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
2641
2642     # host pci devices
2643     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
2644         my $d = parse_hostpci($conf->{"hostpci$i"});
2645         next if !$d;
2646
2647         my $pcie = $d->{pcie};
2648         if($pcie){
2649             die "q35 machine model is not enabled" if !$q35;
2650             $pciaddr = print_pcie_addr("hostpci$i");
2651         }else{
2652             $pciaddr = print_pci_addr("hostpci$i", $bridges);
2653         }
2654
2655         my $rombar = $d->{rombar} && $d->{rombar} eq 'off' ? ",rombar=0" : "";
2656         my $driver = $d->{driver} && $d->{driver} eq 'vfio' ? "vfio-pci" : "pci-assign";
2657         my $xvga = $d->{'x-vga'} && $d->{'x-vga'} eq 'on' ? ",x-vga=on" : "";
2658         if ($xvga && $xvga ne '') {
2659             push @$cpuFlags, 'kvm=off';
2660             $vga = 'none';
2661         }
2662         $driver = "vfio-pci" if $xvga ne '';
2663         my $pcidevices = $d->{pciid};
2664         my $multifunction = 1 if @$pcidevices > 1;
2665
2666         my $j=0;
2667         foreach my $pcidevice (@$pcidevices) {
2668
2669             my $id = "hostpci$i";
2670             $id .= ".$j" if $multifunction;
2671             my $addr = $pciaddr;
2672             $addr .= ".$j" if $multifunction;
2673             my $devicestr = "$driver,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
2674
2675             if($j == 0){
2676                 $devicestr .= "$rombar$xvga";
2677                 $devicestr .= ",multifunction=on" if $multifunction;
2678             }
2679
2680             push @$devices, '-device', $devicestr;
2681             $j++;
2682         }
2683     }
2684
2685     # usb devices
2686     for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2687         my $d = parse_usb_device($conf->{"usb$i"});
2688         next if !$d;
2689         if ($d->{vendorid} && $d->{productid}) {
2690             push @$devices, '-device', "usb-host,vendorid=0x$d->{vendorid},productid=0x$d->{productid}";
2691         } elsif (defined($d->{hostbus}) && defined($d->{hostport})) {
2692             push @$devices, '-device', "usb-host,hostbus=$d->{hostbus},hostport=$d->{hostport}";
2693         } elsif ($d->{spice}) {
2694             # usb redir support for spice
2695             push @$devices, '-chardev', "spicevmc,id=usbredirchardev$i,name=usbredir";
2696             push @$devices, '-device', "usb-redir,chardev=usbredirchardev$i,id=usbredirdev$i,bus=ehci.0";
2697         }
2698     }
2699
2700     # serial devices
2701     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2702         if (my $path = $conf->{"serial$i"}) {
2703             if ($path eq 'socket') {
2704                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
2705                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
2706                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2707             } else {
2708                 die "no such serial device\n" if ! -c $path;
2709                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
2710                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2711             }
2712         }
2713     }
2714
2715     # parallel devices
2716     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
2717         if (my $path = $conf->{"parallel$i"}) {
2718             die "no such parallel device\n" if ! -c $path;
2719             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
2720             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
2721             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
2722         }
2723     }
2724
2725     my $vmname = $conf->{name} || "vm$vmid";
2726
2727     push @$cmd, '-name', $vmname;
2728
2729     my $sockets = 1;
2730     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
2731     $sockets = $conf->{sockets} if  $conf->{sockets};
2732
2733     my $cores = $conf->{cores} || 1;
2734
2735     my $maxcpus = $sockets * $cores;
2736
2737     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
2738
2739     my $allowed_vcpus = $cpuinfo->{cpus};
2740
2741     die "MAX $maxcpus vcpus allowed per VM on this node\n"
2742         if ($allowed_vcpus < $maxcpus);
2743
2744     push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
2745
2746     push @$cmd, '-nodefaults';
2747
2748     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2749
2750     my $bootindex_hash = {};
2751     my $i = 1;
2752     foreach my $o (split(//, $bootorder)) {
2753         $bootindex_hash->{$o} = $i*100;
2754         $i++;
2755     }
2756
2757     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000";
2758
2759     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
2760
2761     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
2762
2763     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
2764
2765     # time drift fix
2766     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
2767
2768     my $nokvm = defined($conf->{kvm}) && $conf->{kvm} == 0 ? 1 : 0;
2769     my $useLocaltime = $conf->{localtime};
2770
2771     if (my $ost = $conf->{ostype}) {
2772         # other, wxp, w2k, w2k3, w2k8, wvista, win7, win8, l24, l26, solaris
2773
2774         if ($ost =~ m/^w/) { # windows
2775             $useLocaltime = 1 if !defined($conf->{localtime});
2776
2777             # use time drift fix when acpi is enabled
2778             if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
2779                 $tdf = 1 if !defined($conf->{tdf});
2780             }
2781         }
2782
2783         if ($ost eq 'win7' || $ost eq 'win8' || $ost eq 'w2k8' ||
2784             $ost eq 'wvista') {
2785             push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
2786             push @$cmd, '-no-hpet';
2787             if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
2788                 push @$cpuFlags , 'hv_spinlocks=0x1fff' if !$nokvm;
2789                 push @$cpuFlags , 'hv_vapic' if !$nokvm;
2790                 push @$cpuFlags , 'hv_time' if !$nokvm;
2791
2792             } else {
2793                 push @$cpuFlags , 'hv_spinlocks=0xffff' if !$nokvm;
2794             }
2795         }
2796
2797         if ($ost eq 'win7' || $ost eq 'win8') {
2798             push @$cpuFlags , 'hv_relaxed' if !$nokvm;
2799         }
2800     }
2801
2802     push @$rtcFlags, 'driftfix=slew' if $tdf;
2803
2804     if ($nokvm) {
2805         push @$machineFlags, 'accel=tcg';
2806     } else {
2807         die "No accelerator found!\n" if !$cpuinfo->{hvm};
2808     }
2809
2810     if ($machine_type) {
2811         push @$machineFlags, "type=${machine_type}";
2812     }
2813
2814     if ($conf->{startdate}) {
2815         push @$rtcFlags, "base=$conf->{startdate}";
2816     } elsif ($useLocaltime) {
2817         push @$rtcFlags, 'base=localtime';
2818     }
2819
2820     my $cpu = $nokvm ? "qemu64" : "kvm64";
2821     $cpu = $conf->{cpu} if $conf->{cpu};
2822
2823     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
2824
2825     push @$cpuFlags , '+x2apic' if !$nokvm && $conf->{ostype} ne 'solaris';
2826
2827     push @$cpuFlags , '-x2apic' if $conf->{ostype} eq 'solaris';
2828
2829     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
2830
2831     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
2832
2833         push @$cpuFlags , '+kvm_pv_unhalt' if !$nokvm;
2834         push @$cpuFlags , '+kvm_pv_eoi' if !$nokvm;
2835     }
2836
2837     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
2838
2839     push @$cmd, '-cpu', "$cpu,enforce";
2840
2841     my $memory = $conf->{memory} || $defaults->{memory};
2842     my $static_memory = 0;
2843     my $dimm_memory = 0;
2844
2845     if ($hotplug_features->{memory}) {
2846         die "Numa need to be enabled for memory hotplug\n" if !$conf->{numa};
2847         die "Total memory is bigger than ${MAX_MEM}MB\n" if $memory > $MAX_MEM;
2848         $static_memory = $STATICMEM;
2849         die "minimum memory must be ${static_memory}MB\n" if($memory < $static_memory);
2850         $dimm_memory = $memory - $static_memory;
2851         push @$cmd, '-m', "size=${static_memory},slots=255,maxmem=${MAX_MEM}M";
2852
2853     } else {
2854
2855         $static_memory = $memory;
2856         push @$cmd, '-m', $static_memory;
2857     }
2858
2859     if ($conf->{numa}) {
2860
2861         my $numa_totalmemory = undef;
2862         for (my $i = 0; $i < $MAX_NUMA; $i++) {
2863             next if !$conf->{"numa$i"};
2864             my $numa = parse_numa($conf->{"numa$i"});
2865             next if !$numa;
2866             # memory
2867             die "missing numa node$i memory value\n" if !$numa->{memory};
2868             my $numa_memory = $numa->{memory};
2869             $numa_totalmemory += $numa_memory;
2870             my $numa_object = "memory-backend-ram,id=ram-node$i,size=${numa_memory}M";
2871
2872             # cpus
2873             my $cpus_start = $numa->{cpus}->{start};
2874             die "missing numa node$i cpus\n" if !defined($cpus_start);
2875             my $cpus_end = $numa->{cpus}->{end} if defined($numa->{cpus}->{end});
2876             my $cpus = $cpus_start;
2877             if (defined($cpus_end)) {
2878                 $cpus .= "-$cpus_end";
2879                 die "numa node$i :  cpu range $cpus is incorrect\n" if $cpus_end <= $cpus_start;
2880             }
2881
2882             # hostnodes
2883             my $hostnodes_start = $numa->{hostnodes}->{start};
2884             if (defined($hostnodes_start)) {
2885                 my $hostnodes_end = $numa->{hostnodes}->{end} if defined($numa->{hostnodes}->{end});
2886                 my $hostnodes = $hostnodes_start;
2887                 if (defined($hostnodes_end)) {
2888                     $hostnodes .= "-$hostnodes_end";
2889                     die "host node $hostnodes range is incorrect\n" if $hostnodes_end <= $hostnodes_start;
2890                 }
2891
2892                 my $hostnodes_end_range = defined($hostnodes_end) ? $hostnodes_end : $hostnodes_start;
2893                 for (my $i = $hostnodes_start; $i <= $hostnodes_end_range; $i++ ) {
2894                     die "host numa node$i don't exist\n" if ! -d "/sys/devices/system/node/node$i/";
2895                 }
2896
2897                 # policy
2898                 my $policy = $numa->{policy};
2899                 die "you need to define a policy for hostnode $hostnodes\n" if !$policy;
2900                 $numa_object .= ",host-nodes=$hostnodes,policy=$policy";
2901             }
2902
2903             push @$cmd, '-object', $numa_object;
2904             push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
2905         }
2906
2907         die "total memory for NUMA nodes must be equal to vm static memory\n"
2908             if $numa_totalmemory && $numa_totalmemory != $static_memory;
2909
2910         #if no custom tology, we split memory and cores across numa nodes
2911         if(!$numa_totalmemory) {
2912
2913             my $numa_memory = ($static_memory / $sockets) . "M";
2914
2915             for (my $i = 0; $i < $sockets; $i++)  {
2916
2917                 my $cpustart = ($cores * $i);
2918                 my $cpuend = ($cpustart + $cores - 1) if $cores && $cores > 1;
2919                 my $cpus = $cpustart;
2920                 $cpus .= "-$cpuend" if $cpuend;
2921
2922                 push @$cmd, '-object', "memory-backend-ram,size=$numa_memory,id=ram-node$i";
2923                 push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
2924             }
2925         }
2926     }
2927
2928     if ($hotplug_features->{memory}) {
2929         foreach_dimm($conf, $vmid, $memory, $sockets, sub {
2930             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
2931             push @$cmd, "-object" , "memory-backend-ram,id=mem-$name,size=${dimm_size}M";
2932             push @$cmd, "-device", "pc-dimm,id=$name,memdev=mem-$name,node=$numanode";
2933
2934             #if dimm_memory is not aligned to dimm map
2935             if($current_size > $memory) {
2936                  $conf->{memory} = $current_size;
2937                  update_config_nolock($vmid, $conf, 1);
2938             }
2939         });
2940     }
2941
2942     push @$cmd, '-S' if $conf->{freeze};
2943
2944     # set keyboard layout
2945     my $kb = $conf->{keyboard} || $defaults->{keyboard};
2946     push @$cmd, '-k', $kb if $kb;
2947
2948     # enable sound
2949     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
2950     #push @$cmd, '-soundhw', 'es1370';
2951     #push @$cmd, '-soundhw', $soundhw if $soundhw;
2952
2953     if($conf->{agent}) {
2954         my $qgasocket = qmp_socket($vmid, 1);
2955         my $pciaddr = print_pci_addr("qga0", $bridges);
2956         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
2957         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
2958         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
2959     }
2960
2961     my $spice_port;
2962
2963     if ($qxlnum) {
2964         if ($qxlnum > 1) {
2965             if ($conf->{ostype} && $conf->{ostype} =~ m/^w/){
2966                 for(my $i = 1; $i < $qxlnum; $i++){
2967                     my $pciaddr = print_pci_addr("vga$i", $bridges);
2968                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
2969                 }
2970             } else {
2971                 # assume other OS works like Linux
2972                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
2973                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
2974             }
2975         }
2976
2977         my $pciaddr = print_pci_addr("spice", $bridges);
2978
2979         my $nodename = PVE::INotify::nodename();
2980         my $pfamily = PVE::Tools::get_host_address_family($nodename);
2981         $spice_port = PVE::Tools::next_spice_port($pfamily);
2982
2983         push @$devices, '-spice', "tls-port=${spice_port},addr=localhost,tls-ciphers=DES-CBC3-SHA,seamless-migration=on";
2984
2985         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
2986         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
2987         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
2988     }
2989
2990     # enable balloon by default, unless explicitly disabled
2991     if (!defined($conf->{balloon}) || $conf->{balloon}) {
2992         $pciaddr = print_pci_addr("balloon0", $bridges);
2993         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
2994     }
2995
2996     if ($conf->{watchdog}) {
2997         my $wdopts = parse_watchdog($conf->{watchdog});
2998         $pciaddr = print_pci_addr("watchdog", $bridges);
2999         my $watchdog = $wdopts->{model} || 'i6300esb';
3000         push @$devices, '-device', "$watchdog$pciaddr";
3001         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3002     }
3003
3004     my $vollist = [];
3005     my $scsicontroller = {};
3006     my $ahcicontroller = {};
3007     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3008
3009     # Add iscsi initiator name if available
3010     if (my $initiator = get_initiator_name()) {
3011         push @$devices, '-iscsi', "initiator-name=$initiator";
3012     }
3013
3014     foreach_drive($conf, sub {
3015         my ($ds, $drive) = @_;
3016
3017         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3018             push @$vollist, $drive->{file};
3019         }
3020
3021         $use_virtio = 1 if $ds =~ m/^virtio/;
3022
3023         if (drive_is_cdrom ($drive)) {
3024             if ($bootindex_hash->{d}) {
3025                 $drive->{bootindex} = $bootindex_hash->{d};
3026                 $bootindex_hash->{d} += 1;
3027             }
3028         } else {
3029             if ($bootindex_hash->{c}) {
3030                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3031                 $bootindex_hash->{c} += 1;
3032             }
3033         }
3034
3035         if($drive->{interface} eq 'virtio'){
3036            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3037         }
3038
3039         if ($drive->{interface} eq 'scsi') {
3040
3041             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3042
3043             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3044             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3045
3046             my $iothread = '';
3047             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3048                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3049                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3050             }
3051
3052             my $queues = '';
3053             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3054                 $queues = ",num_queues=$drive->{queues}";
3055             }
3056
3057             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3058             $scsicontroller->{$controller}=1;
3059         }
3060
3061         if ($drive->{interface} eq 'sata') {
3062            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3063            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3064            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3065            $ahcicontroller->{$controller}=1;
3066         }
3067
3068         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3069         push @$devices, '-drive',$drive_cmd;
3070         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3071     });
3072
3073     for (my $i = 0; $i < $MAX_NETS; $i++) {
3074          next if !$conf->{"net$i"};
3075          my $d = parse_net($conf->{"net$i"});
3076          next if !$d;
3077
3078          $use_virtio = 1 if $d->{model} eq 'virtio';
3079
3080          if ($bootindex_hash->{n}) {
3081             $d->{bootindex} = $bootindex_hash->{n};
3082             $bootindex_hash->{n} += 1;
3083          }
3084
3085          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3086          push @$devices, '-netdev', $netdevfull;
3087
3088          my $netdevicefull = print_netdevice_full($vmid,$conf,$d,"net$i",$bridges);
3089          push @$devices, '-device', $netdevicefull;
3090     }
3091
3092     if (!$q35) {
3093         # add pci bridges
3094         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3095            $bridges->{1} = 1;
3096            $bridges->{2} = 1;
3097         }
3098
3099         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3100
3101         while (my ($k, $v) = each %$bridges) {
3102             $pciaddr = print_pci_addr("pci.$k");
3103             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3104         }
3105     }
3106
3107     # add custom args
3108     if ($conf->{args}) {
3109         my $aa = PVE::Tools::split_args($conf->{args});
3110         push @$cmd, @$aa;
3111     }
3112
3113     push @$cmd, @$devices;
3114     push @$cmd, '-rtc', join(',', @$rtcFlags)
3115         if scalar(@$rtcFlags);
3116     push @$cmd, '-machine', join(',', @$machineFlags)
3117         if scalar(@$machineFlags);
3118     push @$cmd, '-global', join(',', @$globalFlags)
3119         if scalar(@$globalFlags);
3120
3121     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3122 }
3123
3124 sub vnc_socket {
3125     my ($vmid) = @_;
3126     return "${var_run_tmpdir}/$vmid.vnc";
3127 }
3128
3129 sub spice_port {
3130     my ($vmid) = @_;
3131
3132     my $res = vm_mon_cmd($vmid, 'query-spice');
3133
3134     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3135 }
3136
3137 sub qmp_socket {
3138     my ($vmid, $qga) = @_;
3139     my $sockettype = $qga ? 'qga' : 'qmp';
3140     return "${var_run_tmpdir}/$vmid.$sockettype";
3141 }
3142
3143 sub pidfile_name {
3144     my ($vmid) = @_;
3145     return "${var_run_tmpdir}/$vmid.pid";
3146 }
3147
3148 sub vm_devices_list {
3149     my ($vmid) = @_;
3150
3151     my $res = vm_mon_cmd($vmid, 'query-pci');
3152     my $devices = {};
3153     foreach my $pcibus (@$res) {
3154         foreach my $device (@{$pcibus->{devices}}) {
3155             next if !$device->{'qdev_id'};
3156             if ($device->{'pci_bridge'}) {
3157                 $devices->{$device->{'qdev_id'}} = 1;
3158                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3159                     next if !$bridge_device->{'qdev_id'};
3160                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3161                     $devices->{$device->{'qdev_id'}}++;
3162                 }
3163             } else {
3164                 $devices->{$device->{'qdev_id'}} = 1;
3165             }
3166         }
3167     }
3168
3169     my $resblock = vm_mon_cmd($vmid, 'query-block');
3170     foreach my $block (@$resblock) {
3171         if($block->{device} =~ m/^drive-(\S+)/){
3172                 $devices->{$1} = 1;
3173         }
3174     }
3175
3176     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3177     foreach my $mice (@$resmice) {
3178         if ($mice->{name} eq 'QEMU HID Tablet') {
3179             $devices->{tablet} = 1;
3180             last;
3181         }
3182     }
3183
3184     return $devices;
3185 }
3186
3187 sub vm_deviceplug {
3188     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3189
3190     my $q35 = machine_type_is_q35($conf);
3191
3192     my $devices_list = vm_devices_list($vmid);
3193     return 1 if defined($devices_list->{$deviceid});
3194
3195     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3196
3197     if ($deviceid eq 'tablet') {
3198
3199         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3200
3201     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3202
3203         qemu_iothread_add($vmid, $deviceid, $device);
3204
3205         qemu_driveadd($storecfg, $vmid, $device);
3206         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3207
3208         qemu_deviceadd($vmid, $devicefull);
3209         eval { qemu_deviceaddverify($vmid, $deviceid); };
3210         if (my $err = $@) {
3211             eval { qemu_drivedel($vmid, $deviceid); };
3212             warn $@ if $@;
3213             die $err;
3214         }
3215
3216     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3217
3218
3219         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3220         my $pciaddr = print_pci_addr($deviceid);
3221         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3222
3223         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3224
3225         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3226             qemu_iothread_add($vmid, $deviceid, $device);
3227             $devicefull .= ",iothread=iothread-$deviceid";
3228         }
3229
3230         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3231             $devicefull .= ",num_queues=$device->{queues}";
3232         }
3233
3234         qemu_deviceadd($vmid, $devicefull);
3235         qemu_deviceaddverify($vmid, $deviceid);
3236
3237     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3238
3239         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3240         qemu_driveadd($storecfg, $vmid, $device);
3241
3242         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3243         eval { qemu_deviceadd($vmid, $devicefull); };
3244         if (my $err = $@) {
3245             eval { qemu_drivedel($vmid, $deviceid); };
3246             warn $@ if $@;
3247             die $err;
3248         }
3249
3250     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3251
3252         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3253         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid);
3254         qemu_deviceadd($vmid, $netdevicefull);
3255         eval { qemu_deviceaddverify($vmid, $deviceid); };
3256         if (my $err = $@) {
3257             eval { qemu_netdevdel($vmid, $deviceid); };
3258             warn $@ if $@;
3259             die $err;
3260         }
3261
3262     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3263
3264         my $bridgeid = $2;
3265         my $pciaddr = print_pci_addr($deviceid);
3266         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3267
3268         qemu_deviceadd($vmid, $devicefull);
3269         qemu_deviceaddverify($vmid, $deviceid);
3270
3271     } else {
3272         die "can't hotplug device '$deviceid'\n";
3273     }
3274
3275     return 1;
3276 }
3277
3278 # fixme: this should raise exceptions on error!
3279 sub vm_deviceunplug {
3280     my ($vmid, $conf, $deviceid) = @_;
3281
3282     my $devices_list = vm_devices_list($vmid);
3283     return 1 if !defined($devices_list->{$deviceid});
3284
3285     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3286
3287     if ($deviceid eq 'tablet') {
3288
3289         qemu_devicedel($vmid, $deviceid);
3290
3291     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3292
3293         qemu_devicedel($vmid, $deviceid);
3294         qemu_devicedelverify($vmid, $deviceid);
3295         qemu_drivedel($vmid, $deviceid);
3296         qemu_iothread_del($conf, $vmid, $deviceid);
3297
3298     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3299
3300         qemu_devicedel($vmid, $deviceid);
3301         qemu_devicedelverify($vmid, $deviceid);
3302         qemu_iothread_del($conf, $vmid, $deviceid);
3303
3304     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3305
3306         #qemu 2.3 segfault on drive_del with virtioscsi + iothread
3307         my $device = parse_drive($deviceid, $conf->{$deviceid});
3308         die "virtioscsi with iothread is not hot-unplugglable currently" if $device->{iothread};
3309
3310         qemu_devicedel($vmid, $deviceid);
3311         qemu_drivedel($vmid, $deviceid);
3312         qemu_deletescsihw($conf, $vmid, $deviceid);
3313
3314     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3315
3316         qemu_devicedel($vmid, $deviceid);
3317         qemu_devicedelverify($vmid, $deviceid);
3318         qemu_netdevdel($vmid, $deviceid);
3319
3320     } else {
3321         die "can't unplug device '$deviceid'\n";
3322     }
3323
3324     return 1;
3325 }
3326
3327 sub qemu_deviceadd {
3328     my ($vmid, $devicefull) = @_;
3329
3330     $devicefull = "driver=".$devicefull;
3331     my %options =  split(/[=,]/, $devicefull);
3332
3333     vm_mon_cmd($vmid, "device_add" , %options);
3334 }
3335
3336 sub qemu_devicedel {
3337     my ($vmid, $deviceid) = @_;
3338
3339     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3340 }
3341
3342 sub qemu_iothread_add {
3343     my($vmid, $deviceid, $device) = @_;
3344
3345     if ($device->{iothread}) {
3346         my $iothreads = vm_iothreads_list($vmid);
3347         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3348     }
3349 }
3350
3351 sub qemu_iothread_del {
3352     my($conf, $vmid, $deviceid) = @_;
3353
3354     my $device = parse_drive($deviceid, $conf->{$deviceid});
3355     if ($device->{iothread}) {
3356         my $iothreads = vm_iothreads_list($vmid);
3357         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3358     }
3359 }
3360
3361 sub qemu_objectadd {
3362     my($vmid, $objectid, $qomtype) = @_;
3363
3364     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3365
3366     return 1;
3367 }
3368
3369 sub qemu_objectdel {
3370     my($vmid, $objectid) = @_;
3371
3372     vm_mon_cmd($vmid, "object-del", id => $objectid);
3373
3374     return 1;
3375 }
3376
3377 sub qemu_driveadd {
3378     my ($storecfg, $vmid, $device) = @_;
3379
3380     my $drive = print_drive_full($storecfg, $vmid, $device);
3381     $drive =~ s/\\/\\\\/g;
3382     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3383
3384     # If the command succeeds qemu prints: "OK"
3385     return 1 if $ret =~ m/OK/s;
3386
3387     die "adding drive failed: $ret\n";
3388 }
3389
3390 sub qemu_drivedel {
3391     my($vmid, $deviceid) = @_;
3392
3393     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3394     $ret =~ s/^\s+//;
3395
3396     return 1 if $ret eq "";
3397
3398     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3399     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3400
3401     die "deleting drive $deviceid failed : $ret\n";
3402 }
3403
3404 sub qemu_deviceaddverify {
3405     my ($vmid, $deviceid) = @_;
3406
3407     for (my $i = 0; $i <= 5; $i++) {
3408          my $devices_list = vm_devices_list($vmid);
3409          return 1 if defined($devices_list->{$deviceid});
3410          sleep 1;
3411     }
3412
3413     die "error on hotplug device '$deviceid'\n";
3414 }
3415
3416
3417 sub qemu_devicedelverify {
3418     my ($vmid, $deviceid) = @_;
3419
3420     # need to verify that the device is correctly removed as device_del
3421     # is async and empty return is not reliable
3422
3423     for (my $i = 0; $i <= 5; $i++) {
3424          my $devices_list = vm_devices_list($vmid);
3425          return 1 if !defined($devices_list->{$deviceid});
3426          sleep 1;
3427     }
3428
3429     die "error on hot-unplugging device '$deviceid'\n";
3430 }
3431
3432 sub qemu_findorcreatescsihw {
3433     my ($storecfg, $conf, $vmid, $device) = @_;
3434
3435     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3436
3437     my $scsihwid="$controller_prefix$controller";
3438     my $devices_list = vm_devices_list($vmid);
3439
3440     if(!defined($devices_list->{$scsihwid})) {
3441         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3442     }
3443
3444     return 1;
3445 }
3446
3447 sub qemu_deletescsihw {
3448     my ($conf, $vmid, $opt) = @_;
3449
3450     my $device = parse_drive($opt, $conf->{$opt});
3451
3452     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3453         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3454         return 1;
3455     }
3456
3457     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3458
3459     my $devices_list = vm_devices_list($vmid);
3460     foreach my $opt (keys %{$devices_list}) {
3461         if (PVE::QemuServer::valid_drivename($opt)) {
3462             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3463             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3464                 return 1;
3465             }
3466         }
3467     }
3468
3469     my $scsihwid="scsihw$controller";
3470
3471     vm_deviceunplug($vmid, $conf, $scsihwid);
3472
3473     return 1;
3474 }
3475
3476 sub qemu_add_pci_bridge {
3477     my ($storecfg, $conf, $vmid, $device) = @_;
3478
3479     my $bridges = {};
3480
3481     my $bridgeid;
3482
3483     print_pci_addr($device, $bridges);
3484
3485     while (my ($k, $v) = each %$bridges) {
3486         $bridgeid = $k;
3487     }
3488     return 1 if !defined($bridgeid) || $bridgeid < 1;
3489
3490     my $bridge = "pci.$bridgeid";
3491     my $devices_list = vm_devices_list($vmid);
3492
3493     if (!defined($devices_list->{$bridge})) {
3494         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3495     }
3496
3497     return 1;
3498 }
3499
3500 sub qemu_set_link_status {
3501     my ($vmid, $device, $up) = @_;
3502
3503     vm_mon_cmd($vmid, "set_link", name => $device,
3504                up => $up ? JSON::true : JSON::false);
3505 }
3506
3507 sub qemu_netdevadd {
3508     my ($vmid, $conf, $device, $deviceid) = @_;
3509
3510     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid);
3511     my %options =  split(/[=,]/, $netdev);
3512
3513     vm_mon_cmd($vmid, "netdev_add",  %options);
3514     return 1;
3515 }
3516
3517 sub qemu_netdevdel {
3518     my ($vmid, $deviceid) = @_;
3519
3520     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3521 }
3522
3523 sub qemu_cpu_hotplug {
3524     my ($vmid, $conf, $vcpus) = @_;
3525
3526     my $sockets = 1;
3527     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3528     $sockets = $conf->{sockets} if  $conf->{sockets};
3529     my $cores = $conf->{cores} || 1;
3530     my $maxcpus = $sockets * $cores;
3531
3532     $vcpus = $maxcpus if !$vcpus;
3533
3534     die "you can't add more vcpus than maxcpus\n"
3535         if $vcpus > $maxcpus;
3536
3537     my $currentvcpus = $conf->{vcpus} || $maxcpus;
3538     die "online cpu unplug is not yet possible\n"
3539         if $vcpus < $currentvcpus;
3540
3541     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3542     die "vcpus in running vm is different than configuration\n"
3543         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
3544
3545     for (my $i = $currentvcpus; $i < $vcpus; $i++) {
3546         vm_mon_cmd($vmid, "cpu-add", id => int($i));
3547     }
3548 }
3549
3550 sub qemu_memory_hotplug {
3551     my ($vmid, $conf, $defaults, $opt, $value) = @_;
3552
3553     return $value if !check_running($vmid);
3554
3555     my $memory = $conf->{memory} || $defaults->{memory};
3556     $value = $defaults->{memory} if !$value;
3557     return $value if $value == $memory;
3558
3559     my $static_memory = $STATICMEM;
3560     my $dimm_memory = $memory - $static_memory;
3561
3562     die "memory can't be lower than $static_memory MB" if $value < $static_memory;
3563     die "memory unplug is not yet available" if $value < $memory;
3564     die "you cannot add more memory than $MAX_MEM MB!\n" if $memory > $MAX_MEM;
3565
3566
3567     my $sockets = 1;
3568     $sockets = $conf->{sockets} if $conf->{sockets};
3569
3570     foreach_dimm($conf, $vmid, $value, $sockets, sub {
3571         my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3572
3573             return if $current_size <= $conf->{memory};
3574
3575             eval { vm_mon_cmd($vmid, "object-add", 'qom-type' => "memory-backend-ram", id => "mem-$name", props => { size => int($dimm_size*1024*1024) } ) };
3576             if (my $err = $@) {
3577                 eval { qemu_objectdel($vmid, "mem-$name"); };
3578                 die $err;
3579             }
3580
3581             eval { vm_mon_cmd($vmid, "device_add", driver => "pc-dimm", id => "$name", memdev => "mem-$name", node => $numanode) };
3582             if (my $err = $@) {
3583                 eval { qemu_objectdel($vmid, "mem-$name"); };
3584                 die $err;
3585             }
3586             #update conf after each succesful module hotplug
3587             $conf->{memory} = $current_size;
3588             update_config_nolock($vmid, $conf, 1);
3589     });
3590 }
3591
3592 sub qemu_block_set_io_throttle {
3593     my ($vmid, $deviceid, $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr) = @_;
3594
3595     return if !check_running($vmid) ;
3596
3597     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid, bps => int($bps), bps_rd => int($bps_rd), bps_wr => int($bps_wr), iops => int($iops), iops_rd => int($iops_rd), iops_wr => int($iops_wr));
3598
3599 }
3600
3601 # old code, only used to shutdown old VM after update
3602 sub __read_avail {
3603     my ($fh, $timeout) = @_;
3604
3605     my $sel = new IO::Select;
3606     $sel->add($fh);
3607
3608     my $res = '';
3609     my $buf;
3610
3611     my @ready;
3612     while (scalar (@ready = $sel->can_read($timeout))) {
3613         my $count;
3614         if ($count = $fh->sysread($buf, 8192)) {
3615             if ($buf =~ /^(.*)\(qemu\) $/s) {
3616                 $res .= $1;
3617                 last;
3618             } else {
3619                 $res .= $buf;
3620             }
3621         } else {
3622             if (!defined($count)) {
3623                 die "$!\n";
3624             }
3625             last;
3626         }
3627     }
3628
3629     die "monitor read timeout\n" if !scalar(@ready);
3630
3631     return $res;
3632 }
3633
3634 # old code, only used to shutdown old VM after update
3635 sub vm_monitor_command {
3636     my ($vmid, $cmdstr, $nocheck) = @_;
3637
3638     my $res;
3639
3640     eval {
3641         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
3642
3643         my $sname = "${var_run_tmpdir}/$vmid.mon";
3644
3645         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
3646             die "unable to connect to VM $vmid socket - $!\n";
3647
3648         my $timeout = 3;
3649
3650         # hack: migrate sometime blocks the monitor (when migrate_downtime
3651         # is set)
3652         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3653             $timeout = 60*60; # 1 hour
3654         }
3655
3656         # read banner;
3657         my $data = __read_avail($sock, $timeout);
3658
3659         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
3660             die "got unexpected qemu monitor banner\n";
3661         }
3662
3663         my $sel = new IO::Select;
3664         $sel->add($sock);
3665
3666         if (!scalar(my @ready = $sel->can_write($timeout))) {
3667             die "monitor write error - timeout";
3668         }
3669
3670         my $fullcmd = "$cmdstr\r";
3671
3672         # syslog('info', "VM $vmid monitor command: $cmdstr");
3673
3674         my $b;
3675         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
3676             die "monitor write error - $!";
3677         }
3678
3679         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
3680
3681         $timeout = 20;
3682
3683         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3684             $timeout = 60*60; # 1 hour
3685         } elsif ($cmdstr =~ m/^(eject|change)/) {
3686             $timeout = 60; # note: cdrom mount command is slow
3687         }
3688         if ($res = __read_avail($sock, $timeout)) {
3689
3690             my @lines = split("\r?\n", $res);
3691
3692             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
3693
3694             $res = join("\n", @lines);
3695             $res .= "\n";
3696         }
3697     };
3698
3699     my $err = $@;
3700
3701     if ($err) {
3702         syslog("err", "VM $vmid monitor command failed - $err");
3703         die $err;
3704     }
3705
3706     return $res;
3707 }
3708
3709 sub qemu_block_resize {
3710     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
3711
3712     my $running = check_running($vmid);
3713
3714     return if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
3715
3716     return if !$running;
3717
3718     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
3719
3720 }
3721
3722 sub qemu_volume_snapshot {
3723     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3724
3725     my $running = check_running($vmid);
3726
3727     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
3728         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
3729     } else {
3730         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
3731     }
3732 }
3733
3734 sub qemu_volume_snapshot_delete {
3735     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3736
3737     my $running = check_running($vmid);
3738
3739     return if !PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
3740
3741     return if !$running;
3742
3743     vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
3744 }
3745
3746 sub set_migration_caps {
3747     my ($vmid) = @_;
3748
3749     my $cap_ref = [];
3750
3751     my $enabled_cap = {
3752         "auto-converge" => 1,
3753         "xbzrle" => 0,
3754         "x-rdma-pin-all" => 0,
3755         "zero-blocks" => 0,
3756     };
3757
3758     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
3759
3760     for my $supported_capability (@$supported_capabilities) {
3761         push @$cap_ref, {
3762             capability => $supported_capability->{capability},
3763             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
3764         };
3765     }
3766
3767     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
3768 }
3769
3770 my $fast_plug_option = {
3771     'lock' => 1,
3772     'name' => 1,
3773     'onboot' => 1,
3774     'shares' => 1,
3775     'startup' => 1,
3776 };
3777
3778 # hotplug changes in [PENDING]
3779 # $selection hash can be used to only apply specified options, for
3780 # example: { cores => 1 } (only apply changed 'cores')
3781 # $errors ref is used to return error messages
3782 sub vmconfig_hotplug_pending {
3783     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
3784
3785     my $defaults = load_defaults();
3786
3787     # commit values which do not have any impact on running VM first
3788     # Note: those option cannot raise errors, we we do not care about
3789     # $selection and always apply them.
3790
3791     my $add_error = sub {
3792         my ($opt, $msg) = @_;
3793         $errors->{$opt} = "hotplug problem - $msg";
3794     };
3795
3796     my $changes = 0;
3797     foreach my $opt (keys %{$conf->{pending}}) { # add/change
3798         if ($fast_plug_option->{$opt}) {
3799             $conf->{$opt} = $conf->{pending}->{$opt};
3800             delete $conf->{pending}->{$opt};
3801             $changes = 1;
3802         }
3803     }
3804
3805     if ($changes) {
3806         update_config_nolock($vmid, $conf, 1);
3807         $conf = load_config($vmid); # update/reload
3808     }
3809
3810     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3811
3812     my @delete = PVE::Tools::split_list($conf->{pending}->{delete});
3813     foreach my $opt (@delete) {
3814         next if $selection && !$selection->{$opt};
3815         eval {
3816             if ($opt eq 'hotplug') {
3817                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
3818             } elsif ($opt eq 'tablet') {
3819                 die "skip\n" if !$hotplug_features->{usb};
3820                 if ($defaults->{tablet}) {
3821                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
3822                 } else {
3823                     vm_deviceunplug($vmid, $conf, $opt);
3824                 }
3825             } elsif ($opt eq 'vcpus') {
3826                 die "skip\n" if !$hotplug_features->{cpu};
3827                 qemu_cpu_hotplug($vmid, $conf, undef);
3828             } elsif ($opt eq 'balloon') {
3829                 # enable balloon device is not hotpluggable
3830                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
3831             } elsif ($fast_plug_option->{$opt}) {
3832                 # do nothing
3833             } elsif ($opt =~ m/^net(\d+)$/) {
3834                 die "skip\n" if !$hotplug_features->{network};
3835                 vm_deviceunplug($vmid, $conf, $opt);
3836             } elsif (valid_drivename($opt)) {
3837                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
3838                 vm_deviceunplug($vmid, $conf, $opt);
3839                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}));
3840             } elsif ($opt =~ m/^memory$/) {
3841                 die "skip\n" if !$hotplug_features->{memory};
3842                 qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
3843             } elsif ($opt eq 'cpuunits') {
3844                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
3845             } elsif ($opt eq 'cpulimit') {
3846                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
3847             } else {
3848                 die "skip\n";
3849             }
3850         };
3851         if (my $err = $@) {
3852             &$add_error($opt, $err) if $err ne "skip\n";
3853         } else {
3854             # save new config if hotplug was successful
3855             delete $conf->{$opt};
3856             vmconfig_undelete_pending_option($conf, $opt);
3857             update_config_nolock($vmid, $conf, 1);
3858             $conf = load_config($vmid); # update/reload
3859         }
3860     }
3861
3862     foreach my $opt (keys %{$conf->{pending}}) {
3863         next if $selection && !$selection->{$opt};
3864         my $value = $conf->{pending}->{$opt};
3865         eval {
3866             if ($opt eq 'hotplug') {
3867                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
3868             } elsif ($opt eq 'tablet') {
3869                 die "skip\n" if !$hotplug_features->{usb};
3870                 if ($value == 1) {
3871                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
3872                 } elsif ($value == 0) {
3873                     vm_deviceunplug($vmid, $conf, $opt);
3874                 }
3875             } elsif ($opt eq 'vcpus') {
3876                 die "skip\n" if !$hotplug_features->{cpu};
3877                 qemu_cpu_hotplug($vmid, $conf, $value);
3878             } elsif ($opt eq 'balloon') {
3879                 # enable/disable balloning device is not hotpluggable
3880                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
3881                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
3882                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
3883
3884                 # allow manual ballooning if shares is set to zero
3885                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
3886                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
3887                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
3888                 }
3889             } elsif ($opt =~ m/^net(\d+)$/) {
3890                 # some changes can be done without hotplug
3891                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
3892                                     $vmid, $opt, $value);
3893             } elsif (valid_drivename($opt)) {
3894                 # some changes can be done without hotplug
3895                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
3896                                      $vmid, $opt, $value, 1);
3897             } elsif ($opt =~ m/^memory$/) { #dimms
3898                 die "skip\n" if !$hotplug_features->{memory};
3899                 $value = qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
3900             } elsif ($opt eq 'cpuunits') {
3901                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
3902             } elsif ($opt eq 'cpulimit') {
3903                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
3904                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
3905             } else {
3906                 die "skip\n";  # skip non-hot-pluggable options
3907             }
3908         };
3909         if (my $err = $@) {
3910             &$add_error($opt, $err) if $err ne "skip\n";
3911         } else {
3912             # save new config if hotplug was successful
3913             $conf->{$opt} = $value;
3914             delete $conf->{pending}->{$opt};
3915             update_config_nolock($vmid, $conf, 1);
3916             $conf = load_config($vmid); # update/reload
3917         }
3918     }
3919 }
3920
3921 sub vmconfig_apply_pending {
3922     my ($vmid, $conf, $storecfg) = @_;
3923
3924     # cold plug
3925
3926     my @delete = PVE::Tools::split_list($conf->{pending}->{delete});
3927     foreach my $opt (@delete) { # delete
3928         die "internal error" if $opt =~ m/^unused/;
3929         $conf = load_config($vmid); # update/reload
3930         if (!defined($conf->{$opt})) {
3931             vmconfig_undelete_pending_option($conf, $opt);
3932             update_config_nolock($vmid, $conf, 1);
3933         } elsif (valid_drivename($opt)) {
3934             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}));
3935             vmconfig_undelete_pending_option($conf, $opt);
3936             delete $conf->{$opt};
3937             update_config_nolock($vmid, $conf, 1);
3938         } else {
3939             vmconfig_undelete_pending_option($conf, $opt);
3940             delete $conf->{$opt};
3941             update_config_nolock($vmid, $conf, 1);
3942         }
3943     }
3944
3945     $conf = load_config($vmid); # update/reload
3946
3947     foreach my $opt (keys %{$conf->{pending}}) { # add/change
3948         $conf = load_config($vmid); # update/reload
3949
3950         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
3951             # skip if nothing changed
3952         } elsif (valid_drivename($opt)) {
3953             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
3954                 if defined($conf->{$opt});
3955             $conf->{$opt} = $conf->{pending}->{$opt};
3956         } else {
3957             $conf->{$opt} = $conf->{pending}->{$opt};
3958         }
3959
3960         delete $conf->{pending}->{$opt};
3961         update_config_nolock($vmid, $conf, 1);
3962     }
3963 }
3964
3965 my $safe_num_ne = sub {
3966     my ($a, $b) = @_;
3967
3968     return 0 if !defined($a) && !defined($b);
3969     return 1 if !defined($a);
3970     return 1 if !defined($b);
3971
3972     return $a != $b;
3973 };
3974
3975 my $safe_string_ne = sub {
3976     my ($a, $b) = @_;
3977
3978     return 0 if !defined($a) && !defined($b);
3979     return 1 if !defined($a);
3980     return 1 if !defined($b);
3981
3982     return $a ne $b;
3983 };
3984
3985 sub vmconfig_update_net {
3986     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
3987
3988     my $newnet = parse_net($value);
3989
3990     if ($conf->{$opt}) {
3991         my $oldnet = parse_net($conf->{$opt});
3992
3993         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
3994             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
3995             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
3996             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
3997
3998             # for non online change, we try to hot-unplug
3999             die "skip\n" if !$hotplug;
4000             vm_deviceunplug($vmid, $conf, $opt);
4001         } else {
4002
4003             die "internal error" if $opt !~ m/net(\d+)/;
4004             my $iface = "tap${vmid}i$1";
4005
4006             if (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4007                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4008             }
4009
4010             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4011                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4012                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4013                 PVE::Network::tap_unplug($iface);
4014                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall});
4015             }
4016
4017             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4018                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4019             }
4020
4021             return 1;
4022         }
4023     }
4024
4025     if ($hotplug) {
4026         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4027     } else {
4028         die "skip\n";
4029     }
4030 }
4031
4032 sub vmconfig_update_disk {
4033     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4034
4035     # fixme: do we need force?
4036
4037     my $drive = parse_drive($opt, $value);
4038
4039     if ($conf->{$opt}) {
4040
4041         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4042
4043             my $media = $drive->{media} || 'disk';
4044             my $oldmedia = $old_drive->{media} || 'disk';
4045             die "unable to change media type\n" if $media ne $oldmedia;
4046
4047             if (!drive_is_cdrom($old_drive)) {
4048
4049                 if ($drive->{file} ne $old_drive->{file}) {
4050
4051                     die "skip\n" if !$hotplug;
4052
4053                     # unplug and register as unused
4054                     vm_deviceunplug($vmid, $conf, $opt);
4055                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4056
4057                 } else {
4058                     # update existing disk
4059
4060                     # skip non hotpluggable value
4061                     if (&$safe_num_ne($drive->{discard}, $old_drive->{discard}) ||
4062                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4063                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4064                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4065                         die "skip\n";
4066                     }
4067
4068                     # apply throttle
4069                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4070                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4071                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4072                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4073                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4074                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4075                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4076                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4077                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4078                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4079                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4080                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max})) {
4081
4082                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4083                                                    ($drive->{mbps} || 0)*1024*1024,
4084                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4085                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4086                                                    $drive->{iops} || 0,
4087                                                    $drive->{iops_rd} || 0,
4088                                                    $drive->{iops_wr} || 0,
4089                                                    ($drive->{mbps_max} || 0)*1024*1024,
4090                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4091                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4092                                                    $drive->{iops_max} || 0,
4093                                                    $drive->{iops_rd_max} || 0,
4094                                                    $drive->{iops_wr_max} || 0);
4095
4096                     }
4097
4098                     return 1;
4099                 }
4100
4101             } else { # cdrom
4102
4103                 if ($drive->{file} eq 'none') {
4104                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4105                 } else {
4106                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4107                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4108                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4109                 }
4110
4111                 return 1;
4112             }
4113         }
4114     }
4115
4116     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4117     # hotplug new disks
4118     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4119 }
4120
4121 sub vm_start {
4122     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused, $forcemachine, $spice_ticket) = @_;
4123
4124     lock_config($vmid, sub {
4125         my $conf = load_config($vmid, $migratedfrom);
4126
4127         die "you can't start a vm if it's a template\n" if is_template($conf);
4128
4129         check_lock($conf) if !$skiplock;
4130
4131         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4132
4133         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4134             vmconfig_apply_pending($vmid, $conf, $storecfg);
4135             $conf = load_config($vmid); # update/reload
4136         }
4137
4138         my $defaults = load_defaults();
4139
4140         # set environment variable useful inside network script
4141         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4142
4143         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4144
4145         my $migrate_port = 0;
4146         my $migrate_uri;
4147         if ($statefile) {
4148             if ($statefile eq 'tcp') {
4149                 my $localip = "localhost";
4150                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4151                 my $nodename = PVE::INotify::nodename();
4152                 if ($datacenterconf->{migration_unsecure}) {
4153                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4154                 }
4155                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4156                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4157                 $migrate_uri = "tcp:[${localip}]:${migrate_port}";
4158                 push @$cmd, '-incoming', $migrate_uri;
4159                 push @$cmd, '-S';
4160             } else {
4161                 push @$cmd, '-loadstate', $statefile;
4162             }
4163         } elsif ($paused) {
4164             push @$cmd, '-S';
4165         }
4166
4167         # host pci devices
4168         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4169           my $d = parse_hostpci($conf->{"hostpci$i"});
4170           next if !$d;
4171           my $pcidevices = $d->{pciid};
4172           foreach my $pcidevice (@$pcidevices) {
4173                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4174
4175                 my $info = pci_device_info("0000:$pciid");
4176                 die "IOMMU not present\n" if !check_iommu_support();
4177                 die "no pci device info for device '$pciid'\n" if !$info;
4178
4179                 if ($d->{driver} && $d->{driver} eq "vfio") {
4180                     die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4181                 } else {
4182                     die "can't unbind/bind to stub pci device '$pciid'\n" if !pci_dev_bind_to_stub($info);
4183                 }
4184
4185                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4186           }
4187         }
4188
4189         PVE::Storage::activate_volumes($storecfg, $vollist);
4190
4191         eval  { run_command($cmd, timeout => $statefile ? undef : 30,
4192                     umask => 0077); };
4193         my $err = $@;
4194         die "start failed: $err" if $err;
4195
4196         print "migration listens on $migrate_uri\n" if $migrate_uri;
4197
4198         if ($statefile && $statefile ne 'tcp')  {
4199             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4200             warn $@ if $@;
4201         }
4202
4203         if ($migratedfrom) {
4204
4205             eval {
4206                 set_migration_caps($vmid);
4207             };
4208             warn $@ if $@;
4209
4210             if ($spice_port) {
4211                 print "spice listens on port $spice_port\n";
4212                 if ($spice_ticket) {
4213                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4214                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4215                 }
4216             }
4217
4218         } else {
4219
4220             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4221                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4222                     if $conf->{balloon};
4223             }
4224
4225             foreach my $opt (keys %$conf) {
4226                 next if $opt !~  m/^net\d+$/;
4227                 my $nicconf = parse_net($conf->{$opt});
4228                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4229             }
4230         }
4231
4232         vm_mon_cmd_nocheck($vmid, 'qom-set',
4233                     path => "machine/peripheral/balloon0",
4234                     property => "guest-stats-polling-interval",
4235                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4236
4237     });
4238 }
4239
4240 sub vm_mon_cmd {
4241     my ($vmid, $execute, %params) = @_;
4242
4243     my $cmd = { execute => $execute, arguments => \%params };
4244     vm_qmp_command($vmid, $cmd);
4245 }
4246
4247 sub vm_mon_cmd_nocheck {
4248     my ($vmid, $execute, %params) = @_;
4249
4250     my $cmd = { execute => $execute, arguments => \%params };
4251     vm_qmp_command($vmid, $cmd, 1);
4252 }
4253
4254 sub vm_qmp_command {
4255     my ($vmid, $cmd, $nocheck) = @_;
4256
4257     my $res;
4258
4259     my $timeout;
4260     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4261         $timeout = $cmd->{arguments}->{timeout};
4262         delete $cmd->{arguments}->{timeout};
4263     }
4264
4265     eval {
4266         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4267         my $sname = qmp_socket($vmid);
4268         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4269             my $qmpclient = PVE::QMPClient->new();
4270
4271             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4272         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
4273             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
4274                 if scalar(%{$cmd->{arguments}});
4275             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
4276         } else {
4277             die "unable to open monitor socket\n";
4278         }
4279     };
4280     if (my $err = $@) {
4281         syslog("err", "VM $vmid qmp command failed - $err");
4282         die $err;
4283     }
4284
4285     return $res;
4286 }
4287
4288 sub vm_human_monitor_command {
4289     my ($vmid, $cmdline) = @_;
4290
4291     my $res;
4292
4293     my $cmd = {
4294         execute => 'human-monitor-command',
4295         arguments => { 'command-line' => $cmdline},
4296     };
4297
4298     return vm_qmp_command($vmid, $cmd);
4299 }
4300
4301 sub vm_commandline {
4302     my ($storecfg, $vmid) = @_;
4303
4304     my $conf = load_config($vmid);
4305
4306     my $defaults = load_defaults();
4307
4308     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
4309
4310     return join(' ', @$cmd);
4311 }
4312
4313 sub vm_reset {
4314     my ($vmid, $skiplock) = @_;
4315
4316     lock_config($vmid, sub {
4317
4318         my $conf = load_config($vmid);
4319
4320         check_lock($conf) if !$skiplock;
4321
4322         vm_mon_cmd($vmid, "system_reset");
4323     });
4324 }
4325
4326 sub get_vm_volumes {
4327     my ($conf) = @_;
4328
4329     my $vollist = [];
4330     foreach_volid($conf, sub {
4331         my ($volid, $is_cdrom) = @_;
4332
4333         return if $volid =~ m|^/|;
4334
4335         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4336         return if !$sid;
4337
4338         push @$vollist, $volid;
4339     });
4340
4341     return $vollist;
4342 }
4343
4344 sub vm_stop_cleanup {
4345     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
4346
4347     eval {
4348
4349         if (!$keepActive) {
4350             my $vollist = get_vm_volumes($conf);
4351             PVE::Storage::deactivate_volumes($storecfg, $vollist);
4352         }
4353
4354         foreach my $ext (qw(mon qmp pid vnc qga)) {
4355             unlink "/var/run/qemu-server/${vmid}.$ext";
4356         }
4357
4358         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
4359     };
4360     warn $@ if $@; # avoid errors - just warn
4361 }
4362
4363 # Note: use $nockeck to skip tests if VM configuration file exists.
4364 # We need that when migration VMs to other nodes (files already moved)
4365 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
4366 sub vm_stop {
4367     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
4368
4369     $force = 1 if !defined($force) && !$shutdown;
4370
4371     if ($migratedfrom){
4372         my $pid = check_running($vmid, $nocheck, $migratedfrom);
4373         kill 15, $pid if $pid;
4374         my $conf = load_config($vmid, $migratedfrom);
4375         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
4376         return;
4377     }
4378
4379     lock_config($vmid, sub {
4380
4381         my $pid = check_running($vmid, $nocheck);
4382         return if !$pid;
4383
4384         my $conf;
4385         if (!$nocheck) {
4386             $conf = load_config($vmid);
4387             check_lock($conf) if !$skiplock;
4388             if (!defined($timeout) && $shutdown && $conf->{startup}) {
4389                 my $opts = parse_startup($conf->{startup});
4390                 $timeout = $opts->{down} if $opts->{down};
4391             }
4392         }
4393
4394         $timeout = 60 if !defined($timeout);
4395
4396         eval {
4397             if ($shutdown) {
4398                 if (defined($conf) && $conf->{agent}) {
4399                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
4400                 } else {
4401                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
4402                 }
4403             } else {
4404                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
4405             }
4406         };
4407         my $err = $@;
4408
4409         if (!$err) {
4410             my $count = 0;
4411             while (($count < $timeout) && check_running($vmid, $nocheck)) {
4412                 $count++;
4413                 sleep 1;
4414             }
4415
4416             if ($count >= $timeout) {
4417                 if ($force) {
4418                     warn "VM still running - terminating now with SIGTERM\n";
4419                     kill 15, $pid;
4420                 } else {
4421                     die "VM quit/powerdown failed - got timeout\n";
4422                 }
4423             } else {
4424                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4425                 return;
4426             }
4427         } else {
4428             if ($force) {
4429                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
4430                 kill 15, $pid;
4431             } else {
4432                 die "VM quit/powerdown failed\n";
4433             }
4434         }
4435
4436         # wait again
4437         $timeout = 10;
4438
4439         my $count = 0;
4440         while (($count < $timeout) && check_running($vmid, $nocheck)) {
4441             $count++;
4442             sleep 1;
4443         }
4444
4445         if ($count >= $timeout) {
4446             warn "VM still running - terminating now with SIGKILL\n";
4447             kill 9, $pid;
4448             sleep 1;
4449         }
4450
4451         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4452    });
4453 }
4454
4455 sub vm_suspend {
4456     my ($vmid, $skiplock) = @_;
4457
4458     lock_config($vmid, sub {
4459
4460         my $conf = load_config($vmid);
4461
4462         check_lock($conf) if !($skiplock || ($conf->{lock} && $conf->{lock} eq 'backup'));
4463
4464         vm_mon_cmd($vmid, "stop");
4465     });
4466 }
4467
4468 sub vm_resume {
4469     my ($vmid, $skiplock) = @_;
4470
4471     lock_config($vmid, sub {
4472
4473         my $conf = load_config($vmid);
4474
4475         check_lock($conf) if !($skiplock || ($conf->{lock} && $conf->{lock} eq 'backup'));
4476
4477         vm_mon_cmd($vmid, "cont");
4478     });
4479 }
4480
4481 sub vm_sendkey {
4482     my ($vmid, $skiplock, $key) = @_;
4483
4484     lock_config($vmid, sub {
4485
4486         my $conf = load_config($vmid);
4487
4488         # there is no qmp command, so we use the human monitor command
4489         vm_human_monitor_command($vmid, "sendkey $key");
4490     });
4491 }
4492
4493 sub vm_destroy {
4494     my ($storecfg, $vmid, $skiplock) = @_;
4495
4496     lock_config($vmid, sub {
4497
4498         my $conf = load_config($vmid);
4499
4500         check_lock($conf) if !$skiplock;
4501
4502         if (!check_running($vmid)) {
4503             destroy_vm($storecfg, $vmid);
4504         } else {
4505             die "VM $vmid is running - destroy failed\n";
4506         }
4507     });
4508 }
4509
4510 # pci helpers
4511
4512 sub file_write {
4513     my ($filename, $buf) = @_;
4514
4515     my $fh = IO::File->new($filename, "w");
4516     return undef if !$fh;
4517
4518     my $res = print $fh $buf;
4519
4520     $fh->close();
4521
4522     return $res;
4523 }
4524
4525 sub pci_device_info {
4526     my ($name) = @_;
4527
4528     my $res;
4529
4530     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
4531     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
4532
4533     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
4534     return undef if !defined($irq) || $irq !~ m/^\d+$/;
4535
4536     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
4537     return undef if !defined($vendor) || $vendor !~ s/^0x//;
4538
4539     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
4540     return undef if !defined($product) || $product !~ s/^0x//;
4541
4542     $res = {
4543         name => $name,
4544         vendor => $vendor,
4545         product => $product,
4546         domain => $domain,
4547         bus => $bus,
4548         slot => $slot,
4549         func => $func,
4550         irq => $irq,
4551         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
4552     };
4553
4554     return $res;
4555 }
4556
4557 sub pci_dev_reset {
4558     my ($dev) = @_;
4559
4560     my $name = $dev->{name};
4561
4562     my $fn = "$pcisysfs/devices/$name/reset";
4563
4564     return file_write($fn, "1");
4565 }
4566
4567 sub pci_dev_bind_to_stub {
4568     my ($dev) = @_;
4569
4570     my $name = $dev->{name};
4571
4572     my $testdir = "$pcisysfs/drivers/pci-stub/$name";
4573     return 1 if -d $testdir;
4574
4575     my $data = "$dev->{vendor} $dev->{product}";
4576     return undef if !file_write("$pcisysfs/drivers/pci-stub/new_id", $data);
4577
4578     my $fn = "$pcisysfs/devices/$name/driver/unbind";
4579     if (!file_write($fn, $name)) {
4580         return undef if -f $fn;
4581     }
4582
4583     $fn = "$pcisysfs/drivers/pci-stub/bind";
4584     if (! -d $testdir) {
4585         return undef if !file_write($fn, $name);
4586     }
4587
4588     return -d $testdir;
4589 }
4590
4591 sub pci_dev_bind_to_vfio {
4592     my ($dev) = @_;
4593
4594     my $name = $dev->{name};
4595
4596     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4597
4598     if (!-d $vfio_basedir) {
4599         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4600     }
4601     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4602
4603     my $testdir = "$vfio_basedir/$name";
4604     return 1 if -d $testdir;
4605
4606     my $data = "$dev->{vendor} $dev->{product}";
4607     return undef if !file_write("$vfio_basedir/new_id", $data);
4608
4609     my $fn = "$pcisysfs/devices/$name/driver/unbind";
4610     if (!file_write($fn, $name)) {
4611         return undef if -f $fn;
4612     }
4613
4614     $fn = "$vfio_basedir/bind";
4615     if (! -d $testdir) {
4616         return undef if !file_write($fn, $name);
4617     }
4618
4619     return -d $testdir;
4620 }
4621
4622 sub pci_dev_group_bind_to_vfio {
4623     my ($pciid) = @_;
4624
4625     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4626
4627     if (!-d $vfio_basedir) {
4628         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4629     }
4630     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4631
4632     # get IOMMU group devices
4633     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
4634       my @devs = grep /^0000:/, readdir($D);
4635     closedir($D);
4636
4637     foreach my $pciid (@devs) {
4638         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
4639
4640         # pci bridges, switches or root ports are not supported
4641         # they have a pci_bus subdirectory so skip them
4642         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
4643
4644         my $info = pci_device_info($1);
4645         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
4646     }
4647
4648     return 1;
4649 }
4650
4651 sub print_pci_addr {
4652     my ($id, $bridges) = @_;
4653
4654     my $res = '';
4655     my $devices = {
4656         piix3 => { bus => 0, addr => 1 },
4657         #addr2 : first videocard
4658         balloon0 => { bus => 0, addr => 3 },
4659         watchdog => { bus => 0, addr => 4 },
4660         scsihw0 => { bus => 0, addr => 5 },
4661         'pci.3' => { bus => 0, addr => 5 }, #can also be used for virtio-scsi-single bridge
4662         scsihw1 => { bus => 0, addr => 6 },
4663         ahci0 => { bus => 0, addr => 7 },
4664         qga0 => { bus => 0, addr => 8 },
4665         spice => { bus => 0, addr => 9 },
4666         virtio0 => { bus => 0, addr => 10 },
4667         virtio1 => { bus => 0, addr => 11 },
4668         virtio2 => { bus => 0, addr => 12 },
4669         virtio3 => { bus => 0, addr => 13 },
4670         virtio4 => { bus => 0, addr => 14 },
4671         virtio5 => { bus => 0, addr => 15 },
4672         hostpci0 => { bus => 0, addr => 16 },
4673         hostpci1 => { bus => 0, addr => 17 },
4674         net0 => { bus => 0, addr => 18 },
4675         net1 => { bus => 0, addr => 19 },
4676         net2 => { bus => 0, addr => 20 },
4677         net3 => { bus => 0, addr => 21 },
4678         net4 => { bus => 0, addr => 22 },
4679         net5 => { bus => 0, addr => 23 },
4680         vga1 => { bus => 0, addr => 24 },
4681         vga2 => { bus => 0, addr => 25 },
4682         vga3 => { bus => 0, addr => 26 },
4683         hostpci2 => { bus => 0, addr => 27 },
4684         hostpci3 => { bus => 0, addr => 28 },
4685         #addr29 : usb-host (pve-usb.cfg)
4686         'pci.1' => { bus => 0, addr => 30 },
4687         'pci.2' => { bus => 0, addr => 31 },
4688         'net6' => { bus => 1, addr => 1 },
4689         'net7' => { bus => 1, addr => 2 },
4690         'net8' => { bus => 1, addr => 3 },
4691         'net9' => { bus => 1, addr => 4 },
4692         'net10' => { bus => 1, addr => 5 },
4693         'net11' => { bus => 1, addr => 6 },
4694         'net12' => { bus => 1, addr => 7 },
4695         'net13' => { bus => 1, addr => 8 },
4696         'net14' => { bus => 1, addr => 9 },
4697         'net15' => { bus => 1, addr => 10 },
4698         'net16' => { bus => 1, addr => 11 },
4699         'net17' => { bus => 1, addr => 12 },
4700         'net18' => { bus => 1, addr => 13 },
4701         'net19' => { bus => 1, addr => 14 },
4702         'net20' => { bus => 1, addr => 15 },
4703         'net21' => { bus => 1, addr => 16 },
4704         'net22' => { bus => 1, addr => 17 },
4705         'net23' => { bus => 1, addr => 18 },
4706         'net24' => { bus => 1, addr => 19 },
4707         'net25' => { bus => 1, addr => 20 },
4708         'net26' => { bus => 1, addr => 21 },
4709         'net27' => { bus => 1, addr => 22 },
4710         'net28' => { bus => 1, addr => 23 },
4711         'net29' => { bus => 1, addr => 24 },
4712         'net30' => { bus => 1, addr => 25 },
4713         'net31' => { bus => 1, addr => 26 },
4714         'virtio6' => { bus => 2, addr => 1 },
4715         'virtio7' => { bus => 2, addr => 2 },
4716         'virtio8' => { bus => 2, addr => 3 },
4717         'virtio9' => { bus => 2, addr => 4 },
4718         'virtio10' => { bus => 2, addr => 5 },
4719         'virtio11' => { bus => 2, addr => 6 },
4720         'virtio12' => { bus => 2, addr => 7 },
4721         'virtio13' => { bus => 2, addr => 8 },
4722         'virtio14' => { bus => 2, addr => 9 },
4723         'virtio15' => { bus => 2, addr => 10 },
4724         'virtioscsi0' => { bus => 3, addr => 1 },
4725         'virtioscsi1' => { bus => 3, addr => 2 },
4726         'virtioscsi2' => { bus => 3, addr => 3 },
4727         'virtioscsi3' => { bus => 3, addr => 4 },
4728         'virtioscsi4' => { bus => 3, addr => 5 },
4729         'virtioscsi5' => { bus => 3, addr => 6 },
4730         'virtioscsi6' => { bus => 3, addr => 7 },
4731         'virtioscsi7' => { bus => 3, addr => 8 },
4732         'virtioscsi8' => { bus => 3, addr => 9 },
4733         'virtioscsi9' => { bus => 3, addr => 10 },
4734         'virtioscsi10' => { bus => 3, addr => 11 },
4735         'virtioscsi11' => { bus => 3, addr => 12 },
4736         'virtioscsi12' => { bus => 3, addr => 13 },
4737         'virtioscsi13' => { bus => 3, addr => 14 },
4738         'virtioscsi14' => { bus => 3, addr => 15 },
4739         'virtioscsi15' => { bus => 3, addr => 16 },
4740         'virtioscsi16' => { bus => 3, addr => 17 },
4741         'virtioscsi17' => { bus => 3, addr => 18 },
4742         'virtioscsi18' => { bus => 3, addr => 19 },
4743         'virtioscsi19' => { bus => 3, addr => 20 },
4744         'virtioscsi20' => { bus => 3, addr => 21 },
4745         'virtioscsi21' => { bus => 3, addr => 22 },
4746         'virtioscsi22' => { bus => 3, addr => 23 },
4747         'virtioscsi23' => { bus => 3, addr => 24 },
4748         'virtioscsi24' => { bus => 3, addr => 25 },
4749         'virtioscsi25' => { bus => 3, addr => 26 },
4750         'virtioscsi26' => { bus => 3, addr => 27 },
4751         'virtioscsi27' => { bus => 3, addr => 28 },
4752         'virtioscsi28' => { bus => 3, addr => 29 },
4753         'virtioscsi29' => { bus => 3, addr => 30 },
4754         'virtioscsi30' => { bus => 3, addr => 31 },
4755
4756     };
4757
4758     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
4759            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
4760            my $bus = $devices->{$id}->{bus};
4761            $res = ",bus=pci.$bus,addr=$addr";
4762            $bridges->{$bus} = 1 if $bridges;
4763     }
4764     return $res;
4765
4766 }
4767
4768 sub print_pcie_addr {
4769     my ($id) = @_;
4770
4771     my $res = '';
4772     my $devices = {
4773         hostpci0 => { bus => "ich9-pcie-port-1", addr => 0 },
4774         hostpci1 => { bus => "ich9-pcie-port-2", addr => 0 },
4775         hostpci2 => { bus => "ich9-pcie-port-3", addr => 0 },
4776         hostpci3 => { bus => "ich9-pcie-port-4", addr => 0 },
4777     };
4778
4779     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
4780            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
4781            my $bus = $devices->{$id}->{bus};
4782            $res = ",bus=$bus,addr=$addr";
4783     }
4784     return $res;
4785
4786 }
4787
4788 # vzdump restore implementaion
4789
4790 sub tar_archive_read_firstfile {
4791     my $archive = shift;
4792
4793     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
4794
4795     # try to detect archive type first
4796     my $pid = open (TMP, "tar tf '$archive'|") ||
4797         die "unable to open file '$archive'\n";
4798     my $firstfile = <TMP>;
4799     kill 15, $pid;
4800     close TMP;
4801
4802     die "ERROR: archive contaions no data\n" if !$firstfile;
4803     chomp $firstfile;
4804
4805     return $firstfile;
4806 }
4807
4808 sub tar_restore_cleanup {
4809     my ($storecfg, $statfile) = @_;
4810
4811     print STDERR "starting cleanup\n";
4812
4813     if (my $fd = IO::File->new($statfile, "r")) {
4814         while (defined(my $line = <$fd>)) {
4815             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
4816                 my $volid = $2;
4817                 eval {
4818                     if ($volid =~ m|^/|) {
4819                         unlink $volid || die 'unlink failed\n';
4820                     } else {
4821                         PVE::Storage::vdisk_free($storecfg, $volid);
4822                     }
4823                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
4824                 };
4825                 print STDERR "unable to cleanup '$volid' - $@" if $@;
4826             } else {
4827                 print STDERR "unable to parse line in statfile - $line";
4828             }
4829         }
4830         $fd->close();
4831     }
4832 }
4833
4834 sub restore_archive {
4835     my ($archive, $vmid, $user, $opts) = @_;
4836
4837     my $format = $opts->{format};
4838     my $comp;
4839
4840     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
4841         $format = 'tar' if !$format;
4842         $comp = 'gzip';
4843     } elsif ($archive =~ m/\.tar$/) {
4844         $format = 'tar' if !$format;
4845     } elsif ($archive =~ m/.tar.lzo$/) {
4846         $format = 'tar' if !$format;
4847         $comp = 'lzop';
4848     } elsif ($archive =~ m/\.vma$/) {
4849         $format = 'vma' if !$format;
4850     } elsif ($archive =~ m/\.vma\.gz$/) {
4851         $format = 'vma' if !$format;
4852         $comp = 'gzip';
4853     } elsif ($archive =~ m/\.vma\.lzo$/) {
4854         $format = 'vma' if !$format;
4855         $comp = 'lzop';
4856     } else {
4857         $format = 'vma' if !$format; # default
4858     }
4859
4860     # try to detect archive format
4861     if ($format eq 'tar') {
4862         return restore_tar_archive($archive, $vmid, $user, $opts);
4863     } else {
4864         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
4865     }
4866 }
4867
4868 sub restore_update_config_line {
4869     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
4870
4871     return if $line =~ m/^\#qmdump\#/;
4872     return if $line =~ m/^\#vzdump\#/;
4873     return if $line =~ m/^lock:/;
4874     return if $line =~ m/^unused\d+:/;
4875     return if $line =~ m/^parent:/;
4876     return if $line =~ m/^template:/; # restored VM is never a template
4877
4878     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
4879         # try to convert old 1.X settings
4880         my ($id, $ind, $ethcfg) = ($1, $2, $3);
4881         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
4882             my ($model, $macaddr) = split(/\=/, $devconfig);
4883             $macaddr = PVE::Tools::random_ether_addr() if !$macaddr || $unique;
4884             my $net = {
4885                 model => $model,
4886                 bridge => "vmbr$ind",
4887                 macaddr => $macaddr,
4888             };
4889             my $netstr = print_net($net);
4890
4891             print $outfd "net$cookie->{netcount}: $netstr\n";
4892             $cookie->{netcount}++;
4893         }
4894     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
4895         my ($id, $netstr) = ($1, $2);
4896         my $net = parse_net($netstr);
4897         $net->{macaddr} = PVE::Tools::random_ether_addr() if $net->{macaddr};
4898         $netstr = print_net($net);
4899         print $outfd "$id: $netstr\n";
4900     } elsif ($line =~ m/^((ide|scsi|virtio|sata)\d+):\s*(\S+)\s*$/) {
4901         my $virtdev = $1;
4902         my $value = $3;
4903         if ($line =~ m/backup=no/) {
4904             print $outfd "#$line";
4905         } elsif ($virtdev && $map->{$virtdev}) {
4906             my $di = parse_drive($virtdev, $value);
4907             delete $di->{format}; # format can change on restore
4908             $di->{file} = $map->{$virtdev};
4909             $value = print_drive($vmid, $di);
4910             print $outfd "$virtdev: $value\n";
4911         } else {
4912             print $outfd $line;
4913         }
4914     } else {
4915         print $outfd $line;
4916     }
4917 }
4918
4919 sub scan_volids {
4920     my ($cfg, $vmid) = @_;
4921
4922     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
4923
4924     my $volid_hash = {};
4925     foreach my $storeid (keys %$info) {
4926         foreach my $item (@{$info->{$storeid}}) {
4927             next if !($item->{volid} && $item->{size});
4928             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
4929             $volid_hash->{$item->{volid}} = $item;
4930         }
4931     }
4932
4933     return $volid_hash;
4934 }
4935
4936 sub get_used_paths {
4937     my ($vmid, $storecfg, $conf, $scan_snapshots, $skip_drive) = @_;
4938
4939     my $used_path = {};
4940
4941     my $scan_config = sub {
4942         my ($cref, $snapname) = @_;
4943
4944         foreach my $key (keys %$cref) {
4945             my $value = $cref->{$key};
4946             if (valid_drivename($key)) {
4947                 next if $skip_drive && $key eq $skip_drive;
4948                 my $drive = parse_drive($key, $value);
4949                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
4950                 if ($drive->{file} =~ m!^/!) {
4951                     $used_path->{$drive->{file}}++; # = 1;
4952                 } else {
4953                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
4954                     next if !$storeid;
4955                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
4956                     next if !$scfg;
4957                     my $path = PVE::Storage::path($storecfg, $drive->{file}, $snapname);
4958                     $used_path->{$path}++; # = 1;
4959                 }
4960             }
4961         }
4962     };
4963
4964     &$scan_config($conf);
4965
4966     undef $skip_drive;
4967
4968     if ($scan_snapshots) {
4969         foreach my $snapname (keys %{$conf->{snapshots}}) {
4970             &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
4971         }
4972     }
4973
4974     return $used_path;
4975 }
4976
4977 sub update_disksize {
4978     my ($vmid, $conf, $volid_hash) = @_;
4979
4980     my $changes;
4981
4982     my $used = {};
4983
4984     # Note: it is allowed to define multiple storages with same path (alias), so
4985     # we need to check both 'volid' and real 'path' (two different volid can point
4986     # to the same path).
4987
4988     my $usedpath = {};
4989
4990     # update size info
4991     foreach my $opt (keys %$conf) {
4992         if (valid_drivename($opt)) {
4993             my $drive = parse_drive($opt, $conf->{$opt});
4994             my $volid = $drive->{file};
4995             next if !$volid;
4996
4997             $used->{$volid} = 1;
4998             if ($volid_hash->{$volid} &&
4999                 (my $path = $volid_hash->{$volid}->{path})) {
5000                 $usedpath->{$path} = 1;
5001             }
5002
5003             next if drive_is_cdrom($drive);
5004             next if !$volid_hash->{$volid};
5005
5006             $drive->{size} = $volid_hash->{$volid}->{size};
5007             my $new = print_drive($vmid, $drive);
5008             if ($new ne $conf->{$opt}) {
5009                 $changes = 1;
5010                 $conf->{$opt} = $new;
5011             }
5012         }
5013     }
5014
5015     # remove 'unusedX' entry if volume is used
5016     foreach my $opt (keys %$conf) {
5017         next if $opt !~ m/^unused\d+$/;
5018         my $volid = $conf->{$opt};
5019         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5020         if ($used->{$volid} || ($path && $usedpath->{$path})) {
5021             $changes = 1;
5022             delete $conf->{$opt};
5023         }
5024     }
5025
5026     foreach my $volid (sort keys %$volid_hash) {
5027         next if $volid =~ m/vm-$vmid-state-/;
5028         next if $used->{$volid};
5029         my $path = $volid_hash->{$volid}->{path};
5030         next if !$path; # just to be sure
5031         next if $usedpath->{$path};
5032         $changes = 1;
5033         add_unused_volume($conf, $volid);
5034         $usedpath->{$path} = 1; # avoid to add more than once (aliases)
5035     }
5036
5037     return $changes;
5038 }
5039
5040 sub rescan {
5041     my ($vmid, $nolock) = @_;
5042
5043     my $cfg = PVE::Cluster::cfs_read_file("storage.cfg");
5044
5045     my $volid_hash = scan_volids($cfg, $vmid);
5046
5047     my $updatefn =  sub {
5048         my ($vmid) = @_;
5049
5050         my $conf = load_config($vmid);
5051
5052         check_lock($conf);
5053
5054         my $vm_volids = {};
5055         foreach my $volid (keys %$volid_hash) {
5056             my $info = $volid_hash->{$volid};
5057             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5058         }
5059
5060         my $changes = update_disksize($vmid, $conf, $vm_volids);
5061
5062         update_config_nolock($vmid, $conf, 1) if $changes;
5063     };
5064
5065     if (defined($vmid)) {
5066         if ($nolock) {
5067             &$updatefn($vmid);
5068         } else {
5069             lock_config($vmid, $updatefn, $vmid);
5070         }
5071     } else {
5072         my $vmlist = config_list();
5073         foreach my $vmid (keys %$vmlist) {
5074             if ($nolock) {
5075                 &$updatefn($vmid);
5076             } else {
5077                 lock_config($vmid, $updatefn, $vmid);
5078             }
5079         }
5080     }
5081 }
5082
5083 sub restore_vma_archive {
5084     my ($archive, $vmid, $user, $opts, $comp) = @_;
5085
5086     my $input = $archive eq '-' ? "<&STDIN" : undef;
5087     my $readfrom = $archive;
5088
5089     my $uncomp = '';
5090     if ($comp) {
5091         $readfrom = '-';
5092         my $qarchive = PVE::Tools::shellquote($archive);
5093         if ($comp eq 'gzip') {
5094             $uncomp = "zcat $qarchive|";
5095         } elsif ($comp eq 'lzop') {
5096             $uncomp = "lzop -d -c $qarchive|";
5097         } else {
5098             die "unknown compression method '$comp'\n";
5099         }
5100
5101     }
5102
5103     my $tmpdir = "/var/tmp/vzdumptmp$$";
5104     rmtree $tmpdir;
5105
5106     # disable interrupts (always do cleanups)
5107     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5108         warn "got interrupt - ignored\n";
5109     };
5110
5111     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5112     POSIX::mkfifo($mapfifo, 0600);
5113     my $fifofh;
5114
5115     my $openfifo = sub {
5116         open($fifofh, '>', $mapfifo) || die $!;
5117     };
5118
5119     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5120
5121     my $oldtimeout;
5122     my $timeout = 5;
5123
5124     my $devinfo = {};
5125
5126     my $rpcenv = PVE::RPCEnvironment::get();
5127
5128     my $conffile = config_file($vmid);
5129     my $tmpfn = "$conffile.$$.tmp";
5130
5131     # Note: $oldconf is undef if VM does not exists
5132     my $oldconf = PVE::Cluster::cfs_read_file(cfs_config_path($vmid));
5133
5134     my $print_devmap = sub {
5135         my $virtdev_hash = {};
5136
5137         my $cfgfn = "$tmpdir/qemu-server.conf";
5138
5139         # we can read the config - that is already extracted
5140         my $fh = IO::File->new($cfgfn, "r") ||
5141             "unable to read qemu-server.conf - $!\n";
5142
5143         while (defined(my $line = <$fh>)) {
5144             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5145                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5146                 die "archive does not contain data for drive '$virtdev'\n"
5147                     if !$devinfo->{$devname};
5148                 if (defined($opts->{storage})) {
5149                     $storeid = $opts->{storage} || 'local';
5150                 } elsif (!$storeid) {
5151                     $storeid = 'local';
5152                 }
5153                 $format = 'raw' if !$format;
5154                 $devinfo->{$devname}->{devname} = $devname;
5155                 $devinfo->{$devname}->{virtdev} = $virtdev;
5156                 $devinfo->{$devname}->{format} = $format;
5157                 $devinfo->{$devname}->{storeid} = $storeid;
5158
5159                 # check permission on storage
5160                 my $pool = $opts->{pool}; # todo: do we need that?
5161                 if ($user ne 'root@pam') {
5162                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5163                 }
5164
5165                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5166             }
5167         }
5168
5169         foreach my $devname (keys %$devinfo) {
5170             die "found no device mapping information for device '$devname'\n"
5171                 if !$devinfo->{$devname}->{virtdev};
5172         }
5173
5174         my $cfg = cfs_read_file('storage.cfg');
5175
5176         # create empty/temp config
5177         if ($oldconf) {
5178             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5179             foreach_drive($oldconf, sub {
5180                 my ($ds, $drive) = @_;
5181
5182                 return if drive_is_cdrom($drive);
5183
5184                 my $volid = $drive->{file};
5185
5186                 return if !$volid || $volid =~ m|^/|;
5187
5188                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5189                 return if !$path || !$owner || ($owner != $vmid);
5190
5191                 # Note: only delete disk we want to restore
5192                 # other volumes will become unused
5193                 if ($virtdev_hash->{$ds}) {
5194                     PVE::Storage::vdisk_free($cfg, $volid);
5195                 }
5196             });
5197         }
5198
5199         my $map = {};
5200         foreach my $virtdev (sort keys %$virtdev_hash) {
5201             my $d = $virtdev_hash->{$virtdev};
5202             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5203             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5204
5205             # test if requested format is supported
5206             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5207             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5208             $d->{format} = $defFormat if !$supported;
5209
5210             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5211                                                   $d->{format}, undef, $alloc_size);
5212             print STDERR "new volume ID is '$volid'\n";
5213             $d->{volid} = $volid;
5214             my $path = PVE::Storage::path($cfg, $volid);
5215
5216             my $write_zeros = 1;
5217             # fixme: what other storages types initialize volumes with zero?
5218             if ($scfg->{type} eq 'dir' || $scfg->{type} eq 'nfs' || $scfg->{type} eq 'glusterfs' ||
5219                 $scfg->{type} eq 'sheepdog' || $scfg->{type} eq 'rbd') {
5220                 $write_zeros = 0;
5221             }
5222
5223             print $fifofh "${write_zeros}:$d->{devname}=$path\n";
5224
5225             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5226             $map->{$virtdev} = $volid;
5227         }
5228
5229         $fh->seek(0, 0) || die "seek failed - $!\n";
5230
5231         my $outfd = new IO::File ($tmpfn, "w") ||
5232             die "unable to write config for VM $vmid\n";
5233
5234         my $cookie = { netcount => 0 };
5235         while (defined(my $line = <$fh>)) {
5236             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5237         }
5238
5239         $fh->close();
5240         $outfd->close();
5241     };
5242
5243     eval {
5244         # enable interrupts
5245         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5246             die "interrupted by signal\n";
5247         };
5248         local $SIG{ALRM} = sub { die "got timeout\n"; };
5249
5250         $oldtimeout = alarm($timeout);
5251
5252         my $parser = sub {
5253             my $line = shift;
5254
5255             print "$line\n";
5256
5257             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5258                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5259                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5260             } elsif ($line =~ m/^CTIME: /) {
5261                 # we correctly received the vma config, so we can disable
5262                 # the timeout now for disk allocation (set to 10 minutes, so
5263                 # that we always timeout if something goes wrong)
5264                 alarm(600);
5265                 &$print_devmap();
5266                 print $fifofh "done\n";
5267                 my $tmp = $oldtimeout || 0;
5268                 $oldtimeout = undef;
5269                 alarm($tmp);
5270                 close($fifofh);
5271             }
5272         };
5273
5274         print "restore vma archive: $cmd\n";
5275         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5276     };
5277     my $err = $@;
5278
5279     alarm($oldtimeout) if $oldtimeout;
5280
5281     unlink $mapfifo;
5282
5283     if ($err) {
5284         rmtree $tmpdir;
5285         unlink $tmpfn;
5286
5287         my $cfg = cfs_read_file('storage.cfg');
5288         foreach my $devname (keys %$devinfo) {
5289             my $volid = $devinfo->{$devname}->{volid};
5290             next if !$volid;
5291             eval {
5292                 if ($volid =~ m|^/|) {
5293                     unlink $volid || die 'unlink failed\n';
5294                 } else {
5295                     PVE::Storage::vdisk_free($cfg, $volid);
5296                 }
5297                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5298             };
5299             print STDERR "unable to cleanup '$volid' - $@" if $@;
5300         }
5301         die $err;
5302     }
5303
5304     rmtree $tmpdir;
5305
5306     rename($tmpfn, $conffile) ||
5307         die "unable to commit configuration file '$conffile'\n";
5308
5309     PVE::Cluster::cfs_update(); # make sure we read new file
5310
5311     eval { rescan($vmid, 1); };
5312     warn $@ if $@;
5313 }
5314
5315 sub restore_tar_archive {
5316     my ($archive, $vmid, $user, $opts) = @_;
5317
5318     if ($archive ne '-') {
5319         my $firstfile = tar_archive_read_firstfile($archive);
5320         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5321             if $firstfile ne 'qemu-server.conf';
5322     }
5323
5324     my $storecfg = cfs_read_file('storage.cfg');
5325
5326     # destroy existing data - keep empty config
5327     my $vmcfgfn = config_file($vmid);
5328     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5329
5330     my $tocmd = "/usr/lib/qemu-server/qmextract";
5331
5332     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5333     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5334     $tocmd .= ' --prealloc' if $opts->{prealloc};
5335     $tocmd .= ' --info' if $opts->{info};
5336
5337     # tar option "xf" does not autodetect compression when read from STDIN,
5338     # so we pipe to zcat
5339     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5340         PVE::Tools::shellquote("--to-command=$tocmd");
5341
5342     my $tmpdir = "/var/tmp/vzdumptmp$$";
5343     mkpath $tmpdir;
5344
5345     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5346     local $ENV{VZDUMP_VMID} = $vmid;
5347     local $ENV{VZDUMP_USER} = $user;
5348
5349     my $conffile = config_file($vmid);
5350     my $tmpfn = "$conffile.$$.tmp";
5351
5352     # disable interrupts (always do cleanups)
5353     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5354         print STDERR "got interrupt - ignored\n";
5355     };
5356
5357     eval {
5358         # enable interrupts
5359         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5360             die "interrupted by signal\n";
5361         };
5362
5363         if ($archive eq '-') {
5364             print "extracting archive from STDIN\n";
5365             run_command($cmd, input => "<&STDIN");
5366         } else {
5367             print "extracting archive '$archive'\n";
5368             run_command($cmd);
5369         }
5370
5371         return if $opts->{info};
5372
5373         # read new mapping
5374         my $map = {};
5375         my $statfile = "$tmpdir/qmrestore.stat";
5376         if (my $fd = IO::File->new($statfile, "r")) {
5377             while (defined (my $line = <$fd>)) {
5378                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5379                     $map->{$1} = $2 if $1;
5380                 } else {
5381                     print STDERR "unable to parse line in statfile - $line\n";
5382                 }
5383             }
5384             $fd->close();
5385         }
5386
5387         my $confsrc = "$tmpdir/qemu-server.conf";
5388
5389         my $srcfd = new IO::File($confsrc, "r") ||
5390             die "unable to open file '$confsrc'\n";
5391
5392         my $outfd = new IO::File ($tmpfn, "w") ||
5393             die "unable to write config for VM $vmid\n";
5394
5395         my $cookie = { netcount => 0 };
5396         while (defined (my $line = <$srcfd>)) {
5397             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5398         }
5399
5400         $srcfd->close();
5401         $outfd->close();
5402     };
5403     my $err = $@;
5404
5405     if ($err) {
5406
5407         unlink $tmpfn;
5408
5409         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
5410
5411         die $err;
5412     }
5413
5414     rmtree $tmpdir;
5415
5416     rename $tmpfn, $conffile ||
5417         die "unable to commit configuration file '$conffile'\n";
5418
5419     PVE::Cluster::cfs_update(); # make sure we read new file
5420
5421     eval { rescan($vmid, 1); };
5422     warn $@ if $@;
5423 };
5424
5425
5426 # Internal snapshots
5427
5428 # NOTE: Snapshot create/delete involves several non-atomic
5429 # action, and can take a long time.
5430 # So we try to avoid locking the file and use 'lock' variable
5431 # inside the config file instead.
5432
5433 my $snapshot_copy_config = sub {
5434     my ($source, $dest) = @_;
5435
5436     foreach my $k (keys %$source) {
5437         next if $k eq 'snapshots';
5438         next if $k eq 'snapstate';
5439         next if $k eq 'snaptime';
5440         next if $k eq 'vmstate';
5441         next if $k eq 'lock';
5442         next if $k eq 'digest';
5443         next if $k eq 'description';
5444         next if $k =~ m/^unused\d+$/;
5445
5446         $dest->{$k} = $source->{$k};
5447     }
5448 };
5449
5450 my $snapshot_apply_config = sub {
5451     my ($conf, $snap) = @_;
5452
5453     # copy snapshot list
5454     my $newconf = {
5455         snapshots => $conf->{snapshots},
5456     };
5457
5458     # keep description and list of unused disks
5459     foreach my $k (keys %$conf) {
5460         next if !($k =~ m/^unused\d+$/ || $k eq 'description');
5461         $newconf->{$k} = $conf->{$k};
5462     }
5463
5464     &$snapshot_copy_config($snap, $newconf);
5465
5466     return $newconf;
5467 };
5468
5469 sub foreach_writable_storage {
5470     my ($conf, $func) = @_;
5471
5472     my $sidhash = {};
5473
5474     foreach my $ds (keys %$conf) {
5475         next if !valid_drivename($ds);
5476
5477         my $drive = parse_drive($ds, $conf->{$ds});
5478         next if !$drive;
5479         next if drive_is_cdrom($drive);
5480
5481         my $volid = $drive->{file};
5482
5483         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5484         $sidhash->{$sid} = $sid if $sid;
5485     }
5486
5487     foreach my $sid (sort keys %$sidhash) {
5488         &$func($sid);
5489     }
5490 }
5491
5492 my $alloc_vmstate_volid = sub {
5493     my ($storecfg, $vmid, $conf, $snapname) = @_;
5494
5495     # Note: we try to be smart when selecting a $target storage
5496
5497     my $target;
5498
5499     # search shared storage first
5500     foreach_writable_storage($conf, sub {
5501         my ($sid) = @_;
5502         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
5503         return if !$scfg->{shared};
5504
5505         $target = $sid if !$target || $scfg->{path}; # prefer file based storage
5506     });
5507
5508     if (!$target) {
5509         # now search local storage
5510         foreach_writable_storage($conf, sub {
5511             my ($sid) = @_;
5512             my $scfg = PVE::Storage::storage_config($storecfg, $sid);
5513             return if $scfg->{shared};
5514
5515             $target = $sid if !$target || $scfg->{path}; # prefer file based storage;
5516         });
5517     }
5518
5519     $target = 'local' if !$target;
5520
5521     my $driver_state_size = 500; # assume 32MB is enough to safe all driver state;
5522     # we abort live save after $conf->{memory}, so we need at max twice that space
5523     my $size = $conf->{memory}*2 + $driver_state_size;
5524
5525     my $name = "vm-$vmid-state-$snapname";
5526     my $scfg = PVE::Storage::storage_config($storecfg, $target);
5527     $name .= ".raw" if $scfg->{path}; # add filename extension for file base storage
5528     my $volid = PVE::Storage::vdisk_alloc($storecfg, $target, $vmid, 'raw', $name, $size*1024);
5529
5530     return $volid;
5531 };
5532
5533 my $snapshot_prepare = sub {
5534     my ($vmid, $snapname, $save_vmstate, $comment) = @_;
5535
5536     my $snap;
5537
5538     my $updatefn =  sub {
5539
5540         my $conf = load_config($vmid);
5541
5542         die "you can't take a snapshot if it's a template\n"
5543             if is_template($conf);
5544
5545         check_lock($conf);
5546
5547         $conf->{lock} = 'snapshot';
5548
5549         die "snapshot name '$snapname' already used\n"
5550             if defined($conf->{snapshots}->{$snapname});
5551
5552         my $storecfg = PVE::Storage::config();
5553         die "snapshot feature is not available" if !has_feature('snapshot', $conf, $storecfg);
5554
5555         $snap = $conf->{snapshots}->{$snapname} = {};
5556
5557         if ($save_vmstate && check_running($vmid)) {
5558             $snap->{vmstate} = &$alloc_vmstate_volid($storecfg, $vmid, $conf, $snapname);
5559         }
5560
5561         &$snapshot_copy_config($conf, $snap);
5562
5563         $snap->{snapstate} = "prepare";
5564         $snap->{snaptime} = time();
5565         $snap->{description} = $comment if $comment;
5566
5567         # always overwrite machine if we save vmstate. This makes sure we
5568         # can restore it later using correct machine type
5569         $snap->{machine} = get_current_qemu_machine($vmid) if $snap->{vmstate};
5570
5571         update_config_nolock($vmid, $conf, 1);
5572     };
5573
5574     lock_config($vmid, $updatefn);
5575
5576     return $snap;
5577 };
5578
5579 my $snapshot_commit = sub {
5580     my ($vmid, $snapname) = @_;
5581
5582     my $updatefn = sub {
5583
5584         my $conf = load_config($vmid);
5585
5586         die "missing snapshot lock\n"
5587             if !($conf->{lock} && $conf->{lock} eq 'snapshot');
5588
5589         my $has_machine_config = defined($conf->{machine});
5590
5591         my $snap = $conf->{snapshots}->{$snapname};
5592
5593         die "snapshot '$snapname' does not exist\n" if !defined($snap);
5594
5595         die "wrong snapshot state\n"
5596             if !($snap->{snapstate} && $snap->{snapstate} eq "prepare");
5597
5598         delete $snap->{snapstate};
5599         delete $conf->{lock};
5600
5601         my $newconf = &$snapshot_apply_config($conf, $snap);
5602
5603         delete $newconf->{machine} if !$has_machine_config;
5604
5605         $newconf->{parent} = $snapname;
5606
5607         update_config_nolock($vmid, $newconf, 1);
5608     };
5609
5610     lock_config($vmid, $updatefn);
5611 };
5612
5613 sub snapshot_rollback {
5614     my ($vmid, $snapname) = @_;
5615
5616     my $prepare = 1;
5617
5618     my $storecfg = PVE::Storage::config();
5619
5620     my $conf = load_config($vmid);
5621
5622     my $get_snapshot_config = sub {
5623
5624         die "you can't rollback if vm is a template\n" if is_template($conf);
5625
5626         my $res = $conf->{snapshots}->{$snapname};
5627
5628         die "snapshot '$snapname' does not exist\n" if !defined($res);
5629
5630         return $res;
5631     };
5632
5633     my $snap = &$get_snapshot_config();
5634
5635     foreach_drive($snap, sub {
5636         my ($ds, $drive) = @_;
5637
5638         return if drive_is_cdrom($drive);
5639
5640         my $volid = $drive->{file};
5641
5642         PVE::Storage::volume_rollback_is_possible($storecfg, $volid, $snapname);
5643     });
5644
5645     my $updatefn = sub {
5646
5647         $conf = load_config($vmid);
5648
5649         $snap = &$get_snapshot_config();
5650
5651         die "unable to rollback to incomplete snapshot (snapstate = $snap->{snapstate})\n"
5652             if $snap->{snapstate};
5653
5654         if ($prepare) {
5655             check_lock($conf);
5656             vm_stop($storecfg, $vmid, undef, undef, 5, undef, undef);
5657         }
5658
5659         die "unable to rollback vm $vmid: vm is running\n"
5660             if check_running($vmid);
5661
5662         if ($prepare) {
5663             $conf->{lock} = 'rollback';
5664         } else {
5665             die "got wrong lock\n" if !($conf->{lock} && $conf->{lock} eq 'rollback');
5666             delete $conf->{lock};
5667         }
5668
5669         my $forcemachine;
5670
5671         if (!$prepare) {
5672             my $has_machine_config = defined($conf->{machine});
5673
5674             # copy snapshot config to current config
5675             $conf = &$snapshot_apply_config($conf, $snap);
5676             $conf->{parent} = $snapname;
5677
5678             # Note: old code did not store 'machine', so we try to be smart
5679             # and guess the snapshot was generated with kvm 1.4 (pc-i440fx-1.4).
5680             $forcemachine = $conf->{machine} || 'pc-i440fx-1.4';
5681             # we remove the 'machine' configuration if not explicitly specified
5682             # in the original config.
5683             delete $conf->{machine} if $snap->{vmstate} && !$has_machine_config;
5684         }
5685
5686         update_config_nolock($vmid, $conf, 1);
5687
5688         if (!$prepare && $snap->{vmstate}) {
5689             my $statefile = PVE::Storage::path($storecfg, $snap->{vmstate});
5690             vm_start($storecfg, $vmid, $statefile, undef, undef, undef, $forcemachine);
5691         }
5692     };
5693
5694     lock_config($vmid, $updatefn);
5695
5696     foreach_drive($snap, sub {
5697         my ($ds, $drive) = @_;
5698
5699         return if drive_is_cdrom($drive);
5700
5701         my $volid = $drive->{file};
5702         my $device = "drive-$ds";
5703
5704         PVE::Storage::volume_snapshot_rollback($storecfg, $volid, $snapname);
5705     });
5706
5707     $prepare = 0;
5708     lock_config($vmid, $updatefn);
5709 }
5710
5711 my $savevm_wait = sub {
5712     my ($vmid) = @_;
5713
5714     for(;;) {
5715         my $stat = vm_mon_cmd_nocheck($vmid, "query-savevm");
5716         if (!$stat->{status}) {
5717             die "savevm not active\n";
5718         } elsif ($stat->{status} eq 'active') {
5719             sleep(1);
5720             next;
5721         } elsif ($stat->{status} eq 'completed') {
5722             last;
5723         } else {
5724             die "query-savevm returned status '$stat->{status}'\n";
5725         }
5726     }
5727 };
5728
5729 sub do_snapshots_with_qemu {
5730     my ($storecfg, $volid) = @_;
5731
5732     my $storage_name = PVE::Storage::parse_volume_id($volid);
5733
5734     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}} ){
5735         return 1;
5736     }
5737
5738     if ($volid =~ m/\.(qcow2|qed)$/){
5739         return 1;
5740     }
5741
5742     return undef;
5743 }
5744
5745 sub snapshot_create {
5746     my ($vmid, $snapname, $save_vmstate, $comment) = @_;
5747
5748     my $snap = &$snapshot_prepare($vmid, $snapname, $save_vmstate, $comment);
5749
5750     $save_vmstate = 0 if !$snap->{vmstate}; # vm is not running
5751
5752     my $config = load_config($vmid);
5753
5754     my $running = check_running($vmid);
5755
5756     my $freezefs = $running && $config->{agent};
5757     $freezefs = 0 if $snap->{vmstate}; # not needed if we save RAM
5758
5759     my $drivehash = {};
5760
5761     if ($freezefs) {
5762         eval { vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
5763         warn "guest-fsfreeze-freeze problems - $@" if $@;
5764     }
5765
5766     eval {
5767         # create internal snapshots of all drives
5768
5769         my $storecfg = PVE::Storage::config();
5770
5771         if ($running) {
5772             if ($snap->{vmstate}) {
5773                 my $path = PVE::Storage::path($storecfg, $snap->{vmstate});
5774                 vm_mon_cmd($vmid, "savevm-start", statefile => $path);
5775                 &$savevm_wait($vmid);
5776             } else {
5777                 vm_mon_cmd($vmid, "savevm-start");
5778             }
5779         };
5780
5781         foreach_drive($snap, sub {
5782             my ($ds, $drive) = @_;
5783
5784             return if drive_is_cdrom($drive);
5785
5786             my $volid = $drive->{file};
5787             my $device = "drive-$ds";
5788
5789             qemu_volume_snapshot($vmid, $device, $storecfg, $volid, $snapname);
5790             $drivehash->{$ds} = 1;
5791        });
5792     };
5793     my $err = $@;
5794
5795     if ($running) {
5796         eval { vm_mon_cmd($vmid, "savevm-end")  };
5797         warn $@ if $@;
5798
5799         if ($freezefs) {
5800             eval { vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
5801             warn "guest-fsfreeze-thaw problems - $@" if $@;
5802         }
5803
5804         # savevm-end is async, we need to wait
5805         for (;;) {
5806             my $stat = vm_mon_cmd_nocheck($vmid, "query-savevm");
5807             if (!$stat->{bytes}) {
5808                 last;
5809             } else {
5810                 print "savevm not yet finished\n";
5811                 sleep(1);
5812                 next;
5813             }
5814         }
5815     }
5816
5817     if ($err) {
5818         warn "snapshot create failed: starting cleanup\n";
5819         eval { snapshot_delete($vmid, $snapname, 0, $drivehash); };
5820         warn $@ if $@;
5821         die $err;
5822     }
5823
5824     &$snapshot_commit($vmid, $snapname);
5825 }
5826
5827 # Note: $drivehash is only set when called from snapshot_create.
5828 sub snapshot_delete {
5829     my ($vmid, $snapname, $force, $drivehash) = @_;
5830
5831     my $prepare = 1;
5832
5833     my $snap;
5834     my $unused = [];
5835
5836     my $unlink_parent = sub {
5837         my ($confref, $new_parent) = @_;
5838
5839         if ($confref->{parent} && $confref->{parent} eq $snapname) {
5840             if ($new_parent) {
5841                 $confref->{parent} = $new_parent;
5842             } else {
5843                 delete $confref->{parent};
5844             }
5845         }
5846     };
5847
5848     my $updatefn =  sub {
5849         my ($remove_drive) = @_;
5850
5851         my $conf = load_config($vmid);
5852
5853         if (!$drivehash) {
5854             check_lock($conf);
5855             die "you can't delete a snapshot if vm is a template\n"
5856                 if is_template($conf);
5857         }
5858
5859         $snap = $conf->{snapshots}->{$snapname};
5860
5861         die "snapshot '$snapname' does not exist\n" if !defined($snap);
5862
5863         # remove parent refs
5864         if (!$prepare) {
5865             &$unlink_parent($conf, $snap->{parent});
5866             foreach my $sn (keys %{$conf->{snapshots}}) {
5867                 next if $sn eq $snapname;
5868                 &$unlink_parent($conf->{snapshots}->{$sn}, $snap->{parent});
5869             }
5870         }
5871
5872         if ($remove_drive) {
5873             if ($remove_drive eq 'vmstate') {
5874                 delete $snap->{$remove_drive};
5875             } else {
5876                 my $drive = parse_drive($remove_drive, $snap->{$remove_drive});
5877                 my $volid = $drive->{file};
5878                 delete $snap->{$remove_drive};
5879                 add_unused_volume($conf, $volid);
5880             }
5881         }
5882
5883         if ($prepare) {
5884             $snap->{snapstate} = 'delete';
5885         } else {
5886             delete $conf->{snapshots}->{$snapname};
5887             delete $conf->{lock} if $drivehash;
5888             foreach my $volid (@$unused) {
5889                 add_unused_volume($conf, $volid);
5890             }
5891         }
5892
5893         update_config_nolock($vmid, $conf, 1);
5894     };
5895
5896     lock_config($vmid, $updatefn);
5897
5898     # now remove vmstate file
5899
5900     my $storecfg = PVE::Storage::config();
5901
5902     if ($snap->{vmstate}) {
5903         eval {  PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
5904         if (my $err = $@) {
5905             die $err if !$force;
5906             warn $err;
5907         }
5908         # save changes (remove vmstate from snapshot)
5909         lock_config($vmid, $updatefn, 'vmstate') if !$force;
5910     };
5911
5912     # now remove all internal snapshots
5913     foreach_drive($snap, sub {
5914         my ($ds, $drive) = @_;
5915
5916         return if drive_is_cdrom($drive);
5917
5918         my $volid = $drive->{file};
5919         my $device = "drive-$ds";
5920
5921         if (!$drivehash || $drivehash->{$ds}) {
5922             eval { qemu_volume_snapshot_delete($vmid, $device, $storecfg, $volid, $snapname); };
5923             if (my $err = $@) {
5924                 die $err if !$force;
5925                 warn $err;
5926             }
5927         }
5928
5929         # save changes (remove drive fron snapshot)
5930         lock_config($vmid, $updatefn, $ds) if !$force;
5931         push @$unused, $volid;
5932     });
5933
5934     # now cleanup config
5935     $prepare = 0;
5936     lock_config($vmid, $updatefn);
5937 }
5938
5939 sub has_feature {
5940     my ($feature, $conf, $storecfg, $snapname, $running) = @_;
5941
5942     my $err;
5943     foreach_drive($conf, sub {
5944         my ($ds, $drive) = @_;
5945
5946         return if drive_is_cdrom($drive);
5947         my $volid = $drive->{file};
5948         $err = 1 if !PVE::Storage::volume_has_feature($storecfg, $feature, $volid, $snapname, $running);
5949     });
5950
5951     return $err ? 0 : 1;
5952 }
5953
5954 sub template_create {
5955     my ($vmid, $conf, $disk) = @_;
5956
5957     my $storecfg = PVE::Storage::config();
5958
5959     foreach_drive($conf, sub {
5960         my ($ds, $drive) = @_;
5961
5962         return if drive_is_cdrom($drive);
5963         return if $disk && $ds ne $disk;
5964
5965         my $volid = $drive->{file};
5966         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
5967
5968         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
5969         $drive->{file} = $voliddst;
5970         $conf->{$ds} = print_drive($vmid, $drive);
5971         update_config_nolock($vmid, $conf, 1);
5972     });
5973 }
5974
5975 sub is_template {
5976     my ($conf) = @_;
5977
5978     return 1 if defined $conf->{template} && $conf->{template} == 1;
5979 }
5980
5981 sub qemu_img_convert {
5982     my ($src_volid, $dst_volid, $size, $snapname) = @_;
5983
5984     my $storecfg = PVE::Storage::config();
5985     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
5986     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
5987
5988     if ($src_storeid && $dst_storeid) {
5989         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
5990         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5991
5992         my $src_format = qemu_img_format($src_scfg, $src_volname);
5993         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
5994
5995         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
5996         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5997
5998         my $cmd = [];
5999         push @$cmd, '/usr/bin/qemu-img', 'convert', '-t', 'writeback', '-p', '-n';
6000         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
6001         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path, $dst_path;
6002
6003         my $parser = sub {
6004             my $line = shift;
6005             if($line =~ m/\((\S+)\/100\%\)/){
6006                 my $percent = $1;
6007                 my $transferred = int($size * $percent / 100);
6008                 my $remaining = $size - $transferred;
6009
6010                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6011             }
6012
6013         };
6014
6015         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6016         my $err = $@;
6017         die "copy failed: $err" if $err;
6018     }
6019 }
6020
6021 sub qemu_img_format {
6022     my ($scfg, $volname) = @_;
6023
6024     if ($scfg->{path} && $volname =~ m/\.(raw|qcow2|qed|vmdk)$/) {
6025         return $1;
6026     } elsif ($scfg->{type} eq 'iscsi') {
6027         return "host_device";
6028     } else {
6029         return "raw";
6030     }
6031 }
6032
6033 sub qemu_drive_mirror {
6034     my ($vmid, $drive, $dst_volid, $vmiddst) = @_;
6035
6036     my $storecfg = PVE::Storage::config();
6037     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6038
6039     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6040
6041     my $format;
6042     if ($dst_volname =~ m/\.(raw|qcow2)$/){
6043         $format = $1;
6044     }
6045
6046     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6047
6048     #drive-mirror is doing lseek on source image before starting, and this can take a lot of time for big nfs volume
6049     #during this time, qmp socket is hanging
6050     #http://lists.nongnu.org/archive/html/qemu-devel/2015-05/msg01838.html
6051     #so we need to setup a big timeout
6052     my $opts = { timeout => 14400, device => "drive-$drive", mode => "existing", sync => "full", target => $dst_path };
6053     $opts->{format} = $format if $format;
6054
6055     print "drive mirror is starting : this step can take some minutes/hours, depend of disk size and storage speed\n";
6056
6057     vm_mon_cmd($vmid, "drive-mirror", %$opts);
6058     eval {
6059         while (1) {
6060             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6061             my $stat = @$stats[0];
6062             die "mirroring job seem to have die. Maybe do you have bad sectors?" if !$stat;
6063             die "error job is not mirroring" if $stat->{type} ne "mirror";
6064
6065             my $busy = $stat->{busy};
6066             my $ready = $stat->{ready};
6067
6068             if (my $total = $stat->{len}) {
6069                 my $transferred = $stat->{offset} || 0;
6070                 my $remaining = $total - $transferred;
6071                 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6072
6073                 print "transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6074             }
6075
6076
6077             if ($stat->{ready} eq 'true') {
6078
6079                 last if $vmiddst != $vmid;
6080
6081                 # try to switch the disk if source and destination are on the same guest
6082                 eval { vm_mon_cmd($vmid, "block-job-complete", device => "drive-$drive") };
6083                 last if !$@;
6084                 die $@ if $@ !~ m/cannot be completed/;
6085             }
6086             sleep 1;
6087         }
6088
6089
6090     };
6091     my $err = $@;
6092
6093     my $cancel_job = sub {
6094         vm_mon_cmd($vmid, "block-job-cancel", device => "drive-$drive");
6095         while (1) {
6096             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6097             my $stat = @$stats[0];
6098             last if !$stat;
6099             sleep 1;
6100         }
6101     };
6102
6103     if ($err) {
6104         eval { &$cancel_job(); };
6105         die "mirroring error: $err";
6106     }
6107
6108     if ($vmiddst != $vmid) {
6109         # if we clone a disk for a new target vm, we don't switch the disk
6110         &$cancel_job(); # so we call block-job-cancel
6111     }
6112 }
6113
6114 sub clone_disk {
6115     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6116         $newvmid, $storage, $format, $full, $newvollist) = @_;
6117
6118     my $newvolid;
6119
6120     if (!$full) {
6121         print "create linked clone of drive $drivename ($drive->{file})\n";
6122         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6123         push @$newvollist, $newvolid;
6124     } else {
6125         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6126         $storeid = $storage if $storage;
6127
6128         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6129         if (!$format) {
6130             $format = $drive->{format} || $defFormat;
6131         }
6132
6133         # test if requested format is supported - else use default
6134         my $supported = grep { $_ eq $format } @$validFormats;
6135         $format = $defFormat if !$supported;
6136
6137         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6138
6139         print "create full clone of drive $drivename ($drive->{file})\n";
6140         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $format, undef, ($size/1024));
6141         push @$newvollist, $newvolid;
6142
6143         if (!$running || $snapname) {
6144             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname);
6145         } else {
6146             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid);
6147         }
6148     }
6149
6150     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6151
6152     my $disk = $drive;
6153     $disk->{format} = undef;
6154     $disk->{file} = $newvolid;
6155     $disk->{size} = $size;
6156
6157     return $disk;
6158 }
6159
6160 # this only works if VM is running
6161 sub get_current_qemu_machine {
6162     my ($vmid) = @_;
6163
6164     my $cmd = { execute => 'query-machines', arguments => {} };
6165     my $res = vm_qmp_command($vmid, $cmd);
6166
6167     my ($current, $default);
6168     foreach my $e (@$res) {
6169         $default = $e->{name} if $e->{'is-default'};
6170         $current = $e->{name} if $e->{'is-current'};
6171     }
6172
6173     # fallback to the default machine if current is not supported by qemu
6174     return $current || $default || 'pc';
6175 }
6176
6177 sub qemu_machine_feature_enabled {
6178     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6179
6180     my $current_major;
6181     my $current_minor;
6182
6183     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6184
6185         $current_major = $3;
6186         $current_minor = $4;
6187
6188     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6189
6190         $current_major = $1;
6191         $current_minor = $2;
6192     }
6193
6194     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
6195
6196
6197 }
6198
6199 sub lspci {
6200
6201     my $devices = {};
6202
6203     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6204             my (undef, $id, $function) = @_;
6205             my $res = { id => $id, function => $function};
6206             push @{$devices->{$id}}, $res;
6207     });
6208
6209     return $devices;
6210 }
6211
6212 sub vm_iothreads_list {
6213     my ($vmid) = @_;
6214
6215     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6216
6217     my $iothreads = {};
6218     foreach my $iothread (@$res) {
6219         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6220     }
6221
6222     return $iothreads;
6223 }
6224
6225 sub scsihw_infos {
6226     my ($conf, $drive) = @_;
6227
6228     my $maxdev = 0;
6229
6230     if ($conf->{scsihw} && ($conf->{scsihw} =~ m/^lsi/)) {
6231         $maxdev = 7;
6232     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6233         $maxdev = 1;
6234     } else {
6235         $maxdev = 256;
6236     }
6237
6238     my $controller = int($drive->{index} / $maxdev);
6239     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6240
6241     return ($maxdev, $controller, $controller_prefix);
6242 }
6243
6244 1;