PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QMPClient;
  31 use PVE::RPCEnvironment;
  32 use Time::HiRes qw(gettimeofday);
  33
  34 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  35
  36 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  37
  38 # Note about locking: we use flock on the config file protect
  39 # against concurent actions.
  40 # Aditionaly, we have a 'lock' setting in the config file. This
  41 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  42 # allowed when such lock is set. But you can ignore this kind of
  43 # lock with the --skiplock flag.
  44
  45 cfs_register_file('/qemu-server/',
  46                   \&parse_vm_config,
  47                   \&write_vm_config);
  48
  49 PVE::JSONSchema::register_standard_option('skiplock', {
  50     description => "Ignore locks - only root is allowed to use this option.",
  51     type => 'boolean',
  52     optional => 1,
  53 });
  54
  55 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  56     description => "Some command save/restore state from this location.",
  57     type => 'string',
  58     maxLength => 128,
  59     optional => 1,
  60 });
  61
  62 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  63     description => "The name of the snapshot.",
  64     type => 'string', format => 'pve-configid',
  65     maxLength => 40,
  66 });
  67
  68 #no warnings 'redefine';
  69
  70 sub cgroups_write {
  71    my ($controller, $vmid, $option, $value) = @_;
  72
  73    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  74    PVE::ProcFSTools::write_proc_entry($path, $value);
  75
  76 }
  77
  78 my $nodename = PVE::INotify::nodename();
  79
  80 mkdir "/etc/pve/nodes/$nodename";
  81 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  82 mkdir $confdir;
  83
  84 my $var_run_tmpdir = "/var/run/qemu-server";
  85 mkdir $var_run_tmpdir;
  86
  87 my $lock_dir = "/var/lock/qemu-server";
  88 mkdir $lock_dir;
  89
  90 my $pcisysfs = "/sys/bus/pci";
  91
  92 my $confdesc = {
  93     onboot => {
  94         optional => 1,
  95         type => 'boolean',
  96         description => "Specifies whether a VM will be started during system bootup.",
  97         default => 0,
  98     },
  99     autostart => {
 100         optional => 1,
 101         type => 'boolean',
 102         description => "Automatic restart after crash (currently ignored).",
 103         default => 0,
 104     },
 105     hotplug => {
 106         optional => 1,
 107         type => 'string', format => 'pve-hotplug-features',
 108         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 109         default => 'network,disk,usb',
 110     },
 111     reboot => {
 112         optional => 1,
 113         type => 'boolean',
 114         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 115         default => 1,
 116     },
 117     lock => {
 118         optional => 1,
 119         type => 'string',
 120         description => "Lock/unlock the VM.",
 121         enum => [qw(migrate backup snapshot rollback)],
 122     },
 123     cpulimit => {
 124         optional => 1,
 125         type => 'number',
 126         description => "Limit of CPU usage. Note if the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 127         minimum => 0,
 128         maximum => 128,
 129         default => 0,
 130     },
 131     cpuunits => {
 132         optional => 1,
 133         type => 'integer',
 134         description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 135         minimum => 0,
 136         maximum => 500000,
 137         default => 1000,
 138     },
 139     memory => {
 140         optional => 1,
 141         type => 'integer',
 142         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 143         minimum => 16,
 144         default => 512,
 145     },
 146     balloon => {
 147         optional => 1,
 148         type => 'integer',
 149         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 150         minimum => 0,
 151     },
 152     shares => {
 153         optional => 1,
 154         type => 'integer',
 155         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 156         minimum => 0,
 157         maximum => 50000,
 158         default => 1000,
 159     },
 160     keyboard => {
 161         optional => 1,
 162         type => 'string',
 163         description => "Keybord layout for vnc server. Default is read from the datacenter configuration file.",
 164         enum => PVE::Tools::kvmkeymaplist(),
 165         default => 'en-us',
 166     },
 167     name => {
 168         optional => 1,
 169         type => 'string', format => 'dns-name',
 170         description => "Set a name for the VM. Only used on the configuration web interface.",
 171     },
 172     scsihw => {
 173         optional => 1,
 174         type => 'string',
 175         description => "scsi controller model",
 176         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 177         default => 'lsi',
 178     },
 179     description => {
 180         optional => 1,
 181         type => 'string',
 182         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 183     },
 184     ostype => {
 185         optional => 1,
 186         type => 'string',
 187         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 l24 l26 solaris)],
 188         description => <<EODESC,
 189 Used to enable special optimization/features for specific
 190 operating systems:
 191
 192 other  => unspecified OS
 193 wxp    => Microsoft Windows XP
 194 w2k    => Microsoft Windows 2000
 195 w2k3   => Microsoft Windows 2003
 196 w2k8   => Microsoft Windows 2008
 197 wvista => Microsoft Windows Vista
 198 win7   => Microsoft Windows 7
 199 win8   => Microsoft Windows 8/2012
 200 l24    => Linux 2.4 Kernel
 201 l26    => Linux 2.6/3.X Kernel
 202 solaris => solaris/opensolaris/openindiania kernel
 203
 204 other|l24|l26|solaris                       ... no special behaviour
 205 wxp|w2k|w2k3|w2k8|wvista|win7|win8  ... use --localtime switch
 206 EODESC
 207     },
 208     boot => {
 209         optional => 1,
 210         type => 'string',
 211         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 212         pattern => '[acdn]{1,4}',
 213         default => 'cdn',
 214     },
 215     bootdisk => {
 216         optional => 1,
 217         type => 'string', format => 'pve-qm-bootdisk',
 218         description => "Enable booting from specified disk.",
 219         pattern => '(ide|sata|scsi|virtio)\d+',
 220     },
 221     smp => {
 222         optional => 1,
 223         type => 'integer',
 224         description => "The number of CPUs. Please use option -sockets instead.",
 225         minimum => 1,
 226         default => 1,
 227     },
 228     sockets => {
 229         optional => 1,
 230         type => 'integer',
 231         description => "The number of CPU sockets.",
 232         minimum => 1,
 233         default => 1,
 234     },
 235     cores => {
 236         optional => 1,
 237         type => 'integer',
 238         description => "The number of cores per socket.",
 239         minimum => 1,
 240         default => 1,
 241     },
 242     numa => {
 243         optional => 1,
 244         type => 'boolean',
 245         description => "Enable/disable Numa.",
 246         default => 0,
 247     },
 248     vcpus => {
 249         optional => 1,
 250         type => 'integer',
 251         description => "Number of hotplugged vcpus.",
 252         minimum => 1,
 253         default => 0,
 254     },
 255     acpi => {
 256         optional => 1,
 257         type => 'boolean',
 258         description => "Enable/disable ACPI.",
 259         default => 1,
 260     },
 261     agent => {
 262         optional => 1,
 263         type => 'boolean',
 264         description => "Enable/disable Qemu GuestAgent.",
 265         default => 0,
 266     },
 267     kvm => {
 268         optional => 1,
 269         type => 'boolean',
 270         description => "Enable/disable KVM hardware virtualization.",
 271         default => 1,
 272     },
 273     tdf => {
 274         optional => 1,
 275         type => 'boolean',
 276         description => "Enable/disable time drift fix.",
 277         default => 0,
 278     },
 279     localtime => {
 280         optional => 1,
 281         type => 'boolean',
 282         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 283     },
 284     freeze => {
 285         optional => 1,
 286         type => 'boolean',
 287         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 288     },
 289     vga => {
 290         optional => 1,
 291         type => 'string',
 292         description => "Select VGA type. If you want to use high resolution modes (>= 1280x1024x16) then you should use option 'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and 'cirrur' for other OS types. Option 'qxl' enables the SPICE display sever. You can also run without any graphic card using a serial devive as terminal.",
 293         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 294     },
 295     watchdog => {
 296         optional => 1,
 297         type => 'string', format => 'pve-qm-watchdog',
 298         typetext => '[[model=]i6300esb|ib700] [,[action=]reset|shutdown|poweroff|pause|debug|none]',
 299         description => "Create a virtual hardware watchdog device.  Once enabled (by a guest action), the watchdog must be periodically polled by an agent inside the guest or else the guest will be restarted (or execute the action specified)",
 300     },
 301     startdate => {
 302         optional => 1,
 303         type => 'string',
 304         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 305         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 306         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 307         default => 'now',
 308     },
 309     startup =>  get_standard_option('pve-startup-order'),
 310     template => {
 311         optional => 1,
 312         type => 'boolean',
 313         description => "Enable/disable Template.",
 314         default => 0,
 315     },
 316     args => {
 317         optional => 1,
 318         type => 'string',
 319         description => <<EODESCR,
 320 Note: this option is for experts only. It allows you to pass arbitrary arguments to kvm, for example:
 321
 322 args: -no-reboot -no-hpet
 323 EODESCR
 324     },
 325     tablet => {
 326         optional => 1,
 327         type => 'boolean',
 328         default => 1,
 329         description => "Enable/disable the usb tablet device. This device is usually needed to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with normal VNC clients. If you're running lots of console-only guests on one host, you may consider disabling this to save some context switches. This is turned of by default if you use spice (vga=qxl).",
 330     },
 331     migrate_speed => {
 332         optional => 1,
 333         type => 'integer',
 334         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 335         minimum => 0,
 336         default => 0,
 337     },
 338     migrate_downtime => {
 339         optional => 1,
 340         type => 'number',
 341         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 342         minimum => 0,
 343         default => 0.1,
 344     },
 345     cdrom => {
 346         optional => 1,
 347         type => 'string', format => 'pve-qm-drive',
 348         typetext => 'volume',
 349         description => "This is an alias for option -ide2",
 350     },
 351     cpu => {
 352         optional => 1,
 353         description => "Emulated CPU type.",
 354         type => 'string',
 355         enum => [ qw(486 athlon pentium pentium2 pentium3 coreduo core2duo kvm32 kvm64 qemu32 qemu64 phenom Conroe Penryn Nehalem Westmere SandyBridge IvyBridge Haswell Broadwell Opteron_G1 Opteron_G2 Opteron_G3 Opteron_G4 Opteron_G5 host) ],
 356         default => 'kvm64',
 357     },
 358     parent => get_standard_option('pve-snapshot-name', {
 359         optional => 1,
 360         description => "Parent snapshot name. This is used internally, and should not be modified.",
 361     }),
 362     snaptime => {
 363         optional => 1,
 364         description => "Timestamp for snapshots.",
 365         type => 'integer',
 366         minimum => 0,
 367     },
 368     vmstate => {
 369         optional => 1,
 370         type => 'string', format => 'pve-volume-id',
 371         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 372     },
 373     machine => {
 374         description => "Specific the Qemu machine type.",
 375         type => 'string',
 376         pattern => '(pc|pc(-i440fx)?-\d+\.\d+|q35|pc-q35-\d+\.\d+)',
 377         maxLength => 40,
 378         optional => 1,
 379     },
 380     smbios1 => {
 381         description => "Specify SMBIOS type 1 fields.",
 382         type => 'string', format => 'pve-qm-smbios1',
 383         typetext => "[manufacturer=str][,product=str][,version=str][,serial=str] [,uuid=uuid][,sku=str][,family=str]",
 384         maxLength => 256,
 385         optional => 1,
 386     },
 387 };
 388
 389 # what about other qemu settings ?
 390 #cpu => 'string',
 391 #machine => 'string',
 392 #fda => 'file',
 393 #fdb => 'file',
 394 #mtdblock => 'file',
 395 #sd => 'file',
 396 #pflash => 'file',
 397 #snapshot => 'bool',
 398 #bootp => 'file',
 399 ##tftp => 'dir',
 400 ##smb => 'dir',
 401 #kernel => 'file',
 402 #append => 'string',
 403 #initrd => 'file',
 404 ##soundhw => 'string',
 405
 406 while (my ($k, $v) = each %$confdesc) {
 407     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 408 }
 409
 410 my $MAX_IDE_DISKS = 4;
 411 my $MAX_SCSI_DISKS = 14;
 412 my $MAX_VIRTIO_DISKS = 16;
 413 my $MAX_SATA_DISKS = 6;
 414 my $MAX_USB_DEVICES = 5;
 415 my $MAX_NETS = 32;
 416 my $MAX_UNUSED_DISKS = 8;
 417 my $MAX_HOSTPCI_DEVICES = 4;
 418 my $MAX_SERIAL_PORTS = 4;
 419 my $MAX_PARALLEL_PORTS = 3;
 420 my $MAX_NUMA = 8;
 421 my $MAX_MEM = 4194304;
 422 my $STATICMEM = 1024;
 423
 424 my $numadesc = {
 425     optional => 1,
 426     type => 'string', format => 'pve-qm-numanode',
 427     typetext => "cpus=<id[-id],memory=<mb>[[,hostnodes=<id[-id]>] [,policy=<preferred|bind|interleave>]]",
 428     description => "numa topology",
 429 };
 430 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 431
 432 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 433     $confdesc->{"numa$i"} = $numadesc;
 434 }
 435
 436 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 437                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 438                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 439 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 440
 441 my $netdesc = {
 442     optional => 1,
 443     type => 'string', format => 'pve-qm-net',
 444     typetext => "MODEL=XX:XX:XX:XX:XX:XX [,bridge=<dev>][,queues=<nbqueues>][,rate=<mbps>] [,tag=<vlanid>][,firewall=0|1],link_down=0|1]",
 445     description => <<EODESCR,
 446 Specify network devices.
 447
 448 MODEL is one of: $nic_model_list_txt
 449
 450 XX:XX:XX:XX:XX:XX should be an unique MAC address. This is
 451 automatically generated if not specified.
 452
 453 The bridge parameter can be used to automatically add the interface to a bridge device. The Proxmox VE standard bridge is called 'vmbr0'.
 454
 455 Option 'rate' is used to limit traffic bandwidth from and to this interface. It is specified as floating point number, unit is 'Megabytes per second'.
 456
 457 If you specify no bridge, we create a kvm 'user' (NATed) network device, which provides DHCP and DNS services. The following addresses are used:
 458
 459 10.0.2.2   Gateway
 460 10.0.2.3   DNS Server
 461 10.0.2.4   SMB Server
 462
 463 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 464
 465 EODESCR
 466 };
 467 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 468
 469 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 470     $confdesc->{"net$i"} = $netdesc;
 471 }
 472
 473 my $drivename_hash;
 474
 475 my $idedesc = {
 476     optional => 1,
 477     type => 'string', format => 'pve-qm-drive',
 478     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads] [,discard=ignore|on]',
 479     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
 480 };
 481 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
 482
 483 my $scsidesc = {
 484     optional => 1,
 485     type => 'string', format => 'pve-qm-drive',
 486     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads] [,discard=ignore|on] [,iothread=on] [,queues=<nbqueues>]',
 487     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
 488 };
 489 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 490
 491 my $satadesc = {
 492     optional => 1,
 493     type => 'string', format => 'pve-qm-drive',
 494     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads]  [,discard=ignore|on]',
 495     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
 496 };
 497 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
 498
 499 my $virtiodesc = {
 500     optional => 1,
 501     type => 'string', format => 'pve-qm-drive',
 502     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads]  [,discard=ignore|on] [,iothread=on]',
 503     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
 504 };
 505 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
 506
 507 my $usbdesc = {
 508     optional => 1,
 509     type => 'string', format => 'pve-qm-usb-device',
 510     typetext => 'host=HOSTUSBDEVICE|spice',
 511     description => <<EODESCR,
 512 Configure an USB device (n is 0 to 4). This can be used to
 513 pass-through usb devices to the guest. HOSTUSBDEVICE syntax is:
 514
 515 'bus-port(.port)*' (decimal numbers) or
 516 'vendor_id:product_id' (hexadeciaml numbers)
 517
 518 You can use the 'lsusb -t' command to list existing usb devices.
 519
 520 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 521
 522 The value 'spice' can be used to add a usb redirection devices for spice.
 523
 524 EODESCR
 525 };
 526 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
 527
 528 my $hostpcidesc = {
 529         optional => 1,
 530         type => 'string', format => 'pve-qm-hostpci',
 531         typetext => "[host=]HOSTPCIDEVICE [,driver=kvm|vfio] [,rombar=on|off] [,pcie=0|1] [,x-vga=on|off]",
 532         description => <<EODESCR,
 533 Map host pci devices. HOSTPCIDEVICE syntax is:
 534
 535 'bus:dev.func' (hexadecimal numbers)
 536
 537 You can us the 'lspci' command to list existing pci devices.
 538
 539 The 'rombar' option determines whether or not the device's ROM will be visible in the guest's memory map (default is 'on').
 540
 541 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 542
 543 Experimental: user reported problems with this option.
 544 EODESCR
 545 };
 546 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
 547
 548 my $serialdesc = {
 549         optional => 1,
 550         type => 'string',
 551         pattern => '(/dev/.+|socket)',
 552         description =>  <<EODESCR,
 553 Create a serial device inside the VM (n is 0 to 3), and pass through a host serial device (i.e. /dev/ttyS0), or create a unix socket on the host side (use 'qm terminal' to open a terminal connection).
 554
 555 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 556
 557 Experimental: user reported problems with this option.
 558 EODESCR
 559 };
 560
 561 my $paralleldesc= {
 562         optional => 1,
 563         type => 'string',
 564         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
 565         description =>  <<EODESCR,
 566 Map host parallel devices (n is 0 to 2).
 567
 568 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 569
 570 Experimental: user reported problems with this option.
 571 EODESCR
 572 };
 573
 574 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
 575     $confdesc->{"parallel$i"} = $paralleldesc;
 576 }
 577
 578 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
 579     $confdesc->{"serial$i"} = $serialdesc;
 580 }
 581
 582 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
 583     $confdesc->{"hostpci$i"} = $hostpcidesc;
 584 }
 585
 586 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
 587     $drivename_hash->{"ide$i"} = 1;
 588     $confdesc->{"ide$i"} = $idedesc;
 589 }
 590
 591 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
 592     $drivename_hash->{"sata$i"} = 1;
 593     $confdesc->{"sata$i"} = $satadesc;
 594 }
 595
 596 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
 597     $drivename_hash->{"scsi$i"} = 1;
 598     $confdesc->{"scsi$i"} = $scsidesc ;
 599 }
 600
 601 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
 602     $drivename_hash->{"virtio$i"} = 1;
 603     $confdesc->{"virtio$i"} = $virtiodesc;
 604 }
 605
 606 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
 607     $confdesc->{"usb$i"} = $usbdesc;
 608 }
 609
 610 my $unuseddesc = {
 611     optional => 1,
 612     type => 'string', format => 'pve-volume-id',
 613     description => "Reference to unused volumes.",
 614 };
 615
 616 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
 617     $confdesc->{"unused$i"} = $unuseddesc;
 618 }
 619
 620 my $kvm_api_version = 0;
 621
 622 sub kvm_version {
 623
 624     return $kvm_api_version if $kvm_api_version;
 625
 626     my $fh = IO::File->new("</dev/kvm") ||
 627         return 0;
 628
 629     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
 630         $kvm_api_version = $v;
 631     }
 632
 633     $fh->close();
 634
 635     return  $kvm_api_version;
 636 }
 637
 638 my $kvm_user_version;
 639
 640 sub kvm_user_version {
 641
 642     return $kvm_user_version if $kvm_user_version;
 643
 644     $kvm_user_version = 'unknown';
 645
 646     my $tmp = `kvm -help 2>/dev/null`;
 647
 648     if ($tmp =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)[,\s]/) {
 649         $kvm_user_version = $2;
 650     }
 651
 652     return $kvm_user_version;
 653
 654 }
 655
 656 my $kernel_has_vhost_net = -c '/dev/vhost-net';
 657
 658 sub disknames {
 659     # order is important - used to autoselect boot disk
 660     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
 661             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
 662             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
 663             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))));
 664 }
 665
 666 sub valid_drivename {
 667     my $dev = shift;
 668
 669     return defined($drivename_hash->{$dev});
 670 }
 671
 672 sub option_exists {
 673     my $key = shift;
 674     return defined($confdesc->{$key});
 675 }
 676
 677 sub nic_models {
 678     return $nic_model_list;
 679 }
 680
 681 sub os_list_description {
 682
 683     return {
 684         other => 'Other',
 685         wxp => 'Windows XP',
 686         w2k => 'Windows 2000',
 687         w2k3 =>, 'Windows 2003',
 688         w2k8 => 'Windows 2008',
 689         wvista => 'Windows Vista',
 690         win7 => 'Windows 7',
 691         win8 => 'Windows 8/2012',
 692         l24 => 'Linux 2.4',
 693         l26 => 'Linux 2.6',
 694     };
 695 }
 696
 697 my $cdrom_path;
 698
 699 sub get_cdrom_path {
 700
 701     return  $cdrom_path if $cdrom_path;
 702
 703     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
 704     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
 705     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
 706 }
 707
 708 sub get_iso_path {
 709     my ($storecfg, $vmid, $cdrom) = @_;
 710
 711     if ($cdrom eq 'cdrom') {
 712         return get_cdrom_path();
 713     } elsif ($cdrom eq 'none') {
 714         return '';
 715     } elsif ($cdrom =~ m|^/|) {
 716         return $cdrom;
 717     } else {
 718         return PVE::Storage::path($storecfg, $cdrom);
 719     }
 720 }
 721
 722 # try to convert old style file names to volume IDs
 723 sub filename_to_volume_id {
 724     my ($vmid, $file, $media) = @_;
 725
 726     if (!($file eq 'none' || $file eq 'cdrom' ||
 727           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
 728
 729         return undef if $file =~ m|/|;
 730
 731         if ($media && $media eq 'cdrom') {
 732             $file = "local:iso/$file";
 733         } else {
 734             $file = "local:$vmid/$file";
 735         }
 736     }
 737
 738     return $file;
 739 }
 740
 741 sub verify_media_type {
 742     my ($opt, $vtype, $media) = @_;
 743
 744     return if !$media;
 745
 746     my $etype;
 747     if ($media eq 'disk') {
 748         $etype = 'images';
 749     } elsif ($media eq 'cdrom') {
 750         $etype = 'iso';
 751     } else {
 752         die "internal error";
 753     }
 754
 755     return if ($vtype eq $etype);
 756
 757     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
 758 }
 759
 760 sub cleanup_drive_path {
 761     my ($opt, $storecfg, $drive) = @_;
 762
 763     # try to convert filesystem paths to volume IDs
 764
 765     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
 766         ($drive->{file} !~ m|^/dev/.+|) &&
 767         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
 768         ($drive->{file} !~ m/^\d+$/)) {
 769         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
 770         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
 771         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
 772         verify_media_type($opt, $vtype, $drive->{media});
 773         $drive->{file} = $volid;
 774     }
 775
 776     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
 777 }
 778
 779 sub create_conf_nolock {
 780     my ($vmid, $settings) = @_;
 781
 782     my $filename = config_file($vmid);
 783
 784     die "configuration file '$filename' already exists\n" if -f $filename;
 785
 786     my $defaults = load_defaults();
 787
 788     $settings->{name} = "vm$vmid" if !$settings->{name};
 789     $settings->{memory} = $defaults->{memory} if !$settings->{memory};
 790
 791     my $data = '';
 792     foreach my $opt (keys %$settings) {
 793         next if !$confdesc->{$opt};
 794
 795         my $value = $settings->{$opt};
 796         next if !$value;
 797
 798         $data .= "$opt: $value\n";
 799     }
 800
 801     PVE::Tools::file_set_contents($filename, $data);
 802 }
 803
 804 sub parse_hotplug_features {
 805     my ($data) = @_;
 806
 807     my $res = {};
 808
 809     return $res if $data eq '0';
 810
 811     $data = $confdesc->{hotplug}->{default} if $data eq '1';
 812
 813     foreach my $feature (PVE::Tools::split_list($data)) {
 814         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
 815             $res->{$1} = 1;
 816         } else {
 817             warn "ignoring unknown hotplug feature '$feature'\n";
 818         }
 819     }
 820     return $res;
 821 }
 822
 823 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
 824 sub pve_verify_hotplug_features {
 825     my ($value, $noerr) = @_;
 826
 827     return $value if parse_hotplug_features($value);
 828
 829     return undef if $noerr;
 830
 831     die "unable to parse hotplug option\n";
 832 }
 833
 834 my $parse_size = sub {
 835     my ($value) = @_;
 836
 837     return undef if $value !~ m/^(\d+(\.\d+)?)([KMG])?$/;
 838     my ($size, $unit) = ($1, $3);
 839     if ($unit) {
 840         if ($unit eq 'K') {
 841             $size = $size * 1024;
 842         } elsif ($unit eq 'M') {
 843             $size = $size * 1024 * 1024;
 844         } elsif ($unit eq 'G') {
 845             $size = $size * 1024 * 1024 * 1024;
 846         }
 847     }
 848     return int($size);
 849 };
 850
 851 my $format_size = sub {
 852     my ($size) = @_;
 853
 854     $size = int($size);
 855
 856     my $kb = int($size/1024);
 857     return $size if $kb*1024 != $size;
 858
 859     my $mb = int($kb/1024);
 860     return "${kb}K" if $mb*1024 != $kb;
 861
 862     my $gb = int($mb/1024);
 863     return "${mb}M" if $gb*1024 != $mb;
 864
 865     return "${gb}G";
 866 };
 867
 868 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
 869 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
 870 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
 871 #        [,aio=native|threads][,discard=ignore|on][,iothread=on]
 872
 873 sub parse_drive {
 874     my ($key, $data) = @_;
 875
 876     my $res = {};
 877
 878     # $key may be undefined - used to verify JSON parameters
 879     if (!defined($key)) {
 880         $res->{interface} = 'unknown'; # should not harm when used to verify parameters
 881         $res->{index} = 0;
 882     } elsif ($key =~ m/^([^\d]+)(\d+)$/) {
 883         $res->{interface} = $1;
 884         $res->{index} = $2;
 885     } else {
 886         return undef;
 887     }
 888
 889     foreach my $p (split (/,/, $data)) {
 890         next if $p =~ m/^\s*$/;
 891
 892         if ($p =~ m/^(file|volume|cyls|heads|secs|trans|media|snapshot|cache|format|rerror|werror|backup|aio|bps|mbps|mbps_max|bps_rd|mbps_rd|mbps_rd_max|bps_wr|mbps_wr|mbps_wr_max|iops|iops_max|iops_rd|iops_rd_max|iops_wr|iops_wr_max|size|discard|iothread|queues)=(.+)$/) {
 893             my ($k, $v) = ($1, $2);
 894
 895             $k = 'file' if $k eq 'volume';
 896
 897             return undef if defined $res->{$k};
 898
 899             if ($k eq 'bps' || $k eq 'bps_rd' || $k eq 'bps_wr') {
 900                 return undef if !$v || $v !~ m/^\d+/;
 901                 $k = "m$k";
 902                 $v = sprintf("%.3f", $v / (1024*1024));
 903             }
 904             $res->{$k} = $v;
 905         } else {
 906             if (!$res->{file} && $p !~ m/=/) {
 907                 $res->{file} = $p;
 908             } else {
 909                 return undef;
 910             }
 911         }
 912     }
 913
 914     return undef if !$res->{file};
 915
 916     return undef if $res->{cache} &&
 917         $res->{cache} !~ m/^(off|none|writethrough|writeback|unsafe|directsync)$/;
 918     return undef if $res->{snapshot} && $res->{snapshot} !~ m/^(on|off)$/;
 919     return undef if $res->{cyls} && $res->{cyls} !~ m/^\d+$/;
 920     return undef if $res->{heads} && $res->{heads} !~ m/^\d+$/;
 921     return undef if $res->{secs} && $res->{secs} !~ m/^\d+$/;
 922     return undef if $res->{media} && $res->{media} !~ m/^(disk|cdrom)$/;
 923     return undef if $res->{trans} && $res->{trans} !~ m/^(none|lba|auto)$/;
 924     return undef if $res->{format} && $res->{format} !~ m/^(raw|cow|qcow|qed|qcow2|vmdk|cloop)$/;
 925     return undef if $res->{rerror} && $res->{rerror} !~ m/^(ignore|report|stop)$/;
 926     return undef if $res->{werror} && $res->{werror} !~ m/^(enospc|ignore|report|stop)$/;
 927     return undef if $res->{backup} && $res->{backup} !~ m/^(yes|no)$/;
 928     return undef if $res->{aio} && $res->{aio} !~ m/^(native|threads)$/;
 929     return undef if $res->{discard} && $res->{discard} !~ m/^(ignore|on)$/;
 930     return undef if $res->{iothread} && $res->{iothread} !~ m/^(on)$/;
 931     return undef if $res->{queues} && ($res->{queues} !~ m/^\d+$/ || $res->{queues} < 2);
 932
 933     return undef if $res->{mbps_rd} && $res->{mbps};
 934     return undef if $res->{mbps_wr} && $res->{mbps};
 935
 936     return undef if $res->{mbps} && $res->{mbps} !~ m/^\d+(\.\d+)?$/;
 937     return undef if $res->{mbps_max} && $res->{mbps_max} !~ m/^\d+(\.\d+)?$/;
 938     return undef if $res->{mbps_rd} && $res->{mbps_rd} !~ m/^\d+(\.\d+)?$/;
 939     return undef if $res->{mbps_rd_max} && $res->{mbps_rd_max} !~ m/^\d+(\.\d+)?$/;
 940     return undef if $res->{mbps_wr} && $res->{mbps_wr} !~ m/^\d+(\.\d+)?$/;
 941     return undef if $res->{mbps_wr_max} && $res->{mbps_wr_max} !~ m/^\d+(\.\d+)?$/;
 942
 943     return undef if $res->{iops_rd} && $res->{iops};
 944     return undef if $res->{iops_wr} && $res->{iops};
 945
 946
 947     return undef if $res->{iops} && $res->{iops} !~ m/^\d+$/;
 948     return undef if $res->{iops_max} && $res->{iops_max} !~ m/^\d+$/;
 949     return undef if $res->{iops_rd} && $res->{iops_rd} !~ m/^\d+$/;
 950     return undef if $res->{iops_rd_max} && $res->{iops_rd_max} !~ m/^\d+$/;
 951     return undef if $res->{iops_wr} && $res->{iops_wr} !~ m/^\d+$/;
 952     return undef if $res->{iops_wr_max} && $res->{iops_wr_max} !~ m/^\d+$/;
 953
 954
 955     if ($res->{size}) {
 956         return undef if !defined($res->{size} = &$parse_size($res->{size}));
 957     }
 958
 959     if ($res->{media} && ($res->{media} eq 'cdrom')) {
 960         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
 961         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
 962         return undef if $res->{interface} eq 'virtio';
 963     }
 964
 965     # rerror does not work with scsi drives
 966     if ($res->{rerror}) {
 967         return undef if $res->{interface} eq 'scsi';
 968     }
 969
 970     return $res;
 971 }
 972
 973 my @qemu_drive_options = qw(heads secs cyls trans media format cache snapshot rerror werror aio discard iops iops_rd iops_wr iops_max iops_rd_max iops_wr_max);
 974
 975 sub print_drive {
 976     my ($vmid, $drive) = @_;
 977
 978     my $opts = '';
 979     foreach my $o (@qemu_drive_options, 'mbps', 'mbps_rd', 'mbps_wr', 'mbps_max', 'mbps_rd_max', 'mbps_wr_max', 'backup', 'iothread', 'queues') {
 980         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
 981     }
 982
 983     if ($drive->{size}) {
 984         $opts .= ",size=" . &$format_size($drive->{size});
 985     }
 986
 987     return "$drive->{file}$opts";
 988 }
 989
 990 sub scsi_inquiry {
 991     my($fh, $noerr) = @_;
 992
 993     my $SG_IO = 0x2285;
 994     my $SG_GET_VERSION_NUM = 0x2282;
 995
 996     my $versionbuf = "\x00" x 8;
 997     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
 998     if (!$ret) {
 999         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1000         return undef;
1001     }
1002     my $version = unpack("I", $versionbuf);
1003     if ($version < 30000) {
1004         die "scsi generic interface too old\n"  if !$noerr;
1005         return undef;
1006     }
1007
1008     my $buf = "\x00" x 36;
1009     my $sensebuf = "\x00" x 8;
1010     my $cmd = pack("C x3 C x1", 0x12, 36);
1011
1012     # see /usr/include/scsi/sg.h
1013     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1014
1015     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1016                       length($sensebuf), 0, length($buf), $buf,
1017                       $cmd, $sensebuf, 6000);
1018
1019     $ret = ioctl($fh, $SG_IO, $packet);
1020     if (!$ret) {
1021         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1022         return undef;
1023     }
1024
1025     my @res = unpack($sg_io_hdr_t, $packet);
1026     if ($res[17] || $res[18]) {
1027         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1028         return undef;
1029     }
1030
1031     my $res = {};
1032     (my $byte0, my $byte1, $res->{vendor},
1033      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1034
1035     $res->{removable} = $byte1 & 128 ? 1 : 0;
1036     $res->{type} = $byte0 & 31;
1037
1038     return $res;
1039 }
1040
1041 sub path_is_scsi {
1042     my ($path) = @_;
1043
1044     my $fh = IO::File->new("+<$path") || return undef;
1045     my $res = scsi_inquiry($fh, 1);
1046     close($fh);
1047
1048     return $res;
1049 }
1050
1051 sub machine_type_is_q35 {
1052     my ($conf) = @_;
1053
1054     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1055 }
1056
1057 sub print_tabletdevice_full {
1058     my ($conf) = @_;
1059
1060     my $q35 = machine_type_is_q35($conf);
1061
1062     # we use uhci for old VMs because tablet driver was buggy in older qemu
1063     my $usbbus = $q35 ? "ehci" : "uhci";
1064
1065     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1066 }
1067
1068 sub print_drivedevice_full {
1069     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1070
1071     my $device = '';
1072     my $maxdev = 0;
1073
1074     if ($drive->{interface} eq 'virtio') {
1075         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1076         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1077         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1078     } elsif ($drive->{interface} eq 'scsi') {
1079
1080         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1081         my $unit = $drive->{index} % $maxdev;
1082         my $devicetype = 'hd';
1083         my $path = '';
1084         if (drive_is_cdrom($drive)) {
1085               $devicetype = 'cd';
1086         } else {
1087               if ($drive->{file} =~ m|^/|) {
1088                   $path = $drive->{file};
1089               } else {
1090                   $path = PVE::Storage::path($storecfg, $drive->{file});
1091               }
1092
1093               if($path =~ m/^iscsi\:\/\//){
1094                   $devicetype = 'generic';
1095               } else {
1096                   if (my $info = path_is_scsi($path)) {
1097                       if ($info->{type} == 0) {
1098                           $devicetype = 'block';
1099                       } elsif ($info->{type} == 1) { # tape
1100                           $devicetype = 'generic';
1101                       }
1102                   }
1103               }
1104          }
1105
1106         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1107             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1108         } else {
1109             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1110         }
1111
1112     } elsif ($drive->{interface} eq 'ide'){
1113         $maxdev = 2;
1114         my $controller = int($drive->{index} / $maxdev);
1115         my $unit = $drive->{index} % $maxdev;
1116         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1117
1118         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1119     } elsif ($drive->{interface} eq 'sata'){
1120         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1121         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1122         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1123     } elsif ($drive->{interface} eq 'usb') {
1124         die "implement me";
1125         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1126     } else {
1127         die "unsupported interface type";
1128     }
1129
1130     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1131
1132     return $device;
1133 }
1134
1135 sub get_initiator_name {
1136     my $initiator;
1137
1138     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1139     while (defined(my $line = <$fh>)) {
1140         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1141         $initiator = $1;
1142         last;
1143     }
1144     $fh->close();
1145
1146     return $initiator;
1147 }
1148
1149 sub print_drive_full {
1150     my ($storecfg, $vmid, $drive) = @_;
1151
1152     my $path;
1153     my $volid = $drive->{file};
1154     my $format;
1155
1156     if (drive_is_cdrom($drive)) {
1157         $path = get_iso_path($storecfg, $vmid, $volid);
1158     } else {
1159         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1160         if ($storeid) {
1161             $path = PVE::Storage::path($storecfg, $volid);
1162             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1163             $format = qemu_img_format($scfg, $volname);
1164         } else {
1165             $path = $volid;
1166         }
1167    }
1168
1169     my $opts = '';
1170     foreach my $o (@qemu_drive_options) {
1171         next if $o eq 'bootindex';
1172         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1173     }
1174
1175     $opts .= ",format=$format" if $format && !$drive->{format};
1176
1177     foreach my $o (qw(bps bps_rd bps_wr)) {
1178         my $v = $drive->{"m$o"};
1179         $opts .= ",$o=" . int($v*1024*1024) if $v;
1180     }
1181
1182     my $cache_direct = 0;
1183
1184     if (my $cache = $drive->{cache}) {
1185         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1186     } elsif (!drive_is_cdrom($drive)) {
1187         $opts .= ",cache=none";
1188         $cache_direct = 1;
1189     }
1190
1191     # aio native works only with O_DIRECT
1192     if (!$drive->{aio}) {
1193         if($cache_direct) {
1194             $opts .= ",aio=native";
1195         } else {
1196             $opts .= ",aio=threads";
1197         }
1198     }
1199
1200     my $detectzeroes = $drive->{discard} ? "unmap" : "on";
1201     $opts .= ",detect-zeroes=$detectzeroes" if !drive_is_cdrom($drive);
1202
1203     my $pathinfo = $path ? "file=$path," : '';
1204
1205     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1206 }
1207
1208 sub print_netdevice_full {
1209     my ($vmid, $conf, $net, $netid, $bridges) = @_;
1210
1211     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1212
1213     my $device = $net->{model};
1214     if ($net->{model} eq 'virtio') {
1215          $device = 'virtio-net-pci';
1216      };
1217
1218     my $pciaddr = print_pci_addr("$netid", $bridges);
1219     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1220     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1221         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1222         my $vectors = $net->{queues} * 2 + 2;
1223         $tmpstr .= ",vectors=$vectors,mq=on";
1224     }
1225     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1226     return $tmpstr;
1227 }
1228
1229 sub print_netdev_full {
1230     my ($vmid, $conf, $net, $netid) = @_;
1231
1232     my $i = '';
1233     if ($netid =~ m/^net(\d+)$/) {
1234         $i = int($1);
1235     }
1236
1237     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1238
1239     my $ifname = "tap${vmid}i$i";
1240
1241     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1242     die "interface name '$ifname' is too long (max 15 character)\n"
1243         if length($ifname) >= 16;
1244
1245     my $vhostparam = '';
1246     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1247
1248     my $vmname = $conf->{name} || "vm$vmid";
1249
1250     my $netdev = "";
1251
1252     if ($net->{bridge}) {
1253         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/pve-bridge,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1254     } else {
1255         $netdev = "type=user,id=$netid,hostname=$vmname";
1256     }
1257
1258     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1259
1260     return $netdev;
1261 }
1262
1263 sub drive_is_cdrom {
1264     my ($drive) = @_;
1265
1266     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1267
1268 }
1269
1270 sub parse_numa {
1271     my ($data) = @_;
1272
1273     my $res = {};
1274
1275     foreach my $kvp (split(/,/, $data)) {
1276
1277         if ($kvp =~ m/^memory=(\S+)$/) {
1278             $res->{memory} = $1;
1279         } elsif ($kvp =~ m/^policy=(preferred|bind|interleave)$/) {
1280             $res->{policy} = $1;
1281         } elsif ($kvp =~ m/^cpus=(\d+)(-(\d+))?$/) {
1282             $res->{cpus}->{start} = $1;
1283             $res->{cpus}->{end} = $3;
1284         } elsif ($kvp =~ m/^hostnodes=(\d+)(-(\d+))?$/) {
1285             $res->{hostnodes}->{start} = $1;
1286             $res->{hostnodes}->{end} = $3;
1287         } else {
1288             return undef;
1289         }
1290     }
1291
1292     return $res;
1293 }
1294
1295 sub parse_hostpci {
1296     my ($value) = @_;
1297
1298     return undef if !$value;
1299
1300
1301     my @list = split(/,/, $value);
1302     my $found;
1303
1304     my $res = {};
1305     foreach my $kv (@list) {
1306
1307         if ($kv =~ m/^(host=)?([a-f0-9]{2}:[a-f0-9]{2})(\.([a-f0-9]))?$/) {
1308             $found = 1;
1309             if(defined($4)){
1310                 push @{$res->{pciid}}, { id => $2 , function => $4};
1311
1312             }else{
1313                 my $pcidevices = lspci($2);
1314                 $res->{pciid} = $pcidevices->{$2};
1315             }
1316         } elsif ($kv =~ m/^driver=(kvm|vfio)$/) {
1317             $res->{driver} = $1;
1318         } elsif ($kv =~ m/^rombar=(on|off)$/) {
1319             $res->{rombar} = $1;
1320         } elsif ($kv =~ m/^x-vga=(on|off)$/) {
1321             $res->{'x-vga'} = $1;
1322         } elsif ($kv =~ m/^pcie=(\d+)$/) {
1323             $res->{pcie} = 1 if $1 == 1;
1324         } else {
1325             warn "unknown hostpci setting '$kv'\n";
1326         }
1327     }
1328
1329     return undef if !$found;
1330
1331     return $res;
1332 }
1333
1334 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1335 sub parse_net {
1336     my ($data) = @_;
1337
1338     my $res = {};
1339
1340     foreach my $kvp (split(/,/, $data)) {
1341
1342         if ($kvp =~ m/^(ne2k_pci|e1000|e1000-82540em|e1000-82544gc|e1000-82545em|rtl8139|pcnet|virtio|ne2k_isa|i82551|i82557b|i82559er|vmxnet3)(=([0-9a-f]{2}(:[0-9a-f]{2}){5}))?$/i) {
1343             my $model = lc($1);
1344             my $mac = defined($3) ? uc($3) : PVE::Tools::random_ether_addr();
1345             $res->{model} = $model;
1346             $res->{macaddr} = $mac;
1347         } elsif ($kvp =~ m/^bridge=(\S+)$/) {
1348             $res->{bridge} = $1;
1349         } elsif ($kvp =~ m/^queues=(\d+)$/) {
1350             $res->{queues} = $1;
1351         } elsif ($kvp =~ m/^rate=(\d+(\.\d+)?)$/) {
1352             $res->{rate} = $1;
1353         } elsif ($kvp =~ m/^tag=(\d+)$/) {
1354             $res->{tag} = $1;
1355         } elsif ($kvp =~ m/^firewall=([01])$/) {
1356             $res->{firewall} = $1;
1357         } elsif ($kvp =~ m/^link_down=([01])$/) {
1358             $res->{link_down} = $1;
1359         } else {
1360             return undef;
1361         }
1362
1363     }
1364
1365     return undef if !$res->{model};
1366
1367     return $res;
1368 }
1369
1370 sub print_net {
1371     my $net = shift;
1372
1373     my $res = "$net->{model}";
1374     $res .= "=$net->{macaddr}" if $net->{macaddr};
1375     $res .= ",bridge=$net->{bridge}" if $net->{bridge};
1376     $res .= ",rate=$net->{rate}" if $net->{rate};
1377     $res .= ",tag=$net->{tag}" if $net->{tag};
1378     $res .= ",firewall=1" if $net->{firewall};
1379     $res .= ",link_down=1" if $net->{link_down};
1380     $res .= ",queues=$net->{queues}" if $net->{queues};
1381
1382     return $res;
1383 }
1384
1385 sub add_random_macs {
1386     my ($settings) = @_;
1387
1388     foreach my $opt (keys %$settings) {
1389         next if $opt !~ m/^net(\d+)$/;
1390         my $net = parse_net($settings->{$opt});
1391         next if !$net;
1392         $settings->{$opt} = print_net($net);
1393     }
1394 }
1395
1396 sub add_unused_volume {
1397     my ($config, $volid) = @_;
1398
1399     my $key;
1400     for (my $ind = $MAX_UNUSED_DISKS - 1; $ind >= 0; $ind--) {
1401         my $test = "unused$ind";
1402         if (my $vid = $config->{$test}) {
1403             return if $vid eq $volid; # do not add duplicates
1404         } else {
1405             $key = $test;
1406         }
1407     }
1408
1409     die "To many unused volume - please delete them first.\n" if !$key;
1410
1411     $config->{$key} = $volid;
1412
1413     return $key;
1414 }
1415
1416 sub vm_is_volid_owner {
1417     my ($storecfg, $vmid, $volid) = @_;
1418
1419     if ($volid !~  m|^/|) {
1420         my ($path, $owner);
1421         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1422         if ($owner && ($owner == $vmid)) {
1423             return 1;
1424         }
1425     }
1426
1427     return undef;
1428 }
1429
1430 sub vmconfig_delete_pending_option {
1431     my ($conf, $key) = @_;
1432
1433     delete $conf->{pending}->{$key};
1434     my $pending_delete_hash = { $key => 1 };
1435     foreach my $opt (PVE::Tools::split_list($conf->{pending}->{delete})) {
1436         $pending_delete_hash->{$opt} = 1;
1437     }
1438     $conf->{pending}->{delete} = join(',', keys %$pending_delete_hash);
1439 }
1440
1441 sub vmconfig_undelete_pending_option {
1442     my ($conf, $key) = @_;
1443
1444     my $pending_delete_hash = {};
1445     foreach my $opt (PVE::Tools::split_list($conf->{pending}->{delete})) {
1446         $pending_delete_hash->{$opt} = 1;
1447     }
1448     delete $pending_delete_hash->{$key};
1449
1450     my @keylist = keys %$pending_delete_hash;
1451     if (scalar(@keylist)) {
1452         $conf->{pending}->{delete} = join(',', @keylist);
1453     } else {
1454         delete $conf->{pending}->{delete};
1455     }
1456 }
1457
1458 sub vmconfig_register_unused_drive {
1459     my ($storecfg, $vmid, $conf, $drive) = @_;
1460
1461     if (!drive_is_cdrom($drive)) {
1462         my $volid = $drive->{file};
1463         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1464             add_unused_volume($conf, $volid, $vmid);
1465         }
1466     }
1467 }
1468
1469 sub vmconfig_cleanup_pending {
1470     my ($conf) = @_;
1471
1472     # remove pending changes when nothing changed
1473     my $changes;
1474     foreach my $opt (keys %{$conf->{pending}}) {
1475         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
1476             $changes = 1;
1477             delete $conf->{pending}->{$opt};
1478         }
1479     }
1480
1481     # remove delete if option is not set
1482     my $pending_delete_hash = {};
1483     foreach my $opt (PVE::Tools::split_list($conf->{pending}->{delete})) {
1484         if (defined($conf->{$opt})) {
1485             $pending_delete_hash->{$opt} = 1;
1486         } else {
1487             $changes = 1;
1488         }
1489     }
1490
1491     my @keylist = keys %$pending_delete_hash;
1492     if (scalar(@keylist)) {
1493         $conf->{pending}->{delete} = join(',', @keylist);
1494     } else {
1495         delete $conf->{pending}->{delete};
1496     }
1497
1498     return $changes;
1499 }
1500
1501 my $valid_smbios1_options = {
1502     manufacturer => '\S+',
1503     product => '\S+',
1504     version => '\S+',
1505     serial => '\S+',
1506     uuid => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1507     sku => '\S+',
1508     family => '\S+',
1509 };
1510
1511 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
1512 sub parse_smbios1 {
1513     my ($data) = @_;
1514
1515     my $res = {};
1516
1517     foreach my $kvp (split(/,/, $data)) {
1518         return undef if $kvp !~ m/^(\S+)=(.+)$/;
1519         my ($k, $v) = split(/=/, $kvp);
1520         return undef if !defined($k) || !defined($v);
1521         return undef if !$valid_smbios1_options->{$k};
1522         return undef if $v !~ m/^$valid_smbios1_options->{$k}$/;
1523         $res->{$k} = $v;
1524     }
1525
1526     return $res;
1527 }
1528
1529 sub print_smbios1 {
1530     my ($smbios1) = @_;
1531
1532     my $data = '';
1533     foreach my $k (keys %$smbios1) {
1534         next if !defined($smbios1->{$k});
1535         next if !$valid_smbios1_options->{$k};
1536         $data .= ',' if $data;
1537         $data .= "$k=$smbios1->{$k}";
1538     }
1539     return $data;
1540 }
1541
1542 PVE::JSONSchema::register_format('pve-qm-smbios1', \&verify_smbios1);
1543 sub verify_smbios1 {
1544     my ($value, $noerr) = @_;
1545
1546     return $value if parse_smbios1($value);
1547
1548     return undef if $noerr;
1549
1550     die "unable to parse smbios (type 1) options\n";
1551 }
1552
1553 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
1554 sub verify_bootdisk {
1555     my ($value, $noerr) = @_;
1556
1557     return $value if valid_drivename($value);
1558
1559     return undef if $noerr;
1560
1561     die "invalid boot disk '$value'\n";
1562 }
1563
1564 PVE::JSONSchema::register_format('pve-qm-numanode', \&verify_numa);
1565 sub verify_numa {
1566     my ($value, $noerr) = @_;
1567
1568     return $value if parse_numa($value);
1569
1570     return undef if $noerr;
1571
1572     die "unable to parse numa options\n";
1573 }
1574
1575 PVE::JSONSchema::register_format('pve-qm-net', \&verify_net);
1576 sub verify_net {
1577     my ($value, $noerr) = @_;
1578
1579     return $value if parse_net($value);
1580
1581     return undef if $noerr;
1582
1583     die "unable to parse network options\n";
1584 }
1585
1586 PVE::JSONSchema::register_format('pve-qm-drive', \&verify_drive);
1587 sub verify_drive {
1588     my ($value, $noerr) = @_;
1589
1590     return $value if parse_drive(undef, $value);
1591
1592     return undef if $noerr;
1593
1594     die "unable to parse drive options\n";
1595 }
1596
1597 PVE::JSONSchema::register_format('pve-qm-hostpci', \&verify_hostpci);
1598 sub verify_hostpci {
1599     my ($value, $noerr) = @_;
1600
1601     return $value if parse_hostpci($value);
1602
1603     return undef if $noerr;
1604
1605     die "unable to parse pci id\n";
1606 }
1607
1608 PVE::JSONSchema::register_format('pve-qm-watchdog', \&verify_watchdog);
1609 sub verify_watchdog {
1610     my ($value, $noerr) = @_;
1611
1612     return $value if parse_watchdog($value);
1613
1614     return undef if $noerr;
1615
1616     die "unable to parse watchdog options\n";
1617 }
1618
1619 sub parse_watchdog {
1620     my ($value) = @_;
1621
1622     return undef if !$value;
1623
1624     my $res = {};
1625
1626     foreach my $p (split(/,/, $value)) {
1627         next if $p =~ m/^\s*$/;
1628
1629         if ($p =~ m/^(model=)?(i6300esb|ib700)$/) {
1630             $res->{model} = $2;
1631         } elsif ($p =~ m/^(action=)?(reset|shutdown|poweroff|pause|debug|none)$/) {
1632             $res->{action} = $2;
1633         } else {
1634             return undef;
1635         }
1636     }
1637
1638     return $res;
1639 }
1640
1641 sub parse_usb_device {
1642     my ($value) = @_;
1643
1644     return undef if !$value;
1645
1646     my @dl = split(/,/, $value);
1647     my $found;
1648
1649     my $res = {};
1650     foreach my $v (@dl) {
1651         if ($v =~ m/^host=(0x)?([0-9A-Fa-f]{4}):(0x)?([0-9A-Fa-f]{4})$/) {
1652             $found = 1;
1653             $res->{vendorid} = $2;
1654             $res->{productid} = $4;
1655         } elsif ($v =~ m/^host=(\d+)\-(\d+(\.\d+)*)$/) {
1656             $found = 1;
1657             $res->{hostbus} = $1;
1658             $res->{hostport} = $2;
1659         } elsif ($v =~ m/^spice$/) {
1660             $found = 1;
1661             $res->{spice} = 1;
1662         } else {
1663             return undef;
1664         }
1665     }
1666     return undef if !$found;
1667
1668     return $res;
1669 }
1670
1671 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
1672 sub verify_usb_device {
1673     my ($value, $noerr) = @_;
1674
1675     return $value if parse_usb_device($value);
1676
1677     return undef if $noerr;
1678
1679     die "unable to parse usb device\n";
1680 }
1681
1682 # add JSON properties for create and set function
1683 sub json_config_properties {
1684     my $prop = shift;
1685
1686     foreach my $opt (keys %$confdesc) {
1687         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
1688         $prop->{$opt} = $confdesc->{$opt};
1689     }
1690
1691     return $prop;
1692 }
1693
1694 sub check_type {
1695     my ($key, $value) = @_;
1696
1697     die "unknown setting '$key'\n" if !$confdesc->{$key};
1698
1699     my $type = $confdesc->{$key}->{type};
1700
1701     if (!defined($value)) {
1702         die "got undefined value\n";
1703     }
1704
1705     if ($value =~ m/[\n\r]/) {
1706         die "property contains a line feed\n";
1707     }
1708
1709     if ($type eq 'boolean') {
1710         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
1711         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
1712         die "type check ('boolean') failed - got '$value'\n";
1713     } elsif ($type eq 'integer') {
1714         return int($1) if $value =~ m/^(\d+)$/;
1715         die "type check ('integer') failed - got '$value'\n";
1716     } elsif ($type eq 'number') {
1717         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
1718         die "type check ('number') failed - got '$value'\n";
1719     } elsif ($type eq 'string') {
1720         if (my $fmt = $confdesc->{$key}->{format}) {
1721             if ($fmt eq 'pve-qm-drive') {
1722                 # special case - we need to pass $key to parse_drive()
1723                 my $drive = parse_drive($key, $value);
1724                 return $value if $drive;
1725                 die "unable to parse drive options\n";
1726             }
1727             PVE::JSONSchema::check_format($fmt, $value);
1728             return $value;
1729         }
1730         $value =~ s/^\"(.*)\"$/$1/;
1731         return $value;
1732     } else {
1733         die "internal error"
1734     }
1735 }
1736
1737 sub lock_config_full {
1738     my ($vmid, $timeout, $code, @param) = @_;
1739
1740     my $filename = config_file_lock($vmid);
1741
1742     my $res = lock_file($filename, $timeout, $code, @param);
1743
1744     die $@ if $@;
1745
1746     return $res;
1747 }
1748
1749 sub lock_config_mode {
1750     my ($vmid, $timeout, $shared, $code, @param) = @_;
1751
1752     my $filename = config_file_lock($vmid);
1753
1754     my $res = lock_file_full($filename, $timeout, $shared, $code, @param);
1755
1756     die $@ if $@;
1757
1758     return $res;
1759 }
1760
1761 sub lock_config {
1762     my ($vmid, $code, @param) = @_;
1763
1764     return lock_config_full($vmid, 10, $code, @param);
1765 }
1766
1767 sub cfs_config_path {
1768     my ($vmid, $node) = @_;
1769
1770     $node = $nodename if !$node;
1771     return "nodes/$node/qemu-server/$vmid.conf";
1772 }
1773
1774 sub check_iommu_support{
1775     #fixme : need to check IOMMU support
1776     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
1777
1778     my $iommu=1;
1779     return $iommu;
1780
1781 }
1782
1783 sub config_file {
1784     my ($vmid, $node) = @_;
1785
1786     my $cfspath = cfs_config_path($vmid, $node);
1787     return "/etc/pve/$cfspath";
1788 }
1789
1790 sub config_file_lock {
1791     my ($vmid) = @_;
1792
1793     return "$lock_dir/lock-$vmid.conf";
1794 }
1795
1796 sub touch_config {
1797     my ($vmid) = @_;
1798
1799     my $conf = config_file($vmid);
1800     utime undef, undef, $conf;
1801 }
1802
1803 sub destroy_vm {
1804     my ($storecfg, $vmid, $keep_empty_config) = @_;
1805
1806     my $conffile = config_file($vmid);
1807
1808     my $conf = load_config($vmid);
1809
1810     check_lock($conf);
1811
1812     # only remove disks owned by this VM
1813     foreach_drive($conf, sub {
1814         my ($ds, $drive) = @_;
1815
1816         return if drive_is_cdrom($drive);
1817
1818         my $volid = $drive->{file};
1819
1820         return if !$volid || $volid =~ m|^/|;
1821
1822         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
1823         return if !$path || !$owner || ($owner != $vmid);
1824
1825         PVE::Storage::vdisk_free($storecfg, $volid);
1826     });
1827
1828     if ($keep_empty_config) {
1829         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
1830     } else {
1831         unlink $conffile;
1832     }
1833
1834     # also remove unused disk
1835     eval {
1836         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
1837
1838         eval {
1839             PVE::Storage::foreach_volid($dl, sub {
1840                 my ($volid, $sid, $volname, $d) = @_;
1841                 PVE::Storage::vdisk_free($storecfg, $volid);
1842             });
1843         };
1844         warn $@ if $@;
1845
1846     };
1847     warn $@ if $@;
1848 }
1849
1850 sub load_config {
1851     my ($vmid, $node) = @_;
1852
1853     my $cfspath = cfs_config_path($vmid, $node);
1854
1855     my $conf = PVE::Cluster::cfs_read_file($cfspath);
1856
1857     die "no such VM ('$vmid')\n" if !defined($conf);
1858
1859     return $conf;
1860 }
1861
1862 sub parse_vm_config {
1863     my ($filename, $raw) = @_;
1864
1865     return undef if !defined($raw);
1866
1867     my $res = {
1868         digest => Digest::SHA::sha1_hex($raw),
1869         snapshots => {},
1870         pending => {},
1871     };
1872
1873     $filename =~ m|/qemu-server/(\d+)\.conf$|
1874         || die "got strange filename '$filename'";
1875
1876     my $vmid = $1;
1877
1878     my $conf = $res;
1879     my $descr = '';
1880     my $section = '';
1881
1882     my @lines = split(/\n/, $raw);
1883     foreach my $line (@lines) {
1884         next if $line =~ m/^\s*$/;
1885
1886         if ($line =~ m/^\[PENDING\]\s*$/i) {
1887             $section = 'pending';
1888             $conf->{description} = $descr if $descr;
1889             $descr = '';
1890             $conf = $res->{$section} = {};
1891             next;
1892
1893         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
1894             $section = $1;
1895             $conf->{description} = $descr if $descr;
1896             $descr = '';
1897             $conf = $res->{snapshots}->{$section} = {};
1898             next;
1899         }
1900
1901         if ($line =~ m/^\#(.*)\s*$/) {
1902             $descr .= PVE::Tools::decode_text($1) . "\n";
1903             next;
1904         }
1905
1906         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
1907             $descr .= PVE::Tools::decode_text($2);
1908         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
1909             $conf->{snapstate} = $1;
1910         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
1911             my $key = $1;
1912             my $value = $2;
1913             $conf->{$key} = $value;
1914         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
1915             my $value = $1;
1916             if ($section eq 'pending') {
1917                 $conf->{delete} = $value; # we parse this later
1918             } else {
1919                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
1920             }
1921         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
1922             my $key = $1;
1923             my $value = $2;
1924             eval { $value = check_type($key, $value); };
1925             if ($@) {
1926                 warn "vm $vmid - unable to parse value of '$key' - $@";
1927             } else {
1928                 my $fmt = $confdesc->{$key}->{format};
1929                 if ($fmt && $fmt eq 'pve-qm-drive') {
1930                     my $v = parse_drive($key, $value);
1931                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
1932                         $v->{file} = $volid;
1933                         $value = print_drive($vmid, $v);
1934                     } else {
1935                         warn "vm $vmid - unable to parse value of '$key'\n";
1936                         next;
1937                     }
1938                 }
1939
1940                 if ($key eq 'cdrom') {
1941                     $conf->{ide2} = $value;
1942                 } else {
1943                     $conf->{$key} = $value;
1944                 }
1945             }
1946         }
1947     }
1948
1949     $conf->{description} = $descr if $descr;
1950
1951     delete $res->{snapstate}; # just to be sure
1952
1953     return $res;
1954 }
1955
1956 sub write_vm_config {
1957     my ($filename, $conf) = @_;
1958
1959     delete $conf->{snapstate}; # just to be sure
1960
1961     if ($conf->{cdrom}) {
1962         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
1963         $conf->{ide2} = $conf->{cdrom};
1964         delete $conf->{cdrom};
1965     }
1966
1967     # we do not use 'smp' any longer
1968     if ($conf->{sockets}) {
1969         delete $conf->{smp};
1970     } elsif ($conf->{smp}) {
1971         $conf->{sockets} = $conf->{smp};
1972         delete $conf->{cores};
1973         delete $conf->{smp};
1974     }
1975
1976     my $used_volids = {};
1977
1978     my $cleanup_config = sub {
1979         my ($cref, $pending, $snapname) = @_;
1980
1981         foreach my $key (keys %$cref) {
1982             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
1983                 $key eq 'snapstate' || $key eq 'pending';
1984             my $value = $cref->{$key};
1985             if ($key eq 'delete') {
1986                 die "propertry 'delete' is only allowed in [PENDING]\n"
1987                     if !$pending;
1988                 # fixme: check syntax?
1989                 next;
1990             }
1991             eval { $value = check_type($key, $value); };
1992             die "unable to parse value of '$key' - $@" if $@;
1993
1994             $cref->{$key} = $value;
1995
1996             if (!$snapname && valid_drivename($key)) {
1997                 my $drive = parse_drive($key, $value);
1998                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
1999             }
2000         }
2001     };
2002
2003     &$cleanup_config($conf);
2004
2005     &$cleanup_config($conf->{pending}, 1);
2006
2007     foreach my $snapname (keys %{$conf->{snapshots}}) {
2008         die "internal error" if $snapname eq 'pending';
2009         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2010     }
2011
2012     # remove 'unusedX' settings if we re-add a volume
2013     foreach my $key (keys %$conf) {
2014         my $value = $conf->{$key};
2015         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2016             delete $conf->{$key};
2017         }
2018     }
2019
2020     my $generate_raw_config = sub {
2021         my ($conf) = @_;
2022
2023         my $raw = '';
2024
2025         # add description as comment to top of file
2026         my $descr = $conf->{description} || '';
2027         foreach my $cl (split(/\n/, $descr)) {
2028             $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2029         }
2030
2031         foreach my $key (sort keys %$conf) {
2032             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2033             $raw .= "$key: $conf->{$key}\n";
2034         }
2035         return $raw;
2036     };
2037
2038     my $raw = &$generate_raw_config($conf);
2039
2040     if (scalar(keys %{$conf->{pending}})){
2041         $raw .= "\n[PENDING]\n";
2042         $raw .= &$generate_raw_config($conf->{pending});
2043     }
2044
2045     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2046         $raw .= "\n[$snapname]\n";
2047         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2048     }
2049
2050     return $raw;
2051 }
2052
2053 sub update_config_nolock {
2054     my ($vmid, $conf, $skiplock) = @_;
2055
2056     check_lock($conf) if !$skiplock;
2057
2058     my $cfspath = cfs_config_path($vmid);
2059
2060     PVE::Cluster::cfs_write_file($cfspath, $conf);
2061 }
2062
2063 sub update_config {
2064     my ($vmid, $conf, $skiplock) = @_;
2065
2066     lock_config($vmid, &update_config_nolock, $conf, $skiplock);
2067 }
2068
2069 sub load_defaults {
2070
2071     my $res = {};
2072
2073     # we use static defaults from our JSON schema configuration
2074     foreach my $key (keys %$confdesc) {
2075         if (defined(my $default = $confdesc->{$key}->{default})) {
2076             $res->{$key} = $default;
2077         }
2078     }
2079
2080     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2081     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2082
2083     return $res;
2084 }
2085
2086 sub config_list {
2087     my $vmlist = PVE::Cluster::get_vmlist();
2088     my $res = {};
2089     return $res if !$vmlist || !$vmlist->{ids};
2090     my $ids = $vmlist->{ids};
2091
2092     foreach my $vmid (keys %$ids) {
2093         my $d = $ids->{$vmid};
2094         next if !$d->{node} || $d->{node} ne $nodename;
2095         next if !$d->{type} || $d->{type} ne 'qemu';
2096         $res->{$vmid}->{exists} = 1;
2097     }
2098     return $res;
2099 }
2100
2101 # test if VM uses local resources (to prevent migration)
2102 sub check_local_resources {
2103     my ($conf, $noerr) = @_;
2104
2105     my $loc_res = 0;
2106
2107     $loc_res = 1 if $conf->{hostusb}; # old syntax
2108     $loc_res = 1 if $conf->{hostpci}; # old syntax
2109
2110     foreach my $k (keys %$conf) {
2111         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2112         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2113     }
2114
2115     die "VM uses local resources\n" if $loc_res && !$noerr;
2116
2117     return $loc_res;
2118 }
2119
2120 # check if used storages are available on all nodes (use by migrate)
2121 sub check_storage_availability {
2122     my ($storecfg, $conf, $node) = @_;
2123
2124     foreach_drive($conf, sub {
2125         my ($ds, $drive) = @_;
2126
2127         my $volid = $drive->{file};
2128         return if !$volid;
2129
2130         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2131         return if !$sid;
2132
2133         # check if storage is available on both nodes
2134         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2135         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2136    });
2137 }
2138
2139 # list nodes where all VM images are available (used by has_feature API)
2140 sub shared_nodes {
2141     my ($conf, $storecfg) = @_;
2142
2143     my $nodelist = PVE::Cluster::get_nodelist();
2144     my $nodehash = { map { $_ => 1 } @$nodelist };
2145     my $nodename = PVE::INotify::nodename();
2146
2147     foreach_drive($conf, sub {
2148         my ($ds, $drive) = @_;
2149
2150         my $volid = $drive->{file};
2151         return if !$volid;
2152
2153         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2154         if ($storeid) {
2155             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2156             if ($scfg->{disable}) {
2157                 $nodehash = {};
2158             } elsif (my $avail = $scfg->{nodes}) {
2159                 foreach my $node (keys %$nodehash) {
2160                     delete $nodehash->{$node} if !$avail->{$node};
2161                 }
2162             } elsif (!$scfg->{shared}) {
2163                 foreach my $node (keys %$nodehash) {
2164                     delete $nodehash->{$node} if $node ne $nodename
2165                 }
2166             }
2167         }
2168     });
2169
2170     return $nodehash
2171 }
2172
2173 sub check_lock {
2174     my ($conf) = @_;
2175
2176     die "VM is locked ($conf->{lock})\n" if $conf->{lock};
2177 }
2178
2179 sub check_cmdline {
2180     my ($pidfile, $pid) = @_;
2181
2182     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2183     if (defined($fh)) {
2184         my $line = <$fh>;
2185         $fh->close;
2186         return undef if !$line;
2187         my @param = split(/\0/, $line);
2188
2189         my $cmd = $param[0];
2190         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2191
2192         for (my $i = 0; $i < scalar (@param); $i++) {
2193             my $p = $param[$i];
2194             next if !$p;
2195             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2196                 my $p = $param[$i+1];
2197                 return 1 if $p && ($p eq $pidfile);
2198                 return undef;
2199             }
2200         }
2201     }
2202     return undef;
2203 }
2204
2205 sub check_running {
2206     my ($vmid, $nocheck, $node) = @_;
2207
2208     my $filename = config_file($vmid, $node);
2209
2210     die "unable to find configuration file for VM $vmid - no such machine\n"
2211         if !$nocheck && ! -f $filename;
2212
2213     my $pidfile = pidfile_name($vmid);
2214
2215     if (my $fd = IO::File->new("<$pidfile")) {
2216         my $st = stat($fd);
2217         my $line = <$fd>;
2218         close($fd);
2219
2220         my $mtime = $st->mtime;
2221         if ($mtime > time()) {
2222             warn "file '$filename' modified in future\n";
2223         }
2224
2225         if ($line =~ m/^(\d+)$/) {
2226             my $pid = $1;
2227             if (check_cmdline($pidfile, $pid)) {
2228                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2229                     return $pid;
2230                 }
2231             }
2232         }
2233     }
2234
2235     return undef;
2236 }
2237
2238 sub vzlist {
2239
2240     my $vzlist = config_list();
2241
2242     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2243
2244     while (defined(my $de = $fd->read)) {
2245         next if $de !~ m/^(\d+)\.pid$/;
2246         my $vmid = $1;
2247         next if !defined($vzlist->{$vmid});
2248         if (my $pid = check_running($vmid)) {
2249             $vzlist->{$vmid}->{pid} = $pid;
2250         }
2251     }
2252
2253     return $vzlist;
2254 }
2255
2256 sub disksize {
2257     my ($storecfg, $conf) = @_;
2258
2259     my $bootdisk = $conf->{bootdisk};
2260     return undef if !$bootdisk;
2261     return undef if !valid_drivename($bootdisk);
2262
2263     return undef if !$conf->{$bootdisk};
2264
2265     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2266     return undef if !defined($drive);
2267
2268     return undef if drive_is_cdrom($drive);
2269
2270     my $volid = $drive->{file};
2271     return undef if !$volid;
2272
2273     return $drive->{size};
2274 }
2275
2276 my $last_proc_pid_stat;
2277
2278 # get VM status information
2279 # This must be fast and should not block ($full == false)
2280 # We only query KVM using QMP if $full == true (this can be slow)
2281 sub vmstatus {
2282     my ($opt_vmid, $full) = @_;
2283
2284     my $res = {};
2285
2286     my $storecfg = PVE::Storage::config();
2287
2288     my $list = vzlist();
2289     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2290
2291     my $cpucount = $cpuinfo->{cpus} || 1;
2292
2293     foreach my $vmid (keys %$list) {
2294         next if $opt_vmid && ($vmid ne $opt_vmid);
2295
2296         my $cfspath = cfs_config_path($vmid);
2297         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2298
2299         my $d = {};
2300         $d->{pid} = $list->{$vmid}->{pid};
2301
2302         # fixme: better status?
2303         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2304
2305         my $size = disksize($storecfg, $conf);
2306         if (defined($size)) {
2307             $d->{disk} = 0; # no info available
2308             $d->{maxdisk} = $size;
2309         } else {
2310             $d->{disk} = 0;
2311             $d->{maxdisk} = 0;
2312         }
2313
2314         $d->{cpus} = ($conf->{sockets} || 1) * ($conf->{cores} || 1);
2315         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2316         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2317
2318         $d->{name} = $conf->{name} || "VM $vmid";
2319         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024) : 0;
2320
2321         if ($conf->{balloon}) {
2322             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2323             $d->{shares} = defined($conf->{shares}) ? $conf->{shares} : 1000;
2324         }
2325
2326         $d->{uptime} = 0;
2327         $d->{cpu} = 0;
2328         $d->{mem} = 0;
2329
2330         $d->{netout} = 0;
2331         $d->{netin} = 0;
2332
2333         $d->{diskread} = 0;
2334         $d->{diskwrite} = 0;
2335
2336         $d->{template} = is_template($conf);
2337
2338         $res->{$vmid} = $d;
2339     }
2340
2341     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2342     foreach my $dev (keys %$netdev) {
2343         next if $dev !~ m/^tap([1-9]\d*)i/;
2344         my $vmid = $1;
2345         my $d = $res->{$vmid};
2346         next if !$d;
2347
2348         $d->{netout} += $netdev->{$dev}->{receive};
2349         $d->{netin} += $netdev->{$dev}->{transmit};
2350
2351         if ($full) {
2352             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2353             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2354         }
2355
2356     }
2357
2358     my $ctime = gettimeofday;
2359
2360     foreach my $vmid (keys %$list) {
2361
2362         my $d = $res->{$vmid};
2363         my $pid = $d->{pid};
2364         next if !$pid;
2365
2366         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2367         next if !$pstat; # not running
2368
2369         my $used = $pstat->{utime} + $pstat->{stime};
2370
2371         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2372
2373         if ($pstat->{vsize}) {
2374             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2375         }
2376
2377         my $old = $last_proc_pid_stat->{$pid};
2378         if (!$old) {
2379             $last_proc_pid_stat->{$pid} = {
2380                 time => $ctime,
2381                 used => $used,
2382                 cpu => 0,
2383             };
2384             next;
2385         }
2386
2387         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2388
2389         if ($dtime > 1000) {
2390             my $dutime = $used -  $old->{used};
2391
2392             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2393             $last_proc_pid_stat->{$pid} = {
2394                 time => $ctime,
2395                 used => $used,
2396                 cpu => $d->{cpu},
2397             };
2398         } else {
2399             $d->{cpu} = $old->{cpu};
2400         }
2401     }
2402
2403     return $res if !$full;
2404
2405     my $qmpclient = PVE::QMPClient->new();
2406
2407     my $ballooncb = sub {
2408         my ($vmid, $resp) = @_;
2409
2410         my $info = $resp->{'return'};
2411         return if !$info->{max_mem};
2412
2413         my $d = $res->{$vmid};
2414
2415         # use memory assigned to VM
2416         $d->{maxmem} = $info->{max_mem};
2417         $d->{balloon} = $info->{actual};
2418
2419         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2420             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2421             $d->{freemem} = $info->{free_mem};
2422         }
2423
2424         $d->{ballooninfo} = $info;
2425     };
2426
2427     my $blockstatscb = sub {
2428         my ($vmid, $resp) = @_;
2429         my $data = $resp->{'return'} || [];
2430         my $totalrdbytes = 0;
2431         my $totalwrbytes = 0;
2432
2433         for my $blockstat (@$data) {
2434             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2435             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2436
2437             $blockstat->{device} =~ s/drive-//;
2438             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2439         }
2440         $res->{$vmid}->{diskread} = $totalrdbytes;
2441         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2442     };
2443
2444     my $statuscb = sub {
2445         my ($vmid, $resp) = @_;
2446
2447         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2448         # this fails if ballon driver is not loaded, so this must be
2449         # the last commnand (following command are aborted if this fails).
2450         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2451
2452         my $status = 'unknown';
2453         if (!defined($status = $resp->{'return'}->{status})) {
2454             warn "unable to get VM status\n";
2455             return;
2456         }
2457
2458         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2459     };
2460
2461     foreach my $vmid (keys %$list) {
2462         next if $opt_vmid && ($vmid ne $opt_vmid);
2463         next if !$res->{$vmid}->{pid}; # not running
2464         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2465     }
2466
2467     $qmpclient->queue_execute(undef, 1);
2468
2469     foreach my $vmid (keys %$list) {
2470         next if $opt_vmid && ($vmid ne $opt_vmid);
2471         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2472     }
2473
2474     return $res;
2475 }
2476
2477 sub foreach_dimm {
2478     my ($conf, $vmid, $memory, $sockets, $func) = @_;
2479
2480     my $dimm_id = 0;
2481     my $current_size = 1024;
2482     my $dimm_size = 512;
2483     return if $current_size == $memory;
2484
2485     for (my $j = 0; $j < 8; $j++) {
2486         for (my $i = 0; $i < 32; $i++) {
2487             my $name = "dimm${dimm_id}";
2488             $dimm_id++;
2489             my $numanode = $i % $sockets;
2490             $current_size += $dimm_size;
2491             &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
2492             return  $current_size if $current_size >= $memory;
2493         }
2494         $dimm_size *= 2;
2495     }
2496 }
2497
2498 sub foreach_drive {
2499     my ($conf, $func) = @_;
2500
2501     foreach my $ds (keys %$conf) {
2502         next if !valid_drivename($ds);
2503
2504         my $drive = parse_drive($ds, $conf->{$ds});
2505         next if !$drive;
2506
2507         &$func($ds, $drive);
2508     }
2509 }
2510
2511 sub foreach_volid {
2512     my ($conf, $func) = @_;
2513
2514     my $volhash = {};
2515
2516     my $test_volid = sub {
2517         my ($volid, $is_cdrom) = @_;
2518
2519         return if !$volid;
2520
2521         $volhash->{$volid} = $is_cdrom || 0;
2522     };
2523
2524     foreach_drive($conf, sub {
2525         my ($ds, $drive) = @_;
2526         &$test_volid($drive->{file}, drive_is_cdrom($drive));
2527     });
2528
2529     foreach my $snapname (keys %{$conf->{snapshots}}) {
2530         my $snap = $conf->{snapshots}->{$snapname};
2531         &$test_volid($snap->{vmstate}, 0);
2532         foreach_drive($snap, sub {
2533             my ($ds, $drive) = @_;
2534             &$test_volid($drive->{file}, drive_is_cdrom($drive));
2535         });
2536     }
2537
2538     foreach my $volid (keys %$volhash) {
2539         &$func($volid, $volhash->{$volid});
2540     }
2541 }
2542
2543 sub vga_conf_has_spice {
2544     my ($vga) = @_;
2545
2546     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
2547
2548     return $1 || 1;
2549 }
2550
2551 sub config_to_command {
2552     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2553
2554     my $cmd = [];
2555     my $globalFlags = [];
2556     my $machineFlags = [];
2557     my $rtcFlags = [];
2558     my $cpuFlags = [];
2559     my $devices = [];
2560     my $pciaddr = '';
2561     my $bridges = {};
2562     my $kvmver = kvm_user_version();
2563     my $vernum = 0; # unknown
2564     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
2565         $vernum = $1*1000000+$2*1000;
2566     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
2567         $vernum = $1*1000000+$2*1000+$3;
2568     }
2569
2570     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
2571
2572     my $have_ovz = -f '/proc/vz/vestat';
2573
2574     my $q35 = machine_type_is_q35($conf);
2575     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2576     my $machine_type = $forcemachine || $conf->{machine};
2577
2578     my $cpuunits = defined($conf->{cpuunits}) ?
2579             $conf->{cpuunits} : $defaults->{cpuunits};
2580
2581     push @$cmd, '/usr/bin/systemd-run';
2582     push @$cmd, '--scope';
2583     push @$cmd, '--slice', "qemu";
2584     push @$cmd, '--unit', $vmid;
2585     push @$cmd, '-p', "CPUShares=$cpuunits";
2586     if ($conf->{cpulimit}) {
2587         my $cpulimit = int($conf->{cpulimit} * 100);
2588         push @$cmd, '-p', "CPUQuota=$cpulimit\%";
2589     }
2590
2591     push @$cmd, '/usr/bin/kvm';
2592
2593     push @$cmd, '-id', $vmid;
2594
2595     my $use_virtio = 0;
2596
2597     my $qmpsocket = qmp_socket($vmid);
2598     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2599     push @$cmd, '-mon', "chardev=qmp,mode=control";
2600
2601     my $socket = vnc_socket($vmid);
2602     push @$cmd,  '-vnc', "unix:$socket,x509,password";
2603
2604     push @$cmd, '-pidfile' , pidfile_name($vmid);
2605
2606     push @$cmd, '-daemonize';
2607
2608     if ($conf->{smbios1}) {
2609         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
2610     }
2611
2612     if ($q35) {
2613         # the q35 chipset support native usb2, so we enable usb controller
2614         # by default for this machine type
2615         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
2616     } else {
2617         $pciaddr = print_pci_addr("piix3", $bridges);
2618         push @$devices, '-device', "piix3-usb-uhci,id=uhci$pciaddr.0x2";
2619
2620         my $use_usb2 = 0;
2621         for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2622             next if !$conf->{"usb$i"};
2623             $use_usb2 = 1;
2624         }
2625         # include usb device config
2626         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-usb.cfg' if $use_usb2;
2627     }
2628
2629     my $vga = $conf->{vga};
2630
2631     my $qxlnum = vga_conf_has_spice($vga);
2632     $vga = 'qxl' if $qxlnum;
2633
2634     if (!$vga) {
2635         if ($conf->{ostype} && ($conf->{ostype} eq 'win8' ||
2636                                 $conf->{ostype} eq 'win7' ||
2637                                 $conf->{ostype} eq 'w2k8')) {
2638             $vga = 'std';
2639         } else {
2640             $vga = 'cirrus';
2641         }
2642     }
2643
2644     # enable absolute mouse coordinates (needed by vnc)
2645     my $tablet;
2646     if (defined($conf->{tablet})) {
2647         $tablet = $conf->{tablet};
2648     } else {
2649         $tablet = $defaults->{tablet};
2650         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
2651         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
2652     }
2653
2654     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
2655
2656     # host pci devices
2657     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
2658         my $d = parse_hostpci($conf->{"hostpci$i"});
2659         next if !$d;
2660
2661         my $pcie = $d->{pcie};
2662         if($pcie){
2663             die "q35 machine model is not enabled" if !$q35;
2664             $pciaddr = print_pcie_addr("hostpci$i");
2665         }else{
2666             $pciaddr = print_pci_addr("hostpci$i", $bridges);
2667         }
2668
2669         my $rombar = $d->{rombar} && $d->{rombar} eq 'off' ? ",rombar=0" : "";
2670         my $driver = $d->{driver} && $d->{driver} eq 'vfio' ? "vfio-pci" : "pci-assign";
2671         my $xvga = $d->{'x-vga'} && $d->{'x-vga'} eq 'on' ? ",x-vga=on" : "";
2672         if ($xvga && $xvga ne '') {
2673             push @$cpuFlags, 'kvm=off';
2674             $vga = 'none';
2675         }
2676         $driver = "vfio-pci" if $xvga ne '';
2677         my $pcidevices = $d->{pciid};
2678         my $multifunction = 1 if @$pcidevices > 1;
2679
2680         my $j=0;
2681         foreach my $pcidevice (@$pcidevices) {
2682
2683             my $id = "hostpci$i";
2684             $id .= ".$j" if $multifunction;
2685             my $addr = $pciaddr;
2686             $addr .= ".$j" if $multifunction;
2687             my $devicestr = "$driver,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
2688
2689             if($j == 0){
2690                 $devicestr .= "$rombar$xvga";
2691                 $devicestr .= ",multifunction=on" if $multifunction;
2692             }
2693
2694             push @$devices, '-device', $devicestr;
2695             $j++;
2696         }
2697     }
2698
2699     # usb devices
2700     for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2701         my $d = parse_usb_device($conf->{"usb$i"});
2702         next if !$d;
2703         if ($d->{vendorid} && $d->{productid}) {
2704             push @$devices, '-device', "usb-host,vendorid=0x$d->{vendorid},productid=0x$d->{productid}";
2705         } elsif (defined($d->{hostbus}) && defined($d->{hostport})) {
2706             push @$devices, '-device', "usb-host,hostbus=$d->{hostbus},hostport=$d->{hostport}";
2707         } elsif ($d->{spice}) {
2708             # usb redir support for spice
2709             push @$devices, '-chardev', "spicevmc,id=usbredirchardev$i,name=usbredir";
2710             push @$devices, '-device', "usb-redir,chardev=usbredirchardev$i,id=usbredirdev$i,bus=ehci.0";
2711         }
2712     }
2713
2714     # serial devices
2715     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2716         if (my $path = $conf->{"serial$i"}) {
2717             if ($path eq 'socket') {
2718                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
2719                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
2720                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2721             } else {
2722                 die "no such serial device\n" if ! -c $path;
2723                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
2724                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2725             }
2726         }
2727     }
2728
2729     # parallel devices
2730     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
2731         if (my $path = $conf->{"parallel$i"}) {
2732             die "no such parallel device\n" if ! -c $path;
2733             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
2734             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
2735             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
2736         }
2737     }
2738
2739     my $vmname = $conf->{name} || "vm$vmid";
2740
2741     push @$cmd, '-name', $vmname;
2742
2743     my $sockets = 1;
2744     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
2745     $sockets = $conf->{sockets} if  $conf->{sockets};
2746
2747     my $cores = $conf->{cores} || 1;
2748
2749     my $maxcpus = $sockets * $cores;
2750
2751     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
2752
2753     my $allowed_vcpus = $cpuinfo->{cpus};
2754
2755     die "MAX $maxcpus vcpus allowed per VM on this node\n"
2756         if ($allowed_vcpus < $maxcpus);
2757
2758     push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
2759
2760     push @$cmd, '-nodefaults';
2761
2762     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2763
2764     my $bootindex_hash = {};
2765     my $i = 1;
2766     foreach my $o (split(//, $bootorder)) {
2767         $bootindex_hash->{$o} = $i*100;
2768         $i++;
2769     }
2770
2771     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000";
2772
2773     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
2774
2775     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
2776
2777     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
2778
2779     # time drift fix
2780     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
2781
2782     my $nokvm = defined($conf->{kvm}) && $conf->{kvm} == 0 ? 1 : 0;
2783     my $useLocaltime = $conf->{localtime};
2784
2785     if (my $ost = $conf->{ostype}) {
2786         # other, wxp, w2k, w2k3, w2k8, wvista, win7, win8, l24, l26, solaris
2787
2788         if ($ost =~ m/^w/) { # windows
2789             $useLocaltime = 1 if !defined($conf->{localtime});
2790
2791             # use time drift fix when acpi is enabled
2792             if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
2793                 $tdf = 1 if !defined($conf->{tdf});
2794             }
2795         }
2796
2797         if ($ost eq 'win7' || $ost eq 'win8' || $ost eq 'w2k8' ||
2798             $ost eq 'wvista') {
2799             push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
2800             push @$cmd, '-no-hpet';
2801             if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
2802                 push @$cpuFlags , 'hv_spinlocks=0x1fff' if !$nokvm;
2803                 push @$cpuFlags , 'hv_vapic' if !$nokvm;
2804                 push @$cpuFlags , 'hv_time' if !$nokvm;
2805
2806             } else {
2807                 push @$cpuFlags , 'hv_spinlocks=0xffff' if !$nokvm;
2808             }
2809         }
2810
2811         if ($ost eq 'win7' || $ost eq 'win8') {
2812             push @$cpuFlags , 'hv_relaxed' if !$nokvm;
2813         }
2814     }
2815
2816     push @$rtcFlags, 'driftfix=slew' if $tdf;
2817
2818     if ($nokvm) {
2819         push @$machineFlags, 'accel=tcg';
2820     } else {
2821         die "No accelerator found!\n" if !$cpuinfo->{hvm};
2822     }
2823
2824     if ($machine_type) {
2825         push @$machineFlags, "type=${machine_type}";
2826     }
2827
2828     if ($conf->{startdate}) {
2829         push @$rtcFlags, "base=$conf->{startdate}";
2830     } elsif ($useLocaltime) {
2831         push @$rtcFlags, 'base=localtime';
2832     }
2833
2834     my $cpu = $nokvm ? "qemu64" : "kvm64";
2835     $cpu = $conf->{cpu} if $conf->{cpu};
2836
2837     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
2838
2839     push @$cpuFlags , '+x2apic' if !$nokvm && $conf->{ostype} ne 'solaris';
2840
2841     push @$cpuFlags , '-x2apic' if $conf->{ostype} eq 'solaris';
2842
2843     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
2844
2845     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
2846
2847     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
2848
2849         push @$cpuFlags , '+kvm_pv_unhalt' if !$nokvm;
2850         push @$cpuFlags , '+kvm_pv_eoi' if !$nokvm;
2851     }
2852
2853     push @$cpuFlags, 'enforce' if $cpu ne 'host' && !$nokvm;
2854
2855     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
2856
2857     push @$cmd, '-cpu', $cpu;
2858
2859     my $memory = $conf->{memory} || $defaults->{memory};
2860     my $static_memory = 0;
2861     my $dimm_memory = 0;
2862
2863     if ($hotplug_features->{memory}) {
2864         die "Numa need to be enabled for memory hotplug\n" if !$conf->{numa};
2865         die "Total memory is bigger than ${MAX_MEM}MB\n" if $memory > $MAX_MEM;
2866         $static_memory = $STATICMEM;
2867         die "minimum memory must be ${static_memory}MB\n" if($memory < $static_memory);
2868         $dimm_memory = $memory - $static_memory;
2869         push @$cmd, '-m', "size=${static_memory},slots=255,maxmem=${MAX_MEM}M";
2870
2871     } else {
2872
2873         $static_memory = $memory;
2874         push @$cmd, '-m', $static_memory;
2875     }
2876
2877     if ($conf->{numa}) {
2878
2879         my $numa_totalmemory = undef;
2880         for (my $i = 0; $i < $MAX_NUMA; $i++) {
2881             next if !$conf->{"numa$i"};
2882             my $numa = parse_numa($conf->{"numa$i"});
2883             next if !$numa;
2884             # memory
2885             die "missing numa node$i memory value\n" if !$numa->{memory};
2886             my $numa_memory = $numa->{memory};
2887             $numa_totalmemory += $numa_memory;
2888             my $numa_object = "memory-backend-ram,id=ram-node$i,size=${numa_memory}M";
2889
2890             # cpus
2891             my $cpus_start = $numa->{cpus}->{start};
2892             die "missing numa node$i cpus\n" if !defined($cpus_start);
2893             my $cpus_end = $numa->{cpus}->{end} if defined($numa->{cpus}->{end});
2894             my $cpus = $cpus_start;
2895             if (defined($cpus_end)) {
2896                 $cpus .= "-$cpus_end";
2897                 die "numa node$i :  cpu range $cpus is incorrect\n" if $cpus_end <= $cpus_start;
2898             }
2899
2900             # hostnodes
2901             my $hostnodes_start = $numa->{hostnodes}->{start};
2902             if (defined($hostnodes_start)) {
2903                 my $hostnodes_end = $numa->{hostnodes}->{end} if defined($numa->{hostnodes}->{end});
2904                 my $hostnodes = $hostnodes_start;
2905                 if (defined($hostnodes_end)) {
2906                     $hostnodes .= "-$hostnodes_end";
2907                     die "host node $hostnodes range is incorrect\n" if $hostnodes_end <= $hostnodes_start;
2908                 }
2909
2910                 my $hostnodes_end_range = defined($hostnodes_end) ? $hostnodes_end : $hostnodes_start;
2911                 for (my $i = $hostnodes_start; $i <= $hostnodes_end_range; $i++ ) {
2912                     die "host numa node$i don't exist\n" if ! -d "/sys/devices/system/node/node$i/";
2913                 }
2914
2915                 # policy
2916                 my $policy = $numa->{policy};
2917                 die "you need to define a policy for hostnode $hostnodes\n" if !$policy;
2918                 $numa_object .= ",host-nodes=$hostnodes,policy=$policy";
2919             }
2920
2921             push @$cmd, '-object', $numa_object;
2922             push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
2923         }
2924
2925         die "total memory for NUMA nodes must be equal to vm static memory\n"
2926             if $numa_totalmemory && $numa_totalmemory != $static_memory;
2927
2928         #if no custom tology, we split memory and cores across numa nodes
2929         if(!$numa_totalmemory) {
2930
2931             my $numa_memory = ($static_memory / $sockets) . "M";
2932
2933             for (my $i = 0; $i < $sockets; $i++)  {
2934
2935                 my $cpustart = ($cores * $i);
2936                 my $cpuend = ($cpustart + $cores - 1) if $cores && $cores > 1;
2937                 my $cpus = $cpustart;
2938                 $cpus .= "-$cpuend" if $cpuend;
2939
2940                 push @$cmd, '-object', "memory-backend-ram,size=$numa_memory,id=ram-node$i";
2941                 push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
2942             }
2943         }
2944     }
2945
2946     if ($hotplug_features->{memory}) {
2947         foreach_dimm($conf, $vmid, $memory, $sockets, sub {
2948             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
2949             push @$cmd, "-object" , "memory-backend-ram,id=mem-$name,size=${dimm_size}M";
2950             push @$cmd, "-device", "pc-dimm,id=$name,memdev=mem-$name,node=$numanode";
2951
2952             #if dimm_memory is not aligned to dimm map
2953             if($current_size > $memory) {
2954                  $conf->{memory} = $current_size;
2955                  update_config_nolock($vmid, $conf, 1);
2956             }
2957         });
2958     }
2959
2960     push @$cmd, '-S' if $conf->{freeze};
2961
2962     # set keyboard layout
2963     my $kb = $conf->{keyboard} || $defaults->{keyboard};
2964     push @$cmd, '-k', $kb if $kb;
2965
2966     # enable sound
2967     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
2968     #push @$cmd, '-soundhw', 'es1370';
2969     #push @$cmd, '-soundhw', $soundhw if $soundhw;
2970
2971     if($conf->{agent}) {
2972         my $qgasocket = qmp_socket($vmid, 1);
2973         my $pciaddr = print_pci_addr("qga0", $bridges);
2974         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
2975         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
2976         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
2977     }
2978
2979     my $spice_port;
2980
2981     if ($qxlnum) {
2982         if ($qxlnum > 1) {
2983             if ($conf->{ostype} && $conf->{ostype} =~ m/^w/){
2984                 for(my $i = 1; $i < $qxlnum; $i++){
2985                     my $pciaddr = print_pci_addr("vga$i", $bridges);
2986                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
2987                 }
2988             } else {
2989                 # assume other OS works like Linux
2990                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
2991                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
2992             }
2993         }
2994
2995         my $pciaddr = print_pci_addr("spice", $bridges);
2996
2997         my $nodename = PVE::INotify::nodename();
2998         my $pfamily = PVE::Tools::get_host_address_family($nodename);
2999         $spice_port = PVE::Tools::next_spice_port($pfamily);
3000
3001         push @$devices, '-spice', "tls-port=${spice_port},addr=localhost,tls-ciphers=DES-CBC3-SHA,seamless-migration=on";
3002
3003         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3004         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3005         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3006     }
3007
3008     # enable balloon by default, unless explicitly disabled
3009     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3010         $pciaddr = print_pci_addr("balloon0", $bridges);
3011         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3012     }
3013
3014     if ($conf->{watchdog}) {
3015         my $wdopts = parse_watchdog($conf->{watchdog});
3016         $pciaddr = print_pci_addr("watchdog", $bridges);
3017         my $watchdog = $wdopts->{model} || 'i6300esb';
3018         push @$devices, '-device', "$watchdog$pciaddr";
3019         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3020     }
3021
3022     my $vollist = [];
3023     my $scsicontroller = {};
3024     my $ahcicontroller = {};
3025     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3026
3027     # Add iscsi initiator name if available
3028     if (my $initiator = get_initiator_name()) {
3029         push @$devices, '-iscsi', "initiator-name=$initiator";
3030     }
3031
3032     foreach_drive($conf, sub {
3033         my ($ds, $drive) = @_;
3034
3035         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3036             push @$vollist, $drive->{file};
3037         }
3038
3039         $use_virtio = 1 if $ds =~ m/^virtio/;
3040
3041         if (drive_is_cdrom ($drive)) {
3042             if ($bootindex_hash->{d}) {
3043                 $drive->{bootindex} = $bootindex_hash->{d};
3044                 $bootindex_hash->{d} += 1;
3045             }
3046         } else {
3047             if ($bootindex_hash->{c}) {
3048                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3049                 $bootindex_hash->{c} += 1;
3050             }
3051         }
3052
3053         if($drive->{interface} eq 'virtio'){
3054            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3055         }
3056
3057         if ($drive->{interface} eq 'scsi') {
3058
3059             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3060
3061             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3062             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3063
3064             my $iothread = '';
3065             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3066                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3067                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3068             }
3069
3070             my $queues = '';
3071             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3072                 $queues = ",num_queues=$drive->{queues}";
3073             }
3074
3075             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3076             $scsicontroller->{$controller}=1;
3077         }
3078
3079         if ($drive->{interface} eq 'sata') {
3080            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3081            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3082            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3083            $ahcicontroller->{$controller}=1;
3084         }
3085
3086         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3087         push @$devices, '-drive',$drive_cmd;
3088         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3089     });
3090
3091     for (my $i = 0; $i < $MAX_NETS; $i++) {
3092          next if !$conf->{"net$i"};
3093          my $d = parse_net($conf->{"net$i"});
3094          next if !$d;
3095
3096          $use_virtio = 1 if $d->{model} eq 'virtio';
3097
3098          if ($bootindex_hash->{n}) {
3099             $d->{bootindex} = $bootindex_hash->{n};
3100             $bootindex_hash->{n} += 1;
3101          }
3102
3103          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3104          push @$devices, '-netdev', $netdevfull;
3105
3106          my $netdevicefull = print_netdevice_full($vmid,$conf,$d,"net$i",$bridges);
3107          push @$devices, '-device', $netdevicefull;
3108     }
3109
3110     if (!$q35) {
3111         # add pci bridges
3112         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3113            $bridges->{1} = 1;
3114            $bridges->{2} = 1;
3115         }
3116
3117         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3118
3119         while (my ($k, $v) = each %$bridges) {
3120             $pciaddr = print_pci_addr("pci.$k");
3121             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3122         }
3123     }
3124
3125     # add custom args
3126     if ($conf->{args}) {
3127         my $aa = PVE::Tools::split_args($conf->{args});
3128         push @$cmd, @$aa;
3129     }
3130
3131     push @$cmd, @$devices;
3132     push @$cmd, '-rtc', join(',', @$rtcFlags)
3133         if scalar(@$rtcFlags);
3134     push @$cmd, '-machine', join(',', @$machineFlags)
3135         if scalar(@$machineFlags);
3136     push @$cmd, '-global', join(',', @$globalFlags)
3137         if scalar(@$globalFlags);
3138
3139     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3140 }
3141
3142 sub vnc_socket {
3143     my ($vmid) = @_;
3144     return "${var_run_tmpdir}/$vmid.vnc";
3145 }
3146
3147 sub spice_port {
3148     my ($vmid) = @_;
3149
3150     my $res = vm_mon_cmd($vmid, 'query-spice');
3151
3152     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3153 }
3154
3155 sub qmp_socket {
3156     my ($vmid, $qga) = @_;
3157     my $sockettype = $qga ? 'qga' : 'qmp';
3158     return "${var_run_tmpdir}/$vmid.$sockettype";
3159 }
3160
3161 sub pidfile_name {
3162     my ($vmid) = @_;
3163     return "${var_run_tmpdir}/$vmid.pid";
3164 }
3165
3166 sub vm_devices_list {
3167     my ($vmid) = @_;
3168
3169     my $res = vm_mon_cmd($vmid, 'query-pci');
3170     my $devices = {};
3171     foreach my $pcibus (@$res) {
3172         foreach my $device (@{$pcibus->{devices}}) {
3173             next if !$device->{'qdev_id'};
3174             if ($device->{'pci_bridge'}) {
3175                 $devices->{$device->{'qdev_id'}} = 1;
3176                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3177                     next if !$bridge_device->{'qdev_id'};
3178                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3179                     $devices->{$device->{'qdev_id'}}++;
3180                 }
3181             } else {
3182                 $devices->{$device->{'qdev_id'}} = 1;
3183             }
3184         }
3185     }
3186
3187     my $resblock = vm_mon_cmd($vmid, 'query-block');
3188     foreach my $block (@$resblock) {
3189         if($block->{device} =~ m/^drive-(\S+)/){
3190                 $devices->{$1} = 1;
3191         }
3192     }
3193
3194     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3195     foreach my $mice (@$resmice) {
3196         if ($mice->{name} eq 'QEMU HID Tablet') {
3197             $devices->{tablet} = 1;
3198             last;
3199         }
3200     }
3201
3202     return $devices;
3203 }
3204
3205 sub vm_deviceplug {
3206     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3207
3208     my $q35 = machine_type_is_q35($conf);
3209
3210     my $devices_list = vm_devices_list($vmid);
3211     return 1 if defined($devices_list->{$deviceid});
3212
3213     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3214
3215     if ($deviceid eq 'tablet') {
3216
3217         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3218
3219     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3220
3221         qemu_iothread_add($vmid, $deviceid, $device);
3222
3223         qemu_driveadd($storecfg, $vmid, $device);
3224         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3225
3226         qemu_deviceadd($vmid, $devicefull);
3227         eval { qemu_deviceaddverify($vmid, $deviceid); };
3228         if (my $err = $@) {
3229             eval { qemu_drivedel($vmid, $deviceid); };
3230             warn $@ if $@;
3231             die $err;
3232         }
3233
3234     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3235
3236
3237         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3238         my $pciaddr = print_pci_addr($deviceid);
3239         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3240
3241         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3242
3243         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3244             qemu_iothread_add($vmid, $deviceid, $device);
3245             $devicefull .= ",iothread=iothread-$deviceid";
3246         }
3247
3248         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3249             $devicefull .= ",num_queues=$device->{queues}";
3250         }
3251
3252         qemu_deviceadd($vmid, $devicefull);
3253         qemu_deviceaddverify($vmid, $deviceid);
3254
3255     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3256
3257         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3258         qemu_driveadd($storecfg, $vmid, $device);
3259
3260         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3261         eval { qemu_deviceadd($vmid, $devicefull); };
3262         if (my $err = $@) {
3263             eval { qemu_drivedel($vmid, $deviceid); };
3264             warn $@ if $@;
3265             die $err;
3266         }
3267
3268     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3269
3270         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3271         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid);
3272         qemu_deviceadd($vmid, $netdevicefull);
3273         eval { qemu_deviceaddverify($vmid, $deviceid); };
3274         if (my $err = $@) {
3275             eval { qemu_netdevdel($vmid, $deviceid); };
3276             warn $@ if $@;
3277             die $err;
3278         }
3279
3280     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3281
3282         my $bridgeid = $2;
3283         my $pciaddr = print_pci_addr($deviceid);
3284         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3285
3286         qemu_deviceadd($vmid, $devicefull);
3287         qemu_deviceaddverify($vmid, $deviceid);
3288
3289     } else {
3290         die "can't hotplug device '$deviceid'\n";
3291     }
3292
3293     return 1;
3294 }
3295
3296 # fixme: this should raise exceptions on error!
3297 sub vm_deviceunplug {
3298     my ($vmid, $conf, $deviceid) = @_;
3299
3300     my $devices_list = vm_devices_list($vmid);
3301     return 1 if !defined($devices_list->{$deviceid});
3302
3303     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3304
3305     if ($deviceid eq 'tablet') {
3306
3307         qemu_devicedel($vmid, $deviceid);
3308
3309     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3310
3311         qemu_devicedel($vmid, $deviceid);
3312         qemu_devicedelverify($vmid, $deviceid);
3313         qemu_drivedel($vmid, $deviceid);
3314         qemu_iothread_del($conf, $vmid, $deviceid);
3315
3316     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3317
3318         qemu_devicedel($vmid, $deviceid);
3319         qemu_devicedelverify($vmid, $deviceid);
3320         qemu_iothread_del($conf, $vmid, $deviceid);
3321
3322     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3323
3324         #qemu 2.3 segfault on drive_del with virtioscsi + iothread
3325         my $device = parse_drive($deviceid, $conf->{$deviceid});
3326         die "virtioscsi with iothread is not hot-unplugglable currently" if $device->{iothread};
3327
3328         qemu_devicedel($vmid, $deviceid);
3329         qemu_drivedel($vmid, $deviceid);
3330         qemu_deletescsihw($conf, $vmid, $deviceid);
3331
3332     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3333
3334         qemu_devicedel($vmid, $deviceid);
3335         qemu_devicedelverify($vmid, $deviceid);
3336         qemu_netdevdel($vmid, $deviceid);
3337
3338     } else {
3339         die "can't unplug device '$deviceid'\n";
3340     }
3341
3342     return 1;
3343 }
3344
3345 sub qemu_deviceadd {
3346     my ($vmid, $devicefull) = @_;
3347
3348     $devicefull = "driver=".$devicefull;
3349     my %options =  split(/[=,]/, $devicefull);
3350
3351     vm_mon_cmd($vmid, "device_add" , %options);
3352 }
3353
3354 sub qemu_devicedel {
3355     my ($vmid, $deviceid) = @_;
3356
3357     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3358 }
3359
3360 sub qemu_iothread_add {
3361     my($vmid, $deviceid, $device) = @_;
3362
3363     if ($device->{iothread}) {
3364         my $iothreads = vm_iothreads_list($vmid);
3365         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3366     }
3367 }
3368
3369 sub qemu_iothread_del {
3370     my($conf, $vmid, $deviceid) = @_;
3371
3372     my $device = parse_drive($deviceid, $conf->{$deviceid});
3373     if ($device->{iothread}) {
3374         my $iothreads = vm_iothreads_list($vmid);
3375         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3376     }
3377 }
3378
3379 sub qemu_objectadd {
3380     my($vmid, $objectid, $qomtype) = @_;
3381
3382     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3383
3384     return 1;
3385 }
3386
3387 sub qemu_objectdel {
3388     my($vmid, $objectid) = @_;
3389
3390     vm_mon_cmd($vmid, "object-del", id => $objectid);
3391
3392     return 1;
3393 }
3394
3395 sub qemu_driveadd {
3396     my ($storecfg, $vmid, $device) = @_;
3397
3398     my $drive = print_drive_full($storecfg, $vmid, $device);
3399     $drive =~ s/\\/\\\\/g;
3400     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3401
3402     # If the command succeeds qemu prints: "OK"
3403     return 1 if $ret =~ m/OK/s;
3404
3405     die "adding drive failed: $ret\n";
3406 }
3407
3408 sub qemu_drivedel {
3409     my($vmid, $deviceid) = @_;
3410
3411     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3412     $ret =~ s/^\s+//;
3413
3414     return 1 if $ret eq "";
3415
3416     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3417     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3418
3419     die "deleting drive $deviceid failed : $ret\n";
3420 }
3421
3422 sub qemu_deviceaddverify {
3423     my ($vmid, $deviceid) = @_;
3424
3425     for (my $i = 0; $i <= 5; $i++) {
3426          my $devices_list = vm_devices_list($vmid);
3427          return 1 if defined($devices_list->{$deviceid});
3428          sleep 1;
3429     }
3430
3431     die "error on hotplug device '$deviceid'\n";
3432 }
3433
3434
3435 sub qemu_devicedelverify {
3436     my ($vmid, $deviceid) = @_;
3437
3438     # need to verify that the device is correctly removed as device_del
3439     # is async and empty return is not reliable
3440
3441     for (my $i = 0; $i <= 5; $i++) {
3442          my $devices_list = vm_devices_list($vmid);
3443          return 1 if !defined($devices_list->{$deviceid});
3444          sleep 1;
3445     }
3446
3447     die "error on hot-unplugging device '$deviceid'\n";
3448 }
3449
3450 sub qemu_findorcreatescsihw {
3451     my ($storecfg, $conf, $vmid, $device) = @_;
3452
3453     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3454
3455     my $scsihwid="$controller_prefix$controller";
3456     my $devices_list = vm_devices_list($vmid);
3457
3458     if(!defined($devices_list->{$scsihwid})) {
3459         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3460     }
3461
3462     return 1;
3463 }
3464
3465 sub qemu_deletescsihw {
3466     my ($conf, $vmid, $opt) = @_;
3467
3468     my $device = parse_drive($opt, $conf->{$opt});
3469
3470     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3471         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3472         return 1;
3473     }
3474
3475     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3476
3477     my $devices_list = vm_devices_list($vmid);
3478     foreach my $opt (keys %{$devices_list}) {
3479         if (PVE::QemuServer::valid_drivename($opt)) {
3480             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3481             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3482                 return 1;
3483             }
3484         }
3485     }
3486
3487     my $scsihwid="scsihw$controller";
3488
3489     vm_deviceunplug($vmid, $conf, $scsihwid);
3490
3491     return 1;
3492 }
3493
3494 sub qemu_add_pci_bridge {
3495     my ($storecfg, $conf, $vmid, $device) = @_;
3496
3497     my $bridges = {};
3498
3499     my $bridgeid;
3500
3501     print_pci_addr($device, $bridges);
3502
3503     while (my ($k, $v) = each %$bridges) {
3504         $bridgeid = $k;
3505     }
3506     return 1 if !defined($bridgeid) || $bridgeid < 1;
3507
3508     my $bridge = "pci.$bridgeid";
3509     my $devices_list = vm_devices_list($vmid);
3510
3511     if (!defined($devices_list->{$bridge})) {
3512         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3513     }
3514
3515     return 1;
3516 }
3517
3518 sub qemu_set_link_status {
3519     my ($vmid, $device, $up) = @_;
3520
3521     vm_mon_cmd($vmid, "set_link", name => $device,
3522                up => $up ? JSON::true : JSON::false);
3523 }
3524
3525 sub qemu_netdevadd {
3526     my ($vmid, $conf, $device, $deviceid) = @_;
3527
3528     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid);
3529     my %options =  split(/[=,]/, $netdev);
3530
3531     vm_mon_cmd($vmid, "netdev_add",  %options);
3532     return 1;
3533 }
3534
3535 sub qemu_netdevdel {
3536     my ($vmid, $deviceid) = @_;
3537
3538     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3539 }
3540
3541 sub qemu_cpu_hotplug {
3542     my ($vmid, $conf, $vcpus) = @_;
3543
3544     my $sockets = 1;
3545     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3546     $sockets = $conf->{sockets} if  $conf->{sockets};
3547     my $cores = $conf->{cores} || 1;
3548     my $maxcpus = $sockets * $cores;
3549
3550     $vcpus = $maxcpus if !$vcpus;
3551
3552     die "you can't add more vcpus than maxcpus\n"
3553         if $vcpus > $maxcpus;
3554
3555     my $currentvcpus = $conf->{vcpus} || $maxcpus;
3556     die "online cpu unplug is not yet possible\n"
3557         if $vcpus < $currentvcpus;
3558
3559     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3560     die "vcpus in running vm is different than configuration\n"
3561         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
3562
3563     for (my $i = $currentvcpus; $i < $vcpus; $i++) {
3564         vm_mon_cmd($vmid, "cpu-add", id => int($i));
3565     }
3566 }
3567
3568 sub qemu_memory_hotplug {
3569     my ($vmid, $conf, $defaults, $opt, $value) = @_;
3570
3571     return $value if !check_running($vmid);
3572
3573     my $memory = $conf->{memory} || $defaults->{memory};
3574     $value = $defaults->{memory} if !$value;
3575     return $value if $value == $memory;
3576
3577     my $static_memory = $STATICMEM;
3578     my $dimm_memory = $memory - $static_memory;
3579
3580     die "memory can't be lower than $static_memory MB" if $value < $static_memory;
3581     die "memory unplug is not yet available" if $value < $memory;
3582     die "you cannot add more memory than $MAX_MEM MB!\n" if $memory > $MAX_MEM;
3583
3584
3585     my $sockets = 1;
3586     $sockets = $conf->{sockets} if $conf->{sockets};
3587
3588     foreach_dimm($conf, $vmid, $value, $sockets, sub {
3589         my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3590
3591             return if $current_size <= $conf->{memory};
3592
3593             eval { vm_mon_cmd($vmid, "object-add", 'qom-type' => "memory-backend-ram", id => "mem-$name", props => { size => int($dimm_size*1024*1024) } ) };
3594             if (my $err = $@) {
3595                 eval { qemu_objectdel($vmid, "mem-$name"); };
3596                 die $err;
3597             }
3598
3599             eval { vm_mon_cmd($vmid, "device_add", driver => "pc-dimm", id => "$name", memdev => "mem-$name", node => $numanode) };
3600             if (my $err = $@) {
3601                 eval { qemu_objectdel($vmid, "mem-$name"); };
3602                 die $err;
3603             }
3604             #update conf after each succesful module hotplug
3605             $conf->{memory} = $current_size;
3606             update_config_nolock($vmid, $conf, 1);
3607     });
3608 }
3609
3610 sub qemu_block_set_io_throttle {
3611     my ($vmid, $deviceid, $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr) = @_;
3612
3613     return if !check_running($vmid) ;
3614
3615     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid, bps => int($bps), bps_rd => int($bps_rd), bps_wr => int($bps_wr), iops => int($iops), iops_rd => int($iops_rd), iops_wr => int($iops_wr));
3616
3617 }
3618
3619 # old code, only used to shutdown old VM after update
3620 sub __read_avail {
3621     my ($fh, $timeout) = @_;
3622
3623     my $sel = new IO::Select;
3624     $sel->add($fh);
3625
3626     my $res = '';
3627     my $buf;
3628
3629     my @ready;
3630     while (scalar (@ready = $sel->can_read($timeout))) {
3631         my $count;
3632         if ($count = $fh->sysread($buf, 8192)) {
3633             if ($buf =~ /^(.*)\(qemu\) $/s) {
3634                 $res .= $1;
3635                 last;
3636             } else {
3637                 $res .= $buf;
3638             }
3639         } else {
3640             if (!defined($count)) {
3641                 die "$!\n";
3642             }
3643             last;
3644         }
3645     }
3646
3647     die "monitor read timeout\n" if !scalar(@ready);
3648
3649     return $res;
3650 }
3651
3652 # old code, only used to shutdown old VM after update
3653 sub vm_monitor_command {
3654     my ($vmid, $cmdstr, $nocheck) = @_;
3655
3656     my $res;
3657
3658     eval {
3659         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
3660
3661         my $sname = "${var_run_tmpdir}/$vmid.mon";
3662
3663         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
3664             die "unable to connect to VM $vmid socket - $!\n";
3665
3666         my $timeout = 3;
3667
3668         # hack: migrate sometime blocks the monitor (when migrate_downtime
3669         # is set)
3670         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3671             $timeout = 60*60; # 1 hour
3672         }
3673
3674         # read banner;
3675         my $data = __read_avail($sock, $timeout);
3676
3677         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
3678             die "got unexpected qemu monitor banner\n";
3679         }
3680
3681         my $sel = new IO::Select;
3682         $sel->add($sock);
3683
3684         if (!scalar(my @ready = $sel->can_write($timeout))) {
3685             die "monitor write error - timeout";
3686         }
3687
3688         my $fullcmd = "$cmdstr\r";
3689
3690         # syslog('info', "VM $vmid monitor command: $cmdstr");
3691
3692         my $b;
3693         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
3694             die "monitor write error - $!";
3695         }
3696
3697         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
3698
3699         $timeout = 20;
3700
3701         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3702             $timeout = 60*60; # 1 hour
3703         } elsif ($cmdstr =~ m/^(eject|change)/) {
3704             $timeout = 60; # note: cdrom mount command is slow
3705         }
3706         if ($res = __read_avail($sock, $timeout)) {
3707
3708             my @lines = split("\r?\n", $res);
3709
3710             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
3711
3712             $res = join("\n", @lines);
3713             $res .= "\n";
3714         }
3715     };
3716
3717     my $err = $@;
3718
3719     if ($err) {
3720         syslog("err", "VM $vmid monitor command failed - $err");
3721         die $err;
3722     }
3723
3724     return $res;
3725 }
3726
3727 sub qemu_block_resize {
3728     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
3729
3730     my $running = check_running($vmid);
3731
3732     return if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
3733
3734     return if !$running;
3735
3736     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
3737
3738 }
3739
3740 sub qemu_volume_snapshot {
3741     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3742
3743     my $running = check_running($vmid);
3744
3745     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
3746         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
3747     } else {
3748         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
3749     }
3750 }
3751
3752 sub qemu_volume_snapshot_delete {
3753     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3754
3755     my $running = check_running($vmid);
3756
3757     return if !PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
3758
3759     return if !$running;
3760
3761     vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
3762 }
3763
3764 sub set_migration_caps {
3765     my ($vmid) = @_;
3766
3767     my $cap_ref = [];
3768
3769     my $enabled_cap = {
3770         "auto-converge" => 1,
3771         "xbzrle" => 0,
3772         "x-rdma-pin-all" => 0,
3773         "zero-blocks" => 0,
3774     };
3775
3776     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
3777
3778     for my $supported_capability (@$supported_capabilities) {
3779         push @$cap_ref, {
3780             capability => $supported_capability->{capability},
3781             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
3782         };
3783     }
3784
3785     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
3786 }
3787
3788 my $fast_plug_option = {
3789     'lock' => 1,
3790     'name' => 1,
3791     'onboot' => 1,
3792     'shares' => 1,
3793     'startup' => 1,
3794 };
3795
3796 # hotplug changes in [PENDING]
3797 # $selection hash can be used to only apply specified options, for
3798 # example: { cores => 1 } (only apply changed 'cores')
3799 # $errors ref is used to return error messages
3800 sub vmconfig_hotplug_pending {
3801     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
3802
3803     my $defaults = load_defaults();
3804
3805     # commit values which do not have any impact on running VM first
3806     # Note: those option cannot raise errors, we we do not care about
3807     # $selection and always apply them.
3808
3809     my $add_error = sub {
3810         my ($opt, $msg) = @_;
3811         $errors->{$opt} = "hotplug problem - $msg";
3812     };
3813
3814     my $changes = 0;
3815     foreach my $opt (keys %{$conf->{pending}}) { # add/change
3816         if ($fast_plug_option->{$opt}) {
3817             $conf->{$opt} = $conf->{pending}->{$opt};
3818             delete $conf->{pending}->{$opt};
3819             $changes = 1;
3820         }
3821     }
3822
3823     if ($changes) {
3824         update_config_nolock($vmid, $conf, 1);
3825         $conf = load_config($vmid); # update/reload
3826     }
3827
3828     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3829
3830     my @delete = PVE::Tools::split_list($conf->{pending}->{delete});
3831     foreach my $opt (@delete) {
3832         next if $selection && !$selection->{$opt};
3833         eval {
3834             if ($opt eq 'hotplug') {
3835                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
3836             } elsif ($opt eq 'tablet') {
3837                 die "skip\n" if !$hotplug_features->{usb};
3838                 if ($defaults->{tablet}) {
3839                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
3840                 } else {
3841                     vm_deviceunplug($vmid, $conf, $opt);
3842                 }
3843             } elsif ($opt eq 'vcpus') {
3844                 die "skip\n" if !$hotplug_features->{cpu};
3845                 qemu_cpu_hotplug($vmid, $conf, undef);
3846             } elsif ($opt eq 'balloon') {
3847                 # enable balloon device is not hotpluggable
3848                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
3849             } elsif ($fast_plug_option->{$opt}) {
3850                 # do nothing
3851             } elsif ($opt =~ m/^net(\d+)$/) {
3852                 die "skip\n" if !$hotplug_features->{network};
3853                 vm_deviceunplug($vmid, $conf, $opt);
3854             } elsif (valid_drivename($opt)) {
3855                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
3856                 vm_deviceunplug($vmid, $conf, $opt);
3857                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}));
3858             } elsif ($opt =~ m/^memory$/) {
3859                 die "skip\n" if !$hotplug_features->{memory};
3860                 qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
3861             } elsif ($opt eq 'cpuunits') {
3862                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
3863             } elsif ($opt eq 'cpulimit') {
3864                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
3865             } else {
3866                 die "skip\n";
3867             }
3868         };
3869         if (my $err = $@) {
3870             &$add_error($opt, $err) if $err ne "skip\n";
3871         } else {
3872             # save new config if hotplug was successful
3873             delete $conf->{$opt};
3874             vmconfig_undelete_pending_option($conf, $opt);
3875             update_config_nolock($vmid, $conf, 1);
3876             $conf = load_config($vmid); # update/reload
3877         }
3878     }
3879
3880     foreach my $opt (keys %{$conf->{pending}}) {
3881         next if $selection && !$selection->{$opt};
3882         my $value = $conf->{pending}->{$opt};
3883         eval {
3884             if ($opt eq 'hotplug') {
3885                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
3886             } elsif ($opt eq 'tablet') {
3887                 die "skip\n" if !$hotplug_features->{usb};
3888                 if ($value == 1) {
3889                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
3890                 } elsif ($value == 0) {
3891                     vm_deviceunplug($vmid, $conf, $opt);
3892                 }
3893             } elsif ($opt eq 'vcpus') {
3894                 die "skip\n" if !$hotplug_features->{cpu};
3895                 qemu_cpu_hotplug($vmid, $conf, $value);
3896             } elsif ($opt eq 'balloon') {
3897                 # enable/disable balloning device is not hotpluggable
3898                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
3899                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
3900                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
3901
3902                 # allow manual ballooning if shares is set to zero
3903                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
3904                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
3905                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
3906                 }
3907             } elsif ($opt =~ m/^net(\d+)$/) {
3908                 # some changes can be done without hotplug
3909                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
3910                                     $vmid, $opt, $value);
3911             } elsif (valid_drivename($opt)) {
3912                 # some changes can be done without hotplug
3913                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
3914                                      $vmid, $opt, $value, 1);
3915             } elsif ($opt =~ m/^memory$/) { #dimms
3916                 die "skip\n" if !$hotplug_features->{memory};
3917                 $value = qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
3918             } elsif ($opt eq 'cpuunits') {
3919                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
3920             } elsif ($opt eq 'cpulimit') {
3921                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
3922                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
3923             } else {
3924                 die "skip\n";  # skip non-hot-pluggable options
3925             }
3926         };
3927         if (my $err = $@) {
3928             &$add_error($opt, $err) if $err ne "skip\n";
3929         } else {
3930             # save new config if hotplug was successful
3931             $conf->{$opt} = $value;
3932             delete $conf->{pending}->{$opt};
3933             update_config_nolock($vmid, $conf, 1);
3934             $conf = load_config($vmid); # update/reload
3935         }
3936     }
3937 }
3938
3939 sub vmconfig_apply_pending {
3940     my ($vmid, $conf, $storecfg) = @_;
3941
3942     # cold plug
3943
3944     my @delete = PVE::Tools::split_list($conf->{pending}->{delete});
3945     foreach my $opt (@delete) { # delete
3946         die "internal error" if $opt =~ m/^unused/;
3947         $conf = load_config($vmid); # update/reload
3948         if (!defined($conf->{$opt})) {
3949             vmconfig_undelete_pending_option($conf, $opt);
3950             update_config_nolock($vmid, $conf, 1);
3951         } elsif (valid_drivename($opt)) {
3952             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}));
3953             vmconfig_undelete_pending_option($conf, $opt);
3954             delete $conf->{$opt};
3955             update_config_nolock($vmid, $conf, 1);
3956         } else {
3957             vmconfig_undelete_pending_option($conf, $opt);
3958             delete $conf->{$opt};
3959             update_config_nolock($vmid, $conf, 1);
3960         }
3961     }
3962
3963     $conf = load_config($vmid); # update/reload
3964
3965     foreach my $opt (keys %{$conf->{pending}}) { # add/change
3966         $conf = load_config($vmid); # update/reload
3967
3968         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
3969             # skip if nothing changed
3970         } elsif (valid_drivename($opt)) {
3971             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
3972                 if defined($conf->{$opt});
3973             $conf->{$opt} = $conf->{pending}->{$opt};
3974         } else {
3975             $conf->{$opt} = $conf->{pending}->{$opt};
3976         }
3977
3978         delete $conf->{pending}->{$opt};
3979         update_config_nolock($vmid, $conf, 1);
3980     }
3981 }
3982
3983 my $safe_num_ne = sub {
3984     my ($a, $b) = @_;
3985
3986     return 0 if !defined($a) && !defined($b);
3987     return 1 if !defined($a);
3988     return 1 if !defined($b);
3989
3990     return $a != $b;
3991 };
3992
3993 my $safe_string_ne = sub {
3994     my ($a, $b) = @_;
3995
3996     return 0 if !defined($a) && !defined($b);
3997     return 1 if !defined($a);
3998     return 1 if !defined($b);
3999
4000     return $a ne $b;
4001 };
4002
4003 sub vmconfig_update_net {
4004     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4005
4006     my $newnet = parse_net($value);
4007
4008     if ($conf->{$opt}) {
4009         my $oldnet = parse_net($conf->{$opt});
4010
4011         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4012             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4013             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4014             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4015
4016             # for non online change, we try to hot-unplug
4017             die "skip\n" if !$hotplug;
4018             vm_deviceunplug($vmid, $conf, $opt);
4019         } else {
4020
4021             die "internal error" if $opt !~ m/net(\d+)/;
4022             my $iface = "tap${vmid}i$1";
4023
4024             if (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4025                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4026             }
4027
4028             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4029                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4030                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4031                 PVE::Network::tap_unplug($iface);
4032                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall});
4033             }
4034
4035             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4036                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4037             }
4038
4039             return 1;
4040         }
4041     }
4042
4043     if ($hotplug) {
4044         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4045     } else {
4046         die "skip\n";
4047     }
4048 }
4049
4050 sub vmconfig_update_disk {
4051     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4052
4053     # fixme: do we need force?
4054
4055     my $drive = parse_drive($opt, $value);
4056
4057     if ($conf->{$opt}) {
4058
4059         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4060
4061             my $media = $drive->{media} || 'disk';
4062             my $oldmedia = $old_drive->{media} || 'disk';
4063             die "unable to change media type\n" if $media ne $oldmedia;
4064
4065             if (!drive_is_cdrom($old_drive)) {
4066
4067                 if ($drive->{file} ne $old_drive->{file}) {
4068
4069                     die "skip\n" if !$hotplug;
4070
4071                     # unplug and register as unused
4072                     vm_deviceunplug($vmid, $conf, $opt);
4073                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4074
4075                 } else {
4076                     # update existing disk
4077
4078                     # skip non hotpluggable value
4079                     if (&$safe_num_ne($drive->{discard}, $old_drive->{discard}) ||
4080                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4081                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4082                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4083                         die "skip\n";
4084                     }
4085
4086                     # apply throttle
4087                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4088                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4089                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4090                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4091                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4092                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4093                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4094                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4095                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4096                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4097                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4098                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max})) {
4099
4100                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4101                                                    ($drive->{mbps} || 0)*1024*1024,
4102                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4103                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4104                                                    $drive->{iops} || 0,
4105                                                    $drive->{iops_rd} || 0,
4106                                                    $drive->{iops_wr} || 0,
4107                                                    ($drive->{mbps_max} || 0)*1024*1024,
4108                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4109                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4110                                                    $drive->{iops_max} || 0,
4111                                                    $drive->{iops_rd_max} || 0,
4112                                                    $drive->{iops_wr_max} || 0);
4113
4114                     }
4115
4116                     return 1;
4117                 }
4118
4119             } else { # cdrom
4120
4121                 if ($drive->{file} eq 'none') {
4122                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4123                 } else {
4124                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4125                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4126                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4127                 }
4128
4129                 return 1;
4130             }
4131         }
4132     }
4133
4134     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4135     # hotplug new disks
4136     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4137 }
4138
4139 sub vm_start {
4140     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused, $forcemachine, $spice_ticket) = @_;
4141
4142     lock_config($vmid, sub {
4143         my $conf = load_config($vmid, $migratedfrom);
4144
4145         die "you can't start a vm if it's a template\n" if is_template($conf);
4146
4147         check_lock($conf) if !$skiplock;
4148
4149         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4150
4151         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4152             vmconfig_apply_pending($vmid, $conf, $storecfg);
4153             $conf = load_config($vmid); # update/reload
4154         }
4155
4156         my $defaults = load_defaults();
4157
4158         # set environment variable useful inside network script
4159         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4160
4161         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4162
4163         my $migrate_port = 0;
4164         my $migrate_uri;
4165         if ($statefile) {
4166             if ($statefile eq 'tcp') {
4167                 my $localip = "localhost";
4168                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4169                 my $nodename = PVE::INotify::nodename();
4170                 if ($datacenterconf->{migration_unsecure}) {
4171                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4172                 }
4173                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4174                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4175                 $migrate_uri = "tcp:[${localip}]:${migrate_port}";
4176                 push @$cmd, '-incoming', $migrate_uri;
4177                 push @$cmd, '-S';
4178             } else {
4179                 push @$cmd, '-loadstate', $statefile;
4180             }
4181         } elsif ($paused) {
4182             push @$cmd, '-S';
4183         }
4184
4185         # host pci devices
4186         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4187           my $d = parse_hostpci($conf->{"hostpci$i"});
4188           next if !$d;
4189           my $pcidevices = $d->{pciid};
4190           foreach my $pcidevice (@$pcidevices) {
4191                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4192
4193                 my $info = pci_device_info("0000:$pciid");
4194                 die "IOMMU not present\n" if !check_iommu_support();
4195                 die "no pci device info for device '$pciid'\n" if !$info;
4196
4197                 if ($d->{driver} && $d->{driver} eq "vfio") {
4198                     die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4199                 } else {
4200                     die "can't unbind/bind to stub pci device '$pciid'\n" if !pci_dev_bind_to_stub($info);
4201                 }
4202
4203                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4204           }
4205         }
4206
4207         PVE::Storage::activate_volumes($storecfg, $vollist);
4208
4209         eval  { run_command($cmd, timeout => $statefile ? undef : 30,
4210                     umask => 0077); };
4211         my $err = $@;
4212         die "start failed: $err" if $err;
4213
4214         print "migration listens on $migrate_uri\n" if $migrate_uri;
4215
4216         if ($statefile && $statefile ne 'tcp')  {
4217             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4218             warn $@ if $@;
4219         }
4220
4221         if ($migratedfrom) {
4222
4223             eval {
4224                 set_migration_caps($vmid);
4225             };
4226             warn $@ if $@;
4227
4228             if ($spice_port) {
4229                 print "spice listens on port $spice_port\n";
4230                 if ($spice_ticket) {
4231                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4232                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4233                 }
4234             }
4235
4236         } else {
4237
4238             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4239                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4240                     if $conf->{balloon};
4241             }
4242
4243             foreach my $opt (keys %$conf) {
4244                 next if $opt !~  m/^net\d+$/;
4245                 my $nicconf = parse_net($conf->{$opt});
4246                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4247             }
4248         }
4249
4250         vm_mon_cmd_nocheck($vmid, 'qom-set',
4251                     path => "machine/peripheral/balloon0",
4252                     property => "guest-stats-polling-interval",
4253                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4254
4255     });
4256 }
4257
4258 sub vm_mon_cmd {
4259     my ($vmid, $execute, %params) = @_;
4260
4261     my $cmd = { execute => $execute, arguments => \%params };
4262     vm_qmp_command($vmid, $cmd);
4263 }
4264
4265 sub vm_mon_cmd_nocheck {
4266     my ($vmid, $execute, %params) = @_;
4267
4268     my $cmd = { execute => $execute, arguments => \%params };
4269     vm_qmp_command($vmid, $cmd, 1);
4270 }
4271
4272 sub vm_qmp_command {
4273     my ($vmid, $cmd, $nocheck) = @_;
4274
4275     my $res;
4276
4277     my $timeout;
4278     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4279         $timeout = $cmd->{arguments}->{timeout};
4280         delete $cmd->{arguments}->{timeout};
4281     }
4282
4283     eval {
4284         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4285         my $sname = qmp_socket($vmid);
4286         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4287             my $qmpclient = PVE::QMPClient->new();
4288
4289             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4290         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
4291             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
4292                 if scalar(%{$cmd->{arguments}});
4293             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
4294         } else {
4295             die "unable to open monitor socket\n";
4296         }
4297     };
4298     if (my $err = $@) {
4299         syslog("err", "VM $vmid qmp command failed - $err");
4300         die $err;
4301     }
4302
4303     return $res;
4304 }
4305
4306 sub vm_human_monitor_command {
4307     my ($vmid, $cmdline) = @_;
4308
4309     my $res;
4310
4311     my $cmd = {
4312         execute => 'human-monitor-command',
4313         arguments => { 'command-line' => $cmdline},
4314     };
4315
4316     return vm_qmp_command($vmid, $cmd);
4317 }
4318
4319 sub vm_commandline {
4320     my ($storecfg, $vmid) = @_;
4321
4322     my $conf = load_config($vmid);
4323
4324     my $defaults = load_defaults();
4325
4326     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
4327
4328     return join(' ', @$cmd);
4329 }
4330
4331 sub vm_reset {
4332     my ($vmid, $skiplock) = @_;
4333
4334     lock_config($vmid, sub {
4335
4336         my $conf = load_config($vmid);
4337
4338         check_lock($conf) if !$skiplock;
4339
4340         vm_mon_cmd($vmid, "system_reset");
4341     });
4342 }
4343
4344 sub get_vm_volumes {
4345     my ($conf) = @_;
4346
4347     my $vollist = [];
4348     foreach_volid($conf, sub {
4349         my ($volid, $is_cdrom) = @_;
4350
4351         return if $volid =~ m|^/|;
4352
4353         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4354         return if !$sid;
4355
4356         push @$vollist, $volid;
4357     });
4358
4359     return $vollist;
4360 }
4361
4362 sub vm_stop_cleanup {
4363     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
4364
4365     eval {
4366
4367         if (!$keepActive) {
4368             my $vollist = get_vm_volumes($conf);
4369             PVE::Storage::deactivate_volumes($storecfg, $vollist);
4370         }
4371
4372         foreach my $ext (qw(mon qmp pid vnc qga)) {
4373             unlink "/var/run/qemu-server/${vmid}.$ext";
4374         }
4375
4376         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
4377     };
4378     warn $@ if $@; # avoid errors - just warn
4379 }
4380
4381 # Note: use $nockeck to skip tests if VM configuration file exists.
4382 # We need that when migration VMs to other nodes (files already moved)
4383 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
4384 sub vm_stop {
4385     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
4386
4387     $force = 1 if !defined($force) && !$shutdown;
4388
4389     if ($migratedfrom){
4390         my $pid = check_running($vmid, $nocheck, $migratedfrom);
4391         kill 15, $pid if $pid;
4392         my $conf = load_config($vmid, $migratedfrom);
4393         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
4394         return;
4395     }
4396
4397     lock_config($vmid, sub {
4398
4399         my $pid = check_running($vmid, $nocheck);
4400         return if !$pid;
4401
4402         my $conf;
4403         if (!$nocheck) {
4404             $conf = load_config($vmid);
4405             check_lock($conf) if !$skiplock;
4406             if (!defined($timeout) && $shutdown && $conf->{startup}) {
4407                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
4408                 $timeout = $opts->{down} if $opts->{down};
4409             }
4410         }
4411
4412         $timeout = 60 if !defined($timeout);
4413
4414         eval {
4415             if ($shutdown) {
4416                 if (defined($conf) && $conf->{agent}) {
4417                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
4418                 } else {
4419                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
4420                 }
4421             } else {
4422                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
4423             }
4424         };
4425         my $err = $@;
4426
4427         if (!$err) {
4428             my $count = 0;
4429             while (($count < $timeout) && check_running($vmid, $nocheck)) {
4430                 $count++;
4431                 sleep 1;
4432             }
4433
4434             if ($count >= $timeout) {
4435                 if ($force) {
4436                     warn "VM still running - terminating now with SIGTERM\n";
4437                     kill 15, $pid;
4438                 } else {
4439                     die "VM quit/powerdown failed - got timeout\n";
4440                 }
4441             } else {
4442                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4443                 return;
4444             }
4445         } else {
4446             if ($force) {
4447                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
4448                 kill 15, $pid;
4449             } else {
4450                 die "VM quit/powerdown failed\n";
4451             }
4452         }
4453
4454         # wait again
4455         $timeout = 10;
4456
4457         my $count = 0;
4458         while (($count < $timeout) && check_running($vmid, $nocheck)) {
4459             $count++;
4460             sleep 1;
4461         }
4462
4463         if ($count >= $timeout) {
4464             warn "VM still running - terminating now with SIGKILL\n";
4465             kill 9, $pid;
4466             sleep 1;
4467         }
4468
4469         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4470    });
4471 }
4472
4473 sub vm_suspend {
4474     my ($vmid, $skiplock) = @_;
4475
4476     lock_config($vmid, sub {
4477
4478         my $conf = load_config($vmid);
4479
4480         check_lock($conf) if !($skiplock || ($conf->{lock} && $conf->{lock} eq 'backup'));
4481
4482         vm_mon_cmd($vmid, "stop");
4483     });
4484 }
4485
4486 sub vm_resume {
4487     my ($vmid, $skiplock) = @_;
4488
4489     lock_config($vmid, sub {
4490
4491         my $conf = load_config($vmid);
4492
4493         check_lock($conf) if !($skiplock || ($conf->{lock} && $conf->{lock} eq 'backup'));
4494
4495         vm_mon_cmd($vmid, "cont");
4496     });
4497 }
4498
4499 sub vm_sendkey {
4500     my ($vmid, $skiplock, $key) = @_;
4501
4502     lock_config($vmid, sub {
4503
4504         my $conf = load_config($vmid);
4505
4506         # there is no qmp command, so we use the human monitor command
4507         vm_human_monitor_command($vmid, "sendkey $key");
4508     });
4509 }
4510
4511 sub vm_destroy {
4512     my ($storecfg, $vmid, $skiplock) = @_;
4513
4514     lock_config($vmid, sub {
4515
4516         my $conf = load_config($vmid);
4517
4518         check_lock($conf) if !$skiplock;
4519
4520         if (!check_running($vmid)) {
4521             destroy_vm($storecfg, $vmid);
4522         } else {
4523             die "VM $vmid is running - destroy failed\n";
4524         }
4525     });
4526 }
4527
4528 # pci helpers
4529
4530 sub file_write {
4531     my ($filename, $buf) = @_;
4532
4533     my $fh = IO::File->new($filename, "w");
4534     return undef if !$fh;
4535
4536     my $res = print $fh $buf;
4537
4538     $fh->close();
4539
4540     return $res;
4541 }
4542
4543 sub pci_device_info {
4544     my ($name) = @_;
4545
4546     my $res;
4547
4548     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
4549     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
4550
4551     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
4552     return undef if !defined($irq) || $irq !~ m/^\d+$/;
4553
4554     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
4555     return undef if !defined($vendor) || $vendor !~ s/^0x//;
4556
4557     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
4558     return undef if !defined($product) || $product !~ s/^0x//;
4559
4560     $res = {
4561         name => $name,
4562         vendor => $vendor,
4563         product => $product,
4564         domain => $domain,
4565         bus => $bus,
4566         slot => $slot,
4567         func => $func,
4568         irq => $irq,
4569         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
4570     };
4571
4572     return $res;
4573 }
4574
4575 sub pci_dev_reset {
4576     my ($dev) = @_;
4577
4578     my $name = $dev->{name};
4579
4580     my $fn = "$pcisysfs/devices/$name/reset";
4581
4582     return file_write($fn, "1");
4583 }
4584
4585 sub pci_dev_bind_to_stub {
4586     my ($dev) = @_;
4587
4588     my $name = $dev->{name};
4589
4590     my $testdir = "$pcisysfs/drivers/pci-stub/$name";
4591     return 1 if -d $testdir;
4592
4593     my $data = "$dev->{vendor} $dev->{product}";
4594     return undef if !file_write("$pcisysfs/drivers/pci-stub/new_id", $data);
4595
4596     my $fn = "$pcisysfs/devices/$name/driver/unbind";
4597     if (!file_write($fn, $name)) {
4598         return undef if -f $fn;
4599     }
4600
4601     $fn = "$pcisysfs/drivers/pci-stub/bind";
4602     if (! -d $testdir) {
4603         return undef if !file_write($fn, $name);
4604     }
4605
4606     return -d $testdir;
4607 }
4608
4609 sub pci_dev_bind_to_vfio {
4610     my ($dev) = @_;
4611
4612     my $name = $dev->{name};
4613
4614     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4615
4616     if (!-d $vfio_basedir) {
4617         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4618     }
4619     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4620
4621     my $testdir = "$vfio_basedir/$name";
4622     return 1 if -d $testdir;
4623
4624     my $data = "$dev->{vendor} $dev->{product}";
4625     return undef if !file_write("$vfio_basedir/new_id", $data);
4626
4627     my $fn = "$pcisysfs/devices/$name/driver/unbind";
4628     if (!file_write($fn, $name)) {
4629         return undef if -f $fn;
4630     }
4631
4632     $fn = "$vfio_basedir/bind";
4633     if (! -d $testdir) {
4634         return undef if !file_write($fn, $name);
4635     }
4636
4637     return -d $testdir;
4638 }
4639
4640 sub pci_dev_group_bind_to_vfio {
4641     my ($pciid) = @_;
4642
4643     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4644
4645     if (!-d $vfio_basedir) {
4646         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4647     }
4648     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4649
4650     # get IOMMU group devices
4651     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
4652       my @devs = grep /^0000:/, readdir($D);
4653     closedir($D);
4654
4655     foreach my $pciid (@devs) {
4656         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
4657
4658         # pci bridges, switches or root ports are not supported
4659         # they have a pci_bus subdirectory so skip them
4660         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
4661
4662         my $info = pci_device_info($1);
4663         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
4664     }
4665
4666     return 1;
4667 }
4668
4669 sub print_pci_addr {
4670     my ($id, $bridges) = @_;
4671
4672     my $res = '';
4673     my $devices = {
4674         piix3 => { bus => 0, addr => 1 },
4675         #addr2 : first videocard
4676         balloon0 => { bus => 0, addr => 3 },
4677         watchdog => { bus => 0, addr => 4 },
4678         scsihw0 => { bus => 0, addr => 5 },
4679         'pci.3' => { bus => 0, addr => 5 }, #can also be used for virtio-scsi-single bridge
4680         scsihw1 => { bus => 0, addr => 6 },
4681         ahci0 => { bus => 0, addr => 7 },
4682         qga0 => { bus => 0, addr => 8 },
4683         spice => { bus => 0, addr => 9 },
4684         virtio0 => { bus => 0, addr => 10 },
4685         virtio1 => { bus => 0, addr => 11 },
4686         virtio2 => { bus => 0, addr => 12 },
4687         virtio3 => { bus => 0, addr => 13 },
4688         virtio4 => { bus => 0, addr => 14 },
4689         virtio5 => { bus => 0, addr => 15 },
4690         hostpci0 => { bus => 0, addr => 16 },
4691         hostpci1 => { bus => 0, addr => 17 },
4692         net0 => { bus => 0, addr => 18 },
4693         net1 => { bus => 0, addr => 19 },
4694         net2 => { bus => 0, addr => 20 },
4695         net3 => { bus => 0, addr => 21 },
4696         net4 => { bus => 0, addr => 22 },
4697         net5 => { bus => 0, addr => 23 },
4698         vga1 => { bus => 0, addr => 24 },
4699         vga2 => { bus => 0, addr => 25 },
4700         vga3 => { bus => 0, addr => 26 },
4701         hostpci2 => { bus => 0, addr => 27 },
4702         hostpci3 => { bus => 0, addr => 28 },
4703         #addr29 : usb-host (pve-usb.cfg)
4704         'pci.1' => { bus => 0, addr => 30 },
4705         'pci.2' => { bus => 0, addr => 31 },
4706         'net6' => { bus => 1, addr => 1 },
4707         'net7' => { bus => 1, addr => 2 },
4708         'net8' => { bus => 1, addr => 3 },
4709         'net9' => { bus => 1, addr => 4 },
4710         'net10' => { bus => 1, addr => 5 },
4711         'net11' => { bus => 1, addr => 6 },
4712         'net12' => { bus => 1, addr => 7 },
4713         'net13' => { bus => 1, addr => 8 },
4714         'net14' => { bus => 1, addr => 9 },
4715         'net15' => { bus => 1, addr => 10 },
4716         'net16' => { bus => 1, addr => 11 },
4717         'net17' => { bus => 1, addr => 12 },
4718         'net18' => { bus => 1, addr => 13 },
4719         'net19' => { bus => 1, addr => 14 },
4720         'net20' => { bus => 1, addr => 15 },
4721         'net21' => { bus => 1, addr => 16 },
4722         'net22' => { bus => 1, addr => 17 },
4723         'net23' => { bus => 1, addr => 18 },
4724         'net24' => { bus => 1, addr => 19 },
4725         'net25' => { bus => 1, addr => 20 },
4726         'net26' => { bus => 1, addr => 21 },
4727         'net27' => { bus => 1, addr => 22 },
4728         'net28' => { bus => 1, addr => 23 },
4729         'net29' => { bus => 1, addr => 24 },
4730         'net30' => { bus => 1, addr => 25 },
4731         'net31' => { bus => 1, addr => 26 },
4732         'virtio6' => { bus => 2, addr => 1 },
4733         'virtio7' => { bus => 2, addr => 2 },
4734         'virtio8' => { bus => 2, addr => 3 },
4735         'virtio9' => { bus => 2, addr => 4 },
4736         'virtio10' => { bus => 2, addr => 5 },
4737         'virtio11' => { bus => 2, addr => 6 },
4738         'virtio12' => { bus => 2, addr => 7 },
4739         'virtio13' => { bus => 2, addr => 8 },
4740         'virtio14' => { bus => 2, addr => 9 },
4741         'virtio15' => { bus => 2, addr => 10 },
4742         'virtioscsi0' => { bus => 3, addr => 1 },
4743         'virtioscsi1' => { bus => 3, addr => 2 },
4744         'virtioscsi2' => { bus => 3, addr => 3 },
4745         'virtioscsi3' => { bus => 3, addr => 4 },
4746         'virtioscsi4' => { bus => 3, addr => 5 },
4747         'virtioscsi5' => { bus => 3, addr => 6 },
4748         'virtioscsi6' => { bus => 3, addr => 7 },
4749         'virtioscsi7' => { bus => 3, addr => 8 },
4750         'virtioscsi8' => { bus => 3, addr => 9 },
4751         'virtioscsi9' => { bus => 3, addr => 10 },
4752         'virtioscsi10' => { bus => 3, addr => 11 },
4753         'virtioscsi11' => { bus => 3, addr => 12 },
4754         'virtioscsi12' => { bus => 3, addr => 13 },
4755         'virtioscsi13' => { bus => 3, addr => 14 },
4756         'virtioscsi14' => { bus => 3, addr => 15 },
4757         'virtioscsi15' => { bus => 3, addr => 16 },
4758         'virtioscsi16' => { bus => 3, addr => 17 },
4759         'virtioscsi17' => { bus => 3, addr => 18 },
4760         'virtioscsi18' => { bus => 3, addr => 19 },
4761         'virtioscsi19' => { bus => 3, addr => 20 },
4762         'virtioscsi20' => { bus => 3, addr => 21 },
4763         'virtioscsi21' => { bus => 3, addr => 22 },
4764         'virtioscsi22' => { bus => 3, addr => 23 },
4765         'virtioscsi23' => { bus => 3, addr => 24 },
4766         'virtioscsi24' => { bus => 3, addr => 25 },
4767         'virtioscsi25' => { bus => 3, addr => 26 },
4768         'virtioscsi26' => { bus => 3, addr => 27 },
4769         'virtioscsi27' => { bus => 3, addr => 28 },
4770         'virtioscsi28' => { bus => 3, addr => 29 },
4771         'virtioscsi29' => { bus => 3, addr => 30 },
4772         'virtioscsi30' => { bus => 3, addr => 31 },
4773
4774     };
4775
4776     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
4777            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
4778            my $bus = $devices->{$id}->{bus};
4779            $res = ",bus=pci.$bus,addr=$addr";
4780            $bridges->{$bus} = 1 if $bridges;
4781     }
4782     return $res;
4783
4784 }
4785
4786 sub print_pcie_addr {
4787     my ($id) = @_;
4788
4789     my $res = '';
4790     my $devices = {
4791         hostpci0 => { bus => "ich9-pcie-port-1", addr => 0 },
4792         hostpci1 => { bus => "ich9-pcie-port-2", addr => 0 },
4793         hostpci2 => { bus => "ich9-pcie-port-3", addr => 0 },
4794         hostpci3 => { bus => "ich9-pcie-port-4", addr => 0 },
4795     };
4796
4797     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
4798            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
4799            my $bus = $devices->{$id}->{bus};
4800            $res = ",bus=$bus,addr=$addr";
4801     }
4802     return $res;
4803
4804 }
4805
4806 # vzdump restore implementaion
4807
4808 sub tar_archive_read_firstfile {
4809     my $archive = shift;
4810
4811     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
4812
4813     # try to detect archive type first
4814     my $pid = open (TMP, "tar tf '$archive'|") ||
4815         die "unable to open file '$archive'\n";
4816     my $firstfile = <TMP>;
4817     kill 15, $pid;
4818     close TMP;
4819
4820     die "ERROR: archive contaions no data\n" if !$firstfile;
4821     chomp $firstfile;
4822
4823     return $firstfile;
4824 }
4825
4826 sub tar_restore_cleanup {
4827     my ($storecfg, $statfile) = @_;
4828
4829     print STDERR "starting cleanup\n";
4830
4831     if (my $fd = IO::File->new($statfile, "r")) {
4832         while (defined(my $line = <$fd>)) {
4833             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
4834                 my $volid = $2;
4835                 eval {
4836                     if ($volid =~ m|^/|) {
4837                         unlink $volid || die 'unlink failed\n';
4838                     } else {
4839                         PVE::Storage::vdisk_free($storecfg, $volid);
4840                     }
4841                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
4842                 };
4843                 print STDERR "unable to cleanup '$volid' - $@" if $@;
4844             } else {
4845                 print STDERR "unable to parse line in statfile - $line";
4846             }
4847         }
4848         $fd->close();
4849     }
4850 }
4851
4852 sub restore_archive {
4853     my ($archive, $vmid, $user, $opts) = @_;
4854
4855     my $format = $opts->{format};
4856     my $comp;
4857
4858     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
4859         $format = 'tar' if !$format;
4860         $comp = 'gzip';
4861     } elsif ($archive =~ m/\.tar$/) {
4862         $format = 'tar' if !$format;
4863     } elsif ($archive =~ m/.tar.lzo$/) {
4864         $format = 'tar' if !$format;
4865         $comp = 'lzop';
4866     } elsif ($archive =~ m/\.vma$/) {
4867         $format = 'vma' if !$format;
4868     } elsif ($archive =~ m/\.vma\.gz$/) {
4869         $format = 'vma' if !$format;
4870         $comp = 'gzip';
4871     } elsif ($archive =~ m/\.vma\.lzo$/) {
4872         $format = 'vma' if !$format;
4873         $comp = 'lzop';
4874     } else {
4875         $format = 'vma' if !$format; # default
4876     }
4877
4878     # try to detect archive format
4879     if ($format eq 'tar') {
4880         return restore_tar_archive($archive, $vmid, $user, $opts);
4881     } else {
4882         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
4883     }
4884 }
4885
4886 sub restore_update_config_line {
4887     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
4888
4889     return if $line =~ m/^\#qmdump\#/;
4890     return if $line =~ m/^\#vzdump\#/;
4891     return if $line =~ m/^lock:/;
4892     return if $line =~ m/^unused\d+:/;
4893     return if $line =~ m/^parent:/;
4894     return if $line =~ m/^template:/; # restored VM is never a template
4895
4896     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
4897         # try to convert old 1.X settings
4898         my ($id, $ind, $ethcfg) = ($1, $2, $3);
4899         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
4900             my ($model, $macaddr) = split(/\=/, $devconfig);
4901             $macaddr = PVE::Tools::random_ether_addr() if !$macaddr || $unique;
4902             my $net = {
4903                 model => $model,
4904                 bridge => "vmbr$ind",
4905                 macaddr => $macaddr,
4906             };
4907             my $netstr = print_net($net);
4908
4909             print $outfd "net$cookie->{netcount}: $netstr\n";
4910             $cookie->{netcount}++;
4911         }
4912     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
4913         my ($id, $netstr) = ($1, $2);
4914         my $net = parse_net($netstr);
4915         $net->{macaddr} = PVE::Tools::random_ether_addr() if $net->{macaddr};
4916         $netstr = print_net($net);
4917         print $outfd "$id: $netstr\n";
4918     } elsif ($line =~ m/^((ide|scsi|virtio|sata)\d+):\s*(\S+)\s*$/) {
4919         my $virtdev = $1;
4920         my $value = $3;
4921         if ($line =~ m/backup=no/) {
4922             print $outfd "#$line";
4923         } elsif ($virtdev && $map->{$virtdev}) {
4924             my $di = parse_drive($virtdev, $value);
4925             delete $di->{format}; # format can change on restore
4926             $di->{file} = $map->{$virtdev};
4927             $value = print_drive($vmid, $di);
4928             print $outfd "$virtdev: $value\n";
4929         } else {
4930             print $outfd $line;
4931         }
4932     } else {
4933         print $outfd $line;
4934     }
4935 }
4936
4937 sub scan_volids {
4938     my ($cfg, $vmid) = @_;
4939
4940     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
4941
4942     my $volid_hash = {};
4943     foreach my $storeid (keys %$info) {
4944         foreach my $item (@{$info->{$storeid}}) {
4945             next if !($item->{volid} && $item->{size});
4946             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
4947             $volid_hash->{$item->{volid}} = $item;
4948         }
4949     }
4950
4951     return $volid_hash;
4952 }
4953
4954 sub get_used_paths {
4955     my ($vmid, $storecfg, $conf, $scan_snapshots, $skip_drive) = @_;
4956
4957     my $used_path = {};
4958
4959     my $scan_config = sub {
4960         my ($cref, $snapname) = @_;
4961
4962         foreach my $key (keys %$cref) {
4963             my $value = $cref->{$key};
4964             if (valid_drivename($key)) {
4965                 next if $skip_drive && $key eq $skip_drive;
4966                 my $drive = parse_drive($key, $value);
4967                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
4968                 if ($drive->{file} =~ m!^/!) {
4969                     $used_path->{$drive->{file}}++; # = 1;
4970                 } else {
4971                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
4972                     next if !$storeid;
4973                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
4974                     next if !$scfg;
4975                     my $path = PVE::Storage::path($storecfg, $drive->{file}, $snapname);
4976                     $used_path->{$path}++; # = 1;
4977                 }
4978             }
4979         }
4980     };
4981
4982     &$scan_config($conf);
4983
4984     undef $skip_drive;
4985
4986     if ($scan_snapshots) {
4987         foreach my $snapname (keys %{$conf->{snapshots}}) {
4988             &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
4989         }
4990     }
4991
4992     return $used_path;
4993 }
4994
4995 sub update_disksize {
4996     my ($vmid, $conf, $volid_hash) = @_;
4997
4998     my $changes;
4999
5000     my $used = {};
5001
5002     # Note: it is allowed to define multiple storages with same path (alias), so
5003     # we need to check both 'volid' and real 'path' (two different volid can point
5004     # to the same path).
5005
5006     my $usedpath = {};
5007
5008     # update size info
5009     foreach my $opt (keys %$conf) {
5010         if (valid_drivename($opt)) {
5011             my $drive = parse_drive($opt, $conf->{$opt});
5012             my $volid = $drive->{file};
5013             next if !$volid;
5014
5015             $used->{$volid} = 1;
5016             if ($volid_hash->{$volid} &&
5017                 (my $path = $volid_hash->{$volid}->{path})) {
5018                 $usedpath->{$path} = 1;
5019             }
5020
5021             next if drive_is_cdrom($drive);
5022             next if !$volid_hash->{$volid};
5023
5024             $drive->{size} = $volid_hash->{$volid}->{size};
5025             my $new = print_drive($vmid, $drive);
5026             if ($new ne $conf->{$opt}) {
5027                 $changes = 1;
5028                 $conf->{$opt} = $new;
5029             }
5030         }
5031     }
5032
5033     # remove 'unusedX' entry if volume is used
5034     foreach my $opt (keys %$conf) {
5035         next if $opt !~ m/^unused\d+$/;
5036         my $volid = $conf->{$opt};
5037         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5038         if ($used->{$volid} || ($path && $usedpath->{$path})) {
5039             $changes = 1;
5040             delete $conf->{$opt};
5041         }
5042     }
5043
5044     foreach my $volid (sort keys %$volid_hash) {
5045         next if $volid =~ m/vm-$vmid-state-/;
5046         next if $used->{$volid};
5047         my $path = $volid_hash->{$volid}->{path};
5048         next if !$path; # just to be sure
5049         next if $usedpath->{$path};
5050         $changes = 1;
5051         add_unused_volume($conf, $volid);
5052         $usedpath->{$path} = 1; # avoid to add more than once (aliases)
5053     }
5054
5055     return $changes;
5056 }
5057
5058 sub rescan {
5059     my ($vmid, $nolock) = @_;
5060
5061     my $cfg = PVE::Cluster::cfs_read_file("storage.cfg");
5062
5063     my $volid_hash = scan_volids($cfg, $vmid);
5064
5065     my $updatefn =  sub {
5066         my ($vmid) = @_;
5067
5068         my $conf = load_config($vmid);
5069
5070         check_lock($conf);
5071
5072         my $vm_volids = {};
5073         foreach my $volid (keys %$volid_hash) {
5074             my $info = $volid_hash->{$volid};
5075             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5076         }
5077
5078         my $changes = update_disksize($vmid, $conf, $vm_volids);
5079
5080         update_config_nolock($vmid, $conf, 1) if $changes;
5081     };
5082
5083     if (defined($vmid)) {
5084         if ($nolock) {
5085             &$updatefn($vmid);
5086         } else {
5087             lock_config($vmid, $updatefn, $vmid);
5088         }
5089     } else {
5090         my $vmlist = config_list();
5091         foreach my $vmid (keys %$vmlist) {
5092             if ($nolock) {
5093                 &$updatefn($vmid);
5094             } else {
5095                 lock_config($vmid, $updatefn, $vmid);
5096             }
5097         }
5098     }
5099 }
5100
5101 sub restore_vma_archive {
5102     my ($archive, $vmid, $user, $opts, $comp) = @_;
5103
5104     my $input = $archive eq '-' ? "<&STDIN" : undef;
5105     my $readfrom = $archive;
5106
5107     my $uncomp = '';
5108     if ($comp) {
5109         $readfrom = '-';
5110         my $qarchive = PVE::Tools::shellquote($archive);
5111         if ($comp eq 'gzip') {
5112             $uncomp = "zcat $qarchive|";
5113         } elsif ($comp eq 'lzop') {
5114             $uncomp = "lzop -d -c $qarchive|";
5115         } else {
5116             die "unknown compression method '$comp'\n";
5117         }
5118
5119     }
5120
5121     my $tmpdir = "/var/tmp/vzdumptmp$$";
5122     rmtree $tmpdir;
5123
5124     # disable interrupts (always do cleanups)
5125     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5126         warn "got interrupt - ignored\n";
5127     };
5128
5129     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5130     POSIX::mkfifo($mapfifo, 0600);
5131     my $fifofh;
5132
5133     my $openfifo = sub {
5134         open($fifofh, '>', $mapfifo) || die $!;
5135     };
5136
5137     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5138
5139     my $oldtimeout;
5140     my $timeout = 5;
5141
5142     my $devinfo = {};
5143
5144     my $rpcenv = PVE::RPCEnvironment::get();
5145
5146     my $conffile = config_file($vmid);
5147     my $tmpfn = "$conffile.$$.tmp";
5148
5149     # Note: $oldconf is undef if VM does not exists
5150     my $oldconf = PVE::Cluster::cfs_read_file(cfs_config_path($vmid));
5151
5152     my $print_devmap = sub {
5153         my $virtdev_hash = {};
5154
5155         my $cfgfn = "$tmpdir/qemu-server.conf";
5156
5157         # we can read the config - that is already extracted
5158         my $fh = IO::File->new($cfgfn, "r") ||
5159             "unable to read qemu-server.conf - $!\n";
5160
5161         while (defined(my $line = <$fh>)) {
5162             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5163                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5164                 die "archive does not contain data for drive '$virtdev'\n"
5165                     if !$devinfo->{$devname};
5166                 if (defined($opts->{storage})) {
5167                     $storeid = $opts->{storage} || 'local';
5168                 } elsif (!$storeid) {
5169                     $storeid = 'local';
5170                 }
5171                 $format = 'raw' if !$format;
5172                 $devinfo->{$devname}->{devname} = $devname;
5173                 $devinfo->{$devname}->{virtdev} = $virtdev;
5174                 $devinfo->{$devname}->{format} = $format;
5175                 $devinfo->{$devname}->{storeid} = $storeid;
5176
5177                 # check permission on storage
5178                 my $pool = $opts->{pool}; # todo: do we need that?
5179                 if ($user ne 'root@pam') {
5180                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5181                 }
5182
5183                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5184             }
5185         }
5186
5187         foreach my $devname (keys %$devinfo) {
5188             die "found no device mapping information for device '$devname'\n"
5189                 if !$devinfo->{$devname}->{virtdev};
5190         }
5191
5192         my $cfg = cfs_read_file('storage.cfg');
5193
5194         # create empty/temp config
5195         if ($oldconf) {
5196             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5197             foreach_drive($oldconf, sub {
5198                 my ($ds, $drive) = @_;
5199
5200                 return if drive_is_cdrom($drive);
5201
5202                 my $volid = $drive->{file};
5203
5204                 return if !$volid || $volid =~ m|^/|;
5205
5206                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5207                 return if !$path || !$owner || ($owner != $vmid);
5208
5209                 # Note: only delete disk we want to restore
5210                 # other volumes will become unused
5211                 if ($virtdev_hash->{$ds}) {
5212                     PVE::Storage::vdisk_free($cfg, $volid);
5213                 }
5214             });
5215         }
5216
5217         my $map = {};
5218         foreach my $virtdev (sort keys %$virtdev_hash) {
5219             my $d = $virtdev_hash->{$virtdev};
5220             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5221             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5222
5223             # test if requested format is supported
5224             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5225             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5226             $d->{format} = $defFormat if !$supported;
5227
5228             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5229                                                   $d->{format}, undef, $alloc_size);
5230             print STDERR "new volume ID is '$volid'\n";
5231             $d->{volid} = $volid;
5232             my $path = PVE::Storage::path($cfg, $volid);
5233
5234             my $write_zeros = 1;
5235             # fixme: what other storages types initialize volumes with zero?
5236             if ($scfg->{type} eq 'dir' || $scfg->{type} eq 'nfs' || $scfg->{type} eq 'glusterfs' ||
5237                 $scfg->{type} eq 'sheepdog' || $scfg->{type} eq 'rbd') {
5238                 $write_zeros = 0;
5239             }
5240
5241             print $fifofh "${write_zeros}:$d->{devname}=$path\n";
5242
5243             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5244             $map->{$virtdev} = $volid;
5245         }
5246
5247         $fh->seek(0, 0) || die "seek failed - $!\n";
5248
5249         my $outfd = new IO::File ($tmpfn, "w") ||
5250             die "unable to write config for VM $vmid\n";
5251
5252         my $cookie = { netcount => 0 };
5253         while (defined(my $line = <$fh>)) {
5254             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5255         }
5256
5257         $fh->close();
5258         $outfd->close();
5259     };
5260
5261     eval {
5262         # enable interrupts
5263         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5264             die "interrupted by signal\n";
5265         };
5266         local $SIG{ALRM} = sub { die "got timeout\n"; };
5267
5268         $oldtimeout = alarm($timeout);
5269
5270         my $parser = sub {
5271             my $line = shift;
5272
5273             print "$line\n";
5274
5275             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5276                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5277                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5278             } elsif ($line =~ m/^CTIME: /) {
5279                 # we correctly received the vma config, so we can disable
5280                 # the timeout now for disk allocation (set to 10 minutes, so
5281                 # that we always timeout if something goes wrong)
5282                 alarm(600);
5283                 &$print_devmap();
5284                 print $fifofh "done\n";
5285                 my $tmp = $oldtimeout || 0;
5286                 $oldtimeout = undef;
5287                 alarm($tmp);
5288                 close($fifofh);
5289             }
5290         };
5291
5292         print "restore vma archive: $cmd\n";
5293         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5294     };
5295     my $err = $@;
5296
5297     alarm($oldtimeout) if $oldtimeout;
5298
5299     unlink $mapfifo;
5300
5301     if ($err) {
5302         rmtree $tmpdir;
5303         unlink $tmpfn;
5304
5305         my $cfg = cfs_read_file('storage.cfg');
5306         foreach my $devname (keys %$devinfo) {
5307             my $volid = $devinfo->{$devname}->{volid};
5308             next if !$volid;
5309             eval {
5310                 if ($volid =~ m|^/|) {
5311                     unlink $volid || die 'unlink failed\n';
5312                 } else {
5313                     PVE::Storage::vdisk_free($cfg, $volid);
5314                 }
5315                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5316             };
5317             print STDERR "unable to cleanup '$volid' - $@" if $@;
5318         }
5319         die $err;
5320     }
5321
5322     rmtree $tmpdir;
5323
5324     rename($tmpfn, $conffile) ||
5325         die "unable to commit configuration file '$conffile'\n";
5326
5327     PVE::Cluster::cfs_update(); # make sure we read new file
5328
5329     eval { rescan($vmid, 1); };
5330     warn $@ if $@;
5331 }
5332
5333 sub restore_tar_archive {
5334     my ($archive, $vmid, $user, $opts) = @_;
5335
5336     if ($archive ne '-') {
5337         my $firstfile = tar_archive_read_firstfile($archive);
5338         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5339             if $firstfile ne 'qemu-server.conf';
5340     }
5341
5342     my $storecfg = cfs_read_file('storage.cfg');
5343
5344     # destroy existing data - keep empty config
5345     my $vmcfgfn = config_file($vmid);
5346     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5347
5348     my $tocmd = "/usr/lib/qemu-server/qmextract";
5349
5350     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5351     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5352     $tocmd .= ' --prealloc' if $opts->{prealloc};
5353     $tocmd .= ' --info' if $opts->{info};
5354
5355     # tar option "xf" does not autodetect compression when read from STDIN,
5356     # so we pipe to zcat
5357     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5358         PVE::Tools::shellquote("--to-command=$tocmd");
5359
5360     my $tmpdir = "/var/tmp/vzdumptmp$$";
5361     mkpath $tmpdir;
5362
5363     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5364     local $ENV{VZDUMP_VMID} = $vmid;
5365     local $ENV{VZDUMP_USER} = $user;
5366
5367     my $conffile = config_file($vmid);
5368     my $tmpfn = "$conffile.$$.tmp";
5369
5370     # disable interrupts (always do cleanups)
5371     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5372         print STDERR "got interrupt - ignored\n";
5373     };
5374
5375     eval {
5376         # enable interrupts
5377         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5378             die "interrupted by signal\n";
5379         };
5380
5381         if ($archive eq '-') {
5382             print "extracting archive from STDIN\n";
5383             run_command($cmd, input => "<&STDIN");
5384         } else {
5385             print "extracting archive '$archive'\n";
5386             run_command($cmd);
5387         }
5388
5389         return if $opts->{info};
5390
5391         # read new mapping
5392         my $map = {};
5393         my $statfile = "$tmpdir/qmrestore.stat";
5394         if (my $fd = IO::File->new($statfile, "r")) {
5395             while (defined (my $line = <$fd>)) {
5396                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5397                     $map->{$1} = $2 if $1;
5398                 } else {
5399                     print STDERR "unable to parse line in statfile - $line\n";
5400                 }
5401             }
5402             $fd->close();
5403         }
5404
5405         my $confsrc = "$tmpdir/qemu-server.conf";
5406
5407         my $srcfd = new IO::File($confsrc, "r") ||
5408             die "unable to open file '$confsrc'\n";
5409
5410         my $outfd = new IO::File ($tmpfn, "w") ||
5411             die "unable to write config for VM $vmid\n";
5412
5413         my $cookie = { netcount => 0 };
5414         while (defined (my $line = <$srcfd>)) {
5415             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5416         }
5417
5418         $srcfd->close();
5419         $outfd->close();
5420     };
5421     my $err = $@;
5422
5423     if ($err) {
5424
5425         unlink $tmpfn;
5426
5427         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
5428
5429         die $err;
5430     }
5431
5432     rmtree $tmpdir;
5433
5434     rename $tmpfn, $conffile ||
5435         die "unable to commit configuration file '$conffile'\n";
5436
5437     PVE::Cluster::cfs_update(); # make sure we read new file
5438
5439     eval { rescan($vmid, 1); };
5440     warn $@ if $@;
5441 };
5442
5443
5444 # Internal snapshots
5445
5446 # NOTE: Snapshot create/delete involves several non-atomic
5447 # action, and can take a long time.
5448 # So we try to avoid locking the file and use 'lock' variable
5449 # inside the config file instead.
5450
5451 my $snapshot_copy_config = sub {
5452     my ($source, $dest) = @_;
5453
5454     foreach my $k (keys %$source) {
5455         next if $k eq 'snapshots';
5456         next if $k eq 'snapstate';
5457         next if $k eq 'snaptime';
5458         next if $k eq 'vmstate';
5459         next if $k eq 'lock';
5460         next if $k eq 'digest';
5461         next if $k eq 'description';
5462         next if $k =~ m/^unused\d+$/;
5463
5464         $dest->{$k} = $source->{$k};
5465     }
5466 };
5467
5468 my $snapshot_apply_config = sub {
5469     my ($conf, $snap) = @_;
5470
5471     # copy snapshot list
5472     my $newconf = {
5473         snapshots => $conf->{snapshots},
5474     };
5475
5476     # keep description and list of unused disks
5477     foreach my $k (keys %$conf) {
5478         next if !($k =~ m/^unused\d+$/ || $k eq 'description');
5479         $newconf->{$k} = $conf->{$k};
5480     }
5481
5482     &$snapshot_copy_config($snap, $newconf);
5483
5484     return $newconf;
5485 };
5486
5487 sub foreach_writable_storage {
5488     my ($conf, $func) = @_;
5489
5490     my $sidhash = {};
5491
5492     foreach my $ds (keys %$conf) {
5493         next if !valid_drivename($ds);
5494
5495         my $drive = parse_drive($ds, $conf->{$ds});
5496         next if !$drive;
5497         next if drive_is_cdrom($drive);
5498
5499         my $volid = $drive->{file};
5500
5501         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5502         $sidhash->{$sid} = $sid if $sid;
5503     }
5504
5505     foreach my $sid (sort keys %$sidhash) {
5506         &$func($sid);
5507     }
5508 }
5509
5510 my $alloc_vmstate_volid = sub {
5511     my ($storecfg, $vmid, $conf, $snapname) = @_;
5512
5513     # Note: we try to be smart when selecting a $target storage
5514
5515     my $target;
5516
5517     # search shared storage first
5518     foreach_writable_storage($conf, sub {
5519         my ($sid) = @_;
5520         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
5521         return if !$scfg->{shared};
5522
5523         $target = $sid if !$target || $scfg->{path}; # prefer file based storage
5524     });
5525
5526     if (!$target) {
5527         # now search local storage
5528         foreach_writable_storage($conf, sub {
5529             my ($sid) = @_;
5530             my $scfg = PVE::Storage::storage_config($storecfg, $sid);
5531             return if $scfg->{shared};
5532
5533             $target = $sid if !$target || $scfg->{path}; # prefer file based storage;
5534         });
5535     }
5536
5537     $target = 'local' if !$target;
5538
5539     my $driver_state_size = 500; # assume 32MB is enough to safe all driver state;
5540     # we abort live save after $conf->{memory}, so we need at max twice that space
5541     my $size = $conf->{memory}*2 + $driver_state_size;
5542
5543     my $name = "vm-$vmid-state-$snapname";
5544     my $scfg = PVE::Storage::storage_config($storecfg, $target);
5545     $name .= ".raw" if $scfg->{path}; # add filename extension for file base storage
5546     my $volid = PVE::Storage::vdisk_alloc($storecfg, $target, $vmid, 'raw', $name, $size*1024);
5547
5548     return $volid;
5549 };
5550
5551 my $snapshot_prepare = sub {
5552     my ($vmid, $snapname, $save_vmstate, $comment) = @_;
5553
5554     my $snap;
5555
5556     my $updatefn =  sub {
5557
5558         my $conf = load_config($vmid);
5559
5560         die "you can't take a snapshot if it's a template\n"
5561             if is_template($conf);
5562
5563         check_lock($conf);
5564
5565         $conf->{lock} = 'snapshot';
5566
5567         die "snapshot name '$snapname' already used\n"
5568             if defined($conf->{snapshots}->{$snapname});
5569
5570         my $storecfg = PVE::Storage::config();
5571         die "snapshot feature is not available" if !has_feature('snapshot', $conf, $storecfg);
5572
5573         $snap = $conf->{snapshots}->{$snapname} = {};
5574
5575         if ($save_vmstate && check_running($vmid)) {
5576             $snap->{vmstate} = &$alloc_vmstate_volid($storecfg, $vmid, $conf, $snapname);
5577         }
5578
5579         &$snapshot_copy_config($conf, $snap);
5580
5581         $snap->{snapstate} = "prepare";
5582         $snap->{snaptime} = time();
5583         $snap->{description} = $comment if $comment;
5584
5585         # always overwrite machine if we save vmstate. This makes sure we
5586         # can restore it later using correct machine type
5587         $snap->{machine} = get_current_qemu_machine($vmid) if $snap->{vmstate};
5588
5589         update_config_nolock($vmid, $conf, 1);
5590     };
5591
5592     lock_config($vmid, $updatefn);
5593
5594     return $snap;
5595 };
5596
5597 my $snapshot_commit = sub {
5598     my ($vmid, $snapname) = @_;
5599
5600     my $updatefn = sub {
5601
5602         my $conf = load_config($vmid);
5603
5604         die "missing snapshot lock\n"
5605             if !($conf->{lock} && $conf->{lock} eq 'snapshot');
5606
5607         my $has_machine_config = defined($conf->{machine});
5608
5609         my $snap = $conf->{snapshots}->{$snapname};
5610
5611         die "snapshot '$snapname' does not exist\n" if !defined($snap);
5612
5613         die "wrong snapshot state\n"
5614             if !($snap->{snapstate} && $snap->{snapstate} eq "prepare");
5615
5616         delete $snap->{snapstate};
5617         delete $conf->{lock};
5618
5619         my $newconf = &$snapshot_apply_config($conf, $snap);
5620
5621         delete $newconf->{machine} if !$has_machine_config;
5622
5623         $newconf->{parent} = $snapname;
5624
5625         update_config_nolock($vmid, $newconf, 1);
5626     };
5627
5628     lock_config($vmid, $updatefn);
5629 };
5630
5631 sub snapshot_rollback {
5632     my ($vmid, $snapname) = @_;
5633
5634     my $prepare = 1;
5635
5636     my $storecfg = PVE::Storage::config();
5637
5638     my $conf = load_config($vmid);
5639
5640     my $get_snapshot_config = sub {
5641
5642         die "you can't rollback if vm is a template\n" if is_template($conf);
5643
5644         my $res = $conf->{snapshots}->{$snapname};
5645
5646         die "snapshot '$snapname' does not exist\n" if !defined($res);
5647
5648         return $res;
5649     };
5650
5651     my $snap = &$get_snapshot_config();
5652
5653     foreach_drive($snap, sub {
5654         my ($ds, $drive) = @_;
5655
5656         return if drive_is_cdrom($drive);
5657
5658         my $volid = $drive->{file};
5659
5660         PVE::Storage::volume_rollback_is_possible($storecfg, $volid, $snapname);
5661     });
5662
5663     my $updatefn = sub {
5664
5665         $conf = load_config($vmid);
5666
5667         $snap = &$get_snapshot_config();
5668
5669         die "unable to rollback to incomplete snapshot (snapstate = $snap->{snapstate})\n"
5670             if $snap->{snapstate};
5671
5672         if ($prepare) {
5673             check_lock($conf);
5674             vm_stop($storecfg, $vmid, undef, undef, 5, undef, undef);
5675         }
5676
5677         die "unable to rollback vm $vmid: vm is running\n"
5678             if check_running($vmid);
5679
5680         if ($prepare) {
5681             $conf->{lock} = 'rollback';
5682         } else {
5683             die "got wrong lock\n" if !($conf->{lock} && $conf->{lock} eq 'rollback');
5684             delete $conf->{lock};
5685         }
5686
5687         my $forcemachine;
5688
5689         if (!$prepare) {
5690             my $has_machine_config = defined($conf->{machine});
5691
5692             # copy snapshot config to current config
5693             $conf = &$snapshot_apply_config($conf, $snap);
5694             $conf->{parent} = $snapname;
5695
5696             # Note: old code did not store 'machine', so we try to be smart
5697             # and guess the snapshot was generated with kvm 1.4 (pc-i440fx-1.4).
5698             $forcemachine = $conf->{machine} || 'pc-i440fx-1.4';
5699             # we remove the 'machine' configuration if not explicitly specified
5700             # in the original config.
5701             delete $conf->{machine} if $snap->{vmstate} && !$has_machine_config;
5702         }
5703
5704         update_config_nolock($vmid, $conf, 1);
5705
5706         if (!$prepare && $snap->{vmstate}) {
5707             my $statefile = PVE::Storage::path($storecfg, $snap->{vmstate});
5708             vm_start($storecfg, $vmid, $statefile, undef, undef, undef, $forcemachine);
5709         }
5710     };
5711
5712     lock_config($vmid, $updatefn);
5713
5714     foreach_drive($snap, sub {
5715         my ($ds, $drive) = @_;
5716
5717         return if drive_is_cdrom($drive);
5718
5719         my $volid = $drive->{file};
5720         my $device = "drive-$ds";
5721
5722         PVE::Storage::volume_snapshot_rollback($storecfg, $volid, $snapname);
5723     });
5724
5725     $prepare = 0;
5726     lock_config($vmid, $updatefn);
5727 }
5728
5729 my $savevm_wait = sub {
5730     my ($vmid) = @_;
5731
5732     for(;;) {
5733         my $stat = vm_mon_cmd_nocheck($vmid, "query-savevm");
5734         if (!$stat->{status}) {
5735             die "savevm not active\n";
5736         } elsif ($stat->{status} eq 'active') {
5737             sleep(1);
5738             next;
5739         } elsif ($stat->{status} eq 'completed') {
5740             last;
5741         } else {
5742             die "query-savevm returned status '$stat->{status}'\n";
5743         }
5744     }
5745 };
5746
5747 sub do_snapshots_with_qemu {
5748     my ($storecfg, $volid) = @_;
5749
5750     my $storage_name = PVE::Storage::parse_volume_id($volid);
5751
5752     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}} ){
5753         return 1;
5754     }
5755
5756     if ($volid =~ m/\.(qcow2|qed)$/){
5757         return 1;
5758     }
5759
5760     return undef;
5761 }
5762
5763 sub snapshot_create {
5764     my ($vmid, $snapname, $save_vmstate, $comment) = @_;
5765
5766     my $snap = &$snapshot_prepare($vmid, $snapname, $save_vmstate, $comment);
5767
5768     $save_vmstate = 0 if !$snap->{vmstate}; # vm is not running
5769
5770     my $config = load_config($vmid);
5771
5772     my $running = check_running($vmid);
5773
5774     my $freezefs = $running && $config->{agent};
5775     $freezefs = 0 if $snap->{vmstate}; # not needed if we save RAM
5776
5777     my $drivehash = {};
5778
5779     if ($freezefs) {
5780         eval { vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
5781         warn "guest-fsfreeze-freeze problems - $@" if $@;
5782     }
5783
5784     eval {
5785         # create internal snapshots of all drives
5786
5787         my $storecfg = PVE::Storage::config();
5788
5789         if ($running) {
5790             if ($snap->{vmstate}) {
5791                 my $path = PVE::Storage::path($storecfg, $snap->{vmstate});
5792                 vm_mon_cmd($vmid, "savevm-start", statefile => $path);
5793                 &$savevm_wait($vmid);
5794             } else {
5795                 vm_mon_cmd($vmid, "savevm-start");
5796             }
5797         };
5798
5799         foreach_drive($snap, sub {
5800             my ($ds, $drive) = @_;
5801
5802             return if drive_is_cdrom($drive);
5803
5804             my $volid = $drive->{file};
5805             my $device = "drive-$ds";
5806
5807             qemu_volume_snapshot($vmid, $device, $storecfg, $volid, $snapname);
5808             $drivehash->{$ds} = 1;
5809        });
5810     };
5811     my $err = $@;
5812
5813     if ($running) {
5814         eval { vm_mon_cmd($vmid, "savevm-end")  };
5815         warn $@ if $@;
5816
5817         if ($freezefs) {
5818             eval { vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
5819             warn "guest-fsfreeze-thaw problems - $@" if $@;
5820         }
5821
5822         # savevm-end is async, we need to wait
5823         for (;;) {
5824             my $stat = vm_mon_cmd_nocheck($vmid, "query-savevm");
5825             if (!$stat->{bytes}) {
5826                 last;
5827             } else {
5828                 print "savevm not yet finished\n";
5829                 sleep(1);
5830                 next;
5831             }
5832         }
5833     }
5834
5835     if ($err) {
5836         warn "snapshot create failed: starting cleanup\n";
5837         eval { snapshot_delete($vmid, $snapname, 0, $drivehash); };
5838         warn $@ if $@;
5839         die $err;
5840     }
5841
5842     &$snapshot_commit($vmid, $snapname);
5843 }
5844
5845 # Note: $drivehash is only set when called from snapshot_create.
5846 sub snapshot_delete {
5847     my ($vmid, $snapname, $force, $drivehash) = @_;
5848
5849     my $prepare = 1;
5850
5851     my $snap;
5852     my $unused = [];
5853
5854     my $unlink_parent = sub {
5855         my ($confref, $new_parent) = @_;
5856
5857         if ($confref->{parent} && $confref->{parent} eq $snapname) {
5858             if ($new_parent) {
5859                 $confref->{parent} = $new_parent;
5860             } else {
5861                 delete $confref->{parent};
5862             }
5863         }
5864     };
5865
5866     my $updatefn =  sub {
5867         my ($remove_drive) = @_;
5868
5869         my $conf = load_config($vmid);
5870
5871         if (!$drivehash) {
5872             check_lock($conf);
5873             die "you can't delete a snapshot if vm is a template\n"
5874                 if is_template($conf);
5875         }
5876
5877         $snap = $conf->{snapshots}->{$snapname};
5878
5879         die "snapshot '$snapname' does not exist\n" if !defined($snap);
5880
5881         # remove parent refs
5882         if (!$prepare) {
5883             &$unlink_parent($conf, $snap->{parent});
5884             foreach my $sn (keys %{$conf->{snapshots}}) {
5885                 next if $sn eq $snapname;
5886                 &$unlink_parent($conf->{snapshots}->{$sn}, $snap->{parent});
5887             }
5888         }
5889
5890         if ($remove_drive) {
5891             if ($remove_drive eq 'vmstate') {
5892                 delete $snap->{$remove_drive};
5893             } else {
5894                 my $drive = parse_drive($remove_drive, $snap->{$remove_drive});
5895                 my $volid = $drive->{file};
5896                 delete $snap->{$remove_drive};
5897                 add_unused_volume($conf, $volid);
5898             }
5899         }
5900
5901         if ($prepare) {
5902             $snap->{snapstate} = 'delete';
5903         } else {
5904             delete $conf->{snapshots}->{$snapname};
5905             delete $conf->{lock} if $drivehash;
5906             foreach my $volid (@$unused) {
5907                 add_unused_volume($conf, $volid);
5908             }
5909         }
5910
5911         update_config_nolock($vmid, $conf, 1);
5912     };
5913
5914     lock_config($vmid, $updatefn);
5915
5916     # now remove vmstate file
5917
5918     my $storecfg = PVE::Storage::config();
5919
5920     if ($snap->{vmstate}) {
5921         eval {  PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
5922         if (my $err = $@) {
5923             die $err if !$force;
5924             warn $err;
5925         }
5926         # save changes (remove vmstate from snapshot)
5927         lock_config($vmid, $updatefn, 'vmstate') if !$force;
5928     };
5929
5930     # now remove all internal snapshots
5931     foreach_drive($snap, sub {
5932         my ($ds, $drive) = @_;
5933
5934         return if drive_is_cdrom($drive);
5935
5936         my $volid = $drive->{file};
5937         my $device = "drive-$ds";
5938
5939         if (!$drivehash || $drivehash->{$ds}) {
5940             eval { qemu_volume_snapshot_delete($vmid, $device, $storecfg, $volid, $snapname); };
5941             if (my $err = $@) {
5942                 die $err if !$force;
5943                 warn $err;
5944             }
5945         }
5946
5947         # save changes (remove drive fron snapshot)
5948         lock_config($vmid, $updatefn, $ds) if !$force;
5949         push @$unused, $volid;
5950     });
5951
5952     # now cleanup config
5953     $prepare = 0;
5954     lock_config($vmid, $updatefn);
5955 }
5956
5957 sub has_feature {
5958     my ($feature, $conf, $storecfg, $snapname, $running) = @_;
5959
5960     my $err;
5961     foreach_drive($conf, sub {
5962         my ($ds, $drive) = @_;
5963
5964         return if drive_is_cdrom($drive);
5965         my $volid = $drive->{file};
5966         $err = 1 if !PVE::Storage::volume_has_feature($storecfg, $feature, $volid, $snapname, $running);
5967     });
5968
5969     return $err ? 0 : 1;
5970 }
5971
5972 sub template_create {
5973     my ($vmid, $conf, $disk) = @_;
5974
5975     my $storecfg = PVE::Storage::config();
5976
5977     foreach_drive($conf, sub {
5978         my ($ds, $drive) = @_;
5979
5980         return if drive_is_cdrom($drive);
5981         return if $disk && $ds ne $disk;
5982
5983         my $volid = $drive->{file};
5984         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
5985
5986         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
5987         $drive->{file} = $voliddst;
5988         $conf->{$ds} = print_drive($vmid, $drive);
5989         update_config_nolock($vmid, $conf, 1);
5990     });
5991 }
5992
5993 sub is_template {
5994     my ($conf) = @_;
5995
5996     return 1 if defined $conf->{template} && $conf->{template} == 1;
5997 }
5998
5999 sub qemu_img_convert {
6000     my ($src_volid, $dst_volid, $size, $snapname) = @_;
6001
6002     my $storecfg = PVE::Storage::config();
6003     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6004     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6005
6006     if ($src_storeid && $dst_storeid) {
6007         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6008         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6009
6010         my $src_format = qemu_img_format($src_scfg, $src_volname);
6011         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6012
6013         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6014         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6015
6016         my $cmd = [];
6017         push @$cmd, '/usr/bin/qemu-img', 'convert', '-t', 'writeback', '-p', '-n';
6018         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
6019         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path, $dst_path;
6020
6021         my $parser = sub {
6022             my $line = shift;
6023             if($line =~ m/\((\S+)\/100\%\)/){
6024                 my $percent = $1;
6025                 my $transferred = int($size * $percent / 100);
6026                 my $remaining = $size - $transferred;
6027
6028                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6029             }
6030
6031         };
6032
6033         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6034         my $err = $@;
6035         die "copy failed: $err" if $err;
6036     }
6037 }
6038
6039 sub qemu_img_format {
6040     my ($scfg, $volname) = @_;
6041
6042     if ($scfg->{path} && $volname =~ m/\.(raw|cow|qcow|qcow2|qed|vmdk|cloop)$/) {
6043         return $1;
6044     } else {
6045         return "raw";
6046     }
6047 }
6048
6049 sub qemu_drive_mirror {
6050     my ($vmid, $drive, $dst_volid, $vmiddst) = @_;
6051
6052     my $storecfg = PVE::Storage::config();
6053     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6054
6055     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6056
6057     my $format = qemu_img_format($dst_scfg, $dst_volname);
6058
6059     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6060
6061     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $dst_path };
6062     $opts->{format} = $format if $format;
6063
6064     print "drive mirror is starting (scanning bitmap) : this step can take some minutes/hours, depend of disk size and storage speed\n";
6065
6066     eval {
6067     vm_mon_cmd($vmid, "drive-mirror", %$opts);
6068         while (1) {
6069             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6070             my $stat = @$stats[0];
6071             die "mirroring job seem to have die. Maybe do you have bad sectors?" if !$stat;
6072             die "error job is not mirroring" if $stat->{type} ne "mirror";
6073
6074             my $busy = $stat->{busy};
6075             my $ready = $stat->{ready};
6076
6077             if (my $total = $stat->{len}) {
6078                 my $transferred = $stat->{offset} || 0;
6079                 my $remaining = $total - $transferred;
6080                 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6081
6082                 print "transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6083             }
6084
6085
6086             if ($stat->{ready} eq 'true') {
6087
6088                 last if $vmiddst != $vmid;
6089
6090                 # try to switch the disk if source and destination are on the same guest
6091                 eval { vm_mon_cmd($vmid, "block-job-complete", device => "drive-$drive") };
6092                 last if !$@;
6093                 die $@ if $@ !~ m/cannot be completed/;
6094             }
6095             sleep 1;
6096         }
6097
6098
6099     };
6100     my $err = $@;
6101
6102     my $cancel_job = sub {
6103         vm_mon_cmd($vmid, "block-job-cancel", device => "drive-$drive");
6104         while (1) {
6105             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6106             my $stat = @$stats[0];
6107             last if !$stat;
6108             sleep 1;
6109         }
6110     };
6111
6112     if ($err) {
6113         eval { &$cancel_job(); };
6114         die "mirroring error: $err";
6115     }
6116
6117     if ($vmiddst != $vmid) {
6118         # if we clone a disk for a new target vm, we don't switch the disk
6119         &$cancel_job(); # so we call block-job-cancel
6120     }
6121 }
6122
6123 sub clone_disk {
6124     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6125         $newvmid, $storage, $format, $full, $newvollist) = @_;
6126
6127     my $newvolid;
6128
6129     if (!$full) {
6130         print "create linked clone of drive $drivename ($drive->{file})\n";
6131         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6132         push @$newvollist, $newvolid;
6133     } else {
6134         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6135         $storeid = $storage if $storage;
6136
6137         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6138         if (!$format) {
6139             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6140             $format = qemu_img_format($scfg, $volname);
6141         }
6142
6143         # test if requested format is supported - else use default
6144         my $supported = grep { $_ eq $format } @$validFormats;
6145         $format = $defFormat if !$supported;
6146
6147         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6148
6149         print "create full clone of drive $drivename ($drive->{file})\n";
6150         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $format, undef, ($size/1024));
6151         push @$newvollist, $newvolid;
6152
6153         if (!$running || $snapname) {
6154             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname);
6155         } else {
6156             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid);
6157         }
6158     }
6159
6160     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6161
6162     my $disk = $drive;
6163     $disk->{format} = undef;
6164     $disk->{file} = $newvolid;
6165     $disk->{size} = $size;
6166
6167     return $disk;
6168 }
6169
6170 # this only works if VM is running
6171 sub get_current_qemu_machine {
6172     my ($vmid) = @_;
6173
6174     my $cmd = { execute => 'query-machines', arguments => {} };
6175     my $res = vm_qmp_command($vmid, $cmd);
6176
6177     my ($current, $default);
6178     foreach my $e (@$res) {
6179         $default = $e->{name} if $e->{'is-default'};
6180         $current = $e->{name} if $e->{'is-current'};
6181     }
6182
6183     # fallback to the default machine if current is not supported by qemu
6184     return $current || $default || 'pc';
6185 }
6186
6187 sub qemu_machine_feature_enabled {
6188     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6189
6190     my $current_major;
6191     my $current_minor;
6192
6193     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6194
6195         $current_major = $3;
6196         $current_minor = $4;
6197
6198     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6199
6200         $current_major = $1;
6201         $current_minor = $2;
6202     }
6203
6204     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
6205
6206
6207 }
6208
6209 sub lspci {
6210
6211     my $devices = {};
6212
6213     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6214             my (undef, $id, $function) = @_;
6215             my $res = { id => $id, function => $function};
6216             push @{$devices->{$id}}, $res;
6217     });
6218
6219     return $devices;
6220 }
6221
6222 sub vm_iothreads_list {
6223     my ($vmid) = @_;
6224
6225     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6226
6227     my $iothreads = {};
6228     foreach my $iothread (@$res) {
6229         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6230     }
6231
6232     return $iothreads;
6233 }
6234
6235 sub scsihw_infos {
6236     my ($conf, $drive) = @_;
6237
6238     my $maxdev = 0;
6239
6240     if ($conf->{scsihw} && ($conf->{scsihw} =~ m/^lsi/)) {
6241         $maxdev = 7;
6242     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6243         $maxdev = 1;
6244     } else {
6245         $maxdev = 256;
6246     }
6247
6248     my $controller = int($drive->{index} / $maxdev);
6249     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6250
6251     return ($maxdev, $controller, $controller_prefix);
6252 }
6253
6254 1;