PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QMPClient;
  31 use PVE::RPCEnvironment;
  32 use Time::HiRes qw(gettimeofday);
  33
  34 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  35
  36 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  37
  38 # Note about locking: we use flock on the config file protect
  39 # against concurent actions.
  40 # Aditionaly, we have a 'lock' setting in the config file. This
  41 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  42 # allowed when such lock is set. But you can ignore this kind of
  43 # lock with the --skiplock flag.
  44
  45 cfs_register_file('/qemu-server/',
  46                   \&parse_vm_config,
  47                   \&write_vm_config);
  48
  49 PVE::JSONSchema::register_standard_option('skiplock', {
  50     description => "Ignore locks - only root is allowed to use this option.",
  51     type => 'boolean',
  52     optional => 1,
  53 });
  54
  55 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  56     description => "Some command save/restore state from this location.",
  57     type => 'string',
  58     maxLength => 128,
  59     optional => 1,
  60 });
  61
  62 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  63     description => "The name of the snapshot.",
  64     type => 'string', format => 'pve-configid',
  65     maxLength => 40,
  66 });
  67
  68 #no warnings 'redefine';
  69
  70 sub cgroups_write {
  71    my ($controller, $vmid, $option, $value) = @_;
  72
  73    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  74    PVE::ProcFSTools::write_proc_entry($path, $value);
  75
  76 }
  77
  78 my $nodename = PVE::INotify::nodename();
  79
  80 mkdir "/etc/pve/nodes/$nodename";
  81 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  82 mkdir $confdir;
  83
  84 my $var_run_tmpdir = "/var/run/qemu-server";
  85 mkdir $var_run_tmpdir;
  86
  87 my $lock_dir = "/var/lock/qemu-server";
  88 mkdir $lock_dir;
  89
  90 my $pcisysfs = "/sys/bus/pci";
  91
  92 my $confdesc = {
  93     onboot => {
  94         optional => 1,
  95         type => 'boolean',
  96         description => "Specifies whether a VM will be started during system bootup.",
  97         default => 0,
  98     },
  99     autostart => {
 100         optional => 1,
 101         type => 'boolean',
 102         description => "Automatic restart after crash (currently ignored).",
 103         default => 0,
 104     },
 105     hotplug => {
 106         optional => 1,
 107         type => 'string', format => 'pve-hotplug-features',
 108         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 109         default => 'network,disk,usb',
 110     },
 111     reboot => {
 112         optional => 1,
 113         type => 'boolean',
 114         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 115         default => 1,
 116     },
 117     lock => {
 118         optional => 1,
 119         type => 'string',
 120         description => "Lock/unlock the VM.",
 121         enum => [qw(migrate backup snapshot rollback)],
 122     },
 123     cpulimit => {
 124         optional => 1,
 125         type => 'number',
 126         description => "Limit of CPU usage. Note if the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 127         minimum => 0,
 128         maximum => 128,
 129         default => 0,
 130     },
 131     cpuunits => {
 132         optional => 1,
 133         type => 'integer',
 134         description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 135         minimum => 0,
 136         maximum => 500000,
 137         default => 1000,
 138     },
 139     memory => {
 140         optional => 1,
 141         type => 'integer',
 142         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 143         minimum => 16,
 144         default => 512,
 145     },
 146     balloon => {
 147         optional => 1,
 148         type => 'integer',
 149         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 150         minimum => 0,
 151     },
 152     shares => {
 153         optional => 1,
 154         type => 'integer',
 155         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 156         minimum => 0,
 157         maximum => 50000,
 158         default => 1000,
 159     },
 160     keyboard => {
 161         optional => 1,
 162         type => 'string',
 163         description => "Keybord layout for vnc server. Default is read from the datacenter configuration file.",
 164         enum => PVE::Tools::kvmkeymaplist(),
 165         default => 'en-us',
 166     },
 167     name => {
 168         optional => 1,
 169         type => 'string', format => 'dns-name',
 170         description => "Set a name for the VM. Only used on the configuration web interface.",
 171     },
 172     scsihw => {
 173         optional => 1,
 174         type => 'string',
 175         description => "scsi controller model",
 176         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 177         default => 'lsi',
 178     },
 179     description => {
 180         optional => 1,
 181         type => 'string',
 182         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 183     },
 184     ostype => {
 185         optional => 1,
 186         type => 'string',
 187         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 l24 l26 solaris)],
 188         description => <<EODESC,
 189 Used to enable special optimization/features for specific
 190 operating systems:
 191
 192 other  => unspecified OS
 193 wxp    => Microsoft Windows XP
 194 w2k    => Microsoft Windows 2000
 195 w2k3   => Microsoft Windows 2003
 196 w2k8   => Microsoft Windows 2008
 197 wvista => Microsoft Windows Vista
 198 win7   => Microsoft Windows 7
 199 win8   => Microsoft Windows 8/2012
 200 l24    => Linux 2.4 Kernel
 201 l26    => Linux 2.6/3.X Kernel
 202 solaris => solaris/opensolaris/openindiania kernel
 203
 204 other|l24|l26|solaris                       ... no special behaviour
 205 wxp|w2k|w2k3|w2k8|wvista|win7|win8  ... use --localtime switch
 206 EODESC
 207     },
 208     boot => {
 209         optional => 1,
 210         type => 'string',
 211         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 212         pattern => '[acdn]{1,4}',
 213         default => 'cdn',
 214     },
 215     bootdisk => {
 216         optional => 1,
 217         type => 'string', format => 'pve-qm-bootdisk',
 218         description => "Enable booting from specified disk.",
 219         pattern => '(ide|sata|scsi|virtio)\d+',
 220     },
 221     smp => {
 222         optional => 1,
 223         type => 'integer',
 224         description => "The number of CPUs. Please use option -sockets instead.",
 225         minimum => 1,
 226         default => 1,
 227     },
 228     sockets => {
 229         optional => 1,
 230         type => 'integer',
 231         description => "The number of CPU sockets.",
 232         minimum => 1,
 233         default => 1,
 234     },
 235     cores => {
 236         optional => 1,
 237         type => 'integer',
 238         description => "The number of cores per socket.",
 239         minimum => 1,
 240         default => 1,
 241     },
 242     numa => {
 243         optional => 1,
 244         type => 'boolean',
 245         description => "Enable/disable Numa.",
 246         default => 0,
 247     },
 248     vcpus => {
 249         optional => 1,
 250         type => 'integer',
 251         description => "Number of hotplugged vcpus.",
 252         minimum => 1,
 253         default => 0,
 254     },
 255     acpi => {
 256         optional => 1,
 257         type => 'boolean',
 258         description => "Enable/disable ACPI.",
 259         default => 1,
 260     },
 261     agent => {
 262         optional => 1,
 263         type => 'boolean',
 264         description => "Enable/disable Qemu GuestAgent.",
 265         default => 0,
 266     },
 267     kvm => {
 268         optional => 1,
 269         type => 'boolean',
 270         description => "Enable/disable KVM hardware virtualization.",
 271         default => 1,
 272     },
 273     tdf => {
 274         optional => 1,
 275         type => 'boolean',
 276         description => "Enable/disable time drift fix.",
 277         default => 0,
 278     },
 279     localtime => {
 280         optional => 1,
 281         type => 'boolean',
 282         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 283     },
 284     freeze => {
 285         optional => 1,
 286         type => 'boolean',
 287         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 288     },
 289     vga => {
 290         optional => 1,
 291         type => 'string',
 292         description => "Select VGA type. If you want to use high resolution modes (>= 1280x1024x16) then you should use option 'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and 'cirrur' for other OS types. Option 'qxl' enables the SPICE display sever. You can also run without any graphic card using a serial devive as terminal.",
 293         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 294     },
 295     watchdog => {
 296         optional => 1,
 297         type => 'string', format => 'pve-qm-watchdog',
 298         typetext => '[[model=]i6300esb|ib700] [,[action=]reset|shutdown|poweroff|pause|debug|none]',
 299         description => "Create a virtual hardware watchdog device.  Once enabled (by a guest action), the watchdog must be periodically polled by an agent inside the guest or else the guest will be restarted (or execute the action specified)",
 300     },
 301     startdate => {
 302         optional => 1,
 303         type => 'string',
 304         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 305         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 306         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 307         default => 'now',
 308     },
 309     startup =>  get_standard_option('pve-startup-order'),
 310     template => {
 311         optional => 1,
 312         type => 'boolean',
 313         description => "Enable/disable Template.",
 314         default => 0,
 315     },
 316     args => {
 317         optional => 1,
 318         type => 'string',
 319         description => <<EODESCR,
 320 Note: this option is for experts only. It allows you to pass arbitrary arguments to kvm, for example:
 321
 322 args: -no-reboot -no-hpet
 323 EODESCR
 324     },
 325     tablet => {
 326         optional => 1,
 327         type => 'boolean',
 328         default => 1,
 329         description => "Enable/disable the usb tablet device. This device is usually needed to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with normal VNC clients. If you're running lots of console-only guests on one host, you may consider disabling this to save some context switches. This is turned of by default if you use spice (vga=qxl).",
 330     },
 331     migrate_speed => {
 332         optional => 1,
 333         type => 'integer',
 334         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 335         minimum => 0,
 336         default => 0,
 337     },
 338     migrate_downtime => {
 339         optional => 1,
 340         type => 'number',
 341         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 342         minimum => 0,
 343         default => 0.1,
 344     },
 345     cdrom => {
 346         optional => 1,
 347         type => 'string', format => 'pve-qm-drive',
 348         typetext => 'volume',
 349         description => "This is an alias for option -ide2",
 350     },
 351     cpu => {
 352         optional => 1,
 353         description => "Emulated CPU type.",
 354         type => 'string',
 355         enum => [ qw(486 athlon pentium pentium2 pentium3 coreduo core2duo kvm32 kvm64 qemu32 qemu64 phenom Conroe Penryn Nehalem Westmere SandyBridge IvyBridge Haswell Broadwell Opteron_G1 Opteron_G2 Opteron_G3 Opteron_G4 Opteron_G5 host) ],
 356         default => 'kvm64',
 357     },
 358     parent => get_standard_option('pve-snapshot-name', {
 359         optional => 1,
 360         description => "Parent snapshot name. This is used internally, and should not be modified.",
 361     }),
 362     snaptime => {
 363         optional => 1,
 364         description => "Timestamp for snapshots.",
 365         type => 'integer',
 366         minimum => 0,
 367     },
 368     vmstate => {
 369         optional => 1,
 370         type => 'string', format => 'pve-volume-id',
 371         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 372     },
 373     machine => {
 374         description => "Specific the Qemu machine type.",
 375         type => 'string',
 376         pattern => '(pc|pc(-i440fx)?-\d+\.\d+|q35|pc-q35-\d+\.\d+)',
 377         maxLength => 40,
 378         optional => 1,
 379     },
 380     smbios1 => {
 381         description => "Specify SMBIOS type 1 fields.",
 382         type => 'string', format => 'pve-qm-smbios1',
 383         typetext => "[manufacturer=str][,product=str][,version=str][,serial=str] [,uuid=uuid][,sku=str][,family=str]",
 384         maxLength => 256,
 385         optional => 1,
 386     },
 387 };
 388
 389 # what about other qemu settings ?
 390 #cpu => 'string',
 391 #machine => 'string',
 392 #fda => 'file',
 393 #fdb => 'file',
 394 #mtdblock => 'file',
 395 #sd => 'file',
 396 #pflash => 'file',
 397 #snapshot => 'bool',
 398 #bootp => 'file',
 399 ##tftp => 'dir',
 400 ##smb => 'dir',
 401 #kernel => 'file',
 402 #append => 'string',
 403 #initrd => 'file',
 404 ##soundhw => 'string',
 405
 406 while (my ($k, $v) = each %$confdesc) {
 407     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 408 }
 409
 410 my $MAX_IDE_DISKS = 4;
 411 my $MAX_SCSI_DISKS = 14;
 412 my $MAX_VIRTIO_DISKS = 16;
 413 my $MAX_SATA_DISKS = 6;
 414 my $MAX_USB_DEVICES = 5;
 415 my $MAX_NETS = 32;
 416 my $MAX_UNUSED_DISKS = 8;
 417 my $MAX_HOSTPCI_DEVICES = 4;
 418 my $MAX_SERIAL_PORTS = 4;
 419 my $MAX_PARALLEL_PORTS = 3;
 420 my $MAX_NUMA = 8;
 421 my $MAX_MEM = 4194304;
 422 my $STATICMEM = 1024;
 423
 424 my $numadesc = {
 425     optional => 1,
 426     type => 'string', format => 'pve-qm-numanode',
 427     typetext => "cpus=<id[-id],memory=<mb>[[,hostnodes=<id[-id]>] [,policy=<preferred|bind|interleave>]]",
 428     description => "numa topology",
 429 };
 430 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 431
 432 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 433     $confdesc->{"numa$i"} = $numadesc;
 434 }
 435
 436 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 437                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 438                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 439 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 440
 441 my $netdesc = {
 442     optional => 1,
 443     type => 'string', format => 'pve-qm-net',
 444     typetext => "MODEL=XX:XX:XX:XX:XX:XX [,bridge=<dev>][,queues=<nbqueues>][,rate=<mbps>] [,tag=<vlanid>][,firewall=0|1],link_down=0|1]",
 445     description => <<EODESCR,
 446 Specify network devices.
 447
 448 MODEL is one of: $nic_model_list_txt
 449
 450 XX:XX:XX:XX:XX:XX should be an unique MAC address. This is
 451 automatically generated if not specified.
 452
 453 The bridge parameter can be used to automatically add the interface to a bridge device. The Proxmox VE standard bridge is called 'vmbr0'.
 454
 455 Option 'rate' is used to limit traffic bandwidth from and to this interface. It is specified as floating point number, unit is 'Megabytes per second'.
 456
 457 If you specify no bridge, we create a kvm 'user' (NATed) network device, which provides DHCP and DNS services. The following addresses are used:
 458
 459 10.0.2.2   Gateway
 460 10.0.2.3   DNS Server
 461 10.0.2.4   SMB Server
 462
 463 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 464
 465 EODESCR
 466 };
 467 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 468
 469 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 470     $confdesc->{"net$i"} = $netdesc;
 471 }
 472
 473 my $drivename_hash;
 474
 475 my $idedesc = {
 476     optional => 1,
 477     type => 'string', format => 'pve-qm-drive',
 478     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads] [,discard=ignore|on]',
 479     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
 480 };
 481 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
 482
 483 my $scsidesc = {
 484     optional => 1,
 485     type => 'string', format => 'pve-qm-drive',
 486     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads] [,discard=ignore|on] [,iothread=on] [,queues=<nbqueues>]',
 487     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
 488 };
 489 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 490
 491 my $satadesc = {
 492     optional => 1,
 493     type => 'string', format => 'pve-qm-drive',
 494     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads]  [,discard=ignore|on]',
 495     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
 496 };
 497 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
 498
 499 my $virtiodesc = {
 500     optional => 1,
 501     type => 'string', format => 'pve-qm-drive',
 502     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads]  [,discard=ignore|on] [,iothread=on]',
 503     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
 504 };
 505 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
 506
 507 my $usbdesc = {
 508     optional => 1,
 509     type => 'string', format => 'pve-qm-usb-device',
 510     typetext => 'host=HOSTUSBDEVICE|spice',
 511     description => <<EODESCR,
 512 Configure an USB device (n is 0 to 4). This can be used to
 513 pass-through usb devices to the guest. HOSTUSBDEVICE syntax is:
 514
 515 'bus-port(.port)*' (decimal numbers) or
 516 'vendor_id:product_id' (hexadeciaml numbers)
 517
 518 You can use the 'lsusb -t' command to list existing usb devices.
 519
 520 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 521
 522 The value 'spice' can be used to add a usb redirection devices for spice.
 523
 524 EODESCR
 525 };
 526 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
 527
 528 my $hostpcidesc = {
 529         optional => 1,
 530         type => 'string', format => 'pve-qm-hostpci',
 531         typetext => "[host=]HOSTPCIDEVICE [,driver=kvm|vfio] [,rombar=on|off] [,pcie=0|1] [,x-vga=on|off]",
 532         description => <<EODESCR,
 533 Map host pci devices. HOSTPCIDEVICE syntax is:
 534
 535 'bus:dev.func' (hexadecimal numbers)
 536
 537 You can us the 'lspci' command to list existing pci devices.
 538
 539 The 'rombar' option determines whether or not the device's ROM will be visible in the guest's memory map (default is 'on').
 540
 541 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 542
 543 Experimental: user reported problems with this option.
 544 EODESCR
 545 };
 546 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
 547
 548 my $serialdesc = {
 549         optional => 1,
 550         type => 'string',
 551         pattern => '(/dev/.+|socket)',
 552         description =>  <<EODESCR,
 553 Create a serial device inside the VM (n is 0 to 3), and pass through a host serial device (i.e. /dev/ttyS0), or create a unix socket on the host side (use 'qm terminal' to open a terminal connection).
 554
 555 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 556
 557 Experimental: user reported problems with this option.
 558 EODESCR
 559 };
 560
 561 my $paralleldesc= {
 562         optional => 1,
 563         type => 'string',
 564         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
 565         description =>  <<EODESCR,
 566 Map host parallel devices (n is 0 to 2).
 567
 568 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 569
 570 Experimental: user reported problems with this option.
 571 EODESCR
 572 };
 573
 574 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
 575     $confdesc->{"parallel$i"} = $paralleldesc;
 576 }
 577
 578 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
 579     $confdesc->{"serial$i"} = $serialdesc;
 580 }
 581
 582 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
 583     $confdesc->{"hostpci$i"} = $hostpcidesc;
 584 }
 585
 586 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
 587     $drivename_hash->{"ide$i"} = 1;
 588     $confdesc->{"ide$i"} = $idedesc;
 589 }
 590
 591 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
 592     $drivename_hash->{"sata$i"} = 1;
 593     $confdesc->{"sata$i"} = $satadesc;
 594 }
 595
 596 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
 597     $drivename_hash->{"scsi$i"} = 1;
 598     $confdesc->{"scsi$i"} = $scsidesc ;
 599 }
 600
 601 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
 602     $drivename_hash->{"virtio$i"} = 1;
 603     $confdesc->{"virtio$i"} = $virtiodesc;
 604 }
 605
 606 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
 607     $confdesc->{"usb$i"} = $usbdesc;
 608 }
 609
 610 my $unuseddesc = {
 611     optional => 1,
 612     type => 'string', format => 'pve-volume-id',
 613     description => "Reference to unused volumes.",
 614 };
 615
 616 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
 617     $confdesc->{"unused$i"} = $unuseddesc;
 618 }
 619
 620 my $kvm_api_version = 0;
 621
 622 sub kvm_version {
 623
 624     return $kvm_api_version if $kvm_api_version;
 625
 626     my $fh = IO::File->new("</dev/kvm") ||
 627         return 0;
 628
 629     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
 630         $kvm_api_version = $v;
 631     }
 632
 633     $fh->close();
 634
 635     return  $kvm_api_version;
 636 }
 637
 638 my $kvm_user_version;
 639
 640 sub kvm_user_version {
 641
 642     return $kvm_user_version if $kvm_user_version;
 643
 644     $kvm_user_version = 'unknown';
 645
 646     my $tmp = `kvm -help 2>/dev/null`;
 647
 648     if ($tmp =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)[,\s]/) {
 649         $kvm_user_version = $2;
 650     }
 651
 652     return $kvm_user_version;
 653
 654 }
 655
 656 my $kernel_has_vhost_net = -c '/dev/vhost-net';
 657
 658 sub disknames {
 659     # order is important - used to autoselect boot disk
 660     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
 661             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
 662             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
 663             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))));
 664 }
 665
 666 sub valid_drivename {
 667     my $dev = shift;
 668
 669     return defined($drivename_hash->{$dev});
 670 }
 671
 672 sub option_exists {
 673     my $key = shift;
 674     return defined($confdesc->{$key});
 675 }
 676
 677 sub nic_models {
 678     return $nic_model_list;
 679 }
 680
 681 sub os_list_description {
 682
 683     return {
 684         other => 'Other',
 685         wxp => 'Windows XP',
 686         w2k => 'Windows 2000',
 687         w2k3 =>, 'Windows 2003',
 688         w2k8 => 'Windows 2008',
 689         wvista => 'Windows Vista',
 690         win7 => 'Windows 7',
 691         win8 => 'Windows 8/2012',
 692         l24 => 'Linux 2.4',
 693         l26 => 'Linux 2.6',
 694     };
 695 }
 696
 697 my $cdrom_path;
 698
 699 sub get_cdrom_path {
 700
 701     return  $cdrom_path if $cdrom_path;
 702
 703     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
 704     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
 705     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
 706 }
 707
 708 sub get_iso_path {
 709     my ($storecfg, $vmid, $cdrom) = @_;
 710
 711     if ($cdrom eq 'cdrom') {
 712         return get_cdrom_path();
 713     } elsif ($cdrom eq 'none') {
 714         return '';
 715     } elsif ($cdrom =~ m|^/|) {
 716         return $cdrom;
 717     } else {
 718         return PVE::Storage::path($storecfg, $cdrom);
 719     }
 720 }
 721
 722 # try to convert old style file names to volume IDs
 723 sub filename_to_volume_id {
 724     my ($vmid, $file, $media) = @_;
 725
 726     if (!($file eq 'none' || $file eq 'cdrom' ||
 727           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
 728
 729         return undef if $file =~ m|/|;
 730
 731         if ($media && $media eq 'cdrom') {
 732             $file = "local:iso/$file";
 733         } else {
 734             $file = "local:$vmid/$file";
 735         }
 736     }
 737
 738     return $file;
 739 }
 740
 741 sub verify_media_type {
 742     my ($opt, $vtype, $media) = @_;
 743
 744     return if !$media;
 745
 746     my $etype;
 747     if ($media eq 'disk') {
 748         $etype = 'images';
 749     } elsif ($media eq 'cdrom') {
 750         $etype = 'iso';
 751     } else {
 752         die "internal error";
 753     }
 754
 755     return if ($vtype eq $etype);
 756
 757     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
 758 }
 759
 760 sub cleanup_drive_path {
 761     my ($opt, $storecfg, $drive) = @_;
 762
 763     # try to convert filesystem paths to volume IDs
 764
 765     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
 766         ($drive->{file} !~ m|^/dev/.+|) &&
 767         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
 768         ($drive->{file} !~ m/^\d+$/)) {
 769         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
 770         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
 771         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
 772         verify_media_type($opt, $vtype, $drive->{media});
 773         $drive->{file} = $volid;
 774     }
 775
 776     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
 777 }
 778
 779 sub create_conf_nolock {
 780     my ($vmid, $settings) = @_;
 781
 782     my $filename = config_file($vmid);
 783
 784     die "configuration file '$filename' already exists\n" if -f $filename;
 785
 786     my $defaults = load_defaults();
 787
 788     $settings->{name} = "vm$vmid" if !$settings->{name};
 789     $settings->{memory} = $defaults->{memory} if !$settings->{memory};
 790
 791     my $data = '';
 792     foreach my $opt (keys %$settings) {
 793         next if !$confdesc->{$opt};
 794
 795         my $value = $settings->{$opt};
 796         next if !$value;
 797
 798         $data .= "$opt: $value\n";
 799     }
 800
 801     PVE::Tools::file_set_contents($filename, $data);
 802 }
 803
 804 sub parse_hotplug_features {
 805     my ($data) = @_;
 806
 807     my $res = {};
 808
 809     return $res if $data eq '0';
 810
 811     $data = $confdesc->{hotplug}->{default} if $data eq '1';
 812
 813     foreach my $feature (PVE::Tools::split_list($data)) {
 814         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
 815             $res->{$1} = 1;
 816         } else {
 817             warn "ignoring unknown hotplug feature '$feature'\n";
 818         }
 819     }
 820     return $res;
 821 }
 822
 823 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
 824 sub pve_verify_hotplug_features {
 825     my ($value, $noerr) = @_;
 826
 827     return $value if parse_hotplug_features($value);
 828
 829     return undef if $noerr;
 830
 831     die "unable to parse hotplug option\n";
 832 }
 833
 834 my $parse_size = sub {
 835     my ($value) = @_;
 836
 837     return undef if $value !~ m/^(\d+(\.\d+)?)([KMG])?$/;
 838     my ($size, $unit) = ($1, $3);
 839     if ($unit) {
 840         if ($unit eq 'K') {
 841             $size = $size * 1024;
 842         } elsif ($unit eq 'M') {
 843             $size = $size * 1024 * 1024;
 844         } elsif ($unit eq 'G') {
 845             $size = $size * 1024 * 1024 * 1024;
 846         }
 847     }
 848     return int($size);
 849 };
 850
 851 my $format_size = sub {
 852     my ($size) = @_;
 853
 854     $size = int($size);
 855
 856     my $kb = int($size/1024);
 857     return $size if $kb*1024 != $size;
 858
 859     my $mb = int($kb/1024);
 860     return "${kb}K" if $mb*1024 != $kb;
 861
 862     my $gb = int($mb/1024);
 863     return "${mb}M" if $gb*1024 != $mb;
 864
 865     return "${gb}G";
 866 };
 867
 868 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
 869 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
 870 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
 871 #        [,aio=native|threads][,discard=ignore|on][,iothread=on]
 872
 873 sub parse_drive {
 874     my ($key, $data) = @_;
 875
 876     my $res = {};
 877
 878     # $key may be undefined - used to verify JSON parameters
 879     if (!defined($key)) {
 880         $res->{interface} = 'unknown'; # should not harm when used to verify parameters
 881         $res->{index} = 0;
 882     } elsif ($key =~ m/^([^\d]+)(\d+)$/) {
 883         $res->{interface} = $1;
 884         $res->{index} = $2;
 885     } else {
 886         return undef;
 887     }
 888
 889     foreach my $p (split (/,/, $data)) {
 890         next if $p =~ m/^\s*$/;
 891
 892         if ($p =~ m/^(file|volume|cyls|heads|secs|trans|media|snapshot|cache|format|rerror|werror|backup|aio|bps|mbps|mbps_max|bps_rd|mbps_rd|mbps_rd_max|bps_wr|mbps_wr|mbps_wr_max|iops|iops_max|iops_rd|iops_rd_max|iops_wr|iops_wr_max|size|discard|iothread|queues)=(.+)$/) {
 893             my ($k, $v) = ($1, $2);
 894
 895             $k = 'file' if $k eq 'volume';
 896
 897             return undef if defined $res->{$k};
 898
 899             if ($k eq 'bps' || $k eq 'bps_rd' || $k eq 'bps_wr') {
 900                 return undef if !$v || $v !~ m/^\d+/;
 901                 $k = "m$k";
 902                 $v = sprintf("%.3f", $v / (1024*1024));
 903             }
 904             $res->{$k} = $v;
 905         } else {
 906             if (!$res->{file} && $p !~ m/=/) {
 907                 $res->{file} = $p;
 908             } else {
 909                 return undef;
 910             }
 911         }
 912     }
 913
 914     return undef if !$res->{file};
 915
 916     return undef if $res->{cache} &&
 917         $res->{cache} !~ m/^(off|none|writethrough|writeback|unsafe|directsync)$/;
 918     return undef if $res->{snapshot} && $res->{snapshot} !~ m/^(on|off)$/;
 919     return undef if $res->{cyls} && $res->{cyls} !~ m/^\d+$/;
 920     return undef if $res->{heads} && $res->{heads} !~ m/^\d+$/;
 921     return undef if $res->{secs} && $res->{secs} !~ m/^\d+$/;
 922     return undef if $res->{media} && $res->{media} !~ m/^(disk|cdrom)$/;
 923     return undef if $res->{trans} && $res->{trans} !~ m/^(none|lba|auto)$/;
 924     return undef if $res->{format} && $res->{format} !~ m/^(raw|cow|qcow|qed|qcow2|vmdk|cloop)$/;
 925     return undef if $res->{rerror} && $res->{rerror} !~ m/^(ignore|report|stop)$/;
 926     return undef if $res->{werror} && $res->{werror} !~ m/^(enospc|ignore|report|stop)$/;
 927     return undef if $res->{backup} && $res->{backup} !~ m/^(yes|no)$/;
 928     return undef if $res->{aio} && $res->{aio} !~ m/^(native|threads)$/;
 929     return undef if $res->{discard} && $res->{discard} !~ m/^(ignore|on)$/;
 930     return undef if $res->{iothread} && $res->{iothread} !~ m/^(on)$/;
 931     return undef if $res->{queues} && ($res->{queues} !~ m/^\d+$/ || $res->{queues} < 2);
 932
 933     return undef if $res->{mbps_rd} && $res->{mbps};
 934     return undef if $res->{mbps_wr} && $res->{mbps};
 935
 936     return undef if $res->{mbps} && $res->{mbps} !~ m/^\d+(\.\d+)?$/;
 937     return undef if $res->{mbps_max} && $res->{mbps_max} !~ m/^\d+(\.\d+)?$/;
 938     return undef if $res->{mbps_rd} && $res->{mbps_rd} !~ m/^\d+(\.\d+)?$/;
 939     return undef if $res->{mbps_rd_max} && $res->{mbps_rd_max} !~ m/^\d+(\.\d+)?$/;
 940     return undef if $res->{mbps_wr} && $res->{mbps_wr} !~ m/^\d+(\.\d+)?$/;
 941     return undef if $res->{mbps_wr_max} && $res->{mbps_wr_max} !~ m/^\d+(\.\d+)?$/;
 942
 943     return undef if $res->{iops_rd} && $res->{iops};
 944     return undef if $res->{iops_wr} && $res->{iops};
 945
 946
 947     return undef if $res->{iops} && $res->{iops} !~ m/^\d+$/;
 948     return undef if $res->{iops_max} && $res->{iops_max} !~ m/^\d+$/;
 949     return undef if $res->{iops_rd} && $res->{iops_rd} !~ m/^\d+$/;
 950     return undef if $res->{iops_rd_max} && $res->{iops_rd_max} !~ m/^\d+$/;
 951     return undef if $res->{iops_wr} && $res->{iops_wr} !~ m/^\d+$/;
 952     return undef if $res->{iops_wr_max} && $res->{iops_wr_max} !~ m/^\d+$/;
 953
 954
 955     if ($res->{size}) {
 956         return undef if !defined($res->{size} = &$parse_size($res->{size}));
 957     }
 958
 959     if ($res->{media} && ($res->{media} eq 'cdrom')) {
 960         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
 961         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
 962         return undef if $res->{interface} eq 'virtio';
 963     }
 964
 965     # rerror does not work with scsi drives
 966     if ($res->{rerror}) {
 967         return undef if $res->{interface} eq 'scsi';
 968     }
 969
 970     return $res;
 971 }
 972
 973 my @qemu_drive_options = qw(heads secs cyls trans media format cache snapshot rerror werror aio discard iops iops_rd iops_wr iops_max iops_rd_max iops_wr_max);
 974
 975 sub print_drive {
 976     my ($vmid, $drive) = @_;
 977
 978     my $opts = '';
 979     foreach my $o (@qemu_drive_options, 'mbps', 'mbps_rd', 'mbps_wr', 'mbps_max', 'mbps_rd_max', 'mbps_wr_max', 'backup', 'iothread', 'queues') {
 980         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
 981     }
 982
 983     if ($drive->{size}) {
 984         $opts .= ",size=" . &$format_size($drive->{size});
 985     }
 986
 987     return "$drive->{file}$opts";
 988 }
 989
 990 sub scsi_inquiry {
 991     my($fh, $noerr) = @_;
 992
 993     my $SG_IO = 0x2285;
 994     my $SG_GET_VERSION_NUM = 0x2282;
 995
 996     my $versionbuf = "\x00" x 8;
 997     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
 998     if (!$ret) {
 999         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1000         return undef;
1001     }
1002     my $version = unpack("I", $versionbuf);
1003     if ($version < 30000) {
1004         die "scsi generic interface too old\n"  if !$noerr;
1005         return undef;
1006     }
1007
1008     my $buf = "\x00" x 36;
1009     my $sensebuf = "\x00" x 8;
1010     my $cmd = pack("C x3 C x1", 0x12, 36);
1011
1012     # see /usr/include/scsi/sg.h
1013     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1014
1015     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1016                       length($sensebuf), 0, length($buf), $buf,
1017                       $cmd, $sensebuf, 6000);
1018
1019     $ret = ioctl($fh, $SG_IO, $packet);
1020     if (!$ret) {
1021         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1022         return undef;
1023     }
1024
1025     my @res = unpack($sg_io_hdr_t, $packet);
1026     if ($res[17] || $res[18]) {
1027         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1028         return undef;
1029     }
1030
1031     my $res = {};
1032     (my $byte0, my $byte1, $res->{vendor},
1033      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1034
1035     $res->{removable} = $byte1 & 128 ? 1 : 0;
1036     $res->{type} = $byte0 & 31;
1037
1038     return $res;
1039 }
1040
1041 sub path_is_scsi {
1042     my ($path) = @_;
1043
1044     my $fh = IO::File->new("+<$path") || return undef;
1045     my $res = scsi_inquiry($fh, 1);
1046     close($fh);
1047
1048     return $res;
1049 }
1050
1051 sub machine_type_is_q35 {
1052     my ($conf) = @_;
1053
1054     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1055 }
1056
1057 sub print_tabletdevice_full {
1058     my ($conf) = @_;
1059
1060     my $q35 = machine_type_is_q35($conf);
1061
1062     # we use uhci for old VMs because tablet driver was buggy in older qemu
1063     my $usbbus = $q35 ? "ehci" : "uhci";
1064
1065     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1066 }
1067
1068 sub print_drivedevice_full {
1069     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1070
1071     my $device = '';
1072     my $maxdev = 0;
1073
1074     if ($drive->{interface} eq 'virtio') {
1075         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1076         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1077         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1078     } elsif ($drive->{interface} eq 'scsi') {
1079
1080         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1081         my $unit = $drive->{index} % $maxdev;
1082         my $devicetype = 'hd';
1083         my $path = '';
1084         if (drive_is_cdrom($drive)) {
1085               $devicetype = 'cd';
1086         } else {
1087               if ($drive->{file} =~ m|^/|) {
1088                   $path = $drive->{file};
1089               } else {
1090                   $path = PVE::Storage::path($storecfg, $drive->{file});
1091               }
1092
1093               if($path =~ m/^iscsi\:\/\//){
1094                   $devicetype = 'generic';
1095               } else {
1096                   if (my $info = path_is_scsi($path)) {
1097                       if ($info->{type} == 0) {
1098                           $devicetype = 'block';
1099                       } elsif ($info->{type} == 1) { # tape
1100                           $devicetype = 'generic';
1101                       }
1102                   }
1103               }
1104          }
1105
1106         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1107             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1108         } else {
1109             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1110         }
1111
1112     } elsif ($drive->{interface} eq 'ide'){
1113         $maxdev = 2;
1114         my $controller = int($drive->{index} / $maxdev);
1115         my $unit = $drive->{index} % $maxdev;
1116         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1117
1118         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1119     } elsif ($drive->{interface} eq 'sata'){
1120         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1121         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1122         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1123     } elsif ($drive->{interface} eq 'usb') {
1124         die "implement me";
1125         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1126     } else {
1127         die "unsupported interface type";
1128     }
1129
1130     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1131
1132     return $device;
1133 }
1134
1135 sub get_initiator_name {
1136     my $initiator;
1137
1138     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1139     while (defined(my $line = <$fh>)) {
1140         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1141         $initiator = $1;
1142         last;
1143     }
1144     $fh->close();
1145
1146     return $initiator;
1147 }
1148
1149 sub print_drive_full {
1150     my ($storecfg, $vmid, $drive) = @_;
1151
1152     my $path;
1153     my $volid = $drive->{file};
1154     my $format;
1155
1156     if (drive_is_cdrom($drive)) {
1157         $path = get_iso_path($storecfg, $vmid, $volid);
1158     } else {
1159         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1160         if ($storeid) {
1161             $path = PVE::Storage::path($storecfg, $volid);
1162             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1163             $format = qemu_img_format($scfg, $volname);
1164         } else {
1165             $path = $volid;
1166         }
1167    }
1168
1169     my $opts = '';
1170     foreach my $o (@qemu_drive_options) {
1171         next if $o eq 'bootindex';
1172         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1173     }
1174
1175     $opts .= ",format=$format" if $format && !$drive->{format};
1176
1177     foreach my $o (qw(bps bps_rd bps_wr)) {
1178         my $v = $drive->{"m$o"};
1179         $opts .= ",$o=" . int($v*1024*1024) if $v;
1180     }
1181
1182     my $cache_direct = 0;
1183
1184     if (my $cache = $drive->{cache}) {
1185         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1186     } elsif (!drive_is_cdrom($drive)) {
1187         $opts .= ",cache=none";
1188         $cache_direct = 1;
1189     }
1190
1191     # aio native works only with O_DIRECT
1192     if (!$drive->{aio}) {
1193         if($cache_direct) {
1194             $opts .= ",aio=native";
1195         } else {
1196             $opts .= ",aio=threads";
1197         }
1198     }
1199
1200     my $detectzeroes = $drive->{discard} ? "unmap" : "on";
1201     $opts .= ",detect-zeroes=$detectzeroes" if !drive_is_cdrom($drive);
1202
1203     my $pathinfo = $path ? "file=$path," : '';
1204
1205     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1206 }
1207
1208 sub print_netdevice_full {
1209     my ($vmid, $conf, $net, $netid, $bridges) = @_;
1210
1211     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1212
1213     my $device = $net->{model};
1214     if ($net->{model} eq 'virtio') {
1215          $device = 'virtio-net-pci';
1216      };
1217
1218     my $pciaddr = print_pci_addr("$netid", $bridges);
1219     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1220     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1221         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1222         my $vectors = $net->{queues} * 2 + 2;
1223         $tmpstr .= ",vectors=$vectors,mq=on";
1224     }
1225     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1226     return $tmpstr;
1227 }
1228
1229 sub print_netdev_full {
1230     my ($vmid, $conf, $net, $netid) = @_;
1231
1232     my $i = '';
1233     if ($netid =~ m/^net(\d+)$/) {
1234         $i = int($1);
1235     }
1236
1237     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1238
1239     my $ifname = "tap${vmid}i$i";
1240
1241     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1242     die "interface name '$ifname' is too long (max 15 character)\n"
1243         if length($ifname) >= 16;
1244
1245     my $vhostparam = '';
1246     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1247
1248     my $vmname = $conf->{name} || "vm$vmid";
1249
1250     my $netdev = "";
1251
1252     if ($net->{bridge}) {
1253         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/pve-bridge,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1254     } else {
1255         $netdev = "type=user,id=$netid,hostname=$vmname";
1256     }
1257
1258     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1259
1260     return $netdev;
1261 }
1262
1263 sub drive_is_cdrom {
1264     my ($drive) = @_;
1265
1266     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1267
1268 }
1269
1270 sub parse_numa {
1271     my ($data) = @_;
1272
1273     my $res = {};
1274
1275     foreach my $kvp (split(/,/, $data)) {
1276
1277         if ($kvp =~ m/^memory=(\S+)$/) {
1278             $res->{memory} = $1;
1279         } elsif ($kvp =~ m/^policy=(preferred|bind|interleave)$/) {
1280             $res->{policy} = $1;
1281         } elsif ($kvp =~ m/^cpus=(\d+)(-(\d+))?$/) {
1282             $res->{cpus}->{start} = $1;
1283             $res->{cpus}->{end} = $3;
1284         } elsif ($kvp =~ m/^hostnodes=(\d+)(-(\d+))?$/) {
1285             $res->{hostnodes}->{start} = $1;
1286             $res->{hostnodes}->{end} = $3;
1287         } else {
1288             return undef;
1289         }
1290     }
1291
1292     return $res;
1293 }
1294
1295 sub parse_hostpci {
1296     my ($value) = @_;
1297
1298     return undef if !$value;
1299
1300
1301     my @list = split(/,/, $value);
1302     my $found;
1303
1304     my $res = {};
1305     foreach my $kv (@list) {
1306
1307         if ($kv =~ m/^(host=)?([a-f0-9]{2}:[a-f0-9]{2})(\.([a-f0-9]))?$/) {
1308             $found = 1;
1309             if(defined($4)){
1310                 push @{$res->{pciid}}, { id => $2 , function => $4};
1311
1312             }else{
1313                 my $pcidevices = lspci($2);
1314                 $res->{pciid} = $pcidevices->{$2};
1315             }
1316         } elsif ($kv =~ m/^driver=(kvm|vfio)$/) {
1317             $res->{driver} = $1;
1318         } elsif ($kv =~ m/^rombar=(on|off)$/) {
1319             $res->{rombar} = $1;
1320         } elsif ($kv =~ m/^x-vga=(on|off)$/) {
1321             $res->{'x-vga'} = $1;
1322         } elsif ($kv =~ m/^pcie=(\d+)$/) {
1323             $res->{pcie} = 1 if $1 == 1;
1324         } else {
1325             warn "unknown hostpci setting '$kv'\n";
1326         }
1327     }
1328
1329     return undef if !$found;
1330
1331     return $res;
1332 }
1333
1334 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1335 sub parse_net {
1336     my ($data) = @_;
1337
1338     my $res = {};
1339
1340     foreach my $kvp (split(/,/, $data)) {
1341
1342         if ($kvp =~ m/^(ne2k_pci|e1000|e1000-82540em|e1000-82544gc|e1000-82545em|rtl8139|pcnet|virtio|ne2k_isa|i82551|i82557b|i82559er|vmxnet3)(=([0-9a-f]{2}(:[0-9a-f]{2}){5}))?$/i) {
1343             my $model = lc($1);
1344             my $mac = defined($3) ? uc($3) : PVE::Tools::random_ether_addr();
1345             $res->{model} = $model;
1346             $res->{macaddr} = $mac;
1347         } elsif ($kvp =~ m/^bridge=(\S+)$/) {
1348             $res->{bridge} = $1;
1349         } elsif ($kvp =~ m/^queues=(\d+)$/) {
1350             $res->{queues} = $1;
1351         } elsif ($kvp =~ m/^rate=(\d+(\.\d+)?)$/) {
1352             $res->{rate} = $1;
1353         } elsif ($kvp =~ m/^tag=(\d+)$/) {
1354             $res->{tag} = $1;
1355         } elsif ($kvp =~ m/^firewall=([01])$/) {
1356             $res->{firewall} = $1;
1357         } elsif ($kvp =~ m/^link_down=([01])$/) {
1358             $res->{link_down} = $1;
1359         } else {
1360             return undef;
1361         }
1362
1363     }
1364
1365     return undef if !$res->{model};
1366
1367     return $res;
1368 }
1369
1370 sub print_net {
1371     my $net = shift;
1372
1373     my $res = "$net->{model}";
1374     $res .= "=$net->{macaddr}" if $net->{macaddr};
1375     $res .= ",bridge=$net->{bridge}" if $net->{bridge};
1376     $res .= ",rate=$net->{rate}" if $net->{rate};
1377     $res .= ",tag=$net->{tag}" if $net->{tag};
1378     $res .= ",firewall=1" if $net->{firewall};
1379     $res .= ",link_down=1" if $net->{link_down};
1380     $res .= ",queues=$net->{queues}" if $net->{queues};
1381
1382     return $res;
1383 }
1384
1385 sub add_random_macs {
1386     my ($settings) = @_;
1387
1388     foreach my $opt (keys %$settings) {
1389         next if $opt !~ m/^net(\d+)$/;
1390         my $net = parse_net($settings->{$opt});
1391         next if !$net;
1392         $settings->{$opt} = print_net($net);
1393     }
1394 }
1395
1396 sub add_unused_volume {
1397     my ($config, $volid) = @_;
1398
1399     my $key;
1400     for (my $ind = $MAX_UNUSED_DISKS - 1; $ind >= 0; $ind--) {
1401         my $test = "unused$ind";
1402         if (my $vid = $config->{$test}) {
1403             return if $vid eq $volid; # do not add duplicates
1404         } else {
1405             $key = $test;
1406         }
1407     }
1408
1409     die "To many unused volume - please delete them first.\n" if !$key;
1410
1411     $config->{$key} = $volid;
1412
1413     return $key;
1414 }
1415
1416 sub vm_is_volid_owner {
1417     my ($storecfg, $vmid, $volid) = @_;
1418
1419     if ($volid !~  m|^/|) {
1420         my ($path, $owner);
1421         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1422         if ($owner && ($owner == $vmid)) {
1423             return 1;
1424         }
1425     }
1426
1427     return undef;
1428 }
1429
1430 sub vmconfig_delete_pending_option {
1431     my ($conf, $key) = @_;
1432
1433     delete $conf->{pending}->{$key};
1434     my $pending_delete_hash = { $key => 1 };
1435     foreach my $opt (PVE::Tools::split_list($conf->{pending}->{delete})) {
1436         $pending_delete_hash->{$opt} = 1;
1437     }
1438     $conf->{pending}->{delete} = join(',', keys %$pending_delete_hash);
1439 }
1440
1441 sub vmconfig_undelete_pending_option {
1442     my ($conf, $key) = @_;
1443
1444     my $pending_delete_hash = {};
1445     foreach my $opt (PVE::Tools::split_list($conf->{pending}->{delete})) {
1446         $pending_delete_hash->{$opt} = 1;
1447     }
1448     delete $pending_delete_hash->{$key};
1449
1450     my @keylist = keys %$pending_delete_hash;
1451     if (scalar(@keylist)) {
1452         $conf->{pending}->{delete} = join(',', @keylist);
1453     } else {
1454         delete $conf->{pending}->{delete};
1455     }
1456 }
1457
1458 sub vmconfig_register_unused_drive {
1459     my ($storecfg, $vmid, $conf, $drive) = @_;
1460
1461     if (!drive_is_cdrom($drive)) {
1462         my $volid = $drive->{file};
1463         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1464             add_unused_volume($conf, $volid, $vmid);
1465         }
1466     }
1467 }
1468
1469 sub vmconfig_cleanup_pending {
1470     my ($conf) = @_;
1471
1472     # remove pending changes when nothing changed
1473     my $changes;
1474     foreach my $opt (keys %{$conf->{pending}}) {
1475         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
1476             $changes = 1;
1477             delete $conf->{pending}->{$opt};
1478         }
1479     }
1480
1481     # remove delete if option is not set
1482     my $pending_delete_hash = {};
1483     foreach my $opt (PVE::Tools::split_list($conf->{pending}->{delete})) {
1484         if (defined($conf->{$opt})) {
1485             $pending_delete_hash->{$opt} = 1;
1486         } else {
1487             $changes = 1;
1488         }
1489     }
1490
1491     my @keylist = keys %$pending_delete_hash;
1492     if (scalar(@keylist)) {
1493         $conf->{pending}->{delete} = join(',', @keylist);
1494     } else {
1495         delete $conf->{pending}->{delete};
1496     }
1497
1498     return $changes;
1499 }
1500
1501 my $valid_smbios1_options = {
1502     manufacturer => '\S+',
1503     product => '\S+',
1504     version => '\S+',
1505     serial => '\S+',
1506     uuid => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1507     sku => '\S+',
1508     family => '\S+',
1509 };
1510
1511 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
1512 sub parse_smbios1 {
1513     my ($data) = @_;
1514
1515     my $res = {};
1516
1517     foreach my $kvp (split(/,/, $data)) {
1518         return undef if $kvp !~ m/^(\S+)=(.+)$/;
1519         my ($k, $v) = split(/=/, $kvp);
1520         return undef if !defined($k) || !defined($v);
1521         return undef if !$valid_smbios1_options->{$k};
1522         return undef if $v !~ m/^$valid_smbios1_options->{$k}$/;
1523         $res->{$k} = $v;
1524     }
1525
1526     return $res;
1527 }
1528
1529 sub print_smbios1 {
1530     my ($smbios1) = @_;
1531
1532     my $data = '';
1533     foreach my $k (keys %$smbios1) {
1534         next if !defined($smbios1->{$k});
1535         next if !$valid_smbios1_options->{$k};
1536         $data .= ',' if $data;
1537         $data .= "$k=$smbios1->{$k}";
1538     }
1539     return $data;
1540 }
1541
1542 PVE::JSONSchema::register_format('pve-qm-smbios1', \&verify_smbios1);
1543 sub verify_smbios1 {
1544     my ($value, $noerr) = @_;
1545
1546     return $value if parse_smbios1($value);
1547
1548     return undef if $noerr;
1549
1550     die "unable to parse smbios (type 1) options\n";
1551 }
1552
1553 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
1554 sub verify_bootdisk {
1555     my ($value, $noerr) = @_;
1556
1557     return $value if valid_drivename($value);
1558
1559     return undef if $noerr;
1560
1561     die "invalid boot disk '$value'\n";
1562 }
1563
1564 PVE::JSONSchema::register_format('pve-qm-numanode', \&verify_numa);
1565 sub verify_numa {
1566     my ($value, $noerr) = @_;
1567
1568     return $value if parse_numa($value);
1569
1570     return undef if $noerr;
1571
1572     die "unable to parse numa options\n";
1573 }
1574
1575 PVE::JSONSchema::register_format('pve-qm-net', \&verify_net);
1576 sub verify_net {
1577     my ($value, $noerr) = @_;
1578
1579     return $value if parse_net($value);
1580
1581     return undef if $noerr;
1582
1583     die "unable to parse network options\n";
1584 }
1585
1586 PVE::JSONSchema::register_format('pve-qm-drive', \&verify_drive);
1587 sub verify_drive {
1588     my ($value, $noerr) = @_;
1589
1590     return $value if parse_drive(undef, $value);
1591
1592     return undef if $noerr;
1593
1594     die "unable to parse drive options\n";
1595 }
1596
1597 PVE::JSONSchema::register_format('pve-qm-hostpci', \&verify_hostpci);
1598 sub verify_hostpci {
1599     my ($value, $noerr) = @_;
1600
1601     return $value if parse_hostpci($value);
1602
1603     return undef if $noerr;
1604
1605     die "unable to parse pci id\n";
1606 }
1607
1608 PVE::JSONSchema::register_format('pve-qm-watchdog', \&verify_watchdog);
1609 sub verify_watchdog {
1610     my ($value, $noerr) = @_;
1611
1612     return $value if parse_watchdog($value);
1613
1614     return undef if $noerr;
1615
1616     die "unable to parse watchdog options\n";
1617 }
1618
1619 sub parse_watchdog {
1620     my ($value) = @_;
1621
1622     return undef if !$value;
1623
1624     my $res = {};
1625
1626     foreach my $p (split(/,/, $value)) {
1627         next if $p =~ m/^\s*$/;
1628
1629         if ($p =~ m/^(model=)?(i6300esb|ib700)$/) {
1630             $res->{model} = $2;
1631         } elsif ($p =~ m/^(action=)?(reset|shutdown|poweroff|pause|debug|none)$/) {
1632             $res->{action} = $2;
1633         } else {
1634             return undef;
1635         }
1636     }
1637
1638     return $res;
1639 }
1640
1641 sub parse_usb_device {
1642     my ($value) = @_;
1643
1644     return undef if !$value;
1645
1646     my @dl = split(/,/, $value);
1647     my $found;
1648
1649     my $res = {};
1650     foreach my $v (@dl) {
1651         if ($v =~ m/^host=(0x)?([0-9A-Fa-f]{4}):(0x)?([0-9A-Fa-f]{4})$/) {
1652             $found = 1;
1653             $res->{vendorid} = $2;
1654             $res->{productid} = $4;
1655         } elsif ($v =~ m/^host=(\d+)\-(\d+(\.\d+)*)$/) {
1656             $found = 1;
1657             $res->{hostbus} = $1;
1658             $res->{hostport} = $2;
1659         } elsif ($v =~ m/^spice$/) {
1660             $found = 1;
1661             $res->{spice} = 1;
1662         } else {
1663             return undef;
1664         }
1665     }
1666     return undef if !$found;
1667
1668     return $res;
1669 }
1670
1671 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
1672 sub verify_usb_device {
1673     my ($value, $noerr) = @_;
1674
1675     return $value if parse_usb_device($value);
1676
1677     return undef if $noerr;
1678
1679     die "unable to parse usb device\n";
1680 }
1681
1682 # add JSON properties for create and set function
1683 sub json_config_properties {
1684     my $prop = shift;
1685
1686     foreach my $opt (keys %$confdesc) {
1687         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
1688         $prop->{$opt} = $confdesc->{$opt};
1689     }
1690
1691     return $prop;
1692 }
1693
1694 sub check_type {
1695     my ($key, $value) = @_;
1696
1697     die "unknown setting '$key'\n" if !$confdesc->{$key};
1698
1699     my $type = $confdesc->{$key}->{type};
1700
1701     if (!defined($value)) {
1702         die "got undefined value\n";
1703     }
1704
1705     if ($value =~ m/[\n\r]/) {
1706         die "property contains a line feed\n";
1707     }
1708
1709     if ($type eq 'boolean') {
1710         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
1711         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
1712         die "type check ('boolean') failed - got '$value'\n";
1713     } elsif ($type eq 'integer') {
1714         return int($1) if $value =~ m/^(\d+)$/;
1715         die "type check ('integer') failed - got '$value'\n";
1716     } elsif ($type eq 'number') {
1717         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
1718         die "type check ('number') failed - got '$value'\n";
1719     } elsif ($type eq 'string') {
1720         if (my $fmt = $confdesc->{$key}->{format}) {
1721             if ($fmt eq 'pve-qm-drive') {
1722                 # special case - we need to pass $key to parse_drive()
1723                 my $drive = parse_drive($key, $value);
1724                 return $value if $drive;
1725                 die "unable to parse drive options\n";
1726             }
1727             PVE::JSONSchema::check_format($fmt, $value);
1728             return $value;
1729         }
1730         $value =~ s/^\"(.*)\"$/$1/;
1731         return $value;
1732     } else {
1733         die "internal error"
1734     }
1735 }
1736
1737 sub lock_config_full {
1738     my ($vmid, $timeout, $code, @param) = @_;
1739
1740     my $filename = config_file_lock($vmid);
1741
1742     my $res = lock_file($filename, $timeout, $code, @param);
1743
1744     die $@ if $@;
1745
1746     return $res;
1747 }
1748
1749 sub lock_config_mode {
1750     my ($vmid, $timeout, $shared, $code, @param) = @_;
1751
1752     my $filename = config_file_lock($vmid);
1753
1754     my $res = lock_file_full($filename, $timeout, $shared, $code, @param);
1755
1756     die $@ if $@;
1757
1758     return $res;
1759 }
1760
1761 sub lock_config {
1762     my ($vmid, $code, @param) = @_;
1763
1764     return lock_config_full($vmid, 10, $code, @param);
1765 }
1766
1767 sub cfs_config_path {
1768     my ($vmid, $node) = @_;
1769
1770     $node = $nodename if !$node;
1771     return "nodes/$node/qemu-server/$vmid.conf";
1772 }
1773
1774 sub check_iommu_support{
1775     #fixme : need to check IOMMU support
1776     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
1777
1778     my $iommu=1;
1779     return $iommu;
1780
1781 }
1782
1783 sub config_file {
1784     my ($vmid, $node) = @_;
1785
1786     my $cfspath = cfs_config_path($vmid, $node);
1787     return "/etc/pve/$cfspath";
1788 }
1789
1790 sub config_file_lock {
1791     my ($vmid) = @_;
1792
1793     return "$lock_dir/lock-$vmid.conf";
1794 }
1795
1796 sub touch_config {
1797     my ($vmid) = @_;
1798
1799     my $conf = config_file($vmid);
1800     utime undef, undef, $conf;
1801 }
1802
1803 sub destroy_vm {
1804     my ($storecfg, $vmid, $keep_empty_config) = @_;
1805
1806     my $conffile = config_file($vmid);
1807
1808     my $conf = load_config($vmid);
1809
1810     check_lock($conf);
1811
1812     # only remove disks owned by this VM
1813     foreach_drive($conf, sub {
1814         my ($ds, $drive) = @_;
1815
1816         return if drive_is_cdrom($drive);
1817
1818         my $volid = $drive->{file};
1819
1820         return if !$volid || $volid =~ m|^/|;
1821
1822         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
1823         return if !$path || !$owner || ($owner != $vmid);
1824
1825         PVE::Storage::vdisk_free($storecfg, $volid);
1826     });
1827
1828     if ($keep_empty_config) {
1829         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
1830     } else {
1831         unlink $conffile;
1832     }
1833
1834     # also remove unused disk
1835     eval {
1836         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
1837
1838         eval {
1839             PVE::Storage::foreach_volid($dl, sub {
1840                 my ($volid, $sid, $volname, $d) = @_;
1841                 PVE::Storage::vdisk_free($storecfg, $volid);
1842             });
1843         };
1844         warn $@ if $@;
1845
1846     };
1847     warn $@ if $@;
1848 }
1849
1850 sub load_config {
1851     my ($vmid, $node) = @_;
1852
1853     my $cfspath = cfs_config_path($vmid, $node);
1854
1855     my $conf = PVE::Cluster::cfs_read_file($cfspath);
1856
1857     die "no such VM ('$vmid')\n" if !defined($conf);
1858
1859     return $conf;
1860 }
1861
1862 sub parse_vm_config {
1863     my ($filename, $raw) = @_;
1864
1865     return undef if !defined($raw);
1866
1867     my $res = {
1868         digest => Digest::SHA::sha1_hex($raw),
1869         snapshots => {},
1870         pending => {},
1871     };
1872
1873     $filename =~ m|/qemu-server/(\d+)\.conf$|
1874         || die "got strange filename '$filename'";
1875
1876     my $vmid = $1;
1877
1878     my $conf = $res;
1879     my $descr = '';
1880     my $section = '';
1881
1882     my @lines = split(/\n/, $raw);
1883     foreach my $line (@lines) {
1884         next if $line =~ m/^\s*$/;
1885
1886         if ($line =~ m/^\[PENDING\]\s*$/i) {
1887             $section = 'pending';
1888             $conf->{description} = $descr if $descr;
1889             $descr = '';
1890             $conf = $res->{$section} = {};
1891             next;
1892
1893         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
1894             $section = $1;
1895             $conf->{description} = $descr if $descr;
1896             $descr = '';
1897             $conf = $res->{snapshots}->{$section} = {};
1898             next;
1899         }
1900
1901         if ($line =~ m/^\#(.*)\s*$/) {
1902             $descr .= PVE::Tools::decode_text($1) . "\n";
1903             next;
1904         }
1905
1906         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
1907             $descr .= PVE::Tools::decode_text($2);
1908         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
1909             $conf->{snapstate} = $1;
1910         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
1911             my $key = $1;
1912             my $value = $2;
1913             $conf->{$key} = $value;
1914         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
1915             my $value = $1;
1916             if ($section eq 'pending') {
1917                 $conf->{delete} = $value; # we parse this later
1918             } else {
1919                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
1920             }
1921         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
1922             my $key = $1;
1923             my $value = $2;
1924             eval { $value = check_type($key, $value); };
1925             if ($@) {
1926                 warn "vm $vmid - unable to parse value of '$key' - $@";
1927             } else {
1928                 my $fmt = $confdesc->{$key}->{format};
1929                 if ($fmt && $fmt eq 'pve-qm-drive') {
1930                     my $v = parse_drive($key, $value);
1931                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
1932                         $v->{file} = $volid;
1933                         $value = print_drive($vmid, $v);
1934                     } else {
1935                         warn "vm $vmid - unable to parse value of '$key'\n";
1936                         next;
1937                     }
1938                 }
1939
1940                 if ($key eq 'cdrom') {
1941                     $conf->{ide2} = $value;
1942                 } else {
1943                     $conf->{$key} = $value;
1944                 }
1945             }
1946         }
1947     }
1948
1949     $conf->{description} = $descr if $descr;
1950
1951     delete $res->{snapstate}; # just to be sure
1952
1953     return $res;
1954 }
1955
1956 sub write_vm_config {
1957     my ($filename, $conf) = @_;
1958
1959     delete $conf->{snapstate}; # just to be sure
1960
1961     if ($conf->{cdrom}) {
1962         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
1963         $conf->{ide2} = $conf->{cdrom};
1964         delete $conf->{cdrom};
1965     }
1966
1967     # we do not use 'smp' any longer
1968     if ($conf->{sockets}) {
1969         delete $conf->{smp};
1970     } elsif ($conf->{smp}) {
1971         $conf->{sockets} = $conf->{smp};
1972         delete $conf->{cores};
1973         delete $conf->{smp};
1974     }
1975
1976     my $used_volids = {};
1977
1978     my $cleanup_config = sub {
1979         my ($cref, $pending, $snapname) = @_;
1980
1981         foreach my $key (keys %$cref) {
1982             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
1983                 $key eq 'snapstate' || $key eq 'pending';
1984             my $value = $cref->{$key};
1985             if ($key eq 'delete') {
1986                 die "propertry 'delete' is only allowed in [PENDING]\n"
1987                     if !$pending;
1988                 # fixme: check syntax?
1989                 next;
1990             }
1991             eval { $value = check_type($key, $value); };
1992             die "unable to parse value of '$key' - $@" if $@;
1993
1994             $cref->{$key} = $value;
1995
1996             if (!$snapname && valid_drivename($key)) {
1997                 my $drive = parse_drive($key, $value);
1998                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
1999             }
2000         }
2001     };
2002
2003     &$cleanup_config($conf);
2004
2005     &$cleanup_config($conf->{pending}, 1);
2006
2007     foreach my $snapname (keys %{$conf->{snapshots}}) {
2008         die "internal error" if $snapname eq 'pending';
2009         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2010     }
2011
2012     # remove 'unusedX' settings if we re-add a volume
2013     foreach my $key (keys %$conf) {
2014         my $value = $conf->{$key};
2015         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2016             delete $conf->{$key};
2017         }
2018     }
2019
2020     my $generate_raw_config = sub {
2021         my ($conf) = @_;
2022
2023         my $raw = '';
2024
2025         # add description as comment to top of file
2026         my $descr = $conf->{description} || '';
2027         foreach my $cl (split(/\n/, $descr)) {
2028             $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2029         }
2030
2031         foreach my $key (sort keys %$conf) {
2032             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2033             $raw .= "$key: $conf->{$key}\n";
2034         }
2035         return $raw;
2036     };
2037
2038     my $raw = &$generate_raw_config($conf);
2039
2040     if (scalar(keys %{$conf->{pending}})){
2041         $raw .= "\n[PENDING]\n";
2042         $raw .= &$generate_raw_config($conf->{pending});
2043     }
2044
2045     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2046         $raw .= "\n[$snapname]\n";
2047         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2048     }
2049
2050     return $raw;
2051 }
2052
2053 sub update_config_nolock {
2054     my ($vmid, $conf, $skiplock) = @_;
2055
2056     check_lock($conf) if !$skiplock;
2057
2058     my $cfspath = cfs_config_path($vmid);
2059
2060     PVE::Cluster::cfs_write_file($cfspath, $conf);
2061 }
2062
2063 sub update_config {
2064     my ($vmid, $conf, $skiplock) = @_;
2065
2066     lock_config($vmid, &update_config_nolock, $conf, $skiplock);
2067 }
2068
2069 sub load_defaults {
2070
2071     my $res = {};
2072
2073     # we use static defaults from our JSON schema configuration
2074     foreach my $key (keys %$confdesc) {
2075         if (defined(my $default = $confdesc->{$key}->{default})) {
2076             $res->{$key} = $default;
2077         }
2078     }
2079
2080     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2081     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2082
2083     return $res;
2084 }
2085
2086 sub config_list {
2087     my $vmlist = PVE::Cluster::get_vmlist();
2088     my $res = {};
2089     return $res if !$vmlist || !$vmlist->{ids};
2090     my $ids = $vmlist->{ids};
2091
2092     foreach my $vmid (keys %$ids) {
2093         my $d = $ids->{$vmid};
2094         next if !$d->{node} || $d->{node} ne $nodename;
2095         next if !$d->{type} || $d->{type} ne 'qemu';
2096         $res->{$vmid}->{exists} = 1;
2097     }
2098     return $res;
2099 }
2100
2101 # test if VM uses local resources (to prevent migration)
2102 sub check_local_resources {
2103     my ($conf, $noerr) = @_;
2104
2105     my $loc_res = 0;
2106
2107     $loc_res = 1 if $conf->{hostusb}; # old syntax
2108     $loc_res = 1 if $conf->{hostpci}; # old syntax
2109
2110     foreach my $k (keys %$conf) {
2111         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2112         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2113     }
2114
2115     die "VM uses local resources\n" if $loc_res && !$noerr;
2116
2117     return $loc_res;
2118 }
2119
2120 # check if used storages are available on all nodes (use by migrate)
2121 sub check_storage_availability {
2122     my ($storecfg, $conf, $node) = @_;
2123
2124     foreach_drive($conf, sub {
2125         my ($ds, $drive) = @_;
2126
2127         my $volid = $drive->{file};
2128         return if !$volid;
2129
2130         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2131         return if !$sid;
2132
2133         # check if storage is available on both nodes
2134         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2135         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2136    });
2137 }
2138
2139 # list nodes where all VM images are available (used by has_feature API)
2140 sub shared_nodes {
2141     my ($conf, $storecfg) = @_;
2142
2143     my $nodelist = PVE::Cluster::get_nodelist();
2144     my $nodehash = { map { $_ => 1 } @$nodelist };
2145     my $nodename = PVE::INotify::nodename();
2146
2147     foreach_drive($conf, sub {
2148         my ($ds, $drive) = @_;
2149
2150         my $volid = $drive->{file};
2151         return if !$volid;
2152
2153         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2154         if ($storeid) {
2155             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2156             if ($scfg->{disable}) {
2157                 $nodehash = {};
2158             } elsif (my $avail = $scfg->{nodes}) {
2159                 foreach my $node (keys %$nodehash) {
2160                     delete $nodehash->{$node} if !$avail->{$node};
2161                 }
2162             } elsif (!$scfg->{shared}) {
2163                 foreach my $node (keys %$nodehash) {
2164                     delete $nodehash->{$node} if $node ne $nodename
2165                 }
2166             }
2167         }
2168     });
2169
2170     return $nodehash
2171 }
2172
2173 sub check_lock {
2174     my ($conf) = @_;
2175
2176     die "VM is locked ($conf->{lock})\n" if $conf->{lock};
2177 }
2178
2179 sub check_cmdline {
2180     my ($pidfile, $pid) = @_;
2181
2182     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2183     if (defined($fh)) {
2184         my $line = <$fh>;
2185         $fh->close;
2186         return undef if !$line;
2187         my @param = split(/\0/, $line);
2188
2189         my $cmd = $param[0];
2190         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2191
2192         for (my $i = 0; $i < scalar (@param); $i++) {
2193             my $p = $param[$i];
2194             next if !$p;
2195             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2196                 my $p = $param[$i+1];
2197                 return 1 if $p && ($p eq $pidfile);
2198                 return undef;
2199             }
2200         }
2201     }
2202     return undef;
2203 }
2204
2205 sub check_running {
2206     my ($vmid, $nocheck, $node) = @_;
2207
2208     my $filename = config_file($vmid, $node);
2209
2210     die "unable to find configuration file for VM $vmid - no such machine\n"
2211         if !$nocheck && ! -f $filename;
2212
2213     my $pidfile = pidfile_name($vmid);
2214
2215     if (my $fd = IO::File->new("<$pidfile")) {
2216         my $st = stat($fd);
2217         my $line = <$fd>;
2218         close($fd);
2219
2220         my $mtime = $st->mtime;
2221         if ($mtime > time()) {
2222             warn "file '$filename' modified in future\n";
2223         }
2224
2225         if ($line =~ m/^(\d+)$/) {
2226             my $pid = $1;
2227             if (check_cmdline($pidfile, $pid)) {
2228                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2229                     return $pid;
2230                 }
2231             }
2232         }
2233     }
2234
2235     return undef;
2236 }
2237
2238 sub vzlist {
2239
2240     my $vzlist = config_list();
2241
2242     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2243
2244     while (defined(my $de = $fd->read)) {
2245         next if $de !~ m/^(\d+)\.pid$/;
2246         my $vmid = $1;
2247         next if !defined($vzlist->{$vmid});
2248         if (my $pid = check_running($vmid)) {
2249             $vzlist->{$vmid}->{pid} = $pid;
2250         }
2251     }
2252
2253     return $vzlist;
2254 }
2255
2256 sub disksize {
2257     my ($storecfg, $conf) = @_;
2258
2259     my $bootdisk = $conf->{bootdisk};
2260     return undef if !$bootdisk;
2261     return undef if !valid_drivename($bootdisk);
2262
2263     return undef if !$conf->{$bootdisk};
2264
2265     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2266     return undef if !defined($drive);
2267
2268     return undef if drive_is_cdrom($drive);
2269
2270     my $volid = $drive->{file};
2271     return undef if !$volid;
2272
2273     return $drive->{size};
2274 }
2275
2276 my $last_proc_pid_stat;
2277
2278 # get VM status information
2279 # This must be fast and should not block ($full == false)
2280 # We only query KVM using QMP if $full == true (this can be slow)
2281 sub vmstatus {
2282     my ($opt_vmid, $full) = @_;
2283
2284     my $res = {};
2285
2286     my $storecfg = PVE::Storage::config();
2287
2288     my $list = vzlist();
2289     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2290
2291     my $cpucount = $cpuinfo->{cpus} || 1;
2292
2293     foreach my $vmid (keys %$list) {
2294         next if $opt_vmid && ($vmid ne $opt_vmid);
2295
2296         my $cfspath = cfs_config_path($vmid);
2297         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2298
2299         my $d = {};
2300         $d->{pid} = $list->{$vmid}->{pid};
2301
2302         # fixme: better status?
2303         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2304
2305         my $size = disksize($storecfg, $conf);
2306         if (defined($size)) {
2307             $d->{disk} = 0; # no info available
2308             $d->{maxdisk} = $size;
2309         } else {
2310             $d->{disk} = 0;
2311             $d->{maxdisk} = 0;
2312         }
2313
2314         $d->{cpus} = ($conf->{sockets} || 1) * ($conf->{cores} || 1);
2315         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2316         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2317
2318         $d->{name} = $conf->{name} || "VM $vmid";
2319         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024) : 0;
2320
2321         if ($conf->{balloon}) {
2322             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2323             $d->{shares} = defined($conf->{shares}) ? $conf->{shares} : 1000;
2324         }
2325
2326         $d->{uptime} = 0;
2327         $d->{cpu} = 0;
2328         $d->{mem} = 0;
2329
2330         $d->{netout} = 0;
2331         $d->{netin} = 0;
2332
2333         $d->{diskread} = 0;
2334         $d->{diskwrite} = 0;
2335
2336         $d->{template} = is_template($conf);
2337
2338         $res->{$vmid} = $d;
2339     }
2340
2341     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2342     foreach my $dev (keys %$netdev) {
2343         next if $dev !~ m/^tap([1-9]\d*)i/;
2344         my $vmid = $1;
2345         my $d = $res->{$vmid};
2346         next if !$d;
2347
2348         $d->{netout} += $netdev->{$dev}->{receive};
2349         $d->{netin} += $netdev->{$dev}->{transmit};
2350
2351         if ($full) {
2352             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2353             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2354         }
2355
2356     }
2357
2358     my $ctime = gettimeofday;
2359
2360     foreach my $vmid (keys %$list) {
2361
2362         my $d = $res->{$vmid};
2363         my $pid = $d->{pid};
2364         next if !$pid;
2365
2366         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2367         next if !$pstat; # not running
2368
2369         my $used = $pstat->{utime} + $pstat->{stime};
2370
2371         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2372
2373         if ($pstat->{vsize}) {
2374             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2375         }
2376
2377         my $old = $last_proc_pid_stat->{$pid};
2378         if (!$old) {
2379             $last_proc_pid_stat->{$pid} = {
2380                 time => $ctime,
2381                 used => $used,
2382                 cpu => 0,
2383             };
2384             next;
2385         }
2386
2387         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2388
2389         if ($dtime > 1000) {
2390             my $dutime = $used -  $old->{used};
2391
2392             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2393             $last_proc_pid_stat->{$pid} = {
2394                 time => $ctime,
2395                 used => $used,
2396                 cpu => $d->{cpu},
2397             };
2398         } else {
2399             $d->{cpu} = $old->{cpu};
2400         }
2401     }
2402
2403     return $res if !$full;
2404
2405     my $qmpclient = PVE::QMPClient->new();
2406
2407     my $ballooncb = sub {
2408         my ($vmid, $resp) = @_;
2409
2410         my $info = $resp->{'return'};
2411         return if !$info->{max_mem};
2412
2413         my $d = $res->{$vmid};
2414
2415         # use memory assigned to VM
2416         $d->{maxmem} = $info->{max_mem};
2417         $d->{balloon} = $info->{actual};
2418
2419         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2420             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2421             $d->{freemem} = $info->{free_mem};
2422         }
2423
2424         $d->{ballooninfo} = $info;
2425     };
2426
2427     my $blockstatscb = sub {
2428         my ($vmid, $resp) = @_;
2429         my $data = $resp->{'return'} || [];
2430         my $totalrdbytes = 0;
2431         my $totalwrbytes = 0;
2432
2433         for my $blockstat (@$data) {
2434             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2435             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2436
2437             $blockstat->{device} =~ s/drive-//;
2438             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2439         }
2440         $res->{$vmid}->{diskread} = $totalrdbytes;
2441         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2442     };
2443
2444     my $statuscb = sub {
2445         my ($vmid, $resp) = @_;
2446
2447         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2448         # this fails if ballon driver is not loaded, so this must be
2449         # the last commnand (following command are aborted if this fails).
2450         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2451
2452         my $status = 'unknown';
2453         if (!defined($status = $resp->{'return'}->{status})) {
2454             warn "unable to get VM status\n";
2455             return;
2456         }
2457
2458         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2459     };
2460
2461     foreach my $vmid (keys %$list) {
2462         next if $opt_vmid && ($vmid ne $opt_vmid);
2463         next if !$res->{$vmid}->{pid}; # not running
2464         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2465     }
2466
2467     $qmpclient->queue_execute(undef, 1);
2468
2469     foreach my $vmid (keys %$list) {
2470         next if $opt_vmid && ($vmid ne $opt_vmid);
2471         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2472     }
2473
2474     return $res;
2475 }
2476
2477 sub foreach_dimm {
2478     my ($conf, $vmid, $memory, $sockets, $func) = @_;
2479
2480     my $dimm_id = 0;
2481     my $current_size = 1024;
2482     my $dimm_size = 512;
2483     return if $current_size == $memory;
2484
2485     for (my $j = 0; $j < 8; $j++) {
2486         for (my $i = 0; $i < 32; $i++) {
2487             my $name = "dimm${dimm_id}";
2488             $dimm_id++;
2489             my $numanode = $i % $sockets;
2490             $current_size += $dimm_size;
2491             &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
2492             return  $current_size if $current_size >= $memory;
2493         }
2494         $dimm_size *= 2;
2495     }
2496 }
2497
2498 sub foreach_drive {
2499     my ($conf, $func) = @_;
2500
2501     foreach my $ds (keys %$conf) {
2502         next if !valid_drivename($ds);
2503
2504         my $drive = parse_drive($ds, $conf->{$ds});
2505         next if !$drive;
2506
2507         &$func($ds, $drive);
2508     }
2509 }
2510
2511 sub foreach_volid {
2512     my ($conf, $func) = @_;
2513
2514     my $volhash = {};
2515
2516     my $test_volid = sub {
2517         my ($volid, $is_cdrom) = @_;
2518
2519         return if !$volid;
2520
2521         $volhash->{$volid} = $is_cdrom || 0;
2522     };
2523
2524     foreach_drive($conf, sub {
2525         my ($ds, $drive) = @_;
2526         &$test_volid($drive->{file}, drive_is_cdrom($drive));
2527     });
2528
2529     foreach my $snapname (keys %{$conf->{snapshots}}) {
2530         my $snap = $conf->{snapshots}->{$snapname};
2531         &$test_volid($snap->{vmstate}, 0);
2532         foreach_drive($snap, sub {
2533             my ($ds, $drive) = @_;
2534             &$test_volid($drive->{file}, drive_is_cdrom($drive));
2535         });
2536     }
2537
2538     foreach my $volid (keys %$volhash) {
2539         &$func($volid, $volhash->{$volid});
2540     }
2541 }
2542
2543 sub vga_conf_has_spice {
2544     my ($vga) = @_;
2545
2546     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
2547
2548     return $1 || 1;
2549 }
2550
2551 sub config_to_command {
2552     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2553
2554     my $cmd = [];
2555     my $globalFlags = [];
2556     my $machineFlags = [];
2557     my $rtcFlags = [];
2558     my $cpuFlags = [];
2559     my $devices = [];
2560     my $pciaddr = '';
2561     my $bridges = {};
2562     my $kvmver = kvm_user_version();
2563     my $vernum = 0; # unknown
2564     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
2565         $vernum = $1*1000000+$2*1000;
2566     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
2567         $vernum = $1*1000000+$2*1000+$3;
2568     }
2569
2570     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
2571
2572     my $have_ovz = -f '/proc/vz/vestat';
2573
2574     my $q35 = machine_type_is_q35($conf);
2575     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2576     my $machine_type = $forcemachine || $conf->{machine};
2577
2578     my $cpuunits = defined($conf->{cpuunits}) ?
2579             $conf->{cpuunits} : $defaults->{cpuunits};
2580
2581     push @$cmd, '/usr/bin/systemd-run';
2582     push @$cmd, '--scope';
2583     push @$cmd, '--slice', "qemu";
2584     push @$cmd, '--unit', $vmid;
2585     push @$cmd, '-p', "CPUShares=$cpuunits";
2586     if ($conf->{cpulimit}) {
2587         my $cpulimit = int($conf->{cpulimit} * 100);
2588         push @$cmd, '-p', "CPUQuota=$cpulimit\%";
2589     }
2590
2591     push @$cmd, '/usr/bin/kvm';
2592
2593     push @$cmd, '-id', $vmid;
2594
2595     my $use_virtio = 0;
2596
2597     my $qmpsocket = qmp_socket($vmid);
2598     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2599     push @$cmd, '-mon', "chardev=qmp,mode=control";
2600
2601     my $socket = vnc_socket($vmid);
2602     push @$cmd,  '-vnc', "unix:$socket,x509,password";
2603
2604     push @$cmd, '-pidfile' , pidfile_name($vmid);
2605
2606     push @$cmd, '-daemonize';
2607
2608     if ($conf->{smbios1}) {
2609         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
2610     }
2611
2612     if ($q35) {
2613         # the q35 chipset support native usb2, so we enable usb controller
2614         # by default for this machine type
2615         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
2616     } else {
2617         $pciaddr = print_pci_addr("piix3", $bridges);
2618         push @$devices, '-device', "piix3-usb-uhci,id=uhci$pciaddr.0x2";
2619
2620         my $use_usb2 = 0;
2621         for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2622             next if !$conf->{"usb$i"};
2623             $use_usb2 = 1;
2624         }
2625         # include usb device config
2626         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-usb.cfg' if $use_usb2;
2627     }
2628
2629     my $vga = $conf->{vga};
2630
2631     my $qxlnum = vga_conf_has_spice($vga);
2632     $vga = 'qxl' if $qxlnum;
2633
2634     if (!$vga) {
2635         if ($conf->{ostype} && ($conf->{ostype} eq 'win8' ||
2636                                 $conf->{ostype} eq 'win7' ||
2637                                 $conf->{ostype} eq 'w2k8')) {
2638             $vga = 'std';
2639         } else {
2640             $vga = 'cirrus';
2641         }
2642     }
2643
2644     # enable absolute mouse coordinates (needed by vnc)
2645     my $tablet;
2646     if (defined($conf->{tablet})) {
2647         $tablet = $conf->{tablet};
2648     } else {
2649         $tablet = $defaults->{tablet};
2650         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
2651         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
2652     }
2653
2654     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
2655
2656     # host pci devices
2657     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
2658         my $d = parse_hostpci($conf->{"hostpci$i"});
2659         next if !$d;
2660
2661         my $pcie = $d->{pcie};
2662         if($pcie){
2663             die "q35 machine model is not enabled" if !$q35;
2664             $pciaddr = print_pcie_addr("hostpci$i");
2665         }else{
2666             $pciaddr = print_pci_addr("hostpci$i", $bridges);
2667         }
2668
2669         my $rombar = $d->{rombar} && $d->{rombar} eq 'off' ? ",rombar=0" : "";
2670         my $driver = $d->{driver} && $d->{driver} eq 'vfio' ? "vfio-pci" : "pci-assign";
2671         my $xvga = $d->{'x-vga'} && $d->{'x-vga'} eq 'on' ? ",x-vga=on" : "";
2672         if ($xvga && $xvga ne '') {
2673             push @$cpuFlags, 'kvm=off';
2674             $vga = 'none';
2675         }
2676         $driver = "vfio-pci" if $xvga ne '';
2677         my $pcidevices = $d->{pciid};
2678         my $multifunction = 1 if @$pcidevices > 1;
2679
2680         my $j=0;
2681         foreach my $pcidevice (@$pcidevices) {
2682
2683             my $id = "hostpci$i";
2684             $id .= ".$j" if $multifunction;
2685             my $addr = $pciaddr;
2686             $addr .= ".$j" if $multifunction;
2687             my $devicestr = "$driver,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
2688
2689             if($j == 0){
2690                 $devicestr .= "$rombar$xvga";
2691                 $devicestr .= ",multifunction=on" if $multifunction;
2692             }
2693
2694             push @$devices, '-device', $devicestr;
2695             $j++;
2696         }
2697     }
2698
2699     # usb devices
2700     for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2701         my $d = parse_usb_device($conf->{"usb$i"});
2702         next if !$d;
2703         if ($d->{vendorid} && $d->{productid}) {
2704             push @$devices, '-device', "usb-host,vendorid=0x$d->{vendorid},productid=0x$d->{productid}";
2705         } elsif (defined($d->{hostbus}) && defined($d->{hostport})) {
2706             push @$devices, '-device', "usb-host,hostbus=$d->{hostbus},hostport=$d->{hostport}";
2707         } elsif ($d->{spice}) {
2708             # usb redir support for spice
2709             push @$devices, '-chardev', "spicevmc,id=usbredirchardev$i,name=usbredir";
2710             push @$devices, '-device', "usb-redir,chardev=usbredirchardev$i,id=usbredirdev$i,bus=ehci.0";
2711         }
2712     }
2713
2714     # serial devices
2715     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2716         if (my $path = $conf->{"serial$i"}) {
2717             if ($path eq 'socket') {
2718                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
2719                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
2720                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2721             } else {
2722                 die "no such serial device\n" if ! -c $path;
2723                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
2724                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2725             }
2726         }
2727     }
2728
2729     # parallel devices
2730     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
2731         if (my $path = $conf->{"parallel$i"}) {
2732             die "no such parallel device\n" if ! -c $path;
2733             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
2734             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
2735             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
2736         }
2737     }
2738
2739     my $vmname = $conf->{name} || "vm$vmid";
2740
2741     push @$cmd, '-name', $vmname;
2742
2743     my $sockets = 1;
2744     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
2745     $sockets = $conf->{sockets} if  $conf->{sockets};
2746
2747     my $cores = $conf->{cores} || 1;
2748
2749     my $maxcpus = $sockets * $cores;
2750
2751     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
2752
2753     my $allowed_vcpus = $cpuinfo->{cpus};
2754
2755     die "MAX $maxcpus vcpus allowed per VM on this node\n"
2756         if ($allowed_vcpus < $maxcpus);
2757
2758     push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
2759
2760     push @$cmd, '-nodefaults';
2761
2762     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2763
2764     my $bootindex_hash = {};
2765     my $i = 1;
2766     foreach my $o (split(//, $bootorder)) {
2767         $bootindex_hash->{$o} = $i*100;
2768         $i++;
2769     }
2770
2771     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000";
2772
2773     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
2774
2775     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
2776
2777     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
2778
2779     # time drift fix
2780     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
2781
2782     my $nokvm = defined($conf->{kvm}) && $conf->{kvm} == 0 ? 1 : 0;
2783     my $useLocaltime = $conf->{localtime};
2784
2785     if (my $ost = $conf->{ostype}) {
2786         # other, wxp, w2k, w2k3, w2k8, wvista, win7, win8, l24, l26, solaris
2787
2788         if ($ost =~ m/^w/) { # windows
2789             $useLocaltime = 1 if !defined($conf->{localtime});
2790
2791             # use time drift fix when acpi is enabled
2792             if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
2793                 $tdf = 1 if !defined($conf->{tdf});
2794             }
2795         }
2796
2797         if ($ost eq 'win7' || $ost eq 'win8' || $ost eq 'w2k8' ||
2798             $ost eq 'wvista') {
2799             push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
2800             push @$cmd, '-no-hpet';
2801             if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
2802                 push @$cpuFlags , 'hv_spinlocks=0x1fff' if !$nokvm;
2803                 push @$cpuFlags , 'hv_vapic' if !$nokvm;
2804                 push @$cpuFlags , 'hv_time' if !$nokvm;
2805
2806             } else {
2807                 push @$cpuFlags , 'hv_spinlocks=0xffff' if !$nokvm;
2808             }
2809         }
2810
2811         if ($ost eq 'win7' || $ost eq 'win8') {
2812             push @$cpuFlags , 'hv_relaxed' if !$nokvm;
2813         }
2814     }
2815
2816     push @$rtcFlags, 'driftfix=slew' if $tdf;
2817
2818     if ($nokvm) {
2819         push @$machineFlags, 'accel=tcg';
2820     } else {
2821         die "No accelerator found!\n" if !$cpuinfo->{hvm};
2822     }
2823
2824     if ($machine_type) {
2825         push @$machineFlags, "type=${machine_type}";
2826     }
2827
2828     if ($conf->{startdate}) {
2829         push @$rtcFlags, "base=$conf->{startdate}";
2830     } elsif ($useLocaltime) {
2831         push @$rtcFlags, 'base=localtime';
2832     }
2833
2834     my $cpu = $nokvm ? "qemu64" : "kvm64";
2835     $cpu = $conf->{cpu} if $conf->{cpu};
2836
2837     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
2838
2839     push @$cpuFlags , '+x2apic' if !$nokvm && $conf->{ostype} ne 'solaris';
2840
2841     push @$cpuFlags , '-x2apic' if $conf->{ostype} eq 'solaris';
2842
2843     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
2844
2845     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
2846
2847     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
2848
2849         push @$cpuFlags , '+kvm_pv_unhalt' if !$nokvm;
2850         push @$cpuFlags , '+kvm_pv_eoi' if !$nokvm;
2851     }
2852
2853     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
2854
2855     push @$cmd, '-cpu', "$cpu,enforce";
2856
2857     my $memory = $conf->{memory} || $defaults->{memory};
2858     my $static_memory = 0;
2859     my $dimm_memory = 0;
2860
2861     if ($hotplug_features->{memory}) {
2862         die "Numa need to be enabled for memory hotplug\n" if !$conf->{numa};
2863         die "Total memory is bigger than ${MAX_MEM}MB\n" if $memory > $MAX_MEM;
2864         $static_memory = $STATICMEM;
2865         die "minimum memory must be ${static_memory}MB\n" if($memory < $static_memory);
2866         $dimm_memory = $memory - $static_memory;
2867         push @$cmd, '-m', "size=${static_memory},slots=255,maxmem=${MAX_MEM}M";
2868
2869     } else {
2870
2871         $static_memory = $memory;
2872         push @$cmd, '-m', $static_memory;
2873     }
2874
2875     if ($conf->{numa}) {
2876
2877         my $numa_totalmemory = undef;
2878         for (my $i = 0; $i < $MAX_NUMA; $i++) {
2879             next if !$conf->{"numa$i"};
2880             my $numa = parse_numa($conf->{"numa$i"});
2881             next if !$numa;
2882             # memory
2883             die "missing numa node$i memory value\n" if !$numa->{memory};
2884             my $numa_memory = $numa->{memory};
2885             $numa_totalmemory += $numa_memory;
2886             my $numa_object = "memory-backend-ram,id=ram-node$i,size=${numa_memory}M";
2887
2888             # cpus
2889             my $cpus_start = $numa->{cpus}->{start};
2890             die "missing numa node$i cpus\n" if !defined($cpus_start);
2891             my $cpus_end = $numa->{cpus}->{end} if defined($numa->{cpus}->{end});
2892             my $cpus = $cpus_start;
2893             if (defined($cpus_end)) {
2894                 $cpus .= "-$cpus_end";
2895                 die "numa node$i :  cpu range $cpus is incorrect\n" if $cpus_end <= $cpus_start;
2896             }
2897
2898             # hostnodes
2899             my $hostnodes_start = $numa->{hostnodes}->{start};
2900             if (defined($hostnodes_start)) {
2901                 my $hostnodes_end = $numa->{hostnodes}->{end} if defined($numa->{hostnodes}->{end});
2902                 my $hostnodes = $hostnodes_start;
2903                 if (defined($hostnodes_end)) {
2904                     $hostnodes .= "-$hostnodes_end";
2905                     die "host node $hostnodes range is incorrect\n" if $hostnodes_end <= $hostnodes_start;
2906                 }
2907
2908                 my $hostnodes_end_range = defined($hostnodes_end) ? $hostnodes_end : $hostnodes_start;
2909                 for (my $i = $hostnodes_start; $i <= $hostnodes_end_range; $i++ ) {
2910                     die "host numa node$i don't exist\n" if ! -d "/sys/devices/system/node/node$i/";
2911                 }
2912
2913                 # policy
2914                 my $policy = $numa->{policy};
2915                 die "you need to define a policy for hostnode $hostnodes\n" if !$policy;
2916                 $numa_object .= ",host-nodes=$hostnodes,policy=$policy";
2917             }
2918
2919             push @$cmd, '-object', $numa_object;
2920             push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
2921         }
2922
2923         die "total memory for NUMA nodes must be equal to vm static memory\n"
2924             if $numa_totalmemory && $numa_totalmemory != $static_memory;
2925
2926         #if no custom tology, we split memory and cores across numa nodes
2927         if(!$numa_totalmemory) {
2928
2929             my $numa_memory = ($static_memory / $sockets) . "M";
2930
2931             for (my $i = 0; $i < $sockets; $i++)  {
2932
2933                 my $cpustart = ($cores * $i);
2934                 my $cpuend = ($cpustart + $cores - 1) if $cores && $cores > 1;
2935                 my $cpus = $cpustart;
2936                 $cpus .= "-$cpuend" if $cpuend;
2937
2938                 push @$cmd, '-object', "memory-backend-ram,size=$numa_memory,id=ram-node$i";
2939                 push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
2940             }
2941         }
2942     }
2943
2944     if ($hotplug_features->{memory}) {
2945         foreach_dimm($conf, $vmid, $memory, $sockets, sub {
2946             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
2947             push @$cmd, "-object" , "memory-backend-ram,id=mem-$name,size=${dimm_size}M";
2948             push @$cmd, "-device", "pc-dimm,id=$name,memdev=mem-$name,node=$numanode";
2949
2950             #if dimm_memory is not aligned to dimm map
2951             if($current_size > $memory) {
2952                  $conf->{memory} = $current_size;
2953                  update_config_nolock($vmid, $conf, 1);
2954             }
2955         });
2956     }
2957
2958     push @$cmd, '-S' if $conf->{freeze};
2959
2960     # set keyboard layout
2961     my $kb = $conf->{keyboard} || $defaults->{keyboard};
2962     push @$cmd, '-k', $kb if $kb;
2963
2964     # enable sound
2965     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
2966     #push @$cmd, '-soundhw', 'es1370';
2967     #push @$cmd, '-soundhw', $soundhw if $soundhw;
2968
2969     if($conf->{agent}) {
2970         my $qgasocket = qmp_socket($vmid, 1);
2971         my $pciaddr = print_pci_addr("qga0", $bridges);
2972         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
2973         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
2974         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
2975     }
2976
2977     my $spice_port;
2978
2979     if ($qxlnum) {
2980         if ($qxlnum > 1) {
2981             if ($conf->{ostype} && $conf->{ostype} =~ m/^w/){
2982                 for(my $i = 1; $i < $qxlnum; $i++){
2983                     my $pciaddr = print_pci_addr("vga$i", $bridges);
2984                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
2985                 }
2986             } else {
2987                 # assume other OS works like Linux
2988                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
2989                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
2990             }
2991         }
2992
2993         my $pciaddr = print_pci_addr("spice", $bridges);
2994
2995         my $nodename = PVE::INotify::nodename();
2996         my $pfamily = PVE::Tools::get_host_address_family($nodename);
2997         $spice_port = PVE::Tools::next_spice_port($pfamily);
2998
2999         push @$devices, '-spice', "tls-port=${spice_port},addr=localhost,tls-ciphers=DES-CBC3-SHA,seamless-migration=on";
3000
3001         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3002         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3003         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3004     }
3005
3006     # enable balloon by default, unless explicitly disabled
3007     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3008         $pciaddr = print_pci_addr("balloon0", $bridges);
3009         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3010     }
3011
3012     if ($conf->{watchdog}) {
3013         my $wdopts = parse_watchdog($conf->{watchdog});
3014         $pciaddr = print_pci_addr("watchdog", $bridges);
3015         my $watchdog = $wdopts->{model} || 'i6300esb';
3016         push @$devices, '-device', "$watchdog$pciaddr";
3017         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3018     }
3019
3020     my $vollist = [];
3021     my $scsicontroller = {};
3022     my $ahcicontroller = {};
3023     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3024
3025     # Add iscsi initiator name if available
3026     if (my $initiator = get_initiator_name()) {
3027         push @$devices, '-iscsi', "initiator-name=$initiator";
3028     }
3029
3030     foreach_drive($conf, sub {
3031         my ($ds, $drive) = @_;
3032
3033         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3034             push @$vollist, $drive->{file};
3035         }
3036
3037         $use_virtio = 1 if $ds =~ m/^virtio/;
3038
3039         if (drive_is_cdrom ($drive)) {
3040             if ($bootindex_hash->{d}) {
3041                 $drive->{bootindex} = $bootindex_hash->{d};
3042                 $bootindex_hash->{d} += 1;
3043             }
3044         } else {
3045             if ($bootindex_hash->{c}) {
3046                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3047                 $bootindex_hash->{c} += 1;
3048             }
3049         }
3050
3051         if($drive->{interface} eq 'virtio'){
3052            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3053         }
3054
3055         if ($drive->{interface} eq 'scsi') {
3056
3057             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3058
3059             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3060             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3061
3062             my $iothread = '';
3063             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3064                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3065                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3066             }
3067
3068             my $queues = '';
3069             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3070                 $queues = ",num_queues=$drive->{queues}";
3071             }
3072
3073             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3074             $scsicontroller->{$controller}=1;
3075         }
3076
3077         if ($drive->{interface} eq 'sata') {
3078            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3079            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3080            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3081            $ahcicontroller->{$controller}=1;
3082         }
3083
3084         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3085         push @$devices, '-drive',$drive_cmd;
3086         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3087     });
3088
3089     for (my $i = 0; $i < $MAX_NETS; $i++) {
3090          next if !$conf->{"net$i"};
3091          my $d = parse_net($conf->{"net$i"});
3092          next if !$d;
3093
3094          $use_virtio = 1 if $d->{model} eq 'virtio';
3095
3096          if ($bootindex_hash->{n}) {
3097             $d->{bootindex} = $bootindex_hash->{n};
3098             $bootindex_hash->{n} += 1;
3099          }
3100
3101          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3102          push @$devices, '-netdev', $netdevfull;
3103
3104          my $netdevicefull = print_netdevice_full($vmid,$conf,$d,"net$i",$bridges);
3105          push @$devices, '-device', $netdevicefull;
3106     }
3107
3108     if (!$q35) {
3109         # add pci bridges
3110         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3111            $bridges->{1} = 1;
3112            $bridges->{2} = 1;
3113         }
3114
3115         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3116
3117         while (my ($k, $v) = each %$bridges) {
3118             $pciaddr = print_pci_addr("pci.$k");
3119             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3120         }
3121     }
3122
3123     # add custom args
3124     if ($conf->{args}) {
3125         my $aa = PVE::Tools::split_args($conf->{args});
3126         push @$cmd, @$aa;
3127     }
3128
3129     push @$cmd, @$devices;
3130     push @$cmd, '-rtc', join(',', @$rtcFlags)
3131         if scalar(@$rtcFlags);
3132     push @$cmd, '-machine', join(',', @$machineFlags)
3133         if scalar(@$machineFlags);
3134     push @$cmd, '-global', join(',', @$globalFlags)
3135         if scalar(@$globalFlags);
3136
3137     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3138 }
3139
3140 sub vnc_socket {
3141     my ($vmid) = @_;
3142     return "${var_run_tmpdir}/$vmid.vnc";
3143 }
3144
3145 sub spice_port {
3146     my ($vmid) = @_;
3147
3148     my $res = vm_mon_cmd($vmid, 'query-spice');
3149
3150     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3151 }
3152
3153 sub qmp_socket {
3154     my ($vmid, $qga) = @_;
3155     my $sockettype = $qga ? 'qga' : 'qmp';
3156     return "${var_run_tmpdir}/$vmid.$sockettype";
3157 }
3158
3159 sub pidfile_name {
3160     my ($vmid) = @_;
3161     return "${var_run_tmpdir}/$vmid.pid";
3162 }
3163
3164 sub vm_devices_list {
3165     my ($vmid) = @_;
3166
3167     my $res = vm_mon_cmd($vmid, 'query-pci');
3168     my $devices = {};
3169     foreach my $pcibus (@$res) {
3170         foreach my $device (@{$pcibus->{devices}}) {
3171             next if !$device->{'qdev_id'};
3172             if ($device->{'pci_bridge'}) {
3173                 $devices->{$device->{'qdev_id'}} = 1;
3174                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3175                     next if !$bridge_device->{'qdev_id'};
3176                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3177                     $devices->{$device->{'qdev_id'}}++;
3178                 }
3179             } else {
3180                 $devices->{$device->{'qdev_id'}} = 1;
3181             }
3182         }
3183     }
3184
3185     my $resblock = vm_mon_cmd($vmid, 'query-block');
3186     foreach my $block (@$resblock) {
3187         if($block->{device} =~ m/^drive-(\S+)/){
3188                 $devices->{$1} = 1;
3189         }
3190     }
3191
3192     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3193     foreach my $mice (@$resmice) {
3194         if ($mice->{name} eq 'QEMU HID Tablet') {
3195             $devices->{tablet} = 1;
3196             last;
3197         }
3198     }
3199
3200     return $devices;
3201 }
3202
3203 sub vm_deviceplug {
3204     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3205
3206     my $q35 = machine_type_is_q35($conf);
3207
3208     my $devices_list = vm_devices_list($vmid);
3209     return 1 if defined($devices_list->{$deviceid});
3210
3211     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3212
3213     if ($deviceid eq 'tablet') {
3214
3215         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3216
3217     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3218
3219         qemu_iothread_add($vmid, $deviceid, $device);
3220
3221         qemu_driveadd($storecfg, $vmid, $device);
3222         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3223
3224         qemu_deviceadd($vmid, $devicefull);
3225         eval { qemu_deviceaddverify($vmid, $deviceid); };
3226         if (my $err = $@) {
3227             eval { qemu_drivedel($vmid, $deviceid); };
3228             warn $@ if $@;
3229             die $err;
3230         }
3231
3232     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3233
3234
3235         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3236         my $pciaddr = print_pci_addr($deviceid);
3237         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3238
3239         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3240
3241         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3242             qemu_iothread_add($vmid, $deviceid, $device);
3243             $devicefull .= ",iothread=iothread-$deviceid";
3244         }
3245
3246         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3247             $devicefull .= ",num_queues=$device->{queues}";
3248         }
3249
3250         qemu_deviceadd($vmid, $devicefull);
3251         qemu_deviceaddverify($vmid, $deviceid);
3252
3253     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3254
3255         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3256         qemu_driveadd($storecfg, $vmid, $device);
3257
3258         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3259         eval { qemu_deviceadd($vmid, $devicefull); };
3260         if (my $err = $@) {
3261             eval { qemu_drivedel($vmid, $deviceid); };
3262             warn $@ if $@;
3263             die $err;
3264         }
3265
3266     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3267
3268         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3269         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid);
3270         qemu_deviceadd($vmid, $netdevicefull);
3271         eval { qemu_deviceaddverify($vmid, $deviceid); };
3272         if (my $err = $@) {
3273             eval { qemu_netdevdel($vmid, $deviceid); };
3274             warn $@ if $@;
3275             die $err;
3276         }
3277
3278     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3279
3280         my $bridgeid = $2;
3281         my $pciaddr = print_pci_addr($deviceid);
3282         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3283
3284         qemu_deviceadd($vmid, $devicefull);
3285         qemu_deviceaddverify($vmid, $deviceid);
3286
3287     } else {
3288         die "can't hotplug device '$deviceid'\n";
3289     }
3290
3291     return 1;
3292 }
3293
3294 # fixme: this should raise exceptions on error!
3295 sub vm_deviceunplug {
3296     my ($vmid, $conf, $deviceid) = @_;
3297
3298     my $devices_list = vm_devices_list($vmid);
3299     return 1 if !defined($devices_list->{$deviceid});
3300
3301     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3302
3303     if ($deviceid eq 'tablet') {
3304
3305         qemu_devicedel($vmid, $deviceid);
3306
3307     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3308
3309         qemu_devicedel($vmid, $deviceid);
3310         qemu_devicedelverify($vmid, $deviceid);
3311         qemu_drivedel($vmid, $deviceid);
3312         qemu_iothread_del($conf, $vmid, $deviceid);
3313
3314     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3315
3316         qemu_devicedel($vmid, $deviceid);
3317         qemu_devicedelverify($vmid, $deviceid);
3318         qemu_iothread_del($conf, $vmid, $deviceid);
3319
3320     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3321
3322         #qemu 2.3 segfault on drive_del with virtioscsi + iothread
3323         my $device = parse_drive($deviceid, $conf->{$deviceid});
3324         die "virtioscsi with iothread is not hot-unplugglable currently" if $device->{iothread};
3325
3326         qemu_devicedel($vmid, $deviceid);
3327         qemu_drivedel($vmid, $deviceid);
3328         qemu_deletescsihw($conf, $vmid, $deviceid);
3329
3330     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3331
3332         qemu_devicedel($vmid, $deviceid);
3333         qemu_devicedelverify($vmid, $deviceid);
3334         qemu_netdevdel($vmid, $deviceid);
3335
3336     } else {
3337         die "can't unplug device '$deviceid'\n";
3338     }
3339
3340     return 1;
3341 }
3342
3343 sub qemu_deviceadd {
3344     my ($vmid, $devicefull) = @_;
3345
3346     $devicefull = "driver=".$devicefull;
3347     my %options =  split(/[=,]/, $devicefull);
3348
3349     vm_mon_cmd($vmid, "device_add" , %options);
3350 }
3351
3352 sub qemu_devicedel {
3353     my ($vmid, $deviceid) = @_;
3354
3355     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3356 }
3357
3358 sub qemu_iothread_add {
3359     my($vmid, $deviceid, $device) = @_;
3360
3361     if ($device->{iothread}) {
3362         my $iothreads = vm_iothreads_list($vmid);
3363         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3364     }
3365 }
3366
3367 sub qemu_iothread_del {
3368     my($conf, $vmid, $deviceid) = @_;
3369
3370     my $device = parse_drive($deviceid, $conf->{$deviceid});
3371     if ($device->{iothread}) {
3372         my $iothreads = vm_iothreads_list($vmid);
3373         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3374     }
3375 }
3376
3377 sub qemu_objectadd {
3378     my($vmid, $objectid, $qomtype) = @_;
3379
3380     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3381
3382     return 1;
3383 }
3384
3385 sub qemu_objectdel {
3386     my($vmid, $objectid) = @_;
3387
3388     vm_mon_cmd($vmid, "object-del", id => $objectid);
3389
3390     return 1;
3391 }
3392
3393 sub qemu_driveadd {
3394     my ($storecfg, $vmid, $device) = @_;
3395
3396     my $drive = print_drive_full($storecfg, $vmid, $device);
3397     $drive =~ s/\\/\\\\/g;
3398     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3399
3400     # If the command succeeds qemu prints: "OK"
3401     return 1 if $ret =~ m/OK/s;
3402
3403     die "adding drive failed: $ret\n";
3404 }
3405
3406 sub qemu_drivedel {
3407     my($vmid, $deviceid) = @_;
3408
3409     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3410     $ret =~ s/^\s+//;
3411
3412     return 1 if $ret eq "";
3413
3414     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3415     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3416
3417     die "deleting drive $deviceid failed : $ret\n";
3418 }
3419
3420 sub qemu_deviceaddverify {
3421     my ($vmid, $deviceid) = @_;
3422
3423     for (my $i = 0; $i <= 5; $i++) {
3424          my $devices_list = vm_devices_list($vmid);
3425          return 1 if defined($devices_list->{$deviceid});
3426          sleep 1;
3427     }
3428
3429     die "error on hotplug device '$deviceid'\n";
3430 }
3431
3432
3433 sub qemu_devicedelverify {
3434     my ($vmid, $deviceid) = @_;
3435
3436     # need to verify that the device is correctly removed as device_del
3437     # is async and empty return is not reliable
3438
3439     for (my $i = 0; $i <= 5; $i++) {
3440          my $devices_list = vm_devices_list($vmid);
3441          return 1 if !defined($devices_list->{$deviceid});
3442          sleep 1;
3443     }
3444
3445     die "error on hot-unplugging device '$deviceid'\n";
3446 }
3447
3448 sub qemu_findorcreatescsihw {
3449     my ($storecfg, $conf, $vmid, $device) = @_;
3450
3451     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3452
3453     my $scsihwid="$controller_prefix$controller";
3454     my $devices_list = vm_devices_list($vmid);
3455
3456     if(!defined($devices_list->{$scsihwid})) {
3457         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3458     }
3459
3460     return 1;
3461 }
3462
3463 sub qemu_deletescsihw {
3464     my ($conf, $vmid, $opt) = @_;
3465
3466     my $device = parse_drive($opt, $conf->{$opt});
3467
3468     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3469         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3470         return 1;
3471     }
3472
3473     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3474
3475     my $devices_list = vm_devices_list($vmid);
3476     foreach my $opt (keys %{$devices_list}) {
3477         if (PVE::QemuServer::valid_drivename($opt)) {
3478             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3479             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3480                 return 1;
3481             }
3482         }
3483     }
3484
3485     my $scsihwid="scsihw$controller";
3486
3487     vm_deviceunplug($vmid, $conf, $scsihwid);
3488
3489     return 1;
3490 }
3491
3492 sub qemu_add_pci_bridge {
3493     my ($storecfg, $conf, $vmid, $device) = @_;
3494
3495     my $bridges = {};
3496
3497     my $bridgeid;
3498
3499     print_pci_addr($device, $bridges);
3500
3501     while (my ($k, $v) = each %$bridges) {
3502         $bridgeid = $k;
3503     }
3504     return 1 if !defined($bridgeid) || $bridgeid < 1;
3505
3506     my $bridge = "pci.$bridgeid";
3507     my $devices_list = vm_devices_list($vmid);
3508
3509     if (!defined($devices_list->{$bridge})) {
3510         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3511     }
3512
3513     return 1;
3514 }
3515
3516 sub qemu_set_link_status {
3517     my ($vmid, $device, $up) = @_;
3518
3519     vm_mon_cmd($vmid, "set_link", name => $device,
3520                up => $up ? JSON::true : JSON::false);
3521 }
3522
3523 sub qemu_netdevadd {
3524     my ($vmid, $conf, $device, $deviceid) = @_;
3525
3526     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid);
3527     my %options =  split(/[=,]/, $netdev);
3528
3529     vm_mon_cmd($vmid, "netdev_add",  %options);
3530     return 1;
3531 }
3532
3533 sub qemu_netdevdel {
3534     my ($vmid, $deviceid) = @_;
3535
3536     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3537 }
3538
3539 sub qemu_cpu_hotplug {
3540     my ($vmid, $conf, $vcpus) = @_;
3541
3542     my $sockets = 1;
3543     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3544     $sockets = $conf->{sockets} if  $conf->{sockets};
3545     my $cores = $conf->{cores} || 1;
3546     my $maxcpus = $sockets * $cores;
3547
3548     $vcpus = $maxcpus if !$vcpus;
3549
3550     die "you can't add more vcpus than maxcpus\n"
3551         if $vcpus > $maxcpus;
3552
3553     my $currentvcpus = $conf->{vcpus} || $maxcpus;
3554     die "online cpu unplug is not yet possible\n"
3555         if $vcpus < $currentvcpus;
3556
3557     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3558     die "vcpus in running vm is different than configuration\n"
3559         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
3560
3561     for (my $i = $currentvcpus; $i < $vcpus; $i++) {
3562         vm_mon_cmd($vmid, "cpu-add", id => int($i));
3563     }
3564 }
3565
3566 sub qemu_memory_hotplug {
3567     my ($vmid, $conf, $defaults, $opt, $value) = @_;
3568
3569     return $value if !check_running($vmid);
3570
3571     my $memory = $conf->{memory} || $defaults->{memory};
3572     $value = $defaults->{memory} if !$value;
3573     return $value if $value == $memory;
3574
3575     my $static_memory = $STATICMEM;
3576     my $dimm_memory = $memory - $static_memory;
3577
3578     die "memory can't be lower than $static_memory MB" if $value < $static_memory;
3579     die "memory unplug is not yet available" if $value < $memory;
3580     die "you cannot add more memory than $MAX_MEM MB!\n" if $memory > $MAX_MEM;
3581
3582
3583     my $sockets = 1;
3584     $sockets = $conf->{sockets} if $conf->{sockets};
3585
3586     foreach_dimm($conf, $vmid, $value, $sockets, sub {
3587         my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3588
3589             return if $current_size <= $conf->{memory};
3590
3591             eval { vm_mon_cmd($vmid, "object-add", 'qom-type' => "memory-backend-ram", id => "mem-$name", props => { size => int($dimm_size*1024*1024) } ) };
3592             if (my $err = $@) {
3593                 eval { qemu_objectdel($vmid, "mem-$name"); };
3594                 die $err;
3595             }
3596
3597             eval { vm_mon_cmd($vmid, "device_add", driver => "pc-dimm", id => "$name", memdev => "mem-$name", node => $numanode) };
3598             if (my $err = $@) {
3599                 eval { qemu_objectdel($vmid, "mem-$name"); };
3600                 die $err;
3601             }
3602             #update conf after each succesful module hotplug
3603             $conf->{memory} = $current_size;
3604             update_config_nolock($vmid, $conf, 1);
3605     });
3606 }
3607
3608 sub qemu_block_set_io_throttle {
3609     my ($vmid, $deviceid, $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr) = @_;
3610
3611     return if !check_running($vmid) ;
3612
3613     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid, bps => int($bps), bps_rd => int($bps_rd), bps_wr => int($bps_wr), iops => int($iops), iops_rd => int($iops_rd), iops_wr => int($iops_wr));
3614
3615 }
3616
3617 # old code, only used to shutdown old VM after update
3618 sub __read_avail {
3619     my ($fh, $timeout) = @_;
3620
3621     my $sel = new IO::Select;
3622     $sel->add($fh);
3623
3624     my $res = '';
3625     my $buf;
3626
3627     my @ready;
3628     while (scalar (@ready = $sel->can_read($timeout))) {
3629         my $count;
3630         if ($count = $fh->sysread($buf, 8192)) {
3631             if ($buf =~ /^(.*)\(qemu\) $/s) {
3632                 $res .= $1;
3633                 last;
3634             } else {
3635                 $res .= $buf;
3636             }
3637         } else {
3638             if (!defined($count)) {
3639                 die "$!\n";
3640             }
3641             last;
3642         }
3643     }
3644
3645     die "monitor read timeout\n" if !scalar(@ready);
3646
3647     return $res;
3648 }
3649
3650 # old code, only used to shutdown old VM after update
3651 sub vm_monitor_command {
3652     my ($vmid, $cmdstr, $nocheck) = @_;
3653
3654     my $res;
3655
3656     eval {
3657         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
3658
3659         my $sname = "${var_run_tmpdir}/$vmid.mon";
3660
3661         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
3662             die "unable to connect to VM $vmid socket - $!\n";
3663
3664         my $timeout = 3;
3665
3666         # hack: migrate sometime blocks the monitor (when migrate_downtime
3667         # is set)
3668         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3669             $timeout = 60*60; # 1 hour
3670         }
3671
3672         # read banner;
3673         my $data = __read_avail($sock, $timeout);
3674
3675         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
3676             die "got unexpected qemu monitor banner\n";
3677         }
3678
3679         my $sel = new IO::Select;
3680         $sel->add($sock);
3681
3682         if (!scalar(my @ready = $sel->can_write($timeout))) {
3683             die "monitor write error - timeout";
3684         }
3685
3686         my $fullcmd = "$cmdstr\r";
3687
3688         # syslog('info', "VM $vmid monitor command: $cmdstr");
3689
3690         my $b;
3691         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
3692             die "monitor write error - $!";
3693         }
3694
3695         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
3696
3697         $timeout = 20;
3698
3699         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3700             $timeout = 60*60; # 1 hour
3701         } elsif ($cmdstr =~ m/^(eject|change)/) {
3702             $timeout = 60; # note: cdrom mount command is slow
3703         }
3704         if ($res = __read_avail($sock, $timeout)) {
3705
3706             my @lines = split("\r?\n", $res);
3707
3708             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
3709
3710             $res = join("\n", @lines);
3711             $res .= "\n";
3712         }
3713     };
3714
3715     my $err = $@;
3716
3717     if ($err) {
3718         syslog("err", "VM $vmid monitor command failed - $err");
3719         die $err;
3720     }
3721
3722     return $res;
3723 }
3724
3725 sub qemu_block_resize {
3726     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
3727
3728     my $running = check_running($vmid);
3729
3730     return if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
3731
3732     return if !$running;
3733
3734     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
3735
3736 }
3737
3738 sub qemu_volume_snapshot {
3739     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3740
3741     my $running = check_running($vmid);
3742
3743     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
3744         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
3745     } else {
3746         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
3747     }
3748 }
3749
3750 sub qemu_volume_snapshot_delete {
3751     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3752
3753     my $running = check_running($vmid);
3754
3755     return if !PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
3756
3757     return if !$running;
3758
3759     vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
3760 }
3761
3762 sub set_migration_caps {
3763     my ($vmid) = @_;
3764
3765     my $cap_ref = [];
3766
3767     my $enabled_cap = {
3768         "auto-converge" => 1,
3769         "xbzrle" => 0,
3770         "x-rdma-pin-all" => 0,
3771         "zero-blocks" => 0,
3772     };
3773
3774     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
3775
3776     for my $supported_capability (@$supported_capabilities) {
3777         push @$cap_ref, {
3778             capability => $supported_capability->{capability},
3779             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
3780         };
3781     }
3782
3783     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
3784 }
3785
3786 my $fast_plug_option = {
3787     'lock' => 1,
3788     'name' => 1,
3789     'onboot' => 1,
3790     'shares' => 1,
3791     'startup' => 1,
3792 };
3793
3794 # hotplug changes in [PENDING]
3795 # $selection hash can be used to only apply specified options, for
3796 # example: { cores => 1 } (only apply changed 'cores')
3797 # $errors ref is used to return error messages
3798 sub vmconfig_hotplug_pending {
3799     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
3800
3801     my $defaults = load_defaults();
3802
3803     # commit values which do not have any impact on running VM first
3804     # Note: those option cannot raise errors, we we do not care about
3805     # $selection and always apply them.
3806
3807     my $add_error = sub {
3808         my ($opt, $msg) = @_;
3809         $errors->{$opt} = "hotplug problem - $msg";
3810     };
3811
3812     my $changes = 0;
3813     foreach my $opt (keys %{$conf->{pending}}) { # add/change
3814         if ($fast_plug_option->{$opt}) {
3815             $conf->{$opt} = $conf->{pending}->{$opt};
3816             delete $conf->{pending}->{$opt};
3817             $changes = 1;
3818         }
3819     }
3820
3821     if ($changes) {
3822         update_config_nolock($vmid, $conf, 1);
3823         $conf = load_config($vmid); # update/reload
3824     }
3825
3826     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3827
3828     my @delete = PVE::Tools::split_list($conf->{pending}->{delete});
3829     foreach my $opt (@delete) {
3830         next if $selection && !$selection->{$opt};
3831         eval {
3832             if ($opt eq 'hotplug') {
3833                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
3834             } elsif ($opt eq 'tablet') {
3835                 die "skip\n" if !$hotplug_features->{usb};
3836                 if ($defaults->{tablet}) {
3837                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
3838                 } else {
3839                     vm_deviceunplug($vmid, $conf, $opt);
3840                 }
3841             } elsif ($opt eq 'vcpus') {
3842                 die "skip\n" if !$hotplug_features->{cpu};
3843                 qemu_cpu_hotplug($vmid, $conf, undef);
3844             } elsif ($opt eq 'balloon') {
3845                 # enable balloon device is not hotpluggable
3846                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
3847             } elsif ($fast_plug_option->{$opt}) {
3848                 # do nothing
3849             } elsif ($opt =~ m/^net(\d+)$/) {
3850                 die "skip\n" if !$hotplug_features->{network};
3851                 vm_deviceunplug($vmid, $conf, $opt);
3852             } elsif (valid_drivename($opt)) {
3853                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
3854                 vm_deviceunplug($vmid, $conf, $opt);
3855                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}));
3856             } elsif ($opt =~ m/^memory$/) {
3857                 die "skip\n" if !$hotplug_features->{memory};
3858                 qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
3859             } elsif ($opt eq 'cpuunits') {
3860                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
3861             } elsif ($opt eq 'cpulimit') {
3862                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
3863             } else {
3864                 die "skip\n";
3865             }
3866         };
3867         if (my $err = $@) {
3868             &$add_error($opt, $err) if $err ne "skip\n";
3869         } else {
3870             # save new config if hotplug was successful
3871             delete $conf->{$opt};
3872             vmconfig_undelete_pending_option($conf, $opt);
3873             update_config_nolock($vmid, $conf, 1);
3874             $conf = load_config($vmid); # update/reload
3875         }
3876     }
3877
3878     foreach my $opt (keys %{$conf->{pending}}) {
3879         next if $selection && !$selection->{$opt};
3880         my $value = $conf->{pending}->{$opt};
3881         eval {
3882             if ($opt eq 'hotplug') {
3883                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
3884             } elsif ($opt eq 'tablet') {
3885                 die "skip\n" if !$hotplug_features->{usb};
3886                 if ($value == 1) {
3887                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
3888                 } elsif ($value == 0) {
3889                     vm_deviceunplug($vmid, $conf, $opt);
3890                 }
3891             } elsif ($opt eq 'vcpus') {
3892                 die "skip\n" if !$hotplug_features->{cpu};
3893                 qemu_cpu_hotplug($vmid, $conf, $value);
3894             } elsif ($opt eq 'balloon') {
3895                 # enable/disable balloning device is not hotpluggable
3896                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
3897                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
3898                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
3899
3900                 # allow manual ballooning if shares is set to zero
3901                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
3902                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
3903                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
3904                 }
3905             } elsif ($opt =~ m/^net(\d+)$/) {
3906                 # some changes can be done without hotplug
3907                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
3908                                     $vmid, $opt, $value);
3909             } elsif (valid_drivename($opt)) {
3910                 # some changes can be done without hotplug
3911                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
3912                                      $vmid, $opt, $value, 1);
3913             } elsif ($opt =~ m/^memory$/) { #dimms
3914                 die "skip\n" if !$hotplug_features->{memory};
3915                 $value = qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
3916             } elsif ($opt eq 'cpuunits') {
3917                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
3918             } elsif ($opt eq 'cpulimit') {
3919                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
3920                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
3921             } else {
3922                 die "skip\n";  # skip non-hot-pluggable options
3923             }
3924         };
3925         if (my $err = $@) {
3926             &$add_error($opt, $err) if $err ne "skip\n";
3927         } else {
3928             # save new config if hotplug was successful
3929             $conf->{$opt} = $value;
3930             delete $conf->{pending}->{$opt};
3931             update_config_nolock($vmid, $conf, 1);
3932             $conf = load_config($vmid); # update/reload
3933         }
3934     }
3935 }
3936
3937 sub vmconfig_apply_pending {
3938     my ($vmid, $conf, $storecfg) = @_;
3939
3940     # cold plug
3941
3942     my @delete = PVE::Tools::split_list($conf->{pending}->{delete});
3943     foreach my $opt (@delete) { # delete
3944         die "internal error" if $opt =~ m/^unused/;
3945         $conf = load_config($vmid); # update/reload
3946         if (!defined($conf->{$opt})) {
3947             vmconfig_undelete_pending_option($conf, $opt);
3948             update_config_nolock($vmid, $conf, 1);
3949         } elsif (valid_drivename($opt)) {
3950             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}));
3951             vmconfig_undelete_pending_option($conf, $opt);
3952             delete $conf->{$opt};
3953             update_config_nolock($vmid, $conf, 1);
3954         } else {
3955             vmconfig_undelete_pending_option($conf, $opt);
3956             delete $conf->{$opt};
3957             update_config_nolock($vmid, $conf, 1);
3958         }
3959     }
3960
3961     $conf = load_config($vmid); # update/reload
3962
3963     foreach my $opt (keys %{$conf->{pending}}) { # add/change
3964         $conf = load_config($vmid); # update/reload
3965
3966         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
3967             # skip if nothing changed
3968         } elsif (valid_drivename($opt)) {
3969             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
3970                 if defined($conf->{$opt});
3971             $conf->{$opt} = $conf->{pending}->{$opt};
3972         } else {
3973             $conf->{$opt} = $conf->{pending}->{$opt};
3974         }
3975
3976         delete $conf->{pending}->{$opt};
3977         update_config_nolock($vmid, $conf, 1);
3978     }
3979 }
3980
3981 my $safe_num_ne = sub {
3982     my ($a, $b) = @_;
3983
3984     return 0 if !defined($a) && !defined($b);
3985     return 1 if !defined($a);
3986     return 1 if !defined($b);
3987
3988     return $a != $b;
3989 };
3990
3991 my $safe_string_ne = sub {
3992     my ($a, $b) = @_;
3993
3994     return 0 if !defined($a) && !defined($b);
3995     return 1 if !defined($a);
3996     return 1 if !defined($b);
3997
3998     return $a ne $b;
3999 };
4000
4001 sub vmconfig_update_net {
4002     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4003
4004     my $newnet = parse_net($value);
4005
4006     if ($conf->{$opt}) {
4007         my $oldnet = parse_net($conf->{$opt});
4008
4009         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4010             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4011             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4012             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4013
4014             # for non online change, we try to hot-unplug
4015             die "skip\n" if !$hotplug;
4016             vm_deviceunplug($vmid, $conf, $opt);
4017         } else {
4018
4019             die "internal error" if $opt !~ m/net(\d+)/;
4020             my $iface = "tap${vmid}i$1";
4021
4022             if (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4023                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4024             }
4025
4026             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4027                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4028                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4029                 PVE::Network::tap_unplug($iface);
4030                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall});
4031             }
4032
4033             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4034                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4035             }
4036
4037             return 1;
4038         }
4039     }
4040
4041     if ($hotplug) {
4042         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4043     } else {
4044         die "skip\n";
4045     }
4046 }
4047
4048 sub vmconfig_update_disk {
4049     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4050
4051     # fixme: do we need force?
4052
4053     my $drive = parse_drive($opt, $value);
4054
4055     if ($conf->{$opt}) {
4056
4057         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4058
4059             my $media = $drive->{media} || 'disk';
4060             my $oldmedia = $old_drive->{media} || 'disk';
4061             die "unable to change media type\n" if $media ne $oldmedia;
4062
4063             if (!drive_is_cdrom($old_drive)) {
4064
4065                 if ($drive->{file} ne $old_drive->{file}) {
4066
4067                     die "skip\n" if !$hotplug;
4068
4069                     # unplug and register as unused
4070                     vm_deviceunplug($vmid, $conf, $opt);
4071                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4072
4073                 } else {
4074                     # update existing disk
4075
4076                     # skip non hotpluggable value
4077                     if (&$safe_num_ne($drive->{discard}, $old_drive->{discard}) ||
4078                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4079                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4080                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4081                         die "skip\n";
4082                     }
4083
4084                     # apply throttle
4085                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4086                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4087                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4088                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4089                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4090                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4091                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4092                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4093                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4094                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4095                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4096                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max})) {
4097
4098                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4099                                                    ($drive->{mbps} || 0)*1024*1024,
4100                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4101                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4102                                                    $drive->{iops} || 0,
4103                                                    $drive->{iops_rd} || 0,
4104                                                    $drive->{iops_wr} || 0,
4105                                                    ($drive->{mbps_max} || 0)*1024*1024,
4106                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4107                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4108                                                    $drive->{iops_max} || 0,
4109                                                    $drive->{iops_rd_max} || 0,
4110                                                    $drive->{iops_wr_max} || 0);
4111
4112                     }
4113
4114                     return 1;
4115                 }
4116
4117             } else { # cdrom
4118
4119                 if ($drive->{file} eq 'none') {
4120                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4121                 } else {
4122                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4123                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4124                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4125                 }
4126
4127                 return 1;
4128             }
4129         }
4130     }
4131
4132     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4133     # hotplug new disks
4134     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4135 }
4136
4137 sub vm_start {
4138     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused, $forcemachine, $spice_ticket) = @_;
4139
4140     lock_config($vmid, sub {
4141         my $conf = load_config($vmid, $migratedfrom);
4142
4143         die "you can't start a vm if it's a template\n" if is_template($conf);
4144
4145         check_lock($conf) if !$skiplock;
4146
4147         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4148
4149         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4150             vmconfig_apply_pending($vmid, $conf, $storecfg);
4151             $conf = load_config($vmid); # update/reload
4152         }
4153
4154         my $defaults = load_defaults();
4155
4156         # set environment variable useful inside network script
4157         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4158
4159         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4160
4161         my $migrate_port = 0;
4162         my $migrate_uri;
4163         if ($statefile) {
4164             if ($statefile eq 'tcp') {
4165                 my $localip = "localhost";
4166                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4167                 my $nodename = PVE::INotify::nodename();
4168                 if ($datacenterconf->{migration_unsecure}) {
4169                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4170                 }
4171                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4172                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4173                 $migrate_uri = "tcp:[${localip}]:${migrate_port}";
4174                 push @$cmd, '-incoming', $migrate_uri;
4175                 push @$cmd, '-S';
4176             } else {
4177                 push @$cmd, '-loadstate', $statefile;
4178             }
4179         } elsif ($paused) {
4180             push @$cmd, '-S';
4181         }
4182
4183         # host pci devices
4184         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4185           my $d = parse_hostpci($conf->{"hostpci$i"});
4186           next if !$d;
4187           my $pcidevices = $d->{pciid};
4188           foreach my $pcidevice (@$pcidevices) {
4189                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4190
4191                 my $info = pci_device_info("0000:$pciid");
4192                 die "IOMMU not present\n" if !check_iommu_support();
4193                 die "no pci device info for device '$pciid'\n" if !$info;
4194
4195                 if ($d->{driver} && $d->{driver} eq "vfio") {
4196                     die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4197                 } else {
4198                     die "can't unbind/bind to stub pci device '$pciid'\n" if !pci_dev_bind_to_stub($info);
4199                 }
4200
4201                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4202           }
4203         }
4204
4205         PVE::Storage::activate_volumes($storecfg, $vollist);
4206
4207         eval  { run_command($cmd, timeout => $statefile ? undef : 30,
4208                     umask => 0077); };
4209         my $err = $@;
4210         die "start failed: $err" if $err;
4211
4212         print "migration listens on $migrate_uri\n" if $migrate_uri;
4213
4214         if ($statefile && $statefile ne 'tcp')  {
4215             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4216             warn $@ if $@;
4217         }
4218
4219         if ($migratedfrom) {
4220
4221             eval {
4222                 set_migration_caps($vmid);
4223             };
4224             warn $@ if $@;
4225
4226             if ($spice_port) {
4227                 print "spice listens on port $spice_port\n";
4228                 if ($spice_ticket) {
4229                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4230                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4231                 }
4232             }
4233
4234         } else {
4235
4236             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4237                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4238                     if $conf->{balloon};
4239             }
4240
4241             foreach my $opt (keys %$conf) {
4242                 next if $opt !~  m/^net\d+$/;
4243                 my $nicconf = parse_net($conf->{$opt});
4244                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4245             }
4246         }
4247
4248         vm_mon_cmd_nocheck($vmid, 'qom-set',
4249                     path => "machine/peripheral/balloon0",
4250                     property => "guest-stats-polling-interval",
4251                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4252
4253     });
4254 }
4255
4256 sub vm_mon_cmd {
4257     my ($vmid, $execute, %params) = @_;
4258
4259     my $cmd = { execute => $execute, arguments => \%params };
4260     vm_qmp_command($vmid, $cmd);
4261 }
4262
4263 sub vm_mon_cmd_nocheck {
4264     my ($vmid, $execute, %params) = @_;
4265
4266     my $cmd = { execute => $execute, arguments => \%params };
4267     vm_qmp_command($vmid, $cmd, 1);
4268 }
4269
4270 sub vm_qmp_command {
4271     my ($vmid, $cmd, $nocheck) = @_;
4272
4273     my $res;
4274
4275     my $timeout;
4276     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4277         $timeout = $cmd->{arguments}->{timeout};
4278         delete $cmd->{arguments}->{timeout};
4279     }
4280
4281     eval {
4282         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4283         my $sname = qmp_socket($vmid);
4284         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4285             my $qmpclient = PVE::QMPClient->new();
4286
4287             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4288         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
4289             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
4290                 if scalar(%{$cmd->{arguments}});
4291             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
4292         } else {
4293             die "unable to open monitor socket\n";
4294         }
4295     };
4296     if (my $err = $@) {
4297         syslog("err", "VM $vmid qmp command failed - $err");
4298         die $err;
4299     }
4300
4301     return $res;
4302 }
4303
4304 sub vm_human_monitor_command {
4305     my ($vmid, $cmdline) = @_;
4306
4307     my $res;
4308
4309     my $cmd = {
4310         execute => 'human-monitor-command',
4311         arguments => { 'command-line' => $cmdline},
4312     };
4313
4314     return vm_qmp_command($vmid, $cmd);
4315 }
4316
4317 sub vm_commandline {
4318     my ($storecfg, $vmid) = @_;
4319
4320     my $conf = load_config($vmid);
4321
4322     my $defaults = load_defaults();
4323
4324     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
4325
4326     return join(' ', @$cmd);
4327 }
4328
4329 sub vm_reset {
4330     my ($vmid, $skiplock) = @_;
4331
4332     lock_config($vmid, sub {
4333
4334         my $conf = load_config($vmid);
4335
4336         check_lock($conf) if !$skiplock;
4337
4338         vm_mon_cmd($vmid, "system_reset");
4339     });
4340 }
4341
4342 sub get_vm_volumes {
4343     my ($conf) = @_;
4344
4345     my $vollist = [];
4346     foreach_volid($conf, sub {
4347         my ($volid, $is_cdrom) = @_;
4348
4349         return if $volid =~ m|^/|;
4350
4351         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4352         return if !$sid;
4353
4354         push @$vollist, $volid;
4355     });
4356
4357     return $vollist;
4358 }
4359
4360 sub vm_stop_cleanup {
4361     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
4362
4363     eval {
4364
4365         if (!$keepActive) {
4366             my $vollist = get_vm_volumes($conf);
4367             PVE::Storage::deactivate_volumes($storecfg, $vollist);
4368         }
4369
4370         foreach my $ext (qw(mon qmp pid vnc qga)) {
4371             unlink "/var/run/qemu-server/${vmid}.$ext";
4372         }
4373
4374         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
4375     };
4376     warn $@ if $@; # avoid errors - just warn
4377 }
4378
4379 # Note: use $nockeck to skip tests if VM configuration file exists.
4380 # We need that when migration VMs to other nodes (files already moved)
4381 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
4382 sub vm_stop {
4383     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
4384
4385     $force = 1 if !defined($force) && !$shutdown;
4386
4387     if ($migratedfrom){
4388         my $pid = check_running($vmid, $nocheck, $migratedfrom);
4389         kill 15, $pid if $pid;
4390         my $conf = load_config($vmid, $migratedfrom);
4391         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
4392         return;
4393     }
4394
4395     lock_config($vmid, sub {
4396
4397         my $pid = check_running($vmid, $nocheck);
4398         return if !$pid;
4399
4400         my $conf;
4401         if (!$nocheck) {
4402             $conf = load_config($vmid);
4403             check_lock($conf) if !$skiplock;
4404             if (!defined($timeout) && $shutdown && $conf->{startup}) {
4405                 my $opts = parse_startup($conf->{startup});
4406                 $timeout = $opts->{down} if $opts->{down};
4407             }
4408         }
4409
4410         $timeout = 60 if !defined($timeout);
4411
4412         eval {
4413             if ($shutdown) {
4414                 if (defined($conf) && $conf->{agent}) {
4415                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
4416                 } else {
4417                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
4418                 }
4419             } else {
4420                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
4421             }
4422         };
4423         my $err = $@;
4424
4425         if (!$err) {
4426             my $count = 0;
4427             while (($count < $timeout) && check_running($vmid, $nocheck)) {
4428                 $count++;
4429                 sleep 1;
4430             }
4431
4432             if ($count >= $timeout) {
4433                 if ($force) {
4434                     warn "VM still running - terminating now with SIGTERM\n";
4435                     kill 15, $pid;
4436                 } else {
4437                     die "VM quit/powerdown failed - got timeout\n";
4438                 }
4439             } else {
4440                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4441                 return;
4442             }
4443         } else {
4444             if ($force) {
4445                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
4446                 kill 15, $pid;
4447             } else {
4448                 die "VM quit/powerdown failed\n";
4449             }
4450         }
4451
4452         # wait again
4453         $timeout = 10;
4454
4455         my $count = 0;
4456         while (($count < $timeout) && check_running($vmid, $nocheck)) {
4457             $count++;
4458             sleep 1;
4459         }
4460
4461         if ($count >= $timeout) {
4462             warn "VM still running - terminating now with SIGKILL\n";
4463             kill 9, $pid;
4464             sleep 1;
4465         }
4466
4467         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4468    });
4469 }
4470
4471 sub vm_suspend {
4472     my ($vmid, $skiplock) = @_;
4473
4474     lock_config($vmid, sub {
4475
4476         my $conf = load_config($vmid);
4477
4478         check_lock($conf) if !($skiplock || ($conf->{lock} && $conf->{lock} eq 'backup'));
4479
4480         vm_mon_cmd($vmid, "stop");
4481     });
4482 }
4483
4484 sub vm_resume {
4485     my ($vmid, $skiplock) = @_;
4486
4487     lock_config($vmid, sub {
4488
4489         my $conf = load_config($vmid);
4490
4491         check_lock($conf) if !($skiplock || ($conf->{lock} && $conf->{lock} eq 'backup'));
4492
4493         vm_mon_cmd($vmid, "cont");
4494     });
4495 }
4496
4497 sub vm_sendkey {
4498     my ($vmid, $skiplock, $key) = @_;
4499
4500     lock_config($vmid, sub {
4501
4502         my $conf = load_config($vmid);
4503
4504         # there is no qmp command, so we use the human monitor command
4505         vm_human_monitor_command($vmid, "sendkey $key");
4506     });
4507 }
4508
4509 sub vm_destroy {
4510     my ($storecfg, $vmid, $skiplock) = @_;
4511
4512     lock_config($vmid, sub {
4513
4514         my $conf = load_config($vmid);
4515
4516         check_lock($conf) if !$skiplock;
4517
4518         if (!check_running($vmid)) {
4519             destroy_vm($storecfg, $vmid);
4520         } else {
4521             die "VM $vmid is running - destroy failed\n";
4522         }
4523     });
4524 }
4525
4526 # pci helpers
4527
4528 sub file_write {
4529     my ($filename, $buf) = @_;
4530
4531     my $fh = IO::File->new($filename, "w");
4532     return undef if !$fh;
4533
4534     my $res = print $fh $buf;
4535
4536     $fh->close();
4537
4538     return $res;
4539 }
4540
4541 sub pci_device_info {
4542     my ($name) = @_;
4543
4544     my $res;
4545
4546     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
4547     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
4548
4549     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
4550     return undef if !defined($irq) || $irq !~ m/^\d+$/;
4551
4552     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
4553     return undef if !defined($vendor) || $vendor !~ s/^0x//;
4554
4555     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
4556     return undef if !defined($product) || $product !~ s/^0x//;
4557
4558     $res = {
4559         name => $name,
4560         vendor => $vendor,
4561         product => $product,
4562         domain => $domain,
4563         bus => $bus,
4564         slot => $slot,
4565         func => $func,
4566         irq => $irq,
4567         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
4568     };
4569
4570     return $res;
4571 }
4572
4573 sub pci_dev_reset {
4574     my ($dev) = @_;
4575
4576     my $name = $dev->{name};
4577
4578     my $fn = "$pcisysfs/devices/$name/reset";
4579
4580     return file_write($fn, "1");
4581 }
4582
4583 sub pci_dev_bind_to_stub {
4584     my ($dev) = @_;
4585
4586     my $name = $dev->{name};
4587
4588     my $testdir = "$pcisysfs/drivers/pci-stub/$name";
4589     return 1 if -d $testdir;
4590
4591     my $data = "$dev->{vendor} $dev->{product}";
4592     return undef if !file_write("$pcisysfs/drivers/pci-stub/new_id", $data);
4593
4594     my $fn = "$pcisysfs/devices/$name/driver/unbind";
4595     if (!file_write($fn, $name)) {
4596         return undef if -f $fn;
4597     }
4598
4599     $fn = "$pcisysfs/drivers/pci-stub/bind";
4600     if (! -d $testdir) {
4601         return undef if !file_write($fn, $name);
4602     }
4603
4604     return -d $testdir;
4605 }
4606
4607 sub pci_dev_bind_to_vfio {
4608     my ($dev) = @_;
4609
4610     my $name = $dev->{name};
4611
4612     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4613
4614     if (!-d $vfio_basedir) {
4615         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4616     }
4617     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4618
4619     my $testdir = "$vfio_basedir/$name";
4620     return 1 if -d $testdir;
4621
4622     my $data = "$dev->{vendor} $dev->{product}";
4623     return undef if !file_write("$vfio_basedir/new_id", $data);
4624
4625     my $fn = "$pcisysfs/devices/$name/driver/unbind";
4626     if (!file_write($fn, $name)) {
4627         return undef if -f $fn;
4628     }
4629
4630     $fn = "$vfio_basedir/bind";
4631     if (! -d $testdir) {
4632         return undef if !file_write($fn, $name);
4633     }
4634
4635     return -d $testdir;
4636 }
4637
4638 sub pci_dev_group_bind_to_vfio {
4639     my ($pciid) = @_;
4640
4641     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4642
4643     if (!-d $vfio_basedir) {
4644         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4645     }
4646     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4647
4648     # get IOMMU group devices
4649     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
4650       my @devs = grep /^0000:/, readdir($D);
4651     closedir($D);
4652
4653     foreach my $pciid (@devs) {
4654         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
4655
4656         # pci bridges, switches or root ports are not supported
4657         # they have a pci_bus subdirectory so skip them
4658         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
4659
4660         my $info = pci_device_info($1);
4661         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
4662     }
4663
4664     return 1;
4665 }
4666
4667 sub print_pci_addr {
4668     my ($id, $bridges) = @_;
4669
4670     my $res = '';
4671     my $devices = {
4672         piix3 => { bus => 0, addr => 1 },
4673         #addr2 : first videocard
4674         balloon0 => { bus => 0, addr => 3 },
4675         watchdog => { bus => 0, addr => 4 },
4676         scsihw0 => { bus => 0, addr => 5 },
4677         'pci.3' => { bus => 0, addr => 5 }, #can also be used for virtio-scsi-single bridge
4678         scsihw1 => { bus => 0, addr => 6 },
4679         ahci0 => { bus => 0, addr => 7 },
4680         qga0 => { bus => 0, addr => 8 },
4681         spice => { bus => 0, addr => 9 },
4682         virtio0 => { bus => 0, addr => 10 },
4683         virtio1 => { bus => 0, addr => 11 },
4684         virtio2 => { bus => 0, addr => 12 },
4685         virtio3 => { bus => 0, addr => 13 },
4686         virtio4 => { bus => 0, addr => 14 },
4687         virtio5 => { bus => 0, addr => 15 },
4688         hostpci0 => { bus => 0, addr => 16 },
4689         hostpci1 => { bus => 0, addr => 17 },
4690         net0 => { bus => 0, addr => 18 },
4691         net1 => { bus => 0, addr => 19 },
4692         net2 => { bus => 0, addr => 20 },
4693         net3 => { bus => 0, addr => 21 },
4694         net4 => { bus => 0, addr => 22 },
4695         net5 => { bus => 0, addr => 23 },
4696         vga1 => { bus => 0, addr => 24 },
4697         vga2 => { bus => 0, addr => 25 },
4698         vga3 => { bus => 0, addr => 26 },
4699         hostpci2 => { bus => 0, addr => 27 },
4700         hostpci3 => { bus => 0, addr => 28 },
4701         #addr29 : usb-host (pve-usb.cfg)
4702         'pci.1' => { bus => 0, addr => 30 },
4703         'pci.2' => { bus => 0, addr => 31 },
4704         'net6' => { bus => 1, addr => 1 },
4705         'net7' => { bus => 1, addr => 2 },
4706         'net8' => { bus => 1, addr => 3 },
4707         'net9' => { bus => 1, addr => 4 },
4708         'net10' => { bus => 1, addr => 5 },
4709         'net11' => { bus => 1, addr => 6 },
4710         'net12' => { bus => 1, addr => 7 },
4711         'net13' => { bus => 1, addr => 8 },
4712         'net14' => { bus => 1, addr => 9 },
4713         'net15' => { bus => 1, addr => 10 },
4714         'net16' => { bus => 1, addr => 11 },
4715         'net17' => { bus => 1, addr => 12 },
4716         'net18' => { bus => 1, addr => 13 },
4717         'net19' => { bus => 1, addr => 14 },
4718         'net20' => { bus => 1, addr => 15 },
4719         'net21' => { bus => 1, addr => 16 },
4720         'net22' => { bus => 1, addr => 17 },
4721         'net23' => { bus => 1, addr => 18 },
4722         'net24' => { bus => 1, addr => 19 },
4723         'net25' => { bus => 1, addr => 20 },
4724         'net26' => { bus => 1, addr => 21 },
4725         'net27' => { bus => 1, addr => 22 },
4726         'net28' => { bus => 1, addr => 23 },
4727         'net29' => { bus => 1, addr => 24 },
4728         'net30' => { bus => 1, addr => 25 },
4729         'net31' => { bus => 1, addr => 26 },
4730         'virtio6' => { bus => 2, addr => 1 },
4731         'virtio7' => { bus => 2, addr => 2 },
4732         'virtio8' => { bus => 2, addr => 3 },
4733         'virtio9' => { bus => 2, addr => 4 },
4734         'virtio10' => { bus => 2, addr => 5 },
4735         'virtio11' => { bus => 2, addr => 6 },
4736         'virtio12' => { bus => 2, addr => 7 },
4737         'virtio13' => { bus => 2, addr => 8 },
4738         'virtio14' => { bus => 2, addr => 9 },
4739         'virtio15' => { bus => 2, addr => 10 },
4740         'virtioscsi0' => { bus => 3, addr => 1 },
4741         'virtioscsi1' => { bus => 3, addr => 2 },
4742         'virtioscsi2' => { bus => 3, addr => 3 },
4743         'virtioscsi3' => { bus => 3, addr => 4 },
4744         'virtioscsi4' => { bus => 3, addr => 5 },
4745         'virtioscsi5' => { bus => 3, addr => 6 },
4746         'virtioscsi6' => { bus => 3, addr => 7 },
4747         'virtioscsi7' => { bus => 3, addr => 8 },
4748         'virtioscsi8' => { bus => 3, addr => 9 },
4749         'virtioscsi9' => { bus => 3, addr => 10 },
4750         'virtioscsi10' => { bus => 3, addr => 11 },
4751         'virtioscsi11' => { bus => 3, addr => 12 },
4752         'virtioscsi12' => { bus => 3, addr => 13 },
4753         'virtioscsi13' => { bus => 3, addr => 14 },
4754         'virtioscsi14' => { bus => 3, addr => 15 },
4755         'virtioscsi15' => { bus => 3, addr => 16 },
4756         'virtioscsi16' => { bus => 3, addr => 17 },
4757         'virtioscsi17' => { bus => 3, addr => 18 },
4758         'virtioscsi18' => { bus => 3, addr => 19 },
4759         'virtioscsi19' => { bus => 3, addr => 20 },
4760         'virtioscsi20' => { bus => 3, addr => 21 },
4761         'virtioscsi21' => { bus => 3, addr => 22 },
4762         'virtioscsi22' => { bus => 3, addr => 23 },
4763         'virtioscsi23' => { bus => 3, addr => 24 },
4764         'virtioscsi24' => { bus => 3, addr => 25 },
4765         'virtioscsi25' => { bus => 3, addr => 26 },
4766         'virtioscsi26' => { bus => 3, addr => 27 },
4767         'virtioscsi27' => { bus => 3, addr => 28 },
4768         'virtioscsi28' => { bus => 3, addr => 29 },
4769         'virtioscsi29' => { bus => 3, addr => 30 },
4770         'virtioscsi30' => { bus => 3, addr => 31 },
4771
4772     };
4773
4774     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
4775            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
4776            my $bus = $devices->{$id}->{bus};
4777            $res = ",bus=pci.$bus,addr=$addr";
4778            $bridges->{$bus} = 1 if $bridges;
4779     }
4780     return $res;
4781
4782 }
4783
4784 sub print_pcie_addr {
4785     my ($id) = @_;
4786
4787     my $res = '';
4788     my $devices = {
4789         hostpci0 => { bus => "ich9-pcie-port-1", addr => 0 },
4790         hostpci1 => { bus => "ich9-pcie-port-2", addr => 0 },
4791         hostpci2 => { bus => "ich9-pcie-port-3", addr => 0 },
4792         hostpci3 => { bus => "ich9-pcie-port-4", addr => 0 },
4793     };
4794
4795     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
4796            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
4797            my $bus = $devices->{$id}->{bus};
4798            $res = ",bus=$bus,addr=$addr";
4799     }
4800     return $res;
4801
4802 }
4803
4804 # vzdump restore implementaion
4805
4806 sub tar_archive_read_firstfile {
4807     my $archive = shift;
4808
4809     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
4810
4811     # try to detect archive type first
4812     my $pid = open (TMP, "tar tf '$archive'|") ||
4813         die "unable to open file '$archive'\n";
4814     my $firstfile = <TMP>;
4815     kill 15, $pid;
4816     close TMP;
4817
4818     die "ERROR: archive contaions no data\n" if !$firstfile;
4819     chomp $firstfile;
4820
4821     return $firstfile;
4822 }
4823
4824 sub tar_restore_cleanup {
4825     my ($storecfg, $statfile) = @_;
4826
4827     print STDERR "starting cleanup\n";
4828
4829     if (my $fd = IO::File->new($statfile, "r")) {
4830         while (defined(my $line = <$fd>)) {
4831             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
4832                 my $volid = $2;
4833                 eval {
4834                     if ($volid =~ m|^/|) {
4835                         unlink $volid || die 'unlink failed\n';
4836                     } else {
4837                         PVE::Storage::vdisk_free($storecfg, $volid);
4838                     }
4839                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
4840                 };
4841                 print STDERR "unable to cleanup '$volid' - $@" if $@;
4842             } else {
4843                 print STDERR "unable to parse line in statfile - $line";
4844             }
4845         }
4846         $fd->close();
4847     }
4848 }
4849
4850 sub restore_archive {
4851     my ($archive, $vmid, $user, $opts) = @_;
4852
4853     my $format = $opts->{format};
4854     my $comp;
4855
4856     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
4857         $format = 'tar' if !$format;
4858         $comp = 'gzip';
4859     } elsif ($archive =~ m/\.tar$/) {
4860         $format = 'tar' if !$format;
4861     } elsif ($archive =~ m/.tar.lzo$/) {
4862         $format = 'tar' if !$format;
4863         $comp = 'lzop';
4864     } elsif ($archive =~ m/\.vma$/) {
4865         $format = 'vma' if !$format;
4866     } elsif ($archive =~ m/\.vma\.gz$/) {
4867         $format = 'vma' if !$format;
4868         $comp = 'gzip';
4869     } elsif ($archive =~ m/\.vma\.lzo$/) {
4870         $format = 'vma' if !$format;
4871         $comp = 'lzop';
4872     } else {
4873         $format = 'vma' if !$format; # default
4874     }
4875
4876     # try to detect archive format
4877     if ($format eq 'tar') {
4878         return restore_tar_archive($archive, $vmid, $user, $opts);
4879     } else {
4880         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
4881     }
4882 }
4883
4884 sub restore_update_config_line {
4885     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
4886
4887     return if $line =~ m/^\#qmdump\#/;
4888     return if $line =~ m/^\#vzdump\#/;
4889     return if $line =~ m/^lock:/;
4890     return if $line =~ m/^unused\d+:/;
4891     return if $line =~ m/^parent:/;
4892     return if $line =~ m/^template:/; # restored VM is never a template
4893
4894     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
4895         # try to convert old 1.X settings
4896         my ($id, $ind, $ethcfg) = ($1, $2, $3);
4897         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
4898             my ($model, $macaddr) = split(/\=/, $devconfig);
4899             $macaddr = PVE::Tools::random_ether_addr() if !$macaddr || $unique;
4900             my $net = {
4901                 model => $model,
4902                 bridge => "vmbr$ind",
4903                 macaddr => $macaddr,
4904             };
4905             my $netstr = print_net($net);
4906
4907             print $outfd "net$cookie->{netcount}: $netstr\n";
4908             $cookie->{netcount}++;
4909         }
4910     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
4911         my ($id, $netstr) = ($1, $2);
4912         my $net = parse_net($netstr);
4913         $net->{macaddr} = PVE::Tools::random_ether_addr() if $net->{macaddr};
4914         $netstr = print_net($net);
4915         print $outfd "$id: $netstr\n";
4916     } elsif ($line =~ m/^((ide|scsi|virtio|sata)\d+):\s*(\S+)\s*$/) {
4917         my $virtdev = $1;
4918         my $value = $3;
4919         if ($line =~ m/backup=no/) {
4920             print $outfd "#$line";
4921         } elsif ($virtdev && $map->{$virtdev}) {
4922             my $di = parse_drive($virtdev, $value);
4923             delete $di->{format}; # format can change on restore
4924             $di->{file} = $map->{$virtdev};
4925             $value = print_drive($vmid, $di);
4926             print $outfd "$virtdev: $value\n";
4927         } else {
4928             print $outfd $line;
4929         }
4930     } else {
4931         print $outfd $line;
4932     }
4933 }
4934
4935 sub scan_volids {
4936     my ($cfg, $vmid) = @_;
4937
4938     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
4939
4940     my $volid_hash = {};
4941     foreach my $storeid (keys %$info) {
4942         foreach my $item (@{$info->{$storeid}}) {
4943             next if !($item->{volid} && $item->{size});
4944             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
4945             $volid_hash->{$item->{volid}} = $item;
4946         }
4947     }
4948
4949     return $volid_hash;
4950 }
4951
4952 sub get_used_paths {
4953     my ($vmid, $storecfg, $conf, $scan_snapshots, $skip_drive) = @_;
4954
4955     my $used_path = {};
4956
4957     my $scan_config = sub {
4958         my ($cref, $snapname) = @_;
4959
4960         foreach my $key (keys %$cref) {
4961             my $value = $cref->{$key};
4962             if (valid_drivename($key)) {
4963                 next if $skip_drive && $key eq $skip_drive;
4964                 my $drive = parse_drive($key, $value);
4965                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
4966                 if ($drive->{file} =~ m!^/!) {
4967                     $used_path->{$drive->{file}}++; # = 1;
4968                 } else {
4969                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
4970                     next if !$storeid;
4971                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
4972                     next if !$scfg;
4973                     my $path = PVE::Storage::path($storecfg, $drive->{file}, $snapname);
4974                     $used_path->{$path}++; # = 1;
4975                 }
4976             }
4977         }
4978     };
4979
4980     &$scan_config($conf);
4981
4982     undef $skip_drive;
4983
4984     if ($scan_snapshots) {
4985         foreach my $snapname (keys %{$conf->{snapshots}}) {
4986             &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
4987         }
4988     }
4989
4990     return $used_path;
4991 }
4992
4993 sub update_disksize {
4994     my ($vmid, $conf, $volid_hash) = @_;
4995
4996     my $changes;
4997
4998     my $used = {};
4999
5000     # Note: it is allowed to define multiple storages with same path (alias), so
5001     # we need to check both 'volid' and real 'path' (two different volid can point
5002     # to the same path).
5003
5004     my $usedpath = {};
5005
5006     # update size info
5007     foreach my $opt (keys %$conf) {
5008         if (valid_drivename($opt)) {
5009             my $drive = parse_drive($opt, $conf->{$opt});
5010             my $volid = $drive->{file};
5011             next if !$volid;
5012
5013             $used->{$volid} = 1;
5014             if ($volid_hash->{$volid} &&
5015                 (my $path = $volid_hash->{$volid}->{path})) {
5016                 $usedpath->{$path} = 1;
5017             }
5018
5019             next if drive_is_cdrom($drive);
5020             next if !$volid_hash->{$volid};
5021
5022             $drive->{size} = $volid_hash->{$volid}->{size};
5023             my $new = print_drive($vmid, $drive);
5024             if ($new ne $conf->{$opt}) {
5025                 $changes = 1;
5026                 $conf->{$opt} = $new;
5027             }
5028         }
5029     }
5030
5031     # remove 'unusedX' entry if volume is used
5032     foreach my $opt (keys %$conf) {
5033         next if $opt !~ m/^unused\d+$/;
5034         my $volid = $conf->{$opt};
5035         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5036         if ($used->{$volid} || ($path && $usedpath->{$path})) {
5037             $changes = 1;
5038             delete $conf->{$opt};
5039         }
5040     }
5041
5042     foreach my $volid (sort keys %$volid_hash) {
5043         next if $volid =~ m/vm-$vmid-state-/;
5044         next if $used->{$volid};
5045         my $path = $volid_hash->{$volid}->{path};
5046         next if !$path; # just to be sure
5047         next if $usedpath->{$path};
5048         $changes = 1;
5049         add_unused_volume($conf, $volid);
5050         $usedpath->{$path} = 1; # avoid to add more than once (aliases)
5051     }
5052
5053     return $changes;
5054 }
5055
5056 sub rescan {
5057     my ($vmid, $nolock) = @_;
5058
5059     my $cfg = PVE::Cluster::cfs_read_file("storage.cfg");
5060
5061     my $volid_hash = scan_volids($cfg, $vmid);
5062
5063     my $updatefn =  sub {
5064         my ($vmid) = @_;
5065
5066         my $conf = load_config($vmid);
5067
5068         check_lock($conf);
5069
5070         my $vm_volids = {};
5071         foreach my $volid (keys %$volid_hash) {
5072             my $info = $volid_hash->{$volid};
5073             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5074         }
5075
5076         my $changes = update_disksize($vmid, $conf, $vm_volids);
5077
5078         update_config_nolock($vmid, $conf, 1) if $changes;
5079     };
5080
5081     if (defined($vmid)) {
5082         if ($nolock) {
5083             &$updatefn($vmid);
5084         } else {
5085             lock_config($vmid, $updatefn, $vmid);
5086         }
5087     } else {
5088         my $vmlist = config_list();
5089         foreach my $vmid (keys %$vmlist) {
5090             if ($nolock) {
5091                 &$updatefn($vmid);
5092             } else {
5093                 lock_config($vmid, $updatefn, $vmid);
5094             }
5095         }
5096     }
5097 }
5098
5099 sub restore_vma_archive {
5100     my ($archive, $vmid, $user, $opts, $comp) = @_;
5101
5102     my $input = $archive eq '-' ? "<&STDIN" : undef;
5103     my $readfrom = $archive;
5104
5105     my $uncomp = '';
5106     if ($comp) {
5107         $readfrom = '-';
5108         my $qarchive = PVE::Tools::shellquote($archive);
5109         if ($comp eq 'gzip') {
5110             $uncomp = "zcat $qarchive|";
5111         } elsif ($comp eq 'lzop') {
5112             $uncomp = "lzop -d -c $qarchive|";
5113         } else {
5114             die "unknown compression method '$comp'\n";
5115         }
5116
5117     }
5118
5119     my $tmpdir = "/var/tmp/vzdumptmp$$";
5120     rmtree $tmpdir;
5121
5122     # disable interrupts (always do cleanups)
5123     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5124         warn "got interrupt - ignored\n";
5125     };
5126
5127     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5128     POSIX::mkfifo($mapfifo, 0600);
5129     my $fifofh;
5130
5131     my $openfifo = sub {
5132         open($fifofh, '>', $mapfifo) || die $!;
5133     };
5134
5135     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5136
5137     my $oldtimeout;
5138     my $timeout = 5;
5139
5140     my $devinfo = {};
5141
5142     my $rpcenv = PVE::RPCEnvironment::get();
5143
5144     my $conffile = config_file($vmid);
5145     my $tmpfn = "$conffile.$$.tmp";
5146
5147     # Note: $oldconf is undef if VM does not exists
5148     my $oldconf = PVE::Cluster::cfs_read_file(cfs_config_path($vmid));
5149
5150     my $print_devmap = sub {
5151         my $virtdev_hash = {};
5152
5153         my $cfgfn = "$tmpdir/qemu-server.conf";
5154
5155         # we can read the config - that is already extracted
5156         my $fh = IO::File->new($cfgfn, "r") ||
5157             "unable to read qemu-server.conf - $!\n";
5158
5159         while (defined(my $line = <$fh>)) {
5160             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5161                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5162                 die "archive does not contain data for drive '$virtdev'\n"
5163                     if !$devinfo->{$devname};
5164                 if (defined($opts->{storage})) {
5165                     $storeid = $opts->{storage} || 'local';
5166                 } elsif (!$storeid) {
5167                     $storeid = 'local';
5168                 }
5169                 $format = 'raw' if !$format;
5170                 $devinfo->{$devname}->{devname} = $devname;
5171                 $devinfo->{$devname}->{virtdev} = $virtdev;
5172                 $devinfo->{$devname}->{format} = $format;
5173                 $devinfo->{$devname}->{storeid} = $storeid;
5174
5175                 # check permission on storage
5176                 my $pool = $opts->{pool}; # todo: do we need that?
5177                 if ($user ne 'root@pam') {
5178                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5179                 }
5180
5181                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5182             }
5183         }
5184
5185         foreach my $devname (keys %$devinfo) {
5186             die "found no device mapping information for device '$devname'\n"
5187                 if !$devinfo->{$devname}->{virtdev};
5188         }
5189
5190         my $cfg = cfs_read_file('storage.cfg');
5191
5192         # create empty/temp config
5193         if ($oldconf) {
5194             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5195             foreach_drive($oldconf, sub {
5196                 my ($ds, $drive) = @_;
5197
5198                 return if drive_is_cdrom($drive);
5199
5200                 my $volid = $drive->{file};
5201
5202                 return if !$volid || $volid =~ m|^/|;
5203
5204                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5205                 return if !$path || !$owner || ($owner != $vmid);
5206
5207                 # Note: only delete disk we want to restore
5208                 # other volumes will become unused
5209                 if ($virtdev_hash->{$ds}) {
5210                     PVE::Storage::vdisk_free($cfg, $volid);
5211                 }
5212             });
5213         }
5214
5215         my $map = {};
5216         foreach my $virtdev (sort keys %$virtdev_hash) {
5217             my $d = $virtdev_hash->{$virtdev};
5218             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5219             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5220
5221             # test if requested format is supported
5222             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5223             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5224             $d->{format} = $defFormat if !$supported;
5225
5226             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5227                                                   $d->{format}, undef, $alloc_size);
5228             print STDERR "new volume ID is '$volid'\n";
5229             $d->{volid} = $volid;
5230             my $path = PVE::Storage::path($cfg, $volid);
5231
5232             my $write_zeros = 1;
5233             # fixme: what other storages types initialize volumes with zero?
5234             if ($scfg->{type} eq 'dir' || $scfg->{type} eq 'nfs' || $scfg->{type} eq 'glusterfs' ||
5235                 $scfg->{type} eq 'sheepdog' || $scfg->{type} eq 'rbd') {
5236                 $write_zeros = 0;
5237             }
5238
5239             print $fifofh "${write_zeros}:$d->{devname}=$path\n";
5240
5241             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5242             $map->{$virtdev} = $volid;
5243         }
5244
5245         $fh->seek(0, 0) || die "seek failed - $!\n";
5246
5247         my $outfd = new IO::File ($tmpfn, "w") ||
5248             die "unable to write config for VM $vmid\n";
5249
5250         my $cookie = { netcount => 0 };
5251         while (defined(my $line = <$fh>)) {
5252             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5253         }
5254
5255         $fh->close();
5256         $outfd->close();
5257     };
5258
5259     eval {
5260         # enable interrupts
5261         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5262             die "interrupted by signal\n";
5263         };
5264         local $SIG{ALRM} = sub { die "got timeout\n"; };
5265
5266         $oldtimeout = alarm($timeout);
5267
5268         my $parser = sub {
5269             my $line = shift;
5270
5271             print "$line\n";
5272
5273             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5274                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5275                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5276             } elsif ($line =~ m/^CTIME: /) {
5277                 # we correctly received the vma config, so we can disable
5278                 # the timeout now for disk allocation (set to 10 minutes, so
5279                 # that we always timeout if something goes wrong)
5280                 alarm(600);
5281                 &$print_devmap();
5282                 print $fifofh "done\n";
5283                 my $tmp = $oldtimeout || 0;
5284                 $oldtimeout = undef;
5285                 alarm($tmp);
5286                 close($fifofh);
5287             }
5288         };
5289
5290         print "restore vma archive: $cmd\n";
5291         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5292     };
5293     my $err = $@;
5294
5295     alarm($oldtimeout) if $oldtimeout;
5296
5297     unlink $mapfifo;
5298
5299     if ($err) {
5300         rmtree $tmpdir;
5301         unlink $tmpfn;
5302
5303         my $cfg = cfs_read_file('storage.cfg');
5304         foreach my $devname (keys %$devinfo) {
5305             my $volid = $devinfo->{$devname}->{volid};
5306             next if !$volid;
5307             eval {
5308                 if ($volid =~ m|^/|) {
5309                     unlink $volid || die 'unlink failed\n';
5310                 } else {
5311                     PVE::Storage::vdisk_free($cfg, $volid);
5312                 }
5313                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5314             };
5315             print STDERR "unable to cleanup '$volid' - $@" if $@;
5316         }
5317         die $err;
5318     }
5319
5320     rmtree $tmpdir;
5321
5322     rename($tmpfn, $conffile) ||
5323         die "unable to commit configuration file '$conffile'\n";
5324
5325     PVE::Cluster::cfs_update(); # make sure we read new file
5326
5327     eval { rescan($vmid, 1); };
5328     warn $@ if $@;
5329 }
5330
5331 sub restore_tar_archive {
5332     my ($archive, $vmid, $user, $opts) = @_;
5333
5334     if ($archive ne '-') {
5335         my $firstfile = tar_archive_read_firstfile($archive);
5336         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5337             if $firstfile ne 'qemu-server.conf';
5338     }
5339
5340     my $storecfg = cfs_read_file('storage.cfg');
5341
5342     # destroy existing data - keep empty config
5343     my $vmcfgfn = config_file($vmid);
5344     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5345
5346     my $tocmd = "/usr/lib/qemu-server/qmextract";
5347
5348     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5349     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5350     $tocmd .= ' --prealloc' if $opts->{prealloc};
5351     $tocmd .= ' --info' if $opts->{info};
5352
5353     # tar option "xf" does not autodetect compression when read from STDIN,
5354     # so we pipe to zcat
5355     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5356         PVE::Tools::shellquote("--to-command=$tocmd");
5357
5358     my $tmpdir = "/var/tmp/vzdumptmp$$";
5359     mkpath $tmpdir;
5360
5361     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5362     local $ENV{VZDUMP_VMID} = $vmid;
5363     local $ENV{VZDUMP_USER} = $user;
5364
5365     my $conffile = config_file($vmid);
5366     my $tmpfn = "$conffile.$$.tmp";
5367
5368     # disable interrupts (always do cleanups)
5369     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5370         print STDERR "got interrupt - ignored\n";
5371     };
5372
5373     eval {
5374         # enable interrupts
5375         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5376             die "interrupted by signal\n";
5377         };
5378
5379         if ($archive eq '-') {
5380             print "extracting archive from STDIN\n";
5381             run_command($cmd, input => "<&STDIN");
5382         } else {
5383             print "extracting archive '$archive'\n";
5384             run_command($cmd);
5385         }
5386
5387         return if $opts->{info};
5388
5389         # read new mapping
5390         my $map = {};
5391         my $statfile = "$tmpdir/qmrestore.stat";
5392         if (my $fd = IO::File->new($statfile, "r")) {
5393             while (defined (my $line = <$fd>)) {
5394                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5395                     $map->{$1} = $2 if $1;
5396                 } else {
5397                     print STDERR "unable to parse line in statfile - $line\n";
5398                 }
5399             }
5400             $fd->close();
5401         }
5402
5403         my $confsrc = "$tmpdir/qemu-server.conf";
5404
5405         my $srcfd = new IO::File($confsrc, "r") ||
5406             die "unable to open file '$confsrc'\n";
5407
5408         my $outfd = new IO::File ($tmpfn, "w") ||
5409             die "unable to write config for VM $vmid\n";
5410
5411         my $cookie = { netcount => 0 };
5412         while (defined (my $line = <$srcfd>)) {
5413             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5414         }
5415
5416         $srcfd->close();
5417         $outfd->close();
5418     };
5419     my $err = $@;
5420
5421     if ($err) {
5422
5423         unlink $tmpfn;
5424
5425         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
5426
5427         die $err;
5428     }
5429
5430     rmtree $tmpdir;
5431
5432     rename $tmpfn, $conffile ||
5433         die "unable to commit configuration file '$conffile'\n";
5434
5435     PVE::Cluster::cfs_update(); # make sure we read new file
5436
5437     eval { rescan($vmid, 1); };
5438     warn $@ if $@;
5439 };
5440
5441
5442 # Internal snapshots
5443
5444 # NOTE: Snapshot create/delete involves several non-atomic
5445 # action, and can take a long time.
5446 # So we try to avoid locking the file and use 'lock' variable
5447 # inside the config file instead.
5448
5449 my $snapshot_copy_config = sub {
5450     my ($source, $dest) = @_;
5451
5452     foreach my $k (keys %$source) {
5453         next if $k eq 'snapshots';
5454         next if $k eq 'snapstate';
5455         next if $k eq 'snaptime';
5456         next if $k eq 'vmstate';
5457         next if $k eq 'lock';
5458         next if $k eq 'digest';
5459         next if $k eq 'description';
5460         next if $k =~ m/^unused\d+$/;
5461
5462         $dest->{$k} = $source->{$k};
5463     }
5464 };
5465
5466 my $snapshot_apply_config = sub {
5467     my ($conf, $snap) = @_;
5468
5469     # copy snapshot list
5470     my $newconf = {
5471         snapshots => $conf->{snapshots},
5472     };
5473
5474     # keep description and list of unused disks
5475     foreach my $k (keys %$conf) {
5476         next if !($k =~ m/^unused\d+$/ || $k eq 'description');
5477         $newconf->{$k} = $conf->{$k};
5478     }
5479
5480     &$snapshot_copy_config($snap, $newconf);
5481
5482     return $newconf;
5483 };
5484
5485 sub foreach_writable_storage {
5486     my ($conf, $func) = @_;
5487
5488     my $sidhash = {};
5489
5490     foreach my $ds (keys %$conf) {
5491         next if !valid_drivename($ds);
5492
5493         my $drive = parse_drive($ds, $conf->{$ds});
5494         next if !$drive;
5495         next if drive_is_cdrom($drive);
5496
5497         my $volid = $drive->{file};
5498
5499         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5500         $sidhash->{$sid} = $sid if $sid;
5501     }
5502
5503     foreach my $sid (sort keys %$sidhash) {
5504         &$func($sid);
5505     }
5506 }
5507
5508 my $alloc_vmstate_volid = sub {
5509     my ($storecfg, $vmid, $conf, $snapname) = @_;
5510
5511     # Note: we try to be smart when selecting a $target storage
5512
5513     my $target;
5514
5515     # search shared storage first
5516     foreach_writable_storage($conf, sub {
5517         my ($sid) = @_;
5518         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
5519         return if !$scfg->{shared};
5520
5521         $target = $sid if !$target || $scfg->{path}; # prefer file based storage
5522     });
5523
5524     if (!$target) {
5525         # now search local storage
5526         foreach_writable_storage($conf, sub {
5527             my ($sid) = @_;
5528             my $scfg = PVE::Storage::storage_config($storecfg, $sid);
5529             return if $scfg->{shared};
5530
5531             $target = $sid if !$target || $scfg->{path}; # prefer file based storage;
5532         });
5533     }
5534
5535     $target = 'local' if !$target;
5536
5537     my $driver_state_size = 500; # assume 32MB is enough to safe all driver state;
5538     # we abort live save after $conf->{memory}, so we need at max twice that space
5539     my $size = $conf->{memory}*2 + $driver_state_size;
5540
5541     my $name = "vm-$vmid-state-$snapname";
5542     my $scfg = PVE::Storage::storage_config($storecfg, $target);
5543     $name .= ".raw" if $scfg->{path}; # add filename extension for file base storage
5544     my $volid = PVE::Storage::vdisk_alloc($storecfg, $target, $vmid, 'raw', $name, $size*1024);
5545
5546     return $volid;
5547 };
5548
5549 my $snapshot_prepare = sub {
5550     my ($vmid, $snapname, $save_vmstate, $comment) = @_;
5551
5552     my $snap;
5553
5554     my $updatefn =  sub {
5555
5556         my $conf = load_config($vmid);
5557
5558         die "you can't take a snapshot if it's a template\n"
5559             if is_template($conf);
5560
5561         check_lock($conf);
5562
5563         $conf->{lock} = 'snapshot';
5564
5565         die "snapshot name '$snapname' already used\n"
5566             if defined($conf->{snapshots}->{$snapname});
5567
5568         my $storecfg = PVE::Storage::config();
5569         die "snapshot feature is not available" if !has_feature('snapshot', $conf, $storecfg);
5570
5571         $snap = $conf->{snapshots}->{$snapname} = {};
5572
5573         if ($save_vmstate && check_running($vmid)) {
5574             $snap->{vmstate} = &$alloc_vmstate_volid($storecfg, $vmid, $conf, $snapname);
5575         }
5576
5577         &$snapshot_copy_config($conf, $snap);
5578
5579         $snap->{snapstate} = "prepare";
5580         $snap->{snaptime} = time();
5581         $snap->{description} = $comment if $comment;
5582
5583         # always overwrite machine if we save vmstate. This makes sure we
5584         # can restore it later using correct machine type
5585         $snap->{machine} = get_current_qemu_machine($vmid) if $snap->{vmstate};
5586
5587         update_config_nolock($vmid, $conf, 1);
5588     };
5589
5590     lock_config($vmid, $updatefn);
5591
5592     return $snap;
5593 };
5594
5595 my $snapshot_commit = sub {
5596     my ($vmid, $snapname) = @_;
5597
5598     my $updatefn = sub {
5599
5600         my $conf = load_config($vmid);
5601
5602         die "missing snapshot lock\n"
5603             if !($conf->{lock} && $conf->{lock} eq 'snapshot');
5604
5605         my $has_machine_config = defined($conf->{machine});
5606
5607         my $snap = $conf->{snapshots}->{$snapname};
5608
5609         die "snapshot '$snapname' does not exist\n" if !defined($snap);
5610
5611         die "wrong snapshot state\n"
5612             if !($snap->{snapstate} && $snap->{snapstate} eq "prepare");
5613
5614         delete $snap->{snapstate};
5615         delete $conf->{lock};
5616
5617         my $newconf = &$snapshot_apply_config($conf, $snap);
5618
5619         delete $newconf->{machine} if !$has_machine_config;
5620
5621         $newconf->{parent} = $snapname;
5622
5623         update_config_nolock($vmid, $newconf, 1);
5624     };
5625
5626     lock_config($vmid, $updatefn);
5627 };
5628
5629 sub snapshot_rollback {
5630     my ($vmid, $snapname) = @_;
5631
5632     my $prepare = 1;
5633
5634     my $storecfg = PVE::Storage::config();
5635
5636     my $conf = load_config($vmid);
5637
5638     my $get_snapshot_config = sub {
5639
5640         die "you can't rollback if vm is a template\n" if is_template($conf);
5641
5642         my $res = $conf->{snapshots}->{$snapname};
5643
5644         die "snapshot '$snapname' does not exist\n" if !defined($res);
5645
5646         return $res;
5647     };
5648
5649     my $snap = &$get_snapshot_config();
5650
5651     foreach_drive($snap, sub {
5652         my ($ds, $drive) = @_;
5653
5654         return if drive_is_cdrom($drive);
5655
5656         my $volid = $drive->{file};
5657
5658         PVE::Storage::volume_rollback_is_possible($storecfg, $volid, $snapname);
5659     });
5660
5661     my $updatefn = sub {
5662
5663         $conf = load_config($vmid);
5664
5665         $snap = &$get_snapshot_config();
5666
5667         die "unable to rollback to incomplete snapshot (snapstate = $snap->{snapstate})\n"
5668             if $snap->{snapstate};
5669
5670         if ($prepare) {
5671             check_lock($conf);
5672             vm_stop($storecfg, $vmid, undef, undef, 5, undef, undef);
5673         }
5674
5675         die "unable to rollback vm $vmid: vm is running\n"
5676             if check_running($vmid);
5677
5678         if ($prepare) {
5679             $conf->{lock} = 'rollback';
5680         } else {
5681             die "got wrong lock\n" if !($conf->{lock} && $conf->{lock} eq 'rollback');
5682             delete $conf->{lock};
5683         }
5684
5685         my $forcemachine;
5686
5687         if (!$prepare) {
5688             my $has_machine_config = defined($conf->{machine});
5689
5690             # copy snapshot config to current config
5691             $conf = &$snapshot_apply_config($conf, $snap);
5692             $conf->{parent} = $snapname;
5693
5694             # Note: old code did not store 'machine', so we try to be smart
5695             # and guess the snapshot was generated with kvm 1.4 (pc-i440fx-1.4).
5696             $forcemachine = $conf->{machine} || 'pc-i440fx-1.4';
5697             # we remove the 'machine' configuration if not explicitly specified
5698             # in the original config.
5699             delete $conf->{machine} if $snap->{vmstate} && !$has_machine_config;
5700         }
5701
5702         update_config_nolock($vmid, $conf, 1);
5703
5704         if (!$prepare && $snap->{vmstate}) {
5705             my $statefile = PVE::Storage::path($storecfg, $snap->{vmstate});
5706             vm_start($storecfg, $vmid, $statefile, undef, undef, undef, $forcemachine);
5707         }
5708     };
5709
5710     lock_config($vmid, $updatefn);
5711
5712     foreach_drive($snap, sub {
5713         my ($ds, $drive) = @_;
5714
5715         return if drive_is_cdrom($drive);
5716
5717         my $volid = $drive->{file};
5718         my $device = "drive-$ds";
5719
5720         PVE::Storage::volume_snapshot_rollback($storecfg, $volid, $snapname);
5721     });
5722
5723     $prepare = 0;
5724     lock_config($vmid, $updatefn);
5725 }
5726
5727 my $savevm_wait = sub {
5728     my ($vmid) = @_;
5729
5730     for(;;) {
5731         my $stat = vm_mon_cmd_nocheck($vmid, "query-savevm");
5732         if (!$stat->{status}) {
5733             die "savevm not active\n";
5734         } elsif ($stat->{status} eq 'active') {
5735             sleep(1);
5736             next;
5737         } elsif ($stat->{status} eq 'completed') {
5738             last;
5739         } else {
5740             die "query-savevm returned status '$stat->{status}'\n";
5741         }
5742     }
5743 };
5744
5745 sub do_snapshots_with_qemu {
5746     my ($storecfg, $volid) = @_;
5747
5748     my $storage_name = PVE::Storage::parse_volume_id($volid);
5749
5750     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}} ){
5751         return 1;
5752     }
5753
5754     if ($volid =~ m/\.(qcow2|qed)$/){
5755         return 1;
5756     }
5757
5758     return undef;
5759 }
5760
5761 sub snapshot_create {
5762     my ($vmid, $snapname, $save_vmstate, $comment) = @_;
5763
5764     my $snap = &$snapshot_prepare($vmid, $snapname, $save_vmstate, $comment);
5765
5766     $save_vmstate = 0 if !$snap->{vmstate}; # vm is not running
5767
5768     my $config = load_config($vmid);
5769
5770     my $running = check_running($vmid);
5771
5772     my $freezefs = $running && $config->{agent};
5773     $freezefs = 0 if $snap->{vmstate}; # not needed if we save RAM
5774
5775     my $drivehash = {};
5776
5777     if ($freezefs) {
5778         eval { vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
5779         warn "guest-fsfreeze-freeze problems - $@" if $@;
5780     }
5781
5782     eval {
5783         # create internal snapshots of all drives
5784
5785         my $storecfg = PVE::Storage::config();
5786
5787         if ($running) {
5788             if ($snap->{vmstate}) {
5789                 my $path = PVE::Storage::path($storecfg, $snap->{vmstate});
5790                 vm_mon_cmd($vmid, "savevm-start", statefile => $path);
5791                 &$savevm_wait($vmid);
5792             } else {
5793                 vm_mon_cmd($vmid, "savevm-start");
5794             }
5795         };
5796
5797         foreach_drive($snap, sub {
5798             my ($ds, $drive) = @_;
5799
5800             return if drive_is_cdrom($drive);
5801
5802             my $volid = $drive->{file};
5803             my $device = "drive-$ds";
5804
5805             qemu_volume_snapshot($vmid, $device, $storecfg, $volid, $snapname);
5806             $drivehash->{$ds} = 1;
5807        });
5808     };
5809     my $err = $@;
5810
5811     if ($running) {
5812         eval { vm_mon_cmd($vmid, "savevm-end")  };
5813         warn $@ if $@;
5814
5815         if ($freezefs) {
5816             eval { vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
5817             warn "guest-fsfreeze-thaw problems - $@" if $@;
5818         }
5819
5820         # savevm-end is async, we need to wait
5821         for (;;) {
5822             my $stat = vm_mon_cmd_nocheck($vmid, "query-savevm");
5823             if (!$stat->{bytes}) {
5824                 last;
5825             } else {
5826                 print "savevm not yet finished\n";
5827                 sleep(1);
5828                 next;
5829             }
5830         }
5831     }
5832
5833     if ($err) {
5834         warn "snapshot create failed: starting cleanup\n";
5835         eval { snapshot_delete($vmid, $snapname, 0, $drivehash); };
5836         warn $@ if $@;
5837         die $err;
5838     }
5839
5840     &$snapshot_commit($vmid, $snapname);
5841 }
5842
5843 # Note: $drivehash is only set when called from snapshot_create.
5844 sub snapshot_delete {
5845     my ($vmid, $snapname, $force, $drivehash) = @_;
5846
5847     my $prepare = 1;
5848
5849     my $snap;
5850     my $unused = [];
5851
5852     my $unlink_parent = sub {
5853         my ($confref, $new_parent) = @_;
5854
5855         if ($confref->{parent} && $confref->{parent} eq $snapname) {
5856             if ($new_parent) {
5857                 $confref->{parent} = $new_parent;
5858             } else {
5859                 delete $confref->{parent};
5860             }
5861         }
5862     };
5863
5864     my $updatefn =  sub {
5865         my ($remove_drive) = @_;
5866
5867         my $conf = load_config($vmid);
5868
5869         if (!$drivehash) {
5870             check_lock($conf);
5871             die "you can't delete a snapshot if vm is a template\n"
5872                 if is_template($conf);
5873         }
5874
5875         $snap = $conf->{snapshots}->{$snapname};
5876
5877         die "snapshot '$snapname' does not exist\n" if !defined($snap);
5878
5879         # remove parent refs
5880         if (!$prepare) {
5881             &$unlink_parent($conf, $snap->{parent});
5882             foreach my $sn (keys %{$conf->{snapshots}}) {
5883                 next if $sn eq $snapname;
5884                 &$unlink_parent($conf->{snapshots}->{$sn}, $snap->{parent});
5885             }
5886         }
5887
5888         if ($remove_drive) {
5889             if ($remove_drive eq 'vmstate') {
5890                 delete $snap->{$remove_drive};
5891             } else {
5892                 my $drive = parse_drive($remove_drive, $snap->{$remove_drive});
5893                 my $volid = $drive->{file};
5894                 delete $snap->{$remove_drive};
5895                 add_unused_volume($conf, $volid);
5896             }
5897         }
5898
5899         if ($prepare) {
5900             $snap->{snapstate} = 'delete';
5901         } else {
5902             delete $conf->{snapshots}->{$snapname};
5903             delete $conf->{lock} if $drivehash;
5904             foreach my $volid (@$unused) {
5905                 add_unused_volume($conf, $volid);
5906             }
5907         }
5908
5909         update_config_nolock($vmid, $conf, 1);
5910     };
5911
5912     lock_config($vmid, $updatefn);
5913
5914     # now remove vmstate file
5915
5916     my $storecfg = PVE::Storage::config();
5917
5918     if ($snap->{vmstate}) {
5919         eval {  PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
5920         if (my $err = $@) {
5921             die $err if !$force;
5922             warn $err;
5923         }
5924         # save changes (remove vmstate from snapshot)
5925         lock_config($vmid, $updatefn, 'vmstate') if !$force;
5926     };
5927
5928     # now remove all internal snapshots
5929     foreach_drive($snap, sub {
5930         my ($ds, $drive) = @_;
5931
5932         return if drive_is_cdrom($drive);
5933
5934         my $volid = $drive->{file};
5935         my $device = "drive-$ds";
5936
5937         if (!$drivehash || $drivehash->{$ds}) {
5938             eval { qemu_volume_snapshot_delete($vmid, $device, $storecfg, $volid, $snapname); };
5939             if (my $err = $@) {
5940                 die $err if !$force;
5941                 warn $err;
5942             }
5943         }
5944
5945         # save changes (remove drive fron snapshot)
5946         lock_config($vmid, $updatefn, $ds) if !$force;
5947         push @$unused, $volid;
5948     });
5949
5950     # now cleanup config
5951     $prepare = 0;
5952     lock_config($vmid, $updatefn);
5953 }
5954
5955 sub has_feature {
5956     my ($feature, $conf, $storecfg, $snapname, $running) = @_;
5957
5958     my $err;
5959     foreach_drive($conf, sub {
5960         my ($ds, $drive) = @_;
5961
5962         return if drive_is_cdrom($drive);
5963         my $volid = $drive->{file};
5964         $err = 1 if !PVE::Storage::volume_has_feature($storecfg, $feature, $volid, $snapname, $running);
5965     });
5966
5967     return $err ? 0 : 1;
5968 }
5969
5970 sub template_create {
5971     my ($vmid, $conf, $disk) = @_;
5972
5973     my $storecfg = PVE::Storage::config();
5974
5975     foreach_drive($conf, sub {
5976         my ($ds, $drive) = @_;
5977
5978         return if drive_is_cdrom($drive);
5979         return if $disk && $ds ne $disk;
5980
5981         my $volid = $drive->{file};
5982         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
5983
5984         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
5985         $drive->{file} = $voliddst;
5986         $conf->{$ds} = print_drive($vmid, $drive);
5987         update_config_nolock($vmid, $conf, 1);
5988     });
5989 }
5990
5991 sub is_template {
5992     my ($conf) = @_;
5993
5994     return 1 if defined $conf->{template} && $conf->{template} == 1;
5995 }
5996
5997 sub qemu_img_convert {
5998     my ($src_volid, $dst_volid, $size, $snapname) = @_;
5999
6000     my $storecfg = PVE::Storage::config();
6001     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6002     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6003
6004     if ($src_storeid && $dst_storeid) {
6005         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6006         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6007
6008         my $src_format = qemu_img_format($src_scfg, $src_volname);
6009         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6010
6011         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6012         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6013
6014         my $cmd = [];
6015         push @$cmd, '/usr/bin/qemu-img', 'convert', '-t', 'writeback', '-p', '-n';
6016         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
6017         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path, $dst_path;
6018
6019         my $parser = sub {
6020             my $line = shift;
6021             if($line =~ m/\((\S+)\/100\%\)/){
6022                 my $percent = $1;
6023                 my $transferred = int($size * $percent / 100);
6024                 my $remaining = $size - $transferred;
6025
6026                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6027             }
6028
6029         };
6030
6031         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6032         my $err = $@;
6033         die "copy failed: $err" if $err;
6034     }
6035 }
6036
6037 sub qemu_img_format {
6038     my ($scfg, $volname) = @_;
6039
6040     if ($scfg->{path} && $volname =~ m/\.(raw|cow|qcow|qcow2|qed|vmdk|cloop)$/) {
6041         return $1;
6042     } else {
6043         return "raw";
6044     }
6045 }
6046
6047 sub qemu_drive_mirror {
6048     my ($vmid, $drive, $dst_volid, $vmiddst) = @_;
6049
6050     my $storecfg = PVE::Storage::config();
6051     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6052
6053     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6054
6055     my $format = qemu_img_format($dst_scfg, $dst_volname);
6056
6057     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6058
6059     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $dst_path };
6060     $opts->{format} = $format if $format;
6061
6062     print "drive mirror is starting (scanning bitmap) : this step can take some minutes/hours, depend of disk size and storage speed\n";
6063
6064     eval {
6065     vm_mon_cmd($vmid, "drive-mirror", %$opts);
6066         while (1) {
6067             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6068             my $stat = @$stats[0];
6069             die "mirroring job seem to have die. Maybe do you have bad sectors?" if !$stat;
6070             die "error job is not mirroring" if $stat->{type} ne "mirror";
6071
6072             my $busy = $stat->{busy};
6073             my $ready = $stat->{ready};
6074
6075             if (my $total = $stat->{len}) {
6076                 my $transferred = $stat->{offset} || 0;
6077                 my $remaining = $total - $transferred;
6078                 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6079
6080                 print "transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6081             }
6082
6083
6084             if ($stat->{ready} eq 'true') {
6085
6086                 last if $vmiddst != $vmid;
6087
6088                 # try to switch the disk if source and destination are on the same guest
6089                 eval { vm_mon_cmd($vmid, "block-job-complete", device => "drive-$drive") };
6090                 last if !$@;
6091                 die $@ if $@ !~ m/cannot be completed/;
6092             }
6093             sleep 1;
6094         }
6095
6096
6097     };
6098     my $err = $@;
6099
6100     my $cancel_job = sub {
6101         vm_mon_cmd($vmid, "block-job-cancel", device => "drive-$drive");
6102         while (1) {
6103             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6104             my $stat = @$stats[0];
6105             last if !$stat;
6106             sleep 1;
6107         }
6108     };
6109
6110     if ($err) {
6111         eval { &$cancel_job(); };
6112         die "mirroring error: $err";
6113     }
6114
6115     if ($vmiddst != $vmid) {
6116         # if we clone a disk for a new target vm, we don't switch the disk
6117         &$cancel_job(); # so we call block-job-cancel
6118     }
6119 }
6120
6121 sub clone_disk {
6122     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6123         $newvmid, $storage, $format, $full, $newvollist) = @_;
6124
6125     my $newvolid;
6126
6127     if (!$full) {
6128         print "create linked clone of drive $drivename ($drive->{file})\n";
6129         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6130         push @$newvollist, $newvolid;
6131     } else {
6132         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6133         $storeid = $storage if $storage;
6134
6135         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6136         if (!$format) {
6137             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6138             $format = qemu_img_format($scfg, $volname);
6139         }
6140
6141         # test if requested format is supported - else use default
6142         my $supported = grep { $_ eq $format } @$validFormats;
6143         $format = $defFormat if !$supported;
6144
6145         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6146
6147         print "create full clone of drive $drivename ($drive->{file})\n";
6148         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $format, undef, ($size/1024));
6149         push @$newvollist, $newvolid;
6150
6151         if (!$running || $snapname) {
6152             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname);
6153         } else {
6154             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid);
6155         }
6156     }
6157
6158     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6159
6160     my $disk = $drive;
6161     $disk->{format} = undef;
6162     $disk->{file} = $newvolid;
6163     $disk->{size} = $size;
6164
6165     return $disk;
6166 }
6167
6168 # this only works if VM is running
6169 sub get_current_qemu_machine {
6170     my ($vmid) = @_;
6171
6172     my $cmd = { execute => 'query-machines', arguments => {} };
6173     my $res = vm_qmp_command($vmid, $cmd);
6174
6175     my ($current, $default);
6176     foreach my $e (@$res) {
6177         $default = $e->{name} if $e->{'is-default'};
6178         $current = $e->{name} if $e->{'is-current'};
6179     }
6180
6181     # fallback to the default machine if current is not supported by qemu
6182     return $current || $default || 'pc';
6183 }
6184
6185 sub qemu_machine_feature_enabled {
6186     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6187
6188     my $current_major;
6189     my $current_minor;
6190
6191     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6192
6193         $current_major = $3;
6194         $current_minor = $4;
6195
6196     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6197
6198         $current_major = $1;
6199         $current_minor = $2;
6200     }
6201
6202     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
6203
6204
6205 }
6206
6207 sub lspci {
6208
6209     my $devices = {};
6210
6211     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6212             my (undef, $id, $function) = @_;
6213             my $res = { id => $id, function => $function};
6214             push @{$devices->{$id}}, $res;
6215     });
6216
6217     return $devices;
6218 }
6219
6220 sub vm_iothreads_list {
6221     my ($vmid) = @_;
6222
6223     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6224
6225     my $iothreads = {};
6226     foreach my $iothread (@$res) {
6227         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6228     }
6229
6230     return $iothreads;
6231 }
6232
6233 sub scsihw_infos {
6234     my ($conf, $drive) = @_;
6235
6236     my $maxdev = 0;
6237
6238     if ($conf->{scsihw} && ($conf->{scsihw} =~ m/^lsi/)) {
6239         $maxdev = 7;
6240     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6241         $maxdev = 1;
6242     } else {
6243         $maxdev = 256;
6244     }
6245
6246     my $controller = int($drive->{index} / $maxdev);
6247     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6248
6249     return ($maxdev, $controller, $controller_prefix);
6250 }
6251
6252 1;