PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QMPClient;
  31 use PVE::RPCEnvironment;
  32 use Time::HiRes qw(gettimeofday);
  33
  34 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  35
  36 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  37
  38 # Note about locking: we use flock on the config file protect
  39 # against concurent actions.
  40 # Aditionaly, we have a 'lock' setting in the config file. This
  41 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  42 # allowed when such lock is set. But you can ignore this kind of
  43 # lock with the --skiplock flag.
  44
  45 cfs_register_file('/qemu-server/',
  46                   \&parse_vm_config,
  47                   \&write_vm_config);
  48
  49 PVE::JSONSchema::register_standard_option('skiplock', {
  50     description => "Ignore locks - only root is allowed to use this option.",
  51     type => 'boolean',
  52     optional => 1,
  53 });
  54
  55 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  56     description => "Some command save/restore state from this location.",
  57     type => 'string',
  58     maxLength => 128,
  59     optional => 1,
  60 });
  61
  62 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  63     description => "The name of the snapshot.",
  64     type => 'string', format => 'pve-configid',
  65     maxLength => 40,
  66 });
  67
  68 #no warnings 'redefine';
  69
  70 sub cgroups_write {
  71    my ($controller, $vmid, $option, $value) = @_;
  72
  73    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  74    PVE::ProcFSTools::write_proc_entry($path, $value);
  75
  76 }
  77
  78 my $nodename = PVE::INotify::nodename();
  79
  80 mkdir "/etc/pve/nodes/$nodename";
  81 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  82 mkdir $confdir;
  83
  84 my $var_run_tmpdir = "/var/run/qemu-server";
  85 mkdir $var_run_tmpdir;
  86
  87 my $lock_dir = "/var/lock/qemu-server";
  88 mkdir $lock_dir;
  89
  90 my $pcisysfs = "/sys/bus/pci";
  91
  92 my $confdesc = {
  93     onboot => {
  94         optional => 1,
  95         type => 'boolean',
  96         description => "Specifies whether a VM will be started during system bootup.",
  97         default => 0,
  98     },
  99     autostart => {
 100         optional => 1,
 101         type => 'boolean',
 102         description => "Automatic restart after crash (currently ignored).",
 103         default => 0,
 104     },
 105     hotplug => {
 106         optional => 1,
 107         type => 'string', format => 'pve-hotplug-features',
 108         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 109         default => 'network,disk,usb',
 110     },
 111     reboot => {
 112         optional => 1,
 113         type => 'boolean',
 114         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 115         default => 1,
 116     },
 117     lock => {
 118         optional => 1,
 119         type => 'string',
 120         description => "Lock/unlock the VM.",
 121         enum => [qw(migrate backup snapshot rollback)],
 122     },
 123     cpulimit => {
 124         optional => 1,
 125         type => 'number',
 126         description => "Limit of CPU usage. Note if the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 127         minimum => 0,
 128         maximum => 128,
 129         default => 0,
 130     },
 131     cpuunits => {
 132         optional => 1,
 133         type => 'integer',
 134         description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 135         minimum => 0,
 136         maximum => 500000,
 137         default => 1000,
 138     },
 139     memory => {
 140         optional => 1,
 141         type => 'integer',
 142         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 143         minimum => 16,
 144         default => 512,
 145     },
 146     balloon => {
 147         optional => 1,
 148         type => 'integer',
 149         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 150         minimum => 0,
 151     },
 152     shares => {
 153         optional => 1,
 154         type => 'integer',
 155         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 156         minimum => 0,
 157         maximum => 50000,
 158         default => 1000,
 159     },
 160     keyboard => {
 161         optional => 1,
 162         type => 'string',
 163         description => "Keybord layout for vnc server. Default is read from the datacenter configuration file.",
 164         enum => PVE::Tools::kvmkeymaplist(),
 165         default => 'en-us',
 166     },
 167     name => {
 168         optional => 1,
 169         type => 'string', format => 'dns-name',
 170         description => "Set a name for the VM. Only used on the configuration web interface.",
 171     },
 172     scsihw => {
 173         optional => 1,
 174         type => 'string',
 175         description => "scsi controller model",
 176         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 177         default => 'lsi',
 178     },
 179     description => {
 180         optional => 1,
 181         type => 'string',
 182         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 183     },
 184     ostype => {
 185         optional => 1,
 186         type => 'string',
 187         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 l24 l26 solaris)],
 188         description => <<EODESC,
 189 Used to enable special optimization/features for specific
 190 operating systems:
 191
 192 other  => unspecified OS
 193 wxp    => Microsoft Windows XP
 194 w2k    => Microsoft Windows 2000
 195 w2k3   => Microsoft Windows 2003
 196 w2k8   => Microsoft Windows 2008
 197 wvista => Microsoft Windows Vista
 198 win7   => Microsoft Windows 7
 199 win8   => Microsoft Windows 8/2012
 200 l24    => Linux 2.4 Kernel
 201 l26    => Linux 2.6/3.X Kernel
 202 solaris => solaris/opensolaris/openindiania kernel
 203
 204 other|l24|l26|solaris                       ... no special behaviour
 205 wxp|w2k|w2k3|w2k8|wvista|win7|win8  ... use --localtime switch
 206 EODESC
 207     },
 208     boot => {
 209         optional => 1,
 210         type => 'string',
 211         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 212         pattern => '[acdn]{1,4}',
 213         default => 'cdn',
 214     },
 215     bootdisk => {
 216         optional => 1,
 217         type => 'string', format => 'pve-qm-bootdisk',
 218         description => "Enable booting from specified disk.",
 219         pattern => '(ide|sata|scsi|virtio)\d+',
 220     },
 221     smp => {
 222         optional => 1,
 223         type => 'integer',
 224         description => "The number of CPUs. Please use option -sockets instead.",
 225         minimum => 1,
 226         default => 1,
 227     },
 228     sockets => {
 229         optional => 1,
 230         type => 'integer',
 231         description => "The number of CPU sockets.",
 232         minimum => 1,
 233         default => 1,
 234     },
 235     cores => {
 236         optional => 1,
 237         type => 'integer',
 238         description => "The number of cores per socket.",
 239         minimum => 1,
 240         default => 1,
 241     },
 242     numa => {
 243         optional => 1,
 244         type => 'boolean',
 245         description => "Enable/disable Numa.",
 246         default => 0,
 247     },
 248     vcpus => {
 249         optional => 1,
 250         type => 'integer',
 251         description => "Number of hotplugged vcpus.",
 252         minimum => 1,
 253         default => 0,
 254     },
 255     acpi => {
 256         optional => 1,
 257         type => 'boolean',
 258         description => "Enable/disable ACPI.",
 259         default => 1,
 260     },
 261     agent => {
 262         optional => 1,
 263         type => 'boolean',
 264         description => "Enable/disable Qemu GuestAgent.",
 265         default => 0,
 266     },
 267     kvm => {
 268         optional => 1,
 269         type => 'boolean',
 270         description => "Enable/disable KVM hardware virtualization.",
 271         default => 1,
 272     },
 273     tdf => {
 274         optional => 1,
 275         type => 'boolean',
 276         description => "Enable/disable time drift fix.",
 277         default => 0,
 278     },
 279     localtime => {
 280         optional => 1,
 281         type => 'boolean',
 282         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 283     },
 284     freeze => {
 285         optional => 1,
 286         type => 'boolean',
 287         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 288     },
 289     vga => {
 290         optional => 1,
 291         type => 'string',
 292         description => "Select VGA type. If you want to use high resolution modes (>= 1280x1024x16) then you should use option 'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and 'cirrur' for other OS types. Option 'qxl' enables the SPICE display sever. You can also run without any graphic card using a serial devive as terminal.",
 293         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 294     },
 295     watchdog => {
 296         optional => 1,
 297         type => 'string', format => 'pve-qm-watchdog',
 298         typetext => '[[model=]i6300esb|ib700] [,[action=]reset|shutdown|poweroff|pause|debug|none]',
 299         description => "Create a virtual hardware watchdog device.  Once enabled (by a guest action), the watchdog must be periodically polled by an agent inside the guest or else the guest will be restarted (or execute the action specified)",
 300     },
 301     startdate => {
 302         optional => 1,
 303         type => 'string',
 304         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 305         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 306         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 307         default => 'now',
 308     },
 309     startup =>  get_standard_option('pve-startup-order'),
 310     template => {
 311         optional => 1,
 312         type => 'boolean',
 313         description => "Enable/disable Template.",
 314         default => 0,
 315     },
 316     args => {
 317         optional => 1,
 318         type => 'string',
 319         description => <<EODESCR,
 320 Note: this option is for experts only. It allows you to pass arbitrary arguments to kvm, for example:
 321
 322 args: -no-reboot -no-hpet
 323 EODESCR
 324     },
 325     tablet => {
 326         optional => 1,
 327         type => 'boolean',
 328         default => 1,
 329         description => "Enable/disable the usb tablet device. This device is usually needed to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with normal VNC clients. If you're running lots of console-only guests on one host, you may consider disabling this to save some context switches. This is turned of by default if you use spice (vga=qxl).",
 330     },
 331     migrate_speed => {
 332         optional => 1,
 333         type => 'integer',
 334         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 335         minimum => 0,
 336         default => 0,
 337     },
 338     migrate_downtime => {
 339         optional => 1,
 340         type => 'number',
 341         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 342         minimum => 0,
 343         default => 0.1,
 344     },
 345     cdrom => {
 346         optional => 1,
 347         type => 'string', format => 'pve-qm-drive',
 348         typetext => 'volume',
 349         description => "This is an alias for option -ide2",
 350     },
 351     cpu => {
 352         optional => 1,
 353         description => "Emulated CPU type.",
 354         type => 'string',
 355         enum => [ qw(486 athlon pentium pentium2 pentium3 coreduo core2duo kvm32 kvm64 qemu32 qemu64 phenom Conroe Penryn Nehalem Westmere SandyBridge IvyBridge Haswell Broadwell Opteron_G1 Opteron_G2 Opteron_G3 Opteron_G4 Opteron_G5 host) ],
 356         default => 'kvm64',
 357     },
 358     parent => get_standard_option('pve-snapshot-name', {
 359         optional => 1,
 360         description => "Parent snapshot name. This is used internally, and should not be modified.",
 361     }),
 362     snaptime => {
 363         optional => 1,
 364         description => "Timestamp for snapshots.",
 365         type => 'integer',
 366         minimum => 0,
 367     },
 368     vmstate => {
 369         optional => 1,
 370         type => 'string', format => 'pve-volume-id',
 371         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 372     },
 373     machine => {
 374         description => "Specific the Qemu machine type.",
 375         type => 'string',
 376         pattern => '(pc|pc(-i440fx)?-\d+\.\d+|q35|pc-q35-\d+\.\d+)',
 377         maxLength => 40,
 378         optional => 1,
 379     },
 380     smbios1 => {
 381         description => "Specify SMBIOS type 1 fields.",
 382         type => 'string', format => 'pve-qm-smbios1',
 383         typetext => "[manufacturer=str][,product=str][,version=str][,serial=str] [,uuid=uuid][,sku=str][,family=str]",
 384         maxLength => 256,
 385         optional => 1,
 386     },
 387 };
 388
 389 # what about other qemu settings ?
 390 #cpu => 'string',
 391 #machine => 'string',
 392 #fda => 'file',
 393 #fdb => 'file',
 394 #mtdblock => 'file',
 395 #sd => 'file',
 396 #pflash => 'file',
 397 #snapshot => 'bool',
 398 #bootp => 'file',
 399 ##tftp => 'dir',
 400 ##smb => 'dir',
 401 #kernel => 'file',
 402 #append => 'string',
 403 #initrd => 'file',
 404 ##soundhw => 'string',
 405
 406 while (my ($k, $v) = each %$confdesc) {
 407     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 408 }
 409
 410 my $MAX_IDE_DISKS = 4;
 411 my $MAX_SCSI_DISKS = 14;
 412 my $MAX_VIRTIO_DISKS = 16;
 413 my $MAX_SATA_DISKS = 6;
 414 my $MAX_USB_DEVICES = 5;
 415 my $MAX_NETS = 32;
 416 my $MAX_UNUSED_DISKS = 8;
 417 my $MAX_HOSTPCI_DEVICES = 4;
 418 my $MAX_SERIAL_PORTS = 4;
 419 my $MAX_PARALLEL_PORTS = 3;
 420 my $MAX_NUMA = 8;
 421 my $MAX_MEM = 4194304;
 422 my $STATICMEM = 1024;
 423
 424 my $numadesc = {
 425     optional => 1,
 426     type => 'string', format => 'pve-qm-numanode',
 427     typetext => "cpus=<id[-id],memory=<mb>[[,hostnodes=<id[-id]>] [,policy=<preferred|bind|interleave>]]",
 428     description => "numa topology",
 429 };
 430 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 431
 432 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 433     $confdesc->{"numa$i"} = $numadesc;
 434 }
 435
 436 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 437                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 438                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 439 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 440
 441 my $netdesc = {
 442     optional => 1,
 443     type => 'string', format => 'pve-qm-net',
 444     typetext => "MODEL=XX:XX:XX:XX:XX:XX [,bridge=<dev>][,queues=<nbqueues>][,rate=<mbps>] [,tag=<vlanid>][,firewall=0|1],link_down=0|1]",
 445     description => <<EODESCR,
 446 Specify network devices.
 447
 448 MODEL is one of: $nic_model_list_txt
 449
 450 XX:XX:XX:XX:XX:XX should be an unique MAC address. This is
 451 automatically generated if not specified.
 452
 453 The bridge parameter can be used to automatically add the interface to a bridge device. The Proxmox VE standard bridge is called 'vmbr0'.
 454
 455 Option 'rate' is used to limit traffic bandwidth from and to this interface. It is specified as floating point number, unit is 'Megabytes per second'.
 456
 457 If you specify no bridge, we create a kvm 'user' (NATed) network device, which provides DHCP and DNS services. The following addresses are used:
 458
 459 10.0.2.2   Gateway
 460 10.0.2.3   DNS Server
 461 10.0.2.4   SMB Server
 462
 463 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 464
 465 EODESCR
 466 };
 467 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 468
 469 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 470     $confdesc->{"net$i"} = $netdesc;
 471 }
 472
 473 my $drivename_hash;
 474
 475 my $idedesc = {
 476     optional => 1,
 477     type => 'string', format => 'pve-qm-drive',
 478     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads] [,discard=ignore|on]',
 479     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
 480 };
 481 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
 482
 483 my $scsidesc = {
 484     optional => 1,
 485     type => 'string', format => 'pve-qm-drive',
 486     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads] [,discard=ignore|on] [,iothread=on] [,queues=<nbqueues>]',
 487     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
 488 };
 489 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 490
 491 my $satadesc = {
 492     optional => 1,
 493     type => 'string', format => 'pve-qm-drive',
 494     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads]  [,discard=ignore|on]',
 495     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
 496 };
 497 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
 498
 499 my $virtiodesc = {
 500     optional => 1,
 501     type => 'string', format => 'pve-qm-drive',
 502     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads]  [,discard=ignore|on] [,iothread=on]',
 503     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
 504 };
 505 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
 506
 507 my $usbdesc = {
 508     optional => 1,
 509     type => 'string', format => 'pve-qm-usb-device',
 510     typetext => 'host=HOSTUSBDEVICE|spice',
 511     description => <<EODESCR,
 512 Configure an USB device (n is 0 to 4). This can be used to
 513 pass-through usb devices to the guest. HOSTUSBDEVICE syntax is:
 514
 515 'bus-port(.port)*' (decimal numbers) or
 516 'vendor_id:product_id' (hexadeciaml numbers)
 517
 518 You can use the 'lsusb -t' command to list existing usb devices.
 519
 520 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 521
 522 The value 'spice' can be used to add a usb redirection devices for spice.
 523
 524 EODESCR
 525 };
 526 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
 527
 528 my $hostpcidesc = {
 529         optional => 1,
 530         type => 'string', format => 'pve-qm-hostpci',
 531         typetext => "[host=]HOSTPCIDEVICE [,driver=kvm|vfio] [,rombar=on|off] [,pcie=0|1] [,x-vga=on|off]",
 532         description => <<EODESCR,
 533 Map host pci devices. HOSTPCIDEVICE syntax is:
 534
 535 'bus:dev.func' (hexadecimal numbers)
 536
 537 You can us the 'lspci' command to list existing pci devices.
 538
 539 The 'rombar' option determines whether or not the device's ROM will be visible in the guest's memory map (default is 'on').
 540
 541 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 542
 543 Experimental: user reported problems with this option.
 544 EODESCR
 545 };
 546 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
 547
 548 my $serialdesc = {
 549         optional => 1,
 550         type => 'string',
 551         pattern => '(/dev/.+|socket)',
 552         description =>  <<EODESCR,
 553 Create a serial device inside the VM (n is 0 to 3), and pass through a host serial device (i.e. /dev/ttyS0), or create a unix socket on the host side (use 'qm terminal' to open a terminal connection).
 554
 555 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 556
 557 Experimental: user reported problems with this option.
 558 EODESCR
 559 };
 560
 561 my $paralleldesc= {
 562         optional => 1,
 563         type => 'string',
 564         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
 565         description =>  <<EODESCR,
 566 Map host parallel devices (n is 0 to 2).
 567
 568 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 569
 570 Experimental: user reported problems with this option.
 571 EODESCR
 572 };
 573
 574 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
 575     $confdesc->{"parallel$i"} = $paralleldesc;
 576 }
 577
 578 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
 579     $confdesc->{"serial$i"} = $serialdesc;
 580 }
 581
 582 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
 583     $confdesc->{"hostpci$i"} = $hostpcidesc;
 584 }
 585
 586 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
 587     $drivename_hash->{"ide$i"} = 1;
 588     $confdesc->{"ide$i"} = $idedesc;
 589 }
 590
 591 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
 592     $drivename_hash->{"sata$i"} = 1;
 593     $confdesc->{"sata$i"} = $satadesc;
 594 }
 595
 596 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
 597     $drivename_hash->{"scsi$i"} = 1;
 598     $confdesc->{"scsi$i"} = $scsidesc ;
 599 }
 600
 601 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
 602     $drivename_hash->{"virtio$i"} = 1;
 603     $confdesc->{"virtio$i"} = $virtiodesc;
 604 }
 605
 606 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
 607     $confdesc->{"usb$i"} = $usbdesc;
 608 }
 609
 610 my $unuseddesc = {
 611     optional => 1,
 612     type => 'string', format => 'pve-volume-id',
 613     description => "Reference to unused volumes.",
 614 };
 615
 616 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
 617     $confdesc->{"unused$i"} = $unuseddesc;
 618 }
 619
 620 my $kvm_api_version = 0;
 621
 622 sub kvm_version {
 623
 624     return $kvm_api_version if $kvm_api_version;
 625
 626     my $fh = IO::File->new("</dev/kvm") ||
 627         return 0;
 628
 629     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
 630         $kvm_api_version = $v;
 631     }
 632
 633     $fh->close();
 634
 635     return  $kvm_api_version;
 636 }
 637
 638 my $kvm_user_version;
 639
 640 sub kvm_user_version {
 641
 642     return $kvm_user_version if $kvm_user_version;
 643
 644     $kvm_user_version = 'unknown';
 645
 646     my $tmp = `kvm -help 2>/dev/null`;
 647
 648     if ($tmp =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)[,\s]/) {
 649         $kvm_user_version = $2;
 650     }
 651
 652     return $kvm_user_version;
 653
 654 }
 655
 656 my $kernel_has_vhost_net = -c '/dev/vhost-net';
 657
 658 sub disknames {
 659     # order is important - used to autoselect boot disk
 660     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
 661             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
 662             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
 663             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))));
 664 }
 665
 666 sub valid_drivename {
 667     my $dev = shift;
 668
 669     return defined($drivename_hash->{$dev});
 670 }
 671
 672 sub option_exists {
 673     my $key = shift;
 674     return defined($confdesc->{$key});
 675 }
 676
 677 sub nic_models {
 678     return $nic_model_list;
 679 }
 680
 681 sub os_list_description {
 682
 683     return {
 684         other => 'Other',
 685         wxp => 'Windows XP',
 686         w2k => 'Windows 2000',
 687         w2k3 =>, 'Windows 2003',
 688         w2k8 => 'Windows 2008',
 689         wvista => 'Windows Vista',
 690         win7 => 'Windows 7',
 691         win8 => 'Windows 8/2012',
 692         l24 => 'Linux 2.4',
 693         l26 => 'Linux 2.6',
 694     };
 695 }
 696
 697 my $cdrom_path;
 698
 699 sub get_cdrom_path {
 700
 701     return  $cdrom_path if $cdrom_path;
 702
 703     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
 704     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
 705     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
 706 }
 707
 708 sub get_iso_path {
 709     my ($storecfg, $vmid, $cdrom) = @_;
 710
 711     if ($cdrom eq 'cdrom') {
 712         return get_cdrom_path();
 713     } elsif ($cdrom eq 'none') {
 714         return '';
 715     } elsif ($cdrom =~ m|^/|) {
 716         return $cdrom;
 717     } else {
 718         return PVE::Storage::path($storecfg, $cdrom);
 719     }
 720 }
 721
 722 # try to convert old style file names to volume IDs
 723 sub filename_to_volume_id {
 724     my ($vmid, $file, $media) = @_;
 725
 726     if (!($file eq 'none' || $file eq 'cdrom' ||
 727           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
 728
 729         return undef if $file =~ m|/|;
 730
 731         if ($media && $media eq 'cdrom') {
 732             $file = "local:iso/$file";
 733         } else {
 734             $file = "local:$vmid/$file";
 735         }
 736     }
 737
 738     return $file;
 739 }
 740
 741 sub verify_media_type {
 742     my ($opt, $vtype, $media) = @_;
 743
 744     return if !$media;
 745
 746     my $etype;
 747     if ($media eq 'disk') {
 748         $etype = 'images';
 749     } elsif ($media eq 'cdrom') {
 750         $etype = 'iso';
 751     } else {
 752         die "internal error";
 753     }
 754
 755     return if ($vtype eq $etype);
 756
 757     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
 758 }
 759
 760 sub cleanup_drive_path {
 761     my ($opt, $storecfg, $drive) = @_;
 762
 763     # try to convert filesystem paths to volume IDs
 764
 765     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
 766         ($drive->{file} !~ m|^/dev/.+|) &&
 767         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
 768         ($drive->{file} !~ m/^\d+$/)) {
 769         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
 770         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
 771         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
 772         verify_media_type($opt, $vtype, $drive->{media});
 773         $drive->{file} = $volid;
 774     }
 775
 776     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
 777 }
 778
 779 sub create_conf_nolock {
 780     my ($vmid, $settings) = @_;
 781
 782     my $filename = config_file($vmid);
 783
 784     die "configuration file '$filename' already exists\n" if -f $filename;
 785
 786     my $defaults = load_defaults();
 787
 788     $settings->{name} = "vm$vmid" if !$settings->{name};
 789     $settings->{memory} = $defaults->{memory} if !$settings->{memory};
 790
 791     my $data = '';
 792     foreach my $opt (keys %$settings) {
 793         next if !$confdesc->{$opt};
 794
 795         my $value = $settings->{$opt};
 796         next if !$value;
 797
 798         $data .= "$opt: $value\n";
 799     }
 800
 801     PVE::Tools::file_set_contents($filename, $data);
 802 }
 803
 804 sub parse_hotplug_features {
 805     my ($data) = @_;
 806
 807     my $res = {};
 808
 809     return $res if $data eq '0';
 810
 811     $data = $confdesc->{hotplug}->{default} if $data eq '1';
 812
 813     foreach my $feature (PVE::Tools::split_list($data)) {
 814         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
 815             $res->{$1} = 1;
 816         } else {
 817             warn "ignoring unknown hotplug feature '$feature'\n";
 818         }
 819     }
 820     return $res;
 821 }
 822
 823 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
 824 sub pve_verify_hotplug_features {
 825     my ($value, $noerr) = @_;
 826
 827     return $value if parse_hotplug_features($value);
 828
 829     return undef if $noerr;
 830
 831     die "unable to parse hotplug option\n";
 832 }
 833
 834 my $parse_size = sub {
 835     my ($value) = @_;
 836
 837     return undef if $value !~ m/^(\d+(\.\d+)?)([KMG])?$/;
 838     my ($size, $unit) = ($1, $3);
 839     if ($unit) {
 840         if ($unit eq 'K') {
 841             $size = $size * 1024;
 842         } elsif ($unit eq 'M') {
 843             $size = $size * 1024 * 1024;
 844         } elsif ($unit eq 'G') {
 845             $size = $size * 1024 * 1024 * 1024;
 846         }
 847     }
 848     return int($size);
 849 };
 850
 851 my $format_size = sub {
 852     my ($size) = @_;
 853
 854     $size = int($size);
 855
 856     my $kb = int($size/1024);
 857     return $size if $kb*1024 != $size;
 858
 859     my $mb = int($kb/1024);
 860     return "${kb}K" if $mb*1024 != $kb;
 861
 862     my $gb = int($mb/1024);
 863     return "${mb}M" if $gb*1024 != $mb;
 864
 865     return "${gb}G";
 866 };
 867
 868 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
 869 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
 870 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
 871 #        [,aio=native|threads][,discard=ignore|on][,iothread=on]
 872
 873 sub parse_drive {
 874     my ($key, $data) = @_;
 875
 876     my $res = {};
 877
 878     # $key may be undefined - used to verify JSON parameters
 879     if (!defined($key)) {
 880         $res->{interface} = 'unknown'; # should not harm when used to verify parameters
 881         $res->{index} = 0;
 882     } elsif ($key =~ m/^([^\d]+)(\d+)$/) {
 883         $res->{interface} = $1;
 884         $res->{index} = $2;
 885     } else {
 886         return undef;
 887     }
 888
 889     foreach my $p (split (/,/, $data)) {
 890         next if $p =~ m/^\s*$/;
 891
 892         if ($p =~ m/^(file|volume|cyls|heads|secs|trans|media|snapshot|cache|format|rerror|werror|backup|aio|bps|mbps|mbps_max|bps_rd|mbps_rd|mbps_rd_max|bps_wr|mbps_wr|mbps_wr_max|iops|iops_max|iops_rd|iops_rd_max|iops_wr|iops_wr_max|size|discard|iothread|queues)=(.+)$/) {
 893             my ($k, $v) = ($1, $2);
 894
 895             $k = 'file' if $k eq 'volume';
 896
 897             return undef if defined $res->{$k};
 898
 899             if ($k eq 'bps' || $k eq 'bps_rd' || $k eq 'bps_wr') {
 900                 return undef if !$v || $v !~ m/^\d+/;
 901                 $k = "m$k";
 902                 $v = sprintf("%.3f", $v / (1024*1024));
 903             }
 904             $res->{$k} = $v;
 905         } else {
 906             if (!$res->{file} && $p !~ m/=/) {
 907                 $res->{file} = $p;
 908             } else {
 909                 return undef;
 910             }
 911         }
 912     }
 913
 914     return undef if !$res->{file};
 915
 916     return undef if $res->{cache} &&
 917         $res->{cache} !~ m/^(off|none|writethrough|writeback|unsafe|directsync)$/;
 918     return undef if $res->{snapshot} && $res->{snapshot} !~ m/^(on|off)$/;
 919     return undef if $res->{cyls} && $res->{cyls} !~ m/^\d+$/;
 920     return undef if $res->{heads} && $res->{heads} !~ m/^\d+$/;
 921     return undef if $res->{secs} && $res->{secs} !~ m/^\d+$/;
 922     return undef if $res->{media} && $res->{media} !~ m/^(disk|cdrom)$/;
 923     return undef if $res->{trans} && $res->{trans} !~ m/^(none|lba|auto)$/;
 924     return undef if $res->{format} && $res->{format} !~ m/^(raw|cow|qcow|qed|qcow2|vmdk|cloop)$/;
 925     return undef if $res->{rerror} && $res->{rerror} !~ m/^(ignore|report|stop)$/;
 926     return undef if $res->{werror} && $res->{werror} !~ m/^(enospc|ignore|report|stop)$/;
 927     return undef if $res->{backup} && $res->{backup} !~ m/^(yes|no)$/;
 928     return undef if $res->{aio} && $res->{aio} !~ m/^(native|threads)$/;
 929     return undef if $res->{discard} && $res->{discard} !~ m/^(ignore|on)$/;
 930     return undef if $res->{iothread} && $res->{iothread} !~ m/^(on)$/;
 931     return undef if $res->{queues} && ($res->{queues} !~ m/^\d+$/ || $res->{queues} < 2);
 932
 933     return undef if $res->{mbps_rd} && $res->{mbps};
 934     return undef if $res->{mbps_wr} && $res->{mbps};
 935
 936     return undef if $res->{mbps} && $res->{mbps} !~ m/^\d+(\.\d+)?$/;
 937     return undef if $res->{mbps_max} && $res->{mbps_max} !~ m/^\d+(\.\d+)?$/;
 938     return undef if $res->{mbps_rd} && $res->{mbps_rd} !~ m/^\d+(\.\d+)?$/;
 939     return undef if $res->{mbps_rd_max} && $res->{mbps_rd_max} !~ m/^\d+(\.\d+)?$/;
 940     return undef if $res->{mbps_wr} && $res->{mbps_wr} !~ m/^\d+(\.\d+)?$/;
 941     return undef if $res->{mbps_wr_max} && $res->{mbps_wr_max} !~ m/^\d+(\.\d+)?$/;
 942
 943     return undef if $res->{iops_rd} && $res->{iops};
 944     return undef if $res->{iops_wr} && $res->{iops};
 945
 946
 947     return undef if $res->{iops} && $res->{iops} !~ m/^\d+$/;
 948     return undef if $res->{iops_max} && $res->{iops_max} !~ m/^\d+$/;
 949     return undef if $res->{iops_rd} && $res->{iops_rd} !~ m/^\d+$/;
 950     return undef if $res->{iops_rd_max} && $res->{iops_rd_max} !~ m/^\d+$/;
 951     return undef if $res->{iops_wr} && $res->{iops_wr} !~ m/^\d+$/;
 952     return undef if $res->{iops_wr_max} && $res->{iops_wr_max} !~ m/^\d+$/;
 953
 954
 955     if ($res->{size}) {
 956         return undef if !defined($res->{size} = &$parse_size($res->{size}));
 957     }
 958
 959     if ($res->{media} && ($res->{media} eq 'cdrom')) {
 960         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
 961         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
 962         return undef if $res->{interface} eq 'virtio';
 963     }
 964
 965     # rerror does not work with scsi drives
 966     if ($res->{rerror}) {
 967         return undef if $res->{interface} eq 'scsi';
 968     }
 969
 970     return $res;
 971 }
 972
 973 my @qemu_drive_options = qw(heads secs cyls trans media format cache snapshot rerror werror aio discard iops iops_rd iops_wr iops_max iops_rd_max iops_wr_max);
 974
 975 sub print_drive {
 976     my ($vmid, $drive) = @_;
 977
 978     my $opts = '';
 979     foreach my $o (@qemu_drive_options, 'mbps', 'mbps_rd', 'mbps_wr', 'mbps_max', 'mbps_rd_max', 'mbps_wr_max', 'backup', 'iothread', 'queues') {
 980         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
 981     }
 982
 983     if ($drive->{size}) {
 984         $opts .= ",size=" . &$format_size($drive->{size});
 985     }
 986
 987     return "$drive->{file}$opts";
 988 }
 989
 990 sub scsi_inquiry {
 991     my($fh, $noerr) = @_;
 992
 993     my $SG_IO = 0x2285;
 994     my $SG_GET_VERSION_NUM = 0x2282;
 995
 996     my $versionbuf = "\x00" x 8;
 997     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
 998     if (!$ret) {
 999         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1000         return undef;
1001     }
1002     my $version = unpack("I", $versionbuf);
1003     if ($version < 30000) {
1004         die "scsi generic interface too old\n"  if !$noerr;
1005         return undef;
1006     }
1007
1008     my $buf = "\x00" x 36;
1009     my $sensebuf = "\x00" x 8;
1010     my $cmd = pack("C x3 C x1", 0x12, 36);
1011
1012     # see /usr/include/scsi/sg.h
1013     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1014
1015     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1016                       length($sensebuf), 0, length($buf), $buf,
1017                       $cmd, $sensebuf, 6000);
1018
1019     $ret = ioctl($fh, $SG_IO, $packet);
1020     if (!$ret) {
1021         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1022         return undef;
1023     }
1024
1025     my @res = unpack($sg_io_hdr_t, $packet);
1026     if ($res[17] || $res[18]) {
1027         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1028         return undef;
1029     }
1030
1031     my $res = {};
1032     (my $byte0, my $byte1, $res->{vendor},
1033      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1034
1035     $res->{removable} = $byte1 & 128 ? 1 : 0;
1036     $res->{type} = $byte0 & 31;
1037
1038     return $res;
1039 }
1040
1041 sub path_is_scsi {
1042     my ($path) = @_;
1043
1044     my $fh = IO::File->new("+<$path") || return undef;
1045     my $res = scsi_inquiry($fh, 1);
1046     close($fh);
1047
1048     return $res;
1049 }
1050
1051 sub machine_type_is_q35 {
1052     my ($conf) = @_;
1053
1054     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1055 }
1056
1057 sub print_tabletdevice_full {
1058     my ($conf) = @_;
1059
1060     my $q35 = machine_type_is_q35($conf);
1061
1062     # we use uhci for old VMs because tablet driver was buggy in older qemu
1063     my $usbbus = $q35 ? "ehci" : "uhci";
1064
1065     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1066 }
1067
1068 sub print_drivedevice_full {
1069     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1070
1071     my $device = '';
1072     my $maxdev = 0;
1073
1074     if ($drive->{interface} eq 'virtio') {
1075         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1076         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1077         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1078     } elsif ($drive->{interface} eq 'scsi') {
1079
1080         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1081         my $unit = $drive->{index} % $maxdev;
1082         my $devicetype = 'hd';
1083         my $path = '';
1084         if (drive_is_cdrom($drive)) {
1085               $devicetype = 'cd';
1086         } else {
1087               if ($drive->{file} =~ m|^/|) {
1088                   $path = $drive->{file};
1089               } else {
1090                   $path = PVE::Storage::path($storecfg, $drive->{file});
1091               }
1092
1093               if($path =~ m/^iscsi\:\/\//){
1094                   $devicetype = 'generic';
1095               } else {
1096                   if (my $info = path_is_scsi($path)) {
1097                       if ($info->{type} == 0) {
1098                           $devicetype = 'block';
1099                       } elsif ($info->{type} == 1) { # tape
1100                           $devicetype = 'generic';
1101                       }
1102                   }
1103               }
1104          }
1105
1106         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1107             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1108         } else {
1109             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1110         }
1111
1112     } elsif ($drive->{interface} eq 'ide'){
1113         $maxdev = 2;
1114         my $controller = int($drive->{index} / $maxdev);
1115         my $unit = $drive->{index} % $maxdev;
1116         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1117
1118         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1119     } elsif ($drive->{interface} eq 'sata'){
1120         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1121         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1122         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1123     } elsif ($drive->{interface} eq 'usb') {
1124         die "implement me";
1125         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1126     } else {
1127         die "unsupported interface type";
1128     }
1129
1130     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1131
1132     return $device;
1133 }
1134
1135 sub get_initiator_name {
1136     my $initiator;
1137
1138     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1139     while (defined(my $line = <$fh>)) {
1140         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1141         $initiator = $1;
1142         last;
1143     }
1144     $fh->close();
1145
1146     return $initiator;
1147 }
1148
1149 sub print_drive_full {
1150     my ($storecfg, $vmid, $drive) = @_;
1151
1152     my $path;
1153     my $volid = $drive->{file};
1154     my $format;
1155
1156     if (drive_is_cdrom($drive)) {
1157         $path = get_iso_path($storecfg, $vmid, $volid);
1158     } else {
1159         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1160         if ($storeid) {
1161             $path = PVE::Storage::path($storecfg, $volid);
1162             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1163             $format = qemu_img_format($scfg, $volname);
1164         } else {
1165             $path = $volid;
1166         }
1167    }
1168
1169     my $opts = '';
1170     foreach my $o (@qemu_drive_options) {
1171         next if $o eq 'bootindex';
1172         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1173     }
1174
1175     $opts .= ",format=$format" if $format && !$drive->{format};
1176
1177     foreach my $o (qw(bps bps_rd bps_wr)) {
1178         my $v = $drive->{"m$o"};
1179         $opts .= ",$o=" . int($v*1024*1024) if $v;
1180     }
1181
1182     my $cache_direct = 0;
1183
1184     if (my $cache = $drive->{cache}) {
1185         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1186     } elsif (!drive_is_cdrom($drive)) {
1187         $opts .= ",cache=none";
1188         $cache_direct = 1;
1189     }
1190
1191     # aio native works only with O_DIRECT
1192     if (!$drive->{aio}) {
1193         if($cache_direct) {
1194             $opts .= ",aio=native";
1195         } else {
1196             $opts .= ",aio=threads";
1197         }
1198     }
1199
1200     my $detectzeroes = $drive->{discard} ? "unmap" : "on";
1201     $opts .= ",detect-zeroes=$detectzeroes" if !drive_is_cdrom($drive);
1202
1203     my $pathinfo = $path ? "file=$path," : '';
1204
1205     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1206 }
1207
1208 sub print_netdevice_full {
1209     my ($vmid, $conf, $net, $netid, $bridges) = @_;
1210
1211     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1212
1213     my $device = $net->{model};
1214     if ($net->{model} eq 'virtio') {
1215          $device = 'virtio-net-pci';
1216      };
1217
1218     my $pciaddr = print_pci_addr("$netid", $bridges);
1219     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1220     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1221         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1222         my $vectors = $net->{queues} * 2 + 2;
1223         $tmpstr .= ",vectors=$vectors,mq=on";
1224     }
1225     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1226     return $tmpstr;
1227 }
1228
1229 sub print_netdev_full {
1230     my ($vmid, $conf, $net, $netid) = @_;
1231
1232     my $i = '';
1233     if ($netid =~ m/^net(\d+)$/) {
1234         $i = int($1);
1235     }
1236
1237     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1238
1239     my $ifname = "tap${vmid}i$i";
1240
1241     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1242     die "interface name '$ifname' is too long (max 15 character)\n"
1243         if length($ifname) >= 16;
1244
1245     my $vhostparam = '';
1246     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1247
1248     my $vmname = $conf->{name} || "vm$vmid";
1249
1250     my $netdev = "";
1251
1252     if ($net->{bridge}) {
1253         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/pve-bridge,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1254     } else {
1255         $netdev = "type=user,id=$netid,hostname=$vmname";
1256     }
1257
1258     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1259
1260     return $netdev;
1261 }
1262
1263 sub drive_is_cdrom {
1264     my ($drive) = @_;
1265
1266     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1267
1268 }
1269
1270 sub parse_numa {
1271     my ($data) = @_;
1272
1273     my $res = {};
1274
1275     foreach my $kvp (split(/,/, $data)) {
1276
1277         if ($kvp =~ m/^memory=(\S+)$/) {
1278             $res->{memory} = $1;
1279         } elsif ($kvp =~ m/^policy=(preferred|bind|interleave)$/) {
1280             $res->{policy} = $1;
1281         } elsif ($kvp =~ m/^cpus=(\d+)(-(\d+))?$/) {
1282             $res->{cpus}->{start} = $1;
1283             $res->{cpus}->{end} = $3;
1284         } elsif ($kvp =~ m/^hostnodes=(\d+)(-(\d+))?$/) {
1285             $res->{hostnodes}->{start} = $1;
1286             $res->{hostnodes}->{end} = $3;
1287         } else {
1288             return undef;
1289         }
1290     }
1291
1292     return $res;
1293 }
1294
1295 sub parse_hostpci {
1296     my ($value) = @_;
1297
1298     return undef if !$value;
1299
1300
1301     my @list = split(/,/, $value);
1302     my $found;
1303
1304     my $res = {};
1305     foreach my $kv (@list) {
1306
1307         if ($kv =~ m/^(host=)?([a-f0-9]{2}:[a-f0-9]{2})(\.([a-f0-9]))?$/) {
1308             $found = 1;
1309             if(defined($4)){
1310                 push @{$res->{pciid}}, { id => $2 , function => $4};
1311
1312             }else{
1313                 my $pcidevices = lspci($2);
1314                 $res->{pciid} = $pcidevices->{$2};
1315             }
1316         } elsif ($kv =~ m/^driver=(kvm|vfio)$/) {
1317             $res->{driver} = $1;
1318         } elsif ($kv =~ m/^rombar=(on|off)$/) {
1319             $res->{rombar} = $1;
1320         } elsif ($kv =~ m/^x-vga=(on|off)$/) {
1321             $res->{'x-vga'} = $1;
1322         } elsif ($kv =~ m/^pcie=(\d+)$/) {
1323             $res->{pcie} = 1 if $1 == 1;
1324         } else {
1325             warn "unknown hostpci setting '$kv'\n";
1326         }
1327     }
1328
1329     return undef if !$found;
1330
1331     return $res;
1332 }
1333
1334 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1335 sub parse_net {
1336     my ($data) = @_;
1337
1338     my $res = {};
1339
1340     foreach my $kvp (split(/,/, $data)) {
1341
1342         if ($kvp =~ m/^(ne2k_pci|e1000|e1000-82540em|e1000-82544gc|e1000-82545em|rtl8139|pcnet|virtio|ne2k_isa|i82551|i82557b|i82559er|vmxnet3)(=([0-9a-f]{2}(:[0-9a-f]{2}){5}))?$/i) {
1343             my $model = lc($1);
1344             my $mac = defined($3) ? uc($3) : PVE::Tools::random_ether_addr();
1345             $res->{model} = $model;
1346             $res->{macaddr} = $mac;
1347         } elsif ($kvp =~ m/^bridge=(\S+)$/) {
1348             $res->{bridge} = $1;
1349         } elsif ($kvp =~ m/^queues=(\d+)$/) {
1350             $res->{queues} = $1;
1351         } elsif ($kvp =~ m/^rate=(\d+(\.\d+)?)$/) {
1352             $res->{rate} = $1;
1353         } elsif ($kvp =~ m/^tag=(\d+)$/) {
1354             $res->{tag} = $1;
1355         } elsif ($kvp =~ m/^firewall=([01])$/) {
1356             $res->{firewall} = $1;
1357         } elsif ($kvp =~ m/^link_down=([01])$/) {
1358             $res->{link_down} = $1;
1359         } else {
1360             return undef;
1361         }
1362
1363     }
1364
1365     return undef if !$res->{model};
1366
1367     return $res;
1368 }
1369
1370 sub print_net {
1371     my $net = shift;
1372
1373     my $res = "$net->{model}";
1374     $res .= "=$net->{macaddr}" if $net->{macaddr};
1375     $res .= ",bridge=$net->{bridge}" if $net->{bridge};
1376     $res .= ",rate=$net->{rate}" if $net->{rate};
1377     $res .= ",tag=$net->{tag}" if $net->{tag};
1378     $res .= ",firewall=1" if $net->{firewall};
1379     $res .= ",link_down=1" if $net->{link_down};
1380     $res .= ",queues=$net->{queues}" if $net->{queues};
1381
1382     return $res;
1383 }
1384
1385 sub add_random_macs {
1386     my ($settings) = @_;
1387
1388     foreach my $opt (keys %$settings) {
1389         next if $opt !~ m/^net(\d+)$/;
1390         my $net = parse_net($settings->{$opt});
1391         next if !$net;
1392         $settings->{$opt} = print_net($net);
1393     }
1394 }
1395
1396 sub add_unused_volume {
1397     my ($config, $volid) = @_;
1398
1399     my $key;
1400     for (my $ind = $MAX_UNUSED_DISKS - 1; $ind >= 0; $ind--) {
1401         my $test = "unused$ind";
1402         if (my $vid = $config->{$test}) {
1403             return if $vid eq $volid; # do not add duplicates
1404         } else {
1405             $key = $test;
1406         }
1407     }
1408
1409     die "To many unused volume - please delete them first.\n" if !$key;
1410
1411     $config->{$key} = $volid;
1412
1413     return $key;
1414 }
1415
1416 sub vm_is_volid_owner {
1417     my ($storecfg, $vmid, $volid) = @_;
1418
1419     if ($volid !~  m|^/|) {
1420         my ($path, $owner);
1421         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1422         if ($owner && ($owner == $vmid)) {
1423             return 1;
1424         }
1425     }
1426
1427     return undef;
1428 }
1429
1430 sub vmconfig_delete_pending_option {
1431     my ($conf, $key) = @_;
1432
1433     delete $conf->{pending}->{$key};
1434     my $pending_delete_hash = { $key => 1 };
1435     foreach my $opt (PVE::Tools::split_list($conf->{pending}->{delete})) {
1436         $pending_delete_hash->{$opt} = 1;
1437     }
1438     $conf->{pending}->{delete} = join(',', keys %$pending_delete_hash);
1439 }
1440
1441 sub vmconfig_undelete_pending_option {
1442     my ($conf, $key) = @_;
1443
1444     my $pending_delete_hash = {};
1445     foreach my $opt (PVE::Tools::split_list($conf->{pending}->{delete})) {
1446         $pending_delete_hash->{$opt} = 1;
1447     }
1448     delete $pending_delete_hash->{$key};
1449
1450     my @keylist = keys %$pending_delete_hash;
1451     if (scalar(@keylist)) {
1452         $conf->{pending}->{delete} = join(',', @keylist);
1453     } else {
1454         delete $conf->{pending}->{delete};
1455     }
1456 }
1457
1458 sub vmconfig_register_unused_drive {
1459     my ($storecfg, $vmid, $conf, $drive) = @_;
1460
1461     if (!drive_is_cdrom($drive)) {
1462         my $volid = $drive->{file};
1463         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1464             add_unused_volume($conf, $volid, $vmid);
1465         }
1466     }
1467 }
1468
1469 sub vmconfig_cleanup_pending {
1470     my ($conf) = @_;
1471
1472     # remove pending changes when nothing changed
1473     my $changes;
1474     foreach my $opt (keys %{$conf->{pending}}) {
1475         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
1476             $changes = 1;
1477             delete $conf->{pending}->{$opt};
1478         }
1479     }
1480
1481     # remove delete if option is not set
1482     my $pending_delete_hash = {};
1483     foreach my $opt (PVE::Tools::split_list($conf->{pending}->{delete})) {
1484         if (defined($conf->{$opt})) {
1485             $pending_delete_hash->{$opt} = 1;
1486         } else {
1487             $changes = 1;
1488         }
1489     }
1490
1491     my @keylist = keys %$pending_delete_hash;
1492     if (scalar(@keylist)) {
1493         $conf->{pending}->{delete} = join(',', @keylist);
1494     } else {
1495         delete $conf->{pending}->{delete};
1496     }
1497
1498     return $changes;
1499 }
1500
1501 my $valid_smbios1_options = {
1502     manufacturer => '\S+',
1503     product => '\S+',
1504     version => '\S+',
1505     serial => '\S+',
1506     uuid => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1507     sku => '\S+',
1508     family => '\S+',
1509 };
1510
1511 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
1512 sub parse_smbios1 {
1513     my ($data) = @_;
1514
1515     my $res = {};
1516
1517     foreach my $kvp (split(/,/, $data)) {
1518         return undef if $kvp !~ m/^(\S+)=(.+)$/;
1519         my ($k, $v) = split(/=/, $kvp);
1520         return undef if !defined($k) || !defined($v);
1521         return undef if !$valid_smbios1_options->{$k};
1522         return undef if $v !~ m/^$valid_smbios1_options->{$k}$/;
1523         $res->{$k} = $v;
1524     }
1525
1526     return $res;
1527 }
1528
1529 sub print_smbios1 {
1530     my ($smbios1) = @_;
1531
1532     my $data = '';
1533     foreach my $k (keys %$smbios1) {
1534         next if !defined($smbios1->{$k});
1535         next if !$valid_smbios1_options->{$k};
1536         $data .= ',' if $data;
1537         $data .= "$k=$smbios1->{$k}";
1538     }
1539     return $data;
1540 }
1541
1542 PVE::JSONSchema::register_format('pve-qm-smbios1', \&verify_smbios1);
1543 sub verify_smbios1 {
1544     my ($value, $noerr) = @_;
1545
1546     return $value if parse_smbios1($value);
1547
1548     return undef if $noerr;
1549
1550     die "unable to parse smbios (type 1) options\n";
1551 }
1552
1553 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
1554 sub verify_bootdisk {
1555     my ($value, $noerr) = @_;
1556
1557     return $value if valid_drivename($value);
1558
1559     return undef if $noerr;
1560
1561     die "invalid boot disk '$value'\n";
1562 }
1563
1564 PVE::JSONSchema::register_format('pve-qm-numanode', \&verify_numa);
1565 sub verify_numa {
1566     my ($value, $noerr) = @_;
1567
1568     return $value if parse_numa($value);
1569
1570     return undef if $noerr;
1571
1572     die "unable to parse numa options\n";
1573 }
1574
1575 PVE::JSONSchema::register_format('pve-qm-net', \&verify_net);
1576 sub verify_net {
1577     my ($value, $noerr) = @_;
1578
1579     return $value if parse_net($value);
1580
1581     return undef if $noerr;
1582
1583     die "unable to parse network options\n";
1584 }
1585
1586 PVE::JSONSchema::register_format('pve-qm-drive', \&verify_drive);
1587 sub verify_drive {
1588     my ($value, $noerr) = @_;
1589
1590     return $value if parse_drive(undef, $value);
1591
1592     return undef if $noerr;
1593
1594     die "unable to parse drive options\n";
1595 }
1596
1597 PVE::JSONSchema::register_format('pve-qm-hostpci', \&verify_hostpci);
1598 sub verify_hostpci {
1599     my ($value, $noerr) = @_;
1600
1601     return $value if parse_hostpci($value);
1602
1603     return undef if $noerr;
1604
1605     die "unable to parse pci id\n";
1606 }
1607
1608 PVE::JSONSchema::register_format('pve-qm-watchdog', \&verify_watchdog);
1609 sub verify_watchdog {
1610     my ($value, $noerr) = @_;
1611
1612     return $value if parse_watchdog($value);
1613
1614     return undef if $noerr;
1615
1616     die "unable to parse watchdog options\n";
1617 }
1618
1619 sub parse_watchdog {
1620     my ($value) = @_;
1621
1622     return undef if !$value;
1623
1624     my $res = {};
1625
1626     foreach my $p (split(/,/, $value)) {
1627         next if $p =~ m/^\s*$/;
1628
1629         if ($p =~ m/^(model=)?(i6300esb|ib700)$/) {
1630             $res->{model} = $2;
1631         } elsif ($p =~ m/^(action=)?(reset|shutdown|poweroff|pause|debug|none)$/) {
1632             $res->{action} = $2;
1633         } else {
1634             return undef;
1635         }
1636     }
1637
1638     return $res;
1639 }
1640
1641 sub parse_usb_device {
1642     my ($value) = @_;
1643
1644     return undef if !$value;
1645
1646     my @dl = split(/,/, $value);
1647     my $found;
1648
1649     my $res = {};
1650     foreach my $v (@dl) {
1651         if ($v =~ m/^host=(0x)?([0-9A-Fa-f]{4}):(0x)?([0-9A-Fa-f]{4})$/) {
1652             $found = 1;
1653             $res->{vendorid} = $2;
1654             $res->{productid} = $4;
1655         } elsif ($v =~ m/^host=(\d+)\-(\d+(\.\d+)*)$/) {
1656             $found = 1;
1657             $res->{hostbus} = $1;
1658             $res->{hostport} = $2;
1659         } elsif ($v =~ m/^spice$/) {
1660             $found = 1;
1661             $res->{spice} = 1;
1662         } else {
1663             return undef;
1664         }
1665     }
1666     return undef if !$found;
1667
1668     return $res;
1669 }
1670
1671 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
1672 sub verify_usb_device {
1673     my ($value, $noerr) = @_;
1674
1675     return $value if parse_usb_device($value);
1676
1677     return undef if $noerr;
1678
1679     die "unable to parse usb device\n";
1680 }
1681
1682 # add JSON properties for create and set function
1683 sub json_config_properties {
1684     my $prop = shift;
1685
1686     foreach my $opt (keys %$confdesc) {
1687         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
1688         $prop->{$opt} = $confdesc->{$opt};
1689     }
1690
1691     return $prop;
1692 }
1693
1694 sub check_type {
1695     my ($key, $value) = @_;
1696
1697     die "unknown setting '$key'\n" if !$confdesc->{$key};
1698
1699     my $type = $confdesc->{$key}->{type};
1700
1701     if (!defined($value)) {
1702         die "got undefined value\n";
1703     }
1704
1705     if ($value =~ m/[\n\r]/) {
1706         die "property contains a line feed\n";
1707     }
1708
1709     if ($type eq 'boolean') {
1710         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
1711         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
1712         die "type check ('boolean') failed - got '$value'\n";
1713     } elsif ($type eq 'integer') {
1714         return int($1) if $value =~ m/^(\d+)$/;
1715         die "type check ('integer') failed - got '$value'\n";
1716     } elsif ($type eq 'number') {
1717         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
1718         die "type check ('number') failed - got '$value'\n";
1719     } elsif ($type eq 'string') {
1720         if (my $fmt = $confdesc->{$key}->{format}) {
1721             if ($fmt eq 'pve-qm-drive') {
1722                 # special case - we need to pass $key to parse_drive()
1723                 my $drive = parse_drive($key, $value);
1724                 return $value if $drive;
1725                 die "unable to parse drive options\n";
1726             }
1727             PVE::JSONSchema::check_format($fmt, $value);
1728             return $value;
1729         }
1730         $value =~ s/^\"(.*)\"$/$1/;
1731         return $value;
1732     } else {
1733         die "internal error"
1734     }
1735 }
1736
1737 sub lock_config_full {
1738     my ($vmid, $timeout, $code, @param) = @_;
1739
1740     my $filename = config_file_lock($vmid);
1741
1742     my $res = lock_file($filename, $timeout, $code, @param);
1743
1744     die $@ if $@;
1745
1746     return $res;
1747 }
1748
1749 sub lock_config_mode {
1750     my ($vmid, $timeout, $shared, $code, @param) = @_;
1751
1752     my $filename = config_file_lock($vmid);
1753
1754     my $res = lock_file_full($filename, $timeout, $shared, $code, @param);
1755
1756     die $@ if $@;
1757
1758     return $res;
1759 }
1760
1761 sub lock_config {
1762     my ($vmid, $code, @param) = @_;
1763
1764     return lock_config_full($vmid, 10, $code, @param);
1765 }
1766
1767 sub cfs_config_path {
1768     my ($vmid, $node) = @_;
1769
1770     $node = $nodename if !$node;
1771     return "nodes/$node/qemu-server/$vmid.conf";
1772 }
1773
1774 sub check_iommu_support{
1775     #fixme : need to check IOMMU support
1776     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
1777
1778     my $iommu=1;
1779     return $iommu;
1780
1781 }
1782
1783 sub config_file {
1784     my ($vmid, $node) = @_;
1785
1786     my $cfspath = cfs_config_path($vmid, $node);
1787     return "/etc/pve/$cfspath";
1788 }
1789
1790 sub config_file_lock {
1791     my ($vmid) = @_;
1792
1793     return "$lock_dir/lock-$vmid.conf";
1794 }
1795
1796 sub touch_config {
1797     my ($vmid) = @_;
1798
1799     my $conf = config_file($vmid);
1800     utime undef, undef, $conf;
1801 }
1802
1803 sub destroy_vm {
1804     my ($storecfg, $vmid, $keep_empty_config) = @_;
1805
1806     my $conffile = config_file($vmid);
1807
1808     my $conf = load_config($vmid);
1809
1810     check_lock($conf);
1811
1812     # only remove disks owned by this VM
1813     foreach_drive($conf, sub {
1814         my ($ds, $drive) = @_;
1815
1816         return if drive_is_cdrom($drive);
1817
1818         my $volid = $drive->{file};
1819
1820         return if !$volid || $volid =~ m|^/|;
1821
1822         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
1823         return if !$path || !$owner || ($owner != $vmid);
1824
1825         PVE::Storage::vdisk_free($storecfg, $volid);
1826     });
1827
1828     if ($keep_empty_config) {
1829         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
1830     } else {
1831         unlink $conffile;
1832     }
1833
1834     # also remove unused disk
1835     eval {
1836         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
1837
1838         eval {
1839             PVE::Storage::foreach_volid($dl, sub {
1840                 my ($volid, $sid, $volname, $d) = @_;
1841                 PVE::Storage::vdisk_free($storecfg, $volid);
1842             });
1843         };
1844         warn $@ if $@;
1845
1846     };
1847     warn $@ if $@;
1848 }
1849
1850 sub load_config {
1851     my ($vmid, $node) = @_;
1852
1853     my $cfspath = cfs_config_path($vmid, $node);
1854
1855     my $conf = PVE::Cluster::cfs_read_file($cfspath);
1856
1857     die "no such VM ('$vmid')\n" if !defined($conf);
1858
1859     return $conf;
1860 }
1861
1862 sub parse_vm_config {
1863     my ($filename, $raw) = @_;
1864
1865     return undef if !defined($raw);
1866
1867     my $res = {
1868         digest => Digest::SHA::sha1_hex($raw),
1869         snapshots => {},
1870         pending => {},
1871     };
1872
1873     $filename =~ m|/qemu-server/(\d+)\.conf$|
1874         || die "got strange filename '$filename'";
1875
1876     my $vmid = $1;
1877
1878     my $conf = $res;
1879     my $descr = '';
1880     my $section = '';
1881
1882     my @lines = split(/\n/, $raw);
1883     foreach my $line (@lines) {
1884         next if $line =~ m/^\s*$/;
1885
1886         if ($line =~ m/^\[PENDING\]\s*$/i) {
1887             $section = 'pending';
1888             $conf->{description} = $descr if $descr;
1889             $descr = '';
1890             $conf = $res->{$section} = {};
1891             next;
1892
1893         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
1894             $section = $1;
1895             $conf->{description} = $descr if $descr;
1896             $descr = '';
1897             $conf = $res->{snapshots}->{$section} = {};
1898             next;
1899         }
1900
1901         if ($line =~ m/^\#(.*)\s*$/) {
1902             $descr .= PVE::Tools::decode_text($1) . "\n";
1903             next;
1904         }
1905
1906         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
1907             $descr .= PVE::Tools::decode_text($2);
1908         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
1909             $conf->{snapstate} = $1;
1910         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
1911             my $key = $1;
1912             my $value = $2;
1913             $conf->{$key} = $value;
1914         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
1915             my $value = $1;
1916             if ($section eq 'pending') {
1917                 $conf->{delete} = $value; # we parse this later
1918             } else {
1919                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
1920             }
1921         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
1922             my $key = $1;
1923             my $value = $2;
1924             eval { $value = check_type($key, $value); };
1925             if ($@) {
1926                 warn "vm $vmid - unable to parse value of '$key' - $@";
1927             } else {
1928                 my $fmt = $confdesc->{$key}->{format};
1929                 if ($fmt && $fmt eq 'pve-qm-drive') {
1930                     my $v = parse_drive($key, $value);
1931                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
1932                         $v->{file} = $volid;
1933                         $value = print_drive($vmid, $v);
1934                     } else {
1935                         warn "vm $vmid - unable to parse value of '$key'\n";
1936                         next;
1937                     }
1938                 }
1939
1940                 if ($key eq 'cdrom') {
1941                     $conf->{ide2} = $value;
1942                 } else {
1943                     $conf->{$key} = $value;
1944                 }
1945             }
1946         }
1947     }
1948
1949     $conf->{description} = $descr if $descr;
1950
1951     delete $res->{snapstate}; # just to be sure
1952
1953     return $res;
1954 }
1955
1956 sub write_vm_config {
1957     my ($filename, $conf) = @_;
1958
1959     delete $conf->{snapstate}; # just to be sure
1960
1961     if ($conf->{cdrom}) {
1962         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
1963         $conf->{ide2} = $conf->{cdrom};
1964         delete $conf->{cdrom};
1965     }
1966
1967     # we do not use 'smp' any longer
1968     if ($conf->{sockets}) {
1969         delete $conf->{smp};
1970     } elsif ($conf->{smp}) {
1971         $conf->{sockets} = $conf->{smp};
1972         delete $conf->{cores};
1973         delete $conf->{smp};
1974     }
1975
1976     my $used_volids = {};
1977
1978     my $cleanup_config = sub {
1979         my ($cref, $pending, $snapname) = @_;
1980
1981         foreach my $key (keys %$cref) {
1982             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
1983                 $key eq 'snapstate' || $key eq 'pending';
1984             my $value = $cref->{$key};
1985             if ($key eq 'delete') {
1986                 die "propertry 'delete' is only allowed in [PENDING]\n"
1987                     if !$pending;
1988                 # fixme: check syntax?
1989                 next;
1990             }
1991             eval { $value = check_type($key, $value); };
1992             die "unable to parse value of '$key' - $@" if $@;
1993
1994             $cref->{$key} = $value;
1995
1996             if (!$snapname && valid_drivename($key)) {
1997                 my $drive = parse_drive($key, $value);
1998                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
1999             }
2000         }
2001     };
2002
2003     &$cleanup_config($conf);
2004
2005     &$cleanup_config($conf->{pending}, 1);
2006
2007     foreach my $snapname (keys %{$conf->{snapshots}}) {
2008         die "internal error" if $snapname eq 'pending';
2009         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2010     }
2011
2012     # remove 'unusedX' settings if we re-add a volume
2013     foreach my $key (keys %$conf) {
2014         my $value = $conf->{$key};
2015         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2016             delete $conf->{$key};
2017         }
2018     }
2019
2020     my $generate_raw_config = sub {
2021         my ($conf) = @_;
2022
2023         my $raw = '';
2024
2025         # add description as comment to top of file
2026         my $descr = $conf->{description} || '';
2027         foreach my $cl (split(/\n/, $descr)) {
2028             $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2029         }
2030
2031         foreach my $key (sort keys %$conf) {
2032             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2033             $raw .= "$key: $conf->{$key}\n";
2034         }
2035         return $raw;
2036     };
2037
2038     my $raw = &$generate_raw_config($conf);
2039
2040     if (scalar(keys %{$conf->{pending}})){
2041         $raw .= "\n[PENDING]\n";
2042         $raw .= &$generate_raw_config($conf->{pending});
2043     }
2044
2045     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2046         $raw .= "\n[$snapname]\n";
2047         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2048     }
2049
2050     return $raw;
2051 }
2052
2053 sub update_config_nolock {
2054     my ($vmid, $conf, $skiplock) = @_;
2055
2056     check_lock($conf) if !$skiplock;
2057
2058     my $cfspath = cfs_config_path($vmid);
2059
2060     PVE::Cluster::cfs_write_file($cfspath, $conf);
2061 }
2062
2063 sub update_config {
2064     my ($vmid, $conf, $skiplock) = @_;
2065
2066     lock_config($vmid, &update_config_nolock, $conf, $skiplock);
2067 }
2068
2069 sub load_defaults {
2070
2071     my $res = {};
2072
2073     # we use static defaults from our JSON schema configuration
2074     foreach my $key (keys %$confdesc) {
2075         if (defined(my $default = $confdesc->{$key}->{default})) {
2076             $res->{$key} = $default;
2077         }
2078     }
2079
2080     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2081     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2082
2083     return $res;
2084 }
2085
2086 sub config_list {
2087     my $vmlist = PVE::Cluster::get_vmlist();
2088     my $res = {};
2089     return $res if !$vmlist || !$vmlist->{ids};
2090     my $ids = $vmlist->{ids};
2091
2092     foreach my $vmid (keys %$ids) {
2093         my $d = $ids->{$vmid};
2094         next if !$d->{node} || $d->{node} ne $nodename;
2095         next if !$d->{type} || $d->{type} ne 'qemu';
2096         $res->{$vmid}->{exists} = 1;
2097     }
2098     return $res;
2099 }
2100
2101 # test if VM uses local resources (to prevent migration)
2102 sub check_local_resources {
2103     my ($conf, $noerr) = @_;
2104
2105     my $loc_res = 0;
2106
2107     $loc_res = 1 if $conf->{hostusb}; # old syntax
2108     $loc_res = 1 if $conf->{hostpci}; # old syntax
2109
2110     foreach my $k (keys %$conf) {
2111         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2112         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2113     }
2114
2115     die "VM uses local resources\n" if $loc_res && !$noerr;
2116
2117     return $loc_res;
2118 }
2119
2120 # check if used storages are available on all nodes (use by migrate)
2121 sub check_storage_availability {
2122     my ($storecfg, $conf, $node) = @_;
2123
2124     foreach_drive($conf, sub {
2125         my ($ds, $drive) = @_;
2126
2127         my $volid = $drive->{file};
2128         return if !$volid;
2129
2130         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2131         return if !$sid;
2132
2133         # check if storage is available on both nodes
2134         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2135         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2136    });
2137 }
2138
2139 # list nodes where all VM images are available (used by has_feature API)
2140 sub shared_nodes {
2141     my ($conf, $storecfg) = @_;
2142
2143     my $nodelist = PVE::Cluster::get_nodelist();
2144     my $nodehash = { map { $_ => 1 } @$nodelist };
2145     my $nodename = PVE::INotify::nodename();
2146
2147     foreach_drive($conf, sub {
2148         my ($ds, $drive) = @_;
2149
2150         my $volid = $drive->{file};
2151         return if !$volid;
2152
2153         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2154         if ($storeid) {
2155             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2156             if ($scfg->{disable}) {
2157                 $nodehash = {};
2158             } elsif (my $avail = $scfg->{nodes}) {
2159                 foreach my $node (keys %$nodehash) {
2160                     delete $nodehash->{$node} if !$avail->{$node};
2161                 }
2162             } elsif (!$scfg->{shared}) {
2163                 foreach my $node (keys %$nodehash) {
2164                     delete $nodehash->{$node} if $node ne $nodename
2165                 }
2166             }
2167         }
2168     });
2169
2170     return $nodehash
2171 }
2172
2173 sub check_lock {
2174     my ($conf) = @_;
2175
2176     die "VM is locked ($conf->{lock})\n" if $conf->{lock};
2177 }
2178
2179 sub check_cmdline {
2180     my ($pidfile, $pid) = @_;
2181
2182     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2183     if (defined($fh)) {
2184         my $line = <$fh>;
2185         $fh->close;
2186         return undef if !$line;
2187         my @param = split(/\0/, $line);
2188
2189         my $cmd = $param[0];
2190         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2191
2192         for (my $i = 0; $i < scalar (@param); $i++) {
2193             my $p = $param[$i];
2194             next if !$p;
2195             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2196                 my $p = $param[$i+1];
2197                 return 1 if $p && ($p eq $pidfile);
2198                 return undef;
2199             }
2200         }
2201     }
2202     return undef;
2203 }
2204
2205 sub check_running {
2206     my ($vmid, $nocheck, $node) = @_;
2207
2208     my $filename = config_file($vmid, $node);
2209
2210     die "unable to find configuration file for VM $vmid - no such machine\n"
2211         if !$nocheck && ! -f $filename;
2212
2213     my $pidfile = pidfile_name($vmid);
2214
2215     if (my $fd = IO::File->new("<$pidfile")) {
2216         my $st = stat($fd);
2217         my $line = <$fd>;
2218         close($fd);
2219
2220         my $mtime = $st->mtime;
2221         if ($mtime > time()) {
2222             warn "file '$filename' modified in future\n";
2223         }
2224
2225         if ($line =~ m/^(\d+)$/) {
2226             my $pid = $1;
2227             if (check_cmdline($pidfile, $pid)) {
2228                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2229                     return $pid;
2230                 }
2231             }
2232         }
2233     }
2234
2235     return undef;
2236 }
2237
2238 sub vzlist {
2239
2240     my $vzlist = config_list();
2241
2242     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2243
2244     while (defined(my $de = $fd->read)) {
2245         next if $de !~ m/^(\d+)\.pid$/;
2246         my $vmid = $1;
2247         next if !defined($vzlist->{$vmid});
2248         if (my $pid = check_running($vmid)) {
2249             $vzlist->{$vmid}->{pid} = $pid;
2250         }
2251     }
2252
2253     return $vzlist;
2254 }
2255
2256 sub disksize {
2257     my ($storecfg, $conf) = @_;
2258
2259     my $bootdisk = $conf->{bootdisk};
2260     return undef if !$bootdisk;
2261     return undef if !valid_drivename($bootdisk);
2262
2263     return undef if !$conf->{$bootdisk};
2264
2265     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2266     return undef if !defined($drive);
2267
2268     return undef if drive_is_cdrom($drive);
2269
2270     my $volid = $drive->{file};
2271     return undef if !$volid;
2272
2273     return $drive->{size};
2274 }
2275
2276 my $last_proc_pid_stat;
2277
2278 # get VM status information
2279 # This must be fast and should not block ($full == false)
2280 # We only query KVM using QMP if $full == true (this can be slow)
2281 sub vmstatus {
2282     my ($opt_vmid, $full) = @_;
2283
2284     my $res = {};
2285
2286     my $storecfg = PVE::Storage::config();
2287
2288     my $list = vzlist();
2289     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2290
2291     my $cpucount = $cpuinfo->{cpus} || 1;
2292
2293     foreach my $vmid (keys %$list) {
2294         next if $opt_vmid && ($vmid ne $opt_vmid);
2295
2296         my $cfspath = cfs_config_path($vmid);
2297         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2298
2299         my $d = {};
2300         $d->{pid} = $list->{$vmid}->{pid};
2301
2302         # fixme: better status?
2303         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2304
2305         my $size = disksize($storecfg, $conf);
2306         if (defined($size)) {
2307             $d->{disk} = 0; # no info available
2308             $d->{maxdisk} = $size;
2309         } else {
2310             $d->{disk} = 0;
2311             $d->{maxdisk} = 0;
2312         }
2313
2314         $d->{cpus} = ($conf->{sockets} || 1) * ($conf->{cores} || 1);
2315         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2316         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2317
2318         $d->{name} = $conf->{name} || "VM $vmid";
2319         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024) : 0;
2320
2321         if ($conf->{balloon}) {
2322             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2323             $d->{shares} = defined($conf->{shares}) ? $conf->{shares} : 1000;
2324         }
2325
2326         $d->{uptime} = 0;
2327         $d->{cpu} = 0;
2328         $d->{mem} = 0;
2329
2330         $d->{netout} = 0;
2331         $d->{netin} = 0;
2332
2333         $d->{diskread} = 0;
2334         $d->{diskwrite} = 0;
2335
2336         $d->{template} = is_template($conf);
2337
2338         $res->{$vmid} = $d;
2339     }
2340
2341     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2342     foreach my $dev (keys %$netdev) {
2343         next if $dev !~ m/^tap([1-9]\d*)i/;
2344         my $vmid = $1;
2345         my $d = $res->{$vmid};
2346         next if !$d;
2347
2348         $d->{netout} += $netdev->{$dev}->{receive};
2349         $d->{netin} += $netdev->{$dev}->{transmit};
2350
2351         if ($full) {
2352             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2353             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2354         }
2355
2356     }
2357
2358     my $ctime = gettimeofday;
2359
2360     foreach my $vmid (keys %$list) {
2361
2362         my $d = $res->{$vmid};
2363         my $pid = $d->{pid};
2364         next if !$pid;
2365
2366         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2367         next if !$pstat; # not running
2368
2369         my $used = $pstat->{utime} + $pstat->{stime};
2370
2371         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2372
2373         if ($pstat->{vsize}) {
2374             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2375         }
2376
2377         my $old = $last_proc_pid_stat->{$pid};
2378         if (!$old) {
2379             $last_proc_pid_stat->{$pid} = {
2380                 time => $ctime,
2381                 used => $used,
2382                 cpu => 0,
2383             };
2384             next;
2385         }
2386
2387         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2388
2389         if ($dtime > 1000) {
2390             my $dutime = $used -  $old->{used};
2391
2392             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2393             $last_proc_pid_stat->{$pid} = {
2394                 time => $ctime,
2395                 used => $used,
2396                 cpu => $d->{cpu},
2397             };
2398         } else {
2399             $d->{cpu} = $old->{cpu};
2400         }
2401     }
2402
2403     return $res if !$full;
2404
2405     my $qmpclient = PVE::QMPClient->new();
2406
2407     my $ballooncb = sub {
2408         my ($vmid, $resp) = @_;
2409
2410         my $info = $resp->{'return'};
2411         return if !$info->{max_mem};
2412
2413         my $d = $res->{$vmid};
2414
2415         # use memory assigned to VM
2416         $d->{maxmem} = $info->{max_mem};
2417         $d->{balloon} = $info->{actual};
2418
2419         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2420             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2421             $d->{freemem} = $info->{free_mem};
2422         }
2423
2424         $d->{ballooninfo} = $info;
2425     };
2426
2427     my $blockstatscb = sub {
2428         my ($vmid, $resp) = @_;
2429         my $data = $resp->{'return'} || [];
2430         my $totalrdbytes = 0;
2431         my $totalwrbytes = 0;
2432
2433         for my $blockstat (@$data) {
2434             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2435             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2436
2437             $blockstat->{device} =~ s/drive-//;
2438             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2439         }
2440         $res->{$vmid}->{diskread} = $totalrdbytes;
2441         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2442     };
2443
2444     my $statuscb = sub {
2445         my ($vmid, $resp) = @_;
2446
2447         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2448         # this fails if ballon driver is not loaded, so this must be
2449         # the last commnand (following command are aborted if this fails).
2450         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2451
2452         my $status = 'unknown';
2453         if (!defined($status = $resp->{'return'}->{status})) {
2454             warn "unable to get VM status\n";
2455             return;
2456         }
2457
2458         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2459     };
2460
2461     foreach my $vmid (keys %$list) {
2462         next if $opt_vmid && ($vmid ne $opt_vmid);
2463         next if !$res->{$vmid}->{pid}; # not running
2464         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2465     }
2466
2467     $qmpclient->queue_execute(undef, 1);
2468
2469     foreach my $vmid (keys %$list) {
2470         next if $opt_vmid && ($vmid ne $opt_vmid);
2471         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2472     }
2473
2474     return $res;
2475 }
2476
2477 sub foreach_dimm {
2478     my ($conf, $vmid, $memory, $sockets, $func) = @_;
2479
2480     my $dimm_id = 0;
2481     my $current_size = 1024;
2482     my $dimm_size = 512;
2483     return if $current_size == $memory;
2484
2485     for (my $j = 0; $j < 8; $j++) {
2486         for (my $i = 0; $i < 32; $i++) {
2487             my $name = "dimm${dimm_id}";
2488             $dimm_id++;
2489             my $numanode = $i % $sockets;
2490             $current_size += $dimm_size;
2491             &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
2492             return  $current_size if $current_size >= $memory;
2493         }
2494         $dimm_size *= 2;
2495     }
2496 }
2497
2498 sub foreach_drive {
2499     my ($conf, $func) = @_;
2500
2501     foreach my $ds (keys %$conf) {
2502         next if !valid_drivename($ds);
2503
2504         my $drive = parse_drive($ds, $conf->{$ds});
2505         next if !$drive;
2506
2507         &$func($ds, $drive);
2508     }
2509 }
2510
2511 sub foreach_volid {
2512     my ($conf, $func) = @_;
2513
2514     my $volhash = {};
2515
2516     my $test_volid = sub {
2517         my ($volid, $is_cdrom) = @_;
2518
2519         return if !$volid;
2520
2521         $volhash->{$volid} = $is_cdrom || 0;
2522     };
2523
2524     foreach_drive($conf, sub {
2525         my ($ds, $drive) = @_;
2526         &$test_volid($drive->{file}, drive_is_cdrom($drive));
2527     });
2528
2529     foreach my $snapname (keys %{$conf->{snapshots}}) {
2530         my $snap = $conf->{snapshots}->{$snapname};
2531         &$test_volid($snap->{vmstate}, 0);
2532         foreach_drive($snap, sub {
2533             my ($ds, $drive) = @_;
2534             &$test_volid($drive->{file}, drive_is_cdrom($drive));
2535         });
2536     }
2537
2538     foreach my $volid (keys %$volhash) {
2539         &$func($volid, $volhash->{$volid});
2540     }
2541 }
2542
2543 sub vga_conf_has_spice {
2544     my ($vga) = @_;
2545
2546     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
2547
2548     return $1 || 1;
2549 }
2550
2551 sub config_to_command {
2552     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2553
2554     my $cmd = [];
2555     my $globalFlags = [];
2556     my $machineFlags = [];
2557     my $rtcFlags = [];
2558     my $cpuFlags = [];
2559     my $devices = [];
2560     my $pciaddr = '';
2561     my $bridges = {};
2562     my $kvmver = kvm_user_version();
2563     my $vernum = 0; # unknown
2564     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
2565         $vernum = $1*1000000+$2*1000;
2566     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
2567         $vernum = $1*1000000+$2*1000+$3;
2568     }
2569
2570     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
2571
2572     my $have_ovz = -f '/proc/vz/vestat';
2573
2574     my $q35 = machine_type_is_q35($conf);
2575     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2576     my $machine_type = $forcemachine || $conf->{machine};
2577
2578     my $cpuunits = defined($conf->{cpuunits}) ?
2579             $conf->{cpuunits} : $defaults->{cpuunits};
2580
2581     push @$cmd, '/usr/bin/systemd-run';
2582     push @$cmd, '--scope';
2583     push @$cmd, '--slice', "qemu";
2584     push @$cmd, '--unit', $vmid;
2585     push @$cmd, '-p', "CPUShares=$cpuunits";
2586     if ($conf->{cpulimit}) {
2587         my $cpulimit = int($conf->{cpulimit} * 100);
2588         push @$cmd, '-p', "CPUQuota=$cpulimit\%";
2589     }
2590
2591     push @$cmd, '/usr/bin/kvm';
2592
2593     push @$cmd, '-id', $vmid;
2594
2595     my $use_virtio = 0;
2596
2597     my $qmpsocket = qmp_socket($vmid);
2598     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2599     push @$cmd, '-mon', "chardev=qmp,mode=control";
2600
2601     my $socket = vnc_socket($vmid);
2602     push @$cmd,  '-vnc', "unix:$socket,x509,password";
2603
2604     push @$cmd, '-pidfile' , pidfile_name($vmid);
2605
2606     push @$cmd, '-daemonize';
2607
2608     if ($conf->{smbios1}) {
2609         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
2610     }
2611
2612     if ($q35) {
2613         # the q35 chipset support native usb2, so we enable usb controller
2614         # by default for this machine type
2615         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
2616     } else {
2617         $pciaddr = print_pci_addr("piix3", $bridges);
2618         push @$devices, '-device', "piix3-usb-uhci,id=uhci$pciaddr.0x2";
2619
2620         my $use_usb2 = 0;
2621         for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2622             next if !$conf->{"usb$i"};
2623             $use_usb2 = 1;
2624         }
2625         # include usb device config
2626         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-usb.cfg' if $use_usb2;
2627     }
2628
2629     my $vga = $conf->{vga};
2630
2631     my $qxlnum = vga_conf_has_spice($vga);
2632     $vga = 'qxl' if $qxlnum;
2633
2634     if (!$vga) {
2635         if ($conf->{ostype} && ($conf->{ostype} eq 'win8' ||
2636                                 $conf->{ostype} eq 'win7' ||
2637                                 $conf->{ostype} eq 'w2k8')) {
2638             $vga = 'std';
2639         } else {
2640             $vga = 'cirrus';
2641         }
2642     }
2643
2644     # enable absolute mouse coordinates (needed by vnc)
2645     my $tablet;
2646     if (defined($conf->{tablet})) {
2647         $tablet = $conf->{tablet};
2648     } else {
2649         $tablet = $defaults->{tablet};
2650         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
2651         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
2652     }
2653
2654     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
2655
2656     # host pci devices
2657     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
2658         my $d = parse_hostpci($conf->{"hostpci$i"});
2659         next if !$d;
2660
2661         my $pcie = $d->{pcie};
2662         if($pcie){
2663             die "q35 machine model is not enabled" if !$q35;
2664             $pciaddr = print_pcie_addr("hostpci$i");
2665         }else{
2666             $pciaddr = print_pci_addr("hostpci$i", $bridges);
2667         }
2668
2669         my $rombar = $d->{rombar} && $d->{rombar} eq 'off' ? ",rombar=0" : "";
2670         my $driver = $d->{driver} && $d->{driver} eq 'vfio' ? "vfio-pci" : "pci-assign";
2671         my $xvga = $d->{'x-vga'} && $d->{'x-vga'} eq 'on' ? ",x-vga=on" : "";
2672         if ($xvga && $xvga ne '') {
2673             push @$cpuFlags, 'kvm=off';
2674             $vga = 'none';
2675         }
2676         $driver = "vfio-pci" if $xvga ne '';
2677         my $pcidevices = $d->{pciid};
2678         my $multifunction = 1 if @$pcidevices > 1;
2679
2680         my $j=0;
2681         foreach my $pcidevice (@$pcidevices) {
2682
2683             my $id = "hostpci$i";
2684             $id .= ".$j" if $multifunction;
2685             my $addr = $pciaddr;
2686             $addr .= ".$j" if $multifunction;
2687             my $devicestr = "$driver,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
2688
2689             if($j == 0){
2690                 $devicestr .= "$rombar$xvga";
2691                 $devicestr .= ",multifunction=on" if $multifunction;
2692             }
2693
2694             push @$devices, '-device', $devicestr;
2695             $j++;
2696         }
2697     }
2698
2699     # usb devices
2700     for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2701         my $d = parse_usb_device($conf->{"usb$i"});
2702         next if !$d;
2703         if ($d->{vendorid} && $d->{productid}) {
2704             push @$devices, '-device', "usb-host,vendorid=0x$d->{vendorid},productid=0x$d->{productid}";
2705         } elsif (defined($d->{hostbus}) && defined($d->{hostport})) {
2706             push @$devices, '-device', "usb-host,hostbus=$d->{hostbus},hostport=$d->{hostport}";
2707         } elsif ($d->{spice}) {
2708             # usb redir support for spice
2709             push @$devices, '-chardev', "spicevmc,id=usbredirchardev$i,name=usbredir";
2710             push @$devices, '-device', "usb-redir,chardev=usbredirchardev$i,id=usbredirdev$i,bus=ehci.0";
2711         }
2712     }
2713
2714     # serial devices
2715     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2716         if (my $path = $conf->{"serial$i"}) {
2717             if ($path eq 'socket') {
2718                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
2719                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
2720                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2721             } else {
2722                 die "no such serial device\n" if ! -c $path;
2723                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
2724                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2725             }
2726         }
2727     }
2728
2729     # parallel devices
2730     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
2731         if (my $path = $conf->{"parallel$i"}) {
2732             die "no such parallel device\n" if ! -c $path;
2733             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
2734             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
2735             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
2736         }
2737     }
2738
2739     my $vmname = $conf->{name} || "vm$vmid";
2740
2741     push @$cmd, '-name', $vmname;
2742
2743     my $sockets = 1;
2744     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
2745     $sockets = $conf->{sockets} if  $conf->{sockets};
2746
2747     my $cores = $conf->{cores} || 1;
2748
2749     my $maxcpus = $sockets * $cores;
2750
2751     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
2752
2753     my $allowed_vcpus = $cpuinfo->{cpus};
2754
2755     die "MAX $maxcpus vcpus allowed per VM on this node\n"
2756         if ($allowed_vcpus < $maxcpus);
2757
2758     push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
2759
2760     push @$cmd, '-nodefaults';
2761
2762     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2763
2764     my $bootindex_hash = {};
2765     my $i = 1;
2766     foreach my $o (split(//, $bootorder)) {
2767         $bootindex_hash->{$o} = $i*100;
2768         $i++;
2769     }
2770
2771     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000";
2772
2773     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
2774
2775     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
2776
2777     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
2778
2779     # time drift fix
2780     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
2781
2782     my $nokvm = defined($conf->{kvm}) && $conf->{kvm} == 0 ? 1 : 0;
2783     my $useLocaltime = $conf->{localtime};
2784
2785     if (my $ost = $conf->{ostype}) {
2786         # other, wxp, w2k, w2k3, w2k8, wvista, win7, win8, l24, l26, solaris
2787
2788         if ($ost =~ m/^w/) { # windows
2789             $useLocaltime = 1 if !defined($conf->{localtime});
2790
2791             # use time drift fix when acpi is enabled
2792             if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
2793                 $tdf = 1 if !defined($conf->{tdf});
2794             }
2795         }
2796
2797         if ($ost eq 'win7' || $ost eq 'win8' || $ost eq 'w2k8' ||
2798             $ost eq 'wvista') {
2799             push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
2800             push @$cmd, '-no-hpet';
2801             if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
2802                 push @$cpuFlags , 'hv_spinlocks=0x1fff' if !$nokvm;
2803                 push @$cpuFlags , 'hv_vapic' if !$nokvm;
2804                 push @$cpuFlags , 'hv_time' if !$nokvm;
2805
2806             } else {
2807                 push @$cpuFlags , 'hv_spinlocks=0xffff' if !$nokvm;
2808             }
2809         }
2810
2811         if ($ost eq 'win7' || $ost eq 'win8') {
2812             push @$cpuFlags , 'hv_relaxed' if !$nokvm;
2813         }
2814     }
2815
2816     push @$rtcFlags, 'driftfix=slew' if $tdf;
2817
2818     if ($nokvm) {
2819         push @$machineFlags, 'accel=tcg';
2820     } else {
2821         die "No accelerator found!\n" if !$cpuinfo->{hvm};
2822     }
2823
2824     if ($machine_type) {
2825         push @$machineFlags, "type=${machine_type}";
2826     }
2827
2828     if ($conf->{startdate}) {
2829         push @$rtcFlags, "base=$conf->{startdate}";
2830     } elsif ($useLocaltime) {
2831         push @$rtcFlags, 'base=localtime';
2832     }
2833
2834     my $cpu = $nokvm ? "qemu64" : "kvm64";
2835     $cpu = $conf->{cpu} if $conf->{cpu};
2836
2837     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
2838
2839     push @$cpuFlags , '+x2apic' if !$nokvm && $conf->{ostype} ne 'solaris';
2840
2841     push @$cpuFlags , '-x2apic' if $conf->{ostype} eq 'solaris';
2842
2843     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
2844
2845     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
2846
2847         push @$cpuFlags , '+kvm_pv_unhalt' if !$nokvm;
2848         push @$cpuFlags , '+kvm_pv_eoi' if !$nokvm;
2849     }
2850
2851     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
2852
2853     push @$cmd, '-cpu', "$cpu,enforce";
2854
2855     my $memory = $conf->{memory} || $defaults->{memory};
2856     my $static_memory = 0;
2857     my $dimm_memory = 0;
2858
2859     if ($hotplug_features->{memory}) {
2860         die "Numa need to be enabled for memory hotplug\n" if !$conf->{numa};
2861         die "Total memory is bigger than ${MAX_MEM}MB\n" if $memory > $MAX_MEM;
2862         $static_memory = $STATICMEM;
2863         die "minimum memory must be ${static_memory}MB\n" if($memory < $static_memory);
2864         $dimm_memory = $memory - $static_memory;
2865         push @$cmd, '-m', "size=${static_memory},slots=255,maxmem=${MAX_MEM}M";
2866
2867     } else {
2868
2869         $static_memory = $memory;
2870         push @$cmd, '-m', $static_memory;
2871     }
2872
2873     if ($conf->{numa}) {
2874
2875         my $numa_totalmemory = undef;
2876         for (my $i = 0; $i < $MAX_NUMA; $i++) {
2877             next if !$conf->{"numa$i"};
2878             my $numa = parse_numa($conf->{"numa$i"});
2879             next if !$numa;
2880             # memory
2881             die "missing numa node$i memory value\n" if !$numa->{memory};
2882             my $numa_memory = $numa->{memory};
2883             $numa_totalmemory += $numa_memory;
2884             my $numa_object = "memory-backend-ram,id=ram-node$i,size=${numa_memory}M";
2885
2886             # cpus
2887             my $cpus_start = $numa->{cpus}->{start};
2888             die "missing numa node$i cpus\n" if !defined($cpus_start);
2889             my $cpus_end = $numa->{cpus}->{end} if defined($numa->{cpus}->{end});
2890             my $cpus = $cpus_start;
2891             if (defined($cpus_end)) {
2892                 $cpus .= "-$cpus_end";
2893                 die "numa node$i :  cpu range $cpus is incorrect\n" if $cpus_end <= $cpus_start;
2894             }
2895
2896             # hostnodes
2897             my $hostnodes_start = $numa->{hostnodes}->{start};
2898             if (defined($hostnodes_start)) {
2899                 my $hostnodes_end = $numa->{hostnodes}->{end} if defined($numa->{hostnodes}->{end});
2900                 my $hostnodes = $hostnodes_start;
2901                 if (defined($hostnodes_end)) {
2902                     $hostnodes .= "-$hostnodes_end";
2903                     die "host node $hostnodes range is incorrect\n" if $hostnodes_end <= $hostnodes_start;
2904                 }
2905
2906                 my $hostnodes_end_range = defined($hostnodes_end) ? $hostnodes_end : $hostnodes_start;
2907                 for (my $i = $hostnodes_start; $i <= $hostnodes_end_range; $i++ ) {
2908                     die "host numa node$i don't exist\n" if ! -d "/sys/devices/system/node/node$i/";
2909                 }
2910
2911                 # policy
2912                 my $policy = $numa->{policy};
2913                 die "you need to define a policy for hostnode $hostnodes\n" if !$policy;
2914                 $numa_object .= ",host-nodes=$hostnodes,policy=$policy";
2915             }
2916
2917             push @$cmd, '-object', $numa_object;
2918             push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
2919         }
2920
2921         die "total memory for NUMA nodes must be equal to vm static memory\n"
2922             if $numa_totalmemory && $numa_totalmemory != $static_memory;
2923
2924         #if no custom tology, we split memory and cores across numa nodes
2925         if(!$numa_totalmemory) {
2926
2927             my $numa_memory = ($static_memory / $sockets) . "M";
2928
2929             for (my $i = 0; $i < $sockets; $i++)  {
2930
2931                 my $cpustart = ($cores * $i);
2932                 my $cpuend = ($cpustart + $cores - 1) if $cores && $cores > 1;
2933                 my $cpus = $cpustart;
2934                 $cpus .= "-$cpuend" if $cpuend;
2935
2936                 push @$cmd, '-object', "memory-backend-ram,size=$numa_memory,id=ram-node$i";
2937                 push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
2938             }
2939         }
2940     }
2941
2942     if ($hotplug_features->{memory}) {
2943         foreach_dimm($conf, $vmid, $memory, $sockets, sub {
2944             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
2945             push @$cmd, "-object" , "memory-backend-ram,id=mem-$name,size=${dimm_size}M";
2946             push @$cmd, "-device", "pc-dimm,id=$name,memdev=mem-$name,node=$numanode";
2947
2948             #if dimm_memory is not aligned to dimm map
2949             if($current_size > $memory) {
2950                  $conf->{memory} = $current_size;
2951                  update_config_nolock($vmid, $conf, 1);
2952             }
2953         });
2954     }
2955
2956     push @$cmd, '-S' if $conf->{freeze};
2957
2958     # set keyboard layout
2959     my $kb = $conf->{keyboard} || $defaults->{keyboard};
2960     push @$cmd, '-k', $kb if $kb;
2961
2962     # enable sound
2963     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
2964     #push @$cmd, '-soundhw', 'es1370';
2965     #push @$cmd, '-soundhw', $soundhw if $soundhw;
2966
2967     if($conf->{agent}) {
2968         my $qgasocket = qmp_socket($vmid, 1);
2969         my $pciaddr = print_pci_addr("qga0", $bridges);
2970         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
2971         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
2972         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
2973     }
2974
2975     my $spice_port;
2976
2977     if ($qxlnum) {
2978         if ($qxlnum > 1) {
2979             if ($conf->{ostype} && $conf->{ostype} =~ m/^w/){
2980                 for(my $i = 1; $i < $qxlnum; $i++){
2981                     my $pciaddr = print_pci_addr("vga$i", $bridges);
2982                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
2983                 }
2984             } else {
2985                 # assume other OS works like Linux
2986                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
2987                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
2988             }
2989         }
2990
2991         my $pciaddr = print_pci_addr("spice", $bridges);
2992
2993         my $nodename = PVE::INotify::nodename();
2994         my $pfamily = PVE::Tools::get_host_address_family($nodename);
2995         $spice_port = PVE::Tools::next_spice_port($pfamily);
2996
2997         push @$devices, '-spice', "tls-port=${spice_port},addr=localhost,tls-ciphers=DES-CBC3-SHA,seamless-migration=on";
2998
2999         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3000         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3001         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3002     }
3003
3004     # enable balloon by default, unless explicitly disabled
3005     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3006         $pciaddr = print_pci_addr("balloon0", $bridges);
3007         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3008     }
3009
3010     if ($conf->{watchdog}) {
3011         my $wdopts = parse_watchdog($conf->{watchdog});
3012         $pciaddr = print_pci_addr("watchdog", $bridges);
3013         my $watchdog = $wdopts->{model} || 'i6300esb';
3014         push @$devices, '-device', "$watchdog$pciaddr";
3015         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3016     }
3017
3018     my $vollist = [];
3019     my $scsicontroller = {};
3020     my $ahcicontroller = {};
3021     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3022
3023     # Add iscsi initiator name if available
3024     if (my $initiator = get_initiator_name()) {
3025         push @$devices, '-iscsi', "initiator-name=$initiator";
3026     }
3027
3028     foreach_drive($conf, sub {
3029         my ($ds, $drive) = @_;
3030
3031         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3032             push @$vollist, $drive->{file};
3033         }
3034
3035         $use_virtio = 1 if $ds =~ m/^virtio/;
3036
3037         if (drive_is_cdrom ($drive)) {
3038             if ($bootindex_hash->{d}) {
3039                 $drive->{bootindex} = $bootindex_hash->{d};
3040                 $bootindex_hash->{d} += 1;
3041             }
3042         } else {
3043             if ($bootindex_hash->{c}) {
3044                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3045                 $bootindex_hash->{c} += 1;
3046             }
3047         }
3048
3049         if($drive->{interface} eq 'virtio'){
3050            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3051         }
3052
3053         if ($drive->{interface} eq 'scsi') {
3054
3055             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3056
3057             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3058             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3059
3060             my $iothread = '';
3061             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3062                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3063                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3064             }
3065
3066             my $queues = '';
3067             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3068                 $queues = ",num_queues=$drive->{queues}";
3069             }
3070
3071             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3072             $scsicontroller->{$controller}=1;
3073         }
3074
3075         if ($drive->{interface} eq 'sata') {
3076            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3077            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3078            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3079            $ahcicontroller->{$controller}=1;
3080         }
3081
3082         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3083         push @$devices, '-drive',$drive_cmd;
3084         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3085     });
3086
3087     for (my $i = 0; $i < $MAX_NETS; $i++) {
3088          next if !$conf->{"net$i"};
3089          my $d = parse_net($conf->{"net$i"});
3090          next if !$d;
3091
3092          $use_virtio = 1 if $d->{model} eq 'virtio';
3093
3094          if ($bootindex_hash->{n}) {
3095             $d->{bootindex} = $bootindex_hash->{n};
3096             $bootindex_hash->{n} += 1;
3097          }
3098
3099          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3100          push @$devices, '-netdev', $netdevfull;
3101
3102          my $netdevicefull = print_netdevice_full($vmid,$conf,$d,"net$i",$bridges);
3103          push @$devices, '-device', $netdevicefull;
3104     }
3105
3106     if (!$q35) {
3107         # add pci bridges
3108         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3109            $bridges->{1} = 1;
3110            $bridges->{2} = 1;
3111         }
3112
3113         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3114
3115         while (my ($k, $v) = each %$bridges) {
3116             $pciaddr = print_pci_addr("pci.$k");
3117             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3118         }
3119     }
3120
3121     # add custom args
3122     if ($conf->{args}) {
3123         my $aa = PVE::Tools::split_args($conf->{args});
3124         push @$cmd, @$aa;
3125     }
3126
3127     push @$cmd, @$devices;
3128     push @$cmd, '-rtc', join(',', @$rtcFlags)
3129         if scalar(@$rtcFlags);
3130     push @$cmd, '-machine', join(',', @$machineFlags)
3131         if scalar(@$machineFlags);
3132     push @$cmd, '-global', join(',', @$globalFlags)
3133         if scalar(@$globalFlags);
3134
3135     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3136 }
3137
3138 sub vnc_socket {
3139     my ($vmid) = @_;
3140     return "${var_run_tmpdir}/$vmid.vnc";
3141 }
3142
3143 sub spice_port {
3144     my ($vmid) = @_;
3145
3146     my $res = vm_mon_cmd($vmid, 'query-spice');
3147
3148     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3149 }
3150
3151 sub qmp_socket {
3152     my ($vmid, $qga) = @_;
3153     my $sockettype = $qga ? 'qga' : 'qmp';
3154     return "${var_run_tmpdir}/$vmid.$sockettype";
3155 }
3156
3157 sub pidfile_name {
3158     my ($vmid) = @_;
3159     return "${var_run_tmpdir}/$vmid.pid";
3160 }
3161
3162 sub vm_devices_list {
3163     my ($vmid) = @_;
3164
3165     my $res = vm_mon_cmd($vmid, 'query-pci');
3166     my $devices = {};
3167     foreach my $pcibus (@$res) {
3168         foreach my $device (@{$pcibus->{devices}}) {
3169             next if !$device->{'qdev_id'};
3170             if ($device->{'pci_bridge'}) {
3171                 $devices->{$device->{'qdev_id'}} = 1;
3172                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3173                     next if !$bridge_device->{'qdev_id'};
3174                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3175                     $devices->{$device->{'qdev_id'}}++;
3176                 }
3177             } else {
3178                 $devices->{$device->{'qdev_id'}} = 1;
3179             }
3180         }
3181     }
3182
3183     my $resblock = vm_mon_cmd($vmid, 'query-block');
3184     foreach my $block (@$resblock) {
3185         if($block->{device} =~ m/^drive-(\S+)/){
3186                 $devices->{$1} = 1;
3187         }
3188     }
3189
3190     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3191     foreach my $mice (@$resmice) {
3192         if ($mice->{name} eq 'QEMU HID Tablet') {
3193             $devices->{tablet} = 1;
3194             last;
3195         }
3196     }
3197
3198     return $devices;
3199 }
3200
3201 sub vm_deviceplug {
3202     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3203
3204     my $q35 = machine_type_is_q35($conf);
3205
3206     my $devices_list = vm_devices_list($vmid);
3207     return 1 if defined($devices_list->{$deviceid});
3208
3209     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3210
3211     if ($deviceid eq 'tablet') {
3212
3213         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3214
3215     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3216
3217         qemu_iothread_add($vmid, $deviceid, $device);
3218
3219         qemu_driveadd($storecfg, $vmid, $device);
3220         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3221
3222         qemu_deviceadd($vmid, $devicefull);
3223         eval { qemu_deviceaddverify($vmid, $deviceid); };
3224         if (my $err = $@) {
3225             eval { qemu_drivedel($vmid, $deviceid); };
3226             warn $@ if $@;
3227             die $err;
3228         }
3229
3230     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3231
3232
3233         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3234         my $pciaddr = print_pci_addr($deviceid);
3235         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3236
3237         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3238
3239         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3240             qemu_iothread_add($vmid, $deviceid, $device);
3241             $devicefull .= ",iothread=iothread-$deviceid";
3242         }
3243
3244         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3245             $devicefull .= ",num_queues=$device->{queues}";
3246         }
3247
3248         qemu_deviceadd($vmid, $devicefull);
3249         qemu_deviceaddverify($vmid, $deviceid);
3250
3251     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3252
3253         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3254         qemu_driveadd($storecfg, $vmid, $device);
3255
3256         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3257         eval { qemu_deviceadd($vmid, $devicefull); };
3258         if (my $err = $@) {
3259             eval { qemu_drivedel($vmid, $deviceid); };
3260             warn $@ if $@;
3261             die $err;
3262         }
3263
3264     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3265
3266         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3267         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid);
3268         qemu_deviceadd($vmid, $netdevicefull);
3269         eval { qemu_deviceaddverify($vmid, $deviceid); };
3270         if (my $err = $@) {
3271             eval { qemu_netdevdel($vmid, $deviceid); };
3272             warn $@ if $@;
3273             die $err;
3274         }
3275
3276     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3277
3278         my $bridgeid = $2;
3279         my $pciaddr = print_pci_addr($deviceid);
3280         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3281
3282         qemu_deviceadd($vmid, $devicefull);
3283         qemu_deviceaddverify($vmid, $deviceid);
3284
3285     } else {
3286         die "can't hotplug device '$deviceid'\n";
3287     }
3288
3289     return 1;
3290 }
3291
3292 # fixme: this should raise exceptions on error!
3293 sub vm_deviceunplug {
3294     my ($vmid, $conf, $deviceid) = @_;
3295
3296     my $devices_list = vm_devices_list($vmid);
3297     return 1 if !defined($devices_list->{$deviceid});
3298
3299     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3300
3301     if ($deviceid eq 'tablet') {
3302
3303         qemu_devicedel($vmid, $deviceid);
3304
3305     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3306
3307         qemu_devicedel($vmid, $deviceid);
3308         qemu_devicedelverify($vmid, $deviceid);
3309         qemu_drivedel($vmid, $deviceid);
3310         qemu_iothread_del($conf, $vmid, $deviceid);
3311
3312     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3313
3314         qemu_devicedel($vmid, $deviceid);
3315         qemu_devicedelverify($vmid, $deviceid);
3316         qemu_iothread_del($conf, $vmid, $deviceid);
3317
3318     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3319
3320         #qemu 2.3 segfault on drive_del with virtioscsi + iothread
3321         my $device = parse_drive($deviceid, $conf->{$deviceid});
3322         die "virtioscsi with iothread is not hot-unplugglable currently" if $device->{iothread};
3323
3324         qemu_devicedel($vmid, $deviceid);
3325         qemu_drivedel($vmid, $deviceid);
3326         qemu_deletescsihw($conf, $vmid, $deviceid);
3327
3328     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3329
3330         qemu_devicedel($vmid, $deviceid);
3331         qemu_devicedelverify($vmid, $deviceid);
3332         qemu_netdevdel($vmid, $deviceid);
3333
3334     } else {
3335         die "can't unplug device '$deviceid'\n";
3336     }
3337
3338     return 1;
3339 }
3340
3341 sub qemu_deviceadd {
3342     my ($vmid, $devicefull) = @_;
3343
3344     $devicefull = "driver=".$devicefull;
3345     my %options =  split(/[=,]/, $devicefull);
3346
3347     vm_mon_cmd($vmid, "device_add" , %options);
3348 }
3349
3350 sub qemu_devicedel {
3351     my ($vmid, $deviceid) = @_;
3352
3353     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3354 }
3355
3356 sub qemu_iothread_add {
3357     my($vmid, $deviceid, $device) = @_;
3358
3359     if ($device->{iothread}) {
3360         my $iothreads = vm_iothreads_list($vmid);
3361         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3362     }
3363 }
3364
3365 sub qemu_iothread_del {
3366     my($conf, $vmid, $deviceid) = @_;
3367
3368     my $device = parse_drive($deviceid, $conf->{$deviceid});
3369     if ($device->{iothread}) {
3370         my $iothreads = vm_iothreads_list($vmid);
3371         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3372     }
3373 }
3374
3375 sub qemu_objectadd {
3376     my($vmid, $objectid, $qomtype) = @_;
3377
3378     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3379
3380     return 1;
3381 }
3382
3383 sub qemu_objectdel {
3384     my($vmid, $objectid) = @_;
3385
3386     vm_mon_cmd($vmid, "object-del", id => $objectid);
3387
3388     return 1;
3389 }
3390
3391 sub qemu_driveadd {
3392     my ($storecfg, $vmid, $device) = @_;
3393
3394     my $drive = print_drive_full($storecfg, $vmid, $device);
3395     $drive =~ s/\\/\\\\/g;
3396     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3397
3398     # If the command succeeds qemu prints: "OK"
3399     return 1 if $ret =~ m/OK/s;
3400
3401     die "adding drive failed: $ret\n";
3402 }
3403
3404 sub qemu_drivedel {
3405     my($vmid, $deviceid) = @_;
3406
3407     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3408     $ret =~ s/^\s+//;
3409
3410     return 1 if $ret eq "";
3411
3412     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3413     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3414
3415     die "deleting drive $deviceid failed : $ret\n";
3416 }
3417
3418 sub qemu_deviceaddverify {
3419     my ($vmid, $deviceid) = @_;
3420
3421     for (my $i = 0; $i <= 5; $i++) {
3422          my $devices_list = vm_devices_list($vmid);
3423          return 1 if defined($devices_list->{$deviceid});
3424          sleep 1;
3425     }
3426
3427     die "error on hotplug device '$deviceid'\n";
3428 }
3429
3430
3431 sub qemu_devicedelverify {
3432     my ($vmid, $deviceid) = @_;
3433
3434     # need to verify that the device is correctly removed as device_del
3435     # is async and empty return is not reliable
3436
3437     for (my $i = 0; $i <= 5; $i++) {
3438          my $devices_list = vm_devices_list($vmid);
3439          return 1 if !defined($devices_list->{$deviceid});
3440          sleep 1;
3441     }
3442
3443     die "error on hot-unplugging device '$deviceid'\n";
3444 }
3445
3446 sub qemu_findorcreatescsihw {
3447     my ($storecfg, $conf, $vmid, $device) = @_;
3448
3449     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3450
3451     my $scsihwid="$controller_prefix$controller";
3452     my $devices_list = vm_devices_list($vmid);
3453
3454     if(!defined($devices_list->{$scsihwid})) {
3455         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3456     }
3457
3458     return 1;
3459 }
3460
3461 sub qemu_deletescsihw {
3462     my ($conf, $vmid, $opt) = @_;
3463
3464     my $device = parse_drive($opt, $conf->{$opt});
3465
3466     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3467         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3468         return 1;
3469     }
3470
3471     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3472
3473     my $devices_list = vm_devices_list($vmid);
3474     foreach my $opt (keys %{$devices_list}) {
3475         if (PVE::QemuServer::valid_drivename($opt)) {
3476             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3477             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3478                 return 1;
3479             }
3480         }
3481     }
3482
3483     my $scsihwid="scsihw$controller";
3484
3485     vm_deviceunplug($vmid, $conf, $scsihwid);
3486
3487     return 1;
3488 }
3489
3490 sub qemu_add_pci_bridge {
3491     my ($storecfg, $conf, $vmid, $device) = @_;
3492
3493     my $bridges = {};
3494
3495     my $bridgeid;
3496
3497     print_pci_addr($device, $bridges);
3498
3499     while (my ($k, $v) = each %$bridges) {
3500         $bridgeid = $k;
3501     }
3502     return 1 if !defined($bridgeid) || $bridgeid < 1;
3503
3504     my $bridge = "pci.$bridgeid";
3505     my $devices_list = vm_devices_list($vmid);
3506
3507     if (!defined($devices_list->{$bridge})) {
3508         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3509     }
3510
3511     return 1;
3512 }
3513
3514 sub qemu_set_link_status {
3515     my ($vmid, $device, $up) = @_;
3516
3517     vm_mon_cmd($vmid, "set_link", name => $device,
3518                up => $up ? JSON::true : JSON::false);
3519 }
3520
3521 sub qemu_netdevadd {
3522     my ($vmid, $conf, $device, $deviceid) = @_;
3523
3524     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid);
3525     my %options =  split(/[=,]/, $netdev);
3526
3527     vm_mon_cmd($vmid, "netdev_add",  %options);
3528     return 1;
3529 }
3530
3531 sub qemu_netdevdel {
3532     my ($vmid, $deviceid) = @_;
3533
3534     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3535 }
3536
3537 sub qemu_cpu_hotplug {
3538     my ($vmid, $conf, $vcpus) = @_;
3539
3540     my $sockets = 1;
3541     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3542     $sockets = $conf->{sockets} if  $conf->{sockets};
3543     my $cores = $conf->{cores} || 1;
3544     my $maxcpus = $sockets * $cores;
3545
3546     $vcpus = $maxcpus if !$vcpus;
3547
3548     die "you can't add more vcpus than maxcpus\n"
3549         if $vcpus > $maxcpus;
3550
3551     my $currentvcpus = $conf->{vcpus} || $maxcpus;
3552     die "online cpu unplug is not yet possible\n"
3553         if $vcpus < $currentvcpus;
3554
3555     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3556     die "vcpus in running vm is different than configuration\n"
3557         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
3558
3559     for (my $i = $currentvcpus; $i < $vcpus; $i++) {
3560         vm_mon_cmd($vmid, "cpu-add", id => int($i));
3561     }
3562 }
3563
3564 sub qemu_memory_hotplug {
3565     my ($vmid, $conf, $defaults, $opt, $value) = @_;
3566
3567     return $value if !check_running($vmid);
3568
3569     my $memory = $conf->{memory} || $defaults->{memory};
3570     $value = $defaults->{memory} if !$value;
3571     return $value if $value == $memory;
3572
3573     my $static_memory = $STATICMEM;
3574     my $dimm_memory = $memory - $static_memory;
3575
3576     die "memory can't be lower than $static_memory MB" if $value < $static_memory;
3577     die "memory unplug is not yet available" if $value < $memory;
3578     die "you cannot add more memory than $MAX_MEM MB!\n" if $memory > $MAX_MEM;
3579
3580
3581     my $sockets = 1;
3582     $sockets = $conf->{sockets} if $conf->{sockets};
3583
3584     foreach_dimm($conf, $vmid, $value, $sockets, sub {
3585         my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3586
3587             return if $current_size <= $conf->{memory};
3588
3589             eval { vm_mon_cmd($vmid, "object-add", 'qom-type' => "memory-backend-ram", id => "mem-$name", props => { size => int($dimm_size*1024*1024) } ) };
3590             if (my $err = $@) {
3591                 eval { qemu_objectdel($vmid, "mem-$name"); };
3592                 die $err;
3593             }
3594
3595             eval { vm_mon_cmd($vmid, "device_add", driver => "pc-dimm", id => "$name", memdev => "mem-$name", node => $numanode) };
3596             if (my $err = $@) {
3597                 eval { qemu_objectdel($vmid, "mem-$name"); };
3598                 die $err;
3599             }
3600             #update conf after each succesful module hotplug
3601             $conf->{memory} = $current_size;
3602             update_config_nolock($vmid, $conf, 1);
3603     });
3604 }
3605
3606 sub qemu_block_set_io_throttle {
3607     my ($vmid, $deviceid, $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr) = @_;
3608
3609     return if !check_running($vmid) ;
3610
3611     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid, bps => int($bps), bps_rd => int($bps_rd), bps_wr => int($bps_wr), iops => int($iops), iops_rd => int($iops_rd), iops_wr => int($iops_wr));
3612
3613 }
3614
3615 # old code, only used to shutdown old VM after update
3616 sub __read_avail {
3617     my ($fh, $timeout) = @_;
3618
3619     my $sel = new IO::Select;
3620     $sel->add($fh);
3621
3622     my $res = '';
3623     my $buf;
3624
3625     my @ready;
3626     while (scalar (@ready = $sel->can_read($timeout))) {
3627         my $count;
3628         if ($count = $fh->sysread($buf, 8192)) {
3629             if ($buf =~ /^(.*)\(qemu\) $/s) {
3630                 $res .= $1;
3631                 last;
3632             } else {
3633                 $res .= $buf;
3634             }
3635         } else {
3636             if (!defined($count)) {
3637                 die "$!\n";
3638             }
3639             last;
3640         }
3641     }
3642
3643     die "monitor read timeout\n" if !scalar(@ready);
3644
3645     return $res;
3646 }
3647
3648 # old code, only used to shutdown old VM after update
3649 sub vm_monitor_command {
3650     my ($vmid, $cmdstr, $nocheck) = @_;
3651
3652     my $res;
3653
3654     eval {
3655         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
3656
3657         my $sname = "${var_run_tmpdir}/$vmid.mon";
3658
3659         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
3660             die "unable to connect to VM $vmid socket - $!\n";
3661
3662         my $timeout = 3;
3663
3664         # hack: migrate sometime blocks the monitor (when migrate_downtime
3665         # is set)
3666         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3667             $timeout = 60*60; # 1 hour
3668         }
3669
3670         # read banner;
3671         my $data = __read_avail($sock, $timeout);
3672
3673         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
3674             die "got unexpected qemu monitor banner\n";
3675         }
3676
3677         my $sel = new IO::Select;
3678         $sel->add($sock);
3679
3680         if (!scalar(my @ready = $sel->can_write($timeout))) {
3681             die "monitor write error - timeout";
3682         }
3683
3684         my $fullcmd = "$cmdstr\r";
3685
3686         # syslog('info', "VM $vmid monitor command: $cmdstr");
3687
3688         my $b;
3689         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
3690             die "monitor write error - $!";
3691         }
3692
3693         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
3694
3695         $timeout = 20;
3696
3697         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3698             $timeout = 60*60; # 1 hour
3699         } elsif ($cmdstr =~ m/^(eject|change)/) {
3700             $timeout = 60; # note: cdrom mount command is slow
3701         }
3702         if ($res = __read_avail($sock, $timeout)) {
3703
3704             my @lines = split("\r?\n", $res);
3705
3706             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
3707
3708             $res = join("\n", @lines);
3709             $res .= "\n";
3710         }
3711     };
3712
3713     my $err = $@;
3714
3715     if ($err) {
3716         syslog("err", "VM $vmid monitor command failed - $err");
3717         die $err;
3718     }
3719
3720     return $res;
3721 }
3722
3723 sub qemu_block_resize {
3724     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
3725
3726     my $running = check_running($vmid);
3727
3728     return if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
3729
3730     return if !$running;
3731
3732     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
3733
3734 }
3735
3736 sub qemu_volume_snapshot {
3737     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3738
3739     my $running = check_running($vmid);
3740
3741     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
3742         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
3743     } else {
3744         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
3745     }
3746 }
3747
3748 sub qemu_volume_snapshot_delete {
3749     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3750
3751     my $running = check_running($vmid);
3752
3753     return if !PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
3754
3755     return if !$running;
3756
3757     vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
3758 }
3759
3760 sub set_migration_caps {
3761     my ($vmid) = @_;
3762
3763     my $cap_ref = [];
3764
3765     my $enabled_cap = {
3766         "auto-converge" => 1,
3767         "xbzrle" => 0,
3768         "x-rdma-pin-all" => 0,
3769         "zero-blocks" => 0,
3770     };
3771
3772     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
3773
3774     for my $supported_capability (@$supported_capabilities) {
3775         push @$cap_ref, {
3776             capability => $supported_capability->{capability},
3777             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
3778         };
3779     }
3780
3781     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
3782 }
3783
3784 my $fast_plug_option = {
3785     'lock' => 1,
3786     'name' => 1,
3787     'onboot' => 1,
3788     'shares' => 1,
3789     'startup' => 1,
3790 };
3791
3792 # hotplug changes in [PENDING]
3793 # $selection hash can be used to only apply specified options, for
3794 # example: { cores => 1 } (only apply changed 'cores')
3795 # $errors ref is used to return error messages
3796 sub vmconfig_hotplug_pending {
3797     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
3798
3799     my $defaults = load_defaults();
3800
3801     # commit values which do not have any impact on running VM first
3802     # Note: those option cannot raise errors, we we do not care about
3803     # $selection and always apply them.
3804
3805     my $add_error = sub {
3806         my ($opt, $msg) = @_;
3807         $errors->{$opt} = "hotplug problem - $msg";
3808     };
3809
3810     my $changes = 0;
3811     foreach my $opt (keys %{$conf->{pending}}) { # add/change
3812         if ($fast_plug_option->{$opt}) {
3813             $conf->{$opt} = $conf->{pending}->{$opt};
3814             delete $conf->{pending}->{$opt};
3815             $changes = 1;
3816         }
3817     }
3818
3819     if ($changes) {
3820         update_config_nolock($vmid, $conf, 1);
3821         $conf = load_config($vmid); # update/reload
3822     }
3823
3824     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3825
3826     my @delete = PVE::Tools::split_list($conf->{pending}->{delete});
3827     foreach my $opt (@delete) {
3828         next if $selection && !$selection->{$opt};
3829         eval {
3830             if ($opt eq 'hotplug') {
3831                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
3832             } elsif ($opt eq 'tablet') {
3833                 die "skip\n" if !$hotplug_features->{usb};
3834                 if ($defaults->{tablet}) {
3835                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
3836                 } else {
3837                     vm_deviceunplug($vmid, $conf, $opt);
3838                 }
3839             } elsif ($opt eq 'vcpus') {
3840                 die "skip\n" if !$hotplug_features->{cpu};
3841                 qemu_cpu_hotplug($vmid, $conf, undef);
3842             } elsif ($opt eq 'balloon') {
3843                 # enable balloon device is not hotpluggable
3844                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
3845             } elsif ($fast_plug_option->{$opt}) {
3846                 # do nothing
3847             } elsif ($opt =~ m/^net(\d+)$/) {
3848                 die "skip\n" if !$hotplug_features->{network};
3849                 vm_deviceunplug($vmid, $conf, $opt);
3850             } elsif (valid_drivename($opt)) {
3851                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
3852                 vm_deviceunplug($vmid, $conf, $opt);
3853                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}));
3854             } elsif ($opt =~ m/^memory$/) {
3855                 die "skip\n" if !$hotplug_features->{memory};
3856                 qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
3857             } elsif ($opt eq 'cpuunits') {
3858                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
3859             } elsif ($opt eq 'cpulimit') {
3860                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
3861             } else {
3862                 die "skip\n";
3863             }
3864         };
3865         if (my $err = $@) {
3866             &$add_error($opt, $err) if $err ne "skip\n";
3867         } else {
3868             # save new config if hotplug was successful
3869             delete $conf->{$opt};
3870             vmconfig_undelete_pending_option($conf, $opt);
3871             update_config_nolock($vmid, $conf, 1);
3872             $conf = load_config($vmid); # update/reload
3873         }
3874     }
3875
3876     foreach my $opt (keys %{$conf->{pending}}) {
3877         next if $selection && !$selection->{$opt};
3878         my $value = $conf->{pending}->{$opt};
3879         eval {
3880             if ($opt eq 'hotplug') {
3881                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
3882             } elsif ($opt eq 'tablet') {
3883                 die "skip\n" if !$hotplug_features->{usb};
3884                 if ($value == 1) {
3885                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
3886                 } elsif ($value == 0) {
3887                     vm_deviceunplug($vmid, $conf, $opt);
3888                 }
3889             } elsif ($opt eq 'vcpus') {
3890                 die "skip\n" if !$hotplug_features->{cpu};
3891                 qemu_cpu_hotplug($vmid, $conf, $value);
3892             } elsif ($opt eq 'balloon') {
3893                 # enable/disable balloning device is not hotpluggable
3894                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
3895                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
3896                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
3897
3898                 # allow manual ballooning if shares is set to zero
3899                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
3900                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
3901                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
3902                 }
3903             } elsif ($opt =~ m/^net(\d+)$/) {
3904                 # some changes can be done without hotplug
3905                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
3906                                     $vmid, $opt, $value);
3907             } elsif (valid_drivename($opt)) {
3908                 # some changes can be done without hotplug
3909                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
3910                                      $vmid, $opt, $value, 1);
3911             } elsif ($opt =~ m/^memory$/) { #dimms
3912                 die "skip\n" if !$hotplug_features->{memory};
3913                 $value = qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
3914             } elsif ($opt eq 'cpuunits') {
3915                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
3916             } elsif ($opt eq 'cpulimit') {
3917                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
3918                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
3919             } else {
3920                 die "skip\n";  # skip non-hot-pluggable options
3921             }
3922         };
3923         if (my $err = $@) {
3924             &$add_error($opt, $err) if $err ne "skip\n";
3925         } else {
3926             # save new config if hotplug was successful
3927             $conf->{$opt} = $value;
3928             delete $conf->{pending}->{$opt};
3929             update_config_nolock($vmid, $conf, 1);
3930             $conf = load_config($vmid); # update/reload
3931         }
3932     }
3933 }
3934
3935 sub vmconfig_apply_pending {
3936     my ($vmid, $conf, $storecfg) = @_;
3937
3938     # cold plug
3939
3940     my @delete = PVE::Tools::split_list($conf->{pending}->{delete});
3941     foreach my $opt (@delete) { # delete
3942         die "internal error" if $opt =~ m/^unused/;
3943         $conf = load_config($vmid); # update/reload
3944         if (!defined($conf->{$opt})) {
3945             vmconfig_undelete_pending_option($conf, $opt);
3946             update_config_nolock($vmid, $conf, 1);
3947         } elsif (valid_drivename($opt)) {
3948             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}));
3949             vmconfig_undelete_pending_option($conf, $opt);
3950             delete $conf->{$opt};
3951             update_config_nolock($vmid, $conf, 1);
3952         } else {
3953             vmconfig_undelete_pending_option($conf, $opt);
3954             delete $conf->{$opt};
3955             update_config_nolock($vmid, $conf, 1);
3956         }
3957     }
3958
3959     $conf = load_config($vmid); # update/reload
3960
3961     foreach my $opt (keys %{$conf->{pending}}) { # add/change
3962         $conf = load_config($vmid); # update/reload
3963
3964         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
3965             # skip if nothing changed
3966         } elsif (valid_drivename($opt)) {
3967             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
3968                 if defined($conf->{$opt});
3969             $conf->{$opt} = $conf->{pending}->{$opt};
3970         } else {
3971             $conf->{$opt} = $conf->{pending}->{$opt};
3972         }
3973
3974         delete $conf->{pending}->{$opt};
3975         update_config_nolock($vmid, $conf, 1);
3976     }
3977 }
3978
3979 my $safe_num_ne = sub {
3980     my ($a, $b) = @_;
3981
3982     return 0 if !defined($a) && !defined($b);
3983     return 1 if !defined($a);
3984     return 1 if !defined($b);
3985
3986     return $a != $b;
3987 };
3988
3989 my $safe_string_ne = sub {
3990     my ($a, $b) = @_;
3991
3992     return 0 if !defined($a) && !defined($b);
3993     return 1 if !defined($a);
3994     return 1 if !defined($b);
3995
3996     return $a ne $b;
3997 };
3998
3999 sub vmconfig_update_net {
4000     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4001
4002     my $newnet = parse_net($value);
4003
4004     if ($conf->{$opt}) {
4005         my $oldnet = parse_net($conf->{$opt});
4006
4007         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4008             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4009             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4010             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4011
4012             # for non online change, we try to hot-unplug
4013             die "skip\n" if !$hotplug;
4014             vm_deviceunplug($vmid, $conf, $opt);
4015         } else {
4016
4017             die "internal error" if $opt !~ m/net(\d+)/;
4018             my $iface = "tap${vmid}i$1";
4019
4020             if (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4021                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4022             }
4023
4024             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4025                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4026                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4027                 PVE::Network::tap_unplug($iface);
4028                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall});
4029             }
4030
4031             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4032                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4033             }
4034
4035             return 1;
4036         }
4037     }
4038
4039     if ($hotplug) {
4040         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4041     } else {
4042         die "skip\n";
4043     }
4044 }
4045
4046 sub vmconfig_update_disk {
4047     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4048
4049     # fixme: do we need force?
4050
4051     my $drive = parse_drive($opt, $value);
4052
4053     if ($conf->{$opt}) {
4054
4055         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4056
4057             my $media = $drive->{media} || 'disk';
4058             my $oldmedia = $old_drive->{media} || 'disk';
4059             die "unable to change media type\n" if $media ne $oldmedia;
4060
4061             if (!drive_is_cdrom($old_drive)) {
4062
4063                 if ($drive->{file} ne $old_drive->{file}) {
4064
4065                     die "skip\n" if !$hotplug;
4066
4067                     # unplug and register as unused
4068                     vm_deviceunplug($vmid, $conf, $opt);
4069                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4070
4071                 } else {
4072                     # update existing disk
4073
4074                     # skip non hotpluggable value
4075                     if (&$safe_num_ne($drive->{discard}, $old_drive->{discard}) ||
4076                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4077                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4078                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4079                         die "skip\n";
4080                     }
4081
4082                     # apply throttle
4083                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4084                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4085                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4086                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4087                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4088                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4089                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4090                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4091                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4092                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4093                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4094                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max})) {
4095
4096                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4097                                                    ($drive->{mbps} || 0)*1024*1024,
4098                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4099                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4100                                                    $drive->{iops} || 0,
4101                                                    $drive->{iops_rd} || 0,
4102                                                    $drive->{iops_wr} || 0,
4103                                                    ($drive->{mbps_max} || 0)*1024*1024,
4104                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4105                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4106                                                    $drive->{iops_max} || 0,
4107                                                    $drive->{iops_rd_max} || 0,
4108                                                    $drive->{iops_wr_max} || 0);
4109
4110                     }
4111
4112                     return 1;
4113                 }
4114
4115             } else { # cdrom
4116
4117                 if ($drive->{file} eq 'none') {
4118                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4119                 } else {
4120                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4121                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4122                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4123                 }
4124
4125                 return 1;
4126             }
4127         }
4128     }
4129
4130     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4131     # hotplug new disks
4132     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4133 }
4134
4135 sub vm_start {
4136     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused, $forcemachine, $spice_ticket) = @_;
4137
4138     lock_config($vmid, sub {
4139         my $conf = load_config($vmid, $migratedfrom);
4140
4141         die "you can't start a vm if it's a template\n" if is_template($conf);
4142
4143         check_lock($conf) if !$skiplock;
4144
4145         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4146
4147         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4148             vmconfig_apply_pending($vmid, $conf, $storecfg);
4149             $conf = load_config($vmid); # update/reload
4150         }
4151
4152         my $defaults = load_defaults();
4153
4154         # set environment variable useful inside network script
4155         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4156
4157         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4158
4159         my $migrate_port = 0;
4160         my $migrate_uri;
4161         if ($statefile) {
4162             if ($statefile eq 'tcp') {
4163                 my $localip = "localhost";
4164                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4165                 my $nodename = PVE::INotify::nodename();
4166                 if ($datacenterconf->{migration_unsecure}) {
4167                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4168                 }
4169                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4170                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4171                 $migrate_uri = "tcp:[${localip}]:${migrate_port}";
4172                 push @$cmd, '-incoming', $migrate_uri;
4173                 push @$cmd, '-S';
4174             } else {
4175                 push @$cmd, '-loadstate', $statefile;
4176             }
4177         } elsif ($paused) {
4178             push @$cmd, '-S';
4179         }
4180
4181         # host pci devices
4182         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4183           my $d = parse_hostpci($conf->{"hostpci$i"});
4184           next if !$d;
4185           my $pcidevices = $d->{pciid};
4186           foreach my $pcidevice (@$pcidevices) {
4187                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4188
4189                 my $info = pci_device_info("0000:$pciid");
4190                 die "IOMMU not present\n" if !check_iommu_support();
4191                 die "no pci device info for device '$pciid'\n" if !$info;
4192
4193                 if ($d->{driver} && $d->{driver} eq "vfio") {
4194                     die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4195                 } else {
4196                     die "can't unbind/bind to stub pci device '$pciid'\n" if !pci_dev_bind_to_stub($info);
4197                 }
4198
4199                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4200           }
4201         }
4202
4203         PVE::Storage::activate_volumes($storecfg, $vollist);
4204
4205         eval  { run_command($cmd, timeout => $statefile ? undef : 30,
4206                     umask => 0077); };
4207         my $err = $@;
4208         die "start failed: $err" if $err;
4209
4210         print "migration listens on $migrate_uri\n" if $migrate_uri;
4211
4212         if ($statefile && $statefile ne 'tcp')  {
4213             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4214             warn $@ if $@;
4215         }
4216
4217         if ($migratedfrom) {
4218
4219             eval {
4220                 set_migration_caps($vmid);
4221             };
4222             warn $@ if $@;
4223
4224             if ($spice_port) {
4225                 print "spice listens on port $spice_port\n";
4226                 if ($spice_ticket) {
4227                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4228                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4229                 }
4230             }
4231
4232         } else {
4233
4234             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4235                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4236                     if $conf->{balloon};
4237             }
4238
4239             foreach my $opt (keys %$conf) {
4240                 next if $opt !~  m/^net\d+$/;
4241                 my $nicconf = parse_net($conf->{$opt});
4242                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4243             }
4244         }
4245
4246         vm_mon_cmd_nocheck($vmid, 'qom-set',
4247                     path => "machine/peripheral/balloon0",
4248                     property => "guest-stats-polling-interval",
4249                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4250
4251     });
4252 }
4253
4254 sub vm_mon_cmd {
4255     my ($vmid, $execute, %params) = @_;
4256
4257     my $cmd = { execute => $execute, arguments => \%params };
4258     vm_qmp_command($vmid, $cmd);
4259 }
4260
4261 sub vm_mon_cmd_nocheck {
4262     my ($vmid, $execute, %params) = @_;
4263
4264     my $cmd = { execute => $execute, arguments => \%params };
4265     vm_qmp_command($vmid, $cmd, 1);
4266 }
4267
4268 sub vm_qmp_command {
4269     my ($vmid, $cmd, $nocheck) = @_;
4270
4271     my $res;
4272
4273     my $timeout;
4274     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4275         $timeout = $cmd->{arguments}->{timeout};
4276         delete $cmd->{arguments}->{timeout};
4277     }
4278
4279     eval {
4280         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4281         my $sname = qmp_socket($vmid);
4282         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4283             my $qmpclient = PVE::QMPClient->new();
4284
4285             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4286         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
4287             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
4288                 if scalar(%{$cmd->{arguments}});
4289             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
4290         } else {
4291             die "unable to open monitor socket\n";
4292         }
4293     };
4294     if (my $err = $@) {
4295         syslog("err", "VM $vmid qmp command failed - $err");
4296         die $err;
4297     }
4298
4299     return $res;
4300 }
4301
4302 sub vm_human_monitor_command {
4303     my ($vmid, $cmdline) = @_;
4304
4305     my $res;
4306
4307     my $cmd = {
4308         execute => 'human-monitor-command',
4309         arguments => { 'command-line' => $cmdline},
4310     };
4311
4312     return vm_qmp_command($vmid, $cmd);
4313 }
4314
4315 sub vm_commandline {
4316     my ($storecfg, $vmid) = @_;
4317
4318     my $conf = load_config($vmid);
4319
4320     my $defaults = load_defaults();
4321
4322     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
4323
4324     return join(' ', @$cmd);
4325 }
4326
4327 sub vm_reset {
4328     my ($vmid, $skiplock) = @_;
4329
4330     lock_config($vmid, sub {
4331
4332         my $conf = load_config($vmid);
4333
4334         check_lock($conf) if !$skiplock;
4335
4336         vm_mon_cmd($vmid, "system_reset");
4337     });
4338 }
4339
4340 sub get_vm_volumes {
4341     my ($conf) = @_;
4342
4343     my $vollist = [];
4344     foreach_volid($conf, sub {
4345         my ($volid, $is_cdrom) = @_;
4346
4347         return if $volid =~ m|^/|;
4348
4349         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4350         return if !$sid;
4351
4352         push @$vollist, $volid;
4353     });
4354
4355     return $vollist;
4356 }
4357
4358 sub vm_stop_cleanup {
4359     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
4360
4361     eval {
4362
4363         if (!$keepActive) {
4364             my $vollist = get_vm_volumes($conf);
4365             PVE::Storage::deactivate_volumes($storecfg, $vollist);
4366         }
4367
4368         foreach my $ext (qw(mon qmp pid vnc qga)) {
4369             unlink "/var/run/qemu-server/${vmid}.$ext";
4370         }
4371
4372         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
4373     };
4374     warn $@ if $@; # avoid errors - just warn
4375 }
4376
4377 # Note: use $nockeck to skip tests if VM configuration file exists.
4378 # We need that when migration VMs to other nodes (files already moved)
4379 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
4380 sub vm_stop {
4381     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
4382
4383     $force = 1 if !defined($force) && !$shutdown;
4384
4385     if ($migratedfrom){
4386         my $pid = check_running($vmid, $nocheck, $migratedfrom);
4387         kill 15, $pid if $pid;
4388         my $conf = load_config($vmid, $migratedfrom);
4389         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
4390         return;
4391     }
4392
4393     lock_config($vmid, sub {
4394
4395         my $pid = check_running($vmid, $nocheck);
4396         return if !$pid;
4397
4398         my $conf;
4399         if (!$nocheck) {
4400             $conf = load_config($vmid);
4401             check_lock($conf) if !$skiplock;
4402             if (!defined($timeout) && $shutdown && $conf->{startup}) {
4403                 my $opts = parse_startup($conf->{startup});
4404                 $timeout = $opts->{down} if $opts->{down};
4405             }
4406         }
4407
4408         $timeout = 60 if !defined($timeout);
4409
4410         eval {
4411             if ($shutdown) {
4412                 if (defined($conf) && $conf->{agent}) {
4413                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
4414                 } else {
4415                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
4416                 }
4417             } else {
4418                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
4419             }
4420         };
4421         my $err = $@;
4422
4423         if (!$err) {
4424             my $count = 0;
4425             while (($count < $timeout) && check_running($vmid, $nocheck)) {
4426                 $count++;
4427                 sleep 1;
4428             }
4429
4430             if ($count >= $timeout) {
4431                 if ($force) {
4432                     warn "VM still running - terminating now with SIGTERM\n";
4433                     kill 15, $pid;
4434                 } else {
4435                     die "VM quit/powerdown failed - got timeout\n";
4436                 }
4437             } else {
4438                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4439                 return;
4440             }
4441         } else {
4442             if ($force) {
4443                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
4444                 kill 15, $pid;
4445             } else {
4446                 die "VM quit/powerdown failed\n";
4447             }
4448         }
4449
4450         # wait again
4451         $timeout = 10;
4452
4453         my $count = 0;
4454         while (($count < $timeout) && check_running($vmid, $nocheck)) {
4455             $count++;
4456             sleep 1;
4457         }
4458
4459         if ($count >= $timeout) {
4460             warn "VM still running - terminating now with SIGKILL\n";
4461             kill 9, $pid;
4462             sleep 1;
4463         }
4464
4465         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4466    });
4467 }
4468
4469 sub vm_suspend {
4470     my ($vmid, $skiplock) = @_;
4471
4472     lock_config($vmid, sub {
4473
4474         my $conf = load_config($vmid);
4475
4476         check_lock($conf) if !($skiplock || ($conf->{lock} && $conf->{lock} eq 'backup'));
4477
4478         vm_mon_cmd($vmid, "stop");
4479     });
4480 }
4481
4482 sub vm_resume {
4483     my ($vmid, $skiplock) = @_;
4484
4485     lock_config($vmid, sub {
4486
4487         my $conf = load_config($vmid);
4488
4489         check_lock($conf) if !($skiplock || ($conf->{lock} && $conf->{lock} eq 'backup'));
4490
4491         vm_mon_cmd($vmid, "cont");
4492     });
4493 }
4494
4495 sub vm_sendkey {
4496     my ($vmid, $skiplock, $key) = @_;
4497
4498     lock_config($vmid, sub {
4499
4500         my $conf = load_config($vmid);
4501
4502         # there is no qmp command, so we use the human monitor command
4503         vm_human_monitor_command($vmid, "sendkey $key");
4504     });
4505 }
4506
4507 sub vm_destroy {
4508     my ($storecfg, $vmid, $skiplock) = @_;
4509
4510     lock_config($vmid, sub {
4511
4512         my $conf = load_config($vmid);
4513
4514         check_lock($conf) if !$skiplock;
4515
4516         if (!check_running($vmid)) {
4517             destroy_vm($storecfg, $vmid);
4518         } else {
4519             die "VM $vmid is running - destroy failed\n";
4520         }
4521     });
4522 }
4523
4524 # pci helpers
4525
4526 sub file_write {
4527     my ($filename, $buf) = @_;
4528
4529     my $fh = IO::File->new($filename, "w");
4530     return undef if !$fh;
4531
4532     my $res = print $fh $buf;
4533
4534     $fh->close();
4535
4536     return $res;
4537 }
4538
4539 sub pci_device_info {
4540     my ($name) = @_;
4541
4542     my $res;
4543
4544     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
4545     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
4546
4547     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
4548     return undef if !defined($irq) || $irq !~ m/^\d+$/;
4549
4550     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
4551     return undef if !defined($vendor) || $vendor !~ s/^0x//;
4552
4553     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
4554     return undef if !defined($product) || $product !~ s/^0x//;
4555
4556     $res = {
4557         name => $name,
4558         vendor => $vendor,
4559         product => $product,
4560         domain => $domain,
4561         bus => $bus,
4562         slot => $slot,
4563         func => $func,
4564         irq => $irq,
4565         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
4566     };
4567
4568     return $res;
4569 }
4570
4571 sub pci_dev_reset {
4572     my ($dev) = @_;
4573
4574     my $name = $dev->{name};
4575
4576     my $fn = "$pcisysfs/devices/$name/reset";
4577
4578     return file_write($fn, "1");
4579 }
4580
4581 sub pci_dev_bind_to_stub {
4582     my ($dev) = @_;
4583
4584     my $name = $dev->{name};
4585
4586     my $testdir = "$pcisysfs/drivers/pci-stub/$name";
4587     return 1 if -d $testdir;
4588
4589     my $data = "$dev->{vendor} $dev->{product}";
4590     return undef if !file_write("$pcisysfs/drivers/pci-stub/new_id", $data);
4591
4592     my $fn = "$pcisysfs/devices/$name/driver/unbind";
4593     if (!file_write($fn, $name)) {
4594         return undef if -f $fn;
4595     }
4596
4597     $fn = "$pcisysfs/drivers/pci-stub/bind";
4598     if (! -d $testdir) {
4599         return undef if !file_write($fn, $name);
4600     }
4601
4602     return -d $testdir;
4603 }
4604
4605 sub pci_dev_bind_to_vfio {
4606     my ($dev) = @_;
4607
4608     my $name = $dev->{name};
4609
4610     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4611
4612     if (!-d $vfio_basedir) {
4613         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4614     }
4615     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4616
4617     my $testdir = "$vfio_basedir/$name";
4618     return 1 if -d $testdir;
4619
4620     my $data = "$dev->{vendor} $dev->{product}";
4621     return undef if !file_write("$vfio_basedir/new_id", $data);
4622
4623     my $fn = "$pcisysfs/devices/$name/driver/unbind";
4624     if (!file_write($fn, $name)) {
4625         return undef if -f $fn;
4626     }
4627
4628     $fn = "$vfio_basedir/bind";
4629     if (! -d $testdir) {
4630         return undef if !file_write($fn, $name);
4631     }
4632
4633     return -d $testdir;
4634 }
4635
4636 sub pci_dev_group_bind_to_vfio {
4637     my ($pciid) = @_;
4638
4639     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4640
4641     if (!-d $vfio_basedir) {
4642         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4643     }
4644     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4645
4646     # get IOMMU group devices
4647     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
4648       my @devs = grep /^0000:/, readdir($D);
4649     closedir($D);
4650
4651     foreach my $pciid (@devs) {
4652         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
4653
4654         # pci bridges, switches or root ports are not supported
4655         # they have a pci_bus subdirectory so skip them
4656         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
4657
4658         my $info = pci_device_info($1);
4659         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
4660     }
4661
4662     return 1;
4663 }
4664
4665 sub print_pci_addr {
4666     my ($id, $bridges) = @_;
4667
4668     my $res = '';
4669     my $devices = {
4670         piix3 => { bus => 0, addr => 1 },
4671         #addr2 : first videocard
4672         balloon0 => { bus => 0, addr => 3 },
4673         watchdog => { bus => 0, addr => 4 },
4674         scsihw0 => { bus => 0, addr => 5 },
4675         'pci.3' => { bus => 0, addr => 5 }, #can also be used for virtio-scsi-single bridge
4676         scsihw1 => { bus => 0, addr => 6 },
4677         ahci0 => { bus => 0, addr => 7 },
4678         qga0 => { bus => 0, addr => 8 },
4679         spice => { bus => 0, addr => 9 },
4680         virtio0 => { bus => 0, addr => 10 },
4681         virtio1 => { bus => 0, addr => 11 },
4682         virtio2 => { bus => 0, addr => 12 },
4683         virtio3 => { bus => 0, addr => 13 },
4684         virtio4 => { bus => 0, addr => 14 },
4685         virtio5 => { bus => 0, addr => 15 },
4686         hostpci0 => { bus => 0, addr => 16 },
4687         hostpci1 => { bus => 0, addr => 17 },
4688         net0 => { bus => 0, addr => 18 },
4689         net1 => { bus => 0, addr => 19 },
4690         net2 => { bus => 0, addr => 20 },
4691         net3 => { bus => 0, addr => 21 },
4692         net4 => { bus => 0, addr => 22 },
4693         net5 => { bus => 0, addr => 23 },
4694         vga1 => { bus => 0, addr => 24 },
4695         vga2 => { bus => 0, addr => 25 },
4696         vga3 => { bus => 0, addr => 26 },
4697         hostpci2 => { bus => 0, addr => 27 },
4698         hostpci3 => { bus => 0, addr => 28 },
4699         #addr29 : usb-host (pve-usb.cfg)
4700         'pci.1' => { bus => 0, addr => 30 },
4701         'pci.2' => { bus => 0, addr => 31 },
4702         'net6' => { bus => 1, addr => 1 },
4703         'net7' => { bus => 1, addr => 2 },
4704         'net8' => { bus => 1, addr => 3 },
4705         'net9' => { bus => 1, addr => 4 },
4706         'net10' => { bus => 1, addr => 5 },
4707         'net11' => { bus => 1, addr => 6 },
4708         'net12' => { bus => 1, addr => 7 },
4709         'net13' => { bus => 1, addr => 8 },
4710         'net14' => { bus => 1, addr => 9 },
4711         'net15' => { bus => 1, addr => 10 },
4712         'net16' => { bus => 1, addr => 11 },
4713         'net17' => { bus => 1, addr => 12 },
4714         'net18' => { bus => 1, addr => 13 },
4715         'net19' => { bus => 1, addr => 14 },
4716         'net20' => { bus => 1, addr => 15 },
4717         'net21' => { bus => 1, addr => 16 },
4718         'net22' => { bus => 1, addr => 17 },
4719         'net23' => { bus => 1, addr => 18 },
4720         'net24' => { bus => 1, addr => 19 },
4721         'net25' => { bus => 1, addr => 20 },
4722         'net26' => { bus => 1, addr => 21 },
4723         'net27' => { bus => 1, addr => 22 },
4724         'net28' => { bus => 1, addr => 23 },
4725         'net29' => { bus => 1, addr => 24 },
4726         'net30' => { bus => 1, addr => 25 },
4727         'net31' => { bus => 1, addr => 26 },
4728         'virtio6' => { bus => 2, addr => 1 },
4729         'virtio7' => { bus => 2, addr => 2 },
4730         'virtio8' => { bus => 2, addr => 3 },
4731         'virtio9' => { bus => 2, addr => 4 },
4732         'virtio10' => { bus => 2, addr => 5 },
4733         'virtio11' => { bus => 2, addr => 6 },
4734         'virtio12' => { bus => 2, addr => 7 },
4735         'virtio13' => { bus => 2, addr => 8 },
4736         'virtio14' => { bus => 2, addr => 9 },
4737         'virtio15' => { bus => 2, addr => 10 },
4738         'virtioscsi0' => { bus => 3, addr => 1 },
4739         'virtioscsi1' => { bus => 3, addr => 2 },
4740         'virtioscsi2' => { bus => 3, addr => 3 },
4741         'virtioscsi3' => { bus => 3, addr => 4 },
4742         'virtioscsi4' => { bus => 3, addr => 5 },
4743         'virtioscsi5' => { bus => 3, addr => 6 },
4744         'virtioscsi6' => { bus => 3, addr => 7 },
4745         'virtioscsi7' => { bus => 3, addr => 8 },
4746         'virtioscsi8' => { bus => 3, addr => 9 },
4747         'virtioscsi9' => { bus => 3, addr => 10 },
4748         'virtioscsi10' => { bus => 3, addr => 11 },
4749         'virtioscsi11' => { bus => 3, addr => 12 },
4750         'virtioscsi12' => { bus => 3, addr => 13 },
4751         'virtioscsi13' => { bus => 3, addr => 14 },
4752         'virtioscsi14' => { bus => 3, addr => 15 },
4753         'virtioscsi15' => { bus => 3, addr => 16 },
4754         'virtioscsi16' => { bus => 3, addr => 17 },
4755         'virtioscsi17' => { bus => 3, addr => 18 },
4756         'virtioscsi18' => { bus => 3, addr => 19 },
4757         'virtioscsi19' => { bus => 3, addr => 20 },
4758         'virtioscsi20' => { bus => 3, addr => 21 },
4759         'virtioscsi21' => { bus => 3, addr => 22 },
4760         'virtioscsi22' => { bus => 3, addr => 23 },
4761         'virtioscsi23' => { bus => 3, addr => 24 },
4762         'virtioscsi24' => { bus => 3, addr => 25 },
4763         'virtioscsi25' => { bus => 3, addr => 26 },
4764         'virtioscsi26' => { bus => 3, addr => 27 },
4765         'virtioscsi27' => { bus => 3, addr => 28 },
4766         'virtioscsi28' => { bus => 3, addr => 29 },
4767         'virtioscsi29' => { bus => 3, addr => 30 },
4768         'virtioscsi30' => { bus => 3, addr => 31 },
4769
4770     };
4771
4772     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
4773            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
4774            my $bus = $devices->{$id}->{bus};
4775            $res = ",bus=pci.$bus,addr=$addr";
4776            $bridges->{$bus} = 1 if $bridges;
4777     }
4778     return $res;
4779
4780 }
4781
4782 sub print_pcie_addr {
4783     my ($id) = @_;
4784
4785     my $res = '';
4786     my $devices = {
4787         hostpci0 => { bus => "ich9-pcie-port-1", addr => 0 },
4788         hostpci1 => { bus => "ich9-pcie-port-2", addr => 0 },
4789         hostpci2 => { bus => "ich9-pcie-port-3", addr => 0 },
4790         hostpci3 => { bus => "ich9-pcie-port-4", addr => 0 },
4791     };
4792
4793     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
4794            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
4795            my $bus = $devices->{$id}->{bus};
4796            $res = ",bus=$bus,addr=$addr";
4797     }
4798     return $res;
4799
4800 }
4801
4802 # vzdump restore implementaion
4803
4804 sub tar_archive_read_firstfile {
4805     my $archive = shift;
4806
4807     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
4808
4809     # try to detect archive type first
4810     my $pid = open (TMP, "tar tf '$archive'|") ||
4811         die "unable to open file '$archive'\n";
4812     my $firstfile = <TMP>;
4813     kill 15, $pid;
4814     close TMP;
4815
4816     die "ERROR: archive contaions no data\n" if !$firstfile;
4817     chomp $firstfile;
4818
4819     return $firstfile;
4820 }
4821
4822 sub tar_restore_cleanup {
4823     my ($storecfg, $statfile) = @_;
4824
4825     print STDERR "starting cleanup\n";
4826
4827     if (my $fd = IO::File->new($statfile, "r")) {
4828         while (defined(my $line = <$fd>)) {
4829             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
4830                 my $volid = $2;
4831                 eval {
4832                     if ($volid =~ m|^/|) {
4833                         unlink $volid || die 'unlink failed\n';
4834                     } else {
4835                         PVE::Storage::vdisk_free($storecfg, $volid);
4836                     }
4837                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
4838                 };
4839                 print STDERR "unable to cleanup '$volid' - $@" if $@;
4840             } else {
4841                 print STDERR "unable to parse line in statfile - $line";
4842             }
4843         }
4844         $fd->close();
4845     }
4846 }
4847
4848 sub restore_archive {
4849     my ($archive, $vmid, $user, $opts) = @_;
4850
4851     my $format = $opts->{format};
4852     my $comp;
4853
4854     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
4855         $format = 'tar' if !$format;
4856         $comp = 'gzip';
4857     } elsif ($archive =~ m/\.tar$/) {
4858         $format = 'tar' if !$format;
4859     } elsif ($archive =~ m/.tar.lzo$/) {
4860         $format = 'tar' if !$format;
4861         $comp = 'lzop';
4862     } elsif ($archive =~ m/\.vma$/) {
4863         $format = 'vma' if !$format;
4864     } elsif ($archive =~ m/\.vma\.gz$/) {
4865         $format = 'vma' if !$format;
4866         $comp = 'gzip';
4867     } elsif ($archive =~ m/\.vma\.lzo$/) {
4868         $format = 'vma' if !$format;
4869         $comp = 'lzop';
4870     } else {
4871         $format = 'vma' if !$format; # default
4872     }
4873
4874     # try to detect archive format
4875     if ($format eq 'tar') {
4876         return restore_tar_archive($archive, $vmid, $user, $opts);
4877     } else {
4878         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
4879     }
4880 }
4881
4882 sub restore_update_config_line {
4883     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
4884
4885     return if $line =~ m/^\#qmdump\#/;
4886     return if $line =~ m/^\#vzdump\#/;
4887     return if $line =~ m/^lock:/;
4888     return if $line =~ m/^unused\d+:/;
4889     return if $line =~ m/^parent:/;
4890     return if $line =~ m/^template:/; # restored VM is never a template
4891
4892     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
4893         # try to convert old 1.X settings
4894         my ($id, $ind, $ethcfg) = ($1, $2, $3);
4895         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
4896             my ($model, $macaddr) = split(/\=/, $devconfig);
4897             $macaddr = PVE::Tools::random_ether_addr() if !$macaddr || $unique;
4898             my $net = {
4899                 model => $model,
4900                 bridge => "vmbr$ind",
4901                 macaddr => $macaddr,
4902             };
4903             my $netstr = print_net($net);
4904
4905             print $outfd "net$cookie->{netcount}: $netstr\n";
4906             $cookie->{netcount}++;
4907         }
4908     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
4909         my ($id, $netstr) = ($1, $2);
4910         my $net = parse_net($netstr);
4911         $net->{macaddr} = PVE::Tools::random_ether_addr() if $net->{macaddr};
4912         $netstr = print_net($net);
4913         print $outfd "$id: $netstr\n";
4914     } elsif ($line =~ m/^((ide|scsi|virtio|sata)\d+):\s*(\S+)\s*$/) {
4915         my $virtdev = $1;
4916         my $value = $3;
4917         if ($line =~ m/backup=no/) {
4918             print $outfd "#$line";
4919         } elsif ($virtdev && $map->{$virtdev}) {
4920             my $di = parse_drive($virtdev, $value);
4921             delete $di->{format}; # format can change on restore
4922             $di->{file} = $map->{$virtdev};
4923             $value = print_drive($vmid, $di);
4924             print $outfd "$virtdev: $value\n";
4925         } else {
4926             print $outfd $line;
4927         }
4928     } else {
4929         print $outfd $line;
4930     }
4931 }
4932
4933 sub scan_volids {
4934     my ($cfg, $vmid) = @_;
4935
4936     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
4937
4938     my $volid_hash = {};
4939     foreach my $storeid (keys %$info) {
4940         foreach my $item (@{$info->{$storeid}}) {
4941             next if !($item->{volid} && $item->{size});
4942             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
4943             $volid_hash->{$item->{volid}} = $item;
4944         }
4945     }
4946
4947     return $volid_hash;
4948 }
4949
4950 sub get_used_paths {
4951     my ($vmid, $storecfg, $conf, $scan_snapshots, $skip_drive) = @_;
4952
4953     my $used_path = {};
4954
4955     my $scan_config = sub {
4956         my ($cref, $snapname) = @_;
4957
4958         foreach my $key (keys %$cref) {
4959             my $value = $cref->{$key};
4960             if (valid_drivename($key)) {
4961                 next if $skip_drive && $key eq $skip_drive;
4962                 my $drive = parse_drive($key, $value);
4963                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
4964                 if ($drive->{file} =~ m!^/!) {
4965                     $used_path->{$drive->{file}}++; # = 1;
4966                 } else {
4967                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
4968                     next if !$storeid;
4969                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
4970                     next if !$scfg;
4971                     my $path = PVE::Storage::path($storecfg, $drive->{file}, $snapname);
4972                     $used_path->{$path}++; # = 1;
4973                 }
4974             }
4975         }
4976     };
4977
4978     &$scan_config($conf);
4979
4980     undef $skip_drive;
4981
4982     if ($scan_snapshots) {
4983         foreach my $snapname (keys %{$conf->{snapshots}}) {
4984             &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
4985         }
4986     }
4987
4988     return $used_path;
4989 }
4990
4991 sub update_disksize {
4992     my ($vmid, $conf, $volid_hash) = @_;
4993
4994     my $changes;
4995
4996     my $used = {};
4997
4998     # Note: it is allowed to define multiple storages with same path (alias), so
4999     # we need to check both 'volid' and real 'path' (two different volid can point
5000     # to the same path).
5001
5002     my $usedpath = {};
5003
5004     # update size info
5005     foreach my $opt (keys %$conf) {
5006         if (valid_drivename($opt)) {
5007             my $drive = parse_drive($opt, $conf->{$opt});
5008             my $volid = $drive->{file};
5009             next if !$volid;
5010
5011             $used->{$volid} = 1;
5012             if ($volid_hash->{$volid} &&
5013                 (my $path = $volid_hash->{$volid}->{path})) {
5014                 $usedpath->{$path} = 1;
5015             }
5016
5017             next if drive_is_cdrom($drive);
5018             next if !$volid_hash->{$volid};
5019
5020             $drive->{size} = $volid_hash->{$volid}->{size};
5021             my $new = print_drive($vmid, $drive);
5022             if ($new ne $conf->{$opt}) {
5023                 $changes = 1;
5024                 $conf->{$opt} = $new;
5025             }
5026         }
5027     }
5028
5029     # remove 'unusedX' entry if volume is used
5030     foreach my $opt (keys %$conf) {
5031         next if $opt !~ m/^unused\d+$/;
5032         my $volid = $conf->{$opt};
5033         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5034         if ($used->{$volid} || ($path && $usedpath->{$path})) {
5035             $changes = 1;
5036             delete $conf->{$opt};
5037         }
5038     }
5039
5040     foreach my $volid (sort keys %$volid_hash) {
5041         next if $volid =~ m/vm-$vmid-state-/;
5042         next if $used->{$volid};
5043         my $path = $volid_hash->{$volid}->{path};
5044         next if !$path; # just to be sure
5045         next if $usedpath->{$path};
5046         $changes = 1;
5047         add_unused_volume($conf, $volid);
5048         $usedpath->{$path} = 1; # avoid to add more than once (aliases)
5049     }
5050
5051     return $changes;
5052 }
5053
5054 sub rescan {
5055     my ($vmid, $nolock) = @_;
5056
5057     my $cfg = PVE::Cluster::cfs_read_file("storage.cfg");
5058
5059     my $volid_hash = scan_volids($cfg, $vmid);
5060
5061     my $updatefn =  sub {
5062         my ($vmid) = @_;
5063
5064         my $conf = load_config($vmid);
5065
5066         check_lock($conf);
5067
5068         my $vm_volids = {};
5069         foreach my $volid (keys %$volid_hash) {
5070             my $info = $volid_hash->{$volid};
5071             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5072         }
5073
5074         my $changes = update_disksize($vmid, $conf, $vm_volids);
5075
5076         update_config_nolock($vmid, $conf, 1) if $changes;
5077     };
5078
5079     if (defined($vmid)) {
5080         if ($nolock) {
5081             &$updatefn($vmid);
5082         } else {
5083             lock_config($vmid, $updatefn, $vmid);
5084         }
5085     } else {
5086         my $vmlist = config_list();
5087         foreach my $vmid (keys %$vmlist) {
5088             if ($nolock) {
5089                 &$updatefn($vmid);
5090             } else {
5091                 lock_config($vmid, $updatefn, $vmid);
5092             }
5093         }
5094     }
5095 }
5096
5097 sub restore_vma_archive {
5098     my ($archive, $vmid, $user, $opts, $comp) = @_;
5099
5100     my $input = $archive eq '-' ? "<&STDIN" : undef;
5101     my $readfrom = $archive;
5102
5103     my $uncomp = '';
5104     if ($comp) {
5105         $readfrom = '-';
5106         my $qarchive = PVE::Tools::shellquote($archive);
5107         if ($comp eq 'gzip') {
5108             $uncomp = "zcat $qarchive|";
5109         } elsif ($comp eq 'lzop') {
5110             $uncomp = "lzop -d -c $qarchive|";
5111         } else {
5112             die "unknown compression method '$comp'\n";
5113         }
5114
5115     }
5116
5117     my $tmpdir = "/var/tmp/vzdumptmp$$";
5118     rmtree $tmpdir;
5119
5120     # disable interrupts (always do cleanups)
5121     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5122         warn "got interrupt - ignored\n";
5123     };
5124
5125     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5126     POSIX::mkfifo($mapfifo, 0600);
5127     my $fifofh;
5128
5129     my $openfifo = sub {
5130         open($fifofh, '>', $mapfifo) || die $!;
5131     };
5132
5133     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5134
5135     my $oldtimeout;
5136     my $timeout = 5;
5137
5138     my $devinfo = {};
5139
5140     my $rpcenv = PVE::RPCEnvironment::get();
5141
5142     my $conffile = config_file($vmid);
5143     my $tmpfn = "$conffile.$$.tmp";
5144
5145     # Note: $oldconf is undef if VM does not exists
5146     my $oldconf = PVE::Cluster::cfs_read_file(cfs_config_path($vmid));
5147
5148     my $print_devmap = sub {
5149         my $virtdev_hash = {};
5150
5151         my $cfgfn = "$tmpdir/qemu-server.conf";
5152
5153         # we can read the config - that is already extracted
5154         my $fh = IO::File->new($cfgfn, "r") ||
5155             "unable to read qemu-server.conf - $!\n";
5156
5157         while (defined(my $line = <$fh>)) {
5158             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5159                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5160                 die "archive does not contain data for drive '$virtdev'\n"
5161                     if !$devinfo->{$devname};
5162                 if (defined($opts->{storage})) {
5163                     $storeid = $opts->{storage} || 'local';
5164                 } elsif (!$storeid) {
5165                     $storeid = 'local';
5166                 }
5167                 $format = 'raw' if !$format;
5168                 $devinfo->{$devname}->{devname} = $devname;
5169                 $devinfo->{$devname}->{virtdev} = $virtdev;
5170                 $devinfo->{$devname}->{format} = $format;
5171                 $devinfo->{$devname}->{storeid} = $storeid;
5172
5173                 # check permission on storage
5174                 my $pool = $opts->{pool}; # todo: do we need that?
5175                 if ($user ne 'root@pam') {
5176                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5177                 }
5178
5179                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5180             }
5181         }
5182
5183         foreach my $devname (keys %$devinfo) {
5184             die "found no device mapping information for device '$devname'\n"
5185                 if !$devinfo->{$devname}->{virtdev};
5186         }
5187
5188         my $cfg = cfs_read_file('storage.cfg');
5189
5190         # create empty/temp config
5191         if ($oldconf) {
5192             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5193             foreach_drive($oldconf, sub {
5194                 my ($ds, $drive) = @_;
5195
5196                 return if drive_is_cdrom($drive);
5197
5198                 my $volid = $drive->{file};
5199
5200                 return if !$volid || $volid =~ m|^/|;
5201
5202                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5203                 return if !$path || !$owner || ($owner != $vmid);
5204
5205                 # Note: only delete disk we want to restore
5206                 # other volumes will become unused
5207                 if ($virtdev_hash->{$ds}) {
5208                     PVE::Storage::vdisk_free($cfg, $volid);
5209                 }
5210             });
5211         }
5212
5213         my $map = {};
5214         foreach my $virtdev (sort keys %$virtdev_hash) {
5215             my $d = $virtdev_hash->{$virtdev};
5216             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5217             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5218
5219             # test if requested format is supported
5220             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5221             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5222             $d->{format} = $defFormat if !$supported;
5223
5224             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5225                                                   $d->{format}, undef, $alloc_size);
5226             print STDERR "new volume ID is '$volid'\n";
5227             $d->{volid} = $volid;
5228             my $path = PVE::Storage::path($cfg, $volid);
5229
5230             my $write_zeros = 1;
5231             # fixme: what other storages types initialize volumes with zero?
5232             if ($scfg->{type} eq 'dir' || $scfg->{type} eq 'nfs' || $scfg->{type} eq 'glusterfs' ||
5233                 $scfg->{type} eq 'sheepdog' || $scfg->{type} eq 'rbd') {
5234                 $write_zeros = 0;
5235             }
5236
5237             print $fifofh "${write_zeros}:$d->{devname}=$path\n";
5238
5239             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5240             $map->{$virtdev} = $volid;
5241         }
5242
5243         $fh->seek(0, 0) || die "seek failed - $!\n";
5244
5245         my $outfd = new IO::File ($tmpfn, "w") ||
5246             die "unable to write config for VM $vmid\n";
5247
5248         my $cookie = { netcount => 0 };
5249         while (defined(my $line = <$fh>)) {
5250             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5251         }
5252
5253         $fh->close();
5254         $outfd->close();
5255     };
5256
5257     eval {
5258         # enable interrupts
5259         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5260             die "interrupted by signal\n";
5261         };
5262         local $SIG{ALRM} = sub { die "got timeout\n"; };
5263
5264         $oldtimeout = alarm($timeout);
5265
5266         my $parser = sub {
5267             my $line = shift;
5268
5269             print "$line\n";
5270
5271             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5272                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5273                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5274             } elsif ($line =~ m/^CTIME: /) {
5275                 # we correctly received the vma config, so we can disable
5276                 # the timeout now for disk allocation (set to 10 minutes, so
5277                 # that we always timeout if something goes wrong)
5278                 alarm(600);
5279                 &$print_devmap();
5280                 print $fifofh "done\n";
5281                 my $tmp = $oldtimeout || 0;
5282                 $oldtimeout = undef;
5283                 alarm($tmp);
5284                 close($fifofh);
5285             }
5286         };
5287
5288         print "restore vma archive: $cmd\n";
5289         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5290     };
5291     my $err = $@;
5292
5293     alarm($oldtimeout) if $oldtimeout;
5294
5295     unlink $mapfifo;
5296
5297     if ($err) {
5298         rmtree $tmpdir;
5299         unlink $tmpfn;
5300
5301         my $cfg = cfs_read_file('storage.cfg');
5302         foreach my $devname (keys %$devinfo) {
5303             my $volid = $devinfo->{$devname}->{volid};
5304             next if !$volid;
5305             eval {
5306                 if ($volid =~ m|^/|) {
5307                     unlink $volid || die 'unlink failed\n';
5308                 } else {
5309                     PVE::Storage::vdisk_free($cfg, $volid);
5310                 }
5311                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5312             };
5313             print STDERR "unable to cleanup '$volid' - $@" if $@;
5314         }
5315         die $err;
5316     }
5317
5318     rmtree $tmpdir;
5319
5320     rename($tmpfn, $conffile) ||
5321         die "unable to commit configuration file '$conffile'\n";
5322
5323     PVE::Cluster::cfs_update(); # make sure we read new file
5324
5325     eval { rescan($vmid, 1); };
5326     warn $@ if $@;
5327 }
5328
5329 sub restore_tar_archive {
5330     my ($archive, $vmid, $user, $opts) = @_;
5331
5332     if ($archive ne '-') {
5333         my $firstfile = tar_archive_read_firstfile($archive);
5334         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5335             if $firstfile ne 'qemu-server.conf';
5336     }
5337
5338     my $storecfg = cfs_read_file('storage.cfg');
5339
5340     # destroy existing data - keep empty config
5341     my $vmcfgfn = config_file($vmid);
5342     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5343
5344     my $tocmd = "/usr/lib/qemu-server/qmextract";
5345
5346     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5347     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5348     $tocmd .= ' --prealloc' if $opts->{prealloc};
5349     $tocmd .= ' --info' if $opts->{info};
5350
5351     # tar option "xf" does not autodetect compression when read from STDIN,
5352     # so we pipe to zcat
5353     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5354         PVE::Tools::shellquote("--to-command=$tocmd");
5355
5356     my $tmpdir = "/var/tmp/vzdumptmp$$";
5357     mkpath $tmpdir;
5358
5359     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5360     local $ENV{VZDUMP_VMID} = $vmid;
5361     local $ENV{VZDUMP_USER} = $user;
5362
5363     my $conffile = config_file($vmid);
5364     my $tmpfn = "$conffile.$$.tmp";
5365
5366     # disable interrupts (always do cleanups)
5367     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5368         print STDERR "got interrupt - ignored\n";
5369     };
5370
5371     eval {
5372         # enable interrupts
5373         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5374             die "interrupted by signal\n";
5375         };
5376
5377         if ($archive eq '-') {
5378             print "extracting archive from STDIN\n";
5379             run_command($cmd, input => "<&STDIN");
5380         } else {
5381             print "extracting archive '$archive'\n";
5382             run_command($cmd);
5383         }
5384
5385         return if $opts->{info};
5386
5387         # read new mapping
5388         my $map = {};
5389         my $statfile = "$tmpdir/qmrestore.stat";
5390         if (my $fd = IO::File->new($statfile, "r")) {
5391             while (defined (my $line = <$fd>)) {
5392                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5393                     $map->{$1} = $2 if $1;
5394                 } else {
5395                     print STDERR "unable to parse line in statfile - $line\n";
5396                 }
5397             }
5398             $fd->close();
5399         }
5400
5401         my $confsrc = "$tmpdir/qemu-server.conf";
5402
5403         my $srcfd = new IO::File($confsrc, "r") ||
5404             die "unable to open file '$confsrc'\n";
5405
5406         my $outfd = new IO::File ($tmpfn, "w") ||
5407             die "unable to write config for VM $vmid\n";
5408
5409         my $cookie = { netcount => 0 };
5410         while (defined (my $line = <$srcfd>)) {
5411             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5412         }
5413
5414         $srcfd->close();
5415         $outfd->close();
5416     };
5417     my $err = $@;
5418
5419     if ($err) {
5420
5421         unlink $tmpfn;
5422
5423         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
5424
5425         die $err;
5426     }
5427
5428     rmtree $tmpdir;
5429
5430     rename $tmpfn, $conffile ||
5431         die "unable to commit configuration file '$conffile'\n";
5432
5433     PVE::Cluster::cfs_update(); # make sure we read new file
5434
5435     eval { rescan($vmid, 1); };
5436     warn $@ if $@;
5437 };
5438
5439
5440 # Internal snapshots
5441
5442 # NOTE: Snapshot create/delete involves several non-atomic
5443 # action, and can take a long time.
5444 # So we try to avoid locking the file and use 'lock' variable
5445 # inside the config file instead.
5446
5447 my $snapshot_copy_config = sub {
5448     my ($source, $dest) = @_;
5449
5450     foreach my $k (keys %$source) {
5451         next if $k eq 'snapshots';
5452         next if $k eq 'snapstate';
5453         next if $k eq 'snaptime';
5454         next if $k eq 'vmstate';
5455         next if $k eq 'lock';
5456         next if $k eq 'digest';
5457         next if $k eq 'description';
5458         next if $k =~ m/^unused\d+$/;
5459
5460         $dest->{$k} = $source->{$k};
5461     }
5462 };
5463
5464 my $snapshot_apply_config = sub {
5465     my ($conf, $snap) = @_;
5466
5467     # copy snapshot list
5468     my $newconf = {
5469         snapshots => $conf->{snapshots},
5470     };
5471
5472     # keep description and list of unused disks
5473     foreach my $k (keys %$conf) {
5474         next if !($k =~ m/^unused\d+$/ || $k eq 'description');
5475         $newconf->{$k} = $conf->{$k};
5476     }
5477
5478     &$snapshot_copy_config($snap, $newconf);
5479
5480     return $newconf;
5481 };
5482
5483 sub foreach_writable_storage {
5484     my ($conf, $func) = @_;
5485
5486     my $sidhash = {};
5487
5488     foreach my $ds (keys %$conf) {
5489         next if !valid_drivename($ds);
5490
5491         my $drive = parse_drive($ds, $conf->{$ds});
5492         next if !$drive;
5493         next if drive_is_cdrom($drive);
5494
5495         my $volid = $drive->{file};
5496
5497         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5498         $sidhash->{$sid} = $sid if $sid;
5499     }
5500
5501     foreach my $sid (sort keys %$sidhash) {
5502         &$func($sid);
5503     }
5504 }
5505
5506 my $alloc_vmstate_volid = sub {
5507     my ($storecfg, $vmid, $conf, $snapname) = @_;
5508
5509     # Note: we try to be smart when selecting a $target storage
5510
5511     my $target;
5512
5513     # search shared storage first
5514     foreach_writable_storage($conf, sub {
5515         my ($sid) = @_;
5516         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
5517         return if !$scfg->{shared};
5518
5519         $target = $sid if !$target || $scfg->{path}; # prefer file based storage
5520     });
5521
5522     if (!$target) {
5523         # now search local storage
5524         foreach_writable_storage($conf, sub {
5525             my ($sid) = @_;
5526             my $scfg = PVE::Storage::storage_config($storecfg, $sid);
5527             return if $scfg->{shared};
5528
5529             $target = $sid if !$target || $scfg->{path}; # prefer file based storage;
5530         });
5531     }
5532
5533     $target = 'local' if !$target;
5534
5535     my $driver_state_size = 500; # assume 32MB is enough to safe all driver state;
5536     # we abort live save after $conf->{memory}, so we need at max twice that space
5537     my $size = $conf->{memory}*2 + $driver_state_size;
5538
5539     my $name = "vm-$vmid-state-$snapname";
5540     my $scfg = PVE::Storage::storage_config($storecfg, $target);
5541     $name .= ".raw" if $scfg->{path}; # add filename extension for file base storage
5542     my $volid = PVE::Storage::vdisk_alloc($storecfg, $target, $vmid, 'raw', $name, $size*1024);
5543
5544     return $volid;
5545 };
5546
5547 my $snapshot_prepare = sub {
5548     my ($vmid, $snapname, $save_vmstate, $comment) = @_;
5549
5550     my $snap;
5551
5552     my $updatefn =  sub {
5553
5554         my $conf = load_config($vmid);
5555
5556         die "you can't take a snapshot if it's a template\n"
5557             if is_template($conf);
5558
5559         check_lock($conf);
5560
5561         $conf->{lock} = 'snapshot';
5562
5563         die "snapshot name '$snapname' already used\n"
5564             if defined($conf->{snapshots}->{$snapname});
5565
5566         my $storecfg = PVE::Storage::config();
5567         die "snapshot feature is not available" if !has_feature('snapshot', $conf, $storecfg);
5568
5569         $snap = $conf->{snapshots}->{$snapname} = {};
5570
5571         if ($save_vmstate && check_running($vmid)) {
5572             $snap->{vmstate} = &$alloc_vmstate_volid($storecfg, $vmid, $conf, $snapname);
5573         }
5574
5575         &$snapshot_copy_config($conf, $snap);
5576
5577         $snap->{snapstate} = "prepare";
5578         $snap->{snaptime} = time();
5579         $snap->{description} = $comment if $comment;
5580
5581         # always overwrite machine if we save vmstate. This makes sure we
5582         # can restore it later using correct machine type
5583         $snap->{machine} = get_current_qemu_machine($vmid) if $snap->{vmstate};
5584
5585         update_config_nolock($vmid, $conf, 1);
5586     };
5587
5588     lock_config($vmid, $updatefn);
5589
5590     return $snap;
5591 };
5592
5593 my $snapshot_commit = sub {
5594     my ($vmid, $snapname) = @_;
5595
5596     my $updatefn = sub {
5597
5598         my $conf = load_config($vmid);
5599
5600         die "missing snapshot lock\n"
5601             if !($conf->{lock} && $conf->{lock} eq 'snapshot');
5602
5603         my $has_machine_config = defined($conf->{machine});
5604
5605         my $snap = $conf->{snapshots}->{$snapname};
5606
5607         die "snapshot '$snapname' does not exist\n" if !defined($snap);
5608
5609         die "wrong snapshot state\n"
5610             if !($snap->{snapstate} && $snap->{snapstate} eq "prepare");
5611
5612         delete $snap->{snapstate};
5613         delete $conf->{lock};
5614
5615         my $newconf = &$snapshot_apply_config($conf, $snap);
5616
5617         delete $newconf->{machine} if !$has_machine_config;
5618
5619         $newconf->{parent} = $snapname;
5620
5621         update_config_nolock($vmid, $newconf, 1);
5622     };
5623
5624     lock_config($vmid, $updatefn);
5625 };
5626
5627 sub snapshot_rollback {
5628     my ($vmid, $snapname) = @_;
5629
5630     my $prepare = 1;
5631
5632     my $storecfg = PVE::Storage::config();
5633
5634     my $conf = load_config($vmid);
5635
5636     my $get_snapshot_config = sub {
5637
5638         die "you can't rollback if vm is a template\n" if is_template($conf);
5639
5640         my $res = $conf->{snapshots}->{$snapname};
5641
5642         die "snapshot '$snapname' does not exist\n" if !defined($res);
5643
5644         return $res;
5645     };
5646
5647     my $snap = &$get_snapshot_config();
5648
5649     foreach_drive($snap, sub {
5650         my ($ds, $drive) = @_;
5651
5652         return if drive_is_cdrom($drive);
5653
5654         my $volid = $drive->{file};
5655
5656         PVE::Storage::volume_rollback_is_possible($storecfg, $volid, $snapname);
5657     });
5658
5659     my $updatefn = sub {
5660
5661         $conf = load_config($vmid);
5662
5663         $snap = &$get_snapshot_config();
5664
5665         die "unable to rollback to incomplete snapshot (snapstate = $snap->{snapstate})\n"
5666             if $snap->{snapstate};
5667
5668         if ($prepare) {
5669             check_lock($conf);
5670             vm_stop($storecfg, $vmid, undef, undef, 5, undef, undef);
5671         }
5672
5673         die "unable to rollback vm $vmid: vm is running\n"
5674             if check_running($vmid);
5675
5676         if ($prepare) {
5677             $conf->{lock} = 'rollback';
5678         } else {
5679             die "got wrong lock\n" if !($conf->{lock} && $conf->{lock} eq 'rollback');
5680             delete $conf->{lock};
5681         }
5682
5683         my $forcemachine;
5684
5685         if (!$prepare) {
5686             my $has_machine_config = defined($conf->{machine});
5687
5688             # copy snapshot config to current config
5689             $conf = &$snapshot_apply_config($conf, $snap);
5690             $conf->{parent} = $snapname;
5691
5692             # Note: old code did not store 'machine', so we try to be smart
5693             # and guess the snapshot was generated with kvm 1.4 (pc-i440fx-1.4).
5694             $forcemachine = $conf->{machine} || 'pc-i440fx-1.4';
5695             # we remove the 'machine' configuration if not explicitly specified
5696             # in the original config.
5697             delete $conf->{machine} if $snap->{vmstate} && !$has_machine_config;
5698         }
5699
5700         update_config_nolock($vmid, $conf, 1);
5701
5702         if (!$prepare && $snap->{vmstate}) {
5703             my $statefile = PVE::Storage::path($storecfg, $snap->{vmstate});
5704             vm_start($storecfg, $vmid, $statefile, undef, undef, undef, $forcemachine);
5705         }
5706     };
5707
5708     lock_config($vmid, $updatefn);
5709
5710     foreach_drive($snap, sub {
5711         my ($ds, $drive) = @_;
5712
5713         return if drive_is_cdrom($drive);
5714
5715         my $volid = $drive->{file};
5716         my $device = "drive-$ds";
5717
5718         PVE::Storage::volume_snapshot_rollback($storecfg, $volid, $snapname);
5719     });
5720
5721     $prepare = 0;
5722     lock_config($vmid, $updatefn);
5723 }
5724
5725 my $savevm_wait = sub {
5726     my ($vmid) = @_;
5727
5728     for(;;) {
5729         my $stat = vm_mon_cmd_nocheck($vmid, "query-savevm");
5730         if (!$stat->{status}) {
5731             die "savevm not active\n";
5732         } elsif ($stat->{status} eq 'active') {
5733             sleep(1);
5734             next;
5735         } elsif ($stat->{status} eq 'completed') {
5736             last;
5737         } else {
5738             die "query-savevm returned status '$stat->{status}'\n";
5739         }
5740     }
5741 };
5742
5743 sub do_snapshots_with_qemu {
5744     my ($storecfg, $volid) = @_;
5745
5746     my $storage_name = PVE::Storage::parse_volume_id($volid);
5747
5748     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}} ){
5749         return 1;
5750     }
5751
5752     if ($volid =~ m/\.(qcow2|qed)$/){
5753         return 1;
5754     }
5755
5756     return undef;
5757 }
5758
5759 sub snapshot_create {
5760     my ($vmid, $snapname, $save_vmstate, $comment) = @_;
5761
5762     my $snap = &$snapshot_prepare($vmid, $snapname, $save_vmstate, $comment);
5763
5764     $save_vmstate = 0 if !$snap->{vmstate}; # vm is not running
5765
5766     my $config = load_config($vmid);
5767
5768     my $running = check_running($vmid);
5769
5770     my $freezefs = $running && $config->{agent};
5771     $freezefs = 0 if $snap->{vmstate}; # not needed if we save RAM
5772
5773     my $drivehash = {};
5774
5775     if ($freezefs) {
5776         eval { vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
5777         warn "guest-fsfreeze-freeze problems - $@" if $@;
5778     }
5779
5780     eval {
5781         # create internal snapshots of all drives
5782
5783         my $storecfg = PVE::Storage::config();
5784
5785         if ($running) {
5786             if ($snap->{vmstate}) {
5787                 my $path = PVE::Storage::path($storecfg, $snap->{vmstate});
5788                 vm_mon_cmd($vmid, "savevm-start", statefile => $path);
5789                 &$savevm_wait($vmid);
5790             } else {
5791                 vm_mon_cmd($vmid, "savevm-start");
5792             }
5793         };
5794
5795         foreach_drive($snap, sub {
5796             my ($ds, $drive) = @_;
5797
5798             return if drive_is_cdrom($drive);
5799
5800             my $volid = $drive->{file};
5801             my $device = "drive-$ds";
5802
5803             qemu_volume_snapshot($vmid, $device, $storecfg, $volid, $snapname);
5804             $drivehash->{$ds} = 1;
5805        });
5806     };
5807     my $err = $@;
5808
5809     if ($running) {
5810         eval { vm_mon_cmd($vmid, "savevm-end")  };
5811         warn $@ if $@;
5812
5813         if ($freezefs) {
5814             eval { vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
5815             warn "guest-fsfreeze-thaw problems - $@" if $@;
5816         }
5817
5818         # savevm-end is async, we need to wait
5819         for (;;) {
5820             my $stat = vm_mon_cmd_nocheck($vmid, "query-savevm");
5821             if (!$stat->{bytes}) {
5822                 last;
5823             } else {
5824                 print "savevm not yet finished\n";
5825                 sleep(1);
5826                 next;
5827             }
5828         }
5829     }
5830
5831     if ($err) {
5832         warn "snapshot create failed: starting cleanup\n";
5833         eval { snapshot_delete($vmid, $snapname, 0, $drivehash); };
5834         warn $@ if $@;
5835         die $err;
5836     }
5837
5838     &$snapshot_commit($vmid, $snapname);
5839 }
5840
5841 # Note: $drivehash is only set when called from snapshot_create.
5842 sub snapshot_delete {
5843     my ($vmid, $snapname, $force, $drivehash) = @_;
5844
5845     my $prepare = 1;
5846
5847     my $snap;
5848     my $unused = [];
5849
5850     my $unlink_parent = sub {
5851         my ($confref, $new_parent) = @_;
5852
5853         if ($confref->{parent} && $confref->{parent} eq $snapname) {
5854             if ($new_parent) {
5855                 $confref->{parent} = $new_parent;
5856             } else {
5857                 delete $confref->{parent};
5858             }
5859         }
5860     };
5861
5862     my $updatefn =  sub {
5863         my ($remove_drive) = @_;
5864
5865         my $conf = load_config($vmid);
5866
5867         if (!$drivehash) {
5868             check_lock($conf);
5869             die "you can't delete a snapshot if vm is a template\n"
5870                 if is_template($conf);
5871         }
5872
5873         $snap = $conf->{snapshots}->{$snapname};
5874
5875         die "snapshot '$snapname' does not exist\n" if !defined($snap);
5876
5877         # remove parent refs
5878         if (!$prepare) {
5879             &$unlink_parent($conf, $snap->{parent});
5880             foreach my $sn (keys %{$conf->{snapshots}}) {
5881                 next if $sn eq $snapname;
5882                 &$unlink_parent($conf->{snapshots}->{$sn}, $snap->{parent});
5883             }
5884         }
5885
5886         if ($remove_drive) {
5887             if ($remove_drive eq 'vmstate') {
5888                 delete $snap->{$remove_drive};
5889             } else {
5890                 my $drive = parse_drive($remove_drive, $snap->{$remove_drive});
5891                 my $volid = $drive->{file};
5892                 delete $snap->{$remove_drive};
5893                 add_unused_volume($conf, $volid);
5894             }
5895         }
5896
5897         if ($prepare) {
5898             $snap->{snapstate} = 'delete';
5899         } else {
5900             delete $conf->{snapshots}->{$snapname};
5901             delete $conf->{lock} if $drivehash;
5902             foreach my $volid (@$unused) {
5903                 add_unused_volume($conf, $volid);
5904             }
5905         }
5906
5907         update_config_nolock($vmid, $conf, 1);
5908     };
5909
5910     lock_config($vmid, $updatefn);
5911
5912     # now remove vmstate file
5913
5914     my $storecfg = PVE::Storage::config();
5915
5916     if ($snap->{vmstate}) {
5917         eval {  PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
5918         if (my $err = $@) {
5919             die $err if !$force;
5920             warn $err;
5921         }
5922         # save changes (remove vmstate from snapshot)
5923         lock_config($vmid, $updatefn, 'vmstate') if !$force;
5924     };
5925
5926     # now remove all internal snapshots
5927     foreach_drive($snap, sub {
5928         my ($ds, $drive) = @_;
5929
5930         return if drive_is_cdrom($drive);
5931
5932         my $volid = $drive->{file};
5933         my $device = "drive-$ds";
5934
5935         if (!$drivehash || $drivehash->{$ds}) {
5936             eval { qemu_volume_snapshot_delete($vmid, $device, $storecfg, $volid, $snapname); };
5937             if (my $err = $@) {
5938                 die $err if !$force;
5939                 warn $err;
5940             }
5941         }
5942
5943         # save changes (remove drive fron snapshot)
5944         lock_config($vmid, $updatefn, $ds) if !$force;
5945         push @$unused, $volid;
5946     });
5947
5948     # now cleanup config
5949     $prepare = 0;
5950     lock_config($vmid, $updatefn);
5951 }
5952
5953 sub has_feature {
5954     my ($feature, $conf, $storecfg, $snapname, $running) = @_;
5955
5956     my $err;
5957     foreach_drive($conf, sub {
5958         my ($ds, $drive) = @_;
5959
5960         return if drive_is_cdrom($drive);
5961         my $volid = $drive->{file};
5962         $err = 1 if !PVE::Storage::volume_has_feature($storecfg, $feature, $volid, $snapname, $running);
5963     });
5964
5965     return $err ? 0 : 1;
5966 }
5967
5968 sub template_create {
5969     my ($vmid, $conf, $disk) = @_;
5970
5971     my $storecfg = PVE::Storage::config();
5972
5973     foreach_drive($conf, sub {
5974         my ($ds, $drive) = @_;
5975
5976         return if drive_is_cdrom($drive);
5977         return if $disk && $ds ne $disk;
5978
5979         my $volid = $drive->{file};
5980         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
5981
5982         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
5983         $drive->{file} = $voliddst;
5984         $conf->{$ds} = print_drive($vmid, $drive);
5985         update_config_nolock($vmid, $conf, 1);
5986     });
5987 }
5988
5989 sub is_template {
5990     my ($conf) = @_;
5991
5992     return 1 if defined $conf->{template} && $conf->{template} == 1;
5993 }
5994
5995 sub qemu_img_convert {
5996     my ($src_volid, $dst_volid, $size, $snapname) = @_;
5997
5998     my $storecfg = PVE::Storage::config();
5999     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6000     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6001
6002     if ($src_storeid && $dst_storeid) {
6003         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6004         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6005
6006         my $src_format = qemu_img_format($src_scfg, $src_volname);
6007         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6008
6009         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6010         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6011
6012         my $cmd = [];
6013         push @$cmd, '/usr/bin/qemu-img', 'convert', '-t', 'writeback', '-p', '-n';
6014         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
6015         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path, $dst_path;
6016
6017         my $parser = sub {
6018             my $line = shift;
6019             if($line =~ m/\((\S+)\/100\%\)/){
6020                 my $percent = $1;
6021                 my $transferred = int($size * $percent / 100);
6022                 my $remaining = $size - $transferred;
6023
6024                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6025             }
6026
6027         };
6028
6029         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6030         my $err = $@;
6031         die "copy failed: $err" if $err;
6032     }
6033 }
6034
6035 sub qemu_img_format {
6036     my ($scfg, $volname) = @_;
6037
6038     if ($scfg->{path} && $volname =~ m/\.(raw|cow|qcow|qcow2|qed|vmdk|cloop)$/) {
6039         return $1;
6040     } else {
6041         return "raw";
6042     }
6043 }
6044
6045 sub qemu_drive_mirror {
6046     my ($vmid, $drive, $dst_volid, $vmiddst) = @_;
6047
6048     my $storecfg = PVE::Storage::config();
6049     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6050
6051     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6052
6053     my $format = qemu_img_format($dst_scfg, $dst_volname);
6054
6055     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6056
6057     #drive-mirror is doing lseek on source image before starting, and this can take a lot of time for big nfs volume
6058     #during this time, qmp socket is hanging
6059     #http://lists.nongnu.org/archive/html/qemu-devel/2015-05/msg01838.html
6060     #so we need to setup a big timeout
6061     my $opts = { timeout => 14400, device => "drive-$drive", mode => "existing", sync => "full", target => $dst_path };
6062     $opts->{format} = $format if $format;
6063
6064     print "drive mirror is starting : this step can take some minutes/hours, depend of disk size and storage speed\n";
6065
6066     vm_mon_cmd($vmid, "drive-mirror", %$opts);
6067     eval {
6068         while (1) {
6069             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6070             my $stat = @$stats[0];
6071             die "mirroring job seem to have die. Maybe do you have bad sectors?" if !$stat;
6072             die "error job is not mirroring" if $stat->{type} ne "mirror";
6073
6074             my $busy = $stat->{busy};
6075             my $ready = $stat->{ready};
6076
6077             if (my $total = $stat->{len}) {
6078                 my $transferred = $stat->{offset} || 0;
6079                 my $remaining = $total - $transferred;
6080                 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6081
6082                 print "transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6083             }
6084
6085
6086             if ($stat->{ready} eq 'true') {
6087
6088                 last if $vmiddst != $vmid;
6089
6090                 # try to switch the disk if source and destination are on the same guest
6091                 eval { vm_mon_cmd($vmid, "block-job-complete", device => "drive-$drive") };
6092                 last if !$@;
6093                 die $@ if $@ !~ m/cannot be completed/;
6094             }
6095             sleep 1;
6096         }
6097
6098
6099     };
6100     my $err = $@;
6101
6102     my $cancel_job = sub {
6103         vm_mon_cmd($vmid, "block-job-cancel", device => "drive-$drive");
6104         while (1) {
6105             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6106             my $stat = @$stats[0];
6107             last if !$stat;
6108             sleep 1;
6109         }
6110     };
6111
6112     if ($err) {
6113         eval { &$cancel_job(); };
6114         die "mirroring error: $err";
6115     }
6116
6117     if ($vmiddst != $vmid) {
6118         # if we clone a disk for a new target vm, we don't switch the disk
6119         &$cancel_job(); # so we call block-job-cancel
6120     }
6121 }
6122
6123 sub clone_disk {
6124     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6125         $newvmid, $storage, $format, $full, $newvollist) = @_;
6126
6127     my $newvolid;
6128
6129     if (!$full) {
6130         print "create linked clone of drive $drivename ($drive->{file})\n";
6131         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6132         push @$newvollist, $newvolid;
6133     } else {
6134         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6135         $storeid = $storage if $storage;
6136
6137         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6138         if (!$format) {
6139             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6140             $format = qemu_img_format($scfg, $volname);
6141         }
6142
6143         # test if requested format is supported - else use default
6144         my $supported = grep { $_ eq $format } @$validFormats;
6145         $format = $defFormat if !$supported;
6146
6147         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6148
6149         print "create full clone of drive $drivename ($drive->{file})\n";
6150         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $format, undef, ($size/1024));
6151         push @$newvollist, $newvolid;
6152
6153         if (!$running || $snapname) {
6154             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname);
6155         } else {
6156             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid);
6157         }
6158     }
6159
6160     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6161
6162     my $disk = $drive;
6163     $disk->{format} = undef;
6164     $disk->{file} = $newvolid;
6165     $disk->{size} = $size;
6166
6167     return $disk;
6168 }
6169
6170 # this only works if VM is running
6171 sub get_current_qemu_machine {
6172     my ($vmid) = @_;
6173
6174     my $cmd = { execute => 'query-machines', arguments => {} };
6175     my $res = vm_qmp_command($vmid, $cmd);
6176
6177     my ($current, $default);
6178     foreach my $e (@$res) {
6179         $default = $e->{name} if $e->{'is-default'};
6180         $current = $e->{name} if $e->{'is-current'};
6181     }
6182
6183     # fallback to the default machine if current is not supported by qemu
6184     return $current || $default || 'pc';
6185 }
6186
6187 sub qemu_machine_feature_enabled {
6188     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6189
6190     my $current_major;
6191     my $current_minor;
6192
6193     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6194
6195         $current_major = $3;
6196         $current_minor = $4;
6197
6198     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6199
6200         $current_major = $1;
6201         $current_minor = $2;
6202     }
6203
6204     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
6205
6206
6207 }
6208
6209 sub lspci {
6210
6211     my $devices = {};
6212
6213     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6214             my (undef, $id, $function) = @_;
6215             my $res = { id => $id, function => $function};
6216             push @{$devices->{$id}}, $res;
6217     });
6218
6219     return $devices;
6220 }
6221
6222 sub vm_iothreads_list {
6223     my ($vmid) = @_;
6224
6225     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6226
6227     my $iothreads = {};
6228     foreach my $iothread (@$res) {
6229         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6230     }
6231
6232     return $iothreads;
6233 }
6234
6235 sub scsihw_infos {
6236     my ($conf, $drive) = @_;
6237
6238     my $maxdev = 0;
6239
6240     if ($conf->{scsihw} && ($conf->{scsihw} =~ m/^lsi/)) {
6241         $maxdev = 7;
6242     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6243         $maxdev = 1;
6244     } else {
6245         $maxdev = 256;
6246     }
6247
6248     my $controller = int($drive->{index} / $maxdev);
6249     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6250
6251     return ($maxdev, $controller, $controller_prefix);
6252 }
6253
6254 1;