PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QMPClient;
  31 use PVE::RPCEnvironment;
  32 use Time::HiRes qw(gettimeofday);
  33
  34 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  35
  36 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  37
  38 # Note about locking: we use flock on the config file protect
  39 # against concurent actions.
  40 # Aditionaly, we have a 'lock' setting in the config file. This
  41 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  42 # allowed when such lock is set. But you can ignore this kind of
  43 # lock with the --skiplock flag.
  44
  45 cfs_register_file('/qemu-server/',
  46                   \&parse_vm_config,
  47                   \&write_vm_config);
  48
  49 PVE::JSONSchema::register_standard_option('skiplock', {
  50     description => "Ignore locks - only root is allowed to use this option.",
  51     type => 'boolean',
  52     optional => 1,
  53 });
  54
  55 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  56     description => "Some command save/restore state from this location.",
  57     type => 'string',
  58     maxLength => 128,
  59     optional => 1,
  60 });
  61
  62 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  63     description => "The name of the snapshot.",
  64     type => 'string', format => 'pve-configid',
  65     maxLength => 40,
  66 });
  67
  68 #no warnings 'redefine';
  69
  70 sub cgroups_write {
  71    my ($controller, $vmid, $option, $value) = @_;
  72
  73    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  74    PVE::ProcFSTools::write_proc_entry($path, $value);
  75
  76 }
  77
  78 my $nodename = PVE::INotify::nodename();
  79
  80 mkdir "/etc/pve/nodes/$nodename";
  81 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  82 mkdir $confdir;
  83
  84 my $var_run_tmpdir = "/var/run/qemu-server";
  85 mkdir $var_run_tmpdir;
  86
  87 my $lock_dir = "/var/lock/qemu-server";
  88 mkdir $lock_dir;
  89
  90 my $pcisysfs = "/sys/bus/pci";
  91
  92 my $confdesc = {
  93     onboot => {
  94         optional => 1,
  95         type => 'boolean',
  96         description => "Specifies whether a VM will be started during system bootup.",
  97         default => 0,
  98     },
  99     autostart => {
 100         optional => 1,
 101         type => 'boolean',
 102         description => "Automatic restart after crash (currently ignored).",
 103         default => 0,
 104     },
 105     hotplug => {
 106         optional => 1,
 107         type => 'string', format => 'pve-hotplug-features',
 108         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 109         default => 'network,disk,usb',
 110     },
 111     reboot => {
 112         optional => 1,
 113         type => 'boolean',
 114         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 115         default => 1,
 116     },
 117     lock => {
 118         optional => 1,
 119         type => 'string',
 120         description => "Lock/unlock the VM.",
 121         enum => [qw(migrate backup snapshot rollback)],
 122     },
 123     cpulimit => {
 124         optional => 1,
 125         type => 'number',
 126         description => "Limit of CPU usage. Note if the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 127         minimum => 0,
 128         maximum => 128,
 129         default => 0,
 130     },
 131     cpuunits => {
 132         optional => 1,
 133         type => 'integer',
 134         description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 135         minimum => 0,
 136         maximum => 500000,
 137         default => 1000,
 138     },
 139     memory => {
 140         optional => 1,
 141         type => 'integer',
 142         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 143         minimum => 16,
 144         default => 512,
 145     },
 146     balloon => {
 147         optional => 1,
 148         type => 'integer',
 149         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 150         minimum => 0,
 151     },
 152     shares => {
 153         optional => 1,
 154         type => 'integer',
 155         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 156         minimum => 0,
 157         maximum => 50000,
 158         default => 1000,
 159     },
 160     keyboard => {
 161         optional => 1,
 162         type => 'string',
 163         description => "Keybord layout for vnc server. Default is read from the datacenter configuration file.",
 164         enum => PVE::Tools::kvmkeymaplist(),
 165         default => 'en-us',
 166     },
 167     name => {
 168         optional => 1,
 169         type => 'string', format => 'dns-name',
 170         description => "Set a name for the VM. Only used on the configuration web interface.",
 171     },
 172     scsihw => {
 173         optional => 1,
 174         type => 'string',
 175         description => "scsi controller model",
 176         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 177         default => 'lsi',
 178     },
 179     description => {
 180         optional => 1,
 181         type => 'string',
 182         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 183     },
 184     ostype => {
 185         optional => 1,
 186         type => 'string',
 187         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 l24 l26 solaris)],
 188         description => <<EODESC,
 189 Used to enable special optimization/features for specific
 190 operating systems:
 191
 192 other  => unspecified OS
 193 wxp    => Microsoft Windows XP
 194 w2k    => Microsoft Windows 2000
 195 w2k3   => Microsoft Windows 2003
 196 w2k8   => Microsoft Windows 2008
 197 wvista => Microsoft Windows Vista
 198 win7   => Microsoft Windows 7
 199 win8   => Microsoft Windows 8/2012
 200 l24    => Linux 2.4 Kernel
 201 l26    => Linux 2.6/3.X Kernel
 202 solaris => solaris/opensolaris/openindiania kernel
 203
 204 other|l24|l26|solaris                       ... no special behaviour
 205 wxp|w2k|w2k3|w2k8|wvista|win7|win8  ... use --localtime switch
 206 EODESC
 207     },
 208     boot => {
 209         optional => 1,
 210         type => 'string',
 211         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 212         pattern => '[acdn]{1,4}',
 213         default => 'cdn',
 214     },
 215     bootdisk => {
 216         optional => 1,
 217         type => 'string', format => 'pve-qm-bootdisk',
 218         description => "Enable booting from specified disk.",
 219         pattern => '(ide|sata|scsi|virtio)\d+',
 220     },
 221     smp => {
 222         optional => 1,
 223         type => 'integer',
 224         description => "The number of CPUs. Please use option -sockets instead.",
 225         minimum => 1,
 226         default => 1,
 227     },
 228     sockets => {
 229         optional => 1,
 230         type => 'integer',
 231         description => "The number of CPU sockets.",
 232         minimum => 1,
 233         default => 1,
 234     },
 235     cores => {
 236         optional => 1,
 237         type => 'integer',
 238         description => "The number of cores per socket.",
 239         minimum => 1,
 240         default => 1,
 241     },
 242     numa => {
 243         optional => 1,
 244         type => 'boolean',
 245         description => "Enable/disable Numa.",
 246         default => 0,
 247     },
 248     vcpus => {
 249         optional => 1,
 250         type => 'integer',
 251         description => "Number of hotplugged vcpus.",
 252         minimum => 1,
 253         default => 0,
 254     },
 255     acpi => {
 256         optional => 1,
 257         type => 'boolean',
 258         description => "Enable/disable ACPI.",
 259         default => 1,
 260     },
 261     agent => {
 262         optional => 1,
 263         type => 'boolean',
 264         description => "Enable/disable Qemu GuestAgent.",
 265         default => 0,
 266     },
 267     kvm => {
 268         optional => 1,
 269         type => 'boolean',
 270         description => "Enable/disable KVM hardware virtualization.",
 271         default => 1,
 272     },
 273     tdf => {
 274         optional => 1,
 275         type => 'boolean',
 276         description => "Enable/disable time drift fix.",
 277         default => 0,
 278     },
 279     localtime => {
 280         optional => 1,
 281         type => 'boolean',
 282         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 283     },
 284     freeze => {
 285         optional => 1,
 286         type => 'boolean',
 287         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 288     },
 289     vga => {
 290         optional => 1,
 291         type => 'string',
 292         description => "Select VGA type. If you want to use high resolution modes (>= 1280x1024x16) then you should use option 'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and 'cirrur' for other OS types. Option 'qxl' enables the SPICE display sever. You can also run without any graphic card using a serial devive as terminal.",
 293         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 294     },
 295     watchdog => {
 296         optional => 1,
 297         type => 'string', format => 'pve-qm-watchdog',
 298         typetext => '[[model=]i6300esb|ib700] [,[action=]reset|shutdown|poweroff|pause|debug|none]',
 299         description => "Create a virtual hardware watchdog device.  Once enabled (by a guest action), the watchdog must be periodically polled by an agent inside the guest or else the guest will be restarted (or execute the action specified)",
 300     },
 301     startdate => {
 302         optional => 1,
 303         type => 'string',
 304         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 305         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 306         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 307         default => 'now',
 308     },
 309     startup =>  get_standard_option('pve-startup-order'),
 310     template => {
 311         optional => 1,
 312         type => 'boolean',
 313         description => "Enable/disable Template.",
 314         default => 0,
 315     },
 316     args => {
 317         optional => 1,
 318         type => 'string',
 319         description => <<EODESCR,
 320 Note: this option is for experts only. It allows you to pass arbitrary arguments to kvm, for example:
 321
 322 args: -no-reboot -no-hpet
 323 EODESCR
 324     },
 325     tablet => {
 326         optional => 1,
 327         type => 'boolean',
 328         default => 1,
 329         description => "Enable/disable the usb tablet device. This device is usually needed to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with normal VNC clients. If you're running lots of console-only guests on one host, you may consider disabling this to save some context switches. This is turned of by default if you use spice (vga=qxl).",
 330     },
 331     migrate_speed => {
 332         optional => 1,
 333         type => 'integer',
 334         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 335         minimum => 0,
 336         default => 0,
 337     },
 338     migrate_downtime => {
 339         optional => 1,
 340         type => 'number',
 341         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 342         minimum => 0,
 343         default => 0.1,
 344     },
 345     cdrom => {
 346         optional => 1,
 347         type => 'string', format => 'pve-qm-drive',
 348         typetext => 'volume',
 349         description => "This is an alias for option -ide2",
 350     },
 351     cpu => {
 352         optional => 1,
 353         description => "Emulated CPU type.",
 354         type => 'string',
 355         enum => [ qw(486 athlon pentium pentium2 pentium3 coreduo core2duo kvm32 kvm64 qemu32 qemu64 phenom Conroe Penryn Nehalem Westmere SandyBridge IvyBridge Haswell Broadwell Opteron_G1 Opteron_G2 Opteron_G3 Opteron_G4 Opteron_G5 host) ],
 356         default => 'kvm64',
 357     },
 358     parent => get_standard_option('pve-snapshot-name', {
 359         optional => 1,
 360         description => "Parent snapshot name. This is used internally, and should not be modified.",
 361     }),
 362     snaptime => {
 363         optional => 1,
 364         description => "Timestamp for snapshots.",
 365         type => 'integer',
 366         minimum => 0,
 367     },
 368     vmstate => {
 369         optional => 1,
 370         type => 'string', format => 'pve-volume-id',
 371         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 372     },
 373     machine => {
 374         description => "Specific the Qemu machine type.",
 375         type => 'string',
 376         pattern => '(pc|pc(-i440fx)?-\d+\.\d+|q35|pc-q35-\d+\.\d+)',
 377         maxLength => 40,
 378         optional => 1,
 379     },
 380     smbios1 => {
 381         description => "Specify SMBIOS type 1 fields.",
 382         type => 'string', format => 'pve-qm-smbios1',
 383         typetext => "[manufacturer=str][,product=str][,version=str][,serial=str] [,uuid=uuid][,sku=str][,family=str]",
 384         maxLength => 256,
 385         optional => 1,
 386     },
 387     protection => {
 388         optional => 1,
 389         type => 'boolean',
 390         description => "Sets the protection flag of the VM. This will prevent the remove operation.",
 391         default => 0,
 392     },
 393 };
 394
 395 # what about other qemu settings ?
 396 #cpu => 'string',
 397 #machine => 'string',
 398 #fda => 'file',
 399 #fdb => 'file',
 400 #mtdblock => 'file',
 401 #sd => 'file',
 402 #pflash => 'file',
 403 #snapshot => 'bool',
 404 #bootp => 'file',
 405 ##tftp => 'dir',
 406 ##smb => 'dir',
 407 #kernel => 'file',
 408 #append => 'string',
 409 #initrd => 'file',
 410 ##soundhw => 'string',
 411
 412 while (my ($k, $v) = each %$confdesc) {
 413     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 414 }
 415
 416 my $MAX_IDE_DISKS = 4;
 417 my $MAX_SCSI_DISKS = 14;
 418 my $MAX_VIRTIO_DISKS = 16;
 419 my $MAX_SATA_DISKS = 6;
 420 my $MAX_USB_DEVICES = 5;
 421 my $MAX_NETS = 32;
 422 my $MAX_UNUSED_DISKS = 8;
 423 my $MAX_HOSTPCI_DEVICES = 4;
 424 my $MAX_SERIAL_PORTS = 4;
 425 my $MAX_PARALLEL_PORTS = 3;
 426 my $MAX_NUMA = 8;
 427 my $MAX_MEM = 4194304;
 428 my $STATICMEM = 1024;
 429
 430 my $numadesc = {
 431     optional => 1,
 432     type => 'string', format => 'pve-qm-numanode',
 433     typetext => "cpus=<id[-id],memory=<mb>[[,hostnodes=<id[-id]>] [,policy=<preferred|bind|interleave>]]",
 434     description => "numa topology",
 435 };
 436 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 437
 438 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 439     $confdesc->{"numa$i"} = $numadesc;
 440 }
 441
 442 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 443                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 444                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 445 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 446
 447 my $netdesc = {
 448     optional => 1,
 449     type => 'string', format => 'pve-qm-net',
 450     typetext => "MODEL=XX:XX:XX:XX:XX:XX [,bridge=<dev>][,queues=<nbqueues>][,rate=<mbps>] [,tag=<vlanid>][,firewall=0|1],link_down=0|1]",
 451     description => <<EODESCR,
 452 Specify network devices.
 453
 454 MODEL is one of: $nic_model_list_txt
 455
 456 XX:XX:XX:XX:XX:XX should be an unique MAC address. This is
 457 automatically generated if not specified.
 458
 459 The bridge parameter can be used to automatically add the interface to a bridge device. The Proxmox VE standard bridge is called 'vmbr0'.
 460
 461 Option 'rate' is used to limit traffic bandwidth from and to this interface. It is specified as floating point number, unit is 'Megabytes per second'.
 462
 463 If you specify no bridge, we create a kvm 'user' (NATed) network device, which provides DHCP and DNS services. The following addresses are used:
 464
 465 10.0.2.2   Gateway
 466 10.0.2.3   DNS Server
 467 10.0.2.4   SMB Server
 468
 469 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 470
 471 EODESCR
 472 };
 473 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 474
 475 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 476     $confdesc->{"net$i"} = $netdesc;
 477 }
 478
 479 my $drivename_hash;
 480
 481 my $idedesc = {
 482     optional => 1,
 483     type => 'string', format => 'pve-qm-drive',
 484     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads] [,discard=ignore|on]',
 485     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
 486 };
 487 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
 488
 489 my $scsidesc = {
 490     optional => 1,
 491     type => 'string', format => 'pve-qm-drive',
 492     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads] [,discard=ignore|on] [,iothread=on] [,queues=<nbqueues>]',
 493     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
 494 };
 495 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 496
 497 my $satadesc = {
 498     optional => 1,
 499     type => 'string', format => 'pve-qm-drive',
 500     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads]  [,discard=ignore|on]',
 501     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
 502 };
 503 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
 504
 505 my $virtiodesc = {
 506     optional => 1,
 507     type => 'string', format => 'pve-qm-drive',
 508     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads]  [,discard=ignore|on] [,iothread=on]',
 509     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
 510 };
 511 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
 512
 513 my $usbdesc = {
 514     optional => 1,
 515     type => 'string', format => 'pve-qm-usb-device',
 516     typetext => 'host=HOSTUSBDEVICE|spice',
 517     description => <<EODESCR,
 518 Configure an USB device (n is 0 to 4). This can be used to
 519 pass-through usb devices to the guest. HOSTUSBDEVICE syntax is:
 520
 521 'bus-port(.port)*' (decimal numbers) or
 522 'vendor_id:product_id' (hexadeciaml numbers)
 523
 524 You can use the 'lsusb -t' command to list existing usb devices.
 525
 526 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 527
 528 The value 'spice' can be used to add a usb redirection devices for spice.
 529
 530 EODESCR
 531 };
 532 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
 533
 534 my $hostpcidesc = {
 535         optional => 1,
 536         type => 'string', format => 'pve-qm-hostpci',
 537         typetext => "[host=]HOSTPCIDEVICE [,rombar=on|off] [,pcie=0|1] [,x-vga=on|off]",
 538         description => <<EODESCR,
 539 Map host pci devices. HOSTPCIDEVICE syntax is:
 540
 541 'bus:dev.func' (hexadecimal numbers)
 542
 543 You can us the 'lspci' command to list existing pci devices.
 544
 545 The 'rombar' option determines whether or not the device's ROM will be visible in the guest's memory map (default is 'on').
 546
 547 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 548
 549 Experimental: user reported problems with this option.
 550 EODESCR
 551 };
 552 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
 553
 554 my $serialdesc = {
 555         optional => 1,
 556         type => 'string',
 557         pattern => '(/dev/.+|socket)',
 558         description =>  <<EODESCR,
 559 Create a serial device inside the VM (n is 0 to 3), and pass through a host serial device (i.e. /dev/ttyS0), or create a unix socket on the host side (use 'qm terminal' to open a terminal connection).
 560
 561 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 562
 563 Experimental: user reported problems with this option.
 564 EODESCR
 565 };
 566
 567 my $paralleldesc= {
 568         optional => 1,
 569         type => 'string',
 570         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
 571         description =>  <<EODESCR,
 572 Map host parallel devices (n is 0 to 2).
 573
 574 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 575
 576 Experimental: user reported problems with this option.
 577 EODESCR
 578 };
 579
 580 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
 581     $confdesc->{"parallel$i"} = $paralleldesc;
 582 }
 583
 584 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
 585     $confdesc->{"serial$i"} = $serialdesc;
 586 }
 587
 588 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
 589     $confdesc->{"hostpci$i"} = $hostpcidesc;
 590 }
 591
 592 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
 593     $drivename_hash->{"ide$i"} = 1;
 594     $confdesc->{"ide$i"} = $idedesc;
 595 }
 596
 597 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
 598     $drivename_hash->{"sata$i"} = 1;
 599     $confdesc->{"sata$i"} = $satadesc;
 600 }
 601
 602 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
 603     $drivename_hash->{"scsi$i"} = 1;
 604     $confdesc->{"scsi$i"} = $scsidesc ;
 605 }
 606
 607 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
 608     $drivename_hash->{"virtio$i"} = 1;
 609     $confdesc->{"virtio$i"} = $virtiodesc;
 610 }
 611
 612 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
 613     $confdesc->{"usb$i"} = $usbdesc;
 614 }
 615
 616 my $unuseddesc = {
 617     optional => 1,
 618     type => 'string', format => 'pve-volume-id',
 619     description => "Reference to unused volumes.",
 620 };
 621
 622 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
 623     $confdesc->{"unused$i"} = $unuseddesc;
 624 }
 625
 626 my $kvm_api_version = 0;
 627
 628 sub kvm_version {
 629
 630     return $kvm_api_version if $kvm_api_version;
 631
 632     my $fh = IO::File->new("</dev/kvm") ||
 633         return 0;
 634
 635     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
 636         $kvm_api_version = $v;
 637     }
 638
 639     $fh->close();
 640
 641     return  $kvm_api_version;
 642 }
 643
 644 my $kvm_user_version;
 645
 646 sub kvm_user_version {
 647
 648     return $kvm_user_version if $kvm_user_version;
 649
 650     $kvm_user_version = 'unknown';
 651
 652     my $tmp = `kvm -help 2>/dev/null`;
 653
 654     if ($tmp =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
 655         $kvm_user_version = $2;
 656     }
 657
 658     return $kvm_user_version;
 659
 660 }
 661
 662 my $kernel_has_vhost_net = -c '/dev/vhost-net';
 663
 664 sub disknames {
 665     # order is important - used to autoselect boot disk
 666     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
 667             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
 668             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
 669             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))));
 670 }
 671
 672 sub valid_drivename {
 673     my $dev = shift;
 674
 675     return defined($drivename_hash->{$dev});
 676 }
 677
 678 sub option_exists {
 679     my $key = shift;
 680     return defined($confdesc->{$key});
 681 }
 682
 683 sub nic_models {
 684     return $nic_model_list;
 685 }
 686
 687 sub os_list_description {
 688
 689     return {
 690         other => 'Other',
 691         wxp => 'Windows XP',
 692         w2k => 'Windows 2000',
 693         w2k3 =>, 'Windows 2003',
 694         w2k8 => 'Windows 2008',
 695         wvista => 'Windows Vista',
 696         win7 => 'Windows 7',
 697         win8 => 'Windows 8/2012',
 698         l24 => 'Linux 2.4',
 699         l26 => 'Linux 2.6',
 700     };
 701 }
 702
 703 my $cdrom_path;
 704
 705 sub get_cdrom_path {
 706
 707     return  $cdrom_path if $cdrom_path;
 708
 709     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
 710     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
 711     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
 712 }
 713
 714 sub get_iso_path {
 715     my ($storecfg, $vmid, $cdrom) = @_;
 716
 717     if ($cdrom eq 'cdrom') {
 718         return get_cdrom_path();
 719     } elsif ($cdrom eq 'none') {
 720         return '';
 721     } elsif ($cdrom =~ m|^/|) {
 722         return $cdrom;
 723     } else {
 724         return PVE::Storage::path($storecfg, $cdrom);
 725     }
 726 }
 727
 728 # try to convert old style file names to volume IDs
 729 sub filename_to_volume_id {
 730     my ($vmid, $file, $media) = @_;
 731
 732     if (!($file eq 'none' || $file eq 'cdrom' ||
 733           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
 734
 735         return undef if $file =~ m|/|;
 736
 737         if ($media && $media eq 'cdrom') {
 738             $file = "local:iso/$file";
 739         } else {
 740             $file = "local:$vmid/$file";
 741         }
 742     }
 743
 744     return $file;
 745 }
 746
 747 sub verify_media_type {
 748     my ($opt, $vtype, $media) = @_;
 749
 750     return if !$media;
 751
 752     my $etype;
 753     if ($media eq 'disk') {
 754         $etype = 'images';
 755     } elsif ($media eq 'cdrom') {
 756         $etype = 'iso';
 757     } else {
 758         die "internal error";
 759     }
 760
 761     return if ($vtype eq $etype);
 762
 763     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
 764 }
 765
 766 sub cleanup_drive_path {
 767     my ($opt, $storecfg, $drive) = @_;
 768
 769     # try to convert filesystem paths to volume IDs
 770
 771     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
 772         ($drive->{file} !~ m|^/dev/.+|) &&
 773         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
 774         ($drive->{file} !~ m/^\d+$/)) {
 775         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
 776         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
 777         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
 778         verify_media_type($opt, $vtype, $drive->{media});
 779         $drive->{file} = $volid;
 780     }
 781
 782     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
 783 }
 784
 785 sub create_conf_nolock {
 786     my ($vmid, $settings) = @_;
 787
 788     my $filename = config_file($vmid);
 789
 790     die "configuration file '$filename' already exists\n" if -f $filename;
 791
 792     my $defaults = load_defaults();
 793
 794     $settings->{name} = "vm$vmid" if !$settings->{name};
 795     $settings->{memory} = $defaults->{memory} if !$settings->{memory};
 796
 797     my $data = '';
 798     foreach my $opt (keys %$settings) {
 799         next if !$confdesc->{$opt};
 800
 801         my $value = $settings->{$opt};
 802         next if !$value;
 803
 804         $data .= "$opt: $value\n";
 805     }
 806
 807     PVE::Tools::file_set_contents($filename, $data);
 808 }
 809
 810 sub parse_hotplug_features {
 811     my ($data) = @_;
 812
 813     my $res = {};
 814
 815     return $res if $data eq '0';
 816
 817     $data = $confdesc->{hotplug}->{default} if $data eq '1';
 818
 819     foreach my $feature (PVE::Tools::split_list($data)) {
 820         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
 821             $res->{$1} = 1;
 822         } else {
 823             warn "ignoring unknown hotplug feature '$feature'\n";
 824         }
 825     }
 826     return $res;
 827 }
 828
 829 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
 830 sub pve_verify_hotplug_features {
 831     my ($value, $noerr) = @_;
 832
 833     return $value if parse_hotplug_features($value);
 834
 835     return undef if $noerr;
 836
 837     die "unable to parse hotplug option\n";
 838 }
 839
 840 my $parse_size = sub {
 841     my ($value) = @_;
 842
 843     return undef if $value !~ m/^(\d+(\.\d+)?)([KMG])?$/;
 844     my ($size, $unit) = ($1, $3);
 845     if ($unit) {
 846         if ($unit eq 'K') {
 847             $size = $size * 1024;
 848         } elsif ($unit eq 'M') {
 849             $size = $size * 1024 * 1024;
 850         } elsif ($unit eq 'G') {
 851             $size = $size * 1024 * 1024 * 1024;
 852         }
 853     }
 854     return int($size);
 855 };
 856
 857 my $format_size = sub {
 858     my ($size) = @_;
 859
 860     $size = int($size);
 861
 862     my $kb = int($size/1024);
 863     return $size if $kb*1024 != $size;
 864
 865     my $mb = int($kb/1024);
 866     return "${kb}K" if $mb*1024 != $kb;
 867
 868     my $gb = int($mb/1024);
 869     return "${mb}M" if $gb*1024 != $mb;
 870
 871     return "${gb}G";
 872 };
 873
 874 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
 875 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
 876 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
 877 #        [,aio=native|threads][,discard=ignore|on][,iothread=on]
 878
 879 sub parse_drive {
 880     my ($key, $data) = @_;
 881
 882     my $res = {};
 883
 884     # $key may be undefined - used to verify JSON parameters
 885     if (!defined($key)) {
 886         $res->{interface} = 'unknown'; # should not harm when used to verify parameters
 887         $res->{index} = 0;
 888     } elsif ($key =~ m/^([^\d]+)(\d+)$/) {
 889         $res->{interface} = $1;
 890         $res->{index} = $2;
 891     } else {
 892         return undef;
 893     }
 894
 895     foreach my $p (split (/,/, $data)) {
 896         next if $p =~ m/^\s*$/;
 897
 898         if ($p =~ m/^(file|volume|cyls|heads|secs|trans|media|snapshot|cache|format|rerror|werror|backup|aio|bps|mbps|mbps_max|bps_rd|mbps_rd|mbps_rd_max|bps_wr|mbps_wr|mbps_wr_max|iops|iops_max|iops_rd|iops_rd_max|iops_wr|iops_wr_max|size|discard|iothread|queues)=(.+)$/) {
 899             my ($k, $v) = ($1, $2);
 900
 901             $k = 'file' if $k eq 'volume';
 902
 903             return undef if defined $res->{$k};
 904
 905             if ($k eq 'bps' || $k eq 'bps_rd' || $k eq 'bps_wr') {
 906                 return undef if !$v || $v !~ m/^\d+/;
 907                 $k = "m$k";
 908                 $v = sprintf("%.3f", $v / (1024*1024));
 909             }
 910             $res->{$k} = $v;
 911         } else {
 912             if (!$res->{file} && $p !~ m/=/) {
 913                 $res->{file} = $p;
 914             } else {
 915                 return undef;
 916             }
 917         }
 918     }
 919
 920     return undef if !$res->{file};
 921
 922     return undef if $res->{cache} &&
 923         $res->{cache} !~ m/^(off|none|writethrough|writeback|unsafe|directsync)$/;
 924     return undef if $res->{snapshot} && $res->{snapshot} !~ m/^(on|off)$/;
 925     return undef if $res->{cyls} && $res->{cyls} !~ m/^\d+$/;
 926     return undef if $res->{heads} && $res->{heads} !~ m/^\d+$/;
 927     return undef if $res->{secs} && $res->{secs} !~ m/^\d+$/;
 928     return undef if $res->{media} && $res->{media} !~ m/^(disk|cdrom)$/;
 929     return undef if $res->{trans} && $res->{trans} !~ m/^(none|lba|auto)$/;
 930     return undef if $res->{format} && $res->{format} !~ m/^(raw|cow|qcow|qed|qcow2|vmdk|cloop)$/;
 931     return undef if $res->{rerror} && $res->{rerror} !~ m/^(ignore|report|stop)$/;
 932     return undef if $res->{werror} && $res->{werror} !~ m/^(enospc|ignore|report|stop)$/;
 933     return undef if $res->{backup} && $res->{backup} !~ m/^(yes|no)$/;
 934     return undef if $res->{aio} && $res->{aio} !~ m/^(native|threads)$/;
 935     return undef if $res->{discard} && $res->{discard} !~ m/^(ignore|on)$/;
 936     return undef if $res->{iothread} && $res->{iothread} !~ m/^(on)$/;
 937     return undef if $res->{queues} && ($res->{queues} !~ m/^\d+$/ || $res->{queues} < 2);
 938
 939     return undef if $res->{mbps_rd} && $res->{mbps};
 940     return undef if $res->{mbps_wr} && $res->{mbps};
 941
 942     return undef if $res->{mbps} && $res->{mbps} !~ m/^\d+(\.\d+)?$/;
 943     return undef if $res->{mbps_max} && $res->{mbps_max} !~ m/^\d+(\.\d+)?$/;
 944     return undef if $res->{mbps_rd} && $res->{mbps_rd} !~ m/^\d+(\.\d+)?$/;
 945     return undef if $res->{mbps_rd_max} && $res->{mbps_rd_max} !~ m/^\d+(\.\d+)?$/;
 946     return undef if $res->{mbps_wr} && $res->{mbps_wr} !~ m/^\d+(\.\d+)?$/;
 947     return undef if $res->{mbps_wr_max} && $res->{mbps_wr_max} !~ m/^\d+(\.\d+)?$/;
 948
 949     return undef if $res->{iops_rd} && $res->{iops};
 950     return undef if $res->{iops_wr} && $res->{iops};
 951
 952
 953     return undef if $res->{iops} && $res->{iops} !~ m/^\d+$/;
 954     return undef if $res->{iops_max} && $res->{iops_max} !~ m/^\d+$/;
 955     return undef if $res->{iops_rd} && $res->{iops_rd} !~ m/^\d+$/;
 956     return undef if $res->{iops_rd_max} && $res->{iops_rd_max} !~ m/^\d+$/;
 957     return undef if $res->{iops_wr} && $res->{iops_wr} !~ m/^\d+$/;
 958     return undef if $res->{iops_wr_max} && $res->{iops_wr_max} !~ m/^\d+$/;
 959
 960
 961     if ($res->{size}) {
 962         return undef if !defined($res->{size} = &$parse_size($res->{size}));
 963     }
 964
 965     if ($res->{media} && ($res->{media} eq 'cdrom')) {
 966         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
 967         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
 968         return undef if $res->{interface} eq 'virtio';
 969     }
 970
 971     # rerror does not work with scsi drives
 972     if ($res->{rerror}) {
 973         return undef if $res->{interface} eq 'scsi';
 974     }
 975
 976     return $res;
 977 }
 978
 979 my @qemu_drive_options = qw(heads secs cyls trans media format cache snapshot rerror werror aio discard iops iops_rd iops_wr iops_max iops_rd_max iops_wr_max);
 980
 981 sub print_drive {
 982     my ($vmid, $drive) = @_;
 983
 984     my $opts = '';
 985     foreach my $o (@qemu_drive_options, 'mbps', 'mbps_rd', 'mbps_wr', 'mbps_max', 'mbps_rd_max', 'mbps_wr_max', 'backup', 'iothread', 'queues') {
 986         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
 987     }
 988
 989     if ($drive->{size}) {
 990         $opts .= ",size=" . &$format_size($drive->{size});
 991     }
 992
 993     return "$drive->{file}$opts";
 994 }
 995
 996 sub scsi_inquiry {
 997     my($fh, $noerr) = @_;
 998
 999     my $SG_IO = 0x2285;
1000     my $SG_GET_VERSION_NUM = 0x2282;
1001
1002     my $versionbuf = "\x00" x 8;
1003     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1004     if (!$ret) {
1005         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1006         return undef;
1007     }
1008     my $version = unpack("I", $versionbuf);
1009     if ($version < 30000) {
1010         die "scsi generic interface too old\n"  if !$noerr;
1011         return undef;
1012     }
1013
1014     my $buf = "\x00" x 36;
1015     my $sensebuf = "\x00" x 8;
1016     my $cmd = pack("C x3 C x1", 0x12, 36);
1017
1018     # see /usr/include/scsi/sg.h
1019     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1020
1021     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1022                       length($sensebuf), 0, length($buf), $buf,
1023                       $cmd, $sensebuf, 6000);
1024
1025     $ret = ioctl($fh, $SG_IO, $packet);
1026     if (!$ret) {
1027         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1028         return undef;
1029     }
1030
1031     my @res = unpack($sg_io_hdr_t, $packet);
1032     if ($res[17] || $res[18]) {
1033         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1034         return undef;
1035     }
1036
1037     my $res = {};
1038     (my $byte0, my $byte1, $res->{vendor},
1039      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1040
1041     $res->{removable} = $byte1 & 128 ? 1 : 0;
1042     $res->{type} = $byte0 & 31;
1043
1044     return $res;
1045 }
1046
1047 sub path_is_scsi {
1048     my ($path) = @_;
1049
1050     my $fh = IO::File->new("+<$path") || return undef;
1051     my $res = scsi_inquiry($fh, 1);
1052     close($fh);
1053
1054     return $res;
1055 }
1056
1057 sub machine_type_is_q35 {
1058     my ($conf) = @_;
1059
1060     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1061 }
1062
1063 sub print_tabletdevice_full {
1064     my ($conf) = @_;
1065
1066     my $q35 = machine_type_is_q35($conf);
1067
1068     # we use uhci for old VMs because tablet driver was buggy in older qemu
1069     my $usbbus = $q35 ? "ehci" : "uhci";
1070
1071     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1072 }
1073
1074 sub print_drivedevice_full {
1075     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1076
1077     my $device = '';
1078     my $maxdev = 0;
1079
1080     if ($drive->{interface} eq 'virtio') {
1081         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1082         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1083         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1084     } elsif ($drive->{interface} eq 'scsi') {
1085
1086         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1087         my $unit = $drive->{index} % $maxdev;
1088         my $devicetype = 'hd';
1089         my $path = '';
1090         if (drive_is_cdrom($drive)) {
1091               $devicetype = 'cd';
1092         } else {
1093               if ($drive->{file} =~ m|^/|) {
1094                   $path = $drive->{file};
1095               } else {
1096                   $path = PVE::Storage::path($storecfg, $drive->{file});
1097               }
1098
1099               if($path =~ m/^iscsi\:\/\//){
1100                   $devicetype = 'generic';
1101               } else {
1102                   if (my $info = path_is_scsi($path)) {
1103                       if ($info->{type} == 0) {
1104                           $devicetype = 'block';
1105                       } elsif ($info->{type} == 1) { # tape
1106                           $devicetype = 'generic';
1107                       }
1108                   }
1109               }
1110          }
1111
1112         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1113             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1114         } else {
1115             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1116         }
1117
1118     } elsif ($drive->{interface} eq 'ide'){
1119         $maxdev = 2;
1120         my $controller = int($drive->{index} / $maxdev);
1121         my $unit = $drive->{index} % $maxdev;
1122         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1123
1124         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1125     } elsif ($drive->{interface} eq 'sata'){
1126         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1127         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1128         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1129     } elsif ($drive->{interface} eq 'usb') {
1130         die "implement me";
1131         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1132     } else {
1133         die "unsupported interface type";
1134     }
1135
1136     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1137
1138     return $device;
1139 }
1140
1141 sub get_initiator_name {
1142     my $initiator;
1143
1144     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1145     while (defined(my $line = <$fh>)) {
1146         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1147         $initiator = $1;
1148         last;
1149     }
1150     $fh->close();
1151
1152     return $initiator;
1153 }
1154
1155 sub print_drive_full {
1156     my ($storecfg, $vmid, $drive) = @_;
1157
1158     my $path;
1159     my $volid = $drive->{file};
1160     my $format;
1161
1162     if (drive_is_cdrom($drive)) {
1163         $path = get_iso_path($storecfg, $vmid, $volid);
1164     } else {
1165         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1166         if ($storeid) {
1167             $path = PVE::Storage::path($storecfg, $volid);
1168             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1169             $format = qemu_img_format($scfg, $volname);
1170         } else {
1171             $path = $volid;
1172         }
1173    }
1174
1175     my $opts = '';
1176     foreach my $o (@qemu_drive_options) {
1177         next if $o eq 'bootindex';
1178         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1179     }
1180
1181     $opts .= ",format=$format" if $format && !$drive->{format};
1182
1183     foreach my $o (qw(bps bps_rd bps_wr)) {
1184         my $v = $drive->{"m$o"};
1185         $opts .= ",$o=" . int($v*1024*1024) if $v;
1186     }
1187
1188     my $cache_direct = 0;
1189
1190     if (my $cache = $drive->{cache}) {
1191         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1192     } elsif (!drive_is_cdrom($drive)) {
1193         $opts .= ",cache=none";
1194         $cache_direct = 1;
1195     }
1196
1197     # aio native works only with O_DIRECT
1198     if (!$drive->{aio}) {
1199         if($cache_direct) {
1200             $opts .= ",aio=native";
1201         } else {
1202             $opts .= ",aio=threads";
1203         }
1204     }
1205
1206     my $detectzeroes = $drive->{discard} ? "unmap" : "on";
1207     $opts .= ",detect-zeroes=$detectzeroes" if !drive_is_cdrom($drive);
1208
1209     my $pathinfo = $path ? "file=$path," : '';
1210
1211     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1212 }
1213
1214 sub print_netdevice_full {
1215     my ($vmid, $conf, $net, $netid, $bridges) = @_;
1216
1217     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1218
1219     my $device = $net->{model};
1220     if ($net->{model} eq 'virtio') {
1221          $device = 'virtio-net-pci';
1222      };
1223
1224     my $pciaddr = print_pci_addr("$netid", $bridges);
1225     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1226     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1227         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1228         my $vectors = $net->{queues} * 2 + 2;
1229         $tmpstr .= ",vectors=$vectors,mq=on";
1230     }
1231     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1232     return $tmpstr;
1233 }
1234
1235 sub print_netdev_full {
1236     my ($vmid, $conf, $net, $netid) = @_;
1237
1238     my $i = '';
1239     if ($netid =~ m/^net(\d+)$/) {
1240         $i = int($1);
1241     }
1242
1243     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1244
1245     my $ifname = "tap${vmid}i$i";
1246
1247     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1248     die "interface name '$ifname' is too long (max 15 character)\n"
1249         if length($ifname) >= 16;
1250
1251     my $vhostparam = '';
1252     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1253
1254     my $vmname = $conf->{name} || "vm$vmid";
1255
1256     my $netdev = "";
1257
1258     if ($net->{bridge}) {
1259         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/pve-bridge,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1260     } else {
1261         $netdev = "type=user,id=$netid,hostname=$vmname";
1262     }
1263
1264     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1265
1266     return $netdev;
1267 }
1268
1269 sub drive_is_cdrom {
1270     my ($drive) = @_;
1271
1272     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1273
1274 }
1275
1276 sub parse_numa {
1277     my ($data) = @_;
1278
1279     my $res = {};
1280
1281     foreach my $kvp (split(/,/, $data)) {
1282
1283         if ($kvp =~ m/^memory=(\S+)$/) {
1284             $res->{memory} = $1;
1285         } elsif ($kvp =~ m/^policy=(preferred|bind|interleave)$/) {
1286             $res->{policy} = $1;
1287         } elsif ($kvp =~ m/^cpus=(\d+)(-(\d+))?$/) {
1288             $res->{cpus}->{start} = $1;
1289             $res->{cpus}->{end} = $3;
1290         } elsif ($kvp =~ m/^hostnodes=(\d+)(-(\d+))?$/) {
1291             $res->{hostnodes}->{start} = $1;
1292             $res->{hostnodes}->{end} = $3;
1293         } else {
1294             return undef;
1295         }
1296     }
1297
1298     return $res;
1299 }
1300
1301 sub parse_hostpci {
1302     my ($value) = @_;
1303
1304     return undef if !$value;
1305
1306
1307     my @list = split(/,/, $value);
1308     my $found;
1309
1310     my $res = {};
1311     foreach my $kv (@list) {
1312
1313         if ($kv =~ m/^(host=)?([a-f0-9]{2}:[a-f0-9]{2})(\.([a-f0-9]))?$/) {
1314             $found = 1;
1315             if(defined($4)){
1316                 push @{$res->{pciid}}, { id => $2 , function => $4};
1317
1318             }else{
1319                 my $pcidevices = lspci($2);
1320                 $res->{pciid} = $pcidevices->{$2};
1321             }
1322         } elsif ($kv =~ m/^rombar=(on|off)$/) {
1323             $res->{rombar} = $1;
1324         } elsif ($kv =~ m/^x-vga=(on|off)$/) {
1325             $res->{'x-vga'} = $1;
1326         } elsif ($kv =~ m/^pcie=(\d+)$/) {
1327             $res->{pcie} = 1 if $1 == 1;
1328         } else {
1329             warn "unknown hostpci setting '$kv'\n";
1330         }
1331     }
1332
1333     return undef if !$found;
1334
1335     return $res;
1336 }
1337
1338 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1339 sub parse_net {
1340     my ($data) = @_;
1341
1342     my $res = {};
1343
1344     foreach my $kvp (split(/,/, $data)) {
1345
1346         if ($kvp =~ m/^(ne2k_pci|e1000|e1000-82540em|e1000-82544gc|e1000-82545em|rtl8139|pcnet|virtio|ne2k_isa|i82551|i82557b|i82559er|vmxnet3)(=([0-9a-f]{2}(:[0-9a-f]{2}){5}))?$/i) {
1347             my $model = lc($1);
1348             my $mac = defined($3) ? uc($3) : PVE::Tools::random_ether_addr();
1349             $res->{model} = $model;
1350             $res->{macaddr} = $mac;
1351         } elsif ($kvp =~ m/^bridge=(\S+)$/) {
1352             $res->{bridge} = $1;
1353         } elsif ($kvp =~ m/^queues=(\d+)$/) {
1354             $res->{queues} = $1;
1355         } elsif ($kvp =~ m/^rate=(\d+(\.\d+)?)$/) {
1356             $res->{rate} = $1;
1357         } elsif ($kvp =~ m/^tag=(\d+)$/) {
1358             $res->{tag} = $1;
1359         } elsif ($kvp =~ m/^firewall=([01])$/) {
1360             $res->{firewall} = $1;
1361         } elsif ($kvp =~ m/^link_down=([01])$/) {
1362             $res->{link_down} = $1;
1363         } else {
1364             return undef;
1365         }
1366
1367     }
1368
1369     return undef if !$res->{model};
1370
1371     return $res;
1372 }
1373
1374 sub print_net {
1375     my $net = shift;
1376
1377     my $res = "$net->{model}";
1378     $res .= "=$net->{macaddr}" if $net->{macaddr};
1379     $res .= ",bridge=$net->{bridge}" if $net->{bridge};
1380     $res .= ",rate=$net->{rate}" if $net->{rate};
1381     $res .= ",tag=$net->{tag}" if $net->{tag};
1382     $res .= ",firewall=1" if $net->{firewall};
1383     $res .= ",link_down=1" if $net->{link_down};
1384     $res .= ",queues=$net->{queues}" if $net->{queues};
1385
1386     return $res;
1387 }
1388
1389 sub add_random_macs {
1390     my ($settings) = @_;
1391
1392     foreach my $opt (keys %$settings) {
1393         next if $opt !~ m/^net(\d+)$/;
1394         my $net = parse_net($settings->{$opt});
1395         next if !$net;
1396         $settings->{$opt} = print_net($net);
1397     }
1398 }
1399
1400 sub add_unused_volume {
1401     my ($config, $volid) = @_;
1402
1403     my $key;
1404     for (my $ind = $MAX_UNUSED_DISKS - 1; $ind >= 0; $ind--) {
1405         my $test = "unused$ind";
1406         if (my $vid = $config->{$test}) {
1407             return if $vid eq $volid; # do not add duplicates
1408         } else {
1409             $key = $test;
1410         }
1411     }
1412
1413     die "To many unused volume - please delete them first.\n" if !$key;
1414
1415     $config->{$key} = $volid;
1416
1417     return $key;
1418 }
1419
1420 sub vm_is_volid_owner {
1421     my ($storecfg, $vmid, $volid) = @_;
1422
1423     if ($volid !~  m|^/|) {
1424         my ($path, $owner);
1425         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1426         if ($owner && ($owner == $vmid)) {
1427             return 1;
1428         }
1429     }
1430
1431     return undef;
1432 }
1433
1434 sub split_flagged_list {
1435     my $text = shift || '';
1436     $text =~ s/[,;]/ /g;
1437     $text =~ s/^\s+//;
1438     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
1439 }
1440
1441 sub join_flagged_list {
1442     my ($how, $lst) = @_;
1443     join $how, map { $lst->{$_} . $_ } keys %$lst;
1444 }
1445
1446 sub vmconfig_delete_pending_option {
1447     my ($conf, $key, $force) = @_;
1448
1449     delete $conf->{pending}->{$key};
1450     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1451     $pending_delete_hash->{$key} = $force ? '!' : '';
1452     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1453 }
1454
1455 sub vmconfig_undelete_pending_option {
1456     my ($conf, $key) = @_;
1457
1458     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1459     delete $pending_delete_hash->{$key};
1460
1461     if (%$pending_delete_hash) {
1462         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1463     } else {
1464         delete $conf->{pending}->{delete};
1465     }
1466 }
1467
1468 sub vmconfig_register_unused_drive {
1469     my ($storecfg, $vmid, $conf, $drive) = @_;
1470
1471     if (!drive_is_cdrom($drive)) {
1472         my $volid = $drive->{file};
1473         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1474             add_unused_volume($conf, $volid, $vmid);
1475         }
1476     }
1477 }
1478
1479 sub vmconfig_cleanup_pending {
1480     my ($conf) = @_;
1481
1482     # remove pending changes when nothing changed
1483     my $changes;
1484     foreach my $opt (keys %{$conf->{pending}}) {
1485         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
1486             $changes = 1;
1487             delete $conf->{pending}->{$opt};
1488         }
1489     }
1490
1491     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
1492     my $pending_delete_hash = {};
1493     while (my ($opt, $force) = each %$current_delete_hash) {
1494         if (defined($conf->{$opt})) {
1495             $pending_delete_hash->{$opt} = $force;
1496         } else {
1497             $changes = 1;
1498         }
1499     }
1500
1501     if (%$pending_delete_hash) {
1502         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1503     } else {
1504         delete $conf->{pending}->{delete};
1505     }
1506
1507     return $changes;
1508 }
1509
1510 my $valid_smbios1_options = {
1511     manufacturer => '\S+',
1512     product => '\S+',
1513     version => '\S+',
1514     serial => '\S+',
1515     uuid => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1516     sku => '\S+',
1517     family => '\S+',
1518 };
1519
1520 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
1521 sub parse_smbios1 {
1522     my ($data) = @_;
1523
1524     my $res = {};
1525
1526     foreach my $kvp (split(/,/, $data)) {
1527         return undef if $kvp !~ m/^(\S+)=(.+)$/;
1528         my ($k, $v) = split(/=/, $kvp);
1529         return undef if !defined($k) || !defined($v);
1530         return undef if !$valid_smbios1_options->{$k};
1531         return undef if $v !~ m/^$valid_smbios1_options->{$k}$/;
1532         $res->{$k} = $v;
1533     }
1534
1535     return $res;
1536 }
1537
1538 sub print_smbios1 {
1539     my ($smbios1) = @_;
1540
1541     my $data = '';
1542     foreach my $k (keys %$smbios1) {
1543         next if !defined($smbios1->{$k});
1544         next if !$valid_smbios1_options->{$k};
1545         $data .= ',' if $data;
1546         $data .= "$k=$smbios1->{$k}";
1547     }
1548     return $data;
1549 }
1550
1551 PVE::JSONSchema::register_format('pve-qm-smbios1', \&verify_smbios1);
1552 sub verify_smbios1 {
1553     my ($value, $noerr) = @_;
1554
1555     return $value if parse_smbios1($value);
1556
1557     return undef if $noerr;
1558
1559     die "unable to parse smbios (type 1) options\n";
1560 }
1561
1562 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
1563 sub verify_bootdisk {
1564     my ($value, $noerr) = @_;
1565
1566     return $value if valid_drivename($value);
1567
1568     return undef if $noerr;
1569
1570     die "invalid boot disk '$value'\n";
1571 }
1572
1573 PVE::JSONSchema::register_format('pve-qm-numanode', \&verify_numa);
1574 sub verify_numa {
1575     my ($value, $noerr) = @_;
1576
1577     return $value if parse_numa($value);
1578
1579     return undef if $noerr;
1580
1581     die "unable to parse numa options\n";
1582 }
1583
1584 PVE::JSONSchema::register_format('pve-qm-net', \&verify_net);
1585 sub verify_net {
1586     my ($value, $noerr) = @_;
1587
1588     return $value if parse_net($value);
1589
1590     return undef if $noerr;
1591
1592     die "unable to parse network options\n";
1593 }
1594
1595 PVE::JSONSchema::register_format('pve-qm-drive', \&verify_drive);
1596 sub verify_drive {
1597     my ($value, $noerr) = @_;
1598
1599     return $value if parse_drive(undef, $value);
1600
1601     return undef if $noerr;
1602
1603     die "unable to parse drive options\n";
1604 }
1605
1606 PVE::JSONSchema::register_format('pve-qm-hostpci', \&verify_hostpci);
1607 sub verify_hostpci {
1608     my ($value, $noerr) = @_;
1609
1610     return $value if parse_hostpci($value);
1611
1612     return undef if $noerr;
1613
1614     die "unable to parse pci id\n";
1615 }
1616
1617 PVE::JSONSchema::register_format('pve-qm-watchdog', \&verify_watchdog);
1618 sub verify_watchdog {
1619     my ($value, $noerr) = @_;
1620
1621     return $value if parse_watchdog($value);
1622
1623     return undef if $noerr;
1624
1625     die "unable to parse watchdog options\n";
1626 }
1627
1628 sub parse_watchdog {
1629     my ($value) = @_;
1630
1631     return undef if !$value;
1632
1633     my $res = {};
1634
1635     foreach my $p (split(/,/, $value)) {
1636         next if $p =~ m/^\s*$/;
1637
1638         if ($p =~ m/^(model=)?(i6300esb|ib700)$/) {
1639             $res->{model} = $2;
1640         } elsif ($p =~ m/^(action=)?(reset|shutdown|poweroff|pause|debug|none)$/) {
1641             $res->{action} = $2;
1642         } else {
1643             return undef;
1644         }
1645     }
1646
1647     return $res;
1648 }
1649
1650 sub parse_usb_device {
1651     my ($value) = @_;
1652
1653     return undef if !$value;
1654
1655     my @dl = split(/,/, $value);
1656     my $found;
1657
1658     my $res = {};
1659     foreach my $v (@dl) {
1660         if ($v =~ m/^host=(0x)?([0-9A-Fa-f]{4}):(0x)?([0-9A-Fa-f]{4})$/) {
1661             $found = 1;
1662             $res->{vendorid} = $2;
1663             $res->{productid} = $4;
1664         } elsif ($v =~ m/^host=(\d+)\-(\d+(\.\d+)*)$/) {
1665             $found = 1;
1666             $res->{hostbus} = $1;
1667             $res->{hostport} = $2;
1668         } elsif ($v =~ m/^spice$/) {
1669             $found = 1;
1670             $res->{spice} = 1;
1671         } else {
1672             return undef;
1673         }
1674     }
1675     return undef if !$found;
1676
1677     return $res;
1678 }
1679
1680 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
1681 sub verify_usb_device {
1682     my ($value, $noerr) = @_;
1683
1684     return $value if parse_usb_device($value);
1685
1686     return undef if $noerr;
1687
1688     die "unable to parse usb device\n";
1689 }
1690
1691 # add JSON properties for create and set function
1692 sub json_config_properties {
1693     my $prop = shift;
1694
1695     foreach my $opt (keys %$confdesc) {
1696         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
1697         $prop->{$opt} = $confdesc->{$opt};
1698     }
1699
1700     return $prop;
1701 }
1702
1703 sub check_type {
1704     my ($key, $value) = @_;
1705
1706     die "unknown setting '$key'\n" if !$confdesc->{$key};
1707
1708     my $type = $confdesc->{$key}->{type};
1709
1710     if (!defined($value)) {
1711         die "got undefined value\n";
1712     }
1713
1714     if ($value =~ m/[\n\r]/) {
1715         die "property contains a line feed\n";
1716     }
1717
1718     if ($type eq 'boolean') {
1719         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
1720         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
1721         die "type check ('boolean') failed - got '$value'\n";
1722     } elsif ($type eq 'integer') {
1723         return int($1) if $value =~ m/^(\d+)$/;
1724         die "type check ('integer') failed - got '$value'\n";
1725     } elsif ($type eq 'number') {
1726         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
1727         die "type check ('number') failed - got '$value'\n";
1728     } elsif ($type eq 'string') {
1729         if (my $fmt = $confdesc->{$key}->{format}) {
1730             if ($fmt eq 'pve-qm-drive') {
1731                 # special case - we need to pass $key to parse_drive()
1732                 my $drive = parse_drive($key, $value);
1733                 return $value if $drive;
1734                 die "unable to parse drive options\n";
1735             }
1736             PVE::JSONSchema::check_format($fmt, $value);
1737             return $value;
1738         }
1739         $value =~ s/^\"(.*)\"$/$1/;
1740         return $value;
1741     } else {
1742         die "internal error"
1743     }
1744 }
1745
1746 sub lock_config_full {
1747     my ($vmid, $timeout, $code, @param) = @_;
1748
1749     my $filename = config_file_lock($vmid);
1750
1751     my $res = lock_file($filename, $timeout, $code, @param);
1752
1753     die $@ if $@;
1754
1755     return $res;
1756 }
1757
1758 sub lock_config_mode {
1759     my ($vmid, $timeout, $shared, $code, @param) = @_;
1760
1761     my $filename = config_file_lock($vmid);
1762
1763     my $res = lock_file_full($filename, $timeout, $shared, $code, @param);
1764
1765     die $@ if $@;
1766
1767     return $res;
1768 }
1769
1770 sub lock_config {
1771     my ($vmid, $code, @param) = @_;
1772
1773     return lock_config_full($vmid, 10, $code, @param);
1774 }
1775
1776 sub cfs_config_path {
1777     my ($vmid, $node) = @_;
1778
1779     $node = $nodename if !$node;
1780     return "nodes/$node/qemu-server/$vmid.conf";
1781 }
1782
1783 sub check_iommu_support{
1784     #fixme : need to check IOMMU support
1785     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
1786
1787     my $iommu=1;
1788     return $iommu;
1789
1790 }
1791
1792 sub config_file {
1793     my ($vmid, $node) = @_;
1794
1795     my $cfspath = cfs_config_path($vmid, $node);
1796     return "/etc/pve/$cfspath";
1797 }
1798
1799 sub config_file_lock {
1800     my ($vmid) = @_;
1801
1802     return "$lock_dir/lock-$vmid.conf";
1803 }
1804
1805 sub touch_config {
1806     my ($vmid) = @_;
1807
1808     my $conf = config_file($vmid);
1809     utime undef, undef, $conf;
1810 }
1811
1812 sub destroy_vm {
1813     my ($storecfg, $vmid, $keep_empty_config) = @_;
1814
1815     my $conffile = config_file($vmid);
1816
1817     my $conf = load_config($vmid);
1818
1819     check_lock($conf);
1820
1821     # only remove disks owned by this VM
1822     foreach_drive($conf, sub {
1823         my ($ds, $drive) = @_;
1824
1825         return if drive_is_cdrom($drive);
1826
1827         my $volid = $drive->{file};
1828
1829         return if !$volid || $volid =~ m|^/|;
1830
1831         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
1832         return if !$path || !$owner || ($owner != $vmid);
1833
1834         PVE::Storage::vdisk_free($storecfg, $volid);
1835     });
1836
1837     if ($keep_empty_config) {
1838         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
1839     } else {
1840         unlink $conffile;
1841     }
1842
1843     # also remove unused disk
1844     eval {
1845         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
1846
1847         eval {
1848             PVE::Storage::foreach_volid($dl, sub {
1849                 my ($volid, $sid, $volname, $d) = @_;
1850                 PVE::Storage::vdisk_free($storecfg, $volid);
1851             });
1852         };
1853         warn $@ if $@;
1854
1855     };
1856     warn $@ if $@;
1857 }
1858
1859 sub load_config {
1860     my ($vmid, $node) = @_;
1861
1862     my $cfspath = cfs_config_path($vmid, $node);
1863
1864     my $conf = PVE::Cluster::cfs_read_file($cfspath);
1865
1866     die "no such VM ('$vmid')\n" if !defined($conf);
1867
1868     return $conf;
1869 }
1870
1871 sub parse_vm_config {
1872     my ($filename, $raw) = @_;
1873
1874     return undef if !defined($raw);
1875
1876     my $res = {
1877         digest => Digest::SHA::sha1_hex($raw),
1878         snapshots => {},
1879         pending => {},
1880     };
1881
1882     $filename =~ m|/qemu-server/(\d+)\.conf$|
1883         || die "got strange filename '$filename'";
1884
1885     my $vmid = $1;
1886
1887     my $conf = $res;
1888     my $descr;
1889     my $section = '';
1890
1891     my @lines = split(/\n/, $raw);
1892     foreach my $line (@lines) {
1893         next if $line =~ m/^\s*$/;
1894
1895         if ($line =~ m/^\[PENDING\]\s*$/i) {
1896             $section = 'pending';
1897             if (defined($descr)) {
1898                 $descr =~ s/\s+$//;
1899                 $conf->{description} = $descr;
1900             }
1901             $descr = undef;
1902             $conf = $res->{$section} = {};
1903             next;
1904
1905         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
1906             $section = $1;
1907             if (defined($descr)) {
1908                 $descr =~ s/\s+$//;
1909                 $conf->{description} = $descr;
1910             }
1911             $descr = undef;
1912             $conf = $res->{snapshots}->{$section} = {};
1913             next;
1914         }
1915
1916         if ($line =~ m/^\#(.*)\s*$/) {
1917             $descr = '' if !defined($descr);
1918             $descr .= PVE::Tools::decode_text($1) . "\n";
1919             next;
1920         }
1921
1922         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
1923             $descr = '' if !defined($descr);
1924             $descr .= PVE::Tools::decode_text($2);
1925         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
1926             $conf->{snapstate} = $1;
1927         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
1928             my $key = $1;
1929             my $value = $2;
1930             $conf->{$key} = $value;
1931         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
1932             my $value = $1;
1933             if ($section eq 'pending') {
1934                 $conf->{delete} = $value; # we parse this later
1935             } else {
1936                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
1937             }
1938         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
1939             my $key = $1;
1940             my $value = $2;
1941             eval { $value = check_type($key, $value); };
1942             if ($@) {
1943                 warn "vm $vmid - unable to parse value of '$key' - $@";
1944             } else {
1945                 my $fmt = $confdesc->{$key}->{format};
1946                 if ($fmt && $fmt eq 'pve-qm-drive') {
1947                     my $v = parse_drive($key, $value);
1948                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
1949                         $v->{file} = $volid;
1950                         $value = print_drive($vmid, $v);
1951                     } else {
1952                         warn "vm $vmid - unable to parse value of '$key'\n";
1953                         next;
1954                     }
1955                 }
1956
1957                 if ($key eq 'cdrom') {
1958                     $conf->{ide2} = $value;
1959                 } else {
1960                     $conf->{$key} = $value;
1961                 }
1962             }
1963         }
1964     }
1965
1966     if (defined($descr)) {
1967         $descr =~ s/\s+$//;
1968         $conf->{description} = $descr;
1969     }
1970     delete $res->{snapstate}; # just to be sure
1971
1972     return $res;
1973 }
1974
1975 sub write_vm_config {
1976     my ($filename, $conf) = @_;
1977
1978     delete $conf->{snapstate}; # just to be sure
1979
1980     if ($conf->{cdrom}) {
1981         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
1982         $conf->{ide2} = $conf->{cdrom};
1983         delete $conf->{cdrom};
1984     }
1985
1986     # we do not use 'smp' any longer
1987     if ($conf->{sockets}) {
1988         delete $conf->{smp};
1989     } elsif ($conf->{smp}) {
1990         $conf->{sockets} = $conf->{smp};
1991         delete $conf->{cores};
1992         delete $conf->{smp};
1993     }
1994
1995     my $used_volids = {};
1996
1997     my $cleanup_config = sub {
1998         my ($cref, $pending, $snapname) = @_;
1999
2000         foreach my $key (keys %$cref) {
2001             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2002                 $key eq 'snapstate' || $key eq 'pending';
2003             my $value = $cref->{$key};
2004             if ($key eq 'delete') {
2005                 die "propertry 'delete' is only allowed in [PENDING]\n"
2006                     if !$pending;
2007                 # fixme: check syntax?
2008                 next;
2009             }
2010             eval { $value = check_type($key, $value); };
2011             die "unable to parse value of '$key' - $@" if $@;
2012
2013             $cref->{$key} = $value;
2014
2015             if (!$snapname && valid_drivename($key)) {
2016                 my $drive = parse_drive($key, $value);
2017                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2018             }
2019         }
2020     };
2021
2022     &$cleanup_config($conf);
2023
2024     &$cleanup_config($conf->{pending}, 1);
2025
2026     foreach my $snapname (keys %{$conf->{snapshots}}) {
2027         die "internal error" if $snapname eq 'pending';
2028         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2029     }
2030
2031     # remove 'unusedX' settings if we re-add a volume
2032     foreach my $key (keys %$conf) {
2033         my $value = $conf->{$key};
2034         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2035             delete $conf->{$key};
2036         }
2037     }
2038
2039     my $generate_raw_config = sub {
2040         my ($conf, $pending) = @_;
2041
2042         my $raw = '';
2043
2044         # add description as comment to top of file
2045         if (defined(my $descr = $conf->{description})) {
2046             if ($descr) {
2047                 foreach my $cl (split(/\n/, $descr)) {
2048                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2049                 }
2050             } else {
2051                 $raw .= "#\n" if $pending;
2052             }
2053         }
2054
2055         foreach my $key (sort keys %$conf) {
2056             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2057             $raw .= "$key: $conf->{$key}\n";
2058         }
2059         return $raw;
2060     };
2061
2062     my $raw = &$generate_raw_config($conf);
2063
2064     if (scalar(keys %{$conf->{pending}})){
2065         $raw .= "\n[PENDING]\n";
2066         $raw .= &$generate_raw_config($conf->{pending}, 1);
2067     }
2068
2069     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2070         $raw .= "\n[$snapname]\n";
2071         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2072     }
2073
2074     return $raw;
2075 }
2076
2077 sub update_config_nolock {
2078     my ($vmid, $conf, $skiplock) = @_;
2079
2080     check_lock($conf) if !$skiplock;
2081
2082     my $cfspath = cfs_config_path($vmid);
2083
2084     PVE::Cluster::cfs_write_file($cfspath, $conf);
2085 }
2086
2087 sub update_config {
2088     my ($vmid, $conf, $skiplock) = @_;
2089
2090     lock_config($vmid, &update_config_nolock, $conf, $skiplock);
2091 }
2092
2093 sub load_defaults {
2094
2095     my $res = {};
2096
2097     # we use static defaults from our JSON schema configuration
2098     foreach my $key (keys %$confdesc) {
2099         if (defined(my $default = $confdesc->{$key}->{default})) {
2100             $res->{$key} = $default;
2101         }
2102     }
2103
2104     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2105     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2106
2107     return $res;
2108 }
2109
2110 sub config_list {
2111     my $vmlist = PVE::Cluster::get_vmlist();
2112     my $res = {};
2113     return $res if !$vmlist || !$vmlist->{ids};
2114     my $ids = $vmlist->{ids};
2115
2116     foreach my $vmid (keys %$ids) {
2117         my $d = $ids->{$vmid};
2118         next if !$d->{node} || $d->{node} ne $nodename;
2119         next if !$d->{type} || $d->{type} ne 'qemu';
2120         $res->{$vmid}->{exists} = 1;
2121     }
2122     return $res;
2123 }
2124
2125 # test if VM uses local resources (to prevent migration)
2126 sub check_local_resources {
2127     my ($conf, $noerr) = @_;
2128
2129     my $loc_res = 0;
2130
2131     $loc_res = 1 if $conf->{hostusb}; # old syntax
2132     $loc_res = 1 if $conf->{hostpci}; # old syntax
2133
2134     foreach my $k (keys %$conf) {
2135         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2136         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2137     }
2138
2139     die "VM uses local resources\n" if $loc_res && !$noerr;
2140
2141     return $loc_res;
2142 }
2143
2144 # check if used storages are available on all nodes (use by migrate)
2145 sub check_storage_availability {
2146     my ($storecfg, $conf, $node) = @_;
2147
2148     foreach_drive($conf, sub {
2149         my ($ds, $drive) = @_;
2150
2151         my $volid = $drive->{file};
2152         return if !$volid;
2153
2154         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2155         return if !$sid;
2156
2157         # check if storage is available on both nodes
2158         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2159         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2160    });
2161 }
2162
2163 # list nodes where all VM images are available (used by has_feature API)
2164 sub shared_nodes {
2165     my ($conf, $storecfg) = @_;
2166
2167     my $nodelist = PVE::Cluster::get_nodelist();
2168     my $nodehash = { map { $_ => 1 } @$nodelist };
2169     my $nodename = PVE::INotify::nodename();
2170
2171     foreach_drive($conf, sub {
2172         my ($ds, $drive) = @_;
2173
2174         my $volid = $drive->{file};
2175         return if !$volid;
2176
2177         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2178         if ($storeid) {
2179             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2180             if ($scfg->{disable}) {
2181                 $nodehash = {};
2182             } elsif (my $avail = $scfg->{nodes}) {
2183                 foreach my $node (keys %$nodehash) {
2184                     delete $nodehash->{$node} if !$avail->{$node};
2185                 }
2186             } elsif (!$scfg->{shared}) {
2187                 foreach my $node (keys %$nodehash) {
2188                     delete $nodehash->{$node} if $node ne $nodename
2189                 }
2190             }
2191         }
2192     });
2193
2194     return $nodehash
2195 }
2196
2197 sub check_lock {
2198     my ($conf) = @_;
2199
2200     die "VM is locked ($conf->{lock})\n" if $conf->{lock};
2201 }
2202
2203 sub check_cmdline {
2204     my ($pidfile, $pid) = @_;
2205
2206     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2207     if (defined($fh)) {
2208         my $line = <$fh>;
2209         $fh->close;
2210         return undef if !$line;
2211         my @param = split(/\0/, $line);
2212
2213         my $cmd = $param[0];
2214         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2215
2216         for (my $i = 0; $i < scalar (@param); $i++) {
2217             my $p = $param[$i];
2218             next if !$p;
2219             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2220                 my $p = $param[$i+1];
2221                 return 1 if $p && ($p eq $pidfile);
2222                 return undef;
2223             }
2224         }
2225     }
2226     return undef;
2227 }
2228
2229 sub check_running {
2230     my ($vmid, $nocheck, $node) = @_;
2231
2232     my $filename = config_file($vmid, $node);
2233
2234     die "unable to find configuration file for VM $vmid - no such machine\n"
2235         if !$nocheck && ! -f $filename;
2236
2237     my $pidfile = pidfile_name($vmid);
2238
2239     if (my $fd = IO::File->new("<$pidfile")) {
2240         my $st = stat($fd);
2241         my $line = <$fd>;
2242         close($fd);
2243
2244         my $mtime = $st->mtime;
2245         if ($mtime > time()) {
2246             warn "file '$filename' modified in future\n";
2247         }
2248
2249         if ($line =~ m/^(\d+)$/) {
2250             my $pid = $1;
2251             if (check_cmdline($pidfile, $pid)) {
2252                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2253                     return $pid;
2254                 }
2255             }
2256         }
2257     }
2258
2259     return undef;
2260 }
2261
2262 sub vzlist {
2263
2264     my $vzlist = config_list();
2265
2266     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2267
2268     while (defined(my $de = $fd->read)) {
2269         next if $de !~ m/^(\d+)\.pid$/;
2270         my $vmid = $1;
2271         next if !defined($vzlist->{$vmid});
2272         if (my $pid = check_running($vmid)) {
2273             $vzlist->{$vmid}->{pid} = $pid;
2274         }
2275     }
2276
2277     return $vzlist;
2278 }
2279
2280 sub disksize {
2281     my ($storecfg, $conf) = @_;
2282
2283     my $bootdisk = $conf->{bootdisk};
2284     return undef if !$bootdisk;
2285     return undef if !valid_drivename($bootdisk);
2286
2287     return undef if !$conf->{$bootdisk};
2288
2289     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2290     return undef if !defined($drive);
2291
2292     return undef if drive_is_cdrom($drive);
2293
2294     my $volid = $drive->{file};
2295     return undef if !$volid;
2296
2297     return $drive->{size};
2298 }
2299
2300 my $last_proc_pid_stat;
2301
2302 # get VM status information
2303 # This must be fast and should not block ($full == false)
2304 # We only query KVM using QMP if $full == true (this can be slow)
2305 sub vmstatus {
2306     my ($opt_vmid, $full) = @_;
2307
2308     my $res = {};
2309
2310     my $storecfg = PVE::Storage::config();
2311
2312     my $list = vzlist();
2313     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2314
2315     my $cpucount = $cpuinfo->{cpus} || 1;
2316
2317     foreach my $vmid (keys %$list) {
2318         next if $opt_vmid && ($vmid ne $opt_vmid);
2319
2320         my $cfspath = cfs_config_path($vmid);
2321         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2322
2323         my $d = {};
2324         $d->{pid} = $list->{$vmid}->{pid};
2325
2326         # fixme: better status?
2327         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2328
2329         my $size = disksize($storecfg, $conf);
2330         if (defined($size)) {
2331             $d->{disk} = 0; # no info available
2332             $d->{maxdisk} = $size;
2333         } else {
2334             $d->{disk} = 0;
2335             $d->{maxdisk} = 0;
2336         }
2337
2338         $d->{cpus} = ($conf->{sockets} || 1) * ($conf->{cores} || 1);
2339         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2340         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2341
2342         $d->{name} = $conf->{name} || "VM $vmid";
2343         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024) : 0;
2344
2345         if ($conf->{balloon}) {
2346             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2347             $d->{shares} = defined($conf->{shares}) ? $conf->{shares} : 1000;
2348         }
2349
2350         $d->{uptime} = 0;
2351         $d->{cpu} = 0;
2352         $d->{mem} = 0;
2353
2354         $d->{netout} = 0;
2355         $d->{netin} = 0;
2356
2357         $d->{diskread} = 0;
2358         $d->{diskwrite} = 0;
2359
2360         $d->{template} = is_template($conf);
2361
2362         $res->{$vmid} = $d;
2363     }
2364
2365     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2366     foreach my $dev (keys %$netdev) {
2367         next if $dev !~ m/^tap([1-9]\d*)i/;
2368         my $vmid = $1;
2369         my $d = $res->{$vmid};
2370         next if !$d;
2371
2372         $d->{netout} += $netdev->{$dev}->{receive};
2373         $d->{netin} += $netdev->{$dev}->{transmit};
2374
2375         if ($full) {
2376             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2377             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2378         }
2379
2380     }
2381
2382     my $ctime = gettimeofday;
2383
2384     foreach my $vmid (keys %$list) {
2385
2386         my $d = $res->{$vmid};
2387         my $pid = $d->{pid};
2388         next if !$pid;
2389
2390         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2391         next if !$pstat; # not running
2392
2393         my $used = $pstat->{utime} + $pstat->{stime};
2394
2395         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2396
2397         if ($pstat->{vsize}) {
2398             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2399         }
2400
2401         my $old = $last_proc_pid_stat->{$pid};
2402         if (!$old) {
2403             $last_proc_pid_stat->{$pid} = {
2404                 time => $ctime,
2405                 used => $used,
2406                 cpu => 0,
2407             };
2408             next;
2409         }
2410
2411         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2412
2413         if ($dtime > 1000) {
2414             my $dutime = $used -  $old->{used};
2415
2416             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2417             $last_proc_pid_stat->{$pid} = {
2418                 time => $ctime,
2419                 used => $used,
2420                 cpu => $d->{cpu},
2421             };
2422         } else {
2423             $d->{cpu} = $old->{cpu};
2424         }
2425     }
2426
2427     return $res if !$full;
2428
2429     my $qmpclient = PVE::QMPClient->new();
2430
2431     my $ballooncb = sub {
2432         my ($vmid, $resp) = @_;
2433
2434         my $info = $resp->{'return'};
2435         return if !$info->{max_mem};
2436
2437         my $d = $res->{$vmid};
2438
2439         # use memory assigned to VM
2440         $d->{maxmem} = $info->{max_mem};
2441         $d->{balloon} = $info->{actual};
2442
2443         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2444             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2445             $d->{freemem} = $info->{free_mem};
2446         }
2447
2448         $d->{ballooninfo} = $info;
2449     };
2450
2451     my $blockstatscb = sub {
2452         my ($vmid, $resp) = @_;
2453         my $data = $resp->{'return'} || [];
2454         my $totalrdbytes = 0;
2455         my $totalwrbytes = 0;
2456
2457         for my $blockstat (@$data) {
2458             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2459             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2460
2461             $blockstat->{device} =~ s/drive-//;
2462             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2463         }
2464         $res->{$vmid}->{diskread} = $totalrdbytes;
2465         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2466     };
2467
2468     my $statuscb = sub {
2469         my ($vmid, $resp) = @_;
2470
2471         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2472         # this fails if ballon driver is not loaded, so this must be
2473         # the last commnand (following command are aborted if this fails).
2474         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2475
2476         my $status = 'unknown';
2477         if (!defined($status = $resp->{'return'}->{status})) {
2478             warn "unable to get VM status\n";
2479             return;
2480         }
2481
2482         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2483     };
2484
2485     foreach my $vmid (keys %$list) {
2486         next if $opt_vmid && ($vmid ne $opt_vmid);
2487         next if !$res->{$vmid}->{pid}; # not running
2488         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2489     }
2490
2491     $qmpclient->queue_execute(undef, 1);
2492
2493     foreach my $vmid (keys %$list) {
2494         next if $opt_vmid && ($vmid ne $opt_vmid);
2495         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2496     }
2497
2498     return $res;
2499 }
2500
2501 sub foreach_dimm {
2502     my ($conf, $vmid, $memory, $sockets, $func) = @_;
2503
2504     my $dimm_id = 0;
2505     my $current_size = 1024;
2506     my $dimm_size = 512;
2507     return if $current_size == $memory;
2508
2509     for (my $j = 0; $j < 8; $j++) {
2510         for (my $i = 0; $i < 32; $i++) {
2511             my $name = "dimm${dimm_id}";
2512             $dimm_id++;
2513             my $numanode = $i % $sockets;
2514             $current_size += $dimm_size;
2515             &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
2516             return  $current_size if $current_size >= $memory;
2517         }
2518         $dimm_size *= 2;
2519     }
2520 }
2521
2522 sub foreach_reverse_dimm {
2523     my ($conf, $vmid, $memory, $sockets, $func) = @_;
2524
2525     my $dimm_id = 253;
2526     my $current_size = 4177920;
2527     my $dimm_size = 65536;
2528     return if $current_size == $memory;
2529
2530     for (my $j = 0; $j < 8; $j++) {
2531         for (my $i = 0; $i < 32; $i++) {
2532             my $name = "dimm${dimm_id}";
2533             $dimm_id--;
2534             my $numanode = $i % $sockets;
2535             $current_size -= $dimm_size;
2536             &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
2537             return  $current_size if $current_size <= $memory;
2538         }
2539         $dimm_size /= 2;
2540     }
2541 }
2542
2543 sub foreach_drive {
2544     my ($conf, $func) = @_;
2545
2546     foreach my $ds (keys %$conf) {
2547         next if !valid_drivename($ds);
2548
2549         my $drive = parse_drive($ds, $conf->{$ds});
2550         next if !$drive;
2551
2552         &$func($ds, $drive);
2553     }
2554 }
2555
2556 sub foreach_volid {
2557     my ($conf, $func) = @_;
2558
2559     my $volhash = {};
2560
2561     my $test_volid = sub {
2562         my ($volid, $is_cdrom) = @_;
2563
2564         return if !$volid;
2565
2566         $volhash->{$volid} = $is_cdrom || 0;
2567     };
2568
2569     foreach_drive($conf, sub {
2570         my ($ds, $drive) = @_;
2571         &$test_volid($drive->{file}, drive_is_cdrom($drive));
2572     });
2573
2574     foreach my $snapname (keys %{$conf->{snapshots}}) {
2575         my $snap = $conf->{snapshots}->{$snapname};
2576         &$test_volid($snap->{vmstate}, 0);
2577         foreach_drive($snap, sub {
2578             my ($ds, $drive) = @_;
2579             &$test_volid($drive->{file}, drive_is_cdrom($drive));
2580         });
2581     }
2582
2583     foreach my $volid (keys %$volhash) {
2584         &$func($volid, $volhash->{$volid});
2585     }
2586 }
2587
2588 sub vga_conf_has_spice {
2589     my ($vga) = @_;
2590
2591     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
2592
2593     return $1 || 1;
2594 }
2595
2596 sub config_to_command {
2597     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2598
2599     my $cmd = [];
2600     my $globalFlags = [];
2601     my $machineFlags = [];
2602     my $rtcFlags = [];
2603     my $cpuFlags = [];
2604     my $devices = [];
2605     my $pciaddr = '';
2606     my $bridges = {};
2607     my $kvmver = kvm_user_version();
2608     my $vernum = 0; # unknown
2609     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
2610         $vernum = $1*1000000+$2*1000;
2611     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
2612         $vernum = $1*1000000+$2*1000+$3;
2613     }
2614
2615     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
2616
2617     my $have_ovz = -f '/proc/vz/vestat';
2618
2619     my $q35 = machine_type_is_q35($conf);
2620     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2621     my $machine_type = $forcemachine || $conf->{machine};
2622
2623     my $cpuunits = defined($conf->{cpuunits}) ?
2624             $conf->{cpuunits} : $defaults->{cpuunits};
2625
2626     push @$cmd, '/usr/bin/systemd-run';
2627     push @$cmd, '--scope';
2628     push @$cmd, '--slice', "qemu";
2629     push @$cmd, '--unit', $vmid;
2630     push @$cmd, '-p', "CPUShares=$cpuunits";
2631     if ($conf->{cpulimit}) {
2632         my $cpulimit = int($conf->{cpulimit} * 100);
2633         push @$cmd, '-p', "CPUQuota=$cpulimit\%";
2634     }
2635
2636     push @$cmd, '/usr/bin/kvm';
2637
2638     push @$cmd, '-id', $vmid;
2639
2640     my $use_virtio = 0;
2641
2642     my $qmpsocket = qmp_socket($vmid);
2643     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2644     push @$cmd, '-mon', "chardev=qmp,mode=control";
2645
2646     my $socket = vnc_socket($vmid);
2647     push @$cmd,  '-vnc', "unix:$socket,x509,password";
2648
2649     push @$cmd, '-pidfile' , pidfile_name($vmid);
2650
2651     push @$cmd, '-daemonize';
2652
2653     if ($conf->{smbios1}) {
2654         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
2655     }
2656
2657     if ($q35) {
2658         # the q35 chipset support native usb2, so we enable usb controller
2659         # by default for this machine type
2660         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
2661     } else {
2662         $pciaddr = print_pci_addr("piix3", $bridges);
2663         push @$devices, '-device', "piix3-usb-uhci,id=uhci$pciaddr.0x2";
2664
2665         my $use_usb2 = 0;
2666         for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2667             next if !$conf->{"usb$i"};
2668             $use_usb2 = 1;
2669         }
2670         # include usb device config
2671         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-usb.cfg' if $use_usb2;
2672     }
2673
2674     my $vga = $conf->{vga};
2675
2676     my $qxlnum = vga_conf_has_spice($vga);
2677     $vga = 'qxl' if $qxlnum;
2678
2679     if (!$vga) {
2680         if ($conf->{ostype} && ($conf->{ostype} eq 'win8' ||
2681                                 $conf->{ostype} eq 'win7' ||
2682                                 $conf->{ostype} eq 'w2k8')) {
2683             $vga = 'std';
2684         } else {
2685             $vga = 'cirrus';
2686         }
2687     }
2688
2689     # enable absolute mouse coordinates (needed by vnc)
2690     my $tablet;
2691     if (defined($conf->{tablet})) {
2692         $tablet = $conf->{tablet};
2693     } else {
2694         $tablet = $defaults->{tablet};
2695         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
2696         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
2697     }
2698
2699     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
2700
2701     # host pci devices
2702     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
2703         my $d = parse_hostpci($conf->{"hostpci$i"});
2704         next if !$d;
2705
2706         my $pcie = $d->{pcie};
2707         if($pcie){
2708             die "q35 machine model is not enabled" if !$q35;
2709             $pciaddr = print_pcie_addr("hostpci$i");
2710         }else{
2711             $pciaddr = print_pci_addr("hostpci$i", $bridges);
2712         }
2713
2714         my $rombar = $d->{rombar} && $d->{rombar} eq 'off' ? ",rombar=0" : "";
2715         my $xvga = $d->{'x-vga'} && $d->{'x-vga'} eq 'on' ? ",x-vga=on" : "";
2716         if ($xvga && $xvga ne '') {
2717             push @$cpuFlags, 'kvm=off';
2718             $vga = 'none';
2719         }
2720         my $pcidevices = $d->{pciid};
2721         my $multifunction = 1 if @$pcidevices > 1;
2722
2723         my $j=0;
2724         foreach my $pcidevice (@$pcidevices) {
2725
2726             my $id = "hostpci$i";
2727             $id .= ".$j" if $multifunction;
2728             my $addr = $pciaddr;
2729             $addr .= ".$j" if $multifunction;
2730             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
2731
2732             if($j == 0){
2733                 $devicestr .= "$rombar$xvga";
2734                 $devicestr .= ",multifunction=on" if $multifunction;
2735             }
2736
2737             push @$devices, '-device', $devicestr;
2738             $j++;
2739         }
2740     }
2741
2742     # usb devices
2743     for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2744         my $d = parse_usb_device($conf->{"usb$i"});
2745         next if !$d;
2746         if ($d->{vendorid} && $d->{productid}) {
2747             push @$devices, '-device', "usb-host,vendorid=0x$d->{vendorid},productid=0x$d->{productid}";
2748         } elsif (defined($d->{hostbus}) && defined($d->{hostport})) {
2749             push @$devices, '-device', "usb-host,hostbus=$d->{hostbus},hostport=$d->{hostport}";
2750         } elsif ($d->{spice}) {
2751             # usb redir support for spice
2752             push @$devices, '-chardev', "spicevmc,id=usbredirchardev$i,name=usbredir";
2753             push @$devices, '-device', "usb-redir,chardev=usbredirchardev$i,id=usbredirdev$i,bus=ehci.0";
2754         }
2755     }
2756
2757     # serial devices
2758     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2759         if (my $path = $conf->{"serial$i"}) {
2760             if ($path eq 'socket') {
2761                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
2762                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
2763                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2764             } else {
2765                 die "no such serial device\n" if ! -c $path;
2766                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
2767                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2768             }
2769         }
2770     }
2771
2772     # parallel devices
2773     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
2774         if (my $path = $conf->{"parallel$i"}) {
2775             die "no such parallel device\n" if ! -c $path;
2776             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
2777             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
2778             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
2779         }
2780     }
2781
2782     my $vmname = $conf->{name} || "vm$vmid";
2783
2784     push @$cmd, '-name', $vmname;
2785
2786     my $sockets = 1;
2787     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
2788     $sockets = $conf->{sockets} if  $conf->{sockets};
2789
2790     my $cores = $conf->{cores} || 1;
2791
2792     my $maxcpus = $sockets * $cores;
2793
2794     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
2795
2796     my $allowed_vcpus = $cpuinfo->{cpus};
2797
2798     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
2799         if ($allowed_vcpus < $maxcpus);
2800
2801     push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
2802
2803     push @$cmd, '-nodefaults';
2804
2805     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2806
2807     my $bootindex_hash = {};
2808     my $i = 1;
2809     foreach my $o (split(//, $bootorder)) {
2810         $bootindex_hash->{$o} = $i*100;
2811         $i++;
2812     }
2813
2814     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000";
2815
2816     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
2817
2818     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
2819
2820     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
2821
2822     # time drift fix
2823     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
2824
2825     my $nokvm = defined($conf->{kvm}) && $conf->{kvm} == 0 ? 1 : 0;
2826     my $useLocaltime = $conf->{localtime};
2827
2828     if (my $ost = $conf->{ostype}) {
2829         # other, wxp, w2k, w2k3, w2k8, wvista, win7, win8, l24, l26, solaris
2830
2831         if ($ost =~ m/^w/) { # windows
2832             $useLocaltime = 1 if !defined($conf->{localtime});
2833
2834             # use time drift fix when acpi is enabled
2835             if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
2836                 $tdf = 1 if !defined($conf->{tdf});
2837             }
2838         }
2839
2840         if ($ost eq 'win7' || $ost eq 'win8' || $ost eq 'w2k8' ||
2841             $ost eq 'wvista') {
2842             push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
2843             push @$cmd, '-no-hpet';
2844             if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
2845                 push @$cpuFlags , 'hv_spinlocks=0x1fff' if !$nokvm;
2846                 push @$cpuFlags , 'hv_vapic' if !$nokvm;
2847                 push @$cpuFlags , 'hv_time' if !$nokvm;
2848
2849             } else {
2850                 push @$cpuFlags , 'hv_spinlocks=0xffff' if !$nokvm;
2851             }
2852         }
2853
2854         if ($ost eq 'win7' || $ost eq 'win8') {
2855             push @$cpuFlags , 'hv_relaxed' if !$nokvm;
2856         }
2857     }
2858
2859     push @$rtcFlags, 'driftfix=slew' if $tdf;
2860
2861     if ($nokvm) {
2862         push @$machineFlags, 'accel=tcg';
2863     } else {
2864         die "No accelerator found!\n" if !$cpuinfo->{hvm};
2865     }
2866
2867     if ($machine_type) {
2868         push @$machineFlags, "type=${machine_type}";
2869     }
2870
2871     if ($conf->{startdate}) {
2872         push @$rtcFlags, "base=$conf->{startdate}";
2873     } elsif ($useLocaltime) {
2874         push @$rtcFlags, 'base=localtime';
2875     }
2876
2877     my $cpu = $nokvm ? "qemu64" : "kvm64";
2878     $cpu = $conf->{cpu} if $conf->{cpu};
2879
2880     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
2881
2882     push @$cpuFlags , '-x2apic'
2883         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
2884
2885     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
2886
2887     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
2888
2889     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
2890
2891         push @$cpuFlags , '+kvm_pv_unhalt' if !$nokvm;
2892         push @$cpuFlags , '+kvm_pv_eoi' if !$nokvm;
2893     }
2894
2895     push @$cpuFlags, 'enforce' if $cpu ne 'host' && !$nokvm;
2896
2897     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
2898
2899     push @$cmd, '-cpu', $cpu;
2900
2901     my $memory = $conf->{memory} || $defaults->{memory};
2902     my $static_memory = 0;
2903     my $dimm_memory = 0;
2904
2905     if ($hotplug_features->{memory}) {
2906         die "Numa need to be enabled for memory hotplug\n" if !$conf->{numa};
2907         die "Total memory is bigger than ${MAX_MEM}MB\n" if $memory > $MAX_MEM;
2908         $static_memory = $STATICMEM;
2909         die "minimum memory must be ${static_memory}MB\n" if($memory < $static_memory);
2910         $dimm_memory = $memory - $static_memory;
2911         push @$cmd, '-m', "size=${static_memory},slots=255,maxmem=${MAX_MEM}M";
2912
2913     } else {
2914
2915         $static_memory = $memory;
2916         push @$cmd, '-m', $static_memory;
2917     }
2918
2919     if ($conf->{numa}) {
2920
2921         my $numa_totalmemory = undef;
2922         for (my $i = 0; $i < $MAX_NUMA; $i++) {
2923             next if !$conf->{"numa$i"};
2924             my $numa = parse_numa($conf->{"numa$i"});
2925             next if !$numa;
2926             # memory
2927             die "missing numa node$i memory value\n" if !$numa->{memory};
2928             my $numa_memory = $numa->{memory};
2929             $numa_totalmemory += $numa_memory;
2930             my $numa_object = "memory-backend-ram,id=ram-node$i,size=${numa_memory}M";
2931
2932             # cpus
2933             my $cpus_start = $numa->{cpus}->{start};
2934             die "missing numa node$i cpus\n" if !defined($cpus_start);
2935             my $cpus_end = $numa->{cpus}->{end} if defined($numa->{cpus}->{end});
2936             my $cpus = $cpus_start;
2937             if (defined($cpus_end)) {
2938                 $cpus .= "-$cpus_end";
2939                 die "numa node$i :  cpu range $cpus is incorrect\n" if $cpus_end <= $cpus_start;
2940             }
2941
2942             # hostnodes
2943             my $hostnodes_start = $numa->{hostnodes}->{start};
2944             if (defined($hostnodes_start)) {
2945                 my $hostnodes_end = $numa->{hostnodes}->{end} if defined($numa->{hostnodes}->{end});
2946                 my $hostnodes = $hostnodes_start;
2947                 if (defined($hostnodes_end)) {
2948                     $hostnodes .= "-$hostnodes_end";
2949                     die "host node $hostnodes range is incorrect\n" if $hostnodes_end <= $hostnodes_start;
2950                 }
2951
2952                 my $hostnodes_end_range = defined($hostnodes_end) ? $hostnodes_end : $hostnodes_start;
2953                 for (my $i = $hostnodes_start; $i <= $hostnodes_end_range; $i++ ) {
2954                     die "host numa node$i don't exist\n" if ! -d "/sys/devices/system/node/node$i/";
2955                 }
2956
2957                 # policy
2958                 my $policy = $numa->{policy};
2959                 die "you need to define a policy for hostnode $hostnodes\n" if !$policy;
2960                 $numa_object .= ",host-nodes=$hostnodes,policy=$policy";
2961             }
2962
2963             push @$cmd, '-object', $numa_object;
2964             push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
2965         }
2966
2967         die "total memory for NUMA nodes must be equal to vm static memory\n"
2968             if $numa_totalmemory && $numa_totalmemory != $static_memory;
2969
2970         #if no custom tology, we split memory and cores across numa nodes
2971         if(!$numa_totalmemory) {
2972
2973             my $numa_memory = ($static_memory / $sockets) . "M";
2974
2975             for (my $i = 0; $i < $sockets; $i++)  {
2976
2977                 my $cpustart = ($cores * $i);
2978                 my $cpuend = ($cpustart + $cores - 1) if $cores && $cores > 1;
2979                 my $cpus = $cpustart;
2980                 $cpus .= "-$cpuend" if $cpuend;
2981
2982                 push @$cmd, '-object', "memory-backend-ram,size=$numa_memory,id=ram-node$i";
2983                 push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
2984             }
2985         }
2986     }
2987
2988     if ($hotplug_features->{memory}) {
2989         foreach_dimm($conf, $vmid, $memory, $sockets, sub {
2990             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
2991             push @$cmd, "-object" , "memory-backend-ram,id=mem-$name,size=${dimm_size}M";
2992             push @$cmd, "-device", "pc-dimm,id=$name,memdev=mem-$name,node=$numanode";
2993
2994             #if dimm_memory is not aligned to dimm map
2995             if($current_size > $memory) {
2996                  $conf->{memory} = $current_size;
2997                  update_config_nolock($vmid, $conf, 1);
2998             }
2999         });
3000     }
3001
3002     push @$cmd, '-S' if $conf->{freeze};
3003
3004     # set keyboard layout
3005     my $kb = $conf->{keyboard} || $defaults->{keyboard};
3006     push @$cmd, '-k', $kb if $kb;
3007
3008     # enable sound
3009     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3010     #push @$cmd, '-soundhw', 'es1370';
3011     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3012
3013     if($conf->{agent}) {
3014         my $qgasocket = qmp_socket($vmid, 1);
3015         my $pciaddr = print_pci_addr("qga0", $bridges);
3016         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3017         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3018         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3019     }
3020
3021     my $spice_port;
3022
3023     if ($qxlnum) {
3024         if ($qxlnum > 1) {
3025             if ($conf->{ostype} && $conf->{ostype} =~ m/^w/){
3026                 for(my $i = 1; $i < $qxlnum; $i++){
3027                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3028                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3029                 }
3030             } else {
3031                 # assume other OS works like Linux
3032                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3033                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3034             }
3035         }
3036
3037         my $pciaddr = print_pci_addr("spice", $bridges);
3038
3039         my $nodename = PVE::INotify::nodename();
3040         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3041         $spice_port = PVE::Tools::next_spice_port($pfamily);
3042
3043         push @$devices, '-spice', "tls-port=${spice_port},addr=localhost,tls-ciphers=DES-CBC3-SHA,seamless-migration=on";
3044
3045         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3046         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3047         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3048     }
3049
3050     # enable balloon by default, unless explicitly disabled
3051     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3052         $pciaddr = print_pci_addr("balloon0", $bridges);
3053         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3054     }
3055
3056     if ($conf->{watchdog}) {
3057         my $wdopts = parse_watchdog($conf->{watchdog});
3058         $pciaddr = print_pci_addr("watchdog", $bridges);
3059         my $watchdog = $wdopts->{model} || 'i6300esb';
3060         push @$devices, '-device', "$watchdog$pciaddr";
3061         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3062     }
3063
3064     my $vollist = [];
3065     my $scsicontroller = {};
3066     my $ahcicontroller = {};
3067     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3068
3069     # Add iscsi initiator name if available
3070     if (my $initiator = get_initiator_name()) {
3071         push @$devices, '-iscsi', "initiator-name=$initiator";
3072     }
3073
3074     foreach_drive($conf, sub {
3075         my ($ds, $drive) = @_;
3076
3077         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3078             push @$vollist, $drive->{file};
3079         }
3080
3081         $use_virtio = 1 if $ds =~ m/^virtio/;
3082
3083         if (drive_is_cdrom ($drive)) {
3084             if ($bootindex_hash->{d}) {
3085                 $drive->{bootindex} = $bootindex_hash->{d};
3086                 $bootindex_hash->{d} += 1;
3087             }
3088         } else {
3089             if ($bootindex_hash->{c}) {
3090                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3091                 $bootindex_hash->{c} += 1;
3092             }
3093         }
3094
3095         if($drive->{interface} eq 'virtio'){
3096            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3097         }
3098
3099         if ($drive->{interface} eq 'scsi') {
3100
3101             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3102
3103             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3104             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3105
3106             my $iothread = '';
3107             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3108                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3109                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3110             }
3111
3112             my $queues = '';
3113             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3114                 $queues = ",num_queues=$drive->{queues}";
3115             }
3116
3117             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3118             $scsicontroller->{$controller}=1;
3119         }
3120
3121         if ($drive->{interface} eq 'sata') {
3122            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3123            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3124            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3125            $ahcicontroller->{$controller}=1;
3126         }
3127
3128         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3129         push @$devices, '-drive',$drive_cmd;
3130         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3131     });
3132
3133     for (my $i = 0; $i < $MAX_NETS; $i++) {
3134          next if !$conf->{"net$i"};
3135          my $d = parse_net($conf->{"net$i"});
3136          next if !$d;
3137
3138          $use_virtio = 1 if $d->{model} eq 'virtio';
3139
3140          if ($bootindex_hash->{n}) {
3141             $d->{bootindex} = $bootindex_hash->{n};
3142             $bootindex_hash->{n} += 1;
3143          }
3144
3145          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3146          push @$devices, '-netdev', $netdevfull;
3147
3148          my $netdevicefull = print_netdevice_full($vmid,$conf,$d,"net$i",$bridges);
3149          push @$devices, '-device', $netdevicefull;
3150     }
3151
3152     if (!$q35) {
3153         # add pci bridges
3154         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3155            $bridges->{1} = 1;
3156            $bridges->{2} = 1;
3157         }
3158
3159         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3160
3161         while (my ($k, $v) = each %$bridges) {
3162             $pciaddr = print_pci_addr("pci.$k");
3163             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3164         }
3165     }
3166
3167     # add custom args
3168     if ($conf->{args}) {
3169         my $aa = PVE::Tools::split_args($conf->{args});
3170         push @$cmd, @$aa;
3171     }
3172
3173     push @$cmd, @$devices;
3174     push @$cmd, '-rtc', join(',', @$rtcFlags)
3175         if scalar(@$rtcFlags);
3176     push @$cmd, '-machine', join(',', @$machineFlags)
3177         if scalar(@$machineFlags);
3178     push @$cmd, '-global', join(',', @$globalFlags)
3179         if scalar(@$globalFlags);
3180
3181     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3182 }
3183
3184 sub vnc_socket {
3185     my ($vmid) = @_;
3186     return "${var_run_tmpdir}/$vmid.vnc";
3187 }
3188
3189 sub spice_port {
3190     my ($vmid) = @_;
3191
3192     my $res = vm_mon_cmd($vmid, 'query-spice');
3193
3194     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3195 }
3196
3197 sub qmp_socket {
3198     my ($vmid, $qga) = @_;
3199     my $sockettype = $qga ? 'qga' : 'qmp';
3200     return "${var_run_tmpdir}/$vmid.$sockettype";
3201 }
3202
3203 sub pidfile_name {
3204     my ($vmid) = @_;
3205     return "${var_run_tmpdir}/$vmid.pid";
3206 }
3207
3208 sub vm_devices_list {
3209     my ($vmid) = @_;
3210
3211     my $res = vm_mon_cmd($vmid, 'query-pci');
3212     my $devices = {};
3213     foreach my $pcibus (@$res) {
3214         foreach my $device (@{$pcibus->{devices}}) {
3215             next if !$device->{'qdev_id'};
3216             if ($device->{'pci_bridge'}) {
3217                 $devices->{$device->{'qdev_id'}} = 1;
3218                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3219                     next if !$bridge_device->{'qdev_id'};
3220                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3221                     $devices->{$device->{'qdev_id'}}++;
3222                 }
3223             } else {
3224                 $devices->{$device->{'qdev_id'}} = 1;
3225             }
3226         }
3227     }
3228
3229     my $resblock = vm_mon_cmd($vmid, 'query-block');
3230     foreach my $block (@$resblock) {
3231         if($block->{device} =~ m/^drive-(\S+)/){
3232                 $devices->{$1} = 1;
3233         }
3234     }
3235
3236     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3237     foreach my $mice (@$resmice) {
3238         if ($mice->{name} eq 'QEMU HID Tablet') {
3239             $devices->{tablet} = 1;
3240             last;
3241         }
3242     }
3243
3244     return $devices;
3245 }
3246
3247 sub vm_deviceplug {
3248     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3249
3250     my $q35 = machine_type_is_q35($conf);
3251
3252     my $devices_list = vm_devices_list($vmid);
3253     return 1 if defined($devices_list->{$deviceid});
3254
3255     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3256
3257     if ($deviceid eq 'tablet') {
3258
3259         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3260
3261     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3262
3263         qemu_iothread_add($vmid, $deviceid, $device);
3264
3265         qemu_driveadd($storecfg, $vmid, $device);
3266         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3267
3268         qemu_deviceadd($vmid, $devicefull);
3269         eval { qemu_deviceaddverify($vmid, $deviceid); };
3270         if (my $err = $@) {
3271             eval { qemu_drivedel($vmid, $deviceid); };
3272             warn $@ if $@;
3273             die $err;
3274         }
3275
3276     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3277
3278
3279         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3280         my $pciaddr = print_pci_addr($deviceid);
3281         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3282
3283         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3284
3285         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3286             qemu_iothread_add($vmid, $deviceid, $device);
3287             $devicefull .= ",iothread=iothread-$deviceid";
3288         }
3289
3290         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3291             $devicefull .= ",num_queues=$device->{queues}";
3292         }
3293
3294         qemu_deviceadd($vmid, $devicefull);
3295         qemu_deviceaddverify($vmid, $deviceid);
3296
3297     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3298
3299         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3300         qemu_driveadd($storecfg, $vmid, $device);
3301
3302         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3303         eval { qemu_deviceadd($vmid, $devicefull); };
3304         if (my $err = $@) {
3305             eval { qemu_drivedel($vmid, $deviceid); };
3306             warn $@ if $@;
3307             die $err;
3308         }
3309
3310     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3311
3312         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3313         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid);
3314         qemu_deviceadd($vmid, $netdevicefull);
3315         eval { qemu_deviceaddverify($vmid, $deviceid); };
3316         if (my $err = $@) {
3317             eval { qemu_netdevdel($vmid, $deviceid); };
3318             warn $@ if $@;
3319             die $err;
3320         }
3321
3322     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3323
3324         my $bridgeid = $2;
3325         my $pciaddr = print_pci_addr($deviceid);
3326         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3327
3328         qemu_deviceadd($vmid, $devicefull);
3329         qemu_deviceaddverify($vmid, $deviceid);
3330
3331     } else {
3332         die "can't hotplug device '$deviceid'\n";
3333     }
3334
3335     return 1;
3336 }
3337
3338 # fixme: this should raise exceptions on error!
3339 sub vm_deviceunplug {
3340     my ($vmid, $conf, $deviceid) = @_;
3341
3342     my $devices_list = vm_devices_list($vmid);
3343     return 1 if !defined($devices_list->{$deviceid});
3344
3345     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3346
3347     if ($deviceid eq 'tablet') {
3348
3349         qemu_devicedel($vmid, $deviceid);
3350
3351     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3352
3353         qemu_devicedel($vmid, $deviceid);
3354         qemu_devicedelverify($vmid, $deviceid);
3355         qemu_drivedel($vmid, $deviceid);
3356         qemu_iothread_del($conf, $vmid, $deviceid);
3357
3358     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3359
3360         qemu_devicedel($vmid, $deviceid);
3361         qemu_devicedelverify($vmid, $deviceid);
3362         qemu_iothread_del($conf, $vmid, $deviceid);
3363
3364     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3365
3366         #qemu 2.3 segfault on drive_del with virtioscsi + iothread
3367         my $device = parse_drive($deviceid, $conf->{$deviceid});
3368         die "virtioscsi with iothread is not hot-unplugglable currently" if $device->{iothread};
3369
3370         qemu_devicedel($vmid, $deviceid);
3371         qemu_drivedel($vmid, $deviceid);
3372         qemu_deletescsihw($conf, $vmid, $deviceid);
3373
3374     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3375
3376         qemu_devicedel($vmid, $deviceid);
3377         qemu_devicedelverify($vmid, $deviceid);
3378         qemu_netdevdel($vmid, $deviceid);
3379
3380     } else {
3381         die "can't unplug device '$deviceid'\n";
3382     }
3383
3384     return 1;
3385 }
3386
3387 sub qemu_deviceadd {
3388     my ($vmid, $devicefull) = @_;
3389
3390     $devicefull = "driver=".$devicefull;
3391     my %options =  split(/[=,]/, $devicefull);
3392
3393     vm_mon_cmd($vmid, "device_add" , %options);
3394 }
3395
3396 sub qemu_devicedel {
3397     my ($vmid, $deviceid) = @_;
3398
3399     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3400 }
3401
3402 sub qemu_iothread_add {
3403     my($vmid, $deviceid, $device) = @_;
3404
3405     if ($device->{iothread}) {
3406         my $iothreads = vm_iothreads_list($vmid);
3407         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3408     }
3409 }
3410
3411 sub qemu_iothread_del {
3412     my($conf, $vmid, $deviceid) = @_;
3413
3414     my $device = parse_drive($deviceid, $conf->{$deviceid});
3415     if ($device->{iothread}) {
3416         my $iothreads = vm_iothreads_list($vmid);
3417         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3418     }
3419 }
3420
3421 sub qemu_objectadd {
3422     my($vmid, $objectid, $qomtype) = @_;
3423
3424     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3425
3426     return 1;
3427 }
3428
3429 sub qemu_objectdel {
3430     my($vmid, $objectid) = @_;
3431
3432     vm_mon_cmd($vmid, "object-del", id => $objectid);
3433
3434     return 1;
3435 }
3436
3437 sub qemu_driveadd {
3438     my ($storecfg, $vmid, $device) = @_;
3439
3440     my $drive = print_drive_full($storecfg, $vmid, $device);
3441     $drive =~ s/\\/\\\\/g;
3442     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3443
3444     # If the command succeeds qemu prints: "OK"
3445     return 1 if $ret =~ m/OK/s;
3446
3447     die "adding drive failed: $ret\n";
3448 }
3449
3450 sub qemu_drivedel {
3451     my($vmid, $deviceid) = @_;
3452
3453     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3454     $ret =~ s/^\s+//;
3455
3456     return 1 if $ret eq "";
3457
3458     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3459     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3460
3461     die "deleting drive $deviceid failed : $ret\n";
3462 }
3463
3464 sub qemu_deviceaddverify {
3465     my ($vmid, $deviceid) = @_;
3466
3467     for (my $i = 0; $i <= 5; $i++) {
3468          my $devices_list = vm_devices_list($vmid);
3469          return 1 if defined($devices_list->{$deviceid});
3470          sleep 1;
3471     }
3472
3473     die "error on hotplug device '$deviceid'\n";
3474 }
3475
3476
3477 sub qemu_devicedelverify {
3478     my ($vmid, $deviceid) = @_;
3479
3480     # need to verify that the device is correctly removed as device_del
3481     # is async and empty return is not reliable
3482
3483     for (my $i = 0; $i <= 5; $i++) {
3484          my $devices_list = vm_devices_list($vmid);
3485          return 1 if !defined($devices_list->{$deviceid});
3486          sleep 1;
3487     }
3488
3489     die "error on hot-unplugging device '$deviceid'\n";
3490 }
3491
3492 sub qemu_findorcreatescsihw {
3493     my ($storecfg, $conf, $vmid, $device) = @_;
3494
3495     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3496
3497     my $scsihwid="$controller_prefix$controller";
3498     my $devices_list = vm_devices_list($vmid);
3499
3500     if(!defined($devices_list->{$scsihwid})) {
3501         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3502     }
3503
3504     return 1;
3505 }
3506
3507 sub qemu_deletescsihw {
3508     my ($conf, $vmid, $opt) = @_;
3509
3510     my $device = parse_drive($opt, $conf->{$opt});
3511
3512     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3513         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3514         return 1;
3515     }
3516
3517     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3518
3519     my $devices_list = vm_devices_list($vmid);
3520     foreach my $opt (keys %{$devices_list}) {
3521         if (PVE::QemuServer::valid_drivename($opt)) {
3522             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3523             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3524                 return 1;
3525             }
3526         }
3527     }
3528
3529     my $scsihwid="scsihw$controller";
3530
3531     vm_deviceunplug($vmid, $conf, $scsihwid);
3532
3533     return 1;
3534 }
3535
3536 sub qemu_add_pci_bridge {
3537     my ($storecfg, $conf, $vmid, $device) = @_;
3538
3539     my $bridges = {};
3540
3541     my $bridgeid;
3542
3543     print_pci_addr($device, $bridges);
3544
3545     while (my ($k, $v) = each %$bridges) {
3546         $bridgeid = $k;
3547     }
3548     return 1 if !defined($bridgeid) || $bridgeid < 1;
3549
3550     my $bridge = "pci.$bridgeid";
3551     my $devices_list = vm_devices_list($vmid);
3552
3553     if (!defined($devices_list->{$bridge})) {
3554         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3555     }
3556
3557     return 1;
3558 }
3559
3560 sub qemu_set_link_status {
3561     my ($vmid, $device, $up) = @_;
3562
3563     vm_mon_cmd($vmid, "set_link", name => $device,
3564                up => $up ? JSON::true : JSON::false);
3565 }
3566
3567 sub qemu_netdevadd {
3568     my ($vmid, $conf, $device, $deviceid) = @_;
3569
3570     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid);
3571     my %options =  split(/[=,]/, $netdev);
3572
3573     vm_mon_cmd($vmid, "netdev_add",  %options);
3574     return 1;
3575 }
3576
3577 sub qemu_netdevdel {
3578     my ($vmid, $deviceid) = @_;
3579
3580     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3581 }
3582
3583 sub qemu_cpu_hotplug {
3584     my ($vmid, $conf, $vcpus) = @_;
3585
3586     my $sockets = 1;
3587     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3588     $sockets = $conf->{sockets} if  $conf->{sockets};
3589     my $cores = $conf->{cores} || 1;
3590     my $maxcpus = $sockets * $cores;
3591
3592     $vcpus = $maxcpus if !$vcpus;
3593
3594     die "you can't add more vcpus than maxcpus\n"
3595         if $vcpus > $maxcpus;
3596
3597     my $currentvcpus = $conf->{vcpus} || $maxcpus;
3598     die "online cpu unplug is not yet possible\n"
3599         if $vcpus < $currentvcpus;
3600
3601     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3602     die "vcpus in running vm is different than configuration\n"
3603         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
3604
3605     for (my $i = $currentvcpus; $i < $vcpus; $i++) {
3606         vm_mon_cmd($vmid, "cpu-add", id => int($i));
3607     }
3608 }
3609
3610 sub qemu_memory_hotplug {
3611     my ($vmid, $conf, $defaults, $opt, $value) = @_;
3612
3613     return $value if !check_running($vmid);
3614
3615     my $memory = $conf->{memory} || $defaults->{memory};
3616     $value = $defaults->{memory} if !$value;
3617     return $value if $value == $memory;
3618
3619     my $static_memory = $STATICMEM;
3620     my $dimm_memory = $memory - $static_memory;
3621
3622     die "memory can't be lower than $static_memory MB" if $value < $static_memory;
3623     die "you cannot add more memory than $MAX_MEM MB!\n" if $memory > $MAX_MEM;
3624
3625
3626     my $sockets = 1;
3627     $sockets = $conf->{sockets} if $conf->{sockets};
3628
3629     if($value > $memory) {
3630
3631         foreach_dimm($conf, $vmid, $value, $sockets, sub {
3632             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3633
3634                 return if $current_size <= $conf->{memory};
3635
3636                 eval { vm_mon_cmd($vmid, "object-add", 'qom-type' => "memory-backend-ram", id => "mem-$name", props => { size => int($dimm_size*1024*1024) } ) };
3637                 if (my $err = $@) {
3638                     eval { qemu_objectdel($vmid, "mem-$name"); };
3639                     die $err;
3640                 }
3641
3642                 eval { vm_mon_cmd($vmid, "device_add", driver => "pc-dimm", id => "$name", memdev => "mem-$name", node => $numanode) };
3643                 if (my $err = $@) {
3644                     eval { qemu_objectdel($vmid, "mem-$name"); };
3645                     die $err;
3646                 }
3647                 #update conf after each succesful module hotplug
3648                 $conf->{memory} = $current_size;
3649                 update_config_nolock($vmid, $conf, 1);
3650         });
3651
3652     } else {
3653
3654         foreach_reverse_dimm($conf, $vmid, $value, $sockets, sub {
3655             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3656
3657                 return if $current_size >= $conf->{memory};
3658                 print "try to unplug memory dimm $name\n";
3659
3660                 my $retry = 0;
3661                 while (1) {
3662                     eval { qemu_devicedel($vmid, $name) };
3663                     sleep 3;
3664                     my $dimm_list = qemu_dimm_list($vmid);
3665                     last if !$dimm_list->{$name};
3666                     raise_param_exc({ $name => "error unplug memory module" }) if $retry > 5;
3667                     $retry++;
3668                 }
3669
3670                 #update conf after each succesful module unplug
3671                 $conf->{memory} = $current_size;
3672
3673                 eval { qemu_objectdel($vmid, "mem-$name"); };
3674                 update_config_nolock($vmid, $conf, 1);
3675         });
3676     }
3677 }
3678
3679 sub qemu_dimm_list {
3680     my ($vmid) = @_;
3681
3682     my $dimmarray = vm_mon_cmd_nocheck($vmid, "query-memory-devices");
3683     my $dimms = {};
3684
3685     foreach my $dimm (@$dimmarray) {
3686
3687         $dimms->{$dimm->{data}->{id}}->{id} = $dimm->{data}->{id};
3688         $dimms->{$dimm->{data}->{id}}->{node} = $dimm->{data}->{node};
3689         $dimms->{$dimm->{data}->{id}}->{addr} = $dimm->{data}->{addr};
3690         $dimms->{$dimm->{data}->{id}}->{size} = $dimm->{data}->{size};
3691         $dimms->{$dimm->{data}->{id}}->{slot} = $dimm->{data}->{slot};
3692     }
3693     return $dimms;
3694 }
3695
3696 sub qemu_block_set_io_throttle {
3697     my ($vmid, $deviceid, $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr) = @_;
3698
3699     return if !check_running($vmid) ;
3700
3701     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid, bps => int($bps), bps_rd => int($bps_rd), bps_wr => int($bps_wr), iops => int($iops), iops_rd => int($iops_rd), iops_wr => int($iops_wr));
3702
3703 }
3704
3705 # old code, only used to shutdown old VM after update
3706 sub __read_avail {
3707     my ($fh, $timeout) = @_;
3708
3709     my $sel = new IO::Select;
3710     $sel->add($fh);
3711
3712     my $res = '';
3713     my $buf;
3714
3715     my @ready;
3716     while (scalar (@ready = $sel->can_read($timeout))) {
3717         my $count;
3718         if ($count = $fh->sysread($buf, 8192)) {
3719             if ($buf =~ /^(.*)\(qemu\) $/s) {
3720                 $res .= $1;
3721                 last;
3722             } else {
3723                 $res .= $buf;
3724             }
3725         } else {
3726             if (!defined($count)) {
3727                 die "$!\n";
3728             }
3729             last;
3730         }
3731     }
3732
3733     die "monitor read timeout\n" if !scalar(@ready);
3734
3735     return $res;
3736 }
3737
3738 # old code, only used to shutdown old VM after update
3739 sub vm_monitor_command {
3740     my ($vmid, $cmdstr, $nocheck) = @_;
3741
3742     my $res;
3743
3744     eval {
3745         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
3746
3747         my $sname = "${var_run_tmpdir}/$vmid.mon";
3748
3749         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
3750             die "unable to connect to VM $vmid socket - $!\n";
3751
3752         my $timeout = 3;
3753
3754         # hack: migrate sometime blocks the monitor (when migrate_downtime
3755         # is set)
3756         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3757             $timeout = 60*60; # 1 hour
3758         }
3759
3760         # read banner;
3761         my $data = __read_avail($sock, $timeout);
3762
3763         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
3764             die "got unexpected qemu monitor banner\n";
3765         }
3766
3767         my $sel = new IO::Select;
3768         $sel->add($sock);
3769
3770         if (!scalar(my @ready = $sel->can_write($timeout))) {
3771             die "monitor write error - timeout";
3772         }
3773
3774         my $fullcmd = "$cmdstr\r";
3775
3776         # syslog('info', "VM $vmid monitor command: $cmdstr");
3777
3778         my $b;
3779         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
3780             die "monitor write error - $!";
3781         }
3782
3783         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
3784
3785         $timeout = 20;
3786
3787         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3788             $timeout = 60*60; # 1 hour
3789         } elsif ($cmdstr =~ m/^(eject|change)/) {
3790             $timeout = 60; # note: cdrom mount command is slow
3791         }
3792         if ($res = __read_avail($sock, $timeout)) {
3793
3794             my @lines = split("\r?\n", $res);
3795
3796             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
3797
3798             $res = join("\n", @lines);
3799             $res .= "\n";
3800         }
3801     };
3802
3803     my $err = $@;
3804
3805     if ($err) {
3806         syslog("err", "VM $vmid monitor command failed - $err");
3807         die $err;
3808     }
3809
3810     return $res;
3811 }
3812
3813 sub qemu_block_resize {
3814     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
3815
3816     my $running = check_running($vmid);
3817
3818     return if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
3819
3820     return if !$running;
3821
3822     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
3823
3824 }
3825
3826 sub qemu_volume_snapshot {
3827     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3828
3829     my $running = check_running($vmid);
3830
3831     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
3832         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
3833     } else {
3834         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
3835     }
3836 }
3837
3838 sub qemu_volume_snapshot_delete {
3839     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3840
3841     my $running = check_running($vmid);
3842
3843     return if !PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
3844
3845     return if !$running;
3846
3847     vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
3848 }
3849
3850 sub set_migration_caps {
3851     my ($vmid) = @_;
3852
3853     my $cap_ref = [];
3854
3855     my $enabled_cap = {
3856         "auto-converge" => 1,
3857         "xbzrle" => 1,
3858         "x-rdma-pin-all" => 0,
3859         "zero-blocks" => 0,
3860         "compress" => 0
3861     };
3862
3863     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
3864
3865     for my $supported_capability (@$supported_capabilities) {
3866         push @$cap_ref, {
3867             capability => $supported_capability->{capability},
3868             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
3869         };
3870     }
3871
3872     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
3873 }
3874
3875 my $fast_plug_option = {
3876     'lock' => 1,
3877     'name' => 1,
3878     'onboot' => 1,
3879     'shares' => 1,
3880     'startup' => 1,
3881     'description' => 1,
3882 };
3883
3884 # hotplug changes in [PENDING]
3885 # $selection hash can be used to only apply specified options, for
3886 # example: { cores => 1 } (only apply changed 'cores')
3887 # $errors ref is used to return error messages
3888 sub vmconfig_hotplug_pending {
3889     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
3890
3891     my $defaults = load_defaults();
3892
3893     # commit values which do not have any impact on running VM first
3894     # Note: those option cannot raise errors, we we do not care about
3895     # $selection and always apply them.
3896
3897     my $add_error = sub {
3898         my ($opt, $msg) = @_;
3899         $errors->{$opt} = "hotplug problem - $msg";
3900     };
3901
3902     my $changes = 0;
3903     foreach my $opt (keys %{$conf->{pending}}) { # add/change
3904         if ($fast_plug_option->{$opt}) {
3905             $conf->{$opt} = $conf->{pending}->{$opt};
3906             delete $conf->{pending}->{$opt};
3907             $changes = 1;
3908         }
3909     }
3910
3911     if ($changes) {
3912         update_config_nolock($vmid, $conf, 1);
3913         $conf = load_config($vmid); # update/reload
3914     }
3915
3916     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3917
3918     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
3919     while (my ($opt, $force) = each %$pending_delete_hash) {
3920         next if $selection && !$selection->{$opt};
3921         eval {
3922             if ($opt eq 'hotplug') {
3923                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
3924             } elsif ($opt eq 'tablet') {
3925                 die "skip\n" if !$hotplug_features->{usb};
3926                 if ($defaults->{tablet}) {
3927                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
3928                 } else {
3929                     vm_deviceunplug($vmid, $conf, $opt);
3930                 }
3931             } elsif ($opt eq 'vcpus') {
3932                 die "skip\n" if !$hotplug_features->{cpu};
3933                 qemu_cpu_hotplug($vmid, $conf, undef);
3934             } elsif ($opt eq 'balloon') {
3935                 # enable balloon device is not hotpluggable
3936                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
3937             } elsif ($fast_plug_option->{$opt}) {
3938                 # do nothing
3939             } elsif ($opt =~ m/^net(\d+)$/) {
3940                 die "skip\n" if !$hotplug_features->{network};
3941                 vm_deviceunplug($vmid, $conf, $opt);
3942             } elsif (valid_drivename($opt)) {
3943                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
3944                 vm_deviceunplug($vmid, $conf, $opt);
3945                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
3946             } elsif ($opt =~ m/^memory$/) {
3947                 die "skip\n" if !$hotplug_features->{memory};
3948                 qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
3949             } elsif ($opt eq 'cpuunits') {
3950                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
3951             } elsif ($opt eq 'cpulimit') {
3952                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
3953             } else {
3954                 die "skip\n";
3955             }
3956         };
3957         if (my $err = $@) {
3958             &$add_error($opt, $err) if $err ne "skip\n";
3959         } else {
3960             # save new config if hotplug was successful
3961             delete $conf->{$opt};
3962             vmconfig_undelete_pending_option($conf, $opt);
3963             update_config_nolock($vmid, $conf, 1);
3964             $conf = load_config($vmid); # update/reload
3965         }
3966     }
3967
3968     foreach my $opt (keys %{$conf->{pending}}) {
3969         next if $selection && !$selection->{$opt};
3970         my $value = $conf->{pending}->{$opt};
3971         eval {
3972             if ($opt eq 'hotplug') {
3973                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
3974             } elsif ($opt eq 'tablet') {
3975                 die "skip\n" if !$hotplug_features->{usb};
3976                 if ($value == 1) {
3977                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
3978                 } elsif ($value == 0) {
3979                     vm_deviceunplug($vmid, $conf, $opt);
3980                 }
3981             } elsif ($opt eq 'vcpus') {
3982                 die "skip\n" if !$hotplug_features->{cpu};
3983                 qemu_cpu_hotplug($vmid, $conf, $value);
3984             } elsif ($opt eq 'balloon') {
3985                 # enable/disable balloning device is not hotpluggable
3986                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
3987                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
3988                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
3989
3990                 # allow manual ballooning if shares is set to zero
3991                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
3992                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
3993                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
3994                 }
3995             } elsif ($opt =~ m/^net(\d+)$/) {
3996                 # some changes can be done without hotplug
3997                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
3998                                     $vmid, $opt, $value);
3999             } elsif (valid_drivename($opt)) {
4000                 # some changes can be done without hotplug
4001                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4002                                      $vmid, $opt, $value, 1);
4003             } elsif ($opt =~ m/^memory$/) { #dimms
4004                 die "skip\n" if !$hotplug_features->{memory};
4005                 $value = qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4006             } elsif ($opt eq 'cpuunits') {
4007                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4008             } elsif ($opt eq 'cpulimit') {
4009                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4010                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4011             } else {
4012                 die "skip\n";  # skip non-hot-pluggable options
4013             }
4014         };
4015         if (my $err = $@) {
4016             &$add_error($opt, $err) if $err ne "skip\n";
4017         } else {
4018             # save new config if hotplug was successful
4019             $conf->{$opt} = $value;
4020             delete $conf->{pending}->{$opt};
4021             update_config_nolock($vmid, $conf, 1);
4022             $conf = load_config($vmid); # update/reload
4023         }
4024     }
4025 }
4026
4027 sub try_deallocate_drive {
4028     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4029
4030     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4031         my $volid = $drive->{file};
4032         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4033             my $sid = PVE::Storage::parse_volume_id($volid);
4034             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4035
4036             # check if the disk is really unused
4037             my $used_paths = PVE::QemuServer::get_used_paths($vmid, $storecfg, $conf, 1, $key);
4038             my $path = PVE::Storage::path($storecfg, $volid);
4039             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4040                    if $used_paths->{$path};
4041             PVE::Storage::vdisk_free($storecfg, $volid);
4042             return 1;
4043         } else {
4044             # If vm is not owner of this disk remove from config
4045             return 1;
4046         }
4047     }
4048
4049     return undef;
4050 }
4051
4052 sub vmconfig_delete_or_detach_drive {
4053     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4054
4055     my $drive = parse_drive($opt, $conf->{$opt});
4056
4057     my $rpcenv = PVE::RPCEnvironment::get();
4058     my $authuser = $rpcenv->get_user();
4059
4060     if ($force) {
4061         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4062         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4063     } else {
4064         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4065     }
4066 }
4067
4068 sub vmconfig_apply_pending {
4069     my ($vmid, $conf, $storecfg) = @_;
4070
4071     # cold plug
4072
4073     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4074     while (my ($opt, $force) = each %$pending_delete_hash) {
4075         die "internal error" if $opt =~ m/^unused/;
4076         $conf = load_config($vmid); # update/reload
4077         if (!defined($conf->{$opt})) {
4078             vmconfig_undelete_pending_option($conf, $opt);
4079             update_config_nolock($vmid, $conf, 1);
4080         } elsif (valid_drivename($opt)) {
4081             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4082             vmconfig_undelete_pending_option($conf, $opt);
4083             delete $conf->{$opt};
4084             update_config_nolock($vmid, $conf, 1);
4085         } else {
4086             vmconfig_undelete_pending_option($conf, $opt);
4087             delete $conf->{$opt};
4088             update_config_nolock($vmid, $conf, 1);
4089         }
4090     }
4091
4092     $conf = load_config($vmid); # update/reload
4093
4094     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4095         $conf = load_config($vmid); # update/reload
4096
4097         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4098             # skip if nothing changed
4099         } elsif (valid_drivename($opt)) {
4100             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4101                 if defined($conf->{$opt});
4102             $conf->{$opt} = $conf->{pending}->{$opt};
4103         } else {
4104             $conf->{$opt} = $conf->{pending}->{$opt};
4105         }
4106
4107         delete $conf->{pending}->{$opt};
4108         update_config_nolock($vmid, $conf, 1);
4109     }
4110 }
4111
4112 my $safe_num_ne = sub {
4113     my ($a, $b) = @_;
4114
4115     return 0 if !defined($a) && !defined($b);
4116     return 1 if !defined($a);
4117     return 1 if !defined($b);
4118
4119     return $a != $b;
4120 };
4121
4122 my $safe_string_ne = sub {
4123     my ($a, $b) = @_;
4124
4125     return 0 if !defined($a) && !defined($b);
4126     return 1 if !defined($a);
4127     return 1 if !defined($b);
4128
4129     return $a ne $b;
4130 };
4131
4132 sub vmconfig_update_net {
4133     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4134
4135     my $newnet = parse_net($value);
4136
4137     if ($conf->{$opt}) {
4138         my $oldnet = parse_net($conf->{$opt});
4139
4140         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4141             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4142             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4143             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4144
4145             # for non online change, we try to hot-unplug
4146             die "skip\n" if !$hotplug;
4147             vm_deviceunplug($vmid, $conf, $opt);
4148         } else {
4149
4150             die "internal error" if $opt !~ m/net(\d+)/;
4151             my $iface = "tap${vmid}i$1";
4152
4153             if (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4154                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4155             }
4156
4157             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4158                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4159                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4160                 PVE::Network::tap_unplug($iface);
4161                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall});
4162             }
4163
4164             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4165                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4166             }
4167
4168             return 1;
4169         }
4170     }
4171
4172     if ($hotplug) {
4173         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4174     } else {
4175         die "skip\n";
4176     }
4177 }
4178
4179 sub vmconfig_update_disk {
4180     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4181
4182     # fixme: do we need force?
4183
4184     my $drive = parse_drive($opt, $value);
4185
4186     if ($conf->{$opt}) {
4187
4188         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4189
4190             my $media = $drive->{media} || 'disk';
4191             my $oldmedia = $old_drive->{media} || 'disk';
4192             die "unable to change media type\n" if $media ne $oldmedia;
4193
4194             if (!drive_is_cdrom($old_drive)) {
4195
4196                 if ($drive->{file} ne $old_drive->{file}) {
4197
4198                     die "skip\n" if !$hotplug;
4199
4200                     # unplug and register as unused
4201                     vm_deviceunplug($vmid, $conf, $opt);
4202                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4203
4204                 } else {
4205                     # update existing disk
4206
4207                     # skip non hotpluggable value
4208                     if (&$safe_num_ne($drive->{discard}, $old_drive->{discard}) ||
4209                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4210                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4211                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4212                         die "skip\n";
4213                     }
4214
4215                     # apply throttle
4216                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4217                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4218                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4219                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4220                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4221                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4222                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4223                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4224                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4225                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4226                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4227                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max})) {
4228
4229                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4230                                                    ($drive->{mbps} || 0)*1024*1024,
4231                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4232                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4233                                                    $drive->{iops} || 0,
4234                                                    $drive->{iops_rd} || 0,
4235                                                    $drive->{iops_wr} || 0,
4236                                                    ($drive->{mbps_max} || 0)*1024*1024,
4237                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4238                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4239                                                    $drive->{iops_max} || 0,
4240                                                    $drive->{iops_rd_max} || 0,
4241                                                    $drive->{iops_wr_max} || 0);
4242
4243                     }
4244
4245                     return 1;
4246                 }
4247
4248             } else { # cdrom
4249
4250                 if ($drive->{file} eq 'none') {
4251                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4252                 } else {
4253                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4254                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4255                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4256                 }
4257
4258                 return 1;
4259             }
4260         }
4261     }
4262
4263     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4264     # hotplug new disks
4265     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4266 }
4267
4268 sub vm_start {
4269     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused, $forcemachine, $spice_ticket) = @_;
4270
4271     lock_config($vmid, sub {
4272         my $conf = load_config($vmid, $migratedfrom);
4273
4274         die "you can't start a vm if it's a template\n" if is_template($conf);
4275
4276         check_lock($conf) if !$skiplock;
4277
4278         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4279
4280         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4281             vmconfig_apply_pending($vmid, $conf, $storecfg);
4282             $conf = load_config($vmid); # update/reload
4283         }
4284
4285         my $defaults = load_defaults();
4286
4287         # set environment variable useful inside network script
4288         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4289
4290         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4291
4292         my $migrate_port = 0;
4293         my $migrate_uri;
4294         if ($statefile) {
4295             if ($statefile eq 'tcp') {
4296                 my $localip = "localhost";
4297                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4298                 my $nodename = PVE::INotify::nodename();
4299                 if ($datacenterconf->{migration_unsecure}) {
4300                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4301                 }
4302                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4303                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4304                 $migrate_uri = "tcp:[${localip}]:${migrate_port}";
4305                 push @$cmd, '-incoming', $migrate_uri;
4306                 push @$cmd, '-S';
4307             } else {
4308                 push @$cmd, '-loadstate', $statefile;
4309             }
4310         } elsif ($paused) {
4311             push @$cmd, '-S';
4312         }
4313
4314         # host pci devices
4315         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4316           my $d = parse_hostpci($conf->{"hostpci$i"});
4317           next if !$d;
4318           my $pcidevices = $d->{pciid};
4319           foreach my $pcidevice (@$pcidevices) {
4320                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4321
4322                 my $info = pci_device_info("0000:$pciid");
4323                 die "IOMMU not present\n" if !check_iommu_support();
4324                 die "no pci device info for device '$pciid'\n" if !$info;
4325                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4326                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4327           }
4328         }
4329
4330         PVE::Storage::activate_volumes($storecfg, $vollist);
4331
4332         eval  { run_command($cmd, timeout => $statefile ? undef : 30,
4333                     umask => 0077); };
4334         my $err = $@;
4335         die "start failed: $err" if $err;
4336
4337         print "migration listens on $migrate_uri\n" if $migrate_uri;
4338
4339         if ($statefile && $statefile ne 'tcp')  {
4340             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4341             warn $@ if $@;
4342         }
4343
4344         if ($migratedfrom) {
4345
4346             eval {
4347                 set_migration_caps($vmid);
4348             };
4349             warn $@ if $@;
4350
4351             if ($spice_port) {
4352                 print "spice listens on port $spice_port\n";
4353                 if ($spice_ticket) {
4354                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4355                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4356                 }
4357             }
4358
4359         } else {
4360
4361             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4362                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4363                     if $conf->{balloon};
4364             }
4365
4366             foreach my $opt (keys %$conf) {
4367                 next if $opt !~  m/^net\d+$/;
4368                 my $nicconf = parse_net($conf->{$opt});
4369                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4370             }
4371         }
4372
4373         vm_mon_cmd_nocheck($vmid, 'qom-set',
4374                     path => "machine/peripheral/balloon0",
4375                     property => "guest-stats-polling-interval",
4376                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4377
4378     });
4379 }
4380
4381 sub vm_mon_cmd {
4382     my ($vmid, $execute, %params) = @_;
4383
4384     my $cmd = { execute => $execute, arguments => \%params };
4385     vm_qmp_command($vmid, $cmd);
4386 }
4387
4388 sub vm_mon_cmd_nocheck {
4389     my ($vmid, $execute, %params) = @_;
4390
4391     my $cmd = { execute => $execute, arguments => \%params };
4392     vm_qmp_command($vmid, $cmd, 1);
4393 }
4394
4395 sub vm_qmp_command {
4396     my ($vmid, $cmd, $nocheck) = @_;
4397
4398     my $res;
4399
4400     my $timeout;
4401     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4402         $timeout = $cmd->{arguments}->{timeout};
4403         delete $cmd->{arguments}->{timeout};
4404     }
4405
4406     eval {
4407         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4408         my $sname = qmp_socket($vmid);
4409         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4410             my $qmpclient = PVE::QMPClient->new();
4411
4412             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4413         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
4414             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
4415                 if scalar(%{$cmd->{arguments}});
4416             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
4417         } else {
4418             die "unable to open monitor socket\n";
4419         }
4420     };
4421     if (my $err = $@) {
4422         syslog("err", "VM $vmid qmp command failed - $err");
4423         die $err;
4424     }
4425
4426     return $res;
4427 }
4428
4429 sub vm_human_monitor_command {
4430     my ($vmid, $cmdline) = @_;
4431
4432     my $res;
4433
4434     my $cmd = {
4435         execute => 'human-monitor-command',
4436         arguments => { 'command-line' => $cmdline},
4437     };
4438
4439     return vm_qmp_command($vmid, $cmd);
4440 }
4441
4442 sub vm_commandline {
4443     my ($storecfg, $vmid) = @_;
4444
4445     my $conf = load_config($vmid);
4446
4447     my $defaults = load_defaults();
4448
4449     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
4450
4451     return join(' ', @$cmd);
4452 }
4453
4454 sub vm_reset {
4455     my ($vmid, $skiplock) = @_;
4456
4457     lock_config($vmid, sub {
4458
4459         my $conf = load_config($vmid);
4460
4461         check_lock($conf) if !$skiplock;
4462
4463         vm_mon_cmd($vmid, "system_reset");
4464     });
4465 }
4466
4467 sub get_vm_volumes {
4468     my ($conf) = @_;
4469
4470     my $vollist = [];
4471     foreach_volid($conf, sub {
4472         my ($volid, $is_cdrom) = @_;
4473
4474         return if $volid =~ m|^/|;
4475
4476         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4477         return if !$sid;
4478
4479         push @$vollist, $volid;
4480     });
4481
4482     return $vollist;
4483 }
4484
4485 sub vm_stop_cleanup {
4486     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
4487
4488     eval {
4489
4490         if (!$keepActive) {
4491             my $vollist = get_vm_volumes($conf);
4492             PVE::Storage::deactivate_volumes($storecfg, $vollist);
4493         }
4494
4495         foreach my $ext (qw(mon qmp pid vnc qga)) {
4496             unlink "/var/run/qemu-server/${vmid}.$ext";
4497         }
4498
4499         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
4500     };
4501     warn $@ if $@; # avoid errors - just warn
4502 }
4503
4504 # Note: use $nockeck to skip tests if VM configuration file exists.
4505 # We need that when migration VMs to other nodes (files already moved)
4506 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
4507 sub vm_stop {
4508     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
4509
4510     $force = 1 if !defined($force) && !$shutdown;
4511
4512     if ($migratedfrom){
4513         my $pid = check_running($vmid, $nocheck, $migratedfrom);
4514         kill 15, $pid if $pid;
4515         my $conf = load_config($vmid, $migratedfrom);
4516         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
4517         return;
4518     }
4519
4520     lock_config($vmid, sub {
4521
4522         my $pid = check_running($vmid, $nocheck);
4523         return if !$pid;
4524
4525         my $conf;
4526         if (!$nocheck) {
4527             $conf = load_config($vmid);
4528             check_lock($conf) if !$skiplock;
4529             if (!defined($timeout) && $shutdown && $conf->{startup}) {
4530                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
4531                 $timeout = $opts->{down} if $opts->{down};
4532             }
4533         }
4534
4535         $timeout = 60 if !defined($timeout);
4536
4537         eval {
4538             if ($shutdown) {
4539                 if (defined($conf) && $conf->{agent}) {
4540                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
4541                 } else {
4542                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
4543                 }
4544             } else {
4545                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
4546             }
4547         };
4548         my $err = $@;
4549
4550         if (!$err) {
4551             my $count = 0;
4552             while (($count < $timeout) && check_running($vmid, $nocheck)) {
4553                 $count++;
4554                 sleep 1;
4555             }
4556
4557             if ($count >= $timeout) {
4558                 if ($force) {
4559                     warn "VM still running - terminating now with SIGTERM\n";
4560                     kill 15, $pid;
4561                 } else {
4562                     die "VM quit/powerdown failed - got timeout\n";
4563                 }
4564             } else {
4565                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4566                 return;
4567             }
4568         } else {
4569             if ($force) {
4570                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
4571                 kill 15, $pid;
4572             } else {
4573                 die "VM quit/powerdown failed\n";
4574             }
4575         }
4576
4577         # wait again
4578         $timeout = 10;
4579
4580         my $count = 0;
4581         while (($count < $timeout) && check_running($vmid, $nocheck)) {
4582             $count++;
4583             sleep 1;
4584         }
4585
4586         if ($count >= $timeout) {
4587             warn "VM still running - terminating now with SIGKILL\n";
4588             kill 9, $pid;
4589             sleep 1;
4590         }
4591
4592         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4593    });
4594 }
4595
4596 sub vm_suspend {
4597     my ($vmid, $skiplock) = @_;
4598
4599     lock_config($vmid, sub {
4600
4601         my $conf = load_config($vmid);
4602
4603         check_lock($conf) if !($skiplock || ($conf->{lock} && $conf->{lock} eq 'backup'));
4604
4605         vm_mon_cmd($vmid, "stop");
4606     });
4607 }
4608
4609 sub vm_resume {
4610     my ($vmid, $skiplock) = @_;
4611
4612     lock_config($vmid, sub {
4613
4614         my $conf = load_config($vmid);
4615
4616         check_lock($conf) if !($skiplock || ($conf->{lock} && $conf->{lock} eq 'backup'));
4617
4618         vm_mon_cmd($vmid, "cont");
4619     });
4620 }
4621
4622 sub vm_sendkey {
4623     my ($vmid, $skiplock, $key) = @_;
4624
4625     lock_config($vmid, sub {
4626
4627         my $conf = load_config($vmid);
4628
4629         # there is no qmp command, so we use the human monitor command
4630         vm_human_monitor_command($vmid, "sendkey $key");
4631     });
4632 }
4633
4634 sub vm_destroy {
4635     my ($storecfg, $vmid, $skiplock) = @_;
4636
4637     lock_config($vmid, sub {
4638
4639         my $conf = load_config($vmid);
4640
4641         check_lock($conf) if !$skiplock;
4642
4643         if (!check_running($vmid)) {
4644             destroy_vm($storecfg, $vmid);
4645         } else {
4646             die "VM $vmid is running - destroy failed\n";
4647         }
4648     });
4649 }
4650
4651 # pci helpers
4652
4653 sub file_write {
4654     my ($filename, $buf) = @_;
4655
4656     my $fh = IO::File->new($filename, "w");
4657     return undef if !$fh;
4658
4659     my $res = print $fh $buf;
4660
4661     $fh->close();
4662
4663     return $res;
4664 }
4665
4666 sub pci_device_info {
4667     my ($name) = @_;
4668
4669     my $res;
4670
4671     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
4672     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
4673
4674     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
4675     return undef if !defined($irq) || $irq !~ m/^\d+$/;
4676
4677     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
4678     return undef if !defined($vendor) || $vendor !~ s/^0x//;
4679
4680     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
4681     return undef if !defined($product) || $product !~ s/^0x//;
4682
4683     $res = {
4684         name => $name,
4685         vendor => $vendor,
4686         product => $product,
4687         domain => $domain,
4688         bus => $bus,
4689         slot => $slot,
4690         func => $func,
4691         irq => $irq,
4692         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
4693     };
4694
4695     return $res;
4696 }
4697
4698 sub pci_dev_reset {
4699     my ($dev) = @_;
4700
4701     my $name = $dev->{name};
4702
4703     my $fn = "$pcisysfs/devices/$name/reset";
4704
4705     return file_write($fn, "1");
4706 }
4707
4708 sub pci_dev_bind_to_vfio {
4709     my ($dev) = @_;
4710
4711     my $name = $dev->{name};
4712
4713     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4714
4715     if (!-d $vfio_basedir) {
4716         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4717     }
4718     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4719
4720     my $testdir = "$vfio_basedir/$name";
4721     return 1 if -d $testdir;
4722
4723     my $data = "$dev->{vendor} $dev->{product}";
4724     return undef if !file_write("$vfio_basedir/new_id", $data);
4725
4726     my $fn = "$pcisysfs/devices/$name/driver/unbind";
4727     if (!file_write($fn, $name)) {
4728         return undef if -f $fn;
4729     }
4730
4731     $fn = "$vfio_basedir/bind";
4732     if (! -d $testdir) {
4733         return undef if !file_write($fn, $name);
4734     }
4735
4736     return -d $testdir;
4737 }
4738
4739 sub pci_dev_group_bind_to_vfio {
4740     my ($pciid) = @_;
4741
4742     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4743
4744     if (!-d $vfio_basedir) {
4745         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4746     }
4747     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4748
4749     # get IOMMU group devices
4750     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
4751       my @devs = grep /^0000:/, readdir($D);
4752     closedir($D);
4753
4754     foreach my $pciid (@devs) {
4755         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
4756
4757         # pci bridges, switches or root ports are not supported
4758         # they have a pci_bus subdirectory so skip them
4759         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
4760
4761         my $info = pci_device_info($1);
4762         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
4763     }
4764
4765     return 1;
4766 }
4767
4768 sub print_pci_addr {
4769     my ($id, $bridges) = @_;
4770
4771     my $res = '';
4772     my $devices = {
4773         piix3 => { bus => 0, addr => 1 },
4774         #addr2 : first videocard
4775         balloon0 => { bus => 0, addr => 3 },
4776         watchdog => { bus => 0, addr => 4 },
4777         scsihw0 => { bus => 0, addr => 5 },
4778         'pci.3' => { bus => 0, addr => 5 }, #can also be used for virtio-scsi-single bridge
4779         scsihw1 => { bus => 0, addr => 6 },
4780         ahci0 => { bus => 0, addr => 7 },
4781         qga0 => { bus => 0, addr => 8 },
4782         spice => { bus => 0, addr => 9 },
4783         virtio0 => { bus => 0, addr => 10 },
4784         virtio1 => { bus => 0, addr => 11 },
4785         virtio2 => { bus => 0, addr => 12 },
4786         virtio3 => { bus => 0, addr => 13 },
4787         virtio4 => { bus => 0, addr => 14 },
4788         virtio5 => { bus => 0, addr => 15 },
4789         hostpci0 => { bus => 0, addr => 16 },
4790         hostpci1 => { bus => 0, addr => 17 },
4791         net0 => { bus => 0, addr => 18 },
4792         net1 => { bus => 0, addr => 19 },
4793         net2 => { bus => 0, addr => 20 },
4794         net3 => { bus => 0, addr => 21 },
4795         net4 => { bus => 0, addr => 22 },
4796         net5 => { bus => 0, addr => 23 },
4797         vga1 => { bus => 0, addr => 24 },
4798         vga2 => { bus => 0, addr => 25 },
4799         vga3 => { bus => 0, addr => 26 },
4800         hostpci2 => { bus => 0, addr => 27 },
4801         hostpci3 => { bus => 0, addr => 28 },
4802         #addr29 : usb-host (pve-usb.cfg)
4803         'pci.1' => { bus => 0, addr => 30 },
4804         'pci.2' => { bus => 0, addr => 31 },
4805         'net6' => { bus => 1, addr => 1 },
4806         'net7' => { bus => 1, addr => 2 },
4807         'net8' => { bus => 1, addr => 3 },
4808         'net9' => { bus => 1, addr => 4 },
4809         'net10' => { bus => 1, addr => 5 },
4810         'net11' => { bus => 1, addr => 6 },
4811         'net12' => { bus => 1, addr => 7 },
4812         'net13' => { bus => 1, addr => 8 },
4813         'net14' => { bus => 1, addr => 9 },
4814         'net15' => { bus => 1, addr => 10 },
4815         'net16' => { bus => 1, addr => 11 },
4816         'net17' => { bus => 1, addr => 12 },
4817         'net18' => { bus => 1, addr => 13 },
4818         'net19' => { bus => 1, addr => 14 },
4819         'net20' => { bus => 1, addr => 15 },
4820         'net21' => { bus => 1, addr => 16 },
4821         'net22' => { bus => 1, addr => 17 },
4822         'net23' => { bus => 1, addr => 18 },
4823         'net24' => { bus => 1, addr => 19 },
4824         'net25' => { bus => 1, addr => 20 },
4825         'net26' => { bus => 1, addr => 21 },
4826         'net27' => { bus => 1, addr => 22 },
4827         'net28' => { bus => 1, addr => 23 },
4828         'net29' => { bus => 1, addr => 24 },
4829         'net30' => { bus => 1, addr => 25 },
4830         'net31' => { bus => 1, addr => 26 },
4831         'virtio6' => { bus => 2, addr => 1 },
4832         'virtio7' => { bus => 2, addr => 2 },
4833         'virtio8' => { bus => 2, addr => 3 },
4834         'virtio9' => { bus => 2, addr => 4 },
4835         'virtio10' => { bus => 2, addr => 5 },
4836         'virtio11' => { bus => 2, addr => 6 },
4837         'virtio12' => { bus => 2, addr => 7 },
4838         'virtio13' => { bus => 2, addr => 8 },
4839         'virtio14' => { bus => 2, addr => 9 },
4840         'virtio15' => { bus => 2, addr => 10 },
4841         'virtioscsi0' => { bus => 3, addr => 1 },
4842         'virtioscsi1' => { bus => 3, addr => 2 },
4843         'virtioscsi2' => { bus => 3, addr => 3 },
4844         'virtioscsi3' => { bus => 3, addr => 4 },
4845         'virtioscsi4' => { bus => 3, addr => 5 },
4846         'virtioscsi5' => { bus => 3, addr => 6 },
4847         'virtioscsi6' => { bus => 3, addr => 7 },
4848         'virtioscsi7' => { bus => 3, addr => 8 },
4849         'virtioscsi8' => { bus => 3, addr => 9 },
4850         'virtioscsi9' => { bus => 3, addr => 10 },
4851         'virtioscsi10' => { bus => 3, addr => 11 },
4852         'virtioscsi11' => { bus => 3, addr => 12 },
4853         'virtioscsi12' => { bus => 3, addr => 13 },
4854         'virtioscsi13' => { bus => 3, addr => 14 },
4855         'virtioscsi14' => { bus => 3, addr => 15 },
4856         'virtioscsi15' => { bus => 3, addr => 16 },
4857         'virtioscsi16' => { bus => 3, addr => 17 },
4858         'virtioscsi17' => { bus => 3, addr => 18 },
4859         'virtioscsi18' => { bus => 3, addr => 19 },
4860         'virtioscsi19' => { bus => 3, addr => 20 },
4861         'virtioscsi20' => { bus => 3, addr => 21 },
4862         'virtioscsi21' => { bus => 3, addr => 22 },
4863         'virtioscsi22' => { bus => 3, addr => 23 },
4864         'virtioscsi23' => { bus => 3, addr => 24 },
4865         'virtioscsi24' => { bus => 3, addr => 25 },
4866         'virtioscsi25' => { bus => 3, addr => 26 },
4867         'virtioscsi26' => { bus => 3, addr => 27 },
4868         'virtioscsi27' => { bus => 3, addr => 28 },
4869         'virtioscsi28' => { bus => 3, addr => 29 },
4870         'virtioscsi29' => { bus => 3, addr => 30 },
4871         'virtioscsi30' => { bus => 3, addr => 31 },
4872
4873     };
4874
4875     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
4876            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
4877            my $bus = $devices->{$id}->{bus};
4878            $res = ",bus=pci.$bus,addr=$addr";
4879            $bridges->{$bus} = 1 if $bridges;
4880     }
4881     return $res;
4882
4883 }
4884
4885 sub print_pcie_addr {
4886     my ($id) = @_;
4887
4888     my $res = '';
4889     my $devices = {
4890         hostpci0 => { bus => "ich9-pcie-port-1", addr => 0 },
4891         hostpci1 => { bus => "ich9-pcie-port-2", addr => 0 },
4892         hostpci2 => { bus => "ich9-pcie-port-3", addr => 0 },
4893         hostpci3 => { bus => "ich9-pcie-port-4", addr => 0 },
4894     };
4895
4896     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
4897            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
4898            my $bus = $devices->{$id}->{bus};
4899            $res = ",bus=$bus,addr=$addr";
4900     }
4901     return $res;
4902
4903 }
4904
4905 # vzdump restore implementaion
4906
4907 sub tar_archive_read_firstfile {
4908     my $archive = shift;
4909
4910     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
4911
4912     # try to detect archive type first
4913     my $pid = open (TMP, "tar tf '$archive'|") ||
4914         die "unable to open file '$archive'\n";
4915     my $firstfile = <TMP>;
4916     kill 15, $pid;
4917     close TMP;
4918
4919     die "ERROR: archive contaions no data\n" if !$firstfile;
4920     chomp $firstfile;
4921
4922     return $firstfile;
4923 }
4924
4925 sub tar_restore_cleanup {
4926     my ($storecfg, $statfile) = @_;
4927
4928     print STDERR "starting cleanup\n";
4929
4930     if (my $fd = IO::File->new($statfile, "r")) {
4931         while (defined(my $line = <$fd>)) {
4932             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
4933                 my $volid = $2;
4934                 eval {
4935                     if ($volid =~ m|^/|) {
4936                         unlink $volid || die 'unlink failed\n';
4937                     } else {
4938                         PVE::Storage::vdisk_free($storecfg, $volid);
4939                     }
4940                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
4941                 };
4942                 print STDERR "unable to cleanup '$volid' - $@" if $@;
4943             } else {
4944                 print STDERR "unable to parse line in statfile - $line";
4945             }
4946         }
4947         $fd->close();
4948     }
4949 }
4950
4951 sub restore_archive {
4952     my ($archive, $vmid, $user, $opts) = @_;
4953
4954     my $format = $opts->{format};
4955     my $comp;
4956
4957     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
4958         $format = 'tar' if !$format;
4959         $comp = 'gzip';
4960     } elsif ($archive =~ m/\.tar$/) {
4961         $format = 'tar' if !$format;
4962     } elsif ($archive =~ m/.tar.lzo$/) {
4963         $format = 'tar' if !$format;
4964         $comp = 'lzop';
4965     } elsif ($archive =~ m/\.vma$/) {
4966         $format = 'vma' if !$format;
4967     } elsif ($archive =~ m/\.vma\.gz$/) {
4968         $format = 'vma' if !$format;
4969         $comp = 'gzip';
4970     } elsif ($archive =~ m/\.vma\.lzo$/) {
4971         $format = 'vma' if !$format;
4972         $comp = 'lzop';
4973     } else {
4974         $format = 'vma' if !$format; # default
4975     }
4976
4977     # try to detect archive format
4978     if ($format eq 'tar') {
4979         return restore_tar_archive($archive, $vmid, $user, $opts);
4980     } else {
4981         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
4982     }
4983 }
4984
4985 sub restore_update_config_line {
4986     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
4987
4988     return if $line =~ m/^\#qmdump\#/;
4989     return if $line =~ m/^\#vzdump\#/;
4990     return if $line =~ m/^lock:/;
4991     return if $line =~ m/^unused\d+:/;
4992     return if $line =~ m/^parent:/;
4993     return if $line =~ m/^template:/; # restored VM is never a template
4994
4995     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
4996         # try to convert old 1.X settings
4997         my ($id, $ind, $ethcfg) = ($1, $2, $3);
4998         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
4999             my ($model, $macaddr) = split(/\=/, $devconfig);
5000             $macaddr = PVE::Tools::random_ether_addr() if !$macaddr || $unique;
5001             my $net = {
5002                 model => $model,
5003                 bridge => "vmbr$ind",
5004                 macaddr => $macaddr,
5005             };
5006             my $netstr = print_net($net);
5007
5008             print $outfd "net$cookie->{netcount}: $netstr\n";
5009             $cookie->{netcount}++;
5010         }
5011     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5012         my ($id, $netstr) = ($1, $2);
5013         my $net = parse_net($netstr);
5014         $net->{macaddr} = PVE::Tools::random_ether_addr() if $net->{macaddr};
5015         $netstr = print_net($net);
5016         print $outfd "$id: $netstr\n";
5017     } elsif ($line =~ m/^((ide|scsi|virtio|sata)\d+):\s*(\S+)\s*$/) {
5018         my $virtdev = $1;
5019         my $value = $3;
5020         if ($line =~ m/backup=no/) {
5021             print $outfd "#$line";
5022         } elsif ($virtdev && $map->{$virtdev}) {
5023             my $di = parse_drive($virtdev, $value);
5024             delete $di->{format}; # format can change on restore
5025             $di->{file} = $map->{$virtdev};
5026             $value = print_drive($vmid, $di);
5027             print $outfd "$virtdev: $value\n";
5028         } else {
5029             print $outfd $line;
5030         }
5031     } else {
5032         print $outfd $line;
5033     }
5034 }
5035
5036 sub scan_volids {
5037     my ($cfg, $vmid) = @_;
5038
5039     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5040
5041     my $volid_hash = {};
5042     foreach my $storeid (keys %$info) {
5043         foreach my $item (@{$info->{$storeid}}) {
5044             next if !($item->{volid} && $item->{size});
5045             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5046             $volid_hash->{$item->{volid}} = $item;
5047         }
5048     }
5049
5050     return $volid_hash;
5051 }
5052
5053 sub get_used_paths {
5054     my ($vmid, $storecfg, $conf, $scan_snapshots, $skip_drive) = @_;
5055
5056     my $used_path = {};
5057
5058     my $scan_config = sub {
5059         my ($cref, $snapname) = @_;
5060
5061         foreach my $key (keys %$cref) {
5062             my $value = $cref->{$key};
5063             if (valid_drivename($key)) {
5064                 next if $skip_drive && $key eq $skip_drive;
5065                 my $drive = parse_drive($key, $value);
5066                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5067                 if ($drive->{file} =~ m!^/!) {
5068                     $used_path->{$drive->{file}}++; # = 1;
5069                 } else {
5070                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5071                     next if !$storeid;
5072                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5073                     next if !$scfg;
5074                     my $path = PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5075                     $used_path->{$path}++; # = 1;
5076                 }
5077             }
5078         }
5079     };
5080
5081     &$scan_config($conf);
5082
5083     undef $skip_drive;
5084
5085     if ($scan_snapshots) {
5086         foreach my $snapname (keys %{$conf->{snapshots}}) {
5087             &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5088         }
5089     }
5090
5091     return $used_path;
5092 }
5093
5094 sub update_disksize {
5095     my ($vmid, $conf, $volid_hash) = @_;
5096
5097     my $changes;
5098
5099     my $used = {};
5100
5101     # Note: it is allowed to define multiple storages with same path (alias), so
5102     # we need to check both 'volid' and real 'path' (two different volid can point
5103     # to the same path).
5104
5105     my $usedpath = {};
5106
5107     # update size info
5108     foreach my $opt (keys %$conf) {
5109         if (valid_drivename($opt)) {
5110             my $drive = parse_drive($opt, $conf->{$opt});
5111             my $volid = $drive->{file};
5112             next if !$volid;
5113
5114             $used->{$volid} = 1;
5115             if ($volid_hash->{$volid} &&
5116                 (my $path = $volid_hash->{$volid}->{path})) {
5117                 $usedpath->{$path} = 1;
5118             }
5119
5120             next if drive_is_cdrom($drive);
5121             next if !$volid_hash->{$volid};
5122
5123             $drive->{size} = $volid_hash->{$volid}->{size};
5124             my $new = print_drive($vmid, $drive);
5125             if ($new ne $conf->{$opt}) {
5126                 $changes = 1;
5127                 $conf->{$opt} = $new;
5128             }
5129         }
5130     }
5131
5132     # remove 'unusedX' entry if volume is used
5133     foreach my $opt (keys %$conf) {
5134         next if $opt !~ m/^unused\d+$/;
5135         my $volid = $conf->{$opt};
5136         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5137         if ($used->{$volid} || ($path && $usedpath->{$path})) {
5138             $changes = 1;
5139             delete $conf->{$opt};
5140         }
5141     }
5142
5143     foreach my $volid (sort keys %$volid_hash) {
5144         next if $volid =~ m/vm-$vmid-state-/;
5145         next if $used->{$volid};
5146         my $path = $volid_hash->{$volid}->{path};
5147         next if !$path; # just to be sure
5148         next if $usedpath->{$path};
5149         $changes = 1;
5150         add_unused_volume($conf, $volid);
5151         $usedpath->{$path} = 1; # avoid to add more than once (aliases)
5152     }
5153
5154     return $changes;
5155 }
5156
5157 sub rescan {
5158     my ($vmid, $nolock) = @_;
5159
5160     my $cfg = PVE::Cluster::cfs_read_file("storage.cfg");
5161
5162     my $volid_hash = scan_volids($cfg, $vmid);
5163
5164     my $updatefn =  sub {
5165         my ($vmid) = @_;
5166
5167         my $conf = load_config($vmid);
5168
5169         check_lock($conf);
5170
5171         my $vm_volids = {};
5172         foreach my $volid (keys %$volid_hash) {
5173             my $info = $volid_hash->{$volid};
5174             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5175         }
5176
5177         my $changes = update_disksize($vmid, $conf, $vm_volids);
5178
5179         update_config_nolock($vmid, $conf, 1) if $changes;
5180     };
5181
5182     if (defined($vmid)) {
5183         if ($nolock) {
5184             &$updatefn($vmid);
5185         } else {
5186             lock_config($vmid, $updatefn, $vmid);
5187         }
5188     } else {
5189         my $vmlist = config_list();
5190         foreach my $vmid (keys %$vmlist) {
5191             if ($nolock) {
5192                 &$updatefn($vmid);
5193             } else {
5194                 lock_config($vmid, $updatefn, $vmid);
5195             }
5196         }
5197     }
5198 }
5199
5200 sub restore_vma_archive {
5201     my ($archive, $vmid, $user, $opts, $comp) = @_;
5202
5203     my $input = $archive eq '-' ? "<&STDIN" : undef;
5204     my $readfrom = $archive;
5205
5206     my $uncomp = '';
5207     if ($comp) {
5208         $readfrom = '-';
5209         my $qarchive = PVE::Tools::shellquote($archive);
5210         if ($comp eq 'gzip') {
5211             $uncomp = "zcat $qarchive|";
5212         } elsif ($comp eq 'lzop') {
5213             $uncomp = "lzop -d -c $qarchive|";
5214         } else {
5215             die "unknown compression method '$comp'\n";
5216         }
5217
5218     }
5219
5220     my $tmpdir = "/var/tmp/vzdumptmp$$";
5221     rmtree $tmpdir;
5222
5223     # disable interrupts (always do cleanups)
5224     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5225         warn "got interrupt - ignored\n";
5226     };
5227
5228     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5229     POSIX::mkfifo($mapfifo, 0600);
5230     my $fifofh;
5231
5232     my $openfifo = sub {
5233         open($fifofh, '>', $mapfifo) || die $!;
5234     };
5235
5236     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5237
5238     my $oldtimeout;
5239     my $timeout = 5;
5240
5241     my $devinfo = {};
5242
5243     my $rpcenv = PVE::RPCEnvironment::get();
5244
5245     my $conffile = config_file($vmid);
5246     my $tmpfn = "$conffile.$$.tmp";
5247
5248     # Note: $oldconf is undef if VM does not exists
5249     my $oldconf = PVE::Cluster::cfs_read_file(cfs_config_path($vmid));
5250
5251     my $print_devmap = sub {
5252         my $virtdev_hash = {};
5253
5254         my $cfgfn = "$tmpdir/qemu-server.conf";
5255
5256         # we can read the config - that is already extracted
5257         my $fh = IO::File->new($cfgfn, "r") ||
5258             "unable to read qemu-server.conf - $!\n";
5259
5260         while (defined(my $line = <$fh>)) {
5261             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5262                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5263                 die "archive does not contain data for drive '$virtdev'\n"
5264                     if !$devinfo->{$devname};
5265                 if (defined($opts->{storage})) {
5266                     $storeid = $opts->{storage} || 'local';
5267                 } elsif (!$storeid) {
5268                     $storeid = 'local';
5269                 }
5270                 $format = 'raw' if !$format;
5271                 $devinfo->{$devname}->{devname} = $devname;
5272                 $devinfo->{$devname}->{virtdev} = $virtdev;
5273                 $devinfo->{$devname}->{format} = $format;
5274                 $devinfo->{$devname}->{storeid} = $storeid;
5275
5276                 # check permission on storage
5277                 my $pool = $opts->{pool}; # todo: do we need that?
5278                 if ($user ne 'root@pam') {
5279                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5280                 }
5281
5282                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5283             }
5284         }
5285
5286         foreach my $devname (keys %$devinfo) {
5287             die "found no device mapping information for device '$devname'\n"
5288                 if !$devinfo->{$devname}->{virtdev};
5289         }
5290
5291         my $cfg = cfs_read_file('storage.cfg');
5292
5293         # create empty/temp config
5294         if ($oldconf) {
5295             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5296             foreach_drive($oldconf, sub {
5297                 my ($ds, $drive) = @_;
5298
5299                 return if drive_is_cdrom($drive);
5300
5301                 my $volid = $drive->{file};
5302
5303                 return if !$volid || $volid =~ m|^/|;
5304
5305                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5306                 return if !$path || !$owner || ($owner != $vmid);
5307
5308                 # Note: only delete disk we want to restore
5309                 # other volumes will become unused
5310                 if ($virtdev_hash->{$ds}) {
5311                     PVE::Storage::vdisk_free($cfg, $volid);
5312                 }
5313             });
5314         }
5315
5316         my $map = {};
5317         foreach my $virtdev (sort keys %$virtdev_hash) {
5318             my $d = $virtdev_hash->{$virtdev};
5319             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5320             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5321
5322             # test if requested format is supported
5323             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5324             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5325             $d->{format} = $defFormat if !$supported;
5326
5327             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5328                                                   $d->{format}, undef, $alloc_size);
5329             print STDERR "new volume ID is '$volid'\n";
5330             $d->{volid} = $volid;
5331             my $path = PVE::Storage::path($cfg, $volid);
5332
5333             PVE::Storage::activate_volumes($cfg,[$volid]);
5334
5335             my $write_zeros = 1;
5336             # fixme: what other storages types initialize volumes with zero?
5337             if ($scfg->{type} eq 'dir' || $scfg->{type} eq 'nfs' || $scfg->{type} eq 'glusterfs' ||
5338                 $scfg->{type} eq 'sheepdog' || $scfg->{type} eq 'rbd') {
5339                 $write_zeros = 0;
5340             }
5341
5342             print $fifofh "${write_zeros}:$d->{devname}=$path\n";
5343
5344             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5345             $map->{$virtdev} = $volid;
5346         }
5347
5348         $fh->seek(0, 0) || die "seek failed - $!\n";
5349
5350         my $outfd = new IO::File ($tmpfn, "w") ||
5351             die "unable to write config for VM $vmid\n";
5352
5353         my $cookie = { netcount => 0 };
5354         while (defined(my $line = <$fh>)) {
5355             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5356         }
5357
5358         $fh->close();
5359         $outfd->close();
5360     };
5361
5362     eval {
5363         # enable interrupts
5364         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5365             die "interrupted by signal\n";
5366         };
5367         local $SIG{ALRM} = sub { die "got timeout\n"; };
5368
5369         $oldtimeout = alarm($timeout);
5370
5371         my $parser = sub {
5372             my $line = shift;
5373
5374             print "$line\n";
5375
5376             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5377                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5378                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5379             } elsif ($line =~ m/^CTIME: /) {
5380                 # we correctly received the vma config, so we can disable
5381                 # the timeout now for disk allocation (set to 10 minutes, so
5382                 # that we always timeout if something goes wrong)
5383                 alarm(600);
5384                 &$print_devmap();
5385                 print $fifofh "done\n";
5386                 my $tmp = $oldtimeout || 0;
5387                 $oldtimeout = undef;
5388                 alarm($tmp);
5389                 close($fifofh);
5390             }
5391         };
5392
5393         print "restore vma archive: $cmd\n";
5394         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5395     };
5396     my $err = $@;
5397
5398     alarm($oldtimeout) if $oldtimeout;
5399
5400     my $vollist = [];
5401     foreach my $devname (keys %$devinfo) {
5402         my $volid = $devinfo->{$devname}->{volid};
5403         push @$vollist, $volid if $volid;
5404     }
5405
5406     my $cfg = cfs_read_file('storage.cfg');
5407     PVE::Storage::deactivate_volumes($cfg, $vollist);
5408
5409     unlink $mapfifo;
5410
5411     if ($err) {
5412         rmtree $tmpdir;
5413         unlink $tmpfn;
5414
5415         foreach my $devname (keys %$devinfo) {
5416             my $volid = $devinfo->{$devname}->{volid};
5417             next if !$volid;
5418             eval {
5419                 if ($volid =~ m|^/|) {
5420                     unlink $volid || die 'unlink failed\n';
5421                 } else {
5422                     PVE::Storage::vdisk_free($cfg, $volid);
5423                 }
5424                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5425             };
5426             print STDERR "unable to cleanup '$volid' - $@" if $@;
5427         }
5428         die $err;
5429     }
5430
5431     rmtree $tmpdir;
5432
5433     rename($tmpfn, $conffile) ||
5434         die "unable to commit configuration file '$conffile'\n";
5435
5436     PVE::Cluster::cfs_update(); # make sure we read new file
5437
5438     eval { rescan($vmid, 1); };
5439     warn $@ if $@;
5440 }
5441
5442 sub restore_tar_archive {
5443     my ($archive, $vmid, $user, $opts) = @_;
5444
5445     if ($archive ne '-') {
5446         my $firstfile = tar_archive_read_firstfile($archive);
5447         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5448             if $firstfile ne 'qemu-server.conf';
5449     }
5450
5451     my $storecfg = cfs_read_file('storage.cfg');
5452
5453     # destroy existing data - keep empty config
5454     my $vmcfgfn = config_file($vmid);
5455     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5456
5457     my $tocmd = "/usr/lib/qemu-server/qmextract";
5458
5459     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5460     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5461     $tocmd .= ' --prealloc' if $opts->{prealloc};
5462     $tocmd .= ' --info' if $opts->{info};
5463
5464     # tar option "xf" does not autodetect compression when read from STDIN,
5465     # so we pipe to zcat
5466     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5467         PVE::Tools::shellquote("--to-command=$tocmd");
5468
5469     my $tmpdir = "/var/tmp/vzdumptmp$$";
5470     mkpath $tmpdir;
5471
5472     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5473     local $ENV{VZDUMP_VMID} = $vmid;
5474     local $ENV{VZDUMP_USER} = $user;
5475
5476     my $conffile = config_file($vmid);
5477     my $tmpfn = "$conffile.$$.tmp";
5478
5479     # disable interrupts (always do cleanups)
5480     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5481         print STDERR "got interrupt - ignored\n";
5482     };
5483
5484     eval {
5485         # enable interrupts
5486         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5487             die "interrupted by signal\n";
5488         };
5489
5490         if ($archive eq '-') {
5491             print "extracting archive from STDIN\n";
5492             run_command($cmd, input => "<&STDIN");
5493         } else {
5494             print "extracting archive '$archive'\n";
5495             run_command($cmd);
5496         }
5497
5498         return if $opts->{info};
5499
5500         # read new mapping
5501         my $map = {};
5502         my $statfile = "$tmpdir/qmrestore.stat";
5503         if (my $fd = IO::File->new($statfile, "r")) {
5504             while (defined (my $line = <$fd>)) {
5505                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5506                     $map->{$1} = $2 if $1;
5507                 } else {
5508                     print STDERR "unable to parse line in statfile - $line\n";
5509                 }
5510             }
5511             $fd->close();
5512         }
5513
5514         my $confsrc = "$tmpdir/qemu-server.conf";
5515
5516         my $srcfd = new IO::File($confsrc, "r") ||
5517             die "unable to open file '$confsrc'\n";
5518
5519         my $outfd = new IO::File ($tmpfn, "w") ||
5520             die "unable to write config for VM $vmid\n";
5521
5522         my $cookie = { netcount => 0 };
5523         while (defined (my $line = <$srcfd>)) {
5524             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5525         }
5526
5527         $srcfd->close();
5528         $outfd->close();
5529     };
5530     my $err = $@;
5531
5532     if ($err) {
5533
5534         unlink $tmpfn;
5535
5536         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
5537
5538         die $err;
5539     }
5540
5541     rmtree $tmpdir;
5542
5543     rename $tmpfn, $conffile ||
5544         die "unable to commit configuration file '$conffile'\n";
5545
5546     PVE::Cluster::cfs_update(); # make sure we read new file
5547
5548     eval { rescan($vmid, 1); };
5549     warn $@ if $@;
5550 };
5551
5552
5553 # Internal snapshots
5554
5555 # NOTE: Snapshot create/delete involves several non-atomic
5556 # action, and can take a long time.
5557 # So we try to avoid locking the file and use 'lock' variable
5558 # inside the config file instead.
5559
5560 my $snapshot_copy_config = sub {
5561     my ($source, $dest) = @_;
5562
5563     foreach my $k (keys %$source) {
5564         next if $k eq 'snapshots';
5565         next if $k eq 'snapstate';
5566         next if $k eq 'snaptime';
5567         next if $k eq 'vmstate';
5568         next if $k eq 'lock';
5569         next if $k eq 'digest';
5570         next if $k eq 'description';
5571         next if $k =~ m/^unused\d+$/;
5572
5573         $dest->{$k} = $source->{$k};
5574     }
5575 };
5576
5577 my $snapshot_apply_config = sub {
5578     my ($conf, $snap) = @_;
5579
5580     # copy snapshot list
5581     my $newconf = {
5582         snapshots => $conf->{snapshots},
5583     };
5584
5585     # keep description and list of unused disks
5586     foreach my $k (keys %$conf) {
5587         next if !($k =~ m/^unused\d+$/ || $k eq 'description');
5588         $newconf->{$k} = $conf->{$k};
5589     }
5590
5591     &$snapshot_copy_config($snap, $newconf);
5592
5593     return $newconf;
5594 };
5595
5596 sub foreach_writable_storage {
5597     my ($conf, $func) = @_;
5598
5599     my $sidhash = {};
5600
5601     foreach my $ds (keys %$conf) {
5602         next if !valid_drivename($ds);
5603
5604         my $drive = parse_drive($ds, $conf->{$ds});
5605         next if !$drive;
5606         next if drive_is_cdrom($drive);
5607
5608         my $volid = $drive->{file};
5609
5610         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5611         $sidhash->{$sid} = $sid if $sid;
5612     }
5613
5614     foreach my $sid (sort keys %$sidhash) {
5615         &$func($sid);
5616     }
5617 }
5618
5619 my $alloc_vmstate_volid = sub {
5620     my ($storecfg, $vmid, $conf, $snapname) = @_;
5621
5622     # Note: we try to be smart when selecting a $target storage
5623
5624     my $target;
5625
5626     # search shared storage first
5627     foreach_writable_storage($conf, sub {
5628         my ($sid) = @_;
5629         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
5630         return if !$scfg->{shared};
5631
5632         $target = $sid if !$target || $scfg->{path}; # prefer file based storage
5633     });
5634
5635     if (!$target) {
5636         # now search local storage
5637         foreach_writable_storage($conf, sub {
5638             my ($sid) = @_;
5639             my $scfg = PVE::Storage::storage_config($storecfg, $sid);
5640             return if $scfg->{shared};
5641
5642             $target = $sid if !$target || $scfg->{path}; # prefer file based storage;
5643         });
5644     }
5645
5646     $target = 'local' if !$target;
5647
5648     my $driver_state_size = 500; # assume 32MB is enough to safe all driver state;
5649     # we abort live save after $conf->{memory}, so we need at max twice that space
5650     my $size = $conf->{memory}*2 + $driver_state_size;
5651
5652     my $name = "vm-$vmid-state-$snapname";
5653     my $scfg = PVE::Storage::storage_config($storecfg, $target);
5654     $name .= ".raw" if $scfg->{path}; # add filename extension for file base storage
5655     my $volid = PVE::Storage::vdisk_alloc($storecfg, $target, $vmid, 'raw', $name, $size*1024);
5656
5657     return $volid;
5658 };
5659
5660 my $snapshot_prepare = sub {
5661     my ($vmid, $snapname, $save_vmstate, $comment) = @_;
5662
5663     my $snap;
5664
5665     my $updatefn =  sub {
5666
5667         my $conf = load_config($vmid);
5668
5669         die "you can't take a snapshot if it's a template\n"
5670             if is_template($conf);
5671
5672         check_lock($conf);
5673
5674         $conf->{lock} = 'snapshot';
5675
5676         die "snapshot name '$snapname' already used\n"
5677             if defined($conf->{snapshots}->{$snapname});
5678
5679         my $storecfg = PVE::Storage::config();
5680         die "snapshot feature is not available" if !has_feature('snapshot', $conf, $storecfg);
5681
5682         $snap = $conf->{snapshots}->{$snapname} = {};
5683
5684         if ($save_vmstate && check_running($vmid)) {
5685             $snap->{vmstate} = &$alloc_vmstate_volid($storecfg, $vmid, $conf, $snapname);
5686         }
5687
5688         &$snapshot_copy_config($conf, $snap);
5689
5690         $snap->{snapstate} = "prepare";
5691         $snap->{snaptime} = time();
5692         $snap->{description} = $comment if $comment;
5693
5694         # always overwrite machine if we save vmstate. This makes sure we
5695         # can restore it later using correct machine type
5696         $snap->{machine} = get_current_qemu_machine($vmid) if $snap->{vmstate};
5697
5698         update_config_nolock($vmid, $conf, 1);
5699     };
5700
5701     lock_config($vmid, $updatefn);
5702
5703     return $snap;
5704 };
5705
5706 my $snapshot_commit = sub {
5707     my ($vmid, $snapname) = @_;
5708
5709     my $updatefn = sub {
5710
5711         my $conf = load_config($vmid);
5712
5713         die "missing snapshot lock\n"
5714             if !($conf->{lock} && $conf->{lock} eq 'snapshot');
5715
5716         my $has_machine_config = defined($conf->{machine});
5717
5718         my $snap = $conf->{snapshots}->{$snapname};
5719
5720         die "snapshot '$snapname' does not exist\n" if !defined($snap);
5721
5722         die "wrong snapshot state\n"
5723             if !($snap->{snapstate} && $snap->{snapstate} eq "prepare");
5724
5725         delete $snap->{snapstate};
5726         delete $conf->{lock};
5727
5728         my $newconf = &$snapshot_apply_config($conf, $snap);
5729
5730         delete $newconf->{machine} if !$has_machine_config;
5731
5732         $newconf->{parent} = $snapname;
5733
5734         update_config_nolock($vmid, $newconf, 1);
5735     };
5736
5737     lock_config($vmid, $updatefn);
5738 };
5739
5740 sub snapshot_rollback {
5741     my ($vmid, $snapname) = @_;
5742
5743     my $prepare = 1;
5744
5745     my $storecfg = PVE::Storage::config();
5746
5747     my $conf = load_config($vmid);
5748
5749     my $get_snapshot_config = sub {
5750
5751         die "you can't rollback if vm is a template\n" if is_template($conf);
5752
5753         my $res = $conf->{snapshots}->{$snapname};
5754
5755         die "snapshot '$snapname' does not exist\n" if !defined($res);
5756
5757         return $res;
5758     };
5759
5760     my $snap = &$get_snapshot_config();
5761
5762     foreach_drive($snap, sub {
5763         my ($ds, $drive) = @_;
5764
5765         return if drive_is_cdrom($drive);
5766
5767         my $volid = $drive->{file};
5768
5769         PVE::Storage::volume_rollback_is_possible($storecfg, $volid, $snapname);
5770     });
5771
5772     my $updatefn = sub {
5773
5774         $conf = load_config($vmid);
5775
5776         $snap = &$get_snapshot_config();
5777
5778         die "unable to rollback to incomplete snapshot (snapstate = $snap->{snapstate})\n"
5779             if $snap->{snapstate};
5780
5781         if ($prepare) {
5782             check_lock($conf);
5783             vm_stop($storecfg, $vmid, undef, undef, 5, undef, undef);
5784         }
5785
5786         die "unable to rollback vm $vmid: vm is running\n"
5787             if check_running($vmid);
5788
5789         if ($prepare) {
5790             $conf->{lock} = 'rollback';
5791         } else {
5792             die "got wrong lock\n" if !($conf->{lock} && $conf->{lock} eq 'rollback');
5793             delete $conf->{lock};
5794         }
5795
5796         my $forcemachine;
5797
5798         if (!$prepare) {
5799             my $has_machine_config = defined($conf->{machine});
5800
5801             # copy snapshot config to current config
5802             $conf = &$snapshot_apply_config($conf, $snap);
5803             $conf->{parent} = $snapname;
5804
5805             # Note: old code did not store 'machine', so we try to be smart
5806             # and guess the snapshot was generated with kvm 1.4 (pc-i440fx-1.4).
5807             $forcemachine = $conf->{machine} || 'pc-i440fx-1.4';
5808             # we remove the 'machine' configuration if not explicitly specified
5809             # in the original config.
5810             delete $conf->{machine} if $snap->{vmstate} && !$has_machine_config;
5811         }
5812
5813         update_config_nolock($vmid, $conf, 1);
5814
5815         if (!$prepare && $snap->{vmstate}) {
5816             my $statefile = PVE::Storage::path($storecfg, $snap->{vmstate});
5817             vm_start($storecfg, $vmid, $statefile, undef, undef, undef, $forcemachine);
5818         }
5819     };
5820
5821     lock_config($vmid, $updatefn);
5822
5823     foreach_drive($snap, sub {
5824         my ($ds, $drive) = @_;
5825
5826         return if drive_is_cdrom($drive);
5827
5828         my $volid = $drive->{file};
5829         my $device = "drive-$ds";
5830
5831         PVE::Storage::volume_snapshot_rollback($storecfg, $volid, $snapname);
5832     });
5833
5834     $prepare = 0;
5835     lock_config($vmid, $updatefn);
5836 }
5837
5838 my $savevm_wait = sub {
5839     my ($vmid) = @_;
5840
5841     for(;;) {
5842         my $stat = vm_mon_cmd_nocheck($vmid, "query-savevm");
5843         if (!$stat->{status}) {
5844             die "savevm not active\n";
5845         } elsif ($stat->{status} eq 'active') {
5846             sleep(1);
5847             next;
5848         } elsif ($stat->{status} eq 'completed') {
5849             last;
5850         } else {
5851             die "query-savevm returned status '$stat->{status}'\n";
5852         }
5853     }
5854 };
5855
5856 sub do_snapshots_with_qemu {
5857     my ($storecfg, $volid) = @_;
5858
5859     my $storage_name = PVE::Storage::parse_volume_id($volid);
5860
5861     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
5862         && !$storecfg->{ids}->{$storage_name}->{krbd}){
5863         return 1;
5864     }
5865
5866     if ($volid =~ m/\.(qcow2|qed)$/){
5867         return 1;
5868     }
5869
5870     return undef;
5871 }
5872
5873 sub snapshot_create {
5874     my ($vmid, $snapname, $save_vmstate, $comment) = @_;
5875
5876     my $snap = &$snapshot_prepare($vmid, $snapname, $save_vmstate, $comment);
5877
5878     $save_vmstate = 0 if !$snap->{vmstate}; # vm is not running
5879
5880     my $config = load_config($vmid);
5881
5882     my $running = check_running($vmid);
5883
5884     my $freezefs = $running && $config->{agent};
5885     $freezefs = 0 if $snap->{vmstate}; # not needed if we save RAM
5886
5887     my $drivehash = {};
5888
5889     if ($freezefs) {
5890         eval { vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
5891         warn "guest-fsfreeze-freeze problems - $@" if $@;
5892     }
5893
5894     eval {
5895         # create internal snapshots of all drives
5896
5897         my $storecfg = PVE::Storage::config();
5898
5899         if ($running) {
5900             if ($snap->{vmstate}) {
5901                 my $path = PVE::Storage::path($storecfg, $snap->{vmstate});
5902                 vm_mon_cmd($vmid, "savevm-start", statefile => $path);
5903                 &$savevm_wait($vmid);
5904             } else {
5905                 vm_mon_cmd($vmid, "savevm-start");
5906             }
5907         };
5908
5909         foreach_drive($snap, sub {
5910             my ($ds, $drive) = @_;
5911
5912             return if drive_is_cdrom($drive);
5913
5914             my $volid = $drive->{file};
5915             my $device = "drive-$ds";
5916
5917             qemu_volume_snapshot($vmid, $device, $storecfg, $volid, $snapname);
5918             $drivehash->{$ds} = 1;
5919        });
5920     };
5921     my $err = $@;
5922
5923     if ($running) {
5924         eval { vm_mon_cmd($vmid, "savevm-end")  };
5925         warn $@ if $@;
5926
5927         if ($freezefs) {
5928             eval { vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
5929             warn "guest-fsfreeze-thaw problems - $@" if $@;
5930         }
5931
5932         # savevm-end is async, we need to wait
5933         for (;;) {
5934             my $stat = vm_mon_cmd_nocheck($vmid, "query-savevm");
5935             if (!$stat->{bytes}) {
5936                 last;
5937             } else {
5938                 print "savevm not yet finished\n";
5939                 sleep(1);
5940                 next;
5941             }
5942         }
5943     }
5944
5945     if ($err) {
5946         warn "snapshot create failed: starting cleanup\n";
5947         eval { snapshot_delete($vmid, $snapname, 0, $drivehash); };
5948         warn $@ if $@;
5949         die $err;
5950     }
5951
5952     &$snapshot_commit($vmid, $snapname);
5953 }
5954
5955 # Note: $drivehash is only set when called from snapshot_create.
5956 sub snapshot_delete {
5957     my ($vmid, $snapname, $force, $drivehash) = @_;
5958
5959     my $prepare = 1;
5960
5961     my $snap;
5962     my $unused = [];
5963
5964     my $unlink_parent = sub {
5965         my ($confref, $new_parent) = @_;
5966
5967         if ($confref->{parent} && $confref->{parent} eq $snapname) {
5968             if ($new_parent) {
5969                 $confref->{parent} = $new_parent;
5970             } else {
5971                 delete $confref->{parent};
5972             }
5973         }
5974     };
5975
5976     my $updatefn =  sub {
5977         my ($remove_drive) = @_;
5978
5979         my $conf = load_config($vmid);
5980
5981         if (!$drivehash) {
5982             check_lock($conf);
5983             die "you can't delete a snapshot if vm is a template\n"
5984                 if is_template($conf);
5985         }
5986
5987         $snap = $conf->{snapshots}->{$snapname};
5988
5989         die "snapshot '$snapname' does not exist\n" if !defined($snap);
5990
5991         # remove parent refs
5992         if (!$prepare) {
5993             &$unlink_parent($conf, $snap->{parent});
5994             foreach my $sn (keys %{$conf->{snapshots}}) {
5995                 next if $sn eq $snapname;
5996                 &$unlink_parent($conf->{snapshots}->{$sn}, $snap->{parent});
5997             }
5998         }
5999
6000         if ($remove_drive) {
6001             if ($remove_drive eq 'vmstate') {
6002                 delete $snap->{$remove_drive};
6003             } else {
6004                 my $drive = parse_drive($remove_drive, $snap->{$remove_drive});
6005                 my $volid = $drive->{file};
6006                 delete $snap->{$remove_drive};
6007                 add_unused_volume($conf, $volid);
6008             }
6009         }
6010
6011         if ($prepare) {
6012             $snap->{snapstate} = 'delete';
6013         } else {
6014             delete $conf->{snapshots}->{$snapname};
6015             delete $conf->{lock} if $drivehash;
6016             foreach my $volid (@$unused) {
6017                 add_unused_volume($conf, $volid);
6018             }
6019         }
6020
6021         update_config_nolock($vmid, $conf, 1);
6022     };
6023
6024     lock_config($vmid, $updatefn);
6025
6026     # now remove vmstate file
6027
6028     my $storecfg = PVE::Storage::config();
6029
6030     if ($snap->{vmstate}) {
6031         eval {  PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
6032         if (my $err = $@) {
6033             die $err if !$force;
6034             warn $err;
6035         }
6036         # save changes (remove vmstate from snapshot)
6037         lock_config($vmid, $updatefn, 'vmstate') if !$force;
6038     };
6039
6040     # now remove all internal snapshots
6041     foreach_drive($snap, sub {
6042         my ($ds, $drive) = @_;
6043
6044         return if drive_is_cdrom($drive);
6045
6046         my $volid = $drive->{file};
6047         my $device = "drive-$ds";
6048
6049         if (!$drivehash || $drivehash->{$ds}) {
6050             eval { qemu_volume_snapshot_delete($vmid, $device, $storecfg, $volid, $snapname); };
6051             if (my $err = $@) {
6052                 die $err if !$force;
6053                 warn $err;
6054             }
6055         }
6056
6057         # save changes (remove drive fron snapshot)
6058         lock_config($vmid, $updatefn, $ds) if !$force;
6059         push @$unused, $volid;
6060     });
6061
6062     # now cleanup config
6063     $prepare = 0;
6064     lock_config($vmid, $updatefn);
6065 }
6066
6067 sub has_feature {
6068     my ($feature, $conf, $storecfg, $snapname, $running) = @_;
6069
6070     my $err;
6071     foreach_drive($conf, sub {
6072         my ($ds, $drive) = @_;
6073
6074         return if drive_is_cdrom($drive);
6075         my $volid = $drive->{file};
6076         $err = 1 if !PVE::Storage::volume_has_feature($storecfg, $feature, $volid, $snapname, $running);
6077     });
6078
6079     return $err ? 0 : 1;
6080 }
6081
6082 sub template_create {
6083     my ($vmid, $conf, $disk) = @_;
6084
6085     my $storecfg = PVE::Storage::config();
6086
6087     foreach_drive($conf, sub {
6088         my ($ds, $drive) = @_;
6089
6090         return if drive_is_cdrom($drive);
6091         return if $disk && $ds ne $disk;
6092
6093         my $volid = $drive->{file};
6094         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6095
6096         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6097         $drive->{file} = $voliddst;
6098         $conf->{$ds} = print_drive($vmid, $drive);
6099         update_config_nolock($vmid, $conf, 1);
6100     });
6101 }
6102
6103 sub is_template {
6104     my ($conf) = @_;
6105
6106     return 1 if defined $conf->{template} && $conf->{template} == 1;
6107 }
6108
6109 sub qemu_img_convert {
6110     my ($src_volid, $dst_volid, $size, $snapname) = @_;
6111
6112     my $storecfg = PVE::Storage::config();
6113     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6114     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6115
6116     if ($src_storeid && $dst_storeid) {
6117         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6118         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6119
6120         my $src_format = qemu_img_format($src_scfg, $src_volname);
6121         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6122
6123         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6124         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6125
6126         my $cmd = [];
6127         push @$cmd, '/usr/bin/qemu-img', 'convert', '-t', 'writeback', '-p', '-n';
6128         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
6129         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path, $dst_path;
6130
6131         my $parser = sub {
6132             my $line = shift;
6133             if($line =~ m/\((\S+)\/100\%\)/){
6134                 my $percent = $1;
6135                 my $transferred = int($size * $percent / 100);
6136                 my $remaining = $size - $transferred;
6137
6138                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6139             }
6140
6141         };
6142
6143         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6144         my $err = $@;
6145         die "copy failed: $err" if $err;
6146     }
6147 }
6148
6149 sub qemu_img_format {
6150     my ($scfg, $volname) = @_;
6151
6152     if ($scfg->{path} && $volname =~ m/\.(raw|cow|qcow|qcow2|qed|vmdk|cloop)$/) {
6153         return $1;
6154     } else {
6155         return "raw";
6156     }
6157 }
6158
6159 sub qemu_drive_mirror {
6160     my ($vmid, $drive, $dst_volid, $vmiddst) = @_;
6161
6162     my $storecfg = PVE::Storage::config();
6163     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6164
6165     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6166
6167     my $format = qemu_img_format($dst_scfg, $dst_volname);
6168
6169     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6170
6171     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $dst_path };
6172     $opts->{format} = $format if $format;
6173
6174     print "drive mirror is starting (scanning bitmap) : this step can take some minutes/hours, depend of disk size and storage speed\n";
6175
6176     eval {
6177     vm_mon_cmd($vmid, "drive-mirror", %$opts);
6178         while (1) {
6179             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6180             my $stat = @$stats[0];
6181             die "mirroring job seem to have die. Maybe do you have bad sectors?" if !$stat;
6182             die "error job is not mirroring" if $stat->{type} ne "mirror";
6183
6184             my $busy = $stat->{busy};
6185             my $ready = $stat->{ready};
6186
6187             if (my $total = $stat->{len}) {
6188                 my $transferred = $stat->{offset} || 0;
6189                 my $remaining = $total - $transferred;
6190                 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6191
6192                 print "transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6193             }
6194
6195
6196             if ($stat->{ready} eq 'true') {
6197
6198                 last if $vmiddst != $vmid;
6199
6200                 # try to switch the disk if source and destination are on the same guest
6201                 eval { vm_mon_cmd($vmid, "block-job-complete", device => "drive-$drive") };
6202                 last if !$@;
6203                 die $@ if $@ !~ m/cannot be completed/;
6204             }
6205             sleep 1;
6206         }
6207
6208
6209     };
6210     my $err = $@;
6211
6212     my $cancel_job = sub {
6213         vm_mon_cmd($vmid, "block-job-cancel", device => "drive-$drive");
6214         while (1) {
6215             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6216             my $stat = @$stats[0];
6217             last if !$stat;
6218             sleep 1;
6219         }
6220     };
6221
6222     if ($err) {
6223         eval { &$cancel_job(); };
6224         die "mirroring error: $err";
6225     }
6226
6227     if ($vmiddst != $vmid) {
6228         # if we clone a disk for a new target vm, we don't switch the disk
6229         &$cancel_job(); # so we call block-job-cancel
6230     }
6231 }
6232
6233 sub clone_disk {
6234     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6235         $newvmid, $storage, $format, $full, $newvollist) = @_;
6236
6237     my $newvolid;
6238
6239     if (!$full) {
6240         print "create linked clone of drive $drivename ($drive->{file})\n";
6241         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6242         push @$newvollist, $newvolid;
6243     } else {
6244         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6245         $storeid = $storage if $storage;
6246
6247         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6248         if (!$format) {
6249             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6250             $format = qemu_img_format($scfg, $volname);
6251         }
6252
6253         # test if requested format is supported - else use default
6254         my $supported = grep { $_ eq $format } @$validFormats;
6255         $format = $defFormat if !$supported;
6256
6257         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6258
6259         print "create full clone of drive $drivename ($drive->{file})\n";
6260         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $format, undef, ($size/1024));
6261         push @$newvollist, $newvolid;
6262
6263         PVE::Storage::activate_volumes($storecfg, $newvollist);
6264
6265         if (!$running || $snapname) {
6266             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname);
6267         } else {
6268             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid);
6269         }
6270     }
6271
6272     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6273
6274     my $disk = $drive;
6275     $disk->{format} = undef;
6276     $disk->{file} = $newvolid;
6277     $disk->{size} = $size;
6278
6279     return $disk;
6280 }
6281
6282 # this only works if VM is running
6283 sub get_current_qemu_machine {
6284     my ($vmid) = @_;
6285
6286     my $cmd = { execute => 'query-machines', arguments => {} };
6287     my $res = vm_qmp_command($vmid, $cmd);
6288
6289     my ($current, $default);
6290     foreach my $e (@$res) {
6291         $default = $e->{name} if $e->{'is-default'};
6292         $current = $e->{name} if $e->{'is-current'};
6293     }
6294
6295     # fallback to the default machine if current is not supported by qemu
6296     return $current || $default || 'pc';
6297 }
6298
6299 sub qemu_machine_feature_enabled {
6300     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6301
6302     my $current_major;
6303     my $current_minor;
6304
6305     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6306
6307         $current_major = $3;
6308         $current_minor = $4;
6309
6310     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6311
6312         $current_major = $1;
6313         $current_minor = $2;
6314     }
6315
6316     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
6317
6318
6319 }
6320
6321 sub lspci {
6322
6323     my $devices = {};
6324
6325     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6326             my (undef, $id, $function) = @_;
6327             my $res = { id => $id, function => $function};
6328             push @{$devices->{$id}}, $res;
6329     });
6330
6331     return $devices;
6332 }
6333
6334 sub vm_iothreads_list {
6335     my ($vmid) = @_;
6336
6337     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6338
6339     my $iothreads = {};
6340     foreach my $iothread (@$res) {
6341         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6342     }
6343
6344     return $iothreads;
6345 }
6346
6347 sub scsihw_infos {
6348     my ($conf, $drive) = @_;
6349
6350     my $maxdev = 0;
6351
6352     if ($conf->{scsihw} && ($conf->{scsihw} =~ m/^lsi/)) {
6353         $maxdev = 7;
6354     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6355         $maxdev = 1;
6356     } else {
6357         $maxdev = 256;
6358     }
6359
6360     my $controller = int($drive->{index} / $maxdev);
6361     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6362
6363     return ($maxdev, $controller, $controller_prefix);
6364 }
6365
6366 # bash completion helper
6367
6368 sub complete_backup_archives {
6369     my ($cmdname, $pname, $cvalue) = @_;
6370
6371     my $cfg = PVE::Storage::config();
6372
6373     my $storeid;
6374
6375     if ($cvalue =~ m/^([^:]+):/) {
6376         $storeid = $1;
6377     }
6378
6379     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6380
6381     my $res = [];
6382     foreach my $id (keys %$data) {
6383         foreach my $item (@{$data->{$id}}) {
6384             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6385             push @$res, $item->{volid} if defined($item->{volid});
6386         }
6387     }
6388
6389     return $res;
6390 }
6391
6392 my $complete_vmid_full = sub {
6393     my ($running) = @_;
6394
6395     my $idlist = vmstatus();
6396
6397     my $res = [];
6398
6399     foreach my $id (keys %$idlist) {
6400         my $d = $idlist->{$id};
6401         if (defined($running)) {
6402             next if $d->{template};
6403             next if $running && $d->{status} ne 'running';
6404             next if !$running && $d->{status} eq 'running';
6405         }
6406         push @$res, $id;
6407
6408     }
6409     return $res;
6410 };
6411
6412 sub complete_vmid {
6413     return &$complete_vmid_full();
6414 }
6415
6416 sub complete_vmid_stopped {
6417     return &$complete_vmid_full(0);
6418 }
6419
6420 sub complete_vmid_running {
6421     return &$complete_vmid_full(1);
6422 }
6423
6424 sub complete_storage {
6425
6426     my $cfg = PVE::Storage::config();
6427     my $ids = $cfg->{ids};
6428
6429     my $res = [];
6430     foreach my $sid (keys %$ids) {
6431         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6432         push @$res, $sid;
6433     }
6434
6435     return $res;
6436 }
6437
6438 1;