PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QMPClient;
  31 use PVE::RPCEnvironment;
  32 use Time::HiRes qw(gettimeofday);
  33 use File::Copy qw(copy);
  34
  35 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  36
  37 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  38
  39 # Note about locking: we use flock on the config file protect
  40 # against concurent actions.
  41 # Aditionaly, we have a 'lock' setting in the config file. This
  42 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  43 # allowed when such lock is set. But you can ignore this kind of
  44 # lock with the --skiplock flag.
  45
  46 cfs_register_file('/qemu-server/',
  47                   \&parse_vm_config,
  48                   \&write_vm_config);
  49
  50 PVE::JSONSchema::register_standard_option('skiplock', {
  51     description => "Ignore locks - only root is allowed to use this option.",
  52     type => 'boolean',
  53     optional => 1,
  54 });
  55
  56 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  57     description => "Some command save/restore state from this location.",
  58     type => 'string',
  59     maxLength => 128,
  60     optional => 1,
  61 });
  62
  63 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  64     description => "The name of the snapshot.",
  65     type => 'string', format => 'pve-configid',
  66     maxLength => 40,
  67 });
  68
  69 #no warnings 'redefine';
  70
  71 sub cgroups_write {
  72    my ($controller, $vmid, $option, $value) = @_;
  73
  74    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  75    PVE::ProcFSTools::write_proc_entry($path, $value);
  76
  77 }
  78
  79 my $nodename = PVE::INotify::nodename();
  80
  81 mkdir "/etc/pve/nodes/$nodename";
  82 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  83 mkdir $confdir;
  84
  85 my $var_run_tmpdir = "/var/run/qemu-server";
  86 mkdir $var_run_tmpdir;
  87
  88 my $lock_dir = "/var/lock/qemu-server";
  89 mkdir $lock_dir;
  90
  91 my $pcisysfs = "/sys/bus/pci";
  92
  93 my $confdesc = {
  94     onboot => {
  95         optional => 1,
  96         type => 'boolean',
  97         description => "Specifies whether a VM will be started during system bootup.",
  98         default => 0,
  99     },
 100     autostart => {
 101         optional => 1,
 102         type => 'boolean',
 103         description => "Automatic restart after crash (currently ignored).",
 104         default => 0,
 105     },
 106     hotplug => {
 107         optional => 1,
 108         type => 'string', format => 'pve-hotplug-features',
 109         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 110         default => 'network,disk,usb',
 111     },
 112     reboot => {
 113         optional => 1,
 114         type => 'boolean',
 115         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 116         default => 1,
 117     },
 118     lock => {
 119         optional => 1,
 120         type => 'string',
 121         description => "Lock/unlock the VM.",
 122         enum => [qw(migrate backup snapshot rollback)],
 123     },
 124     cpulimit => {
 125         optional => 1,
 126         type => 'number',
 127         description => "Limit of CPU usage. Note if the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 128         minimum => 0,
 129         maximum => 128,
 130         default => 0,
 131     },
 132     cpuunits => {
 133         optional => 1,
 134         type => 'integer',
 135         description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 136         minimum => 0,
 137         maximum => 500000,
 138         default => 1000,
 139     },
 140     memory => {
 141         optional => 1,
 142         type => 'integer',
 143         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 144         minimum => 16,
 145         default => 512,
 146     },
 147     balloon => {
 148         optional => 1,
 149         type => 'integer',
 150         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 151         minimum => 0,
 152     },
 153     shares => {
 154         optional => 1,
 155         type => 'integer',
 156         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 157         minimum => 0,
 158         maximum => 50000,
 159         default => 1000,
 160     },
 161     keyboard => {
 162         optional => 1,
 163         type => 'string',
 164         description => "Keybord layout for vnc server. Default is read from the datacenter configuration file.",
 165         enum => PVE::Tools::kvmkeymaplist(),
 166         default => 'en-us',
 167     },
 168     name => {
 169         optional => 1,
 170         type => 'string', format => 'dns-name',
 171         description => "Set a name for the VM. Only used on the configuration web interface.",
 172     },
 173     scsihw => {
 174         optional => 1,
 175         type => 'string',
 176         description => "scsi controller model",
 177         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 178         default => 'lsi',
 179     },
 180     description => {
 181         optional => 1,
 182         type => 'string',
 183         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 184     },
 185     ostype => {
 186         optional => 1,
 187         type => 'string',
 188         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 l24 l26 solaris)],
 189         description => <<EODESC,
 190 Used to enable special optimization/features for specific
 191 operating systems:
 192
 193 other  => unspecified OS
 194 wxp    => Microsoft Windows XP
 195 w2k    => Microsoft Windows 2000
 196 w2k3   => Microsoft Windows 2003
 197 w2k8   => Microsoft Windows 2008
 198 wvista => Microsoft Windows Vista
 199 win7   => Microsoft Windows 7
 200 win8   => Microsoft Windows 8/2012
 201 l24    => Linux 2.4 Kernel
 202 l26    => Linux 2.6/3.X Kernel
 203 solaris => solaris/opensolaris/openindiania kernel
 204
 205 other|l24|l26|solaris                       ... no special behaviour
 206 wxp|w2k|w2k3|w2k8|wvista|win7|win8  ... use --localtime switch
 207 EODESC
 208     },
 209     boot => {
 210         optional => 1,
 211         type => 'string',
 212         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 213         pattern => '[acdn]{1,4}',
 214         default => 'cdn',
 215     },
 216     bootdisk => {
 217         optional => 1,
 218         type => 'string', format => 'pve-qm-bootdisk',
 219         description => "Enable booting from specified disk.",
 220         pattern => '(ide|sata|scsi|virtio)\d+',
 221     },
 222     smp => {
 223         optional => 1,
 224         type => 'integer',
 225         description => "The number of CPUs. Please use option -sockets instead.",
 226         minimum => 1,
 227         default => 1,
 228     },
 229     sockets => {
 230         optional => 1,
 231         type => 'integer',
 232         description => "The number of CPU sockets.",
 233         minimum => 1,
 234         default => 1,
 235     },
 236     cores => {
 237         optional => 1,
 238         type => 'integer',
 239         description => "The number of cores per socket.",
 240         minimum => 1,
 241         default => 1,
 242     },
 243     numa => {
 244         optional => 1,
 245         type => 'boolean',
 246         description => "Enable/disable Numa.",
 247         default => 0,
 248     },
 249     vcpus => {
 250         optional => 1,
 251         type => 'integer',
 252         description => "Number of hotplugged vcpus.",
 253         minimum => 1,
 254         default => 0,
 255     },
 256     acpi => {
 257         optional => 1,
 258         type => 'boolean',
 259         description => "Enable/disable ACPI.",
 260         default => 1,
 261     },
 262     agent => {
 263         optional => 1,
 264         type => 'boolean',
 265         description => "Enable/disable Qemu GuestAgent.",
 266         default => 0,
 267     },
 268     kvm => {
 269         optional => 1,
 270         type => 'boolean',
 271         description => "Enable/disable KVM hardware virtualization.",
 272         default => 1,
 273     },
 274     tdf => {
 275         optional => 1,
 276         type => 'boolean',
 277         description => "Enable/disable time drift fix.",
 278         default => 0,
 279     },
 280     localtime => {
 281         optional => 1,
 282         type => 'boolean',
 283         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 284     },
 285     freeze => {
 286         optional => 1,
 287         type => 'boolean',
 288         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 289     },
 290     vga => {
 291         optional => 1,
 292         type => 'string',
 293         description => "Select VGA type. If you want to use high resolution modes (>= 1280x1024x16) then you should use option 'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and 'cirrur' for other OS types. Option 'qxl' enables the SPICE display sever. You can also run without any graphic card using a serial devive as terminal.",
 294         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 295     },
 296     watchdog => {
 297         optional => 1,
 298         type => 'string', format => 'pve-qm-watchdog',
 299         typetext => '[[model=]i6300esb|ib700] [,[action=]reset|shutdown|poweroff|pause|debug|none]',
 300         description => "Create a virtual hardware watchdog device.  Once enabled (by a guest action), the watchdog must be periodically polled by an agent inside the guest or else the guest will be restarted (or execute the action specified)",
 301     },
 302     startdate => {
 303         optional => 1,
 304         type => 'string',
 305         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 306         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 307         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 308         default => 'now',
 309     },
 310     startup =>  get_standard_option('pve-startup-order'),
 311     template => {
 312         optional => 1,
 313         type => 'boolean',
 314         description => "Enable/disable Template.",
 315         default => 0,
 316     },
 317     args => {
 318         optional => 1,
 319         type => 'string',
 320         description => <<EODESCR,
 321 Note: this option is for experts only. It allows you to pass arbitrary arguments to kvm, for example:
 322
 323 args: -no-reboot -no-hpet
 324 EODESCR
 325     },
 326     tablet => {
 327         optional => 1,
 328         type => 'boolean',
 329         default => 1,
 330         description => "Enable/disable the usb tablet device. This device is usually needed to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with normal VNC clients. If you're running lots of console-only guests on one host, you may consider disabling this to save some context switches. This is turned of by default if you use spice (vga=qxl).",
 331     },
 332     migrate_speed => {
 333         optional => 1,
 334         type => 'integer',
 335         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 336         minimum => 0,
 337         default => 0,
 338     },
 339     migrate_downtime => {
 340         optional => 1,
 341         type => 'number',
 342         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 343         minimum => 0,
 344         default => 0.1,
 345     },
 346     cdrom => {
 347         optional => 1,
 348         type => 'string', format => 'pve-qm-drive',
 349         typetext => 'volume',
 350         description => "This is an alias for option -ide2",
 351     },
 352     cpu => {
 353         optional => 1,
 354         description => "Emulated CPU type.",
 355         type => 'string',
 356         enum => [ qw(486 athlon pentium pentium2 pentium3 coreduo core2duo kvm32 kvm64 qemu32 qemu64 phenom Conroe Penryn Nehalem Westmere SandyBridge IvyBridge Haswell Broadwell Opteron_G1 Opteron_G2 Opteron_G3 Opteron_G4 Opteron_G5 host) ],
 357         default => 'kvm64',
 358     },
 359     parent => get_standard_option('pve-snapshot-name', {
 360         optional => 1,
 361         description => "Parent snapshot name. This is used internally, and should not be modified.",
 362     }),
 363     snaptime => {
 364         optional => 1,
 365         description => "Timestamp for snapshots.",
 366         type => 'integer',
 367         minimum => 0,
 368     },
 369     vmstate => {
 370         optional => 1,
 371         type => 'string', format => 'pve-volume-id',
 372         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 373     },
 374     machine => {
 375         description => "Specific the Qemu machine type.",
 376         type => 'string',
 377         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
 378         maxLength => 40,
 379         optional => 1,
 380     },
 381     smbios1 => {
 382         description => "Specify SMBIOS type 1 fields.",
 383         type => 'string', format => 'pve-qm-smbios1',
 384         typetext => "[manufacturer=str][,product=str][,version=str][,serial=str] [,uuid=uuid][,sku=str][,family=str]",
 385         maxLength => 256,
 386         optional => 1,
 387     },
 388     protection => {
 389         optional => 1,
 390         type => 'boolean',
 391         description => "Sets the protection flag of the VM. This will prevent the remove operation.",
 392         default => 0,
 393     },
 394     ovmf => {
 395         optional => 1,
 396         type => 'boolean',
 397         description => "Enable ovmf uefi roms.",
 398         default => 0,
 399     },
 400 };
 401
 402 # what about other qemu settings ?
 403 #cpu => 'string',
 404 #machine => 'string',
 405 #fda => 'file',
 406 #fdb => 'file',
 407 #mtdblock => 'file',
 408 #sd => 'file',
 409 #pflash => 'file',
 410 #snapshot => 'bool',
 411 #bootp => 'file',
 412 ##tftp => 'dir',
 413 ##smb => 'dir',
 414 #kernel => 'file',
 415 #append => 'string',
 416 #initrd => 'file',
 417 ##soundhw => 'string',
 418
 419 while (my ($k, $v) = each %$confdesc) {
 420     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 421 }
 422
 423 my $MAX_IDE_DISKS = 4;
 424 my $MAX_SCSI_DISKS = 14;
 425 my $MAX_VIRTIO_DISKS = 16;
 426 my $MAX_SATA_DISKS = 6;
 427 my $MAX_USB_DEVICES = 5;
 428 my $MAX_NETS = 32;
 429 my $MAX_UNUSED_DISKS = 8;
 430 my $MAX_HOSTPCI_DEVICES = 4;
 431 my $MAX_SERIAL_PORTS = 4;
 432 my $MAX_PARALLEL_PORTS = 3;
 433 my $MAX_NUMA = 8;
 434 my $MAX_MEM = 4194304;
 435 my $STATICMEM = 1024;
 436
 437 my $numadesc = {
 438     optional => 1,
 439     type => 'string', format => 'pve-qm-numanode',
 440     typetext => "cpus=<id[-id],memory=<mb>[[,hostnodes=<id[-id]>] [,policy=<preferred|bind|interleave>]]",
 441     description => "numa topology",
 442 };
 443 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 444
 445 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 446     $confdesc->{"numa$i"} = $numadesc;
 447 }
 448
 449 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 450                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 451                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 452 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 453
 454 my $netdesc = {
 455     optional => 1,
 456     type => 'string', format => 'pve-qm-net',
 457     typetext => "MODEL=XX:XX:XX:XX:XX:XX [,bridge=<dev>][,queues=<nbqueues>][,rate=<mbps>] [,tag=<vlanid>][,firewall=0|1],link_down=0|1]",
 458     description => <<EODESCR,
 459 Specify network devices.
 460
 461 MODEL is one of: $nic_model_list_txt
 462
 463 XX:XX:XX:XX:XX:XX should be an unique MAC address. This is
 464 automatically generated if not specified.
 465
 466 The bridge parameter can be used to automatically add the interface to a bridge device. The Proxmox VE standard bridge is called 'vmbr0'.
 467
 468 Option 'rate' is used to limit traffic bandwidth from and to this interface. It is specified as floating point number, unit is 'Megabytes per second'.
 469
 470 If you specify no bridge, we create a kvm 'user' (NATed) network device, which provides DHCP and DNS services. The following addresses are used:
 471
 472 10.0.2.2   Gateway
 473 10.0.2.3   DNS Server
 474 10.0.2.4   SMB Server
 475
 476 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 477
 478 EODESCR
 479 };
 480 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 481
 482 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 483     $confdesc->{"net$i"} = $netdesc;
 484 }
 485
 486 my $drivename_hash;
 487
 488 my $idedesc = {
 489     optional => 1,
 490     type => 'string', format => 'pve-qm-drive',
 491     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads] [,discard=ignore|on] [,serial=serial][,model=model]',
 492     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
 493 };
 494 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
 495
 496 my $scsidesc = {
 497     optional => 1,
 498     type => 'string', format => 'pve-qm-drive',
 499     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads] [,discard=ignore|on] [,iothread=on] [,queues=<nbqueues>] [,serial=serial]',
 500     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
 501 };
 502 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 503
 504 my $satadesc = {
 505     optional => 1,
 506     type => 'string', format => 'pve-qm-drive',
 507     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads]  [,discard=ignore|on] [,serial=serial]',
 508     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
 509 };
 510 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
 511
 512 my $virtiodesc = {
 513     optional => 1,
 514     type => 'string', format => 'pve-qm-drive',
 515     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads]  [,discard=ignore|on] [,iothread=on] [,serial=serial]',
 516     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
 517 };
 518 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
 519
 520 my $usbdesc = {
 521     optional => 1,
 522     type => 'string', format => 'pve-qm-usb-device',
 523     typetext => 'host=HOSTUSBDEVICE|spice',
 524     description => <<EODESCR,
 525 Configure an USB device (n is 0 to 4). This can be used to
 526 pass-through usb devices to the guest. HOSTUSBDEVICE syntax is:
 527
 528 'bus-port(.port)*' (decimal numbers) or
 529 'vendor_id:product_id' (hexadeciaml numbers)
 530
 531 You can use the 'lsusb -t' command to list existing usb devices.
 532
 533 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 534
 535 The value 'spice' can be used to add a usb redirection devices for spice.
 536
 537 EODESCR
 538 };
 539 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
 540
 541 my $hostpcidesc = {
 542         optional => 1,
 543         type => 'string', format => 'pve-qm-hostpci',
 544         typetext => "[host=]HOSTPCIDEVICE [,rombar=on|off] [,pcie=0|1] [,x-vga=on|off]",
 545         description => <<EODESCR,
 546 Map host pci devices. HOSTPCIDEVICE syntax is:
 547
 548 'bus:dev.func' (hexadecimal numbers)
 549
 550 You can us the 'lspci' command to list existing pci devices.
 551
 552 The 'rombar' option determines whether or not the device's ROM will be visible in the guest's memory map (default is 'on').
 553
 554 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 555
 556 Experimental: user reported problems with this option.
 557 EODESCR
 558 };
 559 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
 560
 561 my $serialdesc = {
 562         optional => 1,
 563         type => 'string',
 564         pattern => '(/dev/.+|socket)',
 565         description =>  <<EODESCR,
 566 Create a serial device inside the VM (n is 0 to 3), and pass through a host serial device (i.e. /dev/ttyS0), or create a unix socket on the host side (use 'qm terminal' to open a terminal connection).
 567
 568 Note: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
 569
 570 Experimental: user reported problems with this option.
 571 EODESCR
 572 };
 573
 574 my $paralleldesc= {
 575         optional => 1,
 576         type => 'string',
 577         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
 578         description =>  <<EODESCR,
 579 Map host parallel devices (n is 0 to 2).
 580
 581 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 582
 583 Experimental: user reported problems with this option.
 584 EODESCR
 585 };
 586
 587 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
 588     $confdesc->{"parallel$i"} = $paralleldesc;
 589 }
 590
 591 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
 592     $confdesc->{"serial$i"} = $serialdesc;
 593 }
 594
 595 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
 596     $confdesc->{"hostpci$i"} = $hostpcidesc;
 597 }
 598
 599 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
 600     $drivename_hash->{"ide$i"} = 1;
 601     $confdesc->{"ide$i"} = $idedesc;
 602 }
 603
 604 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
 605     $drivename_hash->{"sata$i"} = 1;
 606     $confdesc->{"sata$i"} = $satadesc;
 607 }
 608
 609 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
 610     $drivename_hash->{"scsi$i"} = 1;
 611     $confdesc->{"scsi$i"} = $scsidesc ;
 612 }
 613
 614 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
 615     $drivename_hash->{"virtio$i"} = 1;
 616     $confdesc->{"virtio$i"} = $virtiodesc;
 617 }
 618
 619 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
 620     $confdesc->{"usb$i"} = $usbdesc;
 621 }
 622
 623 my $unuseddesc = {
 624     optional => 1,
 625     type => 'string', format => 'pve-volume-id',
 626     description => "Reference to unused volumes.",
 627 };
 628
 629 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
 630     $confdesc->{"unused$i"} = $unuseddesc;
 631 }
 632
 633 my $kvm_api_version = 0;
 634
 635 sub kvm_version {
 636
 637     return $kvm_api_version if $kvm_api_version;
 638
 639     my $fh = IO::File->new("</dev/kvm") ||
 640         return 0;
 641
 642     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
 643         $kvm_api_version = $v;
 644     }
 645
 646     $fh->close();
 647
 648     return  $kvm_api_version;
 649 }
 650
 651 my $kvm_user_version;
 652
 653 sub kvm_user_version {
 654
 655     return $kvm_user_version if $kvm_user_version;
 656
 657     $kvm_user_version = 'unknown';
 658
 659     my $tmp = `kvm -help 2>/dev/null`;
 660
 661     if ($tmp =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
 662         $kvm_user_version = $2;
 663     }
 664
 665     return $kvm_user_version;
 666
 667 }
 668
 669 my $kernel_has_vhost_net = -c '/dev/vhost-net';
 670
 671 sub disknames {
 672     # order is important - used to autoselect boot disk
 673     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
 674             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
 675             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
 676             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))));
 677 }
 678
 679 sub valid_drivename {
 680     my $dev = shift;
 681
 682     return defined($drivename_hash->{$dev});
 683 }
 684
 685 sub option_exists {
 686     my $key = shift;
 687     return defined($confdesc->{$key});
 688 }
 689
 690 sub nic_models {
 691     return $nic_model_list;
 692 }
 693
 694 sub os_list_description {
 695
 696     return {
 697         other => 'Other',
 698         wxp => 'Windows XP',
 699         w2k => 'Windows 2000',
 700         w2k3 =>, 'Windows 2003',
 701         w2k8 => 'Windows 2008',
 702         wvista => 'Windows Vista',
 703         win7 => 'Windows 7',
 704         win8 => 'Windows 8/2012',
 705         l24 => 'Linux 2.4',
 706         l26 => 'Linux 2.6',
 707     };
 708 }
 709
 710 my $cdrom_path;
 711
 712 sub get_cdrom_path {
 713
 714     return  $cdrom_path if $cdrom_path;
 715
 716     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
 717     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
 718     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
 719 }
 720
 721 sub get_iso_path {
 722     my ($storecfg, $vmid, $cdrom) = @_;
 723
 724     if ($cdrom eq 'cdrom') {
 725         return get_cdrom_path();
 726     } elsif ($cdrom eq 'none') {
 727         return '';
 728     } elsif ($cdrom =~ m|^/|) {
 729         return $cdrom;
 730     } else {
 731         return PVE::Storage::path($storecfg, $cdrom);
 732     }
 733 }
 734
 735 # try to convert old style file names to volume IDs
 736 sub filename_to_volume_id {
 737     my ($vmid, $file, $media) = @_;
 738
 739     if (!($file eq 'none' || $file eq 'cdrom' ||
 740           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
 741
 742         return undef if $file =~ m|/|;
 743
 744         if ($media && $media eq 'cdrom') {
 745             $file = "local:iso/$file";
 746         } else {
 747             $file = "local:$vmid/$file";
 748         }
 749     }
 750
 751     return $file;
 752 }
 753
 754 sub verify_media_type {
 755     my ($opt, $vtype, $media) = @_;
 756
 757     return if !$media;
 758
 759     my $etype;
 760     if ($media eq 'disk') {
 761         $etype = 'images';
 762     } elsif ($media eq 'cdrom') {
 763         $etype = 'iso';
 764     } else {
 765         die "internal error";
 766     }
 767
 768     return if ($vtype eq $etype);
 769
 770     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
 771 }
 772
 773 sub cleanup_drive_path {
 774     my ($opt, $storecfg, $drive) = @_;
 775
 776     # try to convert filesystem paths to volume IDs
 777
 778     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
 779         ($drive->{file} !~ m|^/dev/.+|) &&
 780         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
 781         ($drive->{file} !~ m/^\d+$/)) {
 782         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
 783         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
 784         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
 785         verify_media_type($opt, $vtype, $drive->{media});
 786         $drive->{file} = $volid;
 787     }
 788
 789     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
 790 }
 791
 792 sub create_conf_nolock {
 793     my ($vmid, $settings) = @_;
 794
 795     my $filename = config_file($vmid);
 796
 797     die "configuration file '$filename' already exists\n" if -f $filename;
 798
 799     my $defaults = load_defaults();
 800
 801     $settings->{name} = "vm$vmid" if !$settings->{name};
 802     $settings->{memory} = $defaults->{memory} if !$settings->{memory};
 803
 804     my $data = '';
 805     foreach my $opt (keys %$settings) {
 806         next if !$confdesc->{$opt};
 807
 808         my $value = $settings->{$opt};
 809         next if !$value;
 810
 811         $data .= "$opt: $value\n";
 812     }
 813
 814     PVE::Tools::file_set_contents($filename, $data);
 815 }
 816
 817 sub parse_hotplug_features {
 818     my ($data) = @_;
 819
 820     my $res = {};
 821
 822     return $res if $data eq '0';
 823
 824     $data = $confdesc->{hotplug}->{default} if $data eq '1';
 825
 826     foreach my $feature (PVE::Tools::split_list($data)) {
 827         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
 828             $res->{$1} = 1;
 829         } else {
 830             warn "ignoring unknown hotplug feature '$feature'\n";
 831         }
 832     }
 833     return $res;
 834 }
 835
 836 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
 837 sub pve_verify_hotplug_features {
 838     my ($value, $noerr) = @_;
 839
 840     return $value if parse_hotplug_features($value);
 841
 842     return undef if $noerr;
 843
 844     die "unable to parse hotplug option\n";
 845 }
 846
 847 my $parse_size = sub {
 848     my ($value) = @_;
 849
 850     return undef if $value !~ m/^(\d+(\.\d+)?)([KMG])?$/;
 851     my ($size, $unit) = ($1, $3);
 852     if ($unit) {
 853         if ($unit eq 'K') {
 854             $size = $size * 1024;
 855         } elsif ($unit eq 'M') {
 856             $size = $size * 1024 * 1024;
 857         } elsif ($unit eq 'G') {
 858             $size = $size * 1024 * 1024 * 1024;
 859         }
 860     }
 861     return int($size);
 862 };
 863
 864 my $format_size = sub {
 865     my ($size) = @_;
 866
 867     $size = int($size);
 868
 869     my $kb = int($size/1024);
 870     return $size if $kb*1024 != $size;
 871
 872     my $mb = int($kb/1024);
 873     return "${kb}K" if $mb*1024 != $kb;
 874
 875     my $gb = int($mb/1024);
 876     return "${mb}M" if $gb*1024 != $mb;
 877
 878     return "${gb}G";
 879 };
 880
 881 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
 882 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
 883 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
 884 #        [,aio=native|threads][,discard=ignore|on][,iothread=on]
 885 #        [,serial=serial][,model=model]
 886
 887 sub parse_drive {
 888     my ($key, $data) = @_;
 889
 890     my $res = {};
 891
 892     # $key may be undefined - used to verify JSON parameters
 893     if (!defined($key)) {
 894         $res->{interface} = 'unknown'; # should not harm when used to verify parameters
 895         $res->{index} = 0;
 896     } elsif ($key =~ m/^([^\d]+)(\d+)$/) {
 897         $res->{interface} = $1;
 898         $res->{index} = $2;
 899     } else {
 900         return undef;
 901     }
 902
 903     foreach my $p (split (/,/, $data)) {
 904         next if $p =~ m/^\s*$/;
 905
 906         if ($p =~ m/^(file|volume|cyls|heads|secs|trans|media|snapshot|cache|format|rerror|werror|backup|aio|bps|mbps|mbps_max|bps_rd|mbps_rd|mbps_rd_max|bps_wr|mbps_wr|mbps_wr_max|iops|iops_max|iops_rd|iops_rd_max|iops_wr|iops_wr_max|size|discard|iothread|queues|serial|model)=(.+)$/) {
 907             my ($k, $v) = ($1, $2);
 908
 909             $k = 'file' if $k eq 'volume';
 910
 911             return undef if defined $res->{$k};
 912
 913             if ($k eq 'bps' || $k eq 'bps_rd' || $k eq 'bps_wr') {
 914                 return undef if !$v || $v !~ m/^\d+/;
 915                 $k = "m$k";
 916                 $v = sprintf("%.3f", $v / (1024*1024));
 917             }
 918             $res->{$k} = $v;
 919         } else {
 920             if (!$res->{file} && $p !~ m/=/) {
 921                 $res->{file} = $p;
 922             } else {
 923                 return undef;
 924             }
 925         }
 926     }
 927
 928     return undef if !$res->{file};
 929
 930     return undef if $res->{cache} &&
 931         $res->{cache} !~ m/^(off|none|writethrough|writeback|unsafe|directsync)$/;
 932     return undef if $res->{snapshot} && $res->{snapshot} !~ m/^(on|off)$/;
 933     return undef if $res->{cyls} && $res->{cyls} !~ m/^\d+$/;
 934     return undef if $res->{heads} && $res->{heads} !~ m/^\d+$/;
 935     return undef if $res->{secs} && $res->{secs} !~ m/^\d+$/;
 936     return undef if $res->{media} && $res->{media} !~ m/^(disk|cdrom)$/;
 937     return undef if $res->{trans} && $res->{trans} !~ m/^(none|lba|auto)$/;
 938     return undef if $res->{format} && $res->{format} !~ m/^(raw|cow|qcow|qed|qcow2|vmdk|cloop)$/;
 939     return undef if $res->{rerror} && $res->{rerror} !~ m/^(ignore|report|stop)$/;
 940     return undef if $res->{werror} && $res->{werror} !~ m/^(enospc|ignore|report|stop)$/;
 941     return undef if $res->{backup} && $res->{backup} !~ m/^(yes|no)$/;
 942     return undef if $res->{aio} && $res->{aio} !~ m/^(native|threads)$/;
 943     return undef if $res->{discard} && $res->{discard} !~ m/^(ignore|on)$/;
 944     return undef if $res->{iothread} && $res->{iothread} !~ m/^(on)$/;
 945     return undef if $res->{queues} && ($res->{queues} !~ m/^\d+$/ || $res->{queues} < 2);
 946
 947     return undef if $res->{mbps_rd} && $res->{mbps};
 948     return undef if $res->{mbps_wr} && $res->{mbps};
 949
 950     return undef if $res->{mbps} && $res->{mbps} !~ m/^\d+(\.\d+)?$/;
 951     return undef if $res->{mbps_max} && $res->{mbps_max} !~ m/^\d+(\.\d+)?$/;
 952     return undef if $res->{mbps_rd} && $res->{mbps_rd} !~ m/^\d+(\.\d+)?$/;
 953     return undef if $res->{mbps_rd_max} && $res->{mbps_rd_max} !~ m/^\d+(\.\d+)?$/;
 954     return undef if $res->{mbps_wr} && $res->{mbps_wr} !~ m/^\d+(\.\d+)?$/;
 955     return undef if $res->{mbps_wr_max} && $res->{mbps_wr_max} !~ m/^\d+(\.\d+)?$/;
 956
 957     return undef if $res->{iops_rd} && $res->{iops};
 958     return undef if $res->{iops_wr} && $res->{iops};
 959
 960
 961     return undef if $res->{iops} && $res->{iops} !~ m/^\d+$/;
 962     return undef if $res->{iops_max} && $res->{iops_max} !~ m/^\d+$/;
 963     return undef if $res->{iops_rd} && $res->{iops_rd} !~ m/^\d+$/;
 964     return undef if $res->{iops_rd_max} && $res->{iops_rd_max} !~ m/^\d+$/;
 965     return undef if $res->{iops_wr} && $res->{iops_wr} !~ m/^\d+$/;
 966     return undef if $res->{iops_wr_max} && $res->{iops_wr_max} !~ m/^\d+$/;
 967
 968     if ($res->{size}) {
 969         return undef if !defined($res->{size} = &$parse_size($res->{size}));
 970     }
 971
 972     if ($res->{media} && ($res->{media} eq 'cdrom')) {
 973         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
 974         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
 975         return undef if $res->{interface} eq 'virtio';
 976     }
 977
 978     # rerror does not work with scsi drives
 979     if ($res->{rerror}) {
 980         return undef if $res->{interface} eq 'scsi';
 981     }
 982
 983     return $res;
 984 }
 985
 986 my @qemu_drive_options = qw(heads secs cyls trans media format cache snapshot rerror werror aio discard iops iops_rd iops_wr iops_max iops_rd_max iops_wr_max serial);
 987
 988 sub print_drive {
 989     my ($vmid, $drive) = @_;
 990
 991     my $opts = '';
 992     foreach my $o (@qemu_drive_options, 'mbps', 'mbps_rd', 'mbps_wr', 'mbps_max', 'mbps_rd_max', 'mbps_wr_max', 'backup', 'iothread', 'queues') {
 993         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
 994     }
 995
 996     if ($drive->{size}) {
 997         $opts .= ",size=" . &$format_size($drive->{size});
 998     }
 999
1000     if (my $model = $drive->{model}) {
1001         $opts .= ",model=$model";
1002     }
1003
1004     return "$drive->{file}$opts";
1005 }
1006
1007 sub scsi_inquiry {
1008     my($fh, $noerr) = @_;
1009
1010     my $SG_IO = 0x2285;
1011     my $SG_GET_VERSION_NUM = 0x2282;
1012
1013     my $versionbuf = "\x00" x 8;
1014     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1015     if (!$ret) {
1016         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1017         return undef;
1018     }
1019     my $version = unpack("I", $versionbuf);
1020     if ($version < 30000) {
1021         die "scsi generic interface too old\n"  if !$noerr;
1022         return undef;
1023     }
1024
1025     my $buf = "\x00" x 36;
1026     my $sensebuf = "\x00" x 8;
1027     my $cmd = pack("C x3 C x1", 0x12, 36);
1028
1029     # see /usr/include/scsi/sg.h
1030     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1031
1032     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1033                       length($sensebuf), 0, length($buf), $buf,
1034                       $cmd, $sensebuf, 6000);
1035
1036     $ret = ioctl($fh, $SG_IO, $packet);
1037     if (!$ret) {
1038         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1039         return undef;
1040     }
1041
1042     my @res = unpack($sg_io_hdr_t, $packet);
1043     if ($res[17] || $res[18]) {
1044         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1045         return undef;
1046     }
1047
1048     my $res = {};
1049     (my $byte0, my $byte1, $res->{vendor},
1050      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1051
1052     $res->{removable} = $byte1 & 128 ? 1 : 0;
1053     $res->{type} = $byte0 & 31;
1054
1055     return $res;
1056 }
1057
1058 sub path_is_scsi {
1059     my ($path) = @_;
1060
1061     my $fh = IO::File->new("+<$path") || return undef;
1062     my $res = scsi_inquiry($fh, 1);
1063     close($fh);
1064
1065     return $res;
1066 }
1067
1068 sub machine_type_is_q35 {
1069     my ($conf) = @_;
1070
1071     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1072 }
1073
1074 sub print_tabletdevice_full {
1075     my ($conf) = @_;
1076
1077     my $q35 = machine_type_is_q35($conf);
1078
1079     # we use uhci for old VMs because tablet driver was buggy in older qemu
1080     my $usbbus = $q35 ? "ehci" : "uhci";
1081
1082     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1083 }
1084
1085 sub print_drivedevice_full {
1086     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1087
1088     my $device = '';
1089     my $maxdev = 0;
1090
1091     if ($drive->{interface} eq 'virtio') {
1092         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1093         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1094         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1095     } elsif ($drive->{interface} eq 'scsi') {
1096
1097         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1098         my $unit = $drive->{index} % $maxdev;
1099         my $devicetype = 'hd';
1100         my $path = '';
1101         if (drive_is_cdrom($drive)) {
1102               $devicetype = 'cd';
1103         } else {
1104               if ($drive->{file} =~ m|^/|) {
1105                   $path = $drive->{file};
1106               } else {
1107                   $path = PVE::Storage::path($storecfg, $drive->{file});
1108               }
1109
1110               if($path =~ m/^iscsi\:\/\//){
1111                   $devicetype = 'generic';
1112               } else {
1113                   if (my $info = path_is_scsi($path)) {
1114                       if ($info->{type} == 0) {
1115                           $devicetype = 'block';
1116                       } elsif ($info->{type} == 1) { # tape
1117                           $devicetype = 'generic';
1118                       }
1119                   }
1120               }
1121          }
1122
1123         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1124             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1125         } else {
1126             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1127         }
1128
1129     } elsif ($drive->{interface} eq 'ide'){
1130         $maxdev = 2;
1131         my $controller = int($drive->{index} / $maxdev);
1132         my $unit = $drive->{index} % $maxdev;
1133         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1134
1135         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1136         if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
1137             $device .= ",model=$model";
1138         }
1139     } elsif ($drive->{interface} eq 'sata'){
1140         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1141         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1142         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1143     } elsif ($drive->{interface} eq 'usb') {
1144         die "implement me";
1145         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1146     } else {
1147         die "unsupported interface type";
1148     }
1149
1150     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1151
1152     return $device;
1153 }
1154
1155 sub get_initiator_name {
1156     my $initiator;
1157
1158     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1159     while (defined(my $line = <$fh>)) {
1160         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1161         $initiator = $1;
1162         last;
1163     }
1164     $fh->close();
1165
1166     return $initiator;
1167 }
1168
1169 sub print_drive_full {
1170     my ($storecfg, $vmid, $drive) = @_;
1171
1172     my $path;
1173     my $volid = $drive->{file};
1174     my $format;
1175
1176     if (drive_is_cdrom($drive)) {
1177         $path = get_iso_path($storecfg, $vmid, $volid);
1178     } else {
1179         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1180         if ($storeid) {
1181             $path = PVE::Storage::path($storecfg, $volid);
1182             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1183             $format = qemu_img_format($scfg, $volname);
1184         } else {
1185             $path = $volid;
1186         }
1187    }
1188
1189     my $opts = '';
1190     foreach my $o (@qemu_drive_options) {
1191         next if $o eq 'bootindex';
1192         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1193     }
1194
1195     $opts .= ",format=$format" if $format && !$drive->{format};
1196
1197     foreach my $o (qw(bps bps_rd bps_wr)) {
1198         my $v = $drive->{"m$o"};
1199         $opts .= ",$o=" . int($v*1024*1024) if $v;
1200     }
1201
1202     my $cache_direct = 0;
1203
1204     if (my $cache = $drive->{cache}) {
1205         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1206     } elsif (!drive_is_cdrom($drive)) {
1207         $opts .= ",cache=none";
1208         $cache_direct = 1;
1209     }
1210
1211     # aio native works only with O_DIRECT
1212     if (!$drive->{aio}) {
1213         if($cache_direct) {
1214             $opts .= ",aio=native";
1215         } else {
1216             $opts .= ",aio=threads";
1217         }
1218     }
1219
1220     my $detectzeroes = $drive->{discard} ? "unmap" : "on";
1221     $opts .= ",detect-zeroes=$detectzeroes" if !drive_is_cdrom($drive);
1222
1223     my $pathinfo = $path ? "file=$path," : '';
1224
1225     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1226 }
1227
1228 sub print_netdevice_full {
1229     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1230
1231     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1232
1233     my $device = $net->{model};
1234     if ($net->{model} eq 'virtio') {
1235          $device = 'virtio-net-pci';
1236      };
1237
1238     my $pciaddr = print_pci_addr("$netid", $bridges);
1239     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1240     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1241         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1242         my $vectors = $net->{queues} * 2 + 2;
1243         $tmpstr .= ",vectors=$vectors,mq=on";
1244     }
1245     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1246
1247     if ($use_old_bios_files) {
1248         my $romfile;
1249         if ($device eq 'virtio-net-pci') {
1250             $romfile = 'pxe-virtio.rom';
1251         } elsif ($device eq 'e1000') {
1252             $romfile = 'pxe-e1000.rom';
1253         } elsif ($device eq 'ne2k') {
1254             $romfile = 'pxe-ne2k_pci.rom';
1255         } elsif ($device eq 'pcnet') {
1256             $romfile = 'pxe-pcnet.rom';
1257         } elsif ($device eq 'rtl8139') {
1258             $romfile = 'pxe-rtl8139.rom';
1259         }
1260         $tmpstr .= ",romfile=$romfile" if $romfile;
1261     }
1262
1263     return $tmpstr;
1264 }
1265
1266 sub print_netdev_full {
1267     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1268
1269     my $i = '';
1270     if ($netid =~ m/^net(\d+)$/) {
1271         $i = int($1);
1272     }
1273
1274     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1275
1276     my $ifname = "tap${vmid}i$i";
1277
1278     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1279     die "interface name '$ifname' is too long (max 15 character)\n"
1280         if length($ifname) >= 16;
1281
1282     my $vhostparam = '';
1283     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1284
1285     my $vmname = $conf->{name} || "vm$vmid";
1286
1287     my $netdev = "";
1288     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1289
1290     if ($net->{bridge}) {
1291         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1292     } else {
1293         $netdev = "type=user,id=$netid,hostname=$vmname";
1294     }
1295
1296     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1297
1298     return $netdev;
1299 }
1300
1301 sub drive_is_cdrom {
1302     my ($drive) = @_;
1303
1304     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1305
1306 }
1307
1308 sub parse_numa {
1309     my ($data) = @_;
1310
1311     my $res = {};
1312
1313     foreach my $kvp (split(/,/, $data)) {
1314
1315         if ($kvp =~ m/^memory=(\S+)$/) {
1316             $res->{memory} = $1;
1317         } elsif ($kvp =~ m/^policy=(preferred|bind|interleave)$/) {
1318             $res->{policy} = $1;
1319         } elsif ($kvp =~ m/^cpus=(\d+)(-(\d+))?$/) {
1320             $res->{cpus}->{start} = $1;
1321             $res->{cpus}->{end} = $3;
1322         } elsif ($kvp =~ m/^hostnodes=(\d+)(-(\d+))?$/) {
1323             $res->{hostnodes}->{start} = $1;
1324             $res->{hostnodes}->{end} = $3;
1325         } else {
1326             return undef;
1327         }
1328     }
1329
1330     return $res;
1331 }
1332
1333 sub parse_hostpci {
1334     my ($value) = @_;
1335
1336     return undef if !$value;
1337
1338
1339     my @list = split(/,/, $value);
1340     my $found;
1341
1342     my $res = {};
1343     foreach my $kv (@list) {
1344
1345         if ($kv =~ m/^(host=)?([a-f0-9]{2}:[a-f0-9]{2})(\.([a-f0-9]))?$/) {
1346             $found = 1;
1347             if(defined($4)){
1348                 push @{$res->{pciid}}, { id => $2 , function => $4};
1349
1350             }else{
1351                 my $pcidevices = lspci($2);
1352                 $res->{pciid} = $pcidevices->{$2};
1353             }
1354         } elsif ($kv =~ m/^rombar=(on|off)$/) {
1355             $res->{rombar} = $1;
1356         } elsif ($kv =~ m/^x-vga=(on|off)$/) {
1357             $res->{'x-vga'} = $1;
1358         } elsif ($kv =~ m/^pcie=(\d+)$/) {
1359             $res->{pcie} = 1 if $1 == 1;
1360         } else {
1361             warn "unknown hostpci setting '$kv'\n";
1362         }
1363     }
1364
1365     return undef if !$found;
1366
1367     return $res;
1368 }
1369
1370 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1371 sub parse_net {
1372     my ($data) = @_;
1373
1374     my $res = {};
1375
1376     foreach my $kvp (split(/,/, $data)) {
1377
1378         if ($kvp =~ m/^(ne2k_pci|e1000|e1000-82540em|e1000-82544gc|e1000-82545em|rtl8139|pcnet|virtio|ne2k_isa|i82551|i82557b|i82559er|vmxnet3)(=([0-9a-f]{2}(:[0-9a-f]{2}){5}))?$/i) {
1379             my $model = lc($1);
1380             my $mac = defined($3) ? uc($3) : PVE::Tools::random_ether_addr();
1381             $res->{model} = $model;
1382             $res->{macaddr} = $mac;
1383         } elsif ($kvp =~ m/^bridge=(\S+)$/) {
1384             $res->{bridge} = $1;
1385         } elsif ($kvp =~ m/^queues=(\d+)$/) {
1386             $res->{queues} = $1;
1387         } elsif ($kvp =~ m/^rate=(\d+(\.\d+)?)$/) {
1388             $res->{rate} = $1;
1389         } elsif ($kvp =~ m/^tag=(\d+)$/) {
1390             $res->{tag} = $1;
1391         } elsif ($kvp =~ m/^firewall=([01])$/) {
1392             $res->{firewall} = $1;
1393         } elsif ($kvp =~ m/^link_down=([01])$/) {
1394             $res->{link_down} = $1;
1395         } else {
1396             return undef;
1397         }
1398
1399     }
1400
1401     return undef if !$res->{model};
1402
1403     return $res;
1404 }
1405
1406 sub print_net {
1407     my $net = shift;
1408
1409     my $res = "$net->{model}";
1410     $res .= "=$net->{macaddr}" if $net->{macaddr};
1411     $res .= ",bridge=$net->{bridge}" if $net->{bridge};
1412     $res .= ",rate=$net->{rate}" if $net->{rate};
1413     $res .= ",tag=$net->{tag}" if $net->{tag};
1414     $res .= ",firewall=1" if $net->{firewall};
1415     $res .= ",link_down=1" if $net->{link_down};
1416     $res .= ",queues=$net->{queues}" if $net->{queues};
1417
1418     return $res;
1419 }
1420
1421 sub add_random_macs {
1422     my ($settings) = @_;
1423
1424     foreach my $opt (keys %$settings) {
1425         next if $opt !~ m/^net(\d+)$/;
1426         my $net = parse_net($settings->{$opt});
1427         next if !$net;
1428         $settings->{$opt} = print_net($net);
1429     }
1430 }
1431
1432 sub add_unused_volume {
1433     my ($config, $volid) = @_;
1434
1435     my $key;
1436     for (my $ind = $MAX_UNUSED_DISKS - 1; $ind >= 0; $ind--) {
1437         my $test = "unused$ind";
1438         if (my $vid = $config->{$test}) {
1439             return if $vid eq $volid; # do not add duplicates
1440         } else {
1441             $key = $test;
1442         }
1443     }
1444
1445     die "To many unused volume - please delete them first.\n" if !$key;
1446
1447     $config->{$key} = $volid;
1448
1449     return $key;
1450 }
1451
1452 sub vm_is_volid_owner {
1453     my ($storecfg, $vmid, $volid) = @_;
1454
1455     if ($volid !~  m|^/|) {
1456         my ($path, $owner);
1457         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1458         if ($owner && ($owner == $vmid)) {
1459             return 1;
1460         }
1461     }
1462
1463     return undef;
1464 }
1465
1466 sub split_flagged_list {
1467     my $text = shift || '';
1468     $text =~ s/[,;]/ /g;
1469     $text =~ s/^\s+//;
1470     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
1471 }
1472
1473 sub join_flagged_list {
1474     my ($how, $lst) = @_;
1475     join $how, map { $lst->{$_} . $_ } keys %$lst;
1476 }
1477
1478 sub vmconfig_delete_pending_option {
1479     my ($conf, $key, $force) = @_;
1480
1481     delete $conf->{pending}->{$key};
1482     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1483     $pending_delete_hash->{$key} = $force ? '!' : '';
1484     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1485 }
1486
1487 sub vmconfig_undelete_pending_option {
1488     my ($conf, $key) = @_;
1489
1490     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1491     delete $pending_delete_hash->{$key};
1492
1493     if (%$pending_delete_hash) {
1494         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1495     } else {
1496         delete $conf->{pending}->{delete};
1497     }
1498 }
1499
1500 sub vmconfig_register_unused_drive {
1501     my ($storecfg, $vmid, $conf, $drive) = @_;
1502
1503     if (!drive_is_cdrom($drive)) {
1504         my $volid = $drive->{file};
1505         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1506             add_unused_volume($conf, $volid, $vmid);
1507         }
1508     }
1509 }
1510
1511 sub vmconfig_cleanup_pending {
1512     my ($conf) = @_;
1513
1514     # remove pending changes when nothing changed
1515     my $changes;
1516     foreach my $opt (keys %{$conf->{pending}}) {
1517         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
1518             $changes = 1;
1519             delete $conf->{pending}->{$opt};
1520         }
1521     }
1522
1523     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
1524     my $pending_delete_hash = {};
1525     while (my ($opt, $force) = each %$current_delete_hash) {
1526         if (defined($conf->{$opt})) {
1527             $pending_delete_hash->{$opt} = $force;
1528         } else {
1529             $changes = 1;
1530         }
1531     }
1532
1533     if (%$pending_delete_hash) {
1534         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1535     } else {
1536         delete $conf->{pending}->{delete};
1537     }
1538
1539     return $changes;
1540 }
1541
1542 my $valid_smbios1_options = {
1543     manufacturer => '\S+',
1544     product => '\S+',
1545     version => '\S+',
1546     serial => '\S+',
1547     uuid => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1548     sku => '\S+',
1549     family => '\S+',
1550 };
1551
1552 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
1553 sub parse_smbios1 {
1554     my ($data) = @_;
1555
1556     my $res = {};
1557
1558     foreach my $kvp (split(/,/, $data)) {
1559         return undef if $kvp !~ m/^(\S+)=(.+)$/;
1560         my ($k, $v) = split(/=/, $kvp);
1561         return undef if !defined($k) || !defined($v);
1562         return undef if !$valid_smbios1_options->{$k};
1563         return undef if $v !~ m/^$valid_smbios1_options->{$k}$/;
1564         $res->{$k} = $v;
1565     }
1566
1567     return $res;
1568 }
1569
1570 sub print_smbios1 {
1571     my ($smbios1) = @_;
1572
1573     my $data = '';
1574     foreach my $k (keys %$smbios1) {
1575         next if !defined($smbios1->{$k});
1576         next if !$valid_smbios1_options->{$k};
1577         $data .= ',' if $data;
1578         $data .= "$k=$smbios1->{$k}";
1579     }
1580     return $data;
1581 }
1582
1583 PVE::JSONSchema::register_format('pve-qm-smbios1', \&verify_smbios1);
1584 sub verify_smbios1 {
1585     my ($value, $noerr) = @_;
1586
1587     return $value if parse_smbios1($value);
1588
1589     return undef if $noerr;
1590
1591     die "unable to parse smbios (type 1) options\n";
1592 }
1593
1594 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
1595 sub verify_bootdisk {
1596     my ($value, $noerr) = @_;
1597
1598     return $value if valid_drivename($value);
1599
1600     return undef if $noerr;
1601
1602     die "invalid boot disk '$value'\n";
1603 }
1604
1605 PVE::JSONSchema::register_format('pve-qm-numanode', \&verify_numa);
1606 sub verify_numa {
1607     my ($value, $noerr) = @_;
1608
1609     return $value if parse_numa($value);
1610
1611     return undef if $noerr;
1612
1613     die "unable to parse numa options\n";
1614 }
1615
1616 PVE::JSONSchema::register_format('pve-qm-net', \&verify_net);
1617 sub verify_net {
1618     my ($value, $noerr) = @_;
1619
1620     return $value if parse_net($value);
1621
1622     return undef if $noerr;
1623
1624     die "unable to parse network options\n";
1625 }
1626
1627 PVE::JSONSchema::register_format('pve-qm-drive', \&verify_drive);
1628 sub verify_drive {
1629     my ($value, $noerr) = @_;
1630
1631     return $value if parse_drive(undef, $value);
1632
1633     return undef if $noerr;
1634
1635     die "unable to parse drive options\n";
1636 }
1637
1638 PVE::JSONSchema::register_format('pve-qm-hostpci', \&verify_hostpci);
1639 sub verify_hostpci {
1640     my ($value, $noerr) = @_;
1641
1642     return $value if parse_hostpci($value);
1643
1644     return undef if $noerr;
1645
1646     die "unable to parse pci id\n";
1647 }
1648
1649 PVE::JSONSchema::register_format('pve-qm-watchdog', \&verify_watchdog);
1650 sub verify_watchdog {
1651     my ($value, $noerr) = @_;
1652
1653     return $value if parse_watchdog($value);
1654
1655     return undef if $noerr;
1656
1657     die "unable to parse watchdog options\n";
1658 }
1659
1660 sub parse_watchdog {
1661     my ($value) = @_;
1662
1663     return undef if !$value;
1664
1665     my $res = {};
1666
1667     foreach my $p (split(/,/, $value)) {
1668         next if $p =~ m/^\s*$/;
1669
1670         if ($p =~ m/^(model=)?(i6300esb|ib700)$/) {
1671             $res->{model} = $2;
1672         } elsif ($p =~ m/^(action=)?(reset|shutdown|poweroff|pause|debug|none)$/) {
1673             $res->{action} = $2;
1674         } else {
1675             return undef;
1676         }
1677     }
1678
1679     return $res;
1680 }
1681
1682 sub parse_usb_device {
1683     my ($value) = @_;
1684
1685     return undef if !$value;
1686
1687     my @dl = split(/,/, $value);
1688     my $found;
1689
1690     my $res = {};
1691     foreach my $v (@dl) {
1692         if ($v =~ m/^host=(0x)?([0-9A-Fa-f]{4}):(0x)?([0-9A-Fa-f]{4})$/) {
1693             $found = 1;
1694             $res->{vendorid} = $2;
1695             $res->{productid} = $4;
1696         } elsif ($v =~ m/^host=(\d+)\-(\d+(\.\d+)*)$/) {
1697             $found = 1;
1698             $res->{hostbus} = $1;
1699             $res->{hostport} = $2;
1700         } elsif ($v =~ m/^spice$/) {
1701             $found = 1;
1702             $res->{spice} = 1;
1703         } else {
1704             return undef;
1705         }
1706     }
1707     return undef if !$found;
1708
1709     return $res;
1710 }
1711
1712 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
1713 sub verify_usb_device {
1714     my ($value, $noerr) = @_;
1715
1716     return $value if parse_usb_device($value);
1717
1718     return undef if $noerr;
1719
1720     die "unable to parse usb device\n";
1721 }
1722
1723 # add JSON properties for create and set function
1724 sub json_config_properties {
1725     my $prop = shift;
1726
1727     foreach my $opt (keys %$confdesc) {
1728         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
1729         $prop->{$opt} = $confdesc->{$opt};
1730     }
1731
1732     return $prop;
1733 }
1734
1735 sub check_type {
1736     my ($key, $value) = @_;
1737
1738     die "unknown setting '$key'\n" if !$confdesc->{$key};
1739
1740     my $type = $confdesc->{$key}->{type};
1741
1742     if (!defined($value)) {
1743         die "got undefined value\n";
1744     }
1745
1746     if ($value =~ m/[\n\r]/) {
1747         die "property contains a line feed\n";
1748     }
1749
1750     if ($type eq 'boolean') {
1751         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
1752         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
1753         die "type check ('boolean') failed - got '$value'\n";
1754     } elsif ($type eq 'integer') {
1755         return int($1) if $value =~ m/^(\d+)$/;
1756         die "type check ('integer') failed - got '$value'\n";
1757     } elsif ($type eq 'number') {
1758         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
1759         die "type check ('number') failed - got '$value'\n";
1760     } elsif ($type eq 'string') {
1761         if (my $fmt = $confdesc->{$key}->{format}) {
1762             if ($fmt eq 'pve-qm-drive') {
1763                 # special case - we need to pass $key to parse_drive()
1764                 my $drive = parse_drive($key, $value);
1765                 return $value if $drive;
1766                 die "unable to parse drive options\n";
1767             }
1768             PVE::JSONSchema::check_format($fmt, $value);
1769             return $value;
1770         }
1771         $value =~ s/^\"(.*)\"$/$1/;
1772         return $value;
1773     } else {
1774         die "internal error"
1775     }
1776 }
1777
1778 sub lock_config_full {
1779     my ($vmid, $timeout, $code, @param) = @_;
1780
1781     my $filename = config_file_lock($vmid);
1782
1783     my $res = lock_file($filename, $timeout, $code, @param);
1784
1785     die $@ if $@;
1786
1787     return $res;
1788 }
1789
1790 sub lock_config_mode {
1791     my ($vmid, $timeout, $shared, $code, @param) = @_;
1792
1793     my $filename = config_file_lock($vmid);
1794
1795     my $res = lock_file_full($filename, $timeout, $shared, $code, @param);
1796
1797     die $@ if $@;
1798
1799     return $res;
1800 }
1801
1802 sub lock_config {
1803     my ($vmid, $code, @param) = @_;
1804
1805     return lock_config_full($vmid, 10, $code, @param);
1806 }
1807
1808 sub cfs_config_path {
1809     my ($vmid, $node) = @_;
1810
1811     $node = $nodename if !$node;
1812     return "nodes/$node/qemu-server/$vmid.conf";
1813 }
1814
1815 sub check_iommu_support{
1816     #fixme : need to check IOMMU support
1817     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
1818
1819     my $iommu=1;
1820     return $iommu;
1821
1822 }
1823
1824 sub config_file {
1825     my ($vmid, $node) = @_;
1826
1827     my $cfspath = cfs_config_path($vmid, $node);
1828     return "/etc/pve/$cfspath";
1829 }
1830
1831 sub config_file_lock {
1832     my ($vmid) = @_;
1833
1834     return "$lock_dir/lock-$vmid.conf";
1835 }
1836
1837 sub touch_config {
1838     my ($vmid) = @_;
1839
1840     my $conf = config_file($vmid);
1841     utime undef, undef, $conf;
1842 }
1843
1844 sub destroy_vm {
1845     my ($storecfg, $vmid, $keep_empty_config) = @_;
1846
1847     my $conffile = config_file($vmid);
1848
1849     my $conf = load_config($vmid);
1850
1851     check_lock($conf);
1852
1853     # only remove disks owned by this VM
1854     foreach_drive($conf, sub {
1855         my ($ds, $drive) = @_;
1856
1857         return if drive_is_cdrom($drive);
1858
1859         my $volid = $drive->{file};
1860
1861         return if !$volid || $volid =~ m|^/|;
1862
1863         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
1864         return if !$path || !$owner || ($owner != $vmid);
1865
1866         PVE::Storage::vdisk_free($storecfg, $volid);
1867     });
1868
1869     if ($keep_empty_config) {
1870         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
1871     } else {
1872         unlink $conffile;
1873     }
1874
1875     # also remove unused disk
1876     eval {
1877         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
1878
1879         eval {
1880             PVE::Storage::foreach_volid($dl, sub {
1881                 my ($volid, $sid, $volname, $d) = @_;
1882                 PVE::Storage::vdisk_free($storecfg, $volid);
1883             });
1884         };
1885         warn $@ if $@;
1886
1887     };
1888     warn $@ if $@;
1889 }
1890
1891 sub load_config {
1892     my ($vmid, $node) = @_;
1893
1894     my $cfspath = cfs_config_path($vmid, $node);
1895
1896     my $conf = PVE::Cluster::cfs_read_file($cfspath);
1897
1898     die "no such VM ('$vmid')\n" if !defined($conf);
1899
1900     return $conf;
1901 }
1902
1903 sub parse_vm_config {
1904     my ($filename, $raw) = @_;
1905
1906     return undef if !defined($raw);
1907
1908     my $res = {
1909         digest => Digest::SHA::sha1_hex($raw),
1910         snapshots => {},
1911         pending => {},
1912     };
1913
1914     $filename =~ m|/qemu-server/(\d+)\.conf$|
1915         || die "got strange filename '$filename'";
1916
1917     my $vmid = $1;
1918
1919     my $conf = $res;
1920     my $descr;
1921     my $section = '';
1922
1923     my @lines = split(/\n/, $raw);
1924     foreach my $line (@lines) {
1925         next if $line =~ m/^\s*$/;
1926
1927         if ($line =~ m/^\[PENDING\]\s*$/i) {
1928             $section = 'pending';
1929             if (defined($descr)) {
1930                 $descr =~ s/\s+$//;
1931                 $conf->{description} = $descr;
1932             }
1933             $descr = undef;
1934             $conf = $res->{$section} = {};
1935             next;
1936
1937         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
1938             $section = $1;
1939             if (defined($descr)) {
1940                 $descr =~ s/\s+$//;
1941                 $conf->{description} = $descr;
1942             }
1943             $descr = undef;
1944             $conf = $res->{snapshots}->{$section} = {};
1945             next;
1946         }
1947
1948         if ($line =~ m/^\#(.*)\s*$/) {
1949             $descr = '' if !defined($descr);
1950             $descr .= PVE::Tools::decode_text($1) . "\n";
1951             next;
1952         }
1953
1954         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
1955             $descr = '' if !defined($descr);
1956             $descr .= PVE::Tools::decode_text($2);
1957         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
1958             $conf->{snapstate} = $1;
1959         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
1960             my $key = $1;
1961             my $value = $2;
1962             $conf->{$key} = $value;
1963         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
1964             my $value = $1;
1965             if ($section eq 'pending') {
1966                 $conf->{delete} = $value; # we parse this later
1967             } else {
1968                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
1969             }
1970         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
1971             my $key = $1;
1972             my $value = $2;
1973             eval { $value = check_type($key, $value); };
1974             if ($@) {
1975                 warn "vm $vmid - unable to parse value of '$key' - $@";
1976             } else {
1977                 my $fmt = $confdesc->{$key}->{format};
1978                 if ($fmt && $fmt eq 'pve-qm-drive') {
1979                     my $v = parse_drive($key, $value);
1980                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
1981                         $v->{file} = $volid;
1982                         $value = print_drive($vmid, $v);
1983                     } else {
1984                         warn "vm $vmid - unable to parse value of '$key'\n";
1985                         next;
1986                     }
1987                 }
1988
1989                 if ($key eq 'cdrom') {
1990                     $conf->{ide2} = $value;
1991                 } else {
1992                     $conf->{$key} = $value;
1993                 }
1994             }
1995         }
1996     }
1997
1998     if (defined($descr)) {
1999         $descr =~ s/\s+$//;
2000         $conf->{description} = $descr;
2001     }
2002     delete $res->{snapstate}; # just to be sure
2003
2004     return $res;
2005 }
2006
2007 sub write_vm_config {
2008     my ($filename, $conf) = @_;
2009
2010     delete $conf->{snapstate}; # just to be sure
2011
2012     if ($conf->{cdrom}) {
2013         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2014         $conf->{ide2} = $conf->{cdrom};
2015         delete $conf->{cdrom};
2016     }
2017
2018     # we do not use 'smp' any longer
2019     if ($conf->{sockets}) {
2020         delete $conf->{smp};
2021     } elsif ($conf->{smp}) {
2022         $conf->{sockets} = $conf->{smp};
2023         delete $conf->{cores};
2024         delete $conf->{smp};
2025     }
2026
2027     my $used_volids = {};
2028
2029     my $cleanup_config = sub {
2030         my ($cref, $pending, $snapname) = @_;
2031
2032         foreach my $key (keys %$cref) {
2033             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2034                 $key eq 'snapstate' || $key eq 'pending';
2035             my $value = $cref->{$key};
2036             if ($key eq 'delete') {
2037                 die "propertry 'delete' is only allowed in [PENDING]\n"
2038                     if !$pending;
2039                 # fixme: check syntax?
2040                 next;
2041             }
2042             eval { $value = check_type($key, $value); };
2043             die "unable to parse value of '$key' - $@" if $@;
2044
2045             $cref->{$key} = $value;
2046
2047             if (!$snapname && valid_drivename($key)) {
2048                 my $drive = parse_drive($key, $value);
2049                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2050             }
2051         }
2052     };
2053
2054     &$cleanup_config($conf);
2055
2056     &$cleanup_config($conf->{pending}, 1);
2057
2058     foreach my $snapname (keys %{$conf->{snapshots}}) {
2059         die "internal error" if $snapname eq 'pending';
2060         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2061     }
2062
2063     # remove 'unusedX' settings if we re-add a volume
2064     foreach my $key (keys %$conf) {
2065         my $value = $conf->{$key};
2066         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2067             delete $conf->{$key};
2068         }
2069     }
2070
2071     my $generate_raw_config = sub {
2072         my ($conf, $pending) = @_;
2073
2074         my $raw = '';
2075
2076         # add description as comment to top of file
2077         if (defined(my $descr = $conf->{description})) {
2078             if ($descr) {
2079                 foreach my $cl (split(/\n/, $descr)) {
2080                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2081                 }
2082             } else {
2083                 $raw .= "#\n" if $pending;
2084             }
2085         }
2086
2087         foreach my $key (sort keys %$conf) {
2088             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2089             $raw .= "$key: $conf->{$key}\n";
2090         }
2091         return $raw;
2092     };
2093
2094     my $raw = &$generate_raw_config($conf);
2095
2096     if (scalar(keys %{$conf->{pending}})){
2097         $raw .= "\n[PENDING]\n";
2098         $raw .= &$generate_raw_config($conf->{pending}, 1);
2099     }
2100
2101     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2102         $raw .= "\n[$snapname]\n";
2103         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2104     }
2105
2106     return $raw;
2107 }
2108
2109 sub update_config_nolock {
2110     my ($vmid, $conf, $skiplock) = @_;
2111
2112     check_lock($conf) if !$skiplock;
2113
2114     my $cfspath = cfs_config_path($vmid);
2115
2116     PVE::Cluster::cfs_write_file($cfspath, $conf);
2117 }
2118
2119 sub update_config {
2120     my ($vmid, $conf, $skiplock) = @_;
2121
2122     lock_config($vmid, &update_config_nolock, $conf, $skiplock);
2123 }
2124
2125 sub load_defaults {
2126
2127     my $res = {};
2128
2129     # we use static defaults from our JSON schema configuration
2130     foreach my $key (keys %$confdesc) {
2131         if (defined(my $default = $confdesc->{$key}->{default})) {
2132             $res->{$key} = $default;
2133         }
2134     }
2135
2136     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2137     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2138
2139     return $res;
2140 }
2141
2142 sub config_list {
2143     my $vmlist = PVE::Cluster::get_vmlist();
2144     my $res = {};
2145     return $res if !$vmlist || !$vmlist->{ids};
2146     my $ids = $vmlist->{ids};
2147
2148     foreach my $vmid (keys %$ids) {
2149         my $d = $ids->{$vmid};
2150         next if !$d->{node} || $d->{node} ne $nodename;
2151         next if !$d->{type} || $d->{type} ne 'qemu';
2152         $res->{$vmid}->{exists} = 1;
2153     }
2154     return $res;
2155 }
2156
2157 # test if VM uses local resources (to prevent migration)
2158 sub check_local_resources {
2159     my ($conf, $noerr) = @_;
2160
2161     my $loc_res = 0;
2162
2163     $loc_res = 1 if $conf->{hostusb}; # old syntax
2164     $loc_res = 1 if $conf->{hostpci}; # old syntax
2165
2166     foreach my $k (keys %$conf) {
2167         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2168         # sockets are safe: they will recreated be on the target side post-migrate
2169         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2170         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2171     }
2172
2173     die "VM uses local resources\n" if $loc_res && !$noerr;
2174
2175     return $loc_res;
2176 }
2177
2178 # check if used storages are available on all nodes (use by migrate)
2179 sub check_storage_availability {
2180     my ($storecfg, $conf, $node) = @_;
2181
2182     foreach_drive($conf, sub {
2183         my ($ds, $drive) = @_;
2184
2185         my $volid = $drive->{file};
2186         return if !$volid;
2187
2188         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2189         return if !$sid;
2190
2191         # check if storage is available on both nodes
2192         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2193         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2194    });
2195 }
2196
2197 # list nodes where all VM images are available (used by has_feature API)
2198 sub shared_nodes {
2199     my ($conf, $storecfg) = @_;
2200
2201     my $nodelist = PVE::Cluster::get_nodelist();
2202     my $nodehash = { map { $_ => 1 } @$nodelist };
2203     my $nodename = PVE::INotify::nodename();
2204
2205     foreach_drive($conf, sub {
2206         my ($ds, $drive) = @_;
2207
2208         my $volid = $drive->{file};
2209         return if !$volid;
2210
2211         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2212         if ($storeid) {
2213             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2214             if ($scfg->{disable}) {
2215                 $nodehash = {};
2216             } elsif (my $avail = $scfg->{nodes}) {
2217                 foreach my $node (keys %$nodehash) {
2218                     delete $nodehash->{$node} if !$avail->{$node};
2219                 }
2220             } elsif (!$scfg->{shared}) {
2221                 foreach my $node (keys %$nodehash) {
2222                     delete $nodehash->{$node} if $node ne $nodename
2223                 }
2224             }
2225         }
2226     });
2227
2228     return $nodehash
2229 }
2230
2231 sub check_lock {
2232     my ($conf) = @_;
2233
2234     die "VM is locked ($conf->{lock})\n" if $conf->{lock};
2235 }
2236
2237 sub check_cmdline {
2238     my ($pidfile, $pid) = @_;
2239
2240     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2241     if (defined($fh)) {
2242         my $line = <$fh>;
2243         $fh->close;
2244         return undef if !$line;
2245         my @param = split(/\0/, $line);
2246
2247         my $cmd = $param[0];
2248         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2249
2250         for (my $i = 0; $i < scalar (@param); $i++) {
2251             my $p = $param[$i];
2252             next if !$p;
2253             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2254                 my $p = $param[$i+1];
2255                 return 1 if $p && ($p eq $pidfile);
2256                 return undef;
2257             }
2258         }
2259     }
2260     return undef;
2261 }
2262
2263 sub check_running {
2264     my ($vmid, $nocheck, $node) = @_;
2265
2266     my $filename = config_file($vmid, $node);
2267
2268     die "unable to find configuration file for VM $vmid - no such machine\n"
2269         if !$nocheck && ! -f $filename;
2270
2271     my $pidfile = pidfile_name($vmid);
2272
2273     if (my $fd = IO::File->new("<$pidfile")) {
2274         my $st = stat($fd);
2275         my $line = <$fd>;
2276         close($fd);
2277
2278         my $mtime = $st->mtime;
2279         if ($mtime > time()) {
2280             warn "file '$filename' modified in future\n";
2281         }
2282
2283         if ($line =~ m/^(\d+)$/) {
2284             my $pid = $1;
2285             if (check_cmdline($pidfile, $pid)) {
2286                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2287                     return $pid;
2288                 }
2289             }
2290         }
2291     }
2292
2293     return undef;
2294 }
2295
2296 sub vzlist {
2297
2298     my $vzlist = config_list();
2299
2300     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2301
2302     while (defined(my $de = $fd->read)) {
2303         next if $de !~ m/^(\d+)\.pid$/;
2304         my $vmid = $1;
2305         next if !defined($vzlist->{$vmid});
2306         if (my $pid = check_running($vmid)) {
2307             $vzlist->{$vmid}->{pid} = $pid;
2308         }
2309     }
2310
2311     return $vzlist;
2312 }
2313
2314 sub disksize {
2315     my ($storecfg, $conf) = @_;
2316
2317     my $bootdisk = $conf->{bootdisk};
2318     return undef if !$bootdisk;
2319     return undef if !valid_drivename($bootdisk);
2320
2321     return undef if !$conf->{$bootdisk};
2322
2323     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2324     return undef if !defined($drive);
2325
2326     return undef if drive_is_cdrom($drive);
2327
2328     my $volid = $drive->{file};
2329     return undef if !$volid;
2330
2331     return $drive->{size};
2332 }
2333
2334 my $last_proc_pid_stat;
2335
2336 # get VM status information
2337 # This must be fast and should not block ($full == false)
2338 # We only query KVM using QMP if $full == true (this can be slow)
2339 sub vmstatus {
2340     my ($opt_vmid, $full) = @_;
2341
2342     my $res = {};
2343
2344     my $storecfg = PVE::Storage::config();
2345
2346     my $list = vzlist();
2347     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2348
2349     my $cpucount = $cpuinfo->{cpus} || 1;
2350
2351     foreach my $vmid (keys %$list) {
2352         next if $opt_vmid && ($vmid ne $opt_vmid);
2353
2354         my $cfspath = cfs_config_path($vmid);
2355         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2356
2357         my $d = {};
2358         $d->{pid} = $list->{$vmid}->{pid};
2359
2360         # fixme: better status?
2361         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2362
2363         my $size = disksize($storecfg, $conf);
2364         if (defined($size)) {
2365             $d->{disk} = 0; # no info available
2366             $d->{maxdisk} = $size;
2367         } else {
2368             $d->{disk} = 0;
2369             $d->{maxdisk} = 0;
2370         }
2371
2372         $d->{cpus} = ($conf->{sockets} || 1) * ($conf->{cores} || 1);
2373         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2374         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2375
2376         $d->{name} = $conf->{name} || "VM $vmid";
2377         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024) : 0;
2378
2379         if ($conf->{balloon}) {
2380             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2381             $d->{shares} = defined($conf->{shares}) ? $conf->{shares} : 1000;
2382         }
2383
2384         $d->{uptime} = 0;
2385         $d->{cpu} = 0;
2386         $d->{mem} = 0;
2387
2388         $d->{netout} = 0;
2389         $d->{netin} = 0;
2390
2391         $d->{diskread} = 0;
2392         $d->{diskwrite} = 0;
2393
2394         $d->{template} = is_template($conf);
2395
2396         $res->{$vmid} = $d;
2397     }
2398
2399     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2400     foreach my $dev (keys %$netdev) {
2401         next if $dev !~ m/^tap([1-9]\d*)i/;
2402         my $vmid = $1;
2403         my $d = $res->{$vmid};
2404         next if !$d;
2405
2406         $d->{netout} += $netdev->{$dev}->{receive};
2407         $d->{netin} += $netdev->{$dev}->{transmit};
2408
2409         if ($full) {
2410             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2411             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2412         }
2413
2414     }
2415
2416     my $ctime = gettimeofday;
2417
2418     foreach my $vmid (keys %$list) {
2419
2420         my $d = $res->{$vmid};
2421         my $pid = $d->{pid};
2422         next if !$pid;
2423
2424         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2425         next if !$pstat; # not running
2426
2427         my $used = $pstat->{utime} + $pstat->{stime};
2428
2429         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2430
2431         if ($pstat->{vsize}) {
2432             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2433         }
2434
2435         my $old = $last_proc_pid_stat->{$pid};
2436         if (!$old) {
2437             $last_proc_pid_stat->{$pid} = {
2438                 time => $ctime,
2439                 used => $used,
2440                 cpu => 0,
2441             };
2442             next;
2443         }
2444
2445         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2446
2447         if ($dtime > 1000) {
2448             my $dutime = $used -  $old->{used};
2449
2450             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2451             $last_proc_pid_stat->{$pid} = {
2452                 time => $ctime,
2453                 used => $used,
2454                 cpu => $d->{cpu},
2455             };
2456         } else {
2457             $d->{cpu} = $old->{cpu};
2458         }
2459     }
2460
2461     return $res if !$full;
2462
2463     my $qmpclient = PVE::QMPClient->new();
2464
2465     my $ballooncb = sub {
2466         my ($vmid, $resp) = @_;
2467
2468         my $info = $resp->{'return'};
2469         return if !$info->{max_mem};
2470
2471         my $d = $res->{$vmid};
2472
2473         # use memory assigned to VM
2474         $d->{maxmem} = $info->{max_mem};
2475         $d->{balloon} = $info->{actual};
2476
2477         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2478             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2479             $d->{freemem} = $info->{free_mem};
2480         }
2481
2482         $d->{ballooninfo} = $info;
2483     };
2484
2485     my $blockstatscb = sub {
2486         my ($vmid, $resp) = @_;
2487         my $data = $resp->{'return'} || [];
2488         my $totalrdbytes = 0;
2489         my $totalwrbytes = 0;
2490
2491         for my $blockstat (@$data) {
2492             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2493             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2494
2495             $blockstat->{device} =~ s/drive-//;
2496             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2497         }
2498         $res->{$vmid}->{diskread} = $totalrdbytes;
2499         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2500     };
2501
2502     my $statuscb = sub {
2503         my ($vmid, $resp) = @_;
2504
2505         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2506         # this fails if ballon driver is not loaded, so this must be
2507         # the last commnand (following command are aborted if this fails).
2508         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2509
2510         my $status = 'unknown';
2511         if (!defined($status = $resp->{'return'}->{status})) {
2512             warn "unable to get VM status\n";
2513             return;
2514         }
2515
2516         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2517     };
2518
2519     foreach my $vmid (keys %$list) {
2520         next if $opt_vmid && ($vmid ne $opt_vmid);
2521         next if !$res->{$vmid}->{pid}; # not running
2522         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2523     }
2524
2525     $qmpclient->queue_execute(undef, 1);
2526
2527     foreach my $vmid (keys %$list) {
2528         next if $opt_vmid && ($vmid ne $opt_vmid);
2529         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2530     }
2531
2532     return $res;
2533 }
2534
2535 sub foreach_dimm {
2536     my ($conf, $vmid, $memory, $sockets, $func) = @_;
2537
2538     my $dimm_id = 0;
2539     my $current_size = 1024;
2540     my $dimm_size = 512;
2541     return if $current_size == $memory;
2542
2543     for (my $j = 0; $j < 8; $j++) {
2544         for (my $i = 0; $i < 32; $i++) {
2545             my $name = "dimm${dimm_id}";
2546             $dimm_id++;
2547             my $numanode = $i % $sockets;
2548             $current_size += $dimm_size;
2549             &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
2550             return  $current_size if $current_size >= $memory;
2551         }
2552         $dimm_size *= 2;
2553     }
2554 }
2555
2556 sub foreach_reverse_dimm {
2557     my ($conf, $vmid, $memory, $sockets, $func) = @_;
2558
2559     my $dimm_id = 253;
2560     my $current_size = 4177920;
2561     my $dimm_size = 65536;
2562     return if $current_size == $memory;
2563
2564     for (my $j = 0; $j < 8; $j++) {
2565         for (my $i = 0; $i < 32; $i++) {
2566             my $name = "dimm${dimm_id}";
2567             $dimm_id--;
2568             my $numanode = $i % $sockets;
2569             $current_size -= $dimm_size;
2570             &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
2571             return  $current_size if $current_size <= $memory;
2572         }
2573         $dimm_size /= 2;
2574     }
2575 }
2576
2577 sub foreach_drive {
2578     my ($conf, $func) = @_;
2579
2580     foreach my $ds (keys %$conf) {
2581         next if !valid_drivename($ds);
2582
2583         my $drive = parse_drive($ds, $conf->{$ds});
2584         next if !$drive;
2585
2586         &$func($ds, $drive);
2587     }
2588 }
2589
2590 sub foreach_volid {
2591     my ($conf, $func) = @_;
2592
2593     my $volhash = {};
2594
2595     my $test_volid = sub {
2596         my ($volid, $is_cdrom) = @_;
2597
2598         return if !$volid;
2599
2600         $volhash->{$volid} = $is_cdrom || 0;
2601     };
2602
2603     foreach_drive($conf, sub {
2604         my ($ds, $drive) = @_;
2605         &$test_volid($drive->{file}, drive_is_cdrom($drive));
2606     });
2607
2608     foreach my $snapname (keys %{$conf->{snapshots}}) {
2609         my $snap = $conf->{snapshots}->{$snapname};
2610         &$test_volid($snap->{vmstate}, 0);
2611         foreach_drive($snap, sub {
2612             my ($ds, $drive) = @_;
2613             &$test_volid($drive->{file}, drive_is_cdrom($drive));
2614         });
2615     }
2616
2617     foreach my $volid (keys %$volhash) {
2618         &$func($volid, $volhash->{$volid});
2619     }
2620 }
2621
2622 sub vga_conf_has_spice {
2623     my ($vga) = @_;
2624
2625     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
2626
2627     return $1 || 1;
2628 }
2629
2630 sub config_to_command {
2631     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2632
2633     my $cmd = [];
2634     my $globalFlags = [];
2635     my $machineFlags = [];
2636     my $rtcFlags = [];
2637     my $cpuFlags = [];
2638     my $devices = [];
2639     my $pciaddr = '';
2640     my $bridges = {};
2641     my $kvmver = kvm_user_version();
2642     my $vernum = 0; # unknown
2643     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
2644         $vernum = $1*1000000+$2*1000;
2645     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
2646         $vernum = $1*1000000+$2*1000+$3;
2647     }
2648
2649     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
2650
2651     my $have_ovz = -f '/proc/vz/vestat';
2652
2653     my $q35 = machine_type_is_q35($conf);
2654     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2655     my $machine_type = $forcemachine || $conf->{machine};
2656     my $use_old_bios_files = undef;
2657     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
2658
2659     my $cpuunits = defined($conf->{cpuunits}) ?
2660             $conf->{cpuunits} : $defaults->{cpuunits};
2661
2662     push @$cmd, '/usr/bin/systemd-run';
2663     push @$cmd, '--scope';
2664     push @$cmd, '--slice', "qemu";
2665     push @$cmd, '--unit', $vmid;
2666     # set KillMode=none, so that systemd don't kill those scopes
2667     # at shutdown (pve-manager service should stop the VMs instead)
2668     push @$cmd, '-p', "KillMode=none";
2669     push @$cmd, '-p', "CPUShares=$cpuunits";
2670     if ($conf->{cpulimit}) {
2671         my $cpulimit = int($conf->{cpulimit} * 100);
2672         push @$cmd, '-p', "CPUQuota=$cpulimit\%";
2673     }
2674
2675     push @$cmd, '/usr/bin/kvm';
2676
2677     push @$cmd, '-id', $vmid;
2678
2679     my $use_virtio = 0;
2680
2681     my $qmpsocket = qmp_socket($vmid);
2682     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2683     push @$cmd, '-mon', "chardev=qmp,mode=control";
2684
2685     my $socket = vnc_socket($vmid);
2686     push @$cmd,  '-vnc', "unix:$socket,x509,password";
2687
2688     push @$cmd, '-pidfile' , pidfile_name($vmid);
2689
2690     push @$cmd, '-daemonize';
2691
2692     if ($conf->{smbios1}) {
2693         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
2694     }
2695
2696     if ($conf->{ovmf}) {
2697         my $ovmfvar = "OVMF_VARS-pure-efi.fd";
2698         my $ovmfvar_src = "/usr/share/kvm/$ovmfvar";
2699         my $ovmfvar_dst = "/tmp/$vmid-$ovmfvar";
2700         copy $ovmfvar_src,$ovmfvar_dst if !(-e $ovmfvar_dst);
2701         push @$cmd, '-drive', "if=pflash,format=raw,readonly,file=/usr/share/kvm/OVMF-pure-efi.fd";
2702         push @$cmd, '-drive', "if=pflash,format=raw,file=$ovmfvar_dst";
2703     }
2704
2705     if ($q35) {
2706         # the q35 chipset support native usb2, so we enable usb controller
2707         # by default for this machine type
2708         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
2709     } else {
2710         $pciaddr = print_pci_addr("piix3", $bridges);
2711         push @$devices, '-device', "piix3-usb-uhci,id=uhci$pciaddr.0x2";
2712
2713         my $use_usb2 = 0;
2714         for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2715             next if !$conf->{"usb$i"};
2716             $use_usb2 = 1;
2717         }
2718         # include usb device config
2719         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-usb.cfg' if $use_usb2;
2720     }
2721
2722     my $vga = $conf->{vga};
2723
2724     my $qxlnum = vga_conf_has_spice($vga);
2725     $vga = 'qxl' if $qxlnum;
2726
2727     if (!$vga) {
2728         if ($conf->{ostype} && ($conf->{ostype} eq 'win8' ||
2729                                 $conf->{ostype} eq 'win7' ||
2730                                 $conf->{ostype} eq 'w2k8')) {
2731             $vga = 'std';
2732         } else {
2733             $vga = 'cirrus';
2734         }
2735     }
2736
2737     # enable absolute mouse coordinates (needed by vnc)
2738     my $tablet;
2739     if (defined($conf->{tablet})) {
2740         $tablet = $conf->{tablet};
2741     } else {
2742         $tablet = $defaults->{tablet};
2743         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
2744         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
2745     }
2746
2747     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
2748
2749     # host pci devices
2750     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
2751         my $d = parse_hostpci($conf->{"hostpci$i"});
2752         next if !$d;
2753
2754         my $pcie = $d->{pcie};
2755         if($pcie){
2756             die "q35 machine model is not enabled" if !$q35;
2757             $pciaddr = print_pcie_addr("hostpci$i");
2758         }else{
2759             $pciaddr = print_pci_addr("hostpci$i", $bridges);
2760         }
2761
2762         my $rombar = $d->{rombar} && $d->{rombar} eq 'off' ? ",rombar=0" : "";
2763         my $xvga = $d->{'x-vga'} && $d->{'x-vga'} eq 'on' ? ",x-vga=on" : "";
2764         if ($xvga && $xvga ne '') {
2765             push @$cpuFlags, 'kvm=off';
2766             $vga = 'none';
2767         }
2768         my $pcidevices = $d->{pciid};
2769         my $multifunction = 1 if @$pcidevices > 1;
2770
2771         my $j=0;
2772         foreach my $pcidevice (@$pcidevices) {
2773
2774             my $id = "hostpci$i";
2775             $id .= ".$j" if $multifunction;
2776             my $addr = $pciaddr;
2777             $addr .= ".$j" if $multifunction;
2778             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
2779
2780             if($j == 0){
2781                 $devicestr .= "$rombar$xvga";
2782                 $devicestr .= ",multifunction=on" if $multifunction;
2783             }
2784
2785             push @$devices, '-device', $devicestr;
2786             $j++;
2787         }
2788     }
2789
2790     # usb devices
2791     for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2792         my $d = parse_usb_device($conf->{"usb$i"});
2793         next if !$d;
2794         if ($d->{vendorid} && $d->{productid}) {
2795             push @$devices, '-device', "usb-host,vendorid=0x$d->{vendorid},productid=0x$d->{productid}";
2796         } elsif (defined($d->{hostbus}) && defined($d->{hostport})) {
2797             push @$devices, '-device', "usb-host,hostbus=$d->{hostbus},hostport=$d->{hostport}";
2798         } elsif ($d->{spice}) {
2799             # usb redir support for spice
2800             push @$devices, '-chardev', "spicevmc,id=usbredirchardev$i,name=usbredir";
2801             push @$devices, '-device', "usb-redir,chardev=usbredirchardev$i,id=usbredirdev$i,bus=ehci.0";
2802         }
2803     }
2804
2805     # serial devices
2806     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2807         if (my $path = $conf->{"serial$i"}) {
2808             if ($path eq 'socket') {
2809                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
2810                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
2811                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2812             } else {
2813                 die "no such serial device\n" if ! -c $path;
2814                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
2815                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2816             }
2817         }
2818     }
2819
2820     # parallel devices
2821     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
2822         if (my $path = $conf->{"parallel$i"}) {
2823             die "no such parallel device\n" if ! -c $path;
2824             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
2825             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
2826             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
2827         }
2828     }
2829
2830     my $vmname = $conf->{name} || "vm$vmid";
2831
2832     push @$cmd, '-name', $vmname;
2833
2834     my $sockets = 1;
2835     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
2836     $sockets = $conf->{sockets} if  $conf->{sockets};
2837
2838     my $cores = $conf->{cores} || 1;
2839
2840     my $maxcpus = $sockets * $cores;
2841
2842     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
2843
2844     my $allowed_vcpus = $cpuinfo->{cpus};
2845
2846     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
2847         if ($allowed_vcpus < $maxcpus);
2848
2849     push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
2850
2851     push @$cmd, '-nodefaults';
2852
2853     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2854
2855     my $bootindex_hash = {};
2856     my $i = 1;
2857     foreach my $o (split(//, $bootorder)) {
2858         $bootindex_hash->{$o} = $i*100;
2859         $i++;
2860     }
2861
2862     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000";
2863
2864     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
2865
2866     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
2867
2868     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
2869
2870     # time drift fix
2871     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
2872
2873     my $nokvm = defined($conf->{kvm}) && $conf->{kvm} == 0 ? 1 : 0;
2874     my $useLocaltime = $conf->{localtime};
2875
2876     if (my $ost = $conf->{ostype}) {
2877         # other, wxp, w2k, w2k3, w2k8, wvista, win7, win8, l24, l26, solaris
2878
2879         if ($ost =~ m/^w/) { # windows
2880             $useLocaltime = 1 if !defined($conf->{localtime});
2881
2882             # use time drift fix when acpi is enabled
2883             if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
2884                 $tdf = 1 if !defined($conf->{tdf});
2885             }
2886         }
2887
2888         if ($ost eq 'win7' || $ost eq 'win8' || $ost eq 'w2k8' ||
2889             $ost eq 'wvista') {
2890             push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
2891             push @$cmd, '-no-hpet';
2892             if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
2893                 push @$cpuFlags , 'hv_spinlocks=0x1fff' if !$nokvm;
2894                 push @$cpuFlags , 'hv_vapic' if !$nokvm;
2895                 push @$cpuFlags , 'hv_time' if !$nokvm;
2896
2897             } else {
2898                 push @$cpuFlags , 'hv_spinlocks=0xffff' if !$nokvm;
2899             }
2900         }
2901
2902         if ($ost eq 'win7' || $ost eq 'win8') {
2903             push @$cpuFlags , 'hv_relaxed' if !$nokvm;
2904         }
2905     }
2906
2907     push @$rtcFlags, 'driftfix=slew' if $tdf;
2908
2909     if ($nokvm) {
2910         push @$machineFlags, 'accel=tcg';
2911     } else {
2912         die "No accelerator found!\n" if !$cpuinfo->{hvm};
2913     }
2914
2915     if ($machine_type) {
2916         push @$machineFlags, "type=${machine_type}";
2917     }
2918
2919     if ($conf->{startdate}) {
2920         push @$rtcFlags, "base=$conf->{startdate}";
2921     } elsif ($useLocaltime) {
2922         push @$rtcFlags, 'base=localtime';
2923     }
2924
2925     my $cpu = $nokvm ? "qemu64" : "kvm64";
2926     $cpu = $conf->{cpu} if $conf->{cpu};
2927
2928     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
2929
2930     push @$cpuFlags , '-x2apic'
2931         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
2932
2933     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
2934
2935     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
2936
2937     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
2938
2939         push @$cpuFlags , '+kvm_pv_unhalt' if !$nokvm;
2940         push @$cpuFlags , '+kvm_pv_eoi' if !$nokvm;
2941     }
2942
2943     push @$cpuFlags, 'enforce' if $cpu ne 'host' && !$nokvm;
2944
2945     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
2946
2947     push @$cmd, '-cpu', $cpu;
2948
2949     my $memory = $conf->{memory} || $defaults->{memory};
2950     my $static_memory = 0;
2951     my $dimm_memory = 0;
2952
2953     if ($hotplug_features->{memory}) {
2954         die "Numa need to be enabled for memory hotplug\n" if !$conf->{numa};
2955         die "Total memory is bigger than ${MAX_MEM}MB\n" if $memory > $MAX_MEM;
2956         $static_memory = $STATICMEM;
2957         die "minimum memory must be ${static_memory}MB\n" if($memory < $static_memory);
2958         $dimm_memory = $memory - $static_memory;
2959         push @$cmd, '-m', "size=${static_memory},slots=255,maxmem=${MAX_MEM}M";
2960
2961     } else {
2962
2963         $static_memory = $memory;
2964         push @$cmd, '-m', $static_memory;
2965     }
2966
2967     if ($conf->{numa}) {
2968
2969         my $numa_totalmemory = undef;
2970         for (my $i = 0; $i < $MAX_NUMA; $i++) {
2971             next if !$conf->{"numa$i"};
2972             my $numa = parse_numa($conf->{"numa$i"});
2973             next if !$numa;
2974             # memory
2975             die "missing numa node$i memory value\n" if !$numa->{memory};
2976             my $numa_memory = $numa->{memory};
2977             $numa_totalmemory += $numa_memory;
2978             my $numa_object = "memory-backend-ram,id=ram-node$i,size=${numa_memory}M";
2979
2980             # cpus
2981             my $cpus_start = $numa->{cpus}->{start};
2982             die "missing numa node$i cpus\n" if !defined($cpus_start);
2983             my $cpus_end = $numa->{cpus}->{end} if defined($numa->{cpus}->{end});
2984             my $cpus = $cpus_start;
2985             if (defined($cpus_end)) {
2986                 $cpus .= "-$cpus_end";
2987                 die "numa node$i :  cpu range $cpus is incorrect\n" if $cpus_end <= $cpus_start;
2988             }
2989
2990             # hostnodes
2991             my $hostnodes_start = $numa->{hostnodes}->{start};
2992             if (defined($hostnodes_start)) {
2993                 my $hostnodes_end = $numa->{hostnodes}->{end} if defined($numa->{hostnodes}->{end});
2994                 my $hostnodes = $hostnodes_start;
2995                 if (defined($hostnodes_end)) {
2996                     $hostnodes .= "-$hostnodes_end";
2997                     die "host node $hostnodes range is incorrect\n" if $hostnodes_end <= $hostnodes_start;
2998                 }
2999
3000                 my $hostnodes_end_range = defined($hostnodes_end) ? $hostnodes_end : $hostnodes_start;
3001                 for (my $i = $hostnodes_start; $i <= $hostnodes_end_range; $i++ ) {
3002                     die "host numa node$i don't exist\n" if ! -d "/sys/devices/system/node/node$i/";
3003                 }
3004
3005                 # policy
3006                 my $policy = $numa->{policy};
3007                 die "you need to define a policy for hostnode $hostnodes\n" if !$policy;
3008                 $numa_object .= ",host-nodes=$hostnodes,policy=$policy";
3009             }
3010
3011             push @$cmd, '-object', $numa_object;
3012             push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
3013         }
3014
3015         die "total memory for NUMA nodes must be equal to vm static memory\n"
3016             if $numa_totalmemory && $numa_totalmemory != $static_memory;
3017
3018         #if no custom tology, we split memory and cores across numa nodes
3019         if(!$numa_totalmemory) {
3020
3021             my $numa_memory = ($static_memory / $sockets) . "M";
3022
3023             for (my $i = 0; $i < $sockets; $i++)  {
3024
3025                 my $cpustart = ($cores * $i);
3026                 my $cpuend = ($cpustart + $cores - 1) if $cores && $cores > 1;
3027                 my $cpus = $cpustart;
3028                 $cpus .= "-$cpuend" if $cpuend;
3029
3030                 push @$cmd, '-object', "memory-backend-ram,size=$numa_memory,id=ram-node$i";
3031                 push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
3032             }
3033         }
3034     }
3035
3036     if ($hotplug_features->{memory}) {
3037         foreach_dimm($conf, $vmid, $memory, $sockets, sub {
3038             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3039             push @$cmd, "-object" , "memory-backend-ram,id=mem-$name,size=${dimm_size}M";
3040             push @$cmd, "-device", "pc-dimm,id=$name,memdev=mem-$name,node=$numanode";
3041
3042             #if dimm_memory is not aligned to dimm map
3043             if($current_size > $memory) {
3044                  $conf->{memory} = $current_size;
3045                  update_config_nolock($vmid, $conf, 1);
3046             }
3047         });
3048     }
3049
3050     push @$cmd, '-S' if $conf->{freeze};
3051
3052     # set keyboard layout
3053     my $kb = $conf->{keyboard} || $defaults->{keyboard};
3054     push @$cmd, '-k', $kb if $kb;
3055
3056     # enable sound
3057     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3058     #push @$cmd, '-soundhw', 'es1370';
3059     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3060
3061     if($conf->{agent}) {
3062         my $qgasocket = qmp_socket($vmid, 1);
3063         my $pciaddr = print_pci_addr("qga0", $bridges);
3064         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3065         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3066         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3067     }
3068
3069     my $spice_port;
3070
3071     if ($qxlnum) {
3072         if ($qxlnum > 1) {
3073             if ($conf->{ostype} && $conf->{ostype} =~ m/^w/){
3074                 for(my $i = 1; $i < $qxlnum; $i++){
3075                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3076                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3077                 }
3078             } else {
3079                 # assume other OS works like Linux
3080                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3081                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3082             }
3083         }
3084
3085         my $pciaddr = print_pci_addr("spice", $bridges);
3086
3087         my $nodename = PVE::INotify::nodename();
3088         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3089         $spice_port = PVE::Tools::next_spice_port($pfamily);
3090
3091         push @$devices, '-spice', "tls-port=${spice_port},addr=localhost,tls-ciphers=DES-CBC3-SHA,seamless-migration=on";
3092
3093         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3094         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3095         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3096     }
3097
3098     # enable balloon by default, unless explicitly disabled
3099     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3100         $pciaddr = print_pci_addr("balloon0", $bridges);
3101         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3102     }
3103
3104     if ($conf->{watchdog}) {
3105         my $wdopts = parse_watchdog($conf->{watchdog});
3106         $pciaddr = print_pci_addr("watchdog", $bridges);
3107         my $watchdog = $wdopts->{model} || 'i6300esb';
3108         push @$devices, '-device', "$watchdog$pciaddr";
3109         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3110     }
3111
3112     my $vollist = [];
3113     my $scsicontroller = {};
3114     my $ahcicontroller = {};
3115     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3116
3117     # Add iscsi initiator name if available
3118     if (my $initiator = get_initiator_name()) {
3119         push @$devices, '-iscsi', "initiator-name=$initiator";
3120     }
3121
3122     foreach_drive($conf, sub {
3123         my ($ds, $drive) = @_;
3124
3125         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3126             push @$vollist, $drive->{file};
3127         }
3128
3129         $use_virtio = 1 if $ds =~ m/^virtio/;
3130
3131         if (drive_is_cdrom ($drive)) {
3132             if ($bootindex_hash->{d}) {
3133                 $drive->{bootindex} = $bootindex_hash->{d};
3134                 $bootindex_hash->{d} += 1;
3135             }
3136         } else {
3137             if ($bootindex_hash->{c}) {
3138                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3139                 $bootindex_hash->{c} += 1;
3140             }
3141         }
3142
3143         if($drive->{interface} eq 'virtio'){
3144            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3145         }
3146
3147         if ($drive->{interface} eq 'scsi') {
3148
3149             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3150
3151             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3152             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3153
3154             my $iothread = '';
3155             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3156                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3157                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3158             }
3159
3160             my $queues = '';
3161             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3162                 $queues = ",num_queues=$drive->{queues}";
3163             }
3164
3165             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3166             $scsicontroller->{$controller}=1;
3167         }
3168
3169         if ($drive->{interface} eq 'sata') {
3170            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3171            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3172            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3173            $ahcicontroller->{$controller}=1;
3174         }
3175
3176         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3177         push @$devices, '-drive',$drive_cmd;
3178         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3179     });
3180
3181     for (my $i = 0; $i < $MAX_NETS; $i++) {
3182          next if !$conf->{"net$i"};
3183          my $d = parse_net($conf->{"net$i"});
3184          next if !$d;
3185
3186          $use_virtio = 1 if $d->{model} eq 'virtio';
3187
3188          if ($bootindex_hash->{n}) {
3189             $d->{bootindex} = $bootindex_hash->{n};
3190             $bootindex_hash->{n} += 1;
3191          }
3192
3193          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3194          push @$devices, '-netdev', $netdevfull;
3195
3196          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3197          push @$devices, '-device', $netdevicefull;
3198     }
3199
3200     if (!$q35) {
3201         # add pci bridges
3202         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3203            $bridges->{1} = 1;
3204            $bridges->{2} = 1;
3205         }
3206
3207         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3208
3209         while (my ($k, $v) = each %$bridges) {
3210             $pciaddr = print_pci_addr("pci.$k");
3211             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3212         }
3213     }
3214
3215     # add custom args
3216     if ($conf->{args}) {
3217         my $aa = PVE::Tools::split_args($conf->{args});
3218         push @$cmd, @$aa;
3219     }
3220
3221     push @$cmd, @$devices;
3222     push @$cmd, '-rtc', join(',', @$rtcFlags)
3223         if scalar(@$rtcFlags);
3224     push @$cmd, '-machine', join(',', @$machineFlags)
3225         if scalar(@$machineFlags);
3226     push @$cmd, '-global', join(',', @$globalFlags)
3227         if scalar(@$globalFlags);
3228
3229     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3230 }
3231
3232 sub vnc_socket {
3233     my ($vmid) = @_;
3234     return "${var_run_tmpdir}/$vmid.vnc";
3235 }
3236
3237 sub spice_port {
3238     my ($vmid) = @_;
3239
3240     my $res = vm_mon_cmd($vmid, 'query-spice');
3241
3242     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3243 }
3244
3245 sub qmp_socket {
3246     my ($vmid, $qga) = @_;
3247     my $sockettype = $qga ? 'qga' : 'qmp';
3248     return "${var_run_tmpdir}/$vmid.$sockettype";
3249 }
3250
3251 sub pidfile_name {
3252     my ($vmid) = @_;
3253     return "${var_run_tmpdir}/$vmid.pid";
3254 }
3255
3256 sub vm_devices_list {
3257     my ($vmid) = @_;
3258
3259     my $res = vm_mon_cmd($vmid, 'query-pci');
3260     my $devices = {};
3261     foreach my $pcibus (@$res) {
3262         foreach my $device (@{$pcibus->{devices}}) {
3263             next if !$device->{'qdev_id'};
3264             if ($device->{'pci_bridge'}) {
3265                 $devices->{$device->{'qdev_id'}} = 1;
3266                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3267                     next if !$bridge_device->{'qdev_id'};
3268                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3269                     $devices->{$device->{'qdev_id'}}++;
3270                 }
3271             } else {
3272                 $devices->{$device->{'qdev_id'}} = 1;
3273             }
3274         }
3275     }
3276
3277     my $resblock = vm_mon_cmd($vmid, 'query-block');
3278     foreach my $block (@$resblock) {
3279         if($block->{device} =~ m/^drive-(\S+)/){
3280                 $devices->{$1} = 1;
3281         }
3282     }
3283
3284     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3285     foreach my $mice (@$resmice) {
3286         if ($mice->{name} eq 'QEMU HID Tablet') {
3287             $devices->{tablet} = 1;
3288             last;
3289         }
3290     }
3291
3292     return $devices;
3293 }
3294
3295 sub vm_deviceplug {
3296     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3297
3298     my $q35 = machine_type_is_q35($conf);
3299
3300     my $devices_list = vm_devices_list($vmid);
3301     return 1 if defined($devices_list->{$deviceid});
3302
3303     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3304
3305     if ($deviceid eq 'tablet') {
3306
3307         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3308
3309     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3310
3311         qemu_iothread_add($vmid, $deviceid, $device);
3312
3313         qemu_driveadd($storecfg, $vmid, $device);
3314         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3315
3316         qemu_deviceadd($vmid, $devicefull);
3317         eval { qemu_deviceaddverify($vmid, $deviceid); };
3318         if (my $err = $@) {
3319             eval { qemu_drivedel($vmid, $deviceid); };
3320             warn $@ if $@;
3321             die $err;
3322         }
3323
3324     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3325
3326
3327         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3328         my $pciaddr = print_pci_addr($deviceid);
3329         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3330
3331         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3332
3333         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3334             qemu_iothread_add($vmid, $deviceid, $device);
3335             $devicefull .= ",iothread=iothread-$deviceid";
3336         }
3337
3338         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3339             $devicefull .= ",num_queues=$device->{queues}";
3340         }
3341
3342         qemu_deviceadd($vmid, $devicefull);
3343         qemu_deviceaddverify($vmid, $deviceid);
3344
3345     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3346
3347         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3348         qemu_driveadd($storecfg, $vmid, $device);
3349
3350         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3351         eval { qemu_deviceadd($vmid, $devicefull); };
3352         if (my $err = $@) {
3353             eval { qemu_drivedel($vmid, $deviceid); };
3354             warn $@ if $@;
3355             die $err;
3356         }
3357
3358     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3359
3360         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3361
3362         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
3363         my $use_old_bios_files = undef;
3364         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3365
3366         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3367         qemu_deviceadd($vmid, $netdevicefull);
3368         eval { qemu_deviceaddverify($vmid, $deviceid); };
3369         if (my $err = $@) {
3370             eval { qemu_netdevdel($vmid, $deviceid); };
3371             warn $@ if $@;
3372             die $err;
3373         }
3374
3375     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3376
3377         my $bridgeid = $2;
3378         my $pciaddr = print_pci_addr($deviceid);
3379         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3380
3381         qemu_deviceadd($vmid, $devicefull);
3382         qemu_deviceaddverify($vmid, $deviceid);
3383
3384     } else {
3385         die "can't hotplug device '$deviceid'\n";
3386     }
3387
3388     return 1;
3389 }
3390
3391 # fixme: this should raise exceptions on error!
3392 sub vm_deviceunplug {
3393     my ($vmid, $conf, $deviceid) = @_;
3394
3395     my $devices_list = vm_devices_list($vmid);
3396     return 1 if !defined($devices_list->{$deviceid});
3397
3398     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3399
3400     if ($deviceid eq 'tablet') {
3401
3402         qemu_devicedel($vmid, $deviceid);
3403
3404     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3405
3406         qemu_devicedel($vmid, $deviceid);
3407         qemu_devicedelverify($vmid, $deviceid);
3408         qemu_drivedel($vmid, $deviceid);
3409         qemu_iothread_del($conf, $vmid, $deviceid);
3410
3411     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3412
3413         qemu_devicedel($vmid, $deviceid);
3414         qemu_devicedelverify($vmid, $deviceid);
3415         qemu_iothread_del($conf, $vmid, $deviceid);
3416
3417     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3418
3419         #qemu 2.3 segfault on drive_del with virtioscsi + iothread
3420         my $device = parse_drive($deviceid, $conf->{$deviceid});
3421         die "virtioscsi with iothread is not hot-unplugglable currently" if $device->{iothread};
3422
3423         qemu_devicedel($vmid, $deviceid);
3424         qemu_drivedel($vmid, $deviceid);
3425         qemu_deletescsihw($conf, $vmid, $deviceid);
3426
3427     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3428
3429         qemu_devicedel($vmid, $deviceid);
3430         qemu_devicedelverify($vmid, $deviceid);
3431         qemu_netdevdel($vmid, $deviceid);
3432
3433     } else {
3434         die "can't unplug device '$deviceid'\n";
3435     }
3436
3437     return 1;
3438 }
3439
3440 sub qemu_deviceadd {
3441     my ($vmid, $devicefull) = @_;
3442
3443     $devicefull = "driver=".$devicefull;
3444     my %options =  split(/[=,]/, $devicefull);
3445
3446     vm_mon_cmd($vmid, "device_add" , %options);
3447 }
3448
3449 sub qemu_devicedel {
3450     my ($vmid, $deviceid) = @_;
3451
3452     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3453 }
3454
3455 sub qemu_iothread_add {
3456     my($vmid, $deviceid, $device) = @_;
3457
3458     if ($device->{iothread}) {
3459         my $iothreads = vm_iothreads_list($vmid);
3460         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3461     }
3462 }
3463
3464 sub qemu_iothread_del {
3465     my($conf, $vmid, $deviceid) = @_;
3466
3467     my $device = parse_drive($deviceid, $conf->{$deviceid});
3468     if ($device->{iothread}) {
3469         my $iothreads = vm_iothreads_list($vmid);
3470         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3471     }
3472 }
3473
3474 sub qemu_objectadd {
3475     my($vmid, $objectid, $qomtype) = @_;
3476
3477     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3478
3479     return 1;
3480 }
3481
3482 sub qemu_objectdel {
3483     my($vmid, $objectid) = @_;
3484
3485     vm_mon_cmd($vmid, "object-del", id => $objectid);
3486
3487     return 1;
3488 }
3489
3490 sub qemu_driveadd {
3491     my ($storecfg, $vmid, $device) = @_;
3492
3493     my $drive = print_drive_full($storecfg, $vmid, $device);
3494     $drive =~ s/\\/\\\\/g;
3495     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3496
3497     # If the command succeeds qemu prints: "OK"
3498     return 1 if $ret =~ m/OK/s;
3499
3500     die "adding drive failed: $ret\n";
3501 }
3502
3503 sub qemu_drivedel {
3504     my($vmid, $deviceid) = @_;
3505
3506     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3507     $ret =~ s/^\s+//;
3508
3509     return 1 if $ret eq "";
3510
3511     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3512     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3513
3514     die "deleting drive $deviceid failed : $ret\n";
3515 }
3516
3517 sub qemu_deviceaddverify {
3518     my ($vmid, $deviceid) = @_;
3519
3520     for (my $i = 0; $i <= 5; $i++) {
3521          my $devices_list = vm_devices_list($vmid);
3522          return 1 if defined($devices_list->{$deviceid});
3523          sleep 1;
3524     }
3525
3526     die "error on hotplug device '$deviceid'\n";
3527 }
3528
3529
3530 sub qemu_devicedelverify {
3531     my ($vmid, $deviceid) = @_;
3532
3533     # need to verify that the device is correctly removed as device_del
3534     # is async and empty return is not reliable
3535
3536     for (my $i = 0; $i <= 5; $i++) {
3537          my $devices_list = vm_devices_list($vmid);
3538          return 1 if !defined($devices_list->{$deviceid});
3539          sleep 1;
3540     }
3541
3542     die "error on hot-unplugging device '$deviceid'\n";
3543 }
3544
3545 sub qemu_findorcreatescsihw {
3546     my ($storecfg, $conf, $vmid, $device) = @_;
3547
3548     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3549
3550     my $scsihwid="$controller_prefix$controller";
3551     my $devices_list = vm_devices_list($vmid);
3552
3553     if(!defined($devices_list->{$scsihwid})) {
3554         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3555     }
3556
3557     return 1;
3558 }
3559
3560 sub qemu_deletescsihw {
3561     my ($conf, $vmid, $opt) = @_;
3562
3563     my $device = parse_drive($opt, $conf->{$opt});
3564
3565     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3566         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3567         return 1;
3568     }
3569
3570     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3571
3572     my $devices_list = vm_devices_list($vmid);
3573     foreach my $opt (keys %{$devices_list}) {
3574         if (PVE::QemuServer::valid_drivename($opt)) {
3575             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3576             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3577                 return 1;
3578             }
3579         }
3580     }
3581
3582     my $scsihwid="scsihw$controller";
3583
3584     vm_deviceunplug($vmid, $conf, $scsihwid);
3585
3586     return 1;
3587 }
3588
3589 sub qemu_add_pci_bridge {
3590     my ($storecfg, $conf, $vmid, $device) = @_;
3591
3592     my $bridges = {};
3593
3594     my $bridgeid;
3595
3596     print_pci_addr($device, $bridges);
3597
3598     while (my ($k, $v) = each %$bridges) {
3599         $bridgeid = $k;
3600     }
3601     return 1 if !defined($bridgeid) || $bridgeid < 1;
3602
3603     my $bridge = "pci.$bridgeid";
3604     my $devices_list = vm_devices_list($vmid);
3605
3606     if (!defined($devices_list->{$bridge})) {
3607         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3608     }
3609
3610     return 1;
3611 }
3612
3613 sub qemu_set_link_status {
3614     my ($vmid, $device, $up) = @_;
3615
3616     vm_mon_cmd($vmid, "set_link", name => $device,
3617                up => $up ? JSON::true : JSON::false);
3618 }
3619
3620 sub qemu_netdevadd {
3621     my ($vmid, $conf, $device, $deviceid) = @_;
3622
3623     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
3624     my %options =  split(/[=,]/, $netdev);
3625
3626     vm_mon_cmd($vmid, "netdev_add",  %options);
3627     return 1;
3628 }
3629
3630 sub qemu_netdevdel {
3631     my ($vmid, $deviceid) = @_;
3632
3633     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3634 }
3635
3636 sub qemu_cpu_hotplug {
3637     my ($vmid, $conf, $vcpus) = @_;
3638
3639     my $sockets = 1;
3640     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3641     $sockets = $conf->{sockets} if  $conf->{sockets};
3642     my $cores = $conf->{cores} || 1;
3643     my $maxcpus = $sockets * $cores;
3644
3645     $vcpus = $maxcpus if !$vcpus;
3646
3647     die "you can't add more vcpus than maxcpus\n"
3648         if $vcpus > $maxcpus;
3649
3650     my $currentvcpus = $conf->{vcpus} || $maxcpus;
3651     die "online cpu unplug is not yet possible\n"
3652         if $vcpus < $currentvcpus;
3653
3654     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3655     die "vcpus in running vm is different than configuration\n"
3656         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
3657
3658     for (my $i = $currentvcpus; $i < $vcpus; $i++) {
3659         vm_mon_cmd($vmid, "cpu-add", id => int($i));
3660     }
3661 }
3662
3663 sub qemu_memory_hotplug {
3664     my ($vmid, $conf, $defaults, $opt, $value) = @_;
3665
3666     return $value if !check_running($vmid);
3667
3668     my $memory = $conf->{memory} || $defaults->{memory};
3669     $value = $defaults->{memory} if !$value;
3670     return $value if $value == $memory;
3671
3672     my $static_memory = $STATICMEM;
3673     my $dimm_memory = $memory - $static_memory;
3674
3675     die "memory can't be lower than $static_memory MB" if $value < $static_memory;
3676     die "you cannot add more memory than $MAX_MEM MB!\n" if $memory > $MAX_MEM;
3677
3678
3679     my $sockets = 1;
3680     $sockets = $conf->{sockets} if $conf->{sockets};
3681
3682     if($value > $memory) {
3683
3684         foreach_dimm($conf, $vmid, $value, $sockets, sub {
3685             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3686
3687                 return if $current_size <= $conf->{memory};
3688
3689                 eval { vm_mon_cmd($vmid, "object-add", 'qom-type' => "memory-backend-ram", id => "mem-$name", props => { size => int($dimm_size*1024*1024) } ) };
3690                 if (my $err = $@) {
3691                     eval { qemu_objectdel($vmid, "mem-$name"); };
3692                     die $err;
3693                 }
3694
3695                 eval { vm_mon_cmd($vmid, "device_add", driver => "pc-dimm", id => "$name", memdev => "mem-$name", node => $numanode) };
3696                 if (my $err = $@) {
3697                     eval { qemu_objectdel($vmid, "mem-$name"); };
3698                     die $err;
3699                 }
3700                 #update conf after each succesful module hotplug
3701                 $conf->{memory} = $current_size;
3702                 update_config_nolock($vmid, $conf, 1);
3703         });
3704
3705     } else {
3706
3707         foreach_reverse_dimm($conf, $vmid, $value, $sockets, sub {
3708             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3709
3710                 return if $current_size >= $conf->{memory};
3711                 print "try to unplug memory dimm $name\n";
3712
3713                 my $retry = 0;
3714                 while (1) {
3715                     eval { qemu_devicedel($vmid, $name) };
3716                     sleep 3;
3717                     my $dimm_list = qemu_dimm_list($vmid);
3718                     last if !$dimm_list->{$name};
3719                     raise_param_exc({ $name => "error unplug memory module" }) if $retry > 5;
3720                     $retry++;
3721                 }
3722
3723                 #update conf after each succesful module unplug
3724                 $conf->{memory} = $current_size;
3725
3726                 eval { qemu_objectdel($vmid, "mem-$name"); };
3727                 update_config_nolock($vmid, $conf, 1);
3728         });
3729     }
3730 }
3731
3732 sub qemu_dimm_list {
3733     my ($vmid) = @_;
3734
3735     my $dimmarray = vm_mon_cmd_nocheck($vmid, "query-memory-devices");
3736     my $dimms = {};
3737
3738     foreach my $dimm (@$dimmarray) {
3739
3740         $dimms->{$dimm->{data}->{id}}->{id} = $dimm->{data}->{id};
3741         $dimms->{$dimm->{data}->{id}}->{node} = $dimm->{data}->{node};
3742         $dimms->{$dimm->{data}->{id}}->{addr} = $dimm->{data}->{addr};
3743         $dimms->{$dimm->{data}->{id}}->{size} = $dimm->{data}->{size};
3744         $dimms->{$dimm->{data}->{id}}->{slot} = $dimm->{data}->{slot};
3745     }
3746     return $dimms;
3747 }
3748
3749 sub qemu_block_set_io_throttle {
3750     my ($vmid, $deviceid, $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr) = @_;
3751
3752     return if !check_running($vmid) ;
3753
3754     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid, bps => int($bps), bps_rd => int($bps_rd), bps_wr => int($bps_wr), iops => int($iops), iops_rd => int($iops_rd), iops_wr => int($iops_wr));
3755
3756 }
3757
3758 # old code, only used to shutdown old VM after update
3759 sub __read_avail {
3760     my ($fh, $timeout) = @_;
3761
3762     my $sel = new IO::Select;
3763     $sel->add($fh);
3764
3765     my $res = '';
3766     my $buf;
3767
3768     my @ready;
3769     while (scalar (@ready = $sel->can_read($timeout))) {
3770         my $count;
3771         if ($count = $fh->sysread($buf, 8192)) {
3772             if ($buf =~ /^(.*)\(qemu\) $/s) {
3773                 $res .= $1;
3774                 last;
3775             } else {
3776                 $res .= $buf;
3777             }
3778         } else {
3779             if (!defined($count)) {
3780                 die "$!\n";
3781             }
3782             last;
3783         }
3784     }
3785
3786     die "monitor read timeout\n" if !scalar(@ready);
3787
3788     return $res;
3789 }
3790
3791 # old code, only used to shutdown old VM after update
3792 sub vm_monitor_command {
3793     my ($vmid, $cmdstr, $nocheck) = @_;
3794
3795     my $res;
3796
3797     eval {
3798         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
3799
3800         my $sname = "${var_run_tmpdir}/$vmid.mon";
3801
3802         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
3803             die "unable to connect to VM $vmid socket - $!\n";
3804
3805         my $timeout = 3;
3806
3807         # hack: migrate sometime blocks the monitor (when migrate_downtime
3808         # is set)
3809         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3810             $timeout = 60*60; # 1 hour
3811         }
3812
3813         # read banner;
3814         my $data = __read_avail($sock, $timeout);
3815
3816         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
3817             die "got unexpected qemu monitor banner\n";
3818         }
3819
3820         my $sel = new IO::Select;
3821         $sel->add($sock);
3822
3823         if (!scalar(my @ready = $sel->can_write($timeout))) {
3824             die "monitor write error - timeout";
3825         }
3826
3827         my $fullcmd = "$cmdstr\r";
3828
3829         # syslog('info', "VM $vmid monitor command: $cmdstr");
3830
3831         my $b;
3832         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
3833             die "monitor write error - $!";
3834         }
3835
3836         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
3837
3838         $timeout = 20;
3839
3840         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3841             $timeout = 60*60; # 1 hour
3842         } elsif ($cmdstr =~ m/^(eject|change)/) {
3843             $timeout = 60; # note: cdrom mount command is slow
3844         }
3845         if ($res = __read_avail($sock, $timeout)) {
3846
3847             my @lines = split("\r?\n", $res);
3848
3849             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
3850
3851             $res = join("\n", @lines);
3852             $res .= "\n";
3853         }
3854     };
3855
3856     my $err = $@;
3857
3858     if ($err) {
3859         syslog("err", "VM $vmid monitor command failed - $err");
3860         die $err;
3861     }
3862
3863     return $res;
3864 }
3865
3866 sub qemu_block_resize {
3867     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
3868
3869     my $running = check_running($vmid);
3870
3871     return if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
3872
3873     return if !$running;
3874
3875     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
3876
3877 }
3878
3879 sub qemu_volume_snapshot {
3880     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3881
3882     my $running = check_running($vmid);
3883
3884     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
3885         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
3886     } else {
3887         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
3888     }
3889 }
3890
3891 sub qemu_volume_snapshot_delete {
3892     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3893
3894     my $running = check_running($vmid);
3895
3896     return if !PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
3897
3898     return if !$running;
3899
3900     vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
3901 }
3902
3903 sub set_migration_caps {
3904     my ($vmid) = @_;
3905
3906     my $cap_ref = [];
3907
3908     my $enabled_cap = {
3909         "auto-converge" => 1,
3910         "xbzrle" => 1,
3911         "x-rdma-pin-all" => 0,
3912         "zero-blocks" => 0,
3913         "compress" => 0
3914     };
3915
3916     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
3917
3918     for my $supported_capability (@$supported_capabilities) {
3919         push @$cap_ref, {
3920             capability => $supported_capability->{capability},
3921             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
3922         };
3923     }
3924
3925     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
3926 }
3927
3928 my $fast_plug_option = {
3929     'lock' => 1,
3930     'name' => 1,
3931     'onboot' => 1,
3932     'shares' => 1,
3933     'startup' => 1,
3934     'description' => 1,
3935 };
3936
3937 # hotplug changes in [PENDING]
3938 # $selection hash can be used to only apply specified options, for
3939 # example: { cores => 1 } (only apply changed 'cores')
3940 # $errors ref is used to return error messages
3941 sub vmconfig_hotplug_pending {
3942     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
3943
3944     my $defaults = load_defaults();
3945
3946     # commit values which do not have any impact on running VM first
3947     # Note: those option cannot raise errors, we we do not care about
3948     # $selection and always apply them.
3949
3950     my $add_error = sub {
3951         my ($opt, $msg) = @_;
3952         $errors->{$opt} = "hotplug problem - $msg";
3953     };
3954
3955     my $changes = 0;
3956     foreach my $opt (keys %{$conf->{pending}}) { # add/change
3957         if ($fast_plug_option->{$opt}) {
3958             $conf->{$opt} = $conf->{pending}->{$opt};
3959             delete $conf->{pending}->{$opt};
3960             $changes = 1;
3961         }
3962     }
3963
3964     if ($changes) {
3965         update_config_nolock($vmid, $conf, 1);
3966         $conf = load_config($vmid); # update/reload
3967     }
3968
3969     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3970
3971     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
3972     while (my ($opt, $force) = each %$pending_delete_hash) {
3973         next if $selection && !$selection->{$opt};
3974         eval {
3975             if ($opt eq 'hotplug') {
3976                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
3977             } elsif ($opt eq 'tablet') {
3978                 die "skip\n" if !$hotplug_features->{usb};
3979                 if ($defaults->{tablet}) {
3980                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
3981                 } else {
3982                     vm_deviceunplug($vmid, $conf, $opt);
3983                 }
3984             } elsif ($opt eq 'vcpus') {
3985                 die "skip\n" if !$hotplug_features->{cpu};
3986                 qemu_cpu_hotplug($vmid, $conf, undef);
3987             } elsif ($opt eq 'balloon') {
3988                 # enable balloon device is not hotpluggable
3989                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
3990             } elsif ($fast_plug_option->{$opt}) {
3991                 # do nothing
3992             } elsif ($opt =~ m/^net(\d+)$/) {
3993                 die "skip\n" if !$hotplug_features->{network};
3994                 vm_deviceunplug($vmid, $conf, $opt);
3995             } elsif (valid_drivename($opt)) {
3996                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
3997                 vm_deviceunplug($vmid, $conf, $opt);
3998                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
3999             } elsif ($opt =~ m/^memory$/) {
4000                 die "skip\n" if !$hotplug_features->{memory};
4001                 qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4002             } elsif ($opt eq 'cpuunits') {
4003                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4004             } elsif ($opt eq 'cpulimit') {
4005                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4006             } else {
4007                 die "skip\n";
4008             }
4009         };
4010         if (my $err = $@) {
4011             &$add_error($opt, $err) if $err ne "skip\n";
4012         } else {
4013             # save new config if hotplug was successful
4014             delete $conf->{$opt};
4015             vmconfig_undelete_pending_option($conf, $opt);
4016             update_config_nolock($vmid, $conf, 1);
4017             $conf = load_config($vmid); # update/reload
4018         }
4019     }
4020
4021     foreach my $opt (keys %{$conf->{pending}}) {
4022         next if $selection && !$selection->{$opt};
4023         my $value = $conf->{pending}->{$opt};
4024         eval {
4025             if ($opt eq 'hotplug') {
4026                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4027             } elsif ($opt eq 'tablet') {
4028                 die "skip\n" if !$hotplug_features->{usb};
4029                 if ($value == 1) {
4030                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4031                 } elsif ($value == 0) {
4032                     vm_deviceunplug($vmid, $conf, $opt);
4033                 }
4034             } elsif ($opt eq 'vcpus') {
4035                 die "skip\n" if !$hotplug_features->{cpu};
4036                 qemu_cpu_hotplug($vmid, $conf, $value);
4037             } elsif ($opt eq 'balloon') {
4038                 # enable/disable balloning device is not hotpluggable
4039                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4040                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4041                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4042
4043                 # allow manual ballooning if shares is set to zero
4044                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4045                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4046                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4047                 }
4048             } elsif ($opt =~ m/^net(\d+)$/) {
4049                 # some changes can be done without hotplug
4050                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4051                                     $vmid, $opt, $value);
4052             } elsif (valid_drivename($opt)) {
4053                 # some changes can be done without hotplug
4054                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4055                                      $vmid, $opt, $value, 1);
4056             } elsif ($opt =~ m/^memory$/) { #dimms
4057                 die "skip\n" if !$hotplug_features->{memory};
4058                 $value = qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4059             } elsif ($opt eq 'cpuunits') {
4060                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4061             } elsif ($opt eq 'cpulimit') {
4062                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4063                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4064             } else {
4065                 die "skip\n";  # skip non-hot-pluggable options
4066             }
4067         };
4068         if (my $err = $@) {
4069             &$add_error($opt, $err) if $err ne "skip\n";
4070         } else {
4071             # save new config if hotplug was successful
4072             $conf->{$opt} = $value;
4073             delete $conf->{pending}->{$opt};
4074             update_config_nolock($vmid, $conf, 1);
4075             $conf = load_config($vmid); # update/reload
4076         }
4077     }
4078 }
4079
4080 sub try_deallocate_drive {
4081     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4082
4083     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4084         my $volid = $drive->{file};
4085         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4086             my $sid = PVE::Storage::parse_volume_id($volid);
4087             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4088
4089             # check if the disk is really unused
4090             my $used_paths = PVE::QemuServer::get_used_paths($vmid, $storecfg, $conf, 1, $key);
4091             my $path = PVE::Storage::path($storecfg, $volid);
4092             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4093                    if $used_paths->{$path};
4094             PVE::Storage::vdisk_free($storecfg, $volid);
4095             return 1;
4096         } else {
4097             # If vm is not owner of this disk remove from config
4098             return 1;
4099         }
4100     }
4101
4102     return undef;
4103 }
4104
4105 sub vmconfig_delete_or_detach_drive {
4106     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4107
4108     my $drive = parse_drive($opt, $conf->{$opt});
4109
4110     my $rpcenv = PVE::RPCEnvironment::get();
4111     my $authuser = $rpcenv->get_user();
4112
4113     if ($force) {
4114         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4115         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4116     } else {
4117         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4118     }
4119 }
4120
4121 sub vmconfig_apply_pending {
4122     my ($vmid, $conf, $storecfg) = @_;
4123
4124     # cold plug
4125
4126     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4127     while (my ($opt, $force) = each %$pending_delete_hash) {
4128         die "internal error" if $opt =~ m/^unused/;
4129         $conf = load_config($vmid); # update/reload
4130         if (!defined($conf->{$opt})) {
4131             vmconfig_undelete_pending_option($conf, $opt);
4132             update_config_nolock($vmid, $conf, 1);
4133         } elsif (valid_drivename($opt)) {
4134             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4135             vmconfig_undelete_pending_option($conf, $opt);
4136             delete $conf->{$opt};
4137             update_config_nolock($vmid, $conf, 1);
4138         } else {
4139             vmconfig_undelete_pending_option($conf, $opt);
4140             delete $conf->{$opt};
4141             update_config_nolock($vmid, $conf, 1);
4142         }
4143     }
4144
4145     $conf = load_config($vmid); # update/reload
4146
4147     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4148         $conf = load_config($vmid); # update/reload
4149
4150         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4151             # skip if nothing changed
4152         } elsif (valid_drivename($opt)) {
4153             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4154                 if defined($conf->{$opt});
4155             $conf->{$opt} = $conf->{pending}->{$opt};
4156         } else {
4157             $conf->{$opt} = $conf->{pending}->{$opt};
4158         }
4159
4160         delete $conf->{pending}->{$opt};
4161         update_config_nolock($vmid, $conf, 1);
4162     }
4163 }
4164
4165 my $safe_num_ne = sub {
4166     my ($a, $b) = @_;
4167
4168     return 0 if !defined($a) && !defined($b);
4169     return 1 if !defined($a);
4170     return 1 if !defined($b);
4171
4172     return $a != $b;
4173 };
4174
4175 my $safe_string_ne = sub {
4176     my ($a, $b) = @_;
4177
4178     return 0 if !defined($a) && !defined($b);
4179     return 1 if !defined($a);
4180     return 1 if !defined($b);
4181
4182     return $a ne $b;
4183 };
4184
4185 sub vmconfig_update_net {
4186     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4187
4188     my $newnet = parse_net($value);
4189
4190     if ($conf->{$opt}) {
4191         my $oldnet = parse_net($conf->{$opt});
4192
4193         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4194             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4195             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4196             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4197
4198             # for non online change, we try to hot-unplug
4199             die "skip\n" if !$hotplug;
4200             vm_deviceunplug($vmid, $conf, $opt);
4201         } else {
4202
4203             die "internal error" if $opt !~ m/net(\d+)/;
4204             my $iface = "tap${vmid}i$1";
4205
4206             if (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4207                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4208             }
4209
4210             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4211                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4212                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4213                 PVE::Network::tap_unplug($iface);
4214                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall});
4215             }
4216
4217             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4218                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4219             }
4220
4221             return 1;
4222         }
4223     }
4224
4225     if ($hotplug) {
4226         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4227     } else {
4228         die "skip\n";
4229     }
4230 }
4231
4232 sub vmconfig_update_disk {
4233     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4234
4235     # fixme: do we need force?
4236
4237     my $drive = parse_drive($opt, $value);
4238
4239     if ($conf->{$opt}) {
4240
4241         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4242
4243             my $media = $drive->{media} || 'disk';
4244             my $oldmedia = $old_drive->{media} || 'disk';
4245             die "unable to change media type\n" if $media ne $oldmedia;
4246
4247             if (!drive_is_cdrom($old_drive)) {
4248
4249                 if ($drive->{file} ne $old_drive->{file}) {
4250
4251                     die "skip\n" if !$hotplug;
4252
4253                     # unplug and register as unused
4254                     vm_deviceunplug($vmid, $conf, $opt);
4255                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4256
4257                 } else {
4258                     # update existing disk
4259
4260                     # skip non hotpluggable value
4261                     if (&$safe_num_ne($drive->{discard}, $old_drive->{discard}) ||
4262                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4263                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4264                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4265                         die "skip\n";
4266                     }
4267
4268                     # apply throttle
4269                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4270                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4271                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4272                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4273                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4274                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4275                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4276                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4277                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4278                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4279                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4280                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max})) {
4281
4282                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4283                                                    ($drive->{mbps} || 0)*1024*1024,
4284                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4285                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4286                                                    $drive->{iops} || 0,
4287                                                    $drive->{iops_rd} || 0,
4288                                                    $drive->{iops_wr} || 0,
4289                                                    ($drive->{mbps_max} || 0)*1024*1024,
4290                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4291                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4292                                                    $drive->{iops_max} || 0,
4293                                                    $drive->{iops_rd_max} || 0,
4294                                                    $drive->{iops_wr_max} || 0);
4295
4296                     }
4297
4298                     return 1;
4299                 }
4300
4301             } else { # cdrom
4302
4303                 if ($drive->{file} eq 'none') {
4304                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4305                 } else {
4306                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4307                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4308                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4309                 }
4310
4311                 return 1;
4312             }
4313         }
4314     }
4315
4316     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4317     # hotplug new disks
4318     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]);
4319     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4320 }
4321
4322 sub vm_start {
4323     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4324         $forcemachine, $spice_ticket) = @_;
4325
4326     lock_config($vmid, sub {
4327         my $conf = load_config($vmid, $migratedfrom);
4328
4329         die "you can't start a vm if it's a template\n" if is_template($conf);
4330
4331         check_lock($conf) if !$skiplock;
4332
4333         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4334
4335         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4336             vmconfig_apply_pending($vmid, $conf, $storecfg);
4337             $conf = load_config($vmid); # update/reload
4338         }
4339
4340         my $defaults = load_defaults();
4341
4342         # set environment variable useful inside network script
4343         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4344
4345         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4346
4347         my $migrate_port = 0;
4348         my $migrate_uri;
4349         if ($statefile) {
4350             if ($statefile eq 'tcp') {
4351                 my $localip = "localhost";
4352                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4353                 my $nodename = PVE::INotify::nodename();
4354                 if ($datacenterconf->{migration_unsecure}) {
4355                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4356                         $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4357                 }
4358                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4359                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4360                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4361                 push @$cmd, '-incoming', $migrate_uri;
4362                 push @$cmd, '-S';
4363             } else {
4364                 push @$cmd, '-loadstate', $statefile;
4365             }
4366         } elsif ($paused) {
4367             push @$cmd, '-S';
4368         }
4369
4370         # host pci devices
4371         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4372           my $d = parse_hostpci($conf->{"hostpci$i"});
4373           next if !$d;
4374           my $pcidevices = $d->{pciid};
4375           foreach my $pcidevice (@$pcidevices) {
4376                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4377
4378                 my $info = pci_device_info("0000:$pciid");
4379                 die "IOMMU not present\n" if !check_iommu_support();
4380                 die "no pci device info for device '$pciid'\n" if !$info;
4381                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4382                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4383           }
4384         }
4385
4386         PVE::Storage::activate_volumes($storecfg, $vollist);
4387
4388         eval  { run_command($cmd, timeout => $statefile ? undef : 30,
4389                     umask => 0077); };
4390         my $err = $@;
4391         die "start failed: $err" if $err;
4392
4393         print "migration listens on $migrate_uri\n" if $migrate_uri;
4394
4395         if ($statefile && $statefile ne 'tcp')  {
4396             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4397             warn $@ if $@;
4398         }
4399
4400         if ($migratedfrom) {
4401
4402             eval {
4403                 set_migration_caps($vmid);
4404             };
4405             warn $@ if $@;
4406
4407             if ($spice_port) {
4408                 print "spice listens on port $spice_port\n";
4409                 if ($spice_ticket) {
4410                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4411                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4412                 }
4413             }
4414
4415         } else {
4416
4417             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4418                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4419                     if $conf->{balloon};
4420             }
4421
4422             foreach my $opt (keys %$conf) {
4423                 next if $opt !~  m/^net\d+$/;
4424                 my $nicconf = parse_net($conf->{$opt});
4425                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4426             }
4427         }
4428
4429         vm_mon_cmd_nocheck($vmid, 'qom-set',
4430                     path => "machine/peripheral/balloon0",
4431                     property => "guest-stats-polling-interval",
4432                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4433
4434     });
4435 }
4436
4437 sub vm_mon_cmd {
4438     my ($vmid, $execute, %params) = @_;
4439
4440     my $cmd = { execute => $execute, arguments => \%params };
4441     vm_qmp_command($vmid, $cmd);
4442 }
4443
4444 sub vm_mon_cmd_nocheck {
4445     my ($vmid, $execute, %params) = @_;
4446
4447     my $cmd = { execute => $execute, arguments => \%params };
4448     vm_qmp_command($vmid, $cmd, 1);
4449 }
4450
4451 sub vm_qmp_command {
4452     my ($vmid, $cmd, $nocheck) = @_;
4453
4454     my $res;
4455
4456     my $timeout;
4457     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4458         $timeout = $cmd->{arguments}->{timeout};
4459         delete $cmd->{arguments}->{timeout};
4460     }
4461
4462     eval {
4463         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4464         my $sname = qmp_socket($vmid);
4465         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4466             my $qmpclient = PVE::QMPClient->new();
4467
4468             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4469         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
4470             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
4471                 if scalar(%{$cmd->{arguments}});
4472             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
4473         } else {
4474             die "unable to open monitor socket\n";
4475         }
4476     };
4477     if (my $err = $@) {
4478         syslog("err", "VM $vmid qmp command failed - $err");
4479         die $err;
4480     }
4481
4482     return $res;
4483 }
4484
4485 sub vm_human_monitor_command {
4486     my ($vmid, $cmdline) = @_;
4487
4488     my $res;
4489
4490     my $cmd = {
4491         execute => 'human-monitor-command',
4492         arguments => { 'command-line' => $cmdline},
4493     };
4494
4495     return vm_qmp_command($vmid, $cmd);
4496 }
4497
4498 sub vm_commandline {
4499     my ($storecfg, $vmid) = @_;
4500
4501     my $conf = load_config($vmid);
4502
4503     my $defaults = load_defaults();
4504
4505     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
4506
4507     return join(' ', @$cmd);
4508 }
4509
4510 sub vm_reset {
4511     my ($vmid, $skiplock) = @_;
4512
4513     lock_config($vmid, sub {
4514
4515         my $conf = load_config($vmid);
4516
4517         check_lock($conf) if !$skiplock;
4518
4519         vm_mon_cmd($vmid, "system_reset");
4520     });
4521 }
4522
4523 sub get_vm_volumes {
4524     my ($conf) = @_;
4525
4526     my $vollist = [];
4527     foreach_volid($conf, sub {
4528         my ($volid, $is_cdrom) = @_;
4529
4530         return if $volid =~ m|^/|;
4531
4532         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4533         return if !$sid;
4534
4535         push @$vollist, $volid;
4536     });
4537
4538     return $vollist;
4539 }
4540
4541 sub vm_stop_cleanup {
4542     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
4543
4544     eval {
4545
4546         if (!$keepActive) {
4547             my $vollist = get_vm_volumes($conf);
4548             PVE::Storage::deactivate_volumes($storecfg, $vollist);
4549         }
4550
4551         foreach my $ext (qw(mon qmp pid vnc qga)) {
4552             unlink "/var/run/qemu-server/${vmid}.$ext";
4553         }
4554
4555         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
4556     };
4557     warn $@ if $@; # avoid errors - just warn
4558 }
4559
4560 # Note: use $nockeck to skip tests if VM configuration file exists.
4561 # We need that when migration VMs to other nodes (files already moved)
4562 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
4563 sub vm_stop {
4564     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
4565
4566     $force = 1 if !defined($force) && !$shutdown;
4567
4568     if ($migratedfrom){
4569         my $pid = check_running($vmid, $nocheck, $migratedfrom);
4570         kill 15, $pid if $pid;
4571         my $conf = load_config($vmid, $migratedfrom);
4572         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
4573         return;
4574     }
4575
4576     lock_config($vmid, sub {
4577
4578         my $pid = check_running($vmid, $nocheck);
4579         return if !$pid;
4580
4581         my $conf;
4582         if (!$nocheck) {
4583             $conf = load_config($vmid);
4584             check_lock($conf) if !$skiplock;
4585             if (!defined($timeout) && $shutdown && $conf->{startup}) {
4586                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
4587                 $timeout = $opts->{down} if $opts->{down};
4588             }
4589         }
4590
4591         $timeout = 60 if !defined($timeout);
4592
4593         eval {
4594             if ($shutdown) {
4595                 if (defined($conf) && $conf->{agent}) {
4596                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
4597                 } else {
4598                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
4599                 }
4600             } else {
4601                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
4602             }
4603         };
4604         my $err = $@;
4605
4606         if (!$err) {
4607             my $count = 0;
4608             while (($count < $timeout) && check_running($vmid, $nocheck)) {
4609                 $count++;
4610                 sleep 1;
4611             }
4612
4613             if ($count >= $timeout) {
4614                 if ($force) {
4615                     warn "VM still running - terminating now with SIGTERM\n";
4616                     kill 15, $pid;
4617                 } else {
4618                     die "VM quit/powerdown failed - got timeout\n";
4619                 }
4620             } else {
4621                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4622                 return;
4623             }
4624         } else {
4625             if ($force) {
4626                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
4627                 kill 15, $pid;
4628             } else {
4629                 die "VM quit/powerdown failed\n";
4630             }
4631         }
4632
4633         # wait again
4634         $timeout = 10;
4635
4636         my $count = 0;
4637         while (($count < $timeout) && check_running($vmid, $nocheck)) {
4638             $count++;
4639             sleep 1;
4640         }
4641
4642         if ($count >= $timeout) {
4643             warn "VM still running - terminating now with SIGKILL\n";
4644             kill 9, $pid;
4645             sleep 1;
4646         }
4647
4648         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4649    });
4650 }
4651
4652 sub vm_suspend {
4653     my ($vmid, $skiplock) = @_;
4654
4655     lock_config($vmid, sub {
4656
4657         my $conf = load_config($vmid);
4658
4659         check_lock($conf) if !($skiplock || ($conf->{lock} && $conf->{lock} eq 'backup'));
4660
4661         vm_mon_cmd($vmid, "stop");
4662     });
4663 }
4664
4665 sub vm_resume {
4666     my ($vmid, $skiplock, $nocheck) = @_;
4667
4668     lock_config($vmid, sub {
4669
4670         if (!$nocheck) {
4671
4672             my $conf = load_config($vmid);
4673
4674             check_lock($conf) if !($skiplock || ($conf->{lock} && $conf->{lock} eq 'backup'));
4675
4676             vm_mon_cmd($vmid, "cont");
4677
4678         } else {
4679             vm_mon_cmd_nocheck($vmid, "cont");
4680         }
4681     });
4682 }
4683
4684 sub vm_sendkey {
4685     my ($vmid, $skiplock, $key) = @_;
4686
4687     lock_config($vmid, sub {
4688
4689         my $conf = load_config($vmid);
4690
4691         # there is no qmp command, so we use the human monitor command
4692         vm_human_monitor_command($vmid, "sendkey $key");
4693     });
4694 }
4695
4696 sub vm_destroy {
4697     my ($storecfg, $vmid, $skiplock) = @_;
4698
4699     lock_config($vmid, sub {
4700
4701         my $conf = load_config($vmid);
4702
4703         check_lock($conf) if !$skiplock;
4704
4705         if (!check_running($vmid)) {
4706             destroy_vm($storecfg, $vmid);
4707         } else {
4708             die "VM $vmid is running - destroy failed\n";
4709         }
4710     });
4711 }
4712
4713 # pci helpers
4714
4715 sub file_write {
4716     my ($filename, $buf) = @_;
4717
4718     my $fh = IO::File->new($filename, "w");
4719     return undef if !$fh;
4720
4721     my $res = print $fh $buf;
4722
4723     $fh->close();
4724
4725     return $res;
4726 }
4727
4728 sub pci_device_info {
4729     my ($name) = @_;
4730
4731     my $res;
4732
4733     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
4734     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
4735
4736     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
4737     return undef if !defined($irq) || $irq !~ m/^\d+$/;
4738
4739     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
4740     return undef if !defined($vendor) || $vendor !~ s/^0x//;
4741
4742     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
4743     return undef if !defined($product) || $product !~ s/^0x//;
4744
4745     $res = {
4746         name => $name,
4747         vendor => $vendor,
4748         product => $product,
4749         domain => $domain,
4750         bus => $bus,
4751         slot => $slot,
4752         func => $func,
4753         irq => $irq,
4754         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
4755     };
4756
4757     return $res;
4758 }
4759
4760 sub pci_dev_reset {
4761     my ($dev) = @_;
4762
4763     my $name = $dev->{name};
4764
4765     my $fn = "$pcisysfs/devices/$name/reset";
4766
4767     return file_write($fn, "1");
4768 }
4769
4770 sub pci_dev_bind_to_vfio {
4771     my ($dev) = @_;
4772
4773     my $name = $dev->{name};
4774
4775     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4776
4777     if (!-d $vfio_basedir) {
4778         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4779     }
4780     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4781
4782     my $testdir = "$vfio_basedir/$name";
4783     return 1 if -d $testdir;
4784
4785     my $data = "$dev->{vendor} $dev->{product}";
4786     return undef if !file_write("$vfio_basedir/new_id", $data);
4787
4788     my $fn = "$pcisysfs/devices/$name/driver/unbind";
4789     if (!file_write($fn, $name)) {
4790         return undef if -f $fn;
4791     }
4792
4793     $fn = "$vfio_basedir/bind";
4794     if (! -d $testdir) {
4795         return undef if !file_write($fn, $name);
4796     }
4797
4798     return -d $testdir;
4799 }
4800
4801 sub pci_dev_group_bind_to_vfio {
4802     my ($pciid) = @_;
4803
4804     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4805
4806     if (!-d $vfio_basedir) {
4807         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4808     }
4809     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4810
4811     # get IOMMU group devices
4812     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
4813       my @devs = grep /^0000:/, readdir($D);
4814     closedir($D);
4815
4816     foreach my $pciid (@devs) {
4817         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
4818
4819         # pci bridges, switches or root ports are not supported
4820         # they have a pci_bus subdirectory so skip them
4821         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
4822
4823         my $info = pci_device_info($1);
4824         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
4825     }
4826
4827     return 1;
4828 }
4829
4830 sub print_pci_addr {
4831     my ($id, $bridges) = @_;
4832
4833     my $res = '';
4834     my $devices = {
4835         piix3 => { bus => 0, addr => 1 },
4836         #addr2 : first videocard
4837         balloon0 => { bus => 0, addr => 3 },
4838         watchdog => { bus => 0, addr => 4 },
4839         scsihw0 => { bus => 0, addr => 5 },
4840         'pci.3' => { bus => 0, addr => 5 }, #can also be used for virtio-scsi-single bridge
4841         scsihw1 => { bus => 0, addr => 6 },
4842         ahci0 => { bus => 0, addr => 7 },
4843         qga0 => { bus => 0, addr => 8 },
4844         spice => { bus => 0, addr => 9 },
4845         virtio0 => { bus => 0, addr => 10 },
4846         virtio1 => { bus => 0, addr => 11 },
4847         virtio2 => { bus => 0, addr => 12 },
4848         virtio3 => { bus => 0, addr => 13 },
4849         virtio4 => { bus => 0, addr => 14 },
4850         virtio5 => { bus => 0, addr => 15 },
4851         hostpci0 => { bus => 0, addr => 16 },
4852         hostpci1 => { bus => 0, addr => 17 },
4853         net0 => { bus => 0, addr => 18 },
4854         net1 => { bus => 0, addr => 19 },
4855         net2 => { bus => 0, addr => 20 },
4856         net3 => { bus => 0, addr => 21 },
4857         net4 => { bus => 0, addr => 22 },
4858         net5 => { bus => 0, addr => 23 },
4859         vga1 => { bus => 0, addr => 24 },
4860         vga2 => { bus => 0, addr => 25 },
4861         vga3 => { bus => 0, addr => 26 },
4862         hostpci2 => { bus => 0, addr => 27 },
4863         hostpci3 => { bus => 0, addr => 28 },
4864         #addr29 : usb-host (pve-usb.cfg)
4865         'pci.1' => { bus => 0, addr => 30 },
4866         'pci.2' => { bus => 0, addr => 31 },
4867         'net6' => { bus => 1, addr => 1 },
4868         'net7' => { bus => 1, addr => 2 },
4869         'net8' => { bus => 1, addr => 3 },
4870         'net9' => { bus => 1, addr => 4 },
4871         'net10' => { bus => 1, addr => 5 },
4872         'net11' => { bus => 1, addr => 6 },
4873         'net12' => { bus => 1, addr => 7 },
4874         'net13' => { bus => 1, addr => 8 },
4875         'net14' => { bus => 1, addr => 9 },
4876         'net15' => { bus => 1, addr => 10 },
4877         'net16' => { bus => 1, addr => 11 },
4878         'net17' => { bus => 1, addr => 12 },
4879         'net18' => { bus => 1, addr => 13 },
4880         'net19' => { bus => 1, addr => 14 },
4881         'net20' => { bus => 1, addr => 15 },
4882         'net21' => { bus => 1, addr => 16 },
4883         'net22' => { bus => 1, addr => 17 },
4884         'net23' => { bus => 1, addr => 18 },
4885         'net24' => { bus => 1, addr => 19 },
4886         'net25' => { bus => 1, addr => 20 },
4887         'net26' => { bus => 1, addr => 21 },
4888         'net27' => { bus => 1, addr => 22 },
4889         'net28' => { bus => 1, addr => 23 },
4890         'net29' => { bus => 1, addr => 24 },
4891         'net30' => { bus => 1, addr => 25 },
4892         'net31' => { bus => 1, addr => 26 },
4893         'virtio6' => { bus => 2, addr => 1 },
4894         'virtio7' => { bus => 2, addr => 2 },
4895         'virtio8' => { bus => 2, addr => 3 },
4896         'virtio9' => { bus => 2, addr => 4 },
4897         'virtio10' => { bus => 2, addr => 5 },
4898         'virtio11' => { bus => 2, addr => 6 },
4899         'virtio12' => { bus => 2, addr => 7 },
4900         'virtio13' => { bus => 2, addr => 8 },
4901         'virtio14' => { bus => 2, addr => 9 },
4902         'virtio15' => { bus => 2, addr => 10 },
4903         'virtioscsi0' => { bus => 3, addr => 1 },
4904         'virtioscsi1' => { bus => 3, addr => 2 },
4905         'virtioscsi2' => { bus => 3, addr => 3 },
4906         'virtioscsi3' => { bus => 3, addr => 4 },
4907         'virtioscsi4' => { bus => 3, addr => 5 },
4908         'virtioscsi5' => { bus => 3, addr => 6 },
4909         'virtioscsi6' => { bus => 3, addr => 7 },
4910         'virtioscsi7' => { bus => 3, addr => 8 },
4911         'virtioscsi8' => { bus => 3, addr => 9 },
4912         'virtioscsi9' => { bus => 3, addr => 10 },
4913         'virtioscsi10' => { bus => 3, addr => 11 },
4914         'virtioscsi11' => { bus => 3, addr => 12 },
4915         'virtioscsi12' => { bus => 3, addr => 13 },
4916         'virtioscsi13' => { bus => 3, addr => 14 },
4917         'virtioscsi14' => { bus => 3, addr => 15 },
4918         'virtioscsi15' => { bus => 3, addr => 16 },
4919         'virtioscsi16' => { bus => 3, addr => 17 },
4920         'virtioscsi17' => { bus => 3, addr => 18 },
4921         'virtioscsi18' => { bus => 3, addr => 19 },
4922         'virtioscsi19' => { bus => 3, addr => 20 },
4923         'virtioscsi20' => { bus => 3, addr => 21 },
4924         'virtioscsi21' => { bus => 3, addr => 22 },
4925         'virtioscsi22' => { bus => 3, addr => 23 },
4926         'virtioscsi23' => { bus => 3, addr => 24 },
4927         'virtioscsi24' => { bus => 3, addr => 25 },
4928         'virtioscsi25' => { bus => 3, addr => 26 },
4929         'virtioscsi26' => { bus => 3, addr => 27 },
4930         'virtioscsi27' => { bus => 3, addr => 28 },
4931         'virtioscsi28' => { bus => 3, addr => 29 },
4932         'virtioscsi29' => { bus => 3, addr => 30 },
4933         'virtioscsi30' => { bus => 3, addr => 31 },
4934
4935     };
4936
4937     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
4938            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
4939            my $bus = $devices->{$id}->{bus};
4940            $res = ",bus=pci.$bus,addr=$addr";
4941            $bridges->{$bus} = 1 if $bridges;
4942     }
4943     return $res;
4944
4945 }
4946
4947 sub print_pcie_addr {
4948     my ($id) = @_;
4949
4950     my $res = '';
4951     my $devices = {
4952         hostpci0 => { bus => "ich9-pcie-port-1", addr => 0 },
4953         hostpci1 => { bus => "ich9-pcie-port-2", addr => 0 },
4954         hostpci2 => { bus => "ich9-pcie-port-3", addr => 0 },
4955         hostpci3 => { bus => "ich9-pcie-port-4", addr => 0 },
4956     };
4957
4958     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
4959            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
4960            my $bus = $devices->{$id}->{bus};
4961            $res = ",bus=$bus,addr=$addr";
4962     }
4963     return $res;
4964
4965 }
4966
4967 # vzdump restore implementaion
4968
4969 sub tar_archive_read_firstfile {
4970     my $archive = shift;
4971
4972     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
4973
4974     # try to detect archive type first
4975     my $pid = open (TMP, "tar tf '$archive'|") ||
4976         die "unable to open file '$archive'\n";
4977     my $firstfile = <TMP>;
4978     kill 15, $pid;
4979     close TMP;
4980
4981     die "ERROR: archive contaions no data\n" if !$firstfile;
4982     chomp $firstfile;
4983
4984     return $firstfile;
4985 }
4986
4987 sub tar_restore_cleanup {
4988     my ($storecfg, $statfile) = @_;
4989
4990     print STDERR "starting cleanup\n";
4991
4992     if (my $fd = IO::File->new($statfile, "r")) {
4993         while (defined(my $line = <$fd>)) {
4994             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
4995                 my $volid = $2;
4996                 eval {
4997                     if ($volid =~ m|^/|) {
4998                         unlink $volid || die 'unlink failed\n';
4999                     } else {
5000                         PVE::Storage::vdisk_free($storecfg, $volid);
5001                     }
5002                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5003                 };
5004                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5005             } else {
5006                 print STDERR "unable to parse line in statfile - $line";
5007             }
5008         }
5009         $fd->close();
5010     }
5011 }
5012
5013 sub restore_archive {
5014     my ($archive, $vmid, $user, $opts) = @_;
5015
5016     my $format = $opts->{format};
5017     my $comp;
5018
5019     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5020         $format = 'tar' if !$format;
5021         $comp = 'gzip';
5022     } elsif ($archive =~ m/\.tar$/) {
5023         $format = 'tar' if !$format;
5024     } elsif ($archive =~ m/.tar.lzo$/) {
5025         $format = 'tar' if !$format;
5026         $comp = 'lzop';
5027     } elsif ($archive =~ m/\.vma$/) {
5028         $format = 'vma' if !$format;
5029     } elsif ($archive =~ m/\.vma\.gz$/) {
5030         $format = 'vma' if !$format;
5031         $comp = 'gzip';
5032     } elsif ($archive =~ m/\.vma\.lzo$/) {
5033         $format = 'vma' if !$format;
5034         $comp = 'lzop';
5035     } else {
5036         $format = 'vma' if !$format; # default
5037     }
5038
5039     # try to detect archive format
5040     if ($format eq 'tar') {
5041         return restore_tar_archive($archive, $vmid, $user, $opts);
5042     } else {
5043         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5044     }
5045 }
5046
5047 sub restore_update_config_line {
5048     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5049
5050     return if $line =~ m/^\#qmdump\#/;
5051     return if $line =~ m/^\#vzdump\#/;
5052     return if $line =~ m/^lock:/;
5053     return if $line =~ m/^unused\d+:/;
5054     return if $line =~ m/^parent:/;
5055     return if $line =~ m/^template:/; # restored VM is never a template
5056
5057     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5058         # try to convert old 1.X settings
5059         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5060         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5061             my ($model, $macaddr) = split(/\=/, $devconfig);
5062             $macaddr = PVE::Tools::random_ether_addr() if !$macaddr || $unique;
5063             my $net = {
5064                 model => $model,
5065                 bridge => "vmbr$ind",
5066                 macaddr => $macaddr,
5067             };
5068             my $netstr = print_net($net);
5069
5070             print $outfd "net$cookie->{netcount}: $netstr\n";
5071             $cookie->{netcount}++;
5072         }
5073     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5074         my ($id, $netstr) = ($1, $2);
5075         my $net = parse_net($netstr);
5076         $net->{macaddr} = PVE::Tools::random_ether_addr() if $net->{macaddr};
5077         $netstr = print_net($net);
5078         print $outfd "$id: $netstr\n";
5079     } elsif ($line =~ m/^((ide|scsi|virtio|sata)\d+):\s*(\S+)\s*$/) {
5080         my $virtdev = $1;
5081         my $value = $3;
5082         if ($line =~ m/backup=no/) {
5083             print $outfd "#$line";
5084         } elsif ($virtdev && $map->{$virtdev}) {
5085             my $di = parse_drive($virtdev, $value);
5086             delete $di->{format}; # format can change on restore
5087             $di->{file} = $map->{$virtdev};
5088             $value = print_drive($vmid, $di);
5089             print $outfd "$virtdev: $value\n";
5090         } else {
5091             print $outfd $line;
5092         }
5093     } else {
5094         print $outfd $line;
5095     }
5096 }
5097
5098 sub scan_volids {
5099     my ($cfg, $vmid) = @_;
5100
5101     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5102
5103     my $volid_hash = {};
5104     foreach my $storeid (keys %$info) {
5105         foreach my $item (@{$info->{$storeid}}) {
5106             next if !($item->{volid} && $item->{size});
5107             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5108             $volid_hash->{$item->{volid}} = $item;
5109         }
5110     }
5111
5112     return $volid_hash;
5113 }
5114
5115 sub get_used_paths {
5116     my ($vmid, $storecfg, $conf, $scan_snapshots, $skip_drive) = @_;
5117
5118     my $used_path = {};
5119
5120     my $scan_config = sub {
5121         my ($cref, $snapname) = @_;
5122
5123         foreach my $key (keys %$cref) {
5124             my $value = $cref->{$key};
5125             if (valid_drivename($key)) {
5126                 next if $skip_drive && $key eq $skip_drive;
5127                 my $drive = parse_drive($key, $value);
5128                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5129                 if ($drive->{file} =~ m!^/!) {
5130                     $used_path->{$drive->{file}}++; # = 1;
5131                 } else {
5132                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5133                     next if !$storeid;
5134                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5135                     next if !$scfg;
5136                     my $path = PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5137                     $used_path->{$path}++; # = 1;
5138                 }
5139             }
5140         }
5141     };
5142
5143     &$scan_config($conf);
5144
5145     undef $skip_drive;
5146
5147     if ($scan_snapshots) {
5148         foreach my $snapname (keys %{$conf->{snapshots}}) {
5149             &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5150         }
5151     }
5152
5153     return $used_path;
5154 }
5155
5156 sub update_disksize {
5157     my ($vmid, $conf, $volid_hash) = @_;
5158
5159     my $changes;
5160
5161     my $used = {};
5162
5163     # Note: it is allowed to define multiple storages with same path (alias), so
5164     # we need to check both 'volid' and real 'path' (two different volid can point
5165     # to the same path).
5166
5167     my $usedpath = {};
5168
5169     # update size info
5170     foreach my $opt (keys %$conf) {
5171         if (valid_drivename($opt)) {
5172             my $drive = parse_drive($opt, $conf->{$opt});
5173             my $volid = $drive->{file};
5174             next if !$volid;
5175
5176             $used->{$volid} = 1;
5177             if ($volid_hash->{$volid} &&
5178                 (my $path = $volid_hash->{$volid}->{path})) {
5179                 $usedpath->{$path} = 1;
5180             }
5181
5182             next if drive_is_cdrom($drive);
5183             next if !$volid_hash->{$volid};
5184
5185             $drive->{size} = $volid_hash->{$volid}->{size};
5186             my $new = print_drive($vmid, $drive);
5187             if ($new ne $conf->{$opt}) {
5188                 $changes = 1;
5189                 $conf->{$opt} = $new;
5190             }
5191         }
5192     }
5193
5194     # remove 'unusedX' entry if volume is used
5195     foreach my $opt (keys %$conf) {
5196         next if $opt !~ m/^unused\d+$/;
5197         my $volid = $conf->{$opt};
5198         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5199         if ($used->{$volid} || ($path && $usedpath->{$path})) {
5200             $changes = 1;
5201             delete $conf->{$opt};
5202         }
5203     }
5204
5205     foreach my $volid (sort keys %$volid_hash) {
5206         next if $volid =~ m/vm-$vmid-state-/;
5207         next if $used->{$volid};
5208         my $path = $volid_hash->{$volid}->{path};
5209         next if !$path; # just to be sure
5210         next if $usedpath->{$path};
5211         $changes = 1;
5212         add_unused_volume($conf, $volid);
5213         $usedpath->{$path} = 1; # avoid to add more than once (aliases)
5214     }
5215
5216     return $changes;
5217 }
5218
5219 sub rescan {
5220     my ($vmid, $nolock) = @_;
5221
5222     my $cfg = PVE::Cluster::cfs_read_file("storage.cfg");
5223
5224     my $volid_hash = scan_volids($cfg, $vmid);
5225
5226     my $updatefn =  sub {
5227         my ($vmid) = @_;
5228
5229         my $conf = load_config($vmid);
5230
5231         check_lock($conf);
5232
5233         my $vm_volids = {};
5234         foreach my $volid (keys %$volid_hash) {
5235             my $info = $volid_hash->{$volid};
5236             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5237         }
5238
5239         my $changes = update_disksize($vmid, $conf, $vm_volids);
5240
5241         update_config_nolock($vmid, $conf, 1) if $changes;
5242     };
5243
5244     if (defined($vmid)) {
5245         if ($nolock) {
5246             &$updatefn($vmid);
5247         } else {
5248             lock_config($vmid, $updatefn, $vmid);
5249         }
5250     } else {
5251         my $vmlist = config_list();
5252         foreach my $vmid (keys %$vmlist) {
5253             if ($nolock) {
5254                 &$updatefn($vmid);
5255             } else {
5256                 lock_config($vmid, $updatefn, $vmid);
5257             }
5258         }
5259     }
5260 }
5261
5262 sub restore_vma_archive {
5263     my ($archive, $vmid, $user, $opts, $comp) = @_;
5264
5265     my $input = $archive eq '-' ? "<&STDIN" : undef;
5266     my $readfrom = $archive;
5267
5268     my $uncomp = '';
5269     if ($comp) {
5270         $readfrom = '-';
5271         my $qarchive = PVE::Tools::shellquote($archive);
5272         if ($comp eq 'gzip') {
5273             $uncomp = "zcat $qarchive|";
5274         } elsif ($comp eq 'lzop') {
5275             $uncomp = "lzop -d -c $qarchive|";
5276         } else {
5277             die "unknown compression method '$comp'\n";
5278         }
5279
5280     }
5281
5282     my $tmpdir = "/var/tmp/vzdumptmp$$";
5283     rmtree $tmpdir;
5284
5285     # disable interrupts (always do cleanups)
5286     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5287         warn "got interrupt - ignored\n";
5288     };
5289
5290     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5291     POSIX::mkfifo($mapfifo, 0600);
5292     my $fifofh;
5293
5294     my $openfifo = sub {
5295         open($fifofh, '>', $mapfifo) || die $!;
5296     };
5297
5298     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5299
5300     my $oldtimeout;
5301     my $timeout = 5;
5302
5303     my $devinfo = {};
5304
5305     my $rpcenv = PVE::RPCEnvironment::get();
5306
5307     my $conffile = config_file($vmid);
5308     my $tmpfn = "$conffile.$$.tmp";
5309
5310     # Note: $oldconf is undef if VM does not exists
5311     my $oldconf = PVE::Cluster::cfs_read_file(cfs_config_path($vmid));
5312
5313     my $print_devmap = sub {
5314         my $virtdev_hash = {};
5315
5316         my $cfgfn = "$tmpdir/qemu-server.conf";
5317
5318         # we can read the config - that is already extracted
5319         my $fh = IO::File->new($cfgfn, "r") ||
5320             "unable to read qemu-server.conf - $!\n";
5321
5322         my $fwcfgfn = "$tmpdir/qemu-server.fw";
5323         PVE::Tools::file_copy($fwcfgfn, "/etc/pve/firewall/$vmid.fw")
5324             if -f $fwcfgfn;
5325
5326         while (defined(my $line = <$fh>)) {
5327             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5328                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5329                 die "archive does not contain data for drive '$virtdev'\n"
5330                     if !$devinfo->{$devname};
5331                 if (defined($opts->{storage})) {
5332                     $storeid = $opts->{storage} || 'local';
5333                 } elsif (!$storeid) {
5334                     $storeid = 'local';
5335                 }
5336                 $format = 'raw' if !$format;
5337                 $devinfo->{$devname}->{devname} = $devname;
5338                 $devinfo->{$devname}->{virtdev} = $virtdev;
5339                 $devinfo->{$devname}->{format} = $format;
5340                 $devinfo->{$devname}->{storeid} = $storeid;
5341
5342                 # check permission on storage
5343                 my $pool = $opts->{pool}; # todo: do we need that?
5344                 if ($user ne 'root@pam') {
5345                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5346                 }
5347
5348                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5349             }
5350         }
5351
5352         foreach my $devname (keys %$devinfo) {
5353             die "found no device mapping information for device '$devname'\n"
5354                 if !$devinfo->{$devname}->{virtdev};
5355         }
5356
5357         my $cfg = cfs_read_file('storage.cfg');
5358
5359         # create empty/temp config
5360         if ($oldconf) {
5361             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5362             foreach_drive($oldconf, sub {
5363                 my ($ds, $drive) = @_;
5364
5365                 return if drive_is_cdrom($drive);
5366
5367                 my $volid = $drive->{file};
5368
5369                 return if !$volid || $volid =~ m|^/|;
5370
5371                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5372                 return if !$path || !$owner || ($owner != $vmid);
5373
5374                 # Note: only delete disk we want to restore
5375                 # other volumes will become unused
5376                 if ($virtdev_hash->{$ds}) {
5377                     PVE::Storage::vdisk_free($cfg, $volid);
5378                 }
5379             });
5380         }
5381
5382         my $map = {};
5383         foreach my $virtdev (sort keys %$virtdev_hash) {
5384             my $d = $virtdev_hash->{$virtdev};
5385             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5386             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5387
5388             # test if requested format is supported
5389             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5390             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5391             $d->{format} = $defFormat if !$supported;
5392
5393             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5394                                                   $d->{format}, undef, $alloc_size);
5395             print STDERR "new volume ID is '$volid'\n";
5396             $d->{volid} = $volid;
5397             my $path = PVE::Storage::path($cfg, $volid);
5398
5399             PVE::Storage::activate_volumes($cfg,[$volid]);
5400
5401             my $write_zeros = 1;
5402             # fixme: what other storages types initialize volumes with zero?
5403             if ($scfg->{type} eq 'dir' || $scfg->{type} eq 'nfs' || $scfg->{type} eq 'glusterfs' ||
5404                 $scfg->{type} eq 'sheepdog' || $scfg->{type} eq 'rbd') {
5405                 $write_zeros = 0;
5406             }
5407
5408             print $fifofh "${write_zeros}:$d->{devname}=$path\n";
5409
5410             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5411             $map->{$virtdev} = $volid;
5412         }
5413
5414         $fh->seek(0, 0) || die "seek failed - $!\n";
5415
5416         my $outfd = new IO::File ($tmpfn, "w") ||
5417             die "unable to write config for VM $vmid\n";
5418
5419         my $cookie = { netcount => 0 };
5420         while (defined(my $line = <$fh>)) {
5421             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5422         }
5423
5424         $fh->close();
5425         $outfd->close();
5426     };
5427
5428     eval {
5429         # enable interrupts
5430         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5431             die "interrupted by signal\n";
5432         };
5433         local $SIG{ALRM} = sub { die "got timeout\n"; };
5434
5435         $oldtimeout = alarm($timeout);
5436
5437         my $parser = sub {
5438             my $line = shift;
5439
5440             print "$line\n";
5441
5442             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5443                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5444                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5445             } elsif ($line =~ m/^CTIME: /) {
5446                 # we correctly received the vma config, so we can disable
5447                 # the timeout now for disk allocation (set to 10 minutes, so
5448                 # that we always timeout if something goes wrong)
5449                 alarm(600);
5450                 &$print_devmap();
5451                 print $fifofh "done\n";
5452                 my $tmp = $oldtimeout || 0;
5453                 $oldtimeout = undef;
5454                 alarm($tmp);
5455                 close($fifofh);
5456             }
5457         };
5458
5459         print "restore vma archive: $cmd\n";
5460         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5461     };
5462     my $err = $@;
5463
5464     alarm($oldtimeout) if $oldtimeout;
5465
5466     my $vollist = [];
5467     foreach my $devname (keys %$devinfo) {
5468         my $volid = $devinfo->{$devname}->{volid};
5469         push @$vollist, $volid if $volid;
5470     }
5471
5472     my $cfg = cfs_read_file('storage.cfg');
5473     PVE::Storage::deactivate_volumes($cfg, $vollist);
5474
5475     unlink $mapfifo;
5476
5477     if ($err) {
5478         rmtree $tmpdir;
5479         unlink $tmpfn;
5480
5481         foreach my $devname (keys %$devinfo) {
5482             my $volid = $devinfo->{$devname}->{volid};
5483             next if !$volid;
5484             eval {
5485                 if ($volid =~ m|^/|) {
5486                     unlink $volid || die 'unlink failed\n';
5487                 } else {
5488                     PVE::Storage::vdisk_free($cfg, $volid);
5489                 }
5490                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5491             };
5492             print STDERR "unable to cleanup '$volid' - $@" if $@;
5493         }
5494         die $err;
5495     }
5496
5497     rmtree $tmpdir;
5498
5499     rename($tmpfn, $conffile) ||
5500         die "unable to commit configuration file '$conffile'\n";
5501
5502     PVE::Cluster::cfs_update(); # make sure we read new file
5503
5504     eval { rescan($vmid, 1); };
5505     warn $@ if $@;
5506 }
5507
5508 sub restore_tar_archive {
5509     my ($archive, $vmid, $user, $opts) = @_;
5510
5511     if ($archive ne '-') {
5512         my $firstfile = tar_archive_read_firstfile($archive);
5513         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5514             if $firstfile ne 'qemu-server.conf';
5515     }
5516
5517     my $storecfg = cfs_read_file('storage.cfg');
5518
5519     # destroy existing data - keep empty config
5520     my $vmcfgfn = config_file($vmid);
5521     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5522
5523     my $tocmd = "/usr/lib/qemu-server/qmextract";
5524
5525     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5526     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5527     $tocmd .= ' --prealloc' if $opts->{prealloc};
5528     $tocmd .= ' --info' if $opts->{info};
5529
5530     # tar option "xf" does not autodetect compression when read from STDIN,
5531     # so we pipe to zcat
5532     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5533         PVE::Tools::shellquote("--to-command=$tocmd");
5534
5535     my $tmpdir = "/var/tmp/vzdumptmp$$";
5536     mkpath $tmpdir;
5537
5538     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5539     local $ENV{VZDUMP_VMID} = $vmid;
5540     local $ENV{VZDUMP_USER} = $user;
5541
5542     my $conffile = config_file($vmid);
5543     my $tmpfn = "$conffile.$$.tmp";
5544
5545     # disable interrupts (always do cleanups)
5546     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5547         print STDERR "got interrupt - ignored\n";
5548     };
5549
5550     eval {
5551         # enable interrupts
5552         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5553             die "interrupted by signal\n";
5554         };
5555
5556         if ($archive eq '-') {
5557             print "extracting archive from STDIN\n";
5558             run_command($cmd, input => "<&STDIN");
5559         } else {
5560             print "extracting archive '$archive'\n";
5561             run_command($cmd);
5562         }
5563
5564         return if $opts->{info};
5565
5566         # read new mapping
5567         my $map = {};
5568         my $statfile = "$tmpdir/qmrestore.stat";
5569         if (my $fd = IO::File->new($statfile, "r")) {
5570             while (defined (my $line = <$fd>)) {
5571                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5572                     $map->{$1} = $2 if $1;
5573                 } else {
5574                     print STDERR "unable to parse line in statfile - $line\n";
5575                 }
5576             }
5577             $fd->close();
5578         }
5579
5580         my $confsrc = "$tmpdir/qemu-server.conf";
5581
5582         my $srcfd = new IO::File($confsrc, "r") ||
5583             die "unable to open file '$confsrc'\n";
5584
5585         my $outfd = new IO::File ($tmpfn, "w") ||
5586             die "unable to write config for VM $vmid\n";
5587
5588         my $cookie = { netcount => 0 };
5589         while (defined (my $line = <$srcfd>)) {
5590             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5591         }
5592
5593         $srcfd->close();
5594         $outfd->close();
5595     };
5596     my $err = $@;
5597
5598     if ($err) {
5599
5600         unlink $tmpfn;
5601
5602         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
5603
5604         die $err;
5605     }
5606
5607     rmtree $tmpdir;
5608
5609     rename $tmpfn, $conffile ||
5610         die "unable to commit configuration file '$conffile'\n";
5611
5612     PVE::Cluster::cfs_update(); # make sure we read new file
5613
5614     eval { rescan($vmid, 1); };
5615     warn $@ if $@;
5616 };
5617
5618
5619 # Internal snapshots
5620
5621 # NOTE: Snapshot create/delete involves several non-atomic
5622 # action, and can take a long time.
5623 # So we try to avoid locking the file and use 'lock' variable
5624 # inside the config file instead.
5625
5626 my $snapshot_copy_config = sub {
5627     my ($source, $dest) = @_;
5628
5629     foreach my $k (keys %$source) {
5630         next if $k eq 'snapshots';
5631         next if $k eq 'snapstate';
5632         next if $k eq 'snaptime';
5633         next if $k eq 'vmstate';
5634         next if $k eq 'lock';
5635         next if $k eq 'digest';
5636         next if $k eq 'description';
5637         next if $k =~ m/^unused\d+$/;
5638
5639         $dest->{$k} = $source->{$k};
5640     }
5641 };
5642
5643 my $snapshot_apply_config = sub {
5644     my ($conf, $snap) = @_;
5645
5646     # copy snapshot list
5647     my $newconf = {
5648         snapshots => $conf->{snapshots},
5649     };
5650
5651     # keep description and list of unused disks
5652     foreach my $k (keys %$conf) {
5653         next if !($k =~ m/^unused\d+$/ || $k eq 'description');
5654         $newconf->{$k} = $conf->{$k};
5655     }
5656
5657     &$snapshot_copy_config($snap, $newconf);
5658
5659     return $newconf;
5660 };
5661
5662 sub foreach_writable_storage {
5663     my ($conf, $func) = @_;
5664
5665     my $sidhash = {};
5666
5667     foreach my $ds (keys %$conf) {
5668         next if !valid_drivename($ds);
5669
5670         my $drive = parse_drive($ds, $conf->{$ds});
5671         next if !$drive;
5672         next if drive_is_cdrom($drive);
5673
5674         my $volid = $drive->{file};
5675
5676         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5677         $sidhash->{$sid} = $sid if $sid;
5678     }
5679
5680     foreach my $sid (sort keys %$sidhash) {
5681         &$func($sid);
5682     }
5683 }
5684
5685 my $alloc_vmstate_volid = sub {
5686     my ($storecfg, $vmid, $conf, $snapname) = @_;
5687
5688     # Note: we try to be smart when selecting a $target storage
5689
5690     my $target;
5691
5692     # search shared storage first
5693     foreach_writable_storage($conf, sub {
5694         my ($sid) = @_;
5695         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
5696         return if !$scfg->{shared};
5697
5698         $target = $sid if !$target || $scfg->{path}; # prefer file based storage
5699     });
5700
5701     if (!$target) {
5702         # now search local storage
5703         foreach_writable_storage($conf, sub {
5704             my ($sid) = @_;
5705             my $scfg = PVE::Storage::storage_config($storecfg, $sid);
5706             return if $scfg->{shared};
5707
5708             $target = $sid if !$target || $scfg->{path}; # prefer file based storage;
5709         });
5710     }
5711
5712     $target = 'local' if !$target;
5713
5714     my $driver_state_size = 500; # assume 32MB is enough to safe all driver state;
5715     # we abort live save after $conf->{memory}, so we need at max twice that space
5716     my $size = $conf->{memory}*2 + $driver_state_size;
5717
5718     my $name = "vm-$vmid-state-$snapname";
5719     my $scfg = PVE::Storage::storage_config($storecfg, $target);
5720     $name .= ".raw" if $scfg->{path}; # add filename extension for file base storage
5721     my $volid = PVE::Storage::vdisk_alloc($storecfg, $target, $vmid, 'raw', $name, $size*1024);
5722
5723     return $volid;
5724 };
5725
5726 my $snapshot_prepare = sub {
5727     my ($vmid, $snapname, $save_vmstate, $comment) = @_;
5728
5729     my $snap;
5730
5731     my $updatefn =  sub {
5732
5733         my $conf = load_config($vmid);
5734
5735         die "you can't take a snapshot if it's a template\n"
5736             if is_template($conf);
5737
5738         check_lock($conf);
5739
5740         $conf->{lock} = 'snapshot';
5741
5742         die "snapshot name '$snapname' already used\n"
5743             if defined($conf->{snapshots}->{$snapname});
5744
5745         my $storecfg = PVE::Storage::config();
5746         die "snapshot feature is not available" if !has_feature('snapshot', $conf, $storecfg);
5747
5748         $snap = $conf->{snapshots}->{$snapname} = {};
5749
5750         if ($save_vmstate && check_running($vmid)) {
5751             $snap->{vmstate} = &$alloc_vmstate_volid($storecfg, $vmid, $conf, $snapname);
5752         }
5753
5754         &$snapshot_copy_config($conf, $snap);
5755
5756         $snap->{snapstate} = "prepare";
5757         $snap->{snaptime} = time();
5758         $snap->{description} = $comment if $comment;
5759
5760         # always overwrite machine if we save vmstate. This makes sure we
5761         # can restore it later using correct machine type
5762         $snap->{machine} = get_current_qemu_machine($vmid) if $snap->{vmstate};
5763
5764         update_config_nolock($vmid, $conf, 1);
5765     };
5766
5767     lock_config($vmid, $updatefn);
5768
5769     return $snap;
5770 };
5771
5772 my $snapshot_commit = sub {
5773     my ($vmid, $snapname) = @_;
5774
5775     my $updatefn = sub {
5776
5777         my $conf = load_config($vmid);
5778
5779         die "missing snapshot lock\n"
5780             if !($conf->{lock} && $conf->{lock} eq 'snapshot');
5781
5782         my $has_machine_config = defined($conf->{machine});
5783
5784         my $snap = $conf->{snapshots}->{$snapname};
5785
5786         die "snapshot '$snapname' does not exist\n" if !defined($snap);
5787
5788         die "wrong snapshot state\n"
5789             if !($snap->{snapstate} && $snap->{snapstate} eq "prepare");
5790
5791         delete $snap->{snapstate};
5792         delete $conf->{lock};
5793
5794         my $newconf = &$snapshot_apply_config($conf, $snap);
5795
5796         delete $newconf->{machine} if !$has_machine_config;
5797
5798         $newconf->{parent} = $snapname;
5799
5800         update_config_nolock($vmid, $newconf, 1);
5801     };
5802
5803     lock_config($vmid, $updatefn);
5804 };
5805
5806 sub snapshot_rollback {
5807     my ($vmid, $snapname) = @_;
5808
5809     my $prepare = 1;
5810
5811     my $storecfg = PVE::Storage::config();
5812
5813     my $conf = load_config($vmid);
5814
5815     my $get_snapshot_config = sub {
5816
5817         die "you can't rollback if vm is a template\n" if is_template($conf);
5818
5819         my $res = $conf->{snapshots}->{$snapname};
5820
5821         die "snapshot '$snapname' does not exist\n" if !defined($res);
5822
5823         return $res;
5824     };
5825
5826     my $snap = &$get_snapshot_config();
5827
5828     foreach_drive($snap, sub {
5829         my ($ds, $drive) = @_;
5830
5831         return if drive_is_cdrom($drive);
5832
5833         my $volid = $drive->{file};
5834
5835         PVE::Storage::volume_rollback_is_possible($storecfg, $volid, $snapname);
5836     });
5837
5838     my $updatefn = sub {
5839
5840         $conf = load_config($vmid);
5841
5842         $snap = &$get_snapshot_config();
5843
5844         die "unable to rollback to incomplete snapshot (snapstate = $snap->{snapstate})\n"
5845             if $snap->{snapstate};
5846
5847         if ($prepare) {
5848             check_lock($conf);
5849             vm_stop($storecfg, $vmid, undef, undef, 5, undef, undef);
5850         }
5851
5852         die "unable to rollback vm $vmid: vm is running\n"
5853             if check_running($vmid);
5854
5855         if ($prepare) {
5856             $conf->{lock} = 'rollback';
5857         } else {
5858             die "got wrong lock\n" if !($conf->{lock} && $conf->{lock} eq 'rollback');
5859             delete $conf->{lock};
5860         }
5861
5862         my $forcemachine;
5863
5864         if (!$prepare) {
5865             my $has_machine_config = defined($conf->{machine});
5866
5867             # copy snapshot config to current config
5868             $conf = &$snapshot_apply_config($conf, $snap);
5869             $conf->{parent} = $snapname;
5870
5871             # Note: old code did not store 'machine', so we try to be smart
5872             # and guess the snapshot was generated with kvm 1.4 (pc-i440fx-1.4).
5873             $forcemachine = $conf->{machine} || 'pc-i440fx-1.4';
5874             # we remove the 'machine' configuration if not explicitly specified
5875             # in the original config.
5876             delete $conf->{machine} if $snap->{vmstate} && !$has_machine_config;
5877         }
5878
5879         update_config_nolock($vmid, $conf, 1);
5880
5881         if (!$prepare && $snap->{vmstate}) {
5882             my $statefile = PVE::Storage::path($storecfg, $snap->{vmstate});
5883             vm_start($storecfg, $vmid, $statefile, undef, undef, undef, $forcemachine);
5884         }
5885     };
5886
5887     lock_config($vmid, $updatefn);
5888
5889     foreach_drive($snap, sub {
5890         my ($ds, $drive) = @_;
5891
5892         return if drive_is_cdrom($drive);
5893
5894         my $volid = $drive->{file};
5895         my $device = "drive-$ds";
5896
5897         PVE::Storage::volume_snapshot_rollback($storecfg, $volid, $snapname);
5898     });
5899
5900     $prepare = 0;
5901     lock_config($vmid, $updatefn);
5902 }
5903
5904 my $savevm_wait = sub {
5905     my ($vmid) = @_;
5906
5907     for(;;) {
5908         my $stat = vm_mon_cmd_nocheck($vmid, "query-savevm");
5909         if (!$stat->{status}) {
5910             die "savevm not active\n";
5911         } elsif ($stat->{status} eq 'active') {
5912             sleep(1);
5913             next;
5914         } elsif ($stat->{status} eq 'completed') {
5915             last;
5916         } else {
5917             die "query-savevm returned status '$stat->{status}'\n";
5918         }
5919     }
5920 };
5921
5922 sub do_snapshots_with_qemu {
5923     my ($storecfg, $volid) = @_;
5924
5925     my $storage_name = PVE::Storage::parse_volume_id($volid);
5926
5927     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
5928         && !$storecfg->{ids}->{$storage_name}->{krbd}){
5929         return 1;
5930     }
5931
5932     if ($volid =~ m/\.(qcow2|qed)$/){
5933         return 1;
5934     }
5935
5936     return undef;
5937 }
5938
5939 sub snapshot_create {
5940     my ($vmid, $snapname, $save_vmstate, $comment) = @_;
5941
5942     my $snap = &$snapshot_prepare($vmid, $snapname, $save_vmstate, $comment);
5943
5944     $save_vmstate = 0 if !$snap->{vmstate}; # vm is not running
5945
5946     my $config = load_config($vmid);
5947
5948     my $running = check_running($vmid);
5949
5950     my $freezefs = $running && $config->{agent};
5951     $freezefs = 0 if $snap->{vmstate}; # not needed if we save RAM
5952
5953     my $drivehash = {};
5954
5955     if ($freezefs) {
5956         eval { vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
5957         warn "guest-fsfreeze-freeze problems - $@" if $@;
5958     }
5959
5960     eval {
5961         # create internal snapshots of all drives
5962
5963         my $storecfg = PVE::Storage::config();
5964
5965         if ($running) {
5966             if ($snap->{vmstate}) {
5967                 my $path = PVE::Storage::path($storecfg, $snap->{vmstate});
5968                 vm_mon_cmd($vmid, "savevm-start", statefile => $path);
5969                 &$savevm_wait($vmid);
5970             } else {
5971                 vm_mon_cmd($vmid, "savevm-start");
5972             }
5973         };
5974
5975         foreach_drive($snap, sub {
5976             my ($ds, $drive) = @_;
5977
5978             return if drive_is_cdrom($drive);
5979
5980             my $volid = $drive->{file};
5981             my $device = "drive-$ds";
5982
5983             qemu_volume_snapshot($vmid, $device, $storecfg, $volid, $snapname);
5984             $drivehash->{$ds} = 1;
5985        });
5986     };
5987     my $err = $@;
5988
5989     if ($running) {
5990         eval { vm_mon_cmd($vmid, "savevm-end")  };
5991         warn $@ if $@;
5992
5993         if ($freezefs) {
5994             eval { vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
5995             warn "guest-fsfreeze-thaw problems - $@" if $@;
5996         }
5997
5998         # savevm-end is async, we need to wait
5999         for (;;) {
6000             my $stat = vm_mon_cmd_nocheck($vmid, "query-savevm");
6001             if (!$stat->{bytes}) {
6002                 last;
6003             } else {
6004                 print "savevm not yet finished\n";
6005                 sleep(1);
6006                 next;
6007             }
6008         }
6009     }
6010
6011     if ($err) {
6012         warn "snapshot create failed: starting cleanup\n";
6013         eval { snapshot_delete($vmid, $snapname, 0, $drivehash); };
6014         warn $@ if $@;
6015         die $err;
6016     }
6017
6018     &$snapshot_commit($vmid, $snapname);
6019 }
6020
6021 # Note: $drivehash is only set when called from snapshot_create.
6022 sub snapshot_delete {
6023     my ($vmid, $snapname, $force, $drivehash) = @_;
6024
6025     my $prepare = 1;
6026
6027     my $snap;
6028     my $unused = [];
6029
6030     my $unlink_parent = sub {
6031         my ($confref, $new_parent) = @_;
6032
6033         if ($confref->{parent} && $confref->{parent} eq $snapname) {
6034             if ($new_parent) {
6035                 $confref->{parent} = $new_parent;
6036             } else {
6037                 delete $confref->{parent};
6038             }
6039         }
6040     };
6041
6042     my $updatefn =  sub {
6043         my ($remove_drive) = @_;
6044
6045         my $conf = load_config($vmid);
6046
6047         if (!$drivehash) {
6048             check_lock($conf);
6049             die "you can't delete a snapshot if vm is a template\n"
6050                 if is_template($conf);
6051         }
6052
6053         $snap = $conf->{snapshots}->{$snapname};
6054
6055         die "snapshot '$snapname' does not exist\n" if !defined($snap);
6056
6057         # remove parent refs
6058         if (!$prepare) {
6059             &$unlink_parent($conf, $snap->{parent});
6060             foreach my $sn (keys %{$conf->{snapshots}}) {
6061                 next if $sn eq $snapname;
6062                 &$unlink_parent($conf->{snapshots}->{$sn}, $snap->{parent});
6063             }
6064         }
6065
6066         if ($remove_drive) {
6067             if ($remove_drive eq 'vmstate') {
6068                 delete $snap->{$remove_drive};
6069             } else {
6070                 my $drive = parse_drive($remove_drive, $snap->{$remove_drive});
6071                 my $volid = $drive->{file};
6072                 delete $snap->{$remove_drive};
6073                 add_unused_volume($conf, $volid);
6074             }
6075         }
6076
6077         if ($prepare) {
6078             $snap->{snapstate} = 'delete';
6079         } else {
6080             delete $conf->{snapshots}->{$snapname};
6081             delete $conf->{lock} if $drivehash;
6082             foreach my $volid (@$unused) {
6083                 add_unused_volume($conf, $volid);
6084             }
6085         }
6086
6087         update_config_nolock($vmid, $conf, 1);
6088     };
6089
6090     lock_config($vmid, $updatefn);
6091
6092     # now remove vmstate file
6093
6094     my $storecfg = PVE::Storage::config();
6095
6096     if ($snap->{vmstate}) {
6097         eval {  PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
6098         if (my $err = $@) {
6099             die $err if !$force;
6100             warn $err;
6101         }
6102         # save changes (remove vmstate from snapshot)
6103         lock_config($vmid, $updatefn, 'vmstate') if !$force;
6104     };
6105
6106     # now remove all internal snapshots
6107     foreach_drive($snap, sub {
6108         my ($ds, $drive) = @_;
6109
6110         return if drive_is_cdrom($drive);
6111
6112         my $volid = $drive->{file};
6113         my $device = "drive-$ds";
6114
6115         if (!$drivehash || $drivehash->{$ds}) {
6116             eval { qemu_volume_snapshot_delete($vmid, $device, $storecfg, $volid, $snapname); };
6117             if (my $err = $@) {
6118                 die $err if !$force;
6119                 warn $err;
6120             }
6121         }
6122
6123         # save changes (remove drive fron snapshot)
6124         lock_config($vmid, $updatefn, $ds) if !$force;
6125         push @$unused, $volid;
6126     });
6127
6128     # now cleanup config
6129     $prepare = 0;
6130     lock_config($vmid, $updatefn);
6131 }
6132
6133 sub has_feature {
6134     my ($feature, $conf, $storecfg, $snapname, $running) = @_;
6135
6136     my $err;
6137     foreach_drive($conf, sub {
6138         my ($ds, $drive) = @_;
6139
6140         return if drive_is_cdrom($drive);
6141         my $volid = $drive->{file};
6142         $err = 1 if !PVE::Storage::volume_has_feature($storecfg, $feature, $volid, $snapname, $running);
6143     });
6144
6145     return $err ? 0 : 1;
6146 }
6147
6148 sub template_create {
6149     my ($vmid, $conf, $disk) = @_;
6150
6151     my $storecfg = PVE::Storage::config();
6152
6153     foreach_drive($conf, sub {
6154         my ($ds, $drive) = @_;
6155
6156         return if drive_is_cdrom($drive);
6157         return if $disk && $ds ne $disk;
6158
6159         my $volid = $drive->{file};
6160         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6161
6162         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6163         $drive->{file} = $voliddst;
6164         $conf->{$ds} = print_drive($vmid, $drive);
6165         update_config_nolock($vmid, $conf, 1);
6166     });
6167 }
6168
6169 sub is_template {
6170     my ($conf) = @_;
6171
6172     return 1 if defined $conf->{template} && $conf->{template} == 1;
6173 }
6174
6175 sub qemu_img_convert {
6176     my ($src_volid, $dst_volid, $size, $snapname) = @_;
6177
6178     my $storecfg = PVE::Storage::config();
6179     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6180     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6181
6182     if ($src_storeid && $dst_storeid) {
6183
6184         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6185
6186         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6187         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6188
6189         my $src_format = qemu_img_format($src_scfg, $src_volname);
6190         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6191
6192         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6193         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6194
6195         my $cmd = [];
6196         push @$cmd, '/usr/bin/qemu-img', 'convert', '-t', 'writeback', '-p', '-n';
6197         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
6198         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path, $dst_path;
6199
6200         my $parser = sub {
6201             my $line = shift;
6202             if($line =~ m/\((\S+)\/100\%\)/){
6203                 my $percent = $1;
6204                 my $transferred = int($size * $percent / 100);
6205                 my $remaining = $size - $transferred;
6206
6207                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6208             }
6209
6210         };
6211
6212         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6213         my $err = $@;
6214         die "copy failed: $err" if $err;
6215     }
6216 }
6217
6218 sub qemu_img_format {
6219     my ($scfg, $volname) = @_;
6220
6221     if ($scfg->{path} && $volname =~ m/\.(raw|cow|qcow|qcow2|qed|vmdk|cloop)$/) {
6222         return $1;
6223     } else {
6224         return "raw";
6225     }
6226 }
6227
6228 sub qemu_drive_mirror {
6229     my ($vmid, $drive, $dst_volid, $vmiddst) = @_;
6230
6231     my $storecfg = PVE::Storage::config();
6232     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6233
6234     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6235
6236     my $format = qemu_img_format($dst_scfg, $dst_volname);
6237
6238     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6239
6240     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $dst_path };
6241     $opts->{format} = $format if $format;
6242
6243     print "drive mirror is starting (scanning bitmap) : this step can take some minutes/hours, depend of disk size and storage speed\n";
6244
6245     eval {
6246     vm_mon_cmd($vmid, "drive-mirror", %$opts);
6247         while (1) {
6248             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6249             my $stat = @$stats[0];
6250             die "mirroring job seem to have die. Maybe do you have bad sectors?" if !$stat;
6251             die "error job is not mirroring" if $stat->{type} ne "mirror";
6252
6253             my $busy = $stat->{busy};
6254             my $ready = $stat->{ready};
6255
6256             if (my $total = $stat->{len}) {
6257                 my $transferred = $stat->{offset} || 0;
6258                 my $remaining = $total - $transferred;
6259                 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6260
6261                 print "transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6262             }
6263
6264
6265             if ($stat->{ready} eq 'true') {
6266
6267                 last if $vmiddst != $vmid;
6268
6269                 # try to switch the disk if source and destination are on the same guest
6270                 eval { vm_mon_cmd($vmid, "block-job-complete", device => "drive-$drive") };
6271                 last if !$@;
6272                 die $@ if $@ !~ m/cannot be completed/;
6273             }
6274             sleep 1;
6275         }
6276
6277
6278     };
6279     my $err = $@;
6280
6281     my $cancel_job = sub {
6282         vm_mon_cmd($vmid, "block-job-cancel", device => "drive-$drive");
6283         while (1) {
6284             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6285             my $stat = @$stats[0];
6286             last if !$stat;
6287             sleep 1;
6288         }
6289     };
6290
6291     if ($err) {
6292         eval { &$cancel_job(); };
6293         die "mirroring error: $err";
6294     }
6295
6296     if ($vmiddst != $vmid) {
6297         # if we clone a disk for a new target vm, we don't switch the disk
6298         &$cancel_job(); # so we call block-job-cancel
6299     }
6300 }
6301
6302 sub clone_disk {
6303     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6304         $newvmid, $storage, $format, $full, $newvollist) = @_;
6305
6306     my $newvolid;
6307
6308     if (!$full) {
6309         print "create linked clone of drive $drivename ($drive->{file})\n";
6310         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6311         push @$newvollist, $newvolid;
6312     } else {
6313         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6314         $storeid = $storage if $storage;
6315
6316         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6317         if (!$format) {
6318             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6319             $format = qemu_img_format($scfg, $volname);
6320         }
6321
6322         # test if requested format is supported - else use default
6323         my $supported = grep { $_ eq $format } @$validFormats;
6324         $format = $defFormat if !$supported;
6325
6326         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6327
6328         print "create full clone of drive $drivename ($drive->{file})\n";
6329         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $format, undef, ($size/1024));
6330         push @$newvollist, $newvolid;
6331
6332         PVE::Storage::activate_volumes($storecfg, $newvollist);
6333
6334         if (!$running || $snapname) {
6335             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname);
6336         } else {
6337             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid);
6338         }
6339     }
6340
6341     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6342
6343     my $disk = $drive;
6344     $disk->{format} = undef;
6345     $disk->{file} = $newvolid;
6346     $disk->{size} = $size;
6347
6348     return $disk;
6349 }
6350
6351 # this only works if VM is running
6352 sub get_current_qemu_machine {
6353     my ($vmid) = @_;
6354
6355     my $cmd = { execute => 'query-machines', arguments => {} };
6356     my $res = vm_qmp_command($vmid, $cmd);
6357
6358     my ($current, $default);
6359     foreach my $e (@$res) {
6360         $default = $e->{name} if $e->{'is-default'};
6361         $current = $e->{name} if $e->{'is-current'};
6362     }
6363
6364     # fallback to the default machine if current is not supported by qemu
6365     return $current || $default || 'pc';
6366 }
6367
6368 sub qemu_machine_feature_enabled {
6369     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6370
6371     my $current_major;
6372     my $current_minor;
6373
6374     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6375
6376         $current_major = $3;
6377         $current_minor = $4;
6378
6379     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6380
6381         $current_major = $1;
6382         $current_minor = $2;
6383     }
6384
6385     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
6386
6387
6388 }
6389
6390 sub qemu_machine_pxe {
6391     my ($vmid, $conf, $machine) = @_;
6392
6393     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6394
6395     foreach my $opt (keys %$conf) {
6396         next if $opt !~ m/^net(\d+)$/;
6397         my $net = PVE::QemuServer::parse_net($conf->{$opt});
6398         next if !$net;
6399         my $romfile = PVE::QemuServer::vm_mon_cmd_nocheck($vmid, 'qom-get', path => $opt, property => 'romfile');
6400         return $machine.".pxe" if $romfile =~ m/pxe/;
6401         last;
6402     }
6403
6404 }
6405
6406 sub qemu_use_old_bios_files {
6407     my ($machine_type) = @_;
6408
6409     return if !$machine_type;
6410
6411     my $use_old_bios_files = undef;
6412
6413     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6414         $machine_type = $1;
6415         $use_old_bios_files = 1;
6416     } else {
6417         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6418         # load new efi bios files on migration. So this hack is required to allow
6419         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6420         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6421         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, undef, 2, 4);
6422     }
6423
6424     return ($use_old_bios_files, $machine_type);
6425 }
6426
6427 sub lspci {
6428
6429     my $devices = {};
6430
6431     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6432             my (undef, $id, $function) = @_;
6433             my $res = { id => $id, function => $function};
6434             push @{$devices->{$id}}, $res;
6435     });
6436
6437     return $devices;
6438 }
6439
6440 sub vm_iothreads_list {
6441     my ($vmid) = @_;
6442
6443     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6444
6445     my $iothreads = {};
6446     foreach my $iothread (@$res) {
6447         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6448     }
6449
6450     return $iothreads;
6451 }
6452
6453 sub scsihw_infos {
6454     my ($conf, $drive) = @_;
6455
6456     my $maxdev = 0;
6457
6458     if ($conf->{scsihw} && ($conf->{scsihw} =~ m/^lsi/)) {
6459         $maxdev = 7;
6460     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6461         $maxdev = 1;
6462     } else {
6463         $maxdev = 256;
6464     }
6465
6466     my $controller = int($drive->{index} / $maxdev);
6467     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6468
6469     return ($maxdev, $controller, $controller_prefix);
6470 }
6471
6472 # bash completion helper
6473
6474 sub complete_backup_archives {
6475     my ($cmdname, $pname, $cvalue) = @_;
6476
6477     my $cfg = PVE::Storage::config();
6478
6479     my $storeid;
6480
6481     if ($cvalue =~ m/^([^:]+):/) {
6482         $storeid = $1;
6483     }
6484
6485     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6486
6487     my $res = [];
6488     foreach my $id (keys %$data) {
6489         foreach my $item (@{$data->{$id}}) {
6490             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6491             push @$res, $item->{volid} if defined($item->{volid});
6492         }
6493     }
6494
6495     return $res;
6496 }
6497
6498 my $complete_vmid_full = sub {
6499     my ($running) = @_;
6500
6501     my $idlist = vmstatus();
6502
6503     my $res = [];
6504
6505     foreach my $id (keys %$idlist) {
6506         my $d = $idlist->{$id};
6507         if (defined($running)) {
6508             next if $d->{template};
6509             next if $running && $d->{status} ne 'running';
6510             next if !$running && $d->{status} eq 'running';
6511         }
6512         push @$res, $id;
6513
6514     }
6515     return $res;
6516 };
6517
6518 sub complete_vmid {
6519     return &$complete_vmid_full();
6520 }
6521
6522 sub complete_vmid_stopped {
6523     return &$complete_vmid_full(0);
6524 }
6525
6526 sub complete_vmid_running {
6527     return &$complete_vmid_full(1);
6528 }
6529
6530 sub complete_storage {
6531
6532     my $cfg = PVE::Storage::config();
6533     my $ids = $cfg->{ids};
6534
6535     my $res = [];
6536     foreach my $sid (keys %$ids) {
6537         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6538         next if !$ids->{$sid}->{content}->{images};
6539         push @$res, $sid;
6540     }
6541
6542     return $res;
6543 }
6544
6545 1;