PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QMPClient;
  31 use PVE::RPCEnvironment;
  32 use Time::HiRes qw(gettimeofday);
  33 use File::Copy qw(copy);
  34
  35 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  36
  37 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  38
  39 # Note about locking: we use flock on the config file protect
  40 # against concurent actions.
  41 # Aditionaly, we have a 'lock' setting in the config file. This
  42 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  43 # allowed when such lock is set. But you can ignore this kind of
  44 # lock with the --skiplock flag.
  45
  46 cfs_register_file('/qemu-server/',
  47                   \&parse_vm_config,
  48                   \&write_vm_config);
  49
  50 PVE::JSONSchema::register_standard_option('skiplock', {
  51     description => "Ignore locks - only root is allowed to use this option.",
  52     type => 'boolean',
  53     optional => 1,
  54 });
  55
  56 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  57     description => "Some command save/restore state from this location.",
  58     type => 'string',
  59     maxLength => 128,
  60     optional => 1,
  61 });
  62
  63 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  64     description => "The name of the snapshot.",
  65     type => 'string', format => 'pve-configid',
  66     maxLength => 40,
  67 });
  68
  69 #no warnings 'redefine';
  70
  71 sub cgroups_write {
  72    my ($controller, $vmid, $option, $value) = @_;
  73
  74    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  75    PVE::ProcFSTools::write_proc_entry($path, $value);
  76
  77 }
  78
  79 my $nodename = PVE::INotify::nodename();
  80
  81 mkdir "/etc/pve/nodes/$nodename";
  82 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  83 mkdir $confdir;
  84
  85 my $var_run_tmpdir = "/var/run/qemu-server";
  86 mkdir $var_run_tmpdir;
  87
  88 my $lock_dir = "/var/lock/qemu-server";
  89 mkdir $lock_dir;
  90
  91 my $pcisysfs = "/sys/bus/pci";
  92
  93 my $confdesc = {
  94     onboot => {
  95         optional => 1,
  96         type => 'boolean',
  97         description => "Specifies whether a VM will be started during system bootup.",
  98         default => 0,
  99     },
 100     autostart => {
 101         optional => 1,
 102         type => 'boolean',
 103         description => "Automatic restart after crash (currently ignored).",
 104         default => 0,
 105     },
 106     hotplug => {
 107         optional => 1,
 108         type => 'string', format => 'pve-hotplug-features',
 109         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 110         default => 'network,disk,usb',
 111     },
 112     reboot => {
 113         optional => 1,
 114         type => 'boolean',
 115         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 116         default => 1,
 117     },
 118     lock => {
 119         optional => 1,
 120         type => 'string',
 121         description => "Lock/unlock the VM.",
 122         enum => [qw(migrate backup snapshot rollback)],
 123     },
 124     cpulimit => {
 125         optional => 1,
 126         type => 'number',
 127         description => "Limit of CPU usage. Note if the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 128         minimum => 0,
 129         maximum => 128,
 130         default => 0,
 131     },
 132     cpuunits => {
 133         optional => 1,
 134         type => 'integer',
 135         description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 136         minimum => 0,
 137         maximum => 500000,
 138         default => 1000,
 139     },
 140     memory => {
 141         optional => 1,
 142         type => 'integer',
 143         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 144         minimum => 16,
 145         default => 512,
 146     },
 147     balloon => {
 148         optional => 1,
 149         type => 'integer',
 150         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 151         minimum => 0,
 152     },
 153     shares => {
 154         optional => 1,
 155         type => 'integer',
 156         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 157         minimum => 0,
 158         maximum => 50000,
 159         default => 1000,
 160     },
 161     keyboard => {
 162         optional => 1,
 163         type => 'string',
 164         description => "Keybord layout for vnc server. Default is read from the datacenter configuration file.",
 165         enum => PVE::Tools::kvmkeymaplist(),
 166         default => 'en-us',
 167     },
 168     name => {
 169         optional => 1,
 170         type => 'string', format => 'dns-name',
 171         description => "Set a name for the VM. Only used on the configuration web interface.",
 172     },
 173     scsihw => {
 174         optional => 1,
 175         type => 'string',
 176         description => "scsi controller model",
 177         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 178         default => 'lsi',
 179     },
 180     description => {
 181         optional => 1,
 182         type => 'string',
 183         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 184     },
 185     ostype => {
 186         optional => 1,
 187         type => 'string',
 188         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 l24 l26 solaris)],
 189         description => <<EODESC,
 190 Used to enable special optimization/features for specific
 191 operating systems:
 192
 193 other  => unspecified OS
 194 wxp    => Microsoft Windows XP
 195 w2k    => Microsoft Windows 2000
 196 w2k3   => Microsoft Windows 2003
 197 w2k8   => Microsoft Windows 2008
 198 wvista => Microsoft Windows Vista
 199 win7   => Microsoft Windows 7
 200 win8   => Microsoft Windows 8/2012
 201 l24    => Linux 2.4 Kernel
 202 l26    => Linux 2.6/3.X Kernel
 203 solaris => solaris/opensolaris/openindiania kernel
 204
 205 other|l24|l26|solaris                       ... no special behaviour
 206 wxp|w2k|w2k3|w2k8|wvista|win7|win8  ... use --localtime switch
 207 EODESC
 208     },
 209     boot => {
 210         optional => 1,
 211         type => 'string',
 212         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 213         pattern => '[acdn]{1,4}',
 214         default => 'cdn',
 215     },
 216     bootdisk => {
 217         optional => 1,
 218         type => 'string', format => 'pve-qm-bootdisk',
 219         description => "Enable booting from specified disk.",
 220         pattern => '(ide|sata|scsi|virtio)\d+',
 221     },
 222     smp => {
 223         optional => 1,
 224         type => 'integer',
 225         description => "The number of CPUs. Please use option -sockets instead.",
 226         minimum => 1,
 227         default => 1,
 228     },
 229     sockets => {
 230         optional => 1,
 231         type => 'integer',
 232         description => "The number of CPU sockets.",
 233         minimum => 1,
 234         default => 1,
 235     },
 236     cores => {
 237         optional => 1,
 238         type => 'integer',
 239         description => "The number of cores per socket.",
 240         minimum => 1,
 241         default => 1,
 242     },
 243     numa => {
 244         optional => 1,
 245         type => 'boolean',
 246         description => "Enable/disable Numa.",
 247         default => 0,
 248     },
 249     vcpus => {
 250         optional => 1,
 251         type => 'integer',
 252         description => "Number of hotplugged vcpus.",
 253         minimum => 1,
 254         default => 0,
 255     },
 256     acpi => {
 257         optional => 1,
 258         type => 'boolean',
 259         description => "Enable/disable ACPI.",
 260         default => 1,
 261     },
 262     agent => {
 263         optional => 1,
 264         type => 'boolean',
 265         description => "Enable/disable Qemu GuestAgent.",
 266         default => 0,
 267     },
 268     kvm => {
 269         optional => 1,
 270         type => 'boolean',
 271         description => "Enable/disable KVM hardware virtualization.",
 272         default => 1,
 273     },
 274     tdf => {
 275         optional => 1,
 276         type => 'boolean',
 277         description => "Enable/disable time drift fix.",
 278         default => 0,
 279     },
 280     localtime => {
 281         optional => 1,
 282         type => 'boolean',
 283         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 284     },
 285     freeze => {
 286         optional => 1,
 287         type => 'boolean',
 288         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 289     },
 290     vga => {
 291         optional => 1,
 292         type => 'string',
 293         description => "Select VGA type. If you want to use high resolution modes (>= 1280x1024x16) then you should use option 'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and 'cirrur' for other OS types. Option 'qxl' enables the SPICE display sever. You can also run without any graphic card using a serial devive as terminal.",
 294         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 295     },
 296     watchdog => {
 297         optional => 1,
 298         type => 'string', format => 'pve-qm-watchdog',
 299         typetext => '[[model=]i6300esb|ib700] [,[action=]reset|shutdown|poweroff|pause|debug|none]',
 300         description => "Create a virtual hardware watchdog device.  Once enabled (by a guest action), the watchdog must be periodically polled by an agent inside the guest or else the guest will be restarted (or execute the action specified)",
 301     },
 302     startdate => {
 303         optional => 1,
 304         type => 'string',
 305         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 306         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 307         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 308         default => 'now',
 309     },
 310     startup =>  get_standard_option('pve-startup-order'),
 311     template => {
 312         optional => 1,
 313         type => 'boolean',
 314         description => "Enable/disable Template.",
 315         default => 0,
 316     },
 317     args => {
 318         optional => 1,
 319         type => 'string',
 320         description => <<EODESCR,
 321 Note: this option is for experts only. It allows you to pass arbitrary arguments to kvm, for example:
 322
 323 args: -no-reboot -no-hpet
 324 EODESCR
 325     },
 326     tablet => {
 327         optional => 1,
 328         type => 'boolean',
 329         default => 1,
 330         description => "Enable/disable the usb tablet device. This device is usually needed to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with normal VNC clients. If you're running lots of console-only guests on one host, you may consider disabling this to save some context switches. This is turned of by default if you use spice (vga=qxl).",
 331     },
 332     migrate_speed => {
 333         optional => 1,
 334         type => 'integer',
 335         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 336         minimum => 0,
 337         default => 0,
 338     },
 339     migrate_downtime => {
 340         optional => 1,
 341         type => 'number',
 342         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 343         minimum => 0,
 344         default => 0.1,
 345     },
 346     cdrom => {
 347         optional => 1,
 348         type => 'string', format => 'pve-qm-drive',
 349         typetext => 'volume',
 350         description => "This is an alias for option -ide2",
 351     },
 352     cpu => {
 353         optional => 1,
 354         description => "Emulated CPU type.",
 355         type => 'string',
 356         enum => [ qw(486 athlon pentium pentium2 pentium3 coreduo core2duo kvm32 kvm64 qemu32 qemu64 phenom Conroe Penryn Nehalem Westmere SandyBridge IvyBridge Haswell Broadwell Opteron_G1 Opteron_G2 Opteron_G3 Opteron_G4 Opteron_G5 host) ],
 357         default => 'kvm64',
 358     },
 359     parent => get_standard_option('pve-snapshot-name', {
 360         optional => 1,
 361         description => "Parent snapshot name. This is used internally, and should not be modified.",
 362     }),
 363     snaptime => {
 364         optional => 1,
 365         description => "Timestamp for snapshots.",
 366         type => 'integer',
 367         minimum => 0,
 368     },
 369     vmstate => {
 370         optional => 1,
 371         type => 'string', format => 'pve-volume-id',
 372         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 373     },
 374     machine => {
 375         description => "Specific the Qemu machine type.",
 376         type => 'string',
 377         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
 378         maxLength => 40,
 379         optional => 1,
 380     },
 381     smbios1 => {
 382         description => "Specify SMBIOS type 1 fields.",
 383         type => 'string', format => 'pve-qm-smbios1',
 384         typetext => "[manufacturer=str][,product=str][,version=str][,serial=str] [,uuid=uuid][,sku=str][,family=str]",
 385         maxLength => 256,
 386         optional => 1,
 387     },
 388     protection => {
 389         optional => 1,
 390         type => 'boolean',
 391         description => "Sets the protection flag of the VM. This will prevent the remove operation.",
 392         default => 0,
 393     },
 394     bios => {
 395         optional => 1,
 396         type => 'string',
 397         enum => [ qw(seabios ovmf) ],
 398         description => "Select BIOS implementation.",
 399         default => 'seabios',
 400     },
 401 };
 402
 403 # what about other qemu settings ?
 404 #cpu => 'string',
 405 #machine => 'string',
 406 #fda => 'file',
 407 #fdb => 'file',
 408 #mtdblock => 'file',
 409 #sd => 'file',
 410 #pflash => 'file',
 411 #snapshot => 'bool',
 412 #bootp => 'file',
 413 ##tftp => 'dir',
 414 ##smb => 'dir',
 415 #kernel => 'file',
 416 #append => 'string',
 417 #initrd => 'file',
 418 ##soundhw => 'string',
 419
 420 while (my ($k, $v) = each %$confdesc) {
 421     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 422 }
 423
 424 my $MAX_IDE_DISKS = 4;
 425 my $MAX_SCSI_DISKS = 14;
 426 my $MAX_VIRTIO_DISKS = 16;
 427 my $MAX_SATA_DISKS = 6;
 428 my $MAX_USB_DEVICES = 5;
 429 my $MAX_NETS = 32;
 430 my $MAX_UNUSED_DISKS = 8;
 431 my $MAX_HOSTPCI_DEVICES = 4;
 432 my $MAX_SERIAL_PORTS = 4;
 433 my $MAX_PARALLEL_PORTS = 3;
 434 my $MAX_NUMA = 8;
 435 my $MAX_MEM = 4194304;
 436 my $STATICMEM = 1024;
 437
 438 my $numadesc = {
 439     optional => 1,
 440     type => 'string', format => 'pve-qm-numanode',
 441     typetext => "cpus=<id[-id],memory=<mb>[[,hostnodes=<id[-id]>] [,policy=<preferred|bind|interleave>]]",
 442     description => "numa topology",
 443 };
 444 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 445
 446 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 447     $confdesc->{"numa$i"} = $numadesc;
 448 }
 449
 450 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 451                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 452                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 453 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 454
 455 my $netdesc = {
 456     optional => 1,
 457     type => 'string', format => 'pve-qm-net',
 458     typetext => "MODEL=XX:XX:XX:XX:XX:XX [,bridge=<dev>][,queues=<nbqueues>][,rate=<mbps>] [,tag=<vlanid>][,firewall=0|1],link_down=0|1]",
 459     description => <<EODESCR,
 460 Specify network devices.
 461
 462 MODEL is one of: $nic_model_list_txt
 463
 464 XX:XX:XX:XX:XX:XX should be an unique MAC address. This is
 465 automatically generated if not specified.
 466
 467 The bridge parameter can be used to automatically add the interface to a bridge device. The Proxmox VE standard bridge is called 'vmbr0'.
 468
 469 Option 'rate' is used to limit traffic bandwidth from and to this interface. It is specified as floating point number, unit is 'Megabytes per second'.
 470
 471 If you specify no bridge, we create a kvm 'user' (NATed) network device, which provides DHCP and DNS services. The following addresses are used:
 472
 473 10.0.2.2   Gateway
 474 10.0.2.3   DNS Server
 475 10.0.2.4   SMB Server
 476
 477 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 478
 479 EODESCR
 480 };
 481 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 482
 483 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 484     $confdesc->{"net$i"} = $netdesc;
 485 }
 486
 487 my $drivename_hash;
 488
 489 my $idedesc = {
 490     optional => 1,
 491     type => 'string', format => 'pve-qm-drive',
 492     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads] [,discard=ignore|on] [,detect_zeroes=on|off] [,serial=serial][,model=model]',
 493     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
 494 };
 495 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
 496
 497 my $scsidesc = {
 498     optional => 1,
 499     type => 'string', format => 'pve-qm-drive',
 500     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads] [,discard=ignore|on] [,detect_zeroes=on|off] [,iothread=on] [,queues=<nbqueues>] [,serial=serial]',
 501     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
 502 };
 503 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 504
 505 my $satadesc = {
 506     optional => 1,
 507     type => 'string', format => 'pve-qm-drive',
 508     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads]  [,discard=ignore|on] [,detect_zeroes=on|off] [,serial=serial]',
 509     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
 510 };
 511 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
 512
 513 my $virtiodesc = {
 514     optional => 1,
 515     type => 'string', format => 'pve-qm-drive',
 516     typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads]  [,discard=ignore|on] [,detect_zeroes=on|off] [,iothread=on] [,serial=serial]',
 517     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
 518 };
 519 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
 520
 521 my $usbdesc = {
 522     optional => 1,
 523     type => 'string', format => 'pve-qm-usb-device',
 524     typetext => 'host=HOSTUSBDEVICE|spice',
 525     description => <<EODESCR,
 526 Configure an USB device (n is 0 to 4). This can be used to
 527 pass-through usb devices to the guest. HOSTUSBDEVICE syntax is:
 528
 529 'bus-port(.port)*' (decimal numbers) or
 530 'vendor_id:product_id' (hexadeciaml numbers)
 531
 532 You can use the 'lsusb -t' command to list existing usb devices.
 533
 534 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 535
 536 The value 'spice' can be used to add a usb redirection devices for spice.
 537
 538 EODESCR
 539 };
 540 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
 541
 542 my $hostpcidesc = {
 543         optional => 1,
 544         type => 'string', format => 'pve-qm-hostpci',
 545         typetext => "[host=]HOSTPCIDEVICE [,rombar=on|off] [,pcie=0|1] [,x-vga=on|off]",
 546         description => <<EODESCR,
 547 Map host pci devices. HOSTPCIDEVICE syntax is:
 548
 549 'bus:dev.func' (hexadecimal numbers)
 550
 551 You can us the 'lspci' command to list existing pci devices.
 552
 553 The 'rombar' option determines whether or not the device's ROM will be visible in the guest's memory map (default is 'on').
 554
 555 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 556
 557 Experimental: user reported problems with this option.
 558 EODESCR
 559 };
 560 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
 561
 562 my $serialdesc = {
 563         optional => 1,
 564         type => 'string',
 565         pattern => '(/dev/.+|socket)',
 566         description =>  <<EODESCR,
 567 Create a serial device inside the VM (n is 0 to 3), and pass through a host serial device (i.e. /dev/ttyS0), or create a unix socket on the host side (use 'qm terminal' to open a terminal connection).
 568
 569 Note: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
 570
 571 Experimental: user reported problems with this option.
 572 EODESCR
 573 };
 574
 575 my $paralleldesc= {
 576         optional => 1,
 577         type => 'string',
 578         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
 579         description =>  <<EODESCR,
 580 Map host parallel devices (n is 0 to 2).
 581
 582 Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 583
 584 Experimental: user reported problems with this option.
 585 EODESCR
 586 };
 587
 588 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
 589     $confdesc->{"parallel$i"} = $paralleldesc;
 590 }
 591
 592 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
 593     $confdesc->{"serial$i"} = $serialdesc;
 594 }
 595
 596 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
 597     $confdesc->{"hostpci$i"} = $hostpcidesc;
 598 }
 599
 600 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
 601     $drivename_hash->{"ide$i"} = 1;
 602     $confdesc->{"ide$i"} = $idedesc;
 603 }
 604
 605 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
 606     $drivename_hash->{"sata$i"} = 1;
 607     $confdesc->{"sata$i"} = $satadesc;
 608 }
 609
 610 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
 611     $drivename_hash->{"scsi$i"} = 1;
 612     $confdesc->{"scsi$i"} = $scsidesc ;
 613 }
 614
 615 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
 616     $drivename_hash->{"virtio$i"} = 1;
 617     $confdesc->{"virtio$i"} = $virtiodesc;
 618 }
 619
 620 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
 621     $confdesc->{"usb$i"} = $usbdesc;
 622 }
 623
 624 my $unuseddesc = {
 625     optional => 1,
 626     type => 'string', format => 'pve-volume-id',
 627     description => "Reference to unused volumes.",
 628 };
 629
 630 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
 631     $confdesc->{"unused$i"} = $unuseddesc;
 632 }
 633
 634 my $kvm_api_version = 0;
 635
 636 sub kvm_version {
 637
 638     return $kvm_api_version if $kvm_api_version;
 639
 640     my $fh = IO::File->new("</dev/kvm") ||
 641         return 0;
 642
 643     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
 644         $kvm_api_version = $v;
 645     }
 646
 647     $fh->close();
 648
 649     return  $kvm_api_version;
 650 }
 651
 652 my $kvm_user_version;
 653
 654 sub kvm_user_version {
 655
 656     return $kvm_user_version if $kvm_user_version;
 657
 658     $kvm_user_version = 'unknown';
 659
 660     my $tmp = `kvm -help 2>/dev/null`;
 661
 662     if ($tmp =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
 663         $kvm_user_version = $2;
 664     }
 665
 666     return $kvm_user_version;
 667
 668 }
 669
 670 my $kernel_has_vhost_net = -c '/dev/vhost-net';
 671
 672 sub disknames {
 673     # order is important - used to autoselect boot disk
 674     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
 675             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
 676             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
 677             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))));
 678 }
 679
 680 sub valid_drivename {
 681     my $dev = shift;
 682
 683     return defined($drivename_hash->{$dev});
 684 }
 685
 686 sub option_exists {
 687     my $key = shift;
 688     return defined($confdesc->{$key});
 689 }
 690
 691 sub nic_models {
 692     return $nic_model_list;
 693 }
 694
 695 sub os_list_description {
 696
 697     return {
 698         other => 'Other',
 699         wxp => 'Windows XP',
 700         w2k => 'Windows 2000',
 701         w2k3 =>, 'Windows 2003',
 702         w2k8 => 'Windows 2008',
 703         wvista => 'Windows Vista',
 704         win7 => 'Windows 7',
 705         win8 => 'Windows 8/2012',
 706         l24 => 'Linux 2.4',
 707         l26 => 'Linux 2.6',
 708     };
 709 }
 710
 711 my $cdrom_path;
 712
 713 sub get_cdrom_path {
 714
 715     return  $cdrom_path if $cdrom_path;
 716
 717     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
 718     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
 719     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
 720 }
 721
 722 sub get_iso_path {
 723     my ($storecfg, $vmid, $cdrom) = @_;
 724
 725     if ($cdrom eq 'cdrom') {
 726         return get_cdrom_path();
 727     } elsif ($cdrom eq 'none') {
 728         return '';
 729     } elsif ($cdrom =~ m|^/|) {
 730         return $cdrom;
 731     } else {
 732         return PVE::Storage::path($storecfg, $cdrom);
 733     }
 734 }
 735
 736 # try to convert old style file names to volume IDs
 737 sub filename_to_volume_id {
 738     my ($vmid, $file, $media) = @_;
 739
 740     if (!($file eq 'none' || $file eq 'cdrom' ||
 741           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
 742
 743         return undef if $file =~ m|/|;
 744
 745         if ($media && $media eq 'cdrom') {
 746             $file = "local:iso/$file";
 747         } else {
 748             $file = "local:$vmid/$file";
 749         }
 750     }
 751
 752     return $file;
 753 }
 754
 755 sub verify_media_type {
 756     my ($opt, $vtype, $media) = @_;
 757
 758     return if !$media;
 759
 760     my $etype;
 761     if ($media eq 'disk') {
 762         $etype = 'images';
 763     } elsif ($media eq 'cdrom') {
 764         $etype = 'iso';
 765     } else {
 766         die "internal error";
 767     }
 768
 769     return if ($vtype eq $etype);
 770
 771     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
 772 }
 773
 774 sub cleanup_drive_path {
 775     my ($opt, $storecfg, $drive) = @_;
 776
 777     # try to convert filesystem paths to volume IDs
 778
 779     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
 780         ($drive->{file} !~ m|^/dev/.+|) &&
 781         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
 782         ($drive->{file} !~ m/^\d+$/)) {
 783         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
 784         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
 785         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
 786         verify_media_type($opt, $vtype, $drive->{media});
 787         $drive->{file} = $volid;
 788     }
 789
 790     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
 791 }
 792
 793 sub create_conf_nolock {
 794     my ($vmid, $settings) = @_;
 795
 796     my $filename = config_file($vmid);
 797
 798     die "configuration file '$filename' already exists\n" if -f $filename;
 799
 800     my $defaults = load_defaults();
 801
 802     $settings->{name} = "vm$vmid" if !$settings->{name};
 803     $settings->{memory} = $defaults->{memory} if !$settings->{memory};
 804
 805     my $data = '';
 806     foreach my $opt (keys %$settings) {
 807         next if !$confdesc->{$opt};
 808
 809         my $value = $settings->{$opt};
 810         next if !$value;
 811
 812         $data .= "$opt: $value\n";
 813     }
 814
 815     PVE::Tools::file_set_contents($filename, $data);
 816 }
 817
 818 sub parse_hotplug_features {
 819     my ($data) = @_;
 820
 821     my $res = {};
 822
 823     return $res if $data eq '0';
 824
 825     $data = $confdesc->{hotplug}->{default} if $data eq '1';
 826
 827     foreach my $feature (PVE::Tools::split_list($data)) {
 828         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
 829             $res->{$1} = 1;
 830         } else {
 831             warn "ignoring unknown hotplug feature '$feature'\n";
 832         }
 833     }
 834     return $res;
 835 }
 836
 837 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
 838 sub pve_verify_hotplug_features {
 839     my ($value, $noerr) = @_;
 840
 841     return $value if parse_hotplug_features($value);
 842
 843     return undef if $noerr;
 844
 845     die "unable to parse hotplug option\n";
 846 }
 847
 848 my $parse_size = sub {
 849     my ($value) = @_;
 850
 851     return undef if $value !~ m/^(\d+(\.\d+)?)([KMG])?$/;
 852     my ($size, $unit) = ($1, $3);
 853     if ($unit) {
 854         if ($unit eq 'K') {
 855             $size = $size * 1024;
 856         } elsif ($unit eq 'M') {
 857             $size = $size * 1024 * 1024;
 858         } elsif ($unit eq 'G') {
 859             $size = $size * 1024 * 1024 * 1024;
 860         }
 861     }
 862     return int($size);
 863 };
 864
 865 my $format_size = sub {
 866     my ($size) = @_;
 867
 868     $size = int($size);
 869
 870     my $kb = int($size/1024);
 871     return $size if $kb*1024 != $size;
 872
 873     my $mb = int($kb/1024);
 874     return "${kb}K" if $mb*1024 != $kb;
 875
 876     my $gb = int($mb/1024);
 877     return "${mb}M" if $gb*1024 != $mb;
 878
 879     return "${gb}G";
 880 };
 881
 882 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
 883 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
 884 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
 885 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
 886 #        [,iothread=on][,serial=serial][,model=model]
 887
 888 sub parse_drive {
 889     my ($key, $data) = @_;
 890
 891     my $res = {};
 892
 893     # $key may be undefined - used to verify JSON parameters
 894     if (!defined($key)) {
 895         $res->{interface} = 'unknown'; # should not harm when used to verify parameters
 896         $res->{index} = 0;
 897     } elsif ($key =~ m/^([^\d]+)(\d+)$/) {
 898         $res->{interface} = $1;
 899         $res->{index} = $2;
 900     } else {
 901         return undef;
 902     }
 903
 904     foreach my $p (split (/,/, $data)) {
 905         next if $p =~ m/^\s*$/;
 906
 907         if ($p =~ m/^(file|volume|cyls|heads|secs|trans|media|snapshot|cache|format|rerror|werror|backup|aio|bps|mbps|mbps_max|bps_rd|mbps_rd|mbps_rd_max|bps_wr|mbps_wr|mbps_wr_max|iops|iops_max|iops_rd|iops_rd_max|iops_wr|iops_wr_max|size|discard|detect_zeroes|iothread|queues|serial|model)=(.+)$/) {
 908             my ($k, $v) = ($1, $2);
 909
 910             $k = 'file' if $k eq 'volume';
 911
 912             return undef if defined $res->{$k};
 913
 914             if ($k eq 'bps' || $k eq 'bps_rd' || $k eq 'bps_wr') {
 915                 return undef if !$v || $v !~ m/^\d+/;
 916                 $k = "m$k";
 917                 $v = sprintf("%.3f", $v / (1024*1024));
 918             }
 919             $res->{$k} = $v;
 920         } else {
 921             if (!$res->{file} && $p !~ m/=/) {
 922                 $res->{file} = $p;
 923             } else {
 924                 return undef;
 925             }
 926         }
 927     }
 928
 929     return undef if !$res->{file};
 930
 931     return undef if $res->{cache} &&
 932         $res->{cache} !~ m/^(off|none|writethrough|writeback|unsafe|directsync)$/;
 933     return undef if $res->{snapshot} && $res->{snapshot} !~ m/^(on|off)$/;
 934     return undef if $res->{cyls} && $res->{cyls} !~ m/^\d+$/;
 935     return undef if $res->{heads} && $res->{heads} !~ m/^\d+$/;
 936     return undef if $res->{secs} && $res->{secs} !~ m/^\d+$/;
 937     return undef if $res->{media} && $res->{media} !~ m/^(disk|cdrom)$/;
 938     return undef if $res->{trans} && $res->{trans} !~ m/^(none|lba|auto)$/;
 939     return undef if $res->{format} && $res->{format} !~ m/^(raw|cow|qcow|qed|qcow2|vmdk|cloop)$/;
 940     return undef if $res->{rerror} && $res->{rerror} !~ m/^(ignore|report|stop)$/;
 941     return undef if $res->{werror} && $res->{werror} !~ m/^(enospc|ignore|report|stop)$/;
 942     return undef if $res->{backup} && $res->{backup} !~ m/^(yes|no)$/;
 943     return undef if $res->{aio} && $res->{aio} !~ m/^(native|threads)$/;
 944     return undef if $res->{discard} && $res->{discard} !~ m/^(ignore|on)$/;
 945     return undef if $res->{detect_zeroes} && $res->{detect_zeroes} !~ m/^(on|off)$/;
 946     return undef if $res->{iothread} && $res->{iothread} !~ m/^(on)$/;
 947     return undef if $res->{queues} && ($res->{queues} !~ m/^\d+$/ || $res->{queues} < 2);
 948
 949     return undef if $res->{mbps_rd} && $res->{mbps};
 950     return undef if $res->{mbps_wr} && $res->{mbps};
 951
 952     return undef if $res->{mbps} && $res->{mbps} !~ m/^\d+(\.\d+)?$/;
 953     return undef if $res->{mbps_max} && $res->{mbps_max} !~ m/^\d+(\.\d+)?$/;
 954     return undef if $res->{mbps_rd} && $res->{mbps_rd} !~ m/^\d+(\.\d+)?$/;
 955     return undef if $res->{mbps_rd_max} && $res->{mbps_rd_max} !~ m/^\d+(\.\d+)?$/;
 956     return undef if $res->{mbps_wr} && $res->{mbps_wr} !~ m/^\d+(\.\d+)?$/;
 957     return undef if $res->{mbps_wr_max} && $res->{mbps_wr_max} !~ m/^\d+(\.\d+)?$/;
 958
 959     return undef if $res->{iops_rd} && $res->{iops};
 960     return undef if $res->{iops_wr} && $res->{iops};
 961
 962
 963     return undef if $res->{iops} && $res->{iops} !~ m/^\d+$/;
 964     return undef if $res->{iops_max} && $res->{iops_max} !~ m/^\d+$/;
 965     return undef if $res->{iops_rd} && $res->{iops_rd} !~ m/^\d+$/;
 966     return undef if $res->{iops_rd_max} && $res->{iops_rd_max} !~ m/^\d+$/;
 967     return undef if $res->{iops_wr} && $res->{iops_wr} !~ m/^\d+$/;
 968     return undef if $res->{iops_wr_max} && $res->{iops_wr_max} !~ m/^\d+$/;
 969
 970     if ($res->{size}) {
 971         return undef if !defined($res->{size} = &$parse_size($res->{size}));
 972     }
 973
 974     if ($res->{media} && ($res->{media} eq 'cdrom')) {
 975         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
 976         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
 977         return undef if $res->{interface} eq 'virtio';
 978     }
 979
 980     # rerror does not work with scsi drives
 981     if ($res->{rerror}) {
 982         return undef if $res->{interface} eq 'scsi';
 983     }
 984
 985     return $res;
 986 }
 987
 988 my @qemu_drive_options = qw(heads secs cyls trans media format cache snapshot rerror werror aio discard iops iops_rd iops_wr iops_max iops_rd_max iops_wr_max serial);
 989
 990 sub print_drive {
 991     my ($vmid, $drive) = @_;
 992
 993     my $opts = '';
 994     foreach my $o (@qemu_drive_options, qw(mbps mbps_rd mbps_wr mbps_max mbps_rd_max mbps_wr_max backup iothread queues detect_zeroes)) {
 995         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
 996     }
 997
 998     if ($drive->{size}) {
 999         $opts .= ",size=" . &$format_size($drive->{size});
1000     }
1001
1002     if (my $model = $drive->{model}) {
1003         $opts .= ",model=$model";
1004     }
1005
1006     return "$drive->{file}$opts";
1007 }
1008
1009 sub scsi_inquiry {
1010     my($fh, $noerr) = @_;
1011
1012     my $SG_IO = 0x2285;
1013     my $SG_GET_VERSION_NUM = 0x2282;
1014
1015     my $versionbuf = "\x00" x 8;
1016     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1017     if (!$ret) {
1018         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1019         return undef;
1020     }
1021     my $version = unpack("I", $versionbuf);
1022     if ($version < 30000) {
1023         die "scsi generic interface too old\n"  if !$noerr;
1024         return undef;
1025     }
1026
1027     my $buf = "\x00" x 36;
1028     my $sensebuf = "\x00" x 8;
1029     my $cmd = pack("C x3 C x1", 0x12, 36);
1030
1031     # see /usr/include/scsi/sg.h
1032     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1033
1034     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1035                       length($sensebuf), 0, length($buf), $buf,
1036                       $cmd, $sensebuf, 6000);
1037
1038     $ret = ioctl($fh, $SG_IO, $packet);
1039     if (!$ret) {
1040         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1041         return undef;
1042     }
1043
1044     my @res = unpack($sg_io_hdr_t, $packet);
1045     if ($res[17] || $res[18]) {
1046         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1047         return undef;
1048     }
1049
1050     my $res = {};
1051     (my $byte0, my $byte1, $res->{vendor},
1052      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1053
1054     $res->{removable} = $byte1 & 128 ? 1 : 0;
1055     $res->{type} = $byte0 & 31;
1056
1057     return $res;
1058 }
1059
1060 sub path_is_scsi {
1061     my ($path) = @_;
1062
1063     my $fh = IO::File->new("+<$path") || return undef;
1064     my $res = scsi_inquiry($fh, 1);
1065     close($fh);
1066
1067     return $res;
1068 }
1069
1070 sub machine_type_is_q35 {
1071     my ($conf) = @_;
1072
1073     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1074 }
1075
1076 sub print_tabletdevice_full {
1077     my ($conf) = @_;
1078
1079     my $q35 = machine_type_is_q35($conf);
1080
1081     # we use uhci for old VMs because tablet driver was buggy in older qemu
1082     my $usbbus = $q35 ? "ehci" : "uhci";
1083
1084     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1085 }
1086
1087 sub print_drivedevice_full {
1088     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1089
1090     my $device = '';
1091     my $maxdev = 0;
1092
1093     if ($drive->{interface} eq 'virtio') {
1094         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1095         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1096         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1097     } elsif ($drive->{interface} eq 'scsi') {
1098
1099         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1100         my $unit = $drive->{index} % $maxdev;
1101         my $devicetype = 'hd';
1102         my $path = '';
1103         if (drive_is_cdrom($drive)) {
1104               $devicetype = 'cd';
1105         } else {
1106               if ($drive->{file} =~ m|^/|) {
1107                   $path = $drive->{file};
1108               } else {
1109                   $path = PVE::Storage::path($storecfg, $drive->{file});
1110               }
1111
1112               if($path =~ m/^iscsi\:\/\//){
1113                   $devicetype = 'generic';
1114               } else {
1115                   if (my $info = path_is_scsi($path)) {
1116                       if ($info->{type} == 0) {
1117                           $devicetype = 'block';
1118                       } elsif ($info->{type} == 1) { # tape
1119                           $devicetype = 'generic';
1120                       }
1121                   }
1122               }
1123          }
1124
1125         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1126             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1127         } else {
1128             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1129         }
1130
1131     } elsif ($drive->{interface} eq 'ide'){
1132         $maxdev = 2;
1133         my $controller = int($drive->{index} / $maxdev);
1134         my $unit = $drive->{index} % $maxdev;
1135         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1136
1137         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1138         if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
1139             $device .= ",model=$model";
1140         }
1141     } elsif ($drive->{interface} eq 'sata'){
1142         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1143         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1144         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1145     } elsif ($drive->{interface} eq 'usb') {
1146         die "implement me";
1147         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1148     } else {
1149         die "unsupported interface type";
1150     }
1151
1152     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1153
1154     return $device;
1155 }
1156
1157 sub get_initiator_name {
1158     my $initiator;
1159
1160     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1161     while (defined(my $line = <$fh>)) {
1162         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1163         $initiator = $1;
1164         last;
1165     }
1166     $fh->close();
1167
1168     return $initiator;
1169 }
1170
1171 sub print_drive_full {
1172     my ($storecfg, $vmid, $drive) = @_;
1173
1174     my $path;
1175     my $volid = $drive->{file};
1176     my $format;
1177
1178     if (drive_is_cdrom($drive)) {
1179         $path = get_iso_path($storecfg, $vmid, $volid);
1180     } else {
1181         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1182         if ($storeid) {
1183             $path = PVE::Storage::path($storecfg, $volid);
1184             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1185             $format = qemu_img_format($scfg, $volname);
1186         } else {
1187             $path = $volid;
1188         }
1189    }
1190
1191     my $opts = '';
1192     foreach my $o (@qemu_drive_options) {
1193         next if $o eq 'bootindex';
1194         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1195     }
1196
1197     $opts .= ",format=$format" if $format && !$drive->{format};
1198
1199     foreach my $o (qw(bps bps_rd bps_wr)) {
1200         my $v = $drive->{"m$o"};
1201         $opts .= ",$o=" . int($v*1024*1024) if $v;
1202     }
1203
1204     my $cache_direct = 0;
1205
1206     if (my $cache = $drive->{cache}) {
1207         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1208     } elsif (!drive_is_cdrom($drive)) {
1209         $opts .= ",cache=none";
1210         $cache_direct = 1;
1211     }
1212
1213     # aio native works only with O_DIRECT
1214     if (!$drive->{aio}) {
1215         if($cache_direct) {
1216             $opts .= ",aio=native";
1217         } else {
1218             $opts .= ",aio=threads";
1219         }
1220     }
1221
1222     if (!drive_is_cdrom($drive)) {
1223         my $detectzeroes;
1224         if ($drive->{detect_zeroes} && $drive->{detect_zeroes} eq 'off') {
1225             $detectzeroes = 'off';
1226         } elsif ($drive->{discard}) {
1227             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1228         } else {
1229             # This used to be our default with discard not being specified:
1230             $detectzeroes = 'on';
1231         }
1232         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1233     }
1234
1235     my $pathinfo = $path ? "file=$path," : '';
1236
1237     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1238 }
1239
1240 sub print_netdevice_full {
1241     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1242
1243     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1244
1245     my $device = $net->{model};
1246     if ($net->{model} eq 'virtio') {
1247          $device = 'virtio-net-pci';
1248      };
1249
1250     my $pciaddr = print_pci_addr("$netid", $bridges);
1251     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1252     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1253         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1254         my $vectors = $net->{queues} * 2 + 2;
1255         $tmpstr .= ",vectors=$vectors,mq=on";
1256     }
1257     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1258
1259     if ($use_old_bios_files) {
1260         my $romfile;
1261         if ($device eq 'virtio-net-pci') {
1262             $romfile = 'pxe-virtio.rom';
1263         } elsif ($device eq 'e1000') {
1264             $romfile = 'pxe-e1000.rom';
1265         } elsif ($device eq 'ne2k') {
1266             $romfile = 'pxe-ne2k_pci.rom';
1267         } elsif ($device eq 'pcnet') {
1268             $romfile = 'pxe-pcnet.rom';
1269         } elsif ($device eq 'rtl8139') {
1270             $romfile = 'pxe-rtl8139.rom';
1271         }
1272         $tmpstr .= ",romfile=$romfile" if $romfile;
1273     }
1274
1275     return $tmpstr;
1276 }
1277
1278 sub print_netdev_full {
1279     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1280
1281     my $i = '';
1282     if ($netid =~ m/^net(\d+)$/) {
1283         $i = int($1);
1284     }
1285
1286     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1287
1288     my $ifname = "tap${vmid}i$i";
1289
1290     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1291     die "interface name '$ifname' is too long (max 15 character)\n"
1292         if length($ifname) >= 16;
1293
1294     my $vhostparam = '';
1295     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1296
1297     my $vmname = $conf->{name} || "vm$vmid";
1298
1299     my $netdev = "";
1300     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1301
1302     if ($net->{bridge}) {
1303         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1304     } else {
1305         $netdev = "type=user,id=$netid,hostname=$vmname";
1306     }
1307
1308     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1309
1310     return $netdev;
1311 }
1312
1313 sub drive_is_cdrom {
1314     my ($drive) = @_;
1315
1316     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1317
1318 }
1319
1320 sub parse_numa {
1321     my ($data) = @_;
1322
1323     my $res = {};
1324
1325     foreach my $kvp (split(/,/, $data)) {
1326
1327         if ($kvp =~ m/^memory=(\S+)$/) {
1328             $res->{memory} = $1;
1329         } elsif ($kvp =~ m/^policy=(preferred|bind|interleave)$/) {
1330             $res->{policy} = $1;
1331         } elsif ($kvp =~ m/^cpus=(\d+)(-(\d+))?$/) {
1332             $res->{cpus}->{start} = $1;
1333             $res->{cpus}->{end} = $3;
1334         } elsif ($kvp =~ m/^hostnodes=(\d+)(-(\d+))?$/) {
1335             $res->{hostnodes}->{start} = $1;
1336             $res->{hostnodes}->{end} = $3;
1337         } else {
1338             return undef;
1339         }
1340     }
1341
1342     return $res;
1343 }
1344
1345 sub parse_hostpci {
1346     my ($value) = @_;
1347
1348     return undef if !$value;
1349
1350
1351     my @list = split(/,/, $value);
1352     my $found;
1353
1354     my $res = {};
1355     foreach my $kv (@list) {
1356
1357         if ($kv =~ m/^(host=)?([a-f0-9]{2}:[a-f0-9]{2})(\.([a-f0-9]))?$/) {
1358             $found = 1;
1359             if(defined($4)){
1360                 push @{$res->{pciid}}, { id => $2 , function => $4};
1361
1362             }else{
1363                 my $pcidevices = lspci($2);
1364                 $res->{pciid} = $pcidevices->{$2};
1365             }
1366         } elsif ($kv =~ m/^rombar=(on|off)$/) {
1367             $res->{rombar} = $1;
1368         } elsif ($kv =~ m/^x-vga=(on|off)$/) {
1369             $res->{'x-vga'} = $1;
1370         } elsif ($kv =~ m/^pcie=(\d+)$/) {
1371             $res->{pcie} = 1 if $1 == 1;
1372         } else {
1373             warn "unknown hostpci setting '$kv'\n";
1374         }
1375     }
1376
1377     return undef if !$found;
1378
1379     return $res;
1380 }
1381
1382 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1383 sub parse_net {
1384     my ($data) = @_;
1385
1386     my $res = {};
1387
1388     foreach my $kvp (split(/,/, $data)) {
1389
1390         if ($kvp =~ m/^(ne2k_pci|e1000|e1000-82540em|e1000-82544gc|e1000-82545em|rtl8139|pcnet|virtio|ne2k_isa|i82551|i82557b|i82559er|vmxnet3)(=([0-9a-f]{2}(:[0-9a-f]{2}){5}))?$/i) {
1391             my $model = lc($1);
1392             my $mac = defined($3) ? uc($3) : PVE::Tools::random_ether_addr();
1393             $res->{model} = $model;
1394             $res->{macaddr} = $mac;
1395         } elsif ($kvp =~ m/^bridge=(\S+)$/) {
1396             $res->{bridge} = $1;
1397         } elsif ($kvp =~ m/^queues=(\d+)$/) {
1398             $res->{queues} = $1;
1399         } elsif ($kvp =~ m/^rate=(\d+(\.\d+)?)$/) {
1400             $res->{rate} = $1;
1401         } elsif ($kvp =~ m/^tag=(\d+)$/) {
1402             $res->{tag} = $1;
1403         } elsif ($kvp =~ m/^firewall=([01])$/) {
1404             $res->{firewall} = $1;
1405         } elsif ($kvp =~ m/^link_down=([01])$/) {
1406             $res->{link_down} = $1;
1407         } else {
1408             return undef;
1409         }
1410
1411     }
1412
1413     return undef if !$res->{model};
1414
1415     return $res;
1416 }
1417
1418 sub print_net {
1419     my $net = shift;
1420
1421     my $res = "$net->{model}";
1422     $res .= "=$net->{macaddr}" if $net->{macaddr};
1423     $res .= ",bridge=$net->{bridge}" if $net->{bridge};
1424     $res .= ",rate=$net->{rate}" if $net->{rate};
1425     $res .= ",tag=$net->{tag}" if $net->{tag};
1426     $res .= ",firewall=1" if $net->{firewall};
1427     $res .= ",link_down=1" if $net->{link_down};
1428     $res .= ",queues=$net->{queues}" if $net->{queues};
1429
1430     return $res;
1431 }
1432
1433 sub add_random_macs {
1434     my ($settings) = @_;
1435
1436     foreach my $opt (keys %$settings) {
1437         next if $opt !~ m/^net(\d+)$/;
1438         my $net = parse_net($settings->{$opt});
1439         next if !$net;
1440         $settings->{$opt} = print_net($net);
1441     }
1442 }
1443
1444 sub add_unused_volume {
1445     my ($config, $volid) = @_;
1446
1447     my $key;
1448     for (my $ind = $MAX_UNUSED_DISKS - 1; $ind >= 0; $ind--) {
1449         my $test = "unused$ind";
1450         if (my $vid = $config->{$test}) {
1451             return if $vid eq $volid; # do not add duplicates
1452         } else {
1453             $key = $test;
1454         }
1455     }
1456
1457     die "To many unused volume - please delete them first.\n" if !$key;
1458
1459     $config->{$key} = $volid;
1460
1461     return $key;
1462 }
1463
1464 sub vm_is_volid_owner {
1465     my ($storecfg, $vmid, $volid) = @_;
1466
1467     if ($volid !~  m|^/|) {
1468         my ($path, $owner);
1469         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1470         if ($owner && ($owner == $vmid)) {
1471             return 1;
1472         }
1473     }
1474
1475     return undef;
1476 }
1477
1478 sub split_flagged_list {
1479     my $text = shift || '';
1480     $text =~ s/[,;]/ /g;
1481     $text =~ s/^\s+//;
1482     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
1483 }
1484
1485 sub join_flagged_list {
1486     my ($how, $lst) = @_;
1487     join $how, map { $lst->{$_} . $_ } keys %$lst;
1488 }
1489
1490 sub vmconfig_delete_pending_option {
1491     my ($conf, $key, $force) = @_;
1492
1493     delete $conf->{pending}->{$key};
1494     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1495     $pending_delete_hash->{$key} = $force ? '!' : '';
1496     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1497 }
1498
1499 sub vmconfig_undelete_pending_option {
1500     my ($conf, $key) = @_;
1501
1502     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1503     delete $pending_delete_hash->{$key};
1504
1505     if (%$pending_delete_hash) {
1506         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1507     } else {
1508         delete $conf->{pending}->{delete};
1509     }
1510 }
1511
1512 sub vmconfig_register_unused_drive {
1513     my ($storecfg, $vmid, $conf, $drive) = @_;
1514
1515     if (!drive_is_cdrom($drive)) {
1516         my $volid = $drive->{file};
1517         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1518             add_unused_volume($conf, $volid, $vmid);
1519         }
1520     }
1521 }
1522
1523 sub vmconfig_cleanup_pending {
1524     my ($conf) = @_;
1525
1526     # remove pending changes when nothing changed
1527     my $changes;
1528     foreach my $opt (keys %{$conf->{pending}}) {
1529         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
1530             $changes = 1;
1531             delete $conf->{pending}->{$opt};
1532         }
1533     }
1534
1535     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
1536     my $pending_delete_hash = {};
1537     while (my ($opt, $force) = each %$current_delete_hash) {
1538         if (defined($conf->{$opt})) {
1539             $pending_delete_hash->{$opt} = $force;
1540         } else {
1541             $changes = 1;
1542         }
1543     }
1544
1545     if (%$pending_delete_hash) {
1546         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1547     } else {
1548         delete $conf->{pending}->{delete};
1549     }
1550
1551     return $changes;
1552 }
1553
1554 my $valid_smbios1_options = {
1555     manufacturer => '\S+',
1556     product => '\S+',
1557     version => '\S+',
1558     serial => '\S+',
1559     uuid => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1560     sku => '\S+',
1561     family => '\S+',
1562 };
1563
1564 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
1565 sub parse_smbios1 {
1566     my ($data) = @_;
1567
1568     my $res = {};
1569
1570     foreach my $kvp (split(/,/, $data)) {
1571         return undef if $kvp !~ m/^(\S+)=(.+)$/;
1572         my ($k, $v) = split(/=/, $kvp);
1573         return undef if !defined($k) || !defined($v);
1574         return undef if !$valid_smbios1_options->{$k};
1575         return undef if $v !~ m/^$valid_smbios1_options->{$k}$/;
1576         $res->{$k} = $v;
1577     }
1578
1579     return $res;
1580 }
1581
1582 sub print_smbios1 {
1583     my ($smbios1) = @_;
1584
1585     my $data = '';
1586     foreach my $k (keys %$smbios1) {
1587         next if !defined($smbios1->{$k});
1588         next if !$valid_smbios1_options->{$k};
1589         $data .= ',' if $data;
1590         $data .= "$k=$smbios1->{$k}";
1591     }
1592     return $data;
1593 }
1594
1595 PVE::JSONSchema::register_format('pve-qm-smbios1', \&verify_smbios1);
1596 sub verify_smbios1 {
1597     my ($value, $noerr) = @_;
1598
1599     return $value if parse_smbios1($value);
1600
1601     return undef if $noerr;
1602
1603     die "unable to parse smbios (type 1) options\n";
1604 }
1605
1606 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
1607 sub verify_bootdisk {
1608     my ($value, $noerr) = @_;
1609
1610     return $value if valid_drivename($value);
1611
1612     return undef if $noerr;
1613
1614     die "invalid boot disk '$value'\n";
1615 }
1616
1617 PVE::JSONSchema::register_format('pve-qm-numanode', \&verify_numa);
1618 sub verify_numa {
1619     my ($value, $noerr) = @_;
1620
1621     return $value if parse_numa($value);
1622
1623     return undef if $noerr;
1624
1625     die "unable to parse numa options\n";
1626 }
1627
1628 PVE::JSONSchema::register_format('pve-qm-net', \&verify_net);
1629 sub verify_net {
1630     my ($value, $noerr) = @_;
1631
1632     return $value if parse_net($value);
1633
1634     return undef if $noerr;
1635
1636     die "unable to parse network options\n";
1637 }
1638
1639 PVE::JSONSchema::register_format('pve-qm-drive', \&verify_drive);
1640 sub verify_drive {
1641     my ($value, $noerr) = @_;
1642
1643     return $value if parse_drive(undef, $value);
1644
1645     return undef if $noerr;
1646
1647     die "unable to parse drive options\n";
1648 }
1649
1650 PVE::JSONSchema::register_format('pve-qm-hostpci', \&verify_hostpci);
1651 sub verify_hostpci {
1652     my ($value, $noerr) = @_;
1653
1654     return $value if parse_hostpci($value);
1655
1656     return undef if $noerr;
1657
1658     die "unable to parse pci id\n";
1659 }
1660
1661 PVE::JSONSchema::register_format('pve-qm-watchdog', \&verify_watchdog);
1662 sub verify_watchdog {
1663     my ($value, $noerr) = @_;
1664
1665     return $value if parse_watchdog($value);
1666
1667     return undef if $noerr;
1668
1669     die "unable to parse watchdog options\n";
1670 }
1671
1672 sub parse_watchdog {
1673     my ($value) = @_;
1674
1675     return undef if !$value;
1676
1677     my $res = {};
1678
1679     foreach my $p (split(/,/, $value)) {
1680         next if $p =~ m/^\s*$/;
1681
1682         if ($p =~ m/^(model=)?(i6300esb|ib700)$/) {
1683             $res->{model} = $2;
1684         } elsif ($p =~ m/^(action=)?(reset|shutdown|poweroff|pause|debug|none)$/) {
1685             $res->{action} = $2;
1686         } else {
1687             return undef;
1688         }
1689     }
1690
1691     return $res;
1692 }
1693
1694 sub parse_usb_device {
1695     my ($value) = @_;
1696
1697     return undef if !$value;
1698
1699     my @dl = split(/,/, $value);
1700     my $found;
1701
1702     my $res = {};
1703     foreach my $v (@dl) {
1704         if ($v =~ m/^host=(0x)?([0-9A-Fa-f]{4}):(0x)?([0-9A-Fa-f]{4})$/) {
1705             $found = 1;
1706             $res->{vendorid} = $2;
1707             $res->{productid} = $4;
1708         } elsif ($v =~ m/^host=(\d+)\-(\d+(\.\d+)*)$/) {
1709             $found = 1;
1710             $res->{hostbus} = $1;
1711             $res->{hostport} = $2;
1712         } elsif ($v =~ m/^spice$/) {
1713             $found = 1;
1714             $res->{spice} = 1;
1715         } else {
1716             return undef;
1717         }
1718     }
1719     return undef if !$found;
1720
1721     return $res;
1722 }
1723
1724 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
1725 sub verify_usb_device {
1726     my ($value, $noerr) = @_;
1727
1728     return $value if parse_usb_device($value);
1729
1730     return undef if $noerr;
1731
1732     die "unable to parse usb device\n";
1733 }
1734
1735 # add JSON properties for create and set function
1736 sub json_config_properties {
1737     my $prop = shift;
1738
1739     foreach my $opt (keys %$confdesc) {
1740         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
1741         $prop->{$opt} = $confdesc->{$opt};
1742     }
1743
1744     return $prop;
1745 }
1746
1747 sub check_type {
1748     my ($key, $value) = @_;
1749
1750     die "unknown setting '$key'\n" if !$confdesc->{$key};
1751
1752     my $type = $confdesc->{$key}->{type};
1753
1754     if (!defined($value)) {
1755         die "got undefined value\n";
1756     }
1757
1758     if ($value =~ m/[\n\r]/) {
1759         die "property contains a line feed\n";
1760     }
1761
1762     if ($type eq 'boolean') {
1763         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
1764         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
1765         die "type check ('boolean') failed - got '$value'\n";
1766     } elsif ($type eq 'integer') {
1767         return int($1) if $value =~ m/^(\d+)$/;
1768         die "type check ('integer') failed - got '$value'\n";
1769     } elsif ($type eq 'number') {
1770         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
1771         die "type check ('number') failed - got '$value'\n";
1772     } elsif ($type eq 'string') {
1773         if (my $fmt = $confdesc->{$key}->{format}) {
1774             if ($fmt eq 'pve-qm-drive') {
1775                 # special case - we need to pass $key to parse_drive()
1776                 my $drive = parse_drive($key, $value);
1777                 return $value if $drive;
1778                 die "unable to parse drive options\n";
1779             }
1780             PVE::JSONSchema::check_format($fmt, $value);
1781             return $value;
1782         }
1783         $value =~ s/^\"(.*)\"$/$1/;
1784         return $value;
1785     } else {
1786         die "internal error"
1787     }
1788 }
1789
1790 sub lock_config_full {
1791     my ($vmid, $timeout, $code, @param) = @_;
1792
1793     my $filename = config_file_lock($vmid);
1794
1795     my $res = lock_file($filename, $timeout, $code, @param);
1796
1797     die $@ if $@;
1798
1799     return $res;
1800 }
1801
1802 sub lock_config_mode {
1803     my ($vmid, $timeout, $shared, $code, @param) = @_;
1804
1805     my $filename = config_file_lock($vmid);
1806
1807     my $res = lock_file_full($filename, $timeout, $shared, $code, @param);
1808
1809     die $@ if $@;
1810
1811     return $res;
1812 }
1813
1814 sub lock_config {
1815     my ($vmid, $code, @param) = @_;
1816
1817     return lock_config_full($vmid, 10, $code, @param);
1818 }
1819
1820 sub cfs_config_path {
1821     my ($vmid, $node) = @_;
1822
1823     $node = $nodename if !$node;
1824     return "nodes/$node/qemu-server/$vmid.conf";
1825 }
1826
1827 sub check_iommu_support{
1828     #fixme : need to check IOMMU support
1829     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
1830
1831     my $iommu=1;
1832     return $iommu;
1833
1834 }
1835
1836 sub config_file {
1837     my ($vmid, $node) = @_;
1838
1839     my $cfspath = cfs_config_path($vmid, $node);
1840     return "/etc/pve/$cfspath";
1841 }
1842
1843 sub config_file_lock {
1844     my ($vmid) = @_;
1845
1846     return "$lock_dir/lock-$vmid.conf";
1847 }
1848
1849 sub touch_config {
1850     my ($vmid) = @_;
1851
1852     my $conf = config_file($vmid);
1853     utime undef, undef, $conf;
1854 }
1855
1856 sub destroy_vm {
1857     my ($storecfg, $vmid, $keep_empty_config) = @_;
1858
1859     my $conffile = config_file($vmid);
1860
1861     my $conf = load_config($vmid);
1862
1863     check_lock($conf);
1864
1865     # only remove disks owned by this VM
1866     foreach_drive($conf, sub {
1867         my ($ds, $drive) = @_;
1868
1869         return if drive_is_cdrom($drive);
1870
1871         my $volid = $drive->{file};
1872
1873         return if !$volid || $volid =~ m|^/|;
1874
1875         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
1876         return if !$path || !$owner || ($owner != $vmid);
1877
1878         PVE::Storage::vdisk_free($storecfg, $volid);
1879     });
1880
1881     if ($keep_empty_config) {
1882         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
1883     } else {
1884         unlink $conffile;
1885     }
1886
1887     # also remove unused disk
1888     eval {
1889         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
1890
1891         eval {
1892             PVE::Storage::foreach_volid($dl, sub {
1893                 my ($volid, $sid, $volname, $d) = @_;
1894                 PVE::Storage::vdisk_free($storecfg, $volid);
1895             });
1896         };
1897         warn $@ if $@;
1898
1899     };
1900     warn $@ if $@;
1901 }
1902
1903 sub load_config {
1904     my ($vmid, $node) = @_;
1905
1906     my $cfspath = cfs_config_path($vmid, $node);
1907
1908     my $conf = PVE::Cluster::cfs_read_file($cfspath);
1909
1910     die "no such VM ('$vmid')\n" if !defined($conf);
1911
1912     return $conf;
1913 }
1914
1915 sub parse_vm_config {
1916     my ($filename, $raw) = @_;
1917
1918     return undef if !defined($raw);
1919
1920     my $res = {
1921         digest => Digest::SHA::sha1_hex($raw),
1922         snapshots => {},
1923         pending => {},
1924     };
1925
1926     $filename =~ m|/qemu-server/(\d+)\.conf$|
1927         || die "got strange filename '$filename'";
1928
1929     my $vmid = $1;
1930
1931     my $conf = $res;
1932     my $descr;
1933     my $section = '';
1934
1935     my @lines = split(/\n/, $raw);
1936     foreach my $line (@lines) {
1937         next if $line =~ m/^\s*$/;
1938
1939         if ($line =~ m/^\[PENDING\]\s*$/i) {
1940             $section = 'pending';
1941             if (defined($descr)) {
1942                 $descr =~ s/\s+$//;
1943                 $conf->{description} = $descr;
1944             }
1945             $descr = undef;
1946             $conf = $res->{$section} = {};
1947             next;
1948
1949         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
1950             $section = $1;
1951             if (defined($descr)) {
1952                 $descr =~ s/\s+$//;
1953                 $conf->{description} = $descr;
1954             }
1955             $descr = undef;
1956             $conf = $res->{snapshots}->{$section} = {};
1957             next;
1958         }
1959
1960         if ($line =~ m/^\#(.*)\s*$/) {
1961             $descr = '' if !defined($descr);
1962             $descr .= PVE::Tools::decode_text($1) . "\n";
1963             next;
1964         }
1965
1966         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
1967             $descr = '' if !defined($descr);
1968             $descr .= PVE::Tools::decode_text($2);
1969         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
1970             $conf->{snapstate} = $1;
1971         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
1972             my $key = $1;
1973             my $value = $2;
1974             $conf->{$key} = $value;
1975         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
1976             my $value = $1;
1977             if ($section eq 'pending') {
1978                 $conf->{delete} = $value; # we parse this later
1979             } else {
1980                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
1981             }
1982         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
1983             my $key = $1;
1984             my $value = $2;
1985             eval { $value = check_type($key, $value); };
1986             if ($@) {
1987                 warn "vm $vmid - unable to parse value of '$key' - $@";
1988             } else {
1989                 my $fmt = $confdesc->{$key}->{format};
1990                 if ($fmt && $fmt eq 'pve-qm-drive') {
1991                     my $v = parse_drive($key, $value);
1992                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
1993                         $v->{file} = $volid;
1994                         $value = print_drive($vmid, $v);
1995                     } else {
1996                         warn "vm $vmid - unable to parse value of '$key'\n";
1997                         next;
1998                     }
1999                 }
2000
2001                 if ($key eq 'cdrom') {
2002                     $conf->{ide2} = $value;
2003                 } else {
2004                     $conf->{$key} = $value;
2005                 }
2006             }
2007         }
2008     }
2009
2010     if (defined($descr)) {
2011         $descr =~ s/\s+$//;
2012         $conf->{description} = $descr;
2013     }
2014     delete $res->{snapstate}; # just to be sure
2015
2016     return $res;
2017 }
2018
2019 sub write_vm_config {
2020     my ($filename, $conf) = @_;
2021
2022     delete $conf->{snapstate}; # just to be sure
2023
2024     if ($conf->{cdrom}) {
2025         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2026         $conf->{ide2} = $conf->{cdrom};
2027         delete $conf->{cdrom};
2028     }
2029
2030     # we do not use 'smp' any longer
2031     if ($conf->{sockets}) {
2032         delete $conf->{smp};
2033     } elsif ($conf->{smp}) {
2034         $conf->{sockets} = $conf->{smp};
2035         delete $conf->{cores};
2036         delete $conf->{smp};
2037     }
2038
2039     my $used_volids = {};
2040
2041     my $cleanup_config = sub {
2042         my ($cref, $pending, $snapname) = @_;
2043
2044         foreach my $key (keys %$cref) {
2045             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2046                 $key eq 'snapstate' || $key eq 'pending';
2047             my $value = $cref->{$key};
2048             if ($key eq 'delete') {
2049                 die "propertry 'delete' is only allowed in [PENDING]\n"
2050                     if !$pending;
2051                 # fixme: check syntax?
2052                 next;
2053             }
2054             eval { $value = check_type($key, $value); };
2055             die "unable to parse value of '$key' - $@" if $@;
2056
2057             $cref->{$key} = $value;
2058
2059             if (!$snapname && valid_drivename($key)) {
2060                 my $drive = parse_drive($key, $value);
2061                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2062             }
2063         }
2064     };
2065
2066     &$cleanup_config($conf);
2067
2068     &$cleanup_config($conf->{pending}, 1);
2069
2070     foreach my $snapname (keys %{$conf->{snapshots}}) {
2071         die "internal error" if $snapname eq 'pending';
2072         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2073     }
2074
2075     # remove 'unusedX' settings if we re-add a volume
2076     foreach my $key (keys %$conf) {
2077         my $value = $conf->{$key};
2078         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2079             delete $conf->{$key};
2080         }
2081     }
2082
2083     my $generate_raw_config = sub {
2084         my ($conf, $pending) = @_;
2085
2086         my $raw = '';
2087
2088         # add description as comment to top of file
2089         if (defined(my $descr = $conf->{description})) {
2090             if ($descr) {
2091                 foreach my $cl (split(/\n/, $descr)) {
2092                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2093                 }
2094             } else {
2095                 $raw .= "#\n" if $pending;
2096             }
2097         }
2098
2099         foreach my $key (sort keys %$conf) {
2100             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2101             $raw .= "$key: $conf->{$key}\n";
2102         }
2103         return $raw;
2104     };
2105
2106     my $raw = &$generate_raw_config($conf);
2107
2108     if (scalar(keys %{$conf->{pending}})){
2109         $raw .= "\n[PENDING]\n";
2110         $raw .= &$generate_raw_config($conf->{pending}, 1);
2111     }
2112
2113     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2114         $raw .= "\n[$snapname]\n";
2115         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2116     }
2117
2118     return $raw;
2119 }
2120
2121 sub update_config_nolock {
2122     my ($vmid, $conf, $skiplock) = @_;
2123
2124     check_lock($conf) if !$skiplock;
2125
2126     my $cfspath = cfs_config_path($vmid);
2127
2128     PVE::Cluster::cfs_write_file($cfspath, $conf);
2129 }
2130
2131 sub update_config {
2132     my ($vmid, $conf, $skiplock) = @_;
2133
2134     lock_config($vmid, &update_config_nolock, $conf, $skiplock);
2135 }
2136
2137 sub load_defaults {
2138
2139     my $res = {};
2140
2141     # we use static defaults from our JSON schema configuration
2142     foreach my $key (keys %$confdesc) {
2143         if (defined(my $default = $confdesc->{$key}->{default})) {
2144             $res->{$key} = $default;
2145         }
2146     }
2147
2148     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2149     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2150
2151     return $res;
2152 }
2153
2154 sub config_list {
2155     my $vmlist = PVE::Cluster::get_vmlist();
2156     my $res = {};
2157     return $res if !$vmlist || !$vmlist->{ids};
2158     my $ids = $vmlist->{ids};
2159
2160     foreach my $vmid (keys %$ids) {
2161         my $d = $ids->{$vmid};
2162         next if !$d->{node} || $d->{node} ne $nodename;
2163         next if !$d->{type} || $d->{type} ne 'qemu';
2164         $res->{$vmid}->{exists} = 1;
2165     }
2166     return $res;
2167 }
2168
2169 # test if VM uses local resources (to prevent migration)
2170 sub check_local_resources {
2171     my ($conf, $noerr) = @_;
2172
2173     my $loc_res = 0;
2174
2175     $loc_res = 1 if $conf->{hostusb}; # old syntax
2176     $loc_res = 1 if $conf->{hostpci}; # old syntax
2177
2178     foreach my $k (keys %$conf) {
2179         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2180         # sockets are safe: they will recreated be on the target side post-migrate
2181         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2182         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2183     }
2184
2185     die "VM uses local resources\n" if $loc_res && !$noerr;
2186
2187     return $loc_res;
2188 }
2189
2190 # check if used storages are available on all nodes (use by migrate)
2191 sub check_storage_availability {
2192     my ($storecfg, $conf, $node) = @_;
2193
2194     foreach_drive($conf, sub {
2195         my ($ds, $drive) = @_;
2196
2197         my $volid = $drive->{file};
2198         return if !$volid;
2199
2200         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2201         return if !$sid;
2202
2203         # check if storage is available on both nodes
2204         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2205         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2206    });
2207 }
2208
2209 # list nodes where all VM images are available (used by has_feature API)
2210 sub shared_nodes {
2211     my ($conf, $storecfg) = @_;
2212
2213     my $nodelist = PVE::Cluster::get_nodelist();
2214     my $nodehash = { map { $_ => 1 } @$nodelist };
2215     my $nodename = PVE::INotify::nodename();
2216
2217     foreach_drive($conf, sub {
2218         my ($ds, $drive) = @_;
2219
2220         my $volid = $drive->{file};
2221         return if !$volid;
2222
2223         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2224         if ($storeid) {
2225             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2226             if ($scfg->{disable}) {
2227                 $nodehash = {};
2228             } elsif (my $avail = $scfg->{nodes}) {
2229                 foreach my $node (keys %$nodehash) {
2230                     delete $nodehash->{$node} if !$avail->{$node};
2231                 }
2232             } elsif (!$scfg->{shared}) {
2233                 foreach my $node (keys %$nodehash) {
2234                     delete $nodehash->{$node} if $node ne $nodename
2235                 }
2236             }
2237         }
2238     });
2239
2240     return $nodehash
2241 }
2242
2243 sub check_lock {
2244     my ($conf) = @_;
2245
2246     die "VM is locked ($conf->{lock})\n" if $conf->{lock};
2247 }
2248
2249 sub check_cmdline {
2250     my ($pidfile, $pid) = @_;
2251
2252     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2253     if (defined($fh)) {
2254         my $line = <$fh>;
2255         $fh->close;
2256         return undef if !$line;
2257         my @param = split(/\0/, $line);
2258
2259         my $cmd = $param[0];
2260         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2261
2262         for (my $i = 0; $i < scalar (@param); $i++) {
2263             my $p = $param[$i];
2264             next if !$p;
2265             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2266                 my $p = $param[$i+1];
2267                 return 1 if $p && ($p eq $pidfile);
2268                 return undef;
2269             }
2270         }
2271     }
2272     return undef;
2273 }
2274
2275 sub check_running {
2276     my ($vmid, $nocheck, $node) = @_;
2277
2278     my $filename = config_file($vmid, $node);
2279
2280     die "unable to find configuration file for VM $vmid - no such machine\n"
2281         if !$nocheck && ! -f $filename;
2282
2283     my $pidfile = pidfile_name($vmid);
2284
2285     if (my $fd = IO::File->new("<$pidfile")) {
2286         my $st = stat($fd);
2287         my $line = <$fd>;
2288         close($fd);
2289
2290         my $mtime = $st->mtime;
2291         if ($mtime > time()) {
2292             warn "file '$filename' modified in future\n";
2293         }
2294
2295         if ($line =~ m/^(\d+)$/) {
2296             my $pid = $1;
2297             if (check_cmdline($pidfile, $pid)) {
2298                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2299                     return $pid;
2300                 }
2301             }
2302         }
2303     }
2304
2305     return undef;
2306 }
2307
2308 sub vzlist {
2309
2310     my $vzlist = config_list();
2311
2312     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2313
2314     while (defined(my $de = $fd->read)) {
2315         next if $de !~ m/^(\d+)\.pid$/;
2316         my $vmid = $1;
2317         next if !defined($vzlist->{$vmid});
2318         if (my $pid = check_running($vmid)) {
2319             $vzlist->{$vmid}->{pid} = $pid;
2320         }
2321     }
2322
2323     return $vzlist;
2324 }
2325
2326 sub disksize {
2327     my ($storecfg, $conf) = @_;
2328
2329     my $bootdisk = $conf->{bootdisk};
2330     return undef if !$bootdisk;
2331     return undef if !valid_drivename($bootdisk);
2332
2333     return undef if !$conf->{$bootdisk};
2334
2335     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2336     return undef if !defined($drive);
2337
2338     return undef if drive_is_cdrom($drive);
2339
2340     my $volid = $drive->{file};
2341     return undef if !$volid;
2342
2343     return $drive->{size};
2344 }
2345
2346 my $last_proc_pid_stat;
2347
2348 # get VM status information
2349 # This must be fast and should not block ($full == false)
2350 # We only query KVM using QMP if $full == true (this can be slow)
2351 sub vmstatus {
2352     my ($opt_vmid, $full) = @_;
2353
2354     my $res = {};
2355
2356     my $storecfg = PVE::Storage::config();
2357
2358     my $list = vzlist();
2359     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2360
2361     my $cpucount = $cpuinfo->{cpus} || 1;
2362
2363     foreach my $vmid (keys %$list) {
2364         next if $opt_vmid && ($vmid ne $opt_vmid);
2365
2366         my $cfspath = cfs_config_path($vmid);
2367         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2368
2369         my $d = {};
2370         $d->{pid} = $list->{$vmid}->{pid};
2371
2372         # fixme: better status?
2373         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2374
2375         my $size = disksize($storecfg, $conf);
2376         if (defined($size)) {
2377             $d->{disk} = 0; # no info available
2378             $d->{maxdisk} = $size;
2379         } else {
2380             $d->{disk} = 0;
2381             $d->{maxdisk} = 0;
2382         }
2383
2384         $d->{cpus} = ($conf->{sockets} || 1) * ($conf->{cores} || 1);
2385         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2386         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2387
2388         $d->{name} = $conf->{name} || "VM $vmid";
2389         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024) : 0;
2390
2391         if ($conf->{balloon}) {
2392             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2393             $d->{shares} = defined($conf->{shares}) ? $conf->{shares} : 1000;
2394         }
2395
2396         $d->{uptime} = 0;
2397         $d->{cpu} = 0;
2398         $d->{mem} = 0;
2399
2400         $d->{netout} = 0;
2401         $d->{netin} = 0;
2402
2403         $d->{diskread} = 0;
2404         $d->{diskwrite} = 0;
2405
2406         $d->{template} = is_template($conf);
2407
2408         $res->{$vmid} = $d;
2409     }
2410
2411     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2412     foreach my $dev (keys %$netdev) {
2413         next if $dev !~ m/^tap([1-9]\d*)i/;
2414         my $vmid = $1;
2415         my $d = $res->{$vmid};
2416         next if !$d;
2417
2418         $d->{netout} += $netdev->{$dev}->{receive};
2419         $d->{netin} += $netdev->{$dev}->{transmit};
2420
2421         if ($full) {
2422             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2423             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2424         }
2425
2426     }
2427
2428     my $ctime = gettimeofday;
2429
2430     foreach my $vmid (keys %$list) {
2431
2432         my $d = $res->{$vmid};
2433         my $pid = $d->{pid};
2434         next if !$pid;
2435
2436         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2437         next if !$pstat; # not running
2438
2439         my $used = $pstat->{utime} + $pstat->{stime};
2440
2441         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2442
2443         if ($pstat->{vsize}) {
2444             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2445         }
2446
2447         my $old = $last_proc_pid_stat->{$pid};
2448         if (!$old) {
2449             $last_proc_pid_stat->{$pid} = {
2450                 time => $ctime,
2451                 used => $used,
2452                 cpu => 0,
2453             };
2454             next;
2455         }
2456
2457         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2458
2459         if ($dtime > 1000) {
2460             my $dutime = $used -  $old->{used};
2461
2462             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2463             $last_proc_pid_stat->{$pid} = {
2464                 time => $ctime,
2465                 used => $used,
2466                 cpu => $d->{cpu},
2467             };
2468         } else {
2469             $d->{cpu} = $old->{cpu};
2470         }
2471     }
2472
2473     return $res if !$full;
2474
2475     my $qmpclient = PVE::QMPClient->new();
2476
2477     my $ballooncb = sub {
2478         my ($vmid, $resp) = @_;
2479
2480         my $info = $resp->{'return'};
2481         return if !$info->{max_mem};
2482
2483         my $d = $res->{$vmid};
2484
2485         # use memory assigned to VM
2486         $d->{maxmem} = $info->{max_mem};
2487         $d->{balloon} = $info->{actual};
2488
2489         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2490             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2491             $d->{freemem} = $info->{free_mem};
2492         }
2493
2494         $d->{ballooninfo} = $info;
2495     };
2496
2497     my $blockstatscb = sub {
2498         my ($vmid, $resp) = @_;
2499         my $data = $resp->{'return'} || [];
2500         my $totalrdbytes = 0;
2501         my $totalwrbytes = 0;
2502
2503         for my $blockstat (@$data) {
2504             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2505             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2506
2507             $blockstat->{device} =~ s/drive-//;
2508             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2509         }
2510         $res->{$vmid}->{diskread} = $totalrdbytes;
2511         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2512     };
2513
2514     my $statuscb = sub {
2515         my ($vmid, $resp) = @_;
2516
2517         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2518         # this fails if ballon driver is not loaded, so this must be
2519         # the last commnand (following command are aborted if this fails).
2520         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2521
2522         my $status = 'unknown';
2523         if (!defined($status = $resp->{'return'}->{status})) {
2524             warn "unable to get VM status\n";
2525             return;
2526         }
2527
2528         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2529     };
2530
2531     foreach my $vmid (keys %$list) {
2532         next if $opt_vmid && ($vmid ne $opt_vmid);
2533         next if !$res->{$vmid}->{pid}; # not running
2534         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2535     }
2536
2537     $qmpclient->queue_execute(undef, 1);
2538
2539     foreach my $vmid (keys %$list) {
2540         next if $opt_vmid && ($vmid ne $opt_vmid);
2541         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2542     }
2543
2544     return $res;
2545 }
2546
2547 sub foreach_dimm {
2548     my ($conf, $vmid, $memory, $sockets, $func) = @_;
2549
2550     my $dimm_id = 0;
2551     my $current_size = 1024;
2552     my $dimm_size = 512;
2553     return if $current_size == $memory;
2554
2555     for (my $j = 0; $j < 8; $j++) {
2556         for (my $i = 0; $i < 32; $i++) {
2557             my $name = "dimm${dimm_id}";
2558             $dimm_id++;
2559             my $numanode = $i % $sockets;
2560             $current_size += $dimm_size;
2561             &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
2562             return  $current_size if $current_size >= $memory;
2563         }
2564         $dimm_size *= 2;
2565     }
2566 }
2567
2568 sub foreach_reverse_dimm {
2569     my ($conf, $vmid, $memory, $sockets, $func) = @_;
2570
2571     my $dimm_id = 253;
2572     my $current_size = 4177920;
2573     my $dimm_size = 65536;
2574     return if $current_size == $memory;
2575
2576     for (my $j = 0; $j < 8; $j++) {
2577         for (my $i = 0; $i < 32; $i++) {
2578             my $name = "dimm${dimm_id}";
2579             $dimm_id--;
2580             my $numanode = $i % $sockets;
2581             $current_size -= $dimm_size;
2582             &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
2583             return  $current_size if $current_size <= $memory;
2584         }
2585         $dimm_size /= 2;
2586     }
2587 }
2588
2589 sub foreach_drive {
2590     my ($conf, $func) = @_;
2591
2592     foreach my $ds (keys %$conf) {
2593         next if !valid_drivename($ds);
2594
2595         my $drive = parse_drive($ds, $conf->{$ds});
2596         next if !$drive;
2597
2598         &$func($ds, $drive);
2599     }
2600 }
2601
2602 sub foreach_volid {
2603     my ($conf, $func) = @_;
2604
2605     my $volhash = {};
2606
2607     my $test_volid = sub {
2608         my ($volid, $is_cdrom) = @_;
2609
2610         return if !$volid;
2611
2612         $volhash->{$volid} = $is_cdrom || 0;
2613     };
2614
2615     foreach_drive($conf, sub {
2616         my ($ds, $drive) = @_;
2617         &$test_volid($drive->{file}, drive_is_cdrom($drive));
2618     });
2619
2620     foreach my $snapname (keys %{$conf->{snapshots}}) {
2621         my $snap = $conf->{snapshots}->{$snapname};
2622         &$test_volid($snap->{vmstate}, 0);
2623         foreach_drive($snap, sub {
2624             my ($ds, $drive) = @_;
2625             &$test_volid($drive->{file}, drive_is_cdrom($drive));
2626         });
2627     }
2628
2629     foreach my $volid (keys %$volhash) {
2630         &$func($volid, $volhash->{$volid});
2631     }
2632 }
2633
2634 sub vga_conf_has_spice {
2635     my ($vga) = @_;
2636
2637     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
2638
2639     return $1 || 1;
2640 }
2641
2642 sub config_to_command {
2643     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2644
2645     my $cmd = [];
2646     my $globalFlags = [];
2647     my $machineFlags = [];
2648     my $rtcFlags = [];
2649     my $cpuFlags = [];
2650     my $devices = [];
2651     my $pciaddr = '';
2652     my $bridges = {};
2653     my $kvmver = kvm_user_version();
2654     my $vernum = 0; # unknown
2655     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
2656         $vernum = $1*1000000+$2*1000;
2657     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
2658         $vernum = $1*1000000+$2*1000+$3;
2659     }
2660
2661     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
2662
2663     my $have_ovz = -f '/proc/vz/vestat';
2664
2665     my $q35 = machine_type_is_q35($conf);
2666     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2667     my $machine_type = $forcemachine || $conf->{machine};
2668     my $use_old_bios_files = undef;
2669     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
2670
2671     my $cpuunits = defined($conf->{cpuunits}) ?
2672             $conf->{cpuunits} : $defaults->{cpuunits};
2673
2674     push @$cmd, '/usr/bin/systemd-run';
2675     push @$cmd, '--scope';
2676     push @$cmd, '--slice', "qemu";
2677     push @$cmd, '--unit', $vmid;
2678     # set KillMode=none, so that systemd don't kill those scopes
2679     # at shutdown (pve-manager service should stop the VMs instead)
2680     push @$cmd, '-p', "KillMode=none";
2681     push @$cmd, '-p', "CPUShares=$cpuunits";
2682     if ($conf->{cpulimit}) {
2683         my $cpulimit = int($conf->{cpulimit} * 100);
2684         push @$cmd, '-p', "CPUQuota=$cpulimit\%";
2685     }
2686
2687     push @$cmd, '/usr/bin/kvm';
2688
2689     push @$cmd, '-id', $vmid;
2690
2691     my $use_virtio = 0;
2692
2693     my $qmpsocket = qmp_socket($vmid);
2694     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2695     push @$cmd, '-mon', "chardev=qmp,mode=control";
2696
2697     my $socket = vnc_socket($vmid);
2698     push @$cmd,  '-vnc', "unix:$socket,x509,password";
2699
2700     push @$cmd, '-pidfile' , pidfile_name($vmid);
2701
2702     push @$cmd, '-daemonize';
2703
2704     if ($conf->{smbios1}) {
2705         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
2706     }
2707
2708     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2709         my $ovmfvar = "OVMF_VARS-pure-efi.fd";
2710         my $ovmfvar_src = "/usr/share/kvm/$ovmfvar";
2711         my $ovmfvar_dst = "/tmp/$vmid-$ovmfvar";
2712         PVE::Tools::file_copy($ovmfvar_src, $ovmfvar_dst, 256*1024);
2713         push @$cmd, '-drive', "if=pflash,format=raw,readonly,file=/usr/share/kvm/OVMF-pure-efi.fd";
2714         push @$cmd, '-drive', "if=pflash,format=raw,file=$ovmfvar_dst";
2715     }
2716
2717     if ($q35) {
2718         # the q35 chipset support native usb2, so we enable usb controller
2719         # by default for this machine type
2720         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
2721     } else {
2722         $pciaddr = print_pci_addr("piix3", $bridges);
2723         push @$devices, '-device', "piix3-usb-uhci,id=uhci$pciaddr.0x2";
2724
2725         my $use_usb2 = 0;
2726         for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2727             next if !$conf->{"usb$i"};
2728             $use_usb2 = 1;
2729         }
2730         # include usb device config
2731         push @$devices, '-readconfig', '/usr/share/qemu-server/pve-usb.cfg' if $use_usb2;
2732     }
2733
2734     my $vga = $conf->{vga};
2735
2736     my $qxlnum = vga_conf_has_spice($vga);
2737     $vga = 'qxl' if $qxlnum;
2738
2739     if (!$vga) {
2740         if ($conf->{ostype} && ($conf->{ostype} eq 'win8' ||
2741                                 $conf->{ostype} eq 'win7' ||
2742                                 $conf->{ostype} eq 'w2k8')) {
2743             $vga = 'std';
2744         } else {
2745             $vga = 'cirrus';
2746         }
2747     }
2748
2749     # enable absolute mouse coordinates (needed by vnc)
2750     my $tablet;
2751     if (defined($conf->{tablet})) {
2752         $tablet = $conf->{tablet};
2753     } else {
2754         $tablet = $defaults->{tablet};
2755         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
2756         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
2757     }
2758
2759     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
2760
2761     my $nohyperv;
2762     # host pci devices
2763     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
2764         my $d = parse_hostpci($conf->{"hostpci$i"});
2765         next if !$d;
2766
2767         my $pcie = $d->{pcie};
2768         if($pcie){
2769             die "q35 machine model is not enabled" if !$q35;
2770             $pciaddr = print_pcie_addr("hostpci$i");
2771         }else{
2772             $pciaddr = print_pci_addr("hostpci$i", $bridges);
2773         }
2774
2775         my $rombar = $d->{rombar} && $d->{rombar} eq 'off' ? ",rombar=0" : "";
2776         my $xvga = $d->{'x-vga'} && $d->{'x-vga'} eq 'on' ? ",x-vga=on" : "";
2777         if ($xvga && $xvga ne '') {
2778             push @$cpuFlags, 'kvm=off';
2779             $vga = 'none';
2780             $nohyperv = 1;
2781         }
2782         my $pcidevices = $d->{pciid};
2783         my $multifunction = 1 if @$pcidevices > 1;
2784
2785         my $j=0;
2786         foreach my $pcidevice (@$pcidevices) {
2787
2788             my $id = "hostpci$i";
2789             $id .= ".$j" if $multifunction;
2790             my $addr = $pciaddr;
2791             $addr .= ".$j" if $multifunction;
2792             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
2793
2794             if($j == 0){
2795                 $devicestr .= "$rombar$xvga";
2796                 $devicestr .= ",multifunction=on" if $multifunction;
2797             }
2798
2799             push @$devices, '-device', $devicestr;
2800             $j++;
2801         }
2802     }
2803
2804     # usb devices
2805     for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
2806         my $d = parse_usb_device($conf->{"usb$i"});
2807         next if !$d;
2808         if ($d->{vendorid} && $d->{productid}) {
2809             push @$devices, '-device', "usb-host,vendorid=0x$d->{vendorid},productid=0x$d->{productid}";
2810         } elsif (defined($d->{hostbus}) && defined($d->{hostport})) {
2811             push @$devices, '-device', "usb-host,hostbus=$d->{hostbus},hostport=$d->{hostport}";
2812         } elsif ($d->{spice}) {
2813             # usb redir support for spice
2814             push @$devices, '-chardev', "spicevmc,id=usbredirchardev$i,name=usbredir";
2815             push @$devices, '-device', "usb-redir,chardev=usbredirchardev$i,id=usbredirdev$i,bus=ehci.0";
2816         }
2817     }
2818
2819     # serial devices
2820     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2821         if (my $path = $conf->{"serial$i"}) {
2822             if ($path eq 'socket') {
2823                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
2824                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
2825                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2826             } else {
2827                 die "no such serial device\n" if ! -c $path;
2828                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
2829                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2830             }
2831         }
2832     }
2833
2834     # parallel devices
2835     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
2836         if (my $path = $conf->{"parallel$i"}) {
2837             die "no such parallel device\n" if ! -c $path;
2838             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
2839             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
2840             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
2841         }
2842     }
2843
2844     my $vmname = $conf->{name} || "vm$vmid";
2845
2846     push @$cmd, '-name', $vmname;
2847
2848     my $sockets = 1;
2849     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
2850     $sockets = $conf->{sockets} if  $conf->{sockets};
2851
2852     my $cores = $conf->{cores} || 1;
2853
2854     my $maxcpus = $sockets * $cores;
2855
2856     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
2857
2858     my $allowed_vcpus = $cpuinfo->{cpus};
2859
2860     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
2861         if ($allowed_vcpus < $maxcpus);
2862
2863     push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
2864
2865     push @$cmd, '-nodefaults';
2866
2867     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2868
2869     my $bootindex_hash = {};
2870     my $i = 1;
2871     foreach my $o (split(//, $bootorder)) {
2872         $bootindex_hash->{$o} = $i*100;
2873         $i++;
2874     }
2875
2876     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000";
2877
2878     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
2879
2880     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
2881
2882     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
2883
2884     # time drift fix
2885     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
2886
2887     my $nokvm = defined($conf->{kvm}) && $conf->{kvm} == 0 ? 1 : 0;
2888     my $useLocaltime = $conf->{localtime};
2889
2890     if (my $ost = $conf->{ostype}) {
2891         # other, wxp, w2k, w2k3, w2k8, wvista, win7, win8, l24, l26, solaris
2892
2893         if ($ost =~ m/^w/) { # windows
2894             $useLocaltime = 1 if !defined($conf->{localtime});
2895
2896             # use time drift fix when acpi is enabled
2897             if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
2898                 $tdf = 1 if !defined($conf->{tdf});
2899             }
2900         }
2901
2902         if ($ost eq 'win7' || $ost eq 'win8' || $ost eq 'w2k8' ||
2903             $ost eq 'wvista') {
2904             push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
2905             push @$cmd, '-no-hpet';
2906             if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
2907                 push @$cpuFlags , 'hv_spinlocks=0x1fff' if !$nokvm && !$nohyperv;
2908                 push @$cpuFlags , 'hv_vapic' if !$nokvm && !$nohyperv;
2909                 push @$cpuFlags , 'hv_time' if !$nokvm && !$nohyperv;
2910
2911             } else {
2912                 push @$cpuFlags , 'hv_spinlocks=0xffff' if !$nokvm && !$nohyperv;
2913             }
2914         }
2915
2916         if ($ost eq 'win7' || $ost eq 'win8') {
2917             push @$cpuFlags , 'hv_relaxed' if !$nokvm && !$nohyperv;
2918         }
2919     }
2920
2921     push @$rtcFlags, 'driftfix=slew' if $tdf;
2922
2923     if ($nokvm) {
2924         push @$machineFlags, 'accel=tcg';
2925     } else {
2926         die "No accelerator found!\n" if !$cpuinfo->{hvm};
2927     }
2928
2929     if ($machine_type) {
2930         push @$machineFlags, "type=${machine_type}";
2931     }
2932
2933     if ($conf->{startdate}) {
2934         push @$rtcFlags, "base=$conf->{startdate}";
2935     } elsif ($useLocaltime) {
2936         push @$rtcFlags, 'base=localtime';
2937     }
2938
2939     my $cpu = $nokvm ? "qemu64" : "kvm64";
2940     $cpu = $conf->{cpu} if $conf->{cpu};
2941
2942     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
2943
2944     push @$cpuFlags , '-x2apic'
2945         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
2946
2947     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
2948
2949     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
2950
2951     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
2952
2953         push @$cpuFlags , '+kvm_pv_unhalt' if !$nokvm;
2954         push @$cpuFlags , '+kvm_pv_eoi' if !$nokvm;
2955     }
2956
2957     push @$cpuFlags, 'enforce' if $cpu ne 'host' && !$nokvm;
2958
2959     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
2960
2961     push @$cmd, '-cpu', $cpu;
2962
2963     my $memory = $conf->{memory} || $defaults->{memory};
2964     my $static_memory = 0;
2965     my $dimm_memory = 0;
2966
2967     if ($hotplug_features->{memory}) {
2968         die "Numa need to be enabled for memory hotplug\n" if !$conf->{numa};
2969         die "Total memory is bigger than ${MAX_MEM}MB\n" if $memory > $MAX_MEM;
2970         $static_memory = $STATICMEM;
2971         die "minimum memory must be ${static_memory}MB\n" if($memory < $static_memory);
2972         $dimm_memory = $memory - $static_memory;
2973         push @$cmd, '-m', "size=${static_memory},slots=255,maxmem=${MAX_MEM}M";
2974
2975     } else {
2976
2977         $static_memory = $memory;
2978         push @$cmd, '-m', $static_memory;
2979     }
2980
2981     if ($conf->{numa}) {
2982
2983         my $numa_totalmemory = undef;
2984         for (my $i = 0; $i < $MAX_NUMA; $i++) {
2985             next if !$conf->{"numa$i"};
2986             my $numa = parse_numa($conf->{"numa$i"});
2987             next if !$numa;
2988             # memory
2989             die "missing numa node$i memory value\n" if !$numa->{memory};
2990             my $numa_memory = $numa->{memory};
2991             $numa_totalmemory += $numa_memory;
2992             my $numa_object = "memory-backend-ram,id=ram-node$i,size=${numa_memory}M";
2993
2994             # cpus
2995             my $cpus_start = $numa->{cpus}->{start};
2996             die "missing numa node$i cpus\n" if !defined($cpus_start);
2997             my $cpus_end = $numa->{cpus}->{end} if defined($numa->{cpus}->{end});
2998             my $cpus = $cpus_start;
2999             if (defined($cpus_end)) {
3000                 $cpus .= "-$cpus_end";
3001                 die "numa node$i :  cpu range $cpus is incorrect\n" if $cpus_end <= $cpus_start;
3002             }
3003
3004             # hostnodes
3005             my $hostnodes_start = $numa->{hostnodes}->{start};
3006             if (defined($hostnodes_start)) {
3007                 my $hostnodes_end = $numa->{hostnodes}->{end} if defined($numa->{hostnodes}->{end});
3008                 my $hostnodes = $hostnodes_start;
3009                 if (defined($hostnodes_end)) {
3010                     $hostnodes .= "-$hostnodes_end";
3011                     die "host node $hostnodes range is incorrect\n" if $hostnodes_end <= $hostnodes_start;
3012                 }
3013
3014                 my $hostnodes_end_range = defined($hostnodes_end) ? $hostnodes_end : $hostnodes_start;
3015                 for (my $i = $hostnodes_start; $i <= $hostnodes_end_range; $i++ ) {
3016                     die "host numa node$i don't exist\n" if ! -d "/sys/devices/system/node/node$i/";
3017                 }
3018
3019                 # policy
3020                 my $policy = $numa->{policy};
3021                 die "you need to define a policy for hostnode $hostnodes\n" if !$policy;
3022                 $numa_object .= ",host-nodes=$hostnodes,policy=$policy";
3023             }
3024
3025             push @$cmd, '-object', $numa_object;
3026             push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
3027         }
3028
3029         die "total memory for NUMA nodes must be equal to vm static memory\n"
3030             if $numa_totalmemory && $numa_totalmemory != $static_memory;
3031
3032         #if no custom tology, we split memory and cores across numa nodes
3033         if(!$numa_totalmemory) {
3034
3035             my $numa_memory = ($static_memory / $sockets) . "M";
3036
3037             for (my $i = 0; $i < $sockets; $i++)  {
3038
3039                 my $cpustart = ($cores * $i);
3040                 my $cpuend = ($cpustart + $cores - 1) if $cores && $cores > 1;
3041                 my $cpus = $cpustart;
3042                 $cpus .= "-$cpuend" if $cpuend;
3043
3044                 push @$cmd, '-object', "memory-backend-ram,size=$numa_memory,id=ram-node$i";
3045                 push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
3046             }
3047         }
3048     }
3049
3050     if ($hotplug_features->{memory}) {
3051         foreach_dimm($conf, $vmid, $memory, $sockets, sub {
3052             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3053             push @$cmd, "-object" , "memory-backend-ram,id=mem-$name,size=${dimm_size}M";
3054             push @$cmd, "-device", "pc-dimm,id=$name,memdev=mem-$name,node=$numanode";
3055
3056             #if dimm_memory is not aligned to dimm map
3057             if($current_size > $memory) {
3058                  $conf->{memory} = $current_size;
3059                  update_config_nolock($vmid, $conf, 1);
3060             }
3061         });
3062     }
3063
3064     push @$cmd, '-S' if $conf->{freeze};
3065
3066     # set keyboard layout
3067     my $kb = $conf->{keyboard} || $defaults->{keyboard};
3068     push @$cmd, '-k', $kb if $kb;
3069
3070     # enable sound
3071     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3072     #push @$cmd, '-soundhw', 'es1370';
3073     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3074
3075     if($conf->{agent}) {
3076         my $qgasocket = qmp_socket($vmid, 1);
3077         my $pciaddr = print_pci_addr("qga0", $bridges);
3078         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3079         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3080         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3081     }
3082
3083     my $spice_port;
3084
3085     if ($qxlnum) {
3086         if ($qxlnum > 1) {
3087             if ($conf->{ostype} && $conf->{ostype} =~ m/^w/){
3088                 for(my $i = 1; $i < $qxlnum; $i++){
3089                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3090                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3091                 }
3092             } else {
3093                 # assume other OS works like Linux
3094                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3095                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3096             }
3097         }
3098
3099         my $pciaddr = print_pci_addr("spice", $bridges);
3100
3101         my $nodename = PVE::INotify::nodename();
3102         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3103         $spice_port = PVE::Tools::next_spice_port($pfamily);
3104
3105         push @$devices, '-spice', "tls-port=${spice_port},addr=localhost,tls-ciphers=DES-CBC3-SHA,seamless-migration=on";
3106
3107         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3108         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3109         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3110     }
3111
3112     # enable balloon by default, unless explicitly disabled
3113     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3114         $pciaddr = print_pci_addr("balloon0", $bridges);
3115         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3116     }
3117
3118     if ($conf->{watchdog}) {
3119         my $wdopts = parse_watchdog($conf->{watchdog});
3120         $pciaddr = print_pci_addr("watchdog", $bridges);
3121         my $watchdog = $wdopts->{model} || 'i6300esb';
3122         push @$devices, '-device', "$watchdog$pciaddr";
3123         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3124     }
3125
3126     my $vollist = [];
3127     my $scsicontroller = {};
3128     my $ahcicontroller = {};
3129     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3130
3131     # Add iscsi initiator name if available
3132     if (my $initiator = get_initiator_name()) {
3133         push @$devices, '-iscsi', "initiator-name=$initiator";
3134     }
3135
3136     foreach_drive($conf, sub {
3137         my ($ds, $drive) = @_;
3138
3139         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3140             push @$vollist, $drive->{file};
3141         }
3142
3143         $use_virtio = 1 if $ds =~ m/^virtio/;
3144
3145         if (drive_is_cdrom ($drive)) {
3146             if ($bootindex_hash->{d}) {
3147                 $drive->{bootindex} = $bootindex_hash->{d};
3148                 $bootindex_hash->{d} += 1;
3149             }
3150         } else {
3151             if ($bootindex_hash->{c}) {
3152                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3153                 $bootindex_hash->{c} += 1;
3154             }
3155         }
3156
3157         if($drive->{interface} eq 'virtio'){
3158            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3159         }
3160
3161         if ($drive->{interface} eq 'scsi') {
3162
3163             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3164
3165             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3166             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3167
3168             my $iothread = '';
3169             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3170                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3171                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3172             }
3173
3174             my $queues = '';
3175             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3176                 $queues = ",num_queues=$drive->{queues}";
3177             }
3178
3179             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3180             $scsicontroller->{$controller}=1;
3181         }
3182
3183         if ($drive->{interface} eq 'sata') {
3184            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3185            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3186            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3187            $ahcicontroller->{$controller}=1;
3188         }
3189
3190         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3191         push @$devices, '-drive',$drive_cmd;
3192         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3193     });
3194
3195     for (my $i = 0; $i < $MAX_NETS; $i++) {
3196          next if !$conf->{"net$i"};
3197          my $d = parse_net($conf->{"net$i"});
3198          next if !$d;
3199
3200          $use_virtio = 1 if $d->{model} eq 'virtio';
3201
3202          if ($bootindex_hash->{n}) {
3203             $d->{bootindex} = $bootindex_hash->{n};
3204             $bootindex_hash->{n} += 1;
3205          }
3206
3207          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3208          push @$devices, '-netdev', $netdevfull;
3209
3210          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3211          push @$devices, '-device', $netdevicefull;
3212     }
3213
3214     if (!$q35) {
3215         # add pci bridges
3216         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3217            $bridges->{1} = 1;
3218            $bridges->{2} = 1;
3219         }
3220
3221         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3222
3223         while (my ($k, $v) = each %$bridges) {
3224             $pciaddr = print_pci_addr("pci.$k");
3225             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3226         }
3227     }
3228
3229     # add custom args
3230     if ($conf->{args}) {
3231         my $aa = PVE::Tools::split_args($conf->{args});
3232         push @$cmd, @$aa;
3233     }
3234
3235     push @$cmd, @$devices;
3236     push @$cmd, '-rtc', join(',', @$rtcFlags)
3237         if scalar(@$rtcFlags);
3238     push @$cmd, '-machine', join(',', @$machineFlags)
3239         if scalar(@$machineFlags);
3240     push @$cmd, '-global', join(',', @$globalFlags)
3241         if scalar(@$globalFlags);
3242
3243     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3244 }
3245
3246 sub vnc_socket {
3247     my ($vmid) = @_;
3248     return "${var_run_tmpdir}/$vmid.vnc";
3249 }
3250
3251 sub spice_port {
3252     my ($vmid) = @_;
3253
3254     my $res = vm_mon_cmd($vmid, 'query-spice');
3255
3256     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3257 }
3258
3259 sub qmp_socket {
3260     my ($vmid, $qga) = @_;
3261     my $sockettype = $qga ? 'qga' : 'qmp';
3262     return "${var_run_tmpdir}/$vmid.$sockettype";
3263 }
3264
3265 sub pidfile_name {
3266     my ($vmid) = @_;
3267     return "${var_run_tmpdir}/$vmid.pid";
3268 }
3269
3270 sub vm_devices_list {
3271     my ($vmid) = @_;
3272
3273     my $res = vm_mon_cmd($vmid, 'query-pci');
3274     my $devices = {};
3275     foreach my $pcibus (@$res) {
3276         foreach my $device (@{$pcibus->{devices}}) {
3277             next if !$device->{'qdev_id'};
3278             if ($device->{'pci_bridge'}) {
3279                 $devices->{$device->{'qdev_id'}} = 1;
3280                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3281                     next if !$bridge_device->{'qdev_id'};
3282                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3283                     $devices->{$device->{'qdev_id'}}++;
3284                 }
3285             } else {
3286                 $devices->{$device->{'qdev_id'}} = 1;
3287             }
3288         }
3289     }
3290
3291     my $resblock = vm_mon_cmd($vmid, 'query-block');
3292     foreach my $block (@$resblock) {
3293         if($block->{device} =~ m/^drive-(\S+)/){
3294                 $devices->{$1} = 1;
3295         }
3296     }
3297
3298     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3299     foreach my $mice (@$resmice) {
3300         if ($mice->{name} eq 'QEMU HID Tablet') {
3301             $devices->{tablet} = 1;
3302             last;
3303         }
3304     }
3305
3306     return $devices;
3307 }
3308
3309 sub vm_deviceplug {
3310     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3311
3312     my $q35 = machine_type_is_q35($conf);
3313
3314     my $devices_list = vm_devices_list($vmid);
3315     return 1 if defined($devices_list->{$deviceid});
3316
3317     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3318
3319     if ($deviceid eq 'tablet') {
3320
3321         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3322
3323     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3324
3325         qemu_iothread_add($vmid, $deviceid, $device);
3326
3327         qemu_driveadd($storecfg, $vmid, $device);
3328         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3329
3330         qemu_deviceadd($vmid, $devicefull);
3331         eval { qemu_deviceaddverify($vmid, $deviceid); };
3332         if (my $err = $@) {
3333             eval { qemu_drivedel($vmid, $deviceid); };
3334             warn $@ if $@;
3335             die $err;
3336         }
3337
3338     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3339
3340
3341         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3342         my $pciaddr = print_pci_addr($deviceid);
3343         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3344
3345         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3346
3347         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3348             qemu_iothread_add($vmid, $deviceid, $device);
3349             $devicefull .= ",iothread=iothread-$deviceid";
3350         }
3351
3352         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3353             $devicefull .= ",num_queues=$device->{queues}";
3354         }
3355
3356         qemu_deviceadd($vmid, $devicefull);
3357         qemu_deviceaddverify($vmid, $deviceid);
3358
3359     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3360
3361         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3362         qemu_driveadd($storecfg, $vmid, $device);
3363
3364         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3365         eval { qemu_deviceadd($vmid, $devicefull); };
3366         if (my $err = $@) {
3367             eval { qemu_drivedel($vmid, $deviceid); };
3368             warn $@ if $@;
3369             die $err;
3370         }
3371
3372     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3373
3374         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3375
3376         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
3377         my $use_old_bios_files = undef;
3378         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3379
3380         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3381         qemu_deviceadd($vmid, $netdevicefull);
3382         eval { qemu_deviceaddverify($vmid, $deviceid); };
3383         if (my $err = $@) {
3384             eval { qemu_netdevdel($vmid, $deviceid); };
3385             warn $@ if $@;
3386             die $err;
3387         }
3388
3389     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3390
3391         my $bridgeid = $2;
3392         my $pciaddr = print_pci_addr($deviceid);
3393         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3394
3395         qemu_deviceadd($vmid, $devicefull);
3396         qemu_deviceaddverify($vmid, $deviceid);
3397
3398     } else {
3399         die "can't hotplug device '$deviceid'\n";
3400     }
3401
3402     return 1;
3403 }
3404
3405 # fixme: this should raise exceptions on error!
3406 sub vm_deviceunplug {
3407     my ($vmid, $conf, $deviceid) = @_;
3408
3409     my $devices_list = vm_devices_list($vmid);
3410     return 1 if !defined($devices_list->{$deviceid});
3411
3412     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3413
3414     if ($deviceid eq 'tablet') {
3415
3416         qemu_devicedel($vmid, $deviceid);
3417
3418     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3419
3420         qemu_devicedel($vmid, $deviceid);
3421         qemu_devicedelverify($vmid, $deviceid);
3422         qemu_drivedel($vmid, $deviceid);
3423         qemu_iothread_del($conf, $vmid, $deviceid);
3424
3425     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3426
3427         qemu_devicedel($vmid, $deviceid);
3428         qemu_devicedelverify($vmid, $deviceid);
3429         qemu_iothread_del($conf, $vmid, $deviceid);
3430
3431     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3432
3433         #qemu 2.3 segfault on drive_del with virtioscsi + iothread
3434         my $device = parse_drive($deviceid, $conf->{$deviceid});
3435         die "virtioscsi with iothread is not hot-unplugglable currently" if $device->{iothread};
3436
3437         qemu_devicedel($vmid, $deviceid);
3438         qemu_drivedel($vmid, $deviceid);
3439         qemu_deletescsihw($conf, $vmid, $deviceid);
3440
3441     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3442
3443         qemu_devicedel($vmid, $deviceid);
3444         qemu_devicedelverify($vmid, $deviceid);
3445         qemu_netdevdel($vmid, $deviceid);
3446
3447     } else {
3448         die "can't unplug device '$deviceid'\n";
3449     }
3450
3451     return 1;
3452 }
3453
3454 sub qemu_deviceadd {
3455     my ($vmid, $devicefull) = @_;
3456
3457     $devicefull = "driver=".$devicefull;
3458     my %options =  split(/[=,]/, $devicefull);
3459
3460     vm_mon_cmd($vmid, "device_add" , %options);
3461 }
3462
3463 sub qemu_devicedel {
3464     my ($vmid, $deviceid) = @_;
3465
3466     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3467 }
3468
3469 sub qemu_iothread_add {
3470     my($vmid, $deviceid, $device) = @_;
3471
3472     if ($device->{iothread}) {
3473         my $iothreads = vm_iothreads_list($vmid);
3474         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3475     }
3476 }
3477
3478 sub qemu_iothread_del {
3479     my($conf, $vmid, $deviceid) = @_;
3480
3481     my $device = parse_drive($deviceid, $conf->{$deviceid});
3482     if ($device->{iothread}) {
3483         my $iothreads = vm_iothreads_list($vmid);
3484         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3485     }
3486 }
3487
3488 sub qemu_objectadd {
3489     my($vmid, $objectid, $qomtype) = @_;
3490
3491     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3492
3493     return 1;
3494 }
3495
3496 sub qemu_objectdel {
3497     my($vmid, $objectid) = @_;
3498
3499     vm_mon_cmd($vmid, "object-del", id => $objectid);
3500
3501     return 1;
3502 }
3503
3504 sub qemu_driveadd {
3505     my ($storecfg, $vmid, $device) = @_;
3506
3507     my $drive = print_drive_full($storecfg, $vmid, $device);
3508     $drive =~ s/\\/\\\\/g;
3509     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3510
3511     # If the command succeeds qemu prints: "OK"
3512     return 1 if $ret =~ m/OK/s;
3513
3514     die "adding drive failed: $ret\n";
3515 }
3516
3517 sub qemu_drivedel {
3518     my($vmid, $deviceid) = @_;
3519
3520     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3521     $ret =~ s/^\s+//;
3522
3523     return 1 if $ret eq "";
3524
3525     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3526     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3527
3528     die "deleting drive $deviceid failed : $ret\n";
3529 }
3530
3531 sub qemu_deviceaddverify {
3532     my ($vmid, $deviceid) = @_;
3533
3534     for (my $i = 0; $i <= 5; $i++) {
3535          my $devices_list = vm_devices_list($vmid);
3536          return 1 if defined($devices_list->{$deviceid});
3537          sleep 1;
3538     }
3539
3540     die "error on hotplug device '$deviceid'\n";
3541 }
3542
3543
3544 sub qemu_devicedelverify {
3545     my ($vmid, $deviceid) = @_;
3546
3547     # need to verify that the device is correctly removed as device_del
3548     # is async and empty return is not reliable
3549
3550     for (my $i = 0; $i <= 5; $i++) {
3551          my $devices_list = vm_devices_list($vmid);
3552          return 1 if !defined($devices_list->{$deviceid});
3553          sleep 1;
3554     }
3555
3556     die "error on hot-unplugging device '$deviceid'\n";
3557 }
3558
3559 sub qemu_findorcreatescsihw {
3560     my ($storecfg, $conf, $vmid, $device) = @_;
3561
3562     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3563
3564     my $scsihwid="$controller_prefix$controller";
3565     my $devices_list = vm_devices_list($vmid);
3566
3567     if(!defined($devices_list->{$scsihwid})) {
3568         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3569     }
3570
3571     return 1;
3572 }
3573
3574 sub qemu_deletescsihw {
3575     my ($conf, $vmid, $opt) = @_;
3576
3577     my $device = parse_drive($opt, $conf->{$opt});
3578
3579     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3580         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3581         return 1;
3582     }
3583
3584     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3585
3586     my $devices_list = vm_devices_list($vmid);
3587     foreach my $opt (keys %{$devices_list}) {
3588         if (PVE::QemuServer::valid_drivename($opt)) {
3589             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3590             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3591                 return 1;
3592             }
3593         }
3594     }
3595
3596     my $scsihwid="scsihw$controller";
3597
3598     vm_deviceunplug($vmid, $conf, $scsihwid);
3599
3600     return 1;
3601 }
3602
3603 sub qemu_add_pci_bridge {
3604     my ($storecfg, $conf, $vmid, $device) = @_;
3605
3606     my $bridges = {};
3607
3608     my $bridgeid;
3609
3610     print_pci_addr($device, $bridges);
3611
3612     while (my ($k, $v) = each %$bridges) {
3613         $bridgeid = $k;
3614     }
3615     return 1 if !defined($bridgeid) || $bridgeid < 1;
3616
3617     my $bridge = "pci.$bridgeid";
3618     my $devices_list = vm_devices_list($vmid);
3619
3620     if (!defined($devices_list->{$bridge})) {
3621         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3622     }
3623
3624     return 1;
3625 }
3626
3627 sub qemu_set_link_status {
3628     my ($vmid, $device, $up) = @_;
3629
3630     vm_mon_cmd($vmid, "set_link", name => $device,
3631                up => $up ? JSON::true : JSON::false);
3632 }
3633
3634 sub qemu_netdevadd {
3635     my ($vmid, $conf, $device, $deviceid) = @_;
3636
3637     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
3638     my %options =  split(/[=,]/, $netdev);
3639
3640     vm_mon_cmd($vmid, "netdev_add",  %options);
3641     return 1;
3642 }
3643
3644 sub qemu_netdevdel {
3645     my ($vmid, $deviceid) = @_;
3646
3647     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3648 }
3649
3650 sub qemu_cpu_hotplug {
3651     my ($vmid, $conf, $vcpus) = @_;
3652
3653     my $sockets = 1;
3654     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3655     $sockets = $conf->{sockets} if  $conf->{sockets};
3656     my $cores = $conf->{cores} || 1;
3657     my $maxcpus = $sockets * $cores;
3658
3659     $vcpus = $maxcpus if !$vcpus;
3660
3661     die "you can't add more vcpus than maxcpus\n"
3662         if $vcpus > $maxcpus;
3663
3664     my $currentvcpus = $conf->{vcpus} || $maxcpus;
3665     die "online cpu unplug is not yet possible\n"
3666         if $vcpus < $currentvcpus;
3667
3668     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3669     die "vcpus in running vm is different than configuration\n"
3670         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
3671
3672     for (my $i = $currentvcpus; $i < $vcpus; $i++) {
3673         vm_mon_cmd($vmid, "cpu-add", id => int($i));
3674     }
3675 }
3676
3677 sub qemu_memory_hotplug {
3678     my ($vmid, $conf, $defaults, $opt, $value) = @_;
3679
3680     return $value if !check_running($vmid);
3681
3682     my $memory = $conf->{memory} || $defaults->{memory};
3683     $value = $defaults->{memory} if !$value;
3684     return $value if $value == $memory;
3685
3686     my $static_memory = $STATICMEM;
3687     my $dimm_memory = $memory - $static_memory;
3688
3689     die "memory can't be lower than $static_memory MB" if $value < $static_memory;
3690     die "you cannot add more memory than $MAX_MEM MB!\n" if $memory > $MAX_MEM;
3691
3692
3693     my $sockets = 1;
3694     $sockets = $conf->{sockets} if $conf->{sockets};
3695
3696     if($value > $memory) {
3697
3698         foreach_dimm($conf, $vmid, $value, $sockets, sub {
3699             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3700
3701                 return if $current_size <= $conf->{memory};
3702
3703                 eval { vm_mon_cmd($vmid, "object-add", 'qom-type' => "memory-backend-ram", id => "mem-$name", props => { size => int($dimm_size*1024*1024) } ) };
3704                 if (my $err = $@) {
3705                     eval { qemu_objectdel($vmid, "mem-$name"); };
3706                     die $err;
3707                 }
3708
3709                 eval { vm_mon_cmd($vmid, "device_add", driver => "pc-dimm", id => "$name", memdev => "mem-$name", node => $numanode) };
3710                 if (my $err = $@) {
3711                     eval { qemu_objectdel($vmid, "mem-$name"); };
3712                     die $err;
3713                 }
3714                 #update conf after each succesful module hotplug
3715                 $conf->{memory} = $current_size;
3716                 update_config_nolock($vmid, $conf, 1);
3717         });
3718
3719     } else {
3720
3721         foreach_reverse_dimm($conf, $vmid, $value, $sockets, sub {
3722             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
3723
3724                 return if $current_size >= $conf->{memory};
3725                 print "try to unplug memory dimm $name\n";
3726
3727                 my $retry = 0;
3728                 while (1) {
3729                     eval { qemu_devicedel($vmid, $name) };
3730                     sleep 3;
3731                     my $dimm_list = qemu_dimm_list($vmid);
3732                     last if !$dimm_list->{$name};
3733                     raise_param_exc({ $name => "error unplug memory module" }) if $retry > 5;
3734                     $retry++;
3735                 }
3736
3737                 #update conf after each succesful module unplug
3738                 $conf->{memory} = $current_size;
3739
3740                 eval { qemu_objectdel($vmid, "mem-$name"); };
3741                 update_config_nolock($vmid, $conf, 1);
3742         });
3743     }
3744 }
3745
3746 sub qemu_dimm_list {
3747     my ($vmid) = @_;
3748
3749     my $dimmarray = vm_mon_cmd_nocheck($vmid, "query-memory-devices");
3750     my $dimms = {};
3751
3752     foreach my $dimm (@$dimmarray) {
3753
3754         $dimms->{$dimm->{data}->{id}}->{id} = $dimm->{data}->{id};
3755         $dimms->{$dimm->{data}->{id}}->{node} = $dimm->{data}->{node};
3756         $dimms->{$dimm->{data}->{id}}->{addr} = $dimm->{data}->{addr};
3757         $dimms->{$dimm->{data}->{id}}->{size} = $dimm->{data}->{size};
3758         $dimms->{$dimm->{data}->{id}}->{slot} = $dimm->{data}->{slot};
3759     }
3760     return $dimms;
3761 }
3762
3763 sub qemu_block_set_io_throttle {
3764     my ($vmid, $deviceid, $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr) = @_;
3765
3766     return if !check_running($vmid) ;
3767
3768     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid, bps => int($bps), bps_rd => int($bps_rd), bps_wr => int($bps_wr), iops => int($iops), iops_rd => int($iops_rd), iops_wr => int($iops_wr));
3769
3770 }
3771
3772 # old code, only used to shutdown old VM after update
3773 sub __read_avail {
3774     my ($fh, $timeout) = @_;
3775
3776     my $sel = new IO::Select;
3777     $sel->add($fh);
3778
3779     my $res = '';
3780     my $buf;
3781
3782     my @ready;
3783     while (scalar (@ready = $sel->can_read($timeout))) {
3784         my $count;
3785         if ($count = $fh->sysread($buf, 8192)) {
3786             if ($buf =~ /^(.*)\(qemu\) $/s) {
3787                 $res .= $1;
3788                 last;
3789             } else {
3790                 $res .= $buf;
3791             }
3792         } else {
3793             if (!defined($count)) {
3794                 die "$!\n";
3795             }
3796             last;
3797         }
3798     }
3799
3800     die "monitor read timeout\n" if !scalar(@ready);
3801
3802     return $res;
3803 }
3804
3805 # old code, only used to shutdown old VM after update
3806 sub vm_monitor_command {
3807     my ($vmid, $cmdstr, $nocheck) = @_;
3808
3809     my $res;
3810
3811     eval {
3812         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
3813
3814         my $sname = "${var_run_tmpdir}/$vmid.mon";
3815
3816         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
3817             die "unable to connect to VM $vmid socket - $!\n";
3818
3819         my $timeout = 3;
3820
3821         # hack: migrate sometime blocks the monitor (when migrate_downtime
3822         # is set)
3823         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3824             $timeout = 60*60; # 1 hour
3825         }
3826
3827         # read banner;
3828         my $data = __read_avail($sock, $timeout);
3829
3830         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
3831             die "got unexpected qemu monitor banner\n";
3832         }
3833
3834         my $sel = new IO::Select;
3835         $sel->add($sock);
3836
3837         if (!scalar(my @ready = $sel->can_write($timeout))) {
3838             die "monitor write error - timeout";
3839         }
3840
3841         my $fullcmd = "$cmdstr\r";
3842
3843         # syslog('info', "VM $vmid monitor command: $cmdstr");
3844
3845         my $b;
3846         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
3847             die "monitor write error - $!";
3848         }
3849
3850         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
3851
3852         $timeout = 20;
3853
3854         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3855             $timeout = 60*60; # 1 hour
3856         } elsif ($cmdstr =~ m/^(eject|change)/) {
3857             $timeout = 60; # note: cdrom mount command is slow
3858         }
3859         if ($res = __read_avail($sock, $timeout)) {
3860
3861             my @lines = split("\r?\n", $res);
3862
3863             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
3864
3865             $res = join("\n", @lines);
3866             $res .= "\n";
3867         }
3868     };
3869
3870     my $err = $@;
3871
3872     if ($err) {
3873         syslog("err", "VM $vmid monitor command failed - $err");
3874         die $err;
3875     }
3876
3877     return $res;
3878 }
3879
3880 sub qemu_block_resize {
3881     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
3882
3883     my $running = check_running($vmid);
3884
3885     return if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
3886
3887     return if !$running;
3888
3889     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
3890
3891 }
3892
3893 sub qemu_volume_snapshot {
3894     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3895
3896     my $running = check_running($vmid);
3897
3898     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
3899         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
3900     } else {
3901         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
3902     }
3903 }
3904
3905 sub qemu_volume_snapshot_delete {
3906     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3907
3908     my $running = check_running($vmid);
3909
3910     return if !PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
3911
3912     return if !$running;
3913
3914     vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
3915 }
3916
3917 sub set_migration_caps {
3918     my ($vmid) = @_;
3919
3920     my $cap_ref = [];
3921
3922     my $enabled_cap = {
3923         "auto-converge" => 1,
3924         "xbzrle" => 1,
3925         "x-rdma-pin-all" => 0,
3926         "zero-blocks" => 0,
3927         "compress" => 0
3928     };
3929
3930     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
3931
3932     for my $supported_capability (@$supported_capabilities) {
3933         push @$cap_ref, {
3934             capability => $supported_capability->{capability},
3935             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
3936         };
3937     }
3938
3939     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
3940 }
3941
3942 my $fast_plug_option = {
3943     'lock' => 1,
3944     'name' => 1,
3945     'onboot' => 1,
3946     'shares' => 1,
3947     'startup' => 1,
3948     'description' => 1,
3949 };
3950
3951 # hotplug changes in [PENDING]
3952 # $selection hash can be used to only apply specified options, for
3953 # example: { cores => 1 } (only apply changed 'cores')
3954 # $errors ref is used to return error messages
3955 sub vmconfig_hotplug_pending {
3956     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
3957
3958     my $defaults = load_defaults();
3959
3960     # commit values which do not have any impact on running VM first
3961     # Note: those option cannot raise errors, we we do not care about
3962     # $selection and always apply them.
3963
3964     my $add_error = sub {
3965         my ($opt, $msg) = @_;
3966         $errors->{$opt} = "hotplug problem - $msg";
3967     };
3968
3969     my $changes = 0;
3970     foreach my $opt (keys %{$conf->{pending}}) { # add/change
3971         if ($fast_plug_option->{$opt}) {
3972             $conf->{$opt} = $conf->{pending}->{$opt};
3973             delete $conf->{pending}->{$opt};
3974             $changes = 1;
3975         }
3976     }
3977
3978     if ($changes) {
3979         update_config_nolock($vmid, $conf, 1);
3980         $conf = load_config($vmid); # update/reload
3981     }
3982
3983     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3984
3985     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
3986     while (my ($opt, $force) = each %$pending_delete_hash) {
3987         next if $selection && !$selection->{$opt};
3988         eval {
3989             if ($opt eq 'hotplug') {
3990                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
3991             } elsif ($opt eq 'tablet') {
3992                 die "skip\n" if !$hotplug_features->{usb};
3993                 if ($defaults->{tablet}) {
3994                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
3995                 } else {
3996                     vm_deviceunplug($vmid, $conf, $opt);
3997                 }
3998             } elsif ($opt eq 'vcpus') {
3999                 die "skip\n" if !$hotplug_features->{cpu};
4000                 qemu_cpu_hotplug($vmid, $conf, undef);
4001             } elsif ($opt eq 'balloon') {
4002                 # enable balloon device is not hotpluggable
4003                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
4004             } elsif ($fast_plug_option->{$opt}) {
4005                 # do nothing
4006             } elsif ($opt =~ m/^net(\d+)$/) {
4007                 die "skip\n" if !$hotplug_features->{network};
4008                 vm_deviceunplug($vmid, $conf, $opt);
4009             } elsif (valid_drivename($opt)) {
4010                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4011                 vm_deviceunplug($vmid, $conf, $opt);
4012                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4013             } elsif ($opt =~ m/^memory$/) {
4014                 die "skip\n" if !$hotplug_features->{memory};
4015                 qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4016             } elsif ($opt eq 'cpuunits') {
4017                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4018             } elsif ($opt eq 'cpulimit') {
4019                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4020             } else {
4021                 die "skip\n";
4022             }
4023         };
4024         if (my $err = $@) {
4025             &$add_error($opt, $err) if $err ne "skip\n";
4026         } else {
4027             # save new config if hotplug was successful
4028             delete $conf->{$opt};
4029             vmconfig_undelete_pending_option($conf, $opt);
4030             update_config_nolock($vmid, $conf, 1);
4031             $conf = load_config($vmid); # update/reload
4032         }
4033     }
4034
4035     foreach my $opt (keys %{$conf->{pending}}) {
4036         next if $selection && !$selection->{$opt};
4037         my $value = $conf->{pending}->{$opt};
4038         eval {
4039             if ($opt eq 'hotplug') {
4040                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4041             } elsif ($opt eq 'tablet') {
4042                 die "skip\n" if !$hotplug_features->{usb};
4043                 if ($value == 1) {
4044                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4045                 } elsif ($value == 0) {
4046                     vm_deviceunplug($vmid, $conf, $opt);
4047                 }
4048             } elsif ($opt eq 'vcpus') {
4049                 die "skip\n" if !$hotplug_features->{cpu};
4050                 qemu_cpu_hotplug($vmid, $conf, $value);
4051             } elsif ($opt eq 'balloon') {
4052                 # enable/disable balloning device is not hotpluggable
4053                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4054                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4055                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4056
4057                 # allow manual ballooning if shares is set to zero
4058                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4059                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4060                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4061                 }
4062             } elsif ($opt =~ m/^net(\d+)$/) {
4063                 # some changes can be done without hotplug
4064                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4065                                     $vmid, $opt, $value);
4066             } elsif (valid_drivename($opt)) {
4067                 # some changes can be done without hotplug
4068                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4069                                      $vmid, $opt, $value, 1);
4070             } elsif ($opt =~ m/^memory$/) { #dimms
4071                 die "skip\n" if !$hotplug_features->{memory};
4072                 $value = qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4073             } elsif ($opt eq 'cpuunits') {
4074                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4075             } elsif ($opt eq 'cpulimit') {
4076                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4077                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4078             } else {
4079                 die "skip\n";  # skip non-hot-pluggable options
4080             }
4081         };
4082         if (my $err = $@) {
4083             &$add_error($opt, $err) if $err ne "skip\n";
4084         } else {
4085             # save new config if hotplug was successful
4086             $conf->{$opt} = $value;
4087             delete $conf->{pending}->{$opt};
4088             update_config_nolock($vmid, $conf, 1);
4089             $conf = load_config($vmid); # update/reload
4090         }
4091     }
4092 }
4093
4094 sub try_deallocate_drive {
4095     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4096
4097     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4098         my $volid = $drive->{file};
4099         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4100             my $sid = PVE::Storage::parse_volume_id($volid);
4101             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4102
4103             # check if the disk is really unused
4104             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4105                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4106             PVE::Storage::vdisk_free($storecfg, $volid);
4107             return 1;
4108         } else {
4109             # If vm is not owner of this disk remove from config
4110             return 1;
4111         }
4112     }
4113
4114     return undef;
4115 }
4116
4117 sub vmconfig_delete_or_detach_drive {
4118     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4119
4120     my $drive = parse_drive($opt, $conf->{$opt});
4121
4122     my $rpcenv = PVE::RPCEnvironment::get();
4123     my $authuser = $rpcenv->get_user();
4124
4125     if ($force) {
4126         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4127         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4128     } else {
4129         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4130     }
4131 }
4132
4133 sub vmconfig_apply_pending {
4134     my ($vmid, $conf, $storecfg) = @_;
4135
4136     # cold plug
4137
4138     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4139     while (my ($opt, $force) = each %$pending_delete_hash) {
4140         die "internal error" if $opt =~ m/^unused/;
4141         $conf = load_config($vmid); # update/reload
4142         if (!defined($conf->{$opt})) {
4143             vmconfig_undelete_pending_option($conf, $opt);
4144             update_config_nolock($vmid, $conf, 1);
4145         } elsif (valid_drivename($opt)) {
4146             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4147             vmconfig_undelete_pending_option($conf, $opt);
4148             delete $conf->{$opt};
4149             update_config_nolock($vmid, $conf, 1);
4150         } else {
4151             vmconfig_undelete_pending_option($conf, $opt);
4152             delete $conf->{$opt};
4153             update_config_nolock($vmid, $conf, 1);
4154         }
4155     }
4156
4157     $conf = load_config($vmid); # update/reload
4158
4159     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4160         $conf = load_config($vmid); # update/reload
4161
4162         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4163             # skip if nothing changed
4164         } elsif (valid_drivename($opt)) {
4165             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4166                 if defined($conf->{$opt});
4167             $conf->{$opt} = $conf->{pending}->{$opt};
4168         } else {
4169             $conf->{$opt} = $conf->{pending}->{$opt};
4170         }
4171
4172         delete $conf->{pending}->{$opt};
4173         update_config_nolock($vmid, $conf, 1);
4174     }
4175 }
4176
4177 my $safe_num_ne = sub {
4178     my ($a, $b) = @_;
4179
4180     return 0 if !defined($a) && !defined($b);
4181     return 1 if !defined($a);
4182     return 1 if !defined($b);
4183
4184     return $a != $b;
4185 };
4186
4187 my $safe_string_ne = sub {
4188     my ($a, $b) = @_;
4189
4190     return 0 if !defined($a) && !defined($b);
4191     return 1 if !defined($a);
4192     return 1 if !defined($b);
4193
4194     return $a ne $b;
4195 };
4196
4197 sub vmconfig_update_net {
4198     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4199
4200     my $newnet = parse_net($value);
4201
4202     if ($conf->{$opt}) {
4203         my $oldnet = parse_net($conf->{$opt});
4204
4205         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4206             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4207             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4208             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4209
4210             # for non online change, we try to hot-unplug
4211             die "skip\n" if !$hotplug;
4212             vm_deviceunplug($vmid, $conf, $opt);
4213         } else {
4214
4215             die "internal error" if $opt !~ m/net(\d+)/;
4216             my $iface = "tap${vmid}i$1";
4217
4218             if (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4219                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4220             }
4221
4222             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4223                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4224                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4225                 PVE::Network::tap_unplug($iface);
4226                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall});
4227             }
4228
4229             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4230                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4231             }
4232
4233             return 1;
4234         }
4235     }
4236
4237     if ($hotplug) {
4238         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4239     } else {
4240         die "skip\n";
4241     }
4242 }
4243
4244 sub vmconfig_update_disk {
4245     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4246
4247     # fixme: do we need force?
4248
4249     my $drive = parse_drive($opt, $value);
4250
4251     if ($conf->{$opt}) {
4252
4253         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4254
4255             my $media = $drive->{media} || 'disk';
4256             my $oldmedia = $old_drive->{media} || 'disk';
4257             die "unable to change media type\n" if $media ne $oldmedia;
4258
4259             if (!drive_is_cdrom($old_drive)) {
4260
4261                 if ($drive->{file} ne $old_drive->{file}) {
4262
4263                     die "skip\n" if !$hotplug;
4264
4265                     # unplug and register as unused
4266                     vm_deviceunplug($vmid, $conf, $opt);
4267                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4268
4269                 } else {
4270                     # update existing disk
4271
4272                     # skip non hotpluggable value
4273                     if (&$safe_num_ne($drive->{discard}, $old_drive->{discard}) ||
4274                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4275                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4276                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4277                         die "skip\n";
4278                     }
4279
4280                     # apply throttle
4281                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4282                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4283                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4284                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4285                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4286                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4287                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4288                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4289                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4290                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4291                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4292                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max})) {
4293
4294                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4295                                                    ($drive->{mbps} || 0)*1024*1024,
4296                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4297                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4298                                                    $drive->{iops} || 0,
4299                                                    $drive->{iops_rd} || 0,
4300                                                    $drive->{iops_wr} || 0,
4301                                                    ($drive->{mbps_max} || 0)*1024*1024,
4302                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4303                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4304                                                    $drive->{iops_max} || 0,
4305                                                    $drive->{iops_rd_max} || 0,
4306                                                    $drive->{iops_wr_max} || 0);
4307
4308                     }
4309
4310                     return 1;
4311                 }
4312
4313             } else { # cdrom
4314
4315                 if ($drive->{file} eq 'none') {
4316                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4317                 } else {
4318                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4319                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4320                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4321                 }
4322
4323                 return 1;
4324             }
4325         }
4326     }
4327
4328     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4329     # hotplug new disks
4330     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]);
4331     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4332 }
4333
4334 sub vm_start {
4335     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4336         $forcemachine, $spice_ticket) = @_;
4337
4338     lock_config($vmid, sub {
4339         my $conf = load_config($vmid, $migratedfrom);
4340
4341         die "you can't start a vm if it's a template\n" if is_template($conf);
4342
4343         check_lock($conf) if !$skiplock;
4344
4345         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4346
4347         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4348             vmconfig_apply_pending($vmid, $conf, $storecfg);
4349             $conf = load_config($vmid); # update/reload
4350         }
4351
4352         my $defaults = load_defaults();
4353
4354         # set environment variable useful inside network script
4355         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4356
4357         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4358
4359         my $migrate_port = 0;
4360         my $migrate_uri;
4361         if ($statefile) {
4362             if ($statefile eq 'tcp') {
4363                 my $localip = "localhost";
4364                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4365                 my $nodename = PVE::INotify::nodename();
4366                 if ($datacenterconf->{migration_unsecure}) {
4367                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4368                         $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4369                 }
4370                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4371                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4372                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4373                 push @$cmd, '-incoming', $migrate_uri;
4374                 push @$cmd, '-S';
4375             } else {
4376                 push @$cmd, '-loadstate', $statefile;
4377             }
4378         } elsif ($paused) {
4379             push @$cmd, '-S';
4380         }
4381
4382         # host pci devices
4383         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4384           my $d = parse_hostpci($conf->{"hostpci$i"});
4385           next if !$d;
4386           my $pcidevices = $d->{pciid};
4387           foreach my $pcidevice (@$pcidevices) {
4388                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4389
4390                 my $info = pci_device_info("0000:$pciid");
4391                 die "IOMMU not present\n" if !check_iommu_support();
4392                 die "no pci device info for device '$pciid'\n" if !$info;
4393                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4394                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4395           }
4396         }
4397
4398         PVE::Storage::activate_volumes($storecfg, $vollist);
4399
4400         eval  { run_command($cmd, timeout => $statefile ? undef : 30,
4401                     umask => 0077); };
4402         my $err = $@;
4403         die "start failed: $err" if $err;
4404
4405         print "migration listens on $migrate_uri\n" if $migrate_uri;
4406
4407         if ($statefile && $statefile ne 'tcp')  {
4408             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4409             warn $@ if $@;
4410         }
4411
4412         if ($migratedfrom) {
4413
4414             eval {
4415                 set_migration_caps($vmid);
4416             };
4417             warn $@ if $@;
4418
4419             if ($spice_port) {
4420                 print "spice listens on port $spice_port\n";
4421                 if ($spice_ticket) {
4422                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4423                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4424                 }
4425             }
4426
4427         } else {
4428
4429             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4430                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4431                     if $conf->{balloon};
4432             }
4433
4434             foreach my $opt (keys %$conf) {
4435                 next if $opt !~  m/^net\d+$/;
4436                 my $nicconf = parse_net($conf->{$opt});
4437                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4438             }
4439         }
4440
4441         vm_mon_cmd_nocheck($vmid, 'qom-set',
4442                     path => "machine/peripheral/balloon0",
4443                     property => "guest-stats-polling-interval",
4444                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4445
4446     });
4447 }
4448
4449 sub vm_mon_cmd {
4450     my ($vmid, $execute, %params) = @_;
4451
4452     my $cmd = { execute => $execute, arguments => \%params };
4453     vm_qmp_command($vmid, $cmd);
4454 }
4455
4456 sub vm_mon_cmd_nocheck {
4457     my ($vmid, $execute, %params) = @_;
4458
4459     my $cmd = { execute => $execute, arguments => \%params };
4460     vm_qmp_command($vmid, $cmd, 1);
4461 }
4462
4463 sub vm_qmp_command {
4464     my ($vmid, $cmd, $nocheck) = @_;
4465
4466     my $res;
4467
4468     my $timeout;
4469     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4470         $timeout = $cmd->{arguments}->{timeout};
4471         delete $cmd->{arguments}->{timeout};
4472     }
4473
4474     eval {
4475         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4476         my $sname = qmp_socket($vmid);
4477         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4478             my $qmpclient = PVE::QMPClient->new();
4479
4480             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4481         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
4482             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
4483                 if scalar(%{$cmd->{arguments}});
4484             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
4485         } else {
4486             die "unable to open monitor socket\n";
4487         }
4488     };
4489     if (my $err = $@) {
4490         syslog("err", "VM $vmid qmp command failed - $err");
4491         die $err;
4492     }
4493
4494     return $res;
4495 }
4496
4497 sub vm_human_monitor_command {
4498     my ($vmid, $cmdline) = @_;
4499
4500     my $res;
4501
4502     my $cmd = {
4503         execute => 'human-monitor-command',
4504         arguments => { 'command-line' => $cmdline},
4505     };
4506
4507     return vm_qmp_command($vmid, $cmd);
4508 }
4509
4510 sub vm_commandline {
4511     my ($storecfg, $vmid) = @_;
4512
4513     my $conf = load_config($vmid);
4514
4515     my $defaults = load_defaults();
4516
4517     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
4518
4519     return join(' ', @$cmd);
4520 }
4521
4522 sub vm_reset {
4523     my ($vmid, $skiplock) = @_;
4524
4525     lock_config($vmid, sub {
4526
4527         my $conf = load_config($vmid);
4528
4529         check_lock($conf) if !$skiplock;
4530
4531         vm_mon_cmd($vmid, "system_reset");
4532     });
4533 }
4534
4535 sub get_vm_volumes {
4536     my ($conf) = @_;
4537
4538     my $vollist = [];
4539     foreach_volid($conf, sub {
4540         my ($volid, $is_cdrom) = @_;
4541
4542         return if $volid =~ m|^/|;
4543
4544         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4545         return if !$sid;
4546
4547         push @$vollist, $volid;
4548     });
4549
4550     return $vollist;
4551 }
4552
4553 sub vm_stop_cleanup {
4554     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
4555
4556     eval {
4557
4558         if (!$keepActive) {
4559             my $vollist = get_vm_volumes($conf);
4560             PVE::Storage::deactivate_volumes($storecfg, $vollist);
4561         }
4562
4563         foreach my $ext (qw(mon qmp pid vnc qga)) {
4564             unlink "/var/run/qemu-server/${vmid}.$ext";
4565         }
4566
4567         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
4568     };
4569     warn $@ if $@; # avoid errors - just warn
4570 }
4571
4572 # Note: use $nockeck to skip tests if VM configuration file exists.
4573 # We need that when migration VMs to other nodes (files already moved)
4574 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
4575 sub vm_stop {
4576     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
4577
4578     $force = 1 if !defined($force) && !$shutdown;
4579
4580     if ($migratedfrom){
4581         my $pid = check_running($vmid, $nocheck, $migratedfrom);
4582         kill 15, $pid if $pid;
4583         my $conf = load_config($vmid, $migratedfrom);
4584         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
4585         return;
4586     }
4587
4588     lock_config($vmid, sub {
4589
4590         my $pid = check_running($vmid, $nocheck);
4591         return if !$pid;
4592
4593         my $conf;
4594         if (!$nocheck) {
4595             $conf = load_config($vmid);
4596             check_lock($conf) if !$skiplock;
4597             if (!defined($timeout) && $shutdown && $conf->{startup}) {
4598                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
4599                 $timeout = $opts->{down} if $opts->{down};
4600             }
4601         }
4602
4603         $timeout = 60 if !defined($timeout);
4604
4605         eval {
4606             if ($shutdown) {
4607                 if (defined($conf) && $conf->{agent}) {
4608                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
4609                 } else {
4610                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
4611                 }
4612             } else {
4613                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
4614             }
4615         };
4616         my $err = $@;
4617
4618         if (!$err) {
4619             my $count = 0;
4620             while (($count < $timeout) && check_running($vmid, $nocheck)) {
4621                 $count++;
4622                 sleep 1;
4623             }
4624
4625             if ($count >= $timeout) {
4626                 if ($force) {
4627                     warn "VM still running - terminating now with SIGTERM\n";
4628                     kill 15, $pid;
4629                 } else {
4630                     die "VM quit/powerdown failed - got timeout\n";
4631                 }
4632             } else {
4633                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4634                 return;
4635             }
4636         } else {
4637             if ($force) {
4638                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
4639                 kill 15, $pid;
4640             } else {
4641                 die "VM quit/powerdown failed\n";
4642             }
4643         }
4644
4645         # wait again
4646         $timeout = 10;
4647
4648         my $count = 0;
4649         while (($count < $timeout) && check_running($vmid, $nocheck)) {
4650             $count++;
4651             sleep 1;
4652         }
4653
4654         if ($count >= $timeout) {
4655             warn "VM still running - terminating now with SIGKILL\n";
4656             kill 9, $pid;
4657             sleep 1;
4658         }
4659
4660         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4661    });
4662 }
4663
4664 sub vm_suspend {
4665     my ($vmid, $skiplock) = @_;
4666
4667     lock_config($vmid, sub {
4668
4669         my $conf = load_config($vmid);
4670
4671         check_lock($conf) if !($skiplock || ($conf->{lock} && $conf->{lock} eq 'backup'));
4672
4673         vm_mon_cmd($vmid, "stop");
4674     });
4675 }
4676
4677 sub vm_resume {
4678     my ($vmid, $skiplock, $nocheck) = @_;
4679
4680     lock_config($vmid, sub {
4681
4682         if (!$nocheck) {
4683
4684             my $conf = load_config($vmid);
4685
4686             check_lock($conf) if !($skiplock || ($conf->{lock} && $conf->{lock} eq 'backup'));
4687
4688             vm_mon_cmd($vmid, "cont");
4689
4690         } else {
4691             vm_mon_cmd_nocheck($vmid, "cont");
4692         }
4693     });
4694 }
4695
4696 sub vm_sendkey {
4697     my ($vmid, $skiplock, $key) = @_;
4698
4699     lock_config($vmid, sub {
4700
4701         my $conf = load_config($vmid);
4702
4703         # there is no qmp command, so we use the human monitor command
4704         vm_human_monitor_command($vmid, "sendkey $key");
4705     });
4706 }
4707
4708 sub vm_destroy {
4709     my ($storecfg, $vmid, $skiplock) = @_;
4710
4711     lock_config($vmid, sub {
4712
4713         my $conf = load_config($vmid);
4714
4715         check_lock($conf) if !$skiplock;
4716
4717         if (!check_running($vmid)) {
4718             destroy_vm($storecfg, $vmid);
4719         } else {
4720             die "VM $vmid is running - destroy failed\n";
4721         }
4722     });
4723 }
4724
4725 # pci helpers
4726
4727 sub file_write {
4728     my ($filename, $buf) = @_;
4729
4730     my $fh = IO::File->new($filename, "w");
4731     return undef if !$fh;
4732
4733     my $res = print $fh $buf;
4734
4735     $fh->close();
4736
4737     return $res;
4738 }
4739
4740 sub pci_device_info {
4741     my ($name) = @_;
4742
4743     my $res;
4744
4745     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
4746     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
4747
4748     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
4749     return undef if !defined($irq) || $irq !~ m/^\d+$/;
4750
4751     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
4752     return undef if !defined($vendor) || $vendor !~ s/^0x//;
4753
4754     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
4755     return undef if !defined($product) || $product !~ s/^0x//;
4756
4757     $res = {
4758         name => $name,
4759         vendor => $vendor,
4760         product => $product,
4761         domain => $domain,
4762         bus => $bus,
4763         slot => $slot,
4764         func => $func,
4765         irq => $irq,
4766         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
4767     };
4768
4769     return $res;
4770 }
4771
4772 sub pci_dev_reset {
4773     my ($dev) = @_;
4774
4775     my $name = $dev->{name};
4776
4777     my $fn = "$pcisysfs/devices/$name/reset";
4778
4779     return file_write($fn, "1");
4780 }
4781
4782 sub pci_dev_bind_to_vfio {
4783     my ($dev) = @_;
4784
4785     my $name = $dev->{name};
4786
4787     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4788
4789     if (!-d $vfio_basedir) {
4790         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4791     }
4792     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4793
4794     my $testdir = "$vfio_basedir/$name";
4795     return 1 if -d $testdir;
4796
4797     my $data = "$dev->{vendor} $dev->{product}";
4798     return undef if !file_write("$vfio_basedir/new_id", $data);
4799
4800     my $fn = "$pcisysfs/devices/$name/driver/unbind";
4801     if (!file_write($fn, $name)) {
4802         return undef if -f $fn;
4803     }
4804
4805     $fn = "$vfio_basedir/bind";
4806     if (! -d $testdir) {
4807         return undef if !file_write($fn, $name);
4808     }
4809
4810     return -d $testdir;
4811 }
4812
4813 sub pci_dev_group_bind_to_vfio {
4814     my ($pciid) = @_;
4815
4816     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4817
4818     if (!-d $vfio_basedir) {
4819         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4820     }
4821     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4822
4823     # get IOMMU group devices
4824     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
4825       my @devs = grep /^0000:/, readdir($D);
4826     closedir($D);
4827
4828     foreach my $pciid (@devs) {
4829         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
4830
4831         # pci bridges, switches or root ports are not supported
4832         # they have a pci_bus subdirectory so skip them
4833         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
4834
4835         my $info = pci_device_info($1);
4836         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
4837     }
4838
4839     return 1;
4840 }
4841
4842 sub print_pci_addr {
4843     my ($id, $bridges) = @_;
4844
4845     my $res = '';
4846     my $devices = {
4847         piix3 => { bus => 0, addr => 1 },
4848         #addr2 : first videocard
4849         balloon0 => { bus => 0, addr => 3 },
4850         watchdog => { bus => 0, addr => 4 },
4851         scsihw0 => { bus => 0, addr => 5 },
4852         'pci.3' => { bus => 0, addr => 5 }, #can also be used for virtio-scsi-single bridge
4853         scsihw1 => { bus => 0, addr => 6 },
4854         ahci0 => { bus => 0, addr => 7 },
4855         qga0 => { bus => 0, addr => 8 },
4856         spice => { bus => 0, addr => 9 },
4857         virtio0 => { bus => 0, addr => 10 },
4858         virtio1 => { bus => 0, addr => 11 },
4859         virtio2 => { bus => 0, addr => 12 },
4860         virtio3 => { bus => 0, addr => 13 },
4861         virtio4 => { bus => 0, addr => 14 },
4862         virtio5 => { bus => 0, addr => 15 },
4863         hostpci0 => { bus => 0, addr => 16 },
4864         hostpci1 => { bus => 0, addr => 17 },
4865         net0 => { bus => 0, addr => 18 },
4866         net1 => { bus => 0, addr => 19 },
4867         net2 => { bus => 0, addr => 20 },
4868         net3 => { bus => 0, addr => 21 },
4869         net4 => { bus => 0, addr => 22 },
4870         net5 => { bus => 0, addr => 23 },
4871         vga1 => { bus => 0, addr => 24 },
4872         vga2 => { bus => 0, addr => 25 },
4873         vga3 => { bus => 0, addr => 26 },
4874         hostpci2 => { bus => 0, addr => 27 },
4875         hostpci3 => { bus => 0, addr => 28 },
4876         #addr29 : usb-host (pve-usb.cfg)
4877         'pci.1' => { bus => 0, addr => 30 },
4878         'pci.2' => { bus => 0, addr => 31 },
4879         'net6' => { bus => 1, addr => 1 },
4880         'net7' => { bus => 1, addr => 2 },
4881         'net8' => { bus => 1, addr => 3 },
4882         'net9' => { bus => 1, addr => 4 },
4883         'net10' => { bus => 1, addr => 5 },
4884         'net11' => { bus => 1, addr => 6 },
4885         'net12' => { bus => 1, addr => 7 },
4886         'net13' => { bus => 1, addr => 8 },
4887         'net14' => { bus => 1, addr => 9 },
4888         'net15' => { bus => 1, addr => 10 },
4889         'net16' => { bus => 1, addr => 11 },
4890         'net17' => { bus => 1, addr => 12 },
4891         'net18' => { bus => 1, addr => 13 },
4892         'net19' => { bus => 1, addr => 14 },
4893         'net20' => { bus => 1, addr => 15 },
4894         'net21' => { bus => 1, addr => 16 },
4895         'net22' => { bus => 1, addr => 17 },
4896         'net23' => { bus => 1, addr => 18 },
4897         'net24' => { bus => 1, addr => 19 },
4898         'net25' => { bus => 1, addr => 20 },
4899         'net26' => { bus => 1, addr => 21 },
4900         'net27' => { bus => 1, addr => 22 },
4901         'net28' => { bus => 1, addr => 23 },
4902         'net29' => { bus => 1, addr => 24 },
4903         'net30' => { bus => 1, addr => 25 },
4904         'net31' => { bus => 1, addr => 26 },
4905         'virtio6' => { bus => 2, addr => 1 },
4906         'virtio7' => { bus => 2, addr => 2 },
4907         'virtio8' => { bus => 2, addr => 3 },
4908         'virtio9' => { bus => 2, addr => 4 },
4909         'virtio10' => { bus => 2, addr => 5 },
4910         'virtio11' => { bus => 2, addr => 6 },
4911         'virtio12' => { bus => 2, addr => 7 },
4912         'virtio13' => { bus => 2, addr => 8 },
4913         'virtio14' => { bus => 2, addr => 9 },
4914         'virtio15' => { bus => 2, addr => 10 },
4915         'virtioscsi0' => { bus => 3, addr => 1 },
4916         'virtioscsi1' => { bus => 3, addr => 2 },
4917         'virtioscsi2' => { bus => 3, addr => 3 },
4918         'virtioscsi3' => { bus => 3, addr => 4 },
4919         'virtioscsi4' => { bus => 3, addr => 5 },
4920         'virtioscsi5' => { bus => 3, addr => 6 },
4921         'virtioscsi6' => { bus => 3, addr => 7 },
4922         'virtioscsi7' => { bus => 3, addr => 8 },
4923         'virtioscsi8' => { bus => 3, addr => 9 },
4924         'virtioscsi9' => { bus => 3, addr => 10 },
4925         'virtioscsi10' => { bus => 3, addr => 11 },
4926         'virtioscsi11' => { bus => 3, addr => 12 },
4927         'virtioscsi12' => { bus => 3, addr => 13 },
4928         'virtioscsi13' => { bus => 3, addr => 14 },
4929         'virtioscsi14' => { bus => 3, addr => 15 },
4930         'virtioscsi15' => { bus => 3, addr => 16 },
4931         'virtioscsi16' => { bus => 3, addr => 17 },
4932         'virtioscsi17' => { bus => 3, addr => 18 },
4933         'virtioscsi18' => { bus => 3, addr => 19 },
4934         'virtioscsi19' => { bus => 3, addr => 20 },
4935         'virtioscsi20' => { bus => 3, addr => 21 },
4936         'virtioscsi21' => { bus => 3, addr => 22 },
4937         'virtioscsi22' => { bus => 3, addr => 23 },
4938         'virtioscsi23' => { bus => 3, addr => 24 },
4939         'virtioscsi24' => { bus => 3, addr => 25 },
4940         'virtioscsi25' => { bus => 3, addr => 26 },
4941         'virtioscsi26' => { bus => 3, addr => 27 },
4942         'virtioscsi27' => { bus => 3, addr => 28 },
4943         'virtioscsi28' => { bus => 3, addr => 29 },
4944         'virtioscsi29' => { bus => 3, addr => 30 },
4945         'virtioscsi30' => { bus => 3, addr => 31 },
4946
4947     };
4948
4949     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
4950            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
4951            my $bus = $devices->{$id}->{bus};
4952            $res = ",bus=pci.$bus,addr=$addr";
4953            $bridges->{$bus} = 1 if $bridges;
4954     }
4955     return $res;
4956
4957 }
4958
4959 sub print_pcie_addr {
4960     my ($id) = @_;
4961
4962     my $res = '';
4963     my $devices = {
4964         hostpci0 => { bus => "ich9-pcie-port-1", addr => 0 },
4965         hostpci1 => { bus => "ich9-pcie-port-2", addr => 0 },
4966         hostpci2 => { bus => "ich9-pcie-port-3", addr => 0 },
4967         hostpci3 => { bus => "ich9-pcie-port-4", addr => 0 },
4968     };
4969
4970     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
4971            my $addr = sprintf("0x%x", $devices->{$id}->{addr});
4972            my $bus = $devices->{$id}->{bus};
4973            $res = ",bus=$bus,addr=$addr";
4974     }
4975     return $res;
4976
4977 }
4978
4979 # vzdump restore implementaion
4980
4981 sub tar_archive_read_firstfile {
4982     my $archive = shift;
4983
4984     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
4985
4986     # try to detect archive type first
4987     my $pid = open (TMP, "tar tf '$archive'|") ||
4988         die "unable to open file '$archive'\n";
4989     my $firstfile = <TMP>;
4990     kill 15, $pid;
4991     close TMP;
4992
4993     die "ERROR: archive contaions no data\n" if !$firstfile;
4994     chomp $firstfile;
4995
4996     return $firstfile;
4997 }
4998
4999 sub tar_restore_cleanup {
5000     my ($storecfg, $statfile) = @_;
5001
5002     print STDERR "starting cleanup\n";
5003
5004     if (my $fd = IO::File->new($statfile, "r")) {
5005         while (defined(my $line = <$fd>)) {
5006             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5007                 my $volid = $2;
5008                 eval {
5009                     if ($volid =~ m|^/|) {
5010                         unlink $volid || die 'unlink failed\n';
5011                     } else {
5012                         PVE::Storage::vdisk_free($storecfg, $volid);
5013                     }
5014                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5015                 };
5016                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5017             } else {
5018                 print STDERR "unable to parse line in statfile - $line";
5019             }
5020         }
5021         $fd->close();
5022     }
5023 }
5024
5025 sub restore_archive {
5026     my ($archive, $vmid, $user, $opts) = @_;
5027
5028     my $format = $opts->{format};
5029     my $comp;
5030
5031     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5032         $format = 'tar' if !$format;
5033         $comp = 'gzip';
5034     } elsif ($archive =~ m/\.tar$/) {
5035         $format = 'tar' if !$format;
5036     } elsif ($archive =~ m/.tar.lzo$/) {
5037         $format = 'tar' if !$format;
5038         $comp = 'lzop';
5039     } elsif ($archive =~ m/\.vma$/) {
5040         $format = 'vma' if !$format;
5041     } elsif ($archive =~ m/\.vma\.gz$/) {
5042         $format = 'vma' if !$format;
5043         $comp = 'gzip';
5044     } elsif ($archive =~ m/\.vma\.lzo$/) {
5045         $format = 'vma' if !$format;
5046         $comp = 'lzop';
5047     } else {
5048         $format = 'vma' if !$format; # default
5049     }
5050
5051     # try to detect archive format
5052     if ($format eq 'tar') {
5053         return restore_tar_archive($archive, $vmid, $user, $opts);
5054     } else {
5055         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5056     }
5057 }
5058
5059 sub restore_update_config_line {
5060     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5061
5062     return if $line =~ m/^\#qmdump\#/;
5063     return if $line =~ m/^\#vzdump\#/;
5064     return if $line =~ m/^lock:/;
5065     return if $line =~ m/^unused\d+:/;
5066     return if $line =~ m/^parent:/;
5067     return if $line =~ m/^template:/; # restored VM is never a template
5068
5069     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5070         # try to convert old 1.X settings
5071         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5072         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5073             my ($model, $macaddr) = split(/\=/, $devconfig);
5074             $macaddr = PVE::Tools::random_ether_addr() if !$macaddr || $unique;
5075             my $net = {
5076                 model => $model,
5077                 bridge => "vmbr$ind",
5078                 macaddr => $macaddr,
5079             };
5080             my $netstr = print_net($net);
5081
5082             print $outfd "net$cookie->{netcount}: $netstr\n";
5083             $cookie->{netcount}++;
5084         }
5085     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5086         my ($id, $netstr) = ($1, $2);
5087         my $net = parse_net($netstr);
5088         $net->{macaddr} = PVE::Tools::random_ether_addr() if $net->{macaddr};
5089         $netstr = print_net($net);
5090         print $outfd "$id: $netstr\n";
5091     } elsif ($line =~ m/^((ide|scsi|virtio|sata)\d+):\s*(\S+)\s*$/) {
5092         my $virtdev = $1;
5093         my $value = $3;
5094         if ($line =~ m/backup=no/) {
5095             print $outfd "#$line";
5096         } elsif ($virtdev && $map->{$virtdev}) {
5097             my $di = parse_drive($virtdev, $value);
5098             delete $di->{format}; # format can change on restore
5099             $di->{file} = $map->{$virtdev};
5100             $value = print_drive($vmid, $di);
5101             print $outfd "$virtdev: $value\n";
5102         } else {
5103             print $outfd $line;
5104         }
5105     } else {
5106         print $outfd $line;
5107     }
5108 }
5109
5110 sub scan_volids {
5111     my ($cfg, $vmid) = @_;
5112
5113     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5114
5115     my $volid_hash = {};
5116     foreach my $storeid (keys %$info) {
5117         foreach my $item (@{$info->{$storeid}}) {
5118             next if !($item->{volid} && $item->{size});
5119             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5120             $volid_hash->{$item->{volid}} = $item;
5121         }
5122     }
5123
5124     return $volid_hash;
5125 }
5126
5127 sub is_volume_in_use {
5128     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5129
5130     my $path = PVE::Storage::path($storecfg, $volid);
5131
5132     my $scan_config = sub {
5133         my ($cref, $snapname) = @_;
5134
5135         foreach my $key (keys %$cref) {
5136             my $value = $cref->{$key};
5137             if (valid_drivename($key)) {
5138                 next if $skip_drive && $key eq $skip_drive;
5139                 my $drive = parse_drive($key, $value);
5140                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5141                 return 1 if $volid eq $drive->{file};
5142                 if ($drive->{file} =~ m!^/!) {
5143                     return 1 if $drive->{file} eq $path;
5144                 } else {
5145                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5146                     next if !$storeid;
5147                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5148                     next if !$scfg;
5149                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5150                 }
5151             }
5152         }
5153
5154         return 0;
5155     };
5156
5157     return 1 if &$scan_config($conf);
5158
5159     undef $skip_drive;
5160
5161     foreach my $snapname (keys %{$conf->{snapshots}}) {
5162         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5163     }
5164
5165     return 0;
5166 }
5167
5168 sub update_disksize {
5169     my ($vmid, $conf, $volid_hash) = @_;
5170
5171     my $changes;
5172
5173     my $used = {};
5174
5175     # Note: it is allowed to define multiple storages with same path (alias), so
5176     # we need to check both 'volid' and real 'path' (two different volid can point
5177     # to the same path).
5178
5179     my $usedpath = {};
5180
5181     # update size info
5182     foreach my $opt (keys %$conf) {
5183         if (valid_drivename($opt)) {
5184             my $drive = parse_drive($opt, $conf->{$opt});
5185             my $volid = $drive->{file};
5186             next if !$volid;
5187
5188             $used->{$volid} = 1;
5189             if ($volid_hash->{$volid} &&
5190                 (my $path = $volid_hash->{$volid}->{path})) {
5191                 $usedpath->{$path} = 1;
5192             }
5193
5194             next if drive_is_cdrom($drive);
5195             next if !$volid_hash->{$volid};
5196
5197             $drive->{size} = $volid_hash->{$volid}->{size};
5198             my $new = print_drive($vmid, $drive);
5199             if ($new ne $conf->{$opt}) {
5200                 $changes = 1;
5201                 $conf->{$opt} = $new;
5202             }
5203         }
5204     }
5205
5206     # remove 'unusedX' entry if volume is used
5207     foreach my $opt (keys %$conf) {
5208         next if $opt !~ m/^unused\d+$/;
5209         my $volid = $conf->{$opt};
5210         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5211         if ($used->{$volid} || ($path && $usedpath->{$path})) {
5212             $changes = 1;
5213             delete $conf->{$opt};
5214         }
5215     }
5216
5217     foreach my $volid (sort keys %$volid_hash) {
5218         next if $volid =~ m/vm-$vmid-state-/;
5219         next if $used->{$volid};
5220         my $path = $volid_hash->{$volid}->{path};
5221         next if !$path; # just to be sure
5222         next if $usedpath->{$path};
5223         $changes = 1;
5224         add_unused_volume($conf, $volid);
5225         $usedpath->{$path} = 1; # avoid to add more than once (aliases)
5226     }
5227
5228     return $changes;
5229 }
5230
5231 sub rescan {
5232     my ($vmid, $nolock) = @_;
5233
5234     my $cfg = PVE::Cluster::cfs_read_file("storage.cfg");
5235
5236     my $volid_hash = scan_volids($cfg, $vmid);
5237
5238     my $updatefn =  sub {
5239         my ($vmid) = @_;
5240
5241         my $conf = load_config($vmid);
5242
5243         check_lock($conf);
5244
5245         my $vm_volids = {};
5246         foreach my $volid (keys %$volid_hash) {
5247             my $info = $volid_hash->{$volid};
5248             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5249         }
5250
5251         my $changes = update_disksize($vmid, $conf, $vm_volids);
5252
5253         update_config_nolock($vmid, $conf, 1) if $changes;
5254     };
5255
5256     if (defined($vmid)) {
5257         if ($nolock) {
5258             &$updatefn($vmid);
5259         } else {
5260             lock_config($vmid, $updatefn, $vmid);
5261         }
5262     } else {
5263         my $vmlist = config_list();
5264         foreach my $vmid (keys %$vmlist) {
5265             if ($nolock) {
5266                 &$updatefn($vmid);
5267             } else {
5268                 lock_config($vmid, $updatefn, $vmid);
5269             }
5270         }
5271     }
5272 }
5273
5274 sub restore_vma_archive {
5275     my ($archive, $vmid, $user, $opts, $comp) = @_;
5276
5277     my $input = $archive eq '-' ? "<&STDIN" : undef;
5278     my $readfrom = $archive;
5279
5280     my $uncomp = '';
5281     if ($comp) {
5282         $readfrom = '-';
5283         my $qarchive = PVE::Tools::shellquote($archive);
5284         if ($comp eq 'gzip') {
5285             $uncomp = "zcat $qarchive|";
5286         } elsif ($comp eq 'lzop') {
5287             $uncomp = "lzop -d -c $qarchive|";
5288         } else {
5289             die "unknown compression method '$comp'\n";
5290         }
5291
5292     }
5293
5294     my $tmpdir = "/var/tmp/vzdumptmp$$";
5295     rmtree $tmpdir;
5296
5297     # disable interrupts (always do cleanups)
5298     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5299         warn "got interrupt - ignored\n";
5300     };
5301
5302     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5303     POSIX::mkfifo($mapfifo, 0600);
5304     my $fifofh;
5305
5306     my $openfifo = sub {
5307         open($fifofh, '>', $mapfifo) || die $!;
5308     };
5309
5310     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5311
5312     my $oldtimeout;
5313     my $timeout = 5;
5314
5315     my $devinfo = {};
5316
5317     my $rpcenv = PVE::RPCEnvironment::get();
5318
5319     my $conffile = config_file($vmid);
5320     my $tmpfn = "$conffile.$$.tmp";
5321
5322     # Note: $oldconf is undef if VM does not exists
5323     my $oldconf = PVE::Cluster::cfs_read_file(cfs_config_path($vmid));
5324
5325     my $print_devmap = sub {
5326         my $virtdev_hash = {};
5327
5328         my $cfgfn = "$tmpdir/qemu-server.conf";
5329
5330         # we can read the config - that is already extracted
5331         my $fh = IO::File->new($cfgfn, "r") ||
5332             "unable to read qemu-server.conf - $!\n";
5333
5334         my $fwcfgfn = "$tmpdir/qemu-server.fw";
5335         PVE::Tools::file_copy($fwcfgfn, "/etc/pve/firewall/$vmid.fw")
5336             if -f $fwcfgfn;
5337
5338         while (defined(my $line = <$fh>)) {
5339             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5340                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5341                 die "archive does not contain data for drive '$virtdev'\n"
5342                     if !$devinfo->{$devname};
5343                 if (defined($opts->{storage})) {
5344                     $storeid = $opts->{storage} || 'local';
5345                 } elsif (!$storeid) {
5346                     $storeid = 'local';
5347                 }
5348                 $format = 'raw' if !$format;
5349                 $devinfo->{$devname}->{devname} = $devname;
5350                 $devinfo->{$devname}->{virtdev} = $virtdev;
5351                 $devinfo->{$devname}->{format} = $format;
5352                 $devinfo->{$devname}->{storeid} = $storeid;
5353
5354                 # check permission on storage
5355                 my $pool = $opts->{pool}; # todo: do we need that?
5356                 if ($user ne 'root@pam') {
5357                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5358                 }
5359
5360                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5361             }
5362         }
5363
5364         foreach my $devname (keys %$devinfo) {
5365             die "found no device mapping information for device '$devname'\n"
5366                 if !$devinfo->{$devname}->{virtdev};
5367         }
5368
5369         my $cfg = cfs_read_file('storage.cfg');
5370
5371         # create empty/temp config
5372         if ($oldconf) {
5373             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5374             foreach_drive($oldconf, sub {
5375                 my ($ds, $drive) = @_;
5376
5377                 return if drive_is_cdrom($drive);
5378
5379                 my $volid = $drive->{file};
5380
5381                 return if !$volid || $volid =~ m|^/|;
5382
5383                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5384                 return if !$path || !$owner || ($owner != $vmid);
5385
5386                 # Note: only delete disk we want to restore
5387                 # other volumes will become unused
5388                 if ($virtdev_hash->{$ds}) {
5389                     PVE::Storage::vdisk_free($cfg, $volid);
5390                 }
5391             });
5392         }
5393
5394         my $map = {};
5395         foreach my $virtdev (sort keys %$virtdev_hash) {
5396             my $d = $virtdev_hash->{$virtdev};
5397             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5398             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5399
5400             # test if requested format is supported
5401             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5402             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5403             $d->{format} = $defFormat if !$supported;
5404
5405             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5406                                                   $d->{format}, undef, $alloc_size);
5407             print STDERR "new volume ID is '$volid'\n";
5408             $d->{volid} = $volid;
5409             my $path = PVE::Storage::path($cfg, $volid);
5410
5411             PVE::Storage::activate_volumes($cfg,[$volid]);
5412
5413             my $write_zeros = 1;
5414             # fixme: what other storages types initialize volumes with zero?
5415             if ($scfg->{type} eq 'dir' || $scfg->{type} eq 'nfs' || $scfg->{type} eq 'glusterfs' ||
5416                 $scfg->{type} eq 'sheepdog' || $scfg->{type} eq 'rbd') {
5417                 $write_zeros = 0;
5418             }
5419
5420             print $fifofh "${write_zeros}:$d->{devname}=$path\n";
5421
5422             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5423             $map->{$virtdev} = $volid;
5424         }
5425
5426         $fh->seek(0, 0) || die "seek failed - $!\n";
5427
5428         my $outfd = new IO::File ($tmpfn, "w") ||
5429             die "unable to write config for VM $vmid\n";
5430
5431         my $cookie = { netcount => 0 };
5432         while (defined(my $line = <$fh>)) {
5433             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5434         }
5435
5436         $fh->close();
5437         $outfd->close();
5438     };
5439
5440     eval {
5441         # enable interrupts
5442         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5443             die "interrupted by signal\n";
5444         };
5445         local $SIG{ALRM} = sub { die "got timeout\n"; };
5446
5447         $oldtimeout = alarm($timeout);
5448
5449         my $parser = sub {
5450             my $line = shift;
5451
5452             print "$line\n";
5453
5454             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5455                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5456                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5457             } elsif ($line =~ m/^CTIME: /) {
5458                 # we correctly received the vma config, so we can disable
5459                 # the timeout now for disk allocation (set to 10 minutes, so
5460                 # that we always timeout if something goes wrong)
5461                 alarm(600);
5462                 &$print_devmap();
5463                 print $fifofh "done\n";
5464                 my $tmp = $oldtimeout || 0;
5465                 $oldtimeout = undef;
5466                 alarm($tmp);
5467                 close($fifofh);
5468             }
5469         };
5470
5471         print "restore vma archive: $cmd\n";
5472         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5473     };
5474     my $err = $@;
5475
5476     alarm($oldtimeout) if $oldtimeout;
5477
5478     my $vollist = [];
5479     foreach my $devname (keys %$devinfo) {
5480         my $volid = $devinfo->{$devname}->{volid};
5481         push @$vollist, $volid if $volid;
5482     }
5483
5484     my $cfg = cfs_read_file('storage.cfg');
5485     PVE::Storage::deactivate_volumes($cfg, $vollist);
5486
5487     unlink $mapfifo;
5488
5489     if ($err) {
5490         rmtree $tmpdir;
5491         unlink $tmpfn;
5492
5493         foreach my $devname (keys %$devinfo) {
5494             my $volid = $devinfo->{$devname}->{volid};
5495             next if !$volid;
5496             eval {
5497                 if ($volid =~ m|^/|) {
5498                     unlink $volid || die 'unlink failed\n';
5499                 } else {
5500                     PVE::Storage::vdisk_free($cfg, $volid);
5501                 }
5502                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5503             };
5504             print STDERR "unable to cleanup '$volid' - $@" if $@;
5505         }
5506         die $err;
5507     }
5508
5509     rmtree $tmpdir;
5510
5511     rename($tmpfn, $conffile) ||
5512         die "unable to commit configuration file '$conffile'\n";
5513
5514     PVE::Cluster::cfs_update(); # make sure we read new file
5515
5516     eval { rescan($vmid, 1); };
5517     warn $@ if $@;
5518 }
5519
5520 sub restore_tar_archive {
5521     my ($archive, $vmid, $user, $opts) = @_;
5522
5523     if ($archive ne '-') {
5524         my $firstfile = tar_archive_read_firstfile($archive);
5525         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5526             if $firstfile ne 'qemu-server.conf';
5527     }
5528
5529     my $storecfg = cfs_read_file('storage.cfg');
5530
5531     # destroy existing data - keep empty config
5532     my $vmcfgfn = config_file($vmid);
5533     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5534
5535     my $tocmd = "/usr/lib/qemu-server/qmextract";
5536
5537     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5538     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5539     $tocmd .= ' --prealloc' if $opts->{prealloc};
5540     $tocmd .= ' --info' if $opts->{info};
5541
5542     # tar option "xf" does not autodetect compression when read from STDIN,
5543     # so we pipe to zcat
5544     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5545         PVE::Tools::shellquote("--to-command=$tocmd");
5546
5547     my $tmpdir = "/var/tmp/vzdumptmp$$";
5548     mkpath $tmpdir;
5549
5550     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5551     local $ENV{VZDUMP_VMID} = $vmid;
5552     local $ENV{VZDUMP_USER} = $user;
5553
5554     my $conffile = config_file($vmid);
5555     my $tmpfn = "$conffile.$$.tmp";
5556
5557     # disable interrupts (always do cleanups)
5558     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5559         print STDERR "got interrupt - ignored\n";
5560     };
5561
5562     eval {
5563         # enable interrupts
5564         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5565             die "interrupted by signal\n";
5566         };
5567
5568         if ($archive eq '-') {
5569             print "extracting archive from STDIN\n";
5570             run_command($cmd, input => "<&STDIN");
5571         } else {
5572             print "extracting archive '$archive'\n";
5573             run_command($cmd);
5574         }
5575
5576         return if $opts->{info};
5577
5578         # read new mapping
5579         my $map = {};
5580         my $statfile = "$tmpdir/qmrestore.stat";
5581         if (my $fd = IO::File->new($statfile, "r")) {
5582             while (defined (my $line = <$fd>)) {
5583                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5584                     $map->{$1} = $2 if $1;
5585                 } else {
5586                     print STDERR "unable to parse line in statfile - $line\n";
5587                 }
5588             }
5589             $fd->close();
5590         }
5591
5592         my $confsrc = "$tmpdir/qemu-server.conf";
5593
5594         my $srcfd = new IO::File($confsrc, "r") ||
5595             die "unable to open file '$confsrc'\n";
5596
5597         my $outfd = new IO::File ($tmpfn, "w") ||
5598             die "unable to write config for VM $vmid\n";
5599
5600         my $cookie = { netcount => 0 };
5601         while (defined (my $line = <$srcfd>)) {
5602             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5603         }
5604
5605         $srcfd->close();
5606         $outfd->close();
5607     };
5608     my $err = $@;
5609
5610     if ($err) {
5611
5612         unlink $tmpfn;
5613
5614         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
5615
5616         die $err;
5617     }
5618
5619     rmtree $tmpdir;
5620
5621     rename $tmpfn, $conffile ||
5622         die "unable to commit configuration file '$conffile'\n";
5623
5624     PVE::Cluster::cfs_update(); # make sure we read new file
5625
5626     eval { rescan($vmid, 1); };
5627     warn $@ if $@;
5628 };
5629
5630
5631 # Internal snapshots
5632
5633 # NOTE: Snapshot create/delete involves several non-atomic
5634 # action, and can take a long time.
5635 # So we try to avoid locking the file and use 'lock' variable
5636 # inside the config file instead.
5637
5638 my $snapshot_copy_config = sub {
5639     my ($source, $dest) = @_;
5640
5641     foreach my $k (keys %$source) {
5642         next if $k eq 'snapshots';
5643         next if $k eq 'snapstate';
5644         next if $k eq 'snaptime';
5645         next if $k eq 'vmstate';
5646         next if $k eq 'lock';
5647         next if $k eq 'digest';
5648         next if $k eq 'description';
5649         next if $k =~ m/^unused\d+$/;
5650
5651         $dest->{$k} = $source->{$k};
5652     }
5653 };
5654
5655 my $snapshot_apply_config = sub {
5656     my ($conf, $snap) = @_;
5657
5658     # copy snapshot list
5659     my $newconf = {
5660         snapshots => $conf->{snapshots},
5661     };
5662
5663     # keep description and list of unused disks
5664     foreach my $k (keys %$conf) {
5665         next if !($k =~ m/^unused\d+$/ || $k eq 'description');
5666         $newconf->{$k} = $conf->{$k};
5667     }
5668
5669     &$snapshot_copy_config($snap, $newconf);
5670
5671     return $newconf;
5672 };
5673
5674 sub foreach_writable_storage {
5675     my ($conf, $func) = @_;
5676
5677     my $sidhash = {};
5678
5679     foreach my $ds (keys %$conf) {
5680         next if !valid_drivename($ds);
5681
5682         my $drive = parse_drive($ds, $conf->{$ds});
5683         next if !$drive;
5684         next if drive_is_cdrom($drive);
5685
5686         my $volid = $drive->{file};
5687
5688         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5689         $sidhash->{$sid} = $sid if $sid;
5690     }
5691
5692     foreach my $sid (sort keys %$sidhash) {
5693         &$func($sid);
5694     }
5695 }
5696
5697 my $alloc_vmstate_volid = sub {
5698     my ($storecfg, $vmid, $conf, $snapname) = @_;
5699
5700     # Note: we try to be smart when selecting a $target storage
5701
5702     my $target;
5703
5704     # search shared storage first
5705     foreach_writable_storage($conf, sub {
5706         my ($sid) = @_;
5707         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
5708         return if !$scfg->{shared};
5709
5710         $target = $sid if !$target || $scfg->{path}; # prefer file based storage
5711     });
5712
5713     if (!$target) {
5714         # now search local storage
5715         foreach_writable_storage($conf, sub {
5716             my ($sid) = @_;
5717             my $scfg = PVE::Storage::storage_config($storecfg, $sid);
5718             return if $scfg->{shared};
5719
5720             $target = $sid if !$target || $scfg->{path}; # prefer file based storage;
5721         });
5722     }
5723
5724     $target = 'local' if !$target;
5725
5726     my $driver_state_size = 500; # assume 32MB is enough to safe all driver state;
5727     # we abort live save after $conf->{memory}, so we need at max twice that space
5728     my $size = $conf->{memory}*2 + $driver_state_size;
5729
5730     my $name = "vm-$vmid-state-$snapname";
5731     my $scfg = PVE::Storage::storage_config($storecfg, $target);
5732     $name .= ".raw" if $scfg->{path}; # add filename extension for file base storage
5733     my $volid = PVE::Storage::vdisk_alloc($storecfg, $target, $vmid, 'raw', $name, $size*1024);
5734
5735     return $volid;
5736 };
5737
5738 my $snapshot_prepare = sub {
5739     my ($vmid, $snapname, $save_vmstate, $comment) = @_;
5740
5741     my $snap;
5742
5743     my $updatefn =  sub {
5744
5745         my $conf = load_config($vmid);
5746
5747         die "you can't take a snapshot if it's a template\n"
5748             if is_template($conf);
5749
5750         check_lock($conf);
5751
5752         $conf->{lock} = 'snapshot';
5753
5754         die "snapshot name '$snapname' already used\n"
5755             if defined($conf->{snapshots}->{$snapname});
5756
5757         my $storecfg = PVE::Storage::config();
5758         die "snapshot feature is not available" if !has_feature('snapshot', $conf, $storecfg);
5759
5760         $snap = $conf->{snapshots}->{$snapname} = {};
5761
5762         if ($save_vmstate && check_running($vmid)) {
5763             $snap->{vmstate} = &$alloc_vmstate_volid($storecfg, $vmid, $conf, $snapname);
5764         }
5765
5766         &$snapshot_copy_config($conf, $snap);
5767
5768         $snap->{snapstate} = "prepare";
5769         $snap->{snaptime} = time();
5770         $snap->{description} = $comment if $comment;
5771
5772         # always overwrite machine if we save vmstate. This makes sure we
5773         # can restore it later using correct machine type
5774         $snap->{machine} = get_current_qemu_machine($vmid) if $snap->{vmstate};
5775
5776         update_config_nolock($vmid, $conf, 1);
5777     };
5778
5779     lock_config($vmid, $updatefn);
5780
5781     return $snap;
5782 };
5783
5784 my $snapshot_commit = sub {
5785     my ($vmid, $snapname) = @_;
5786
5787     my $updatefn = sub {
5788
5789         my $conf = load_config($vmid);
5790
5791         die "missing snapshot lock\n"
5792             if !($conf->{lock} && $conf->{lock} eq 'snapshot');
5793
5794         my $has_machine_config = defined($conf->{machine});
5795
5796         my $snap = $conf->{snapshots}->{$snapname};
5797
5798         die "snapshot '$snapname' does not exist\n" if !defined($snap);
5799
5800         die "wrong snapshot state\n"
5801             if !($snap->{snapstate} && $snap->{snapstate} eq "prepare");
5802
5803         delete $snap->{snapstate};
5804         delete $conf->{lock};
5805
5806         my $newconf = &$snapshot_apply_config($conf, $snap);
5807
5808         delete $newconf->{machine} if !$has_machine_config;
5809
5810         $newconf->{parent} = $snapname;
5811
5812         update_config_nolock($vmid, $newconf, 1);
5813     };
5814
5815     lock_config($vmid, $updatefn);
5816 };
5817
5818 sub snapshot_rollback {
5819     my ($vmid, $snapname) = @_;
5820
5821     my $prepare = 1;
5822
5823     my $storecfg = PVE::Storage::config();
5824
5825     my $conf = load_config($vmid);
5826
5827     my $get_snapshot_config = sub {
5828
5829         die "you can't rollback if vm is a template\n" if is_template($conf);
5830
5831         my $res = $conf->{snapshots}->{$snapname};
5832
5833         die "snapshot '$snapname' does not exist\n" if !defined($res);
5834
5835         return $res;
5836     };
5837
5838     my $snap = &$get_snapshot_config();
5839
5840     foreach_drive($snap, sub {
5841         my ($ds, $drive) = @_;
5842
5843         return if drive_is_cdrom($drive);
5844
5845         my $volid = $drive->{file};
5846
5847         PVE::Storage::volume_rollback_is_possible($storecfg, $volid, $snapname);
5848     });
5849
5850     my $updatefn = sub {
5851
5852         $conf = load_config($vmid);
5853
5854         $snap = &$get_snapshot_config();
5855
5856         die "unable to rollback to incomplete snapshot (snapstate = $snap->{snapstate})\n"
5857             if $snap->{snapstate};
5858
5859         if ($prepare) {
5860             check_lock($conf);
5861             vm_stop($storecfg, $vmid, undef, undef, 5, undef, undef);
5862         }
5863
5864         die "unable to rollback vm $vmid: vm is running\n"
5865             if check_running($vmid);
5866
5867         if ($prepare) {
5868             $conf->{lock} = 'rollback';
5869         } else {
5870             die "got wrong lock\n" if !($conf->{lock} && $conf->{lock} eq 'rollback');
5871             delete $conf->{lock};
5872         }
5873
5874         my $forcemachine;
5875
5876         if (!$prepare) {
5877             my $has_machine_config = defined($conf->{machine});
5878
5879             # copy snapshot config to current config
5880             $conf = &$snapshot_apply_config($conf, $snap);
5881             $conf->{parent} = $snapname;
5882
5883             # Note: old code did not store 'machine', so we try to be smart
5884             # and guess the snapshot was generated with kvm 1.4 (pc-i440fx-1.4).
5885             $forcemachine = $conf->{machine} || 'pc-i440fx-1.4';
5886             # we remove the 'machine' configuration if not explicitly specified
5887             # in the original config.
5888             delete $conf->{machine} if $snap->{vmstate} && !$has_machine_config;
5889         }
5890
5891         update_config_nolock($vmid, $conf, 1);
5892
5893         if (!$prepare && $snap->{vmstate}) {
5894             my $statefile = PVE::Storage::path($storecfg, $snap->{vmstate});
5895             vm_start($storecfg, $vmid, $statefile, undef, undef, undef, $forcemachine);
5896         }
5897     };
5898
5899     lock_config($vmid, $updatefn);
5900
5901     foreach_drive($snap, sub {
5902         my ($ds, $drive) = @_;
5903
5904         return if drive_is_cdrom($drive);
5905
5906         my $volid = $drive->{file};
5907         my $device = "drive-$ds";
5908
5909         PVE::Storage::volume_snapshot_rollback($storecfg, $volid, $snapname);
5910     });
5911
5912     $prepare = 0;
5913     lock_config($vmid, $updatefn);
5914 }
5915
5916 my $savevm_wait = sub {
5917     my ($vmid) = @_;
5918
5919     for(;;) {
5920         my $stat = vm_mon_cmd_nocheck($vmid, "query-savevm");
5921         if (!$stat->{status}) {
5922             die "savevm not active\n";
5923         } elsif ($stat->{status} eq 'active') {
5924             sleep(1);
5925             next;
5926         } elsif ($stat->{status} eq 'completed') {
5927             last;
5928         } else {
5929             die "query-savevm returned status '$stat->{status}'\n";
5930         }
5931     }
5932 };
5933
5934 sub do_snapshots_with_qemu {
5935     my ($storecfg, $volid) = @_;
5936
5937     my $storage_name = PVE::Storage::parse_volume_id($volid);
5938
5939     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
5940         && !$storecfg->{ids}->{$storage_name}->{krbd}){
5941         return 1;
5942     }
5943
5944     if ($volid =~ m/\.(qcow2|qed)$/){
5945         return 1;
5946     }
5947
5948     return undef;
5949 }
5950
5951 sub snapshot_create {
5952     my ($vmid, $snapname, $save_vmstate, $comment) = @_;
5953
5954     my $snap = &$snapshot_prepare($vmid, $snapname, $save_vmstate, $comment);
5955
5956     $save_vmstate = 0 if !$snap->{vmstate}; # vm is not running
5957
5958     my $config = load_config($vmid);
5959
5960     my $running = check_running($vmid);
5961
5962     my $freezefs = $running && $config->{agent};
5963     $freezefs = 0 if $snap->{vmstate}; # not needed if we save RAM
5964
5965     my $drivehash = {};
5966
5967     if ($freezefs) {
5968         eval { vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
5969         warn "guest-fsfreeze-freeze problems - $@" if $@;
5970     }
5971
5972     eval {
5973         # create internal snapshots of all drives
5974
5975         my $storecfg = PVE::Storage::config();
5976
5977         if ($running) {
5978             if ($snap->{vmstate}) {
5979                 my $path = PVE::Storage::path($storecfg, $snap->{vmstate});
5980                 vm_mon_cmd($vmid, "savevm-start", statefile => $path);
5981                 &$savevm_wait($vmid);
5982             } else {
5983                 vm_mon_cmd($vmid, "savevm-start");
5984             }
5985         };
5986
5987         foreach_drive($snap, sub {
5988             my ($ds, $drive) = @_;
5989
5990             return if drive_is_cdrom($drive);
5991
5992             my $volid = $drive->{file};
5993             my $device = "drive-$ds";
5994
5995             qemu_volume_snapshot($vmid, $device, $storecfg, $volid, $snapname);
5996             $drivehash->{$ds} = 1;
5997        });
5998     };
5999     my $err = $@;
6000
6001     if ($running) {
6002         eval { vm_mon_cmd($vmid, "savevm-end")  };
6003         warn $@ if $@;
6004
6005         if ($freezefs) {
6006             eval { vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6007             warn "guest-fsfreeze-thaw problems - $@" if $@;
6008         }
6009
6010         # savevm-end is async, we need to wait
6011         for (;;) {
6012             my $stat = vm_mon_cmd_nocheck($vmid, "query-savevm");
6013             if (!$stat->{bytes}) {
6014                 last;
6015             } else {
6016                 print "savevm not yet finished\n";
6017                 sleep(1);
6018                 next;
6019             }
6020         }
6021     }
6022
6023     if ($err) {
6024         warn "snapshot create failed: starting cleanup\n";
6025         eval { snapshot_delete($vmid, $snapname, 0, $drivehash); };
6026         warn $@ if $@;
6027         die $err;
6028     }
6029
6030     &$snapshot_commit($vmid, $snapname);
6031 }
6032
6033 # Note: $drivehash is only set when called from snapshot_create.
6034 sub snapshot_delete {
6035     my ($vmid, $snapname, $force, $drivehash) = @_;
6036
6037     my $prepare = 1;
6038
6039     my $snap;
6040     my $unused = [];
6041
6042     my $unlink_parent = sub {
6043         my ($confref, $new_parent) = @_;
6044
6045         if ($confref->{parent} && $confref->{parent} eq $snapname) {
6046             if ($new_parent) {
6047                 $confref->{parent} = $new_parent;
6048             } else {
6049                 delete $confref->{parent};
6050             }
6051         }
6052     };
6053
6054     my $updatefn =  sub {
6055         my ($remove_drive) = @_;
6056
6057         my $conf = load_config($vmid);
6058
6059         if (!$drivehash) {
6060             check_lock($conf);
6061             die "you can't delete a snapshot if vm is a template\n"
6062                 if is_template($conf);
6063         }
6064
6065         $snap = $conf->{snapshots}->{$snapname};
6066
6067         die "snapshot '$snapname' does not exist\n" if !defined($snap);
6068
6069         # remove parent refs
6070         if (!$prepare) {
6071             &$unlink_parent($conf, $snap->{parent});
6072             foreach my $sn (keys %{$conf->{snapshots}}) {
6073                 next if $sn eq $snapname;
6074                 &$unlink_parent($conf->{snapshots}->{$sn}, $snap->{parent});
6075             }
6076         }
6077
6078         if ($remove_drive) {
6079             if ($remove_drive eq 'vmstate') {
6080                 delete $snap->{$remove_drive};
6081             } else {
6082                 my $drive = parse_drive($remove_drive, $snap->{$remove_drive});
6083                 my $volid = $drive->{file};
6084                 delete $snap->{$remove_drive};
6085                 add_unused_volume($conf, $volid);
6086             }
6087         }
6088
6089         if ($prepare) {
6090             $snap->{snapstate} = 'delete';
6091         } else {
6092             delete $conf->{snapshots}->{$snapname};
6093             delete $conf->{lock} if $drivehash;
6094             foreach my $volid (@$unused) {
6095                 add_unused_volume($conf, $volid);
6096             }
6097         }
6098
6099         update_config_nolock($vmid, $conf, 1);
6100     };
6101
6102     lock_config($vmid, $updatefn);
6103
6104     # now remove vmstate file
6105
6106     my $storecfg = PVE::Storage::config();
6107
6108     if ($snap->{vmstate}) {
6109         eval {  PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
6110         if (my $err = $@) {
6111             die $err if !$force;
6112             warn $err;
6113         }
6114         # save changes (remove vmstate from snapshot)
6115         lock_config($vmid, $updatefn, 'vmstate') if !$force;
6116     };
6117
6118     # now remove all internal snapshots
6119     foreach_drive($snap, sub {
6120         my ($ds, $drive) = @_;
6121
6122         return if drive_is_cdrom($drive);
6123
6124         my $volid = $drive->{file};
6125         my $device = "drive-$ds";
6126
6127         if (!$drivehash || $drivehash->{$ds}) {
6128             eval { qemu_volume_snapshot_delete($vmid, $device, $storecfg, $volid, $snapname); };
6129             if (my $err = $@) {
6130                 die $err if !$force;
6131                 warn $err;
6132             }
6133         }
6134
6135         # save changes (remove drive fron snapshot)
6136         lock_config($vmid, $updatefn, $ds) if !$force;
6137         push @$unused, $volid;
6138     });
6139
6140     # now cleanup config
6141     $prepare = 0;
6142     lock_config($vmid, $updatefn);
6143 }
6144
6145 sub has_feature {
6146     my ($feature, $conf, $storecfg, $snapname, $running) = @_;
6147
6148     my $err;
6149     foreach_drive($conf, sub {
6150         my ($ds, $drive) = @_;
6151
6152         return if drive_is_cdrom($drive);
6153         my $volid = $drive->{file};
6154         $err = 1 if !PVE::Storage::volume_has_feature($storecfg, $feature, $volid, $snapname, $running);
6155     });
6156
6157     return $err ? 0 : 1;
6158 }
6159
6160 sub template_create {
6161     my ($vmid, $conf, $disk) = @_;
6162
6163     my $storecfg = PVE::Storage::config();
6164
6165     foreach_drive($conf, sub {
6166         my ($ds, $drive) = @_;
6167
6168         return if drive_is_cdrom($drive);
6169         return if $disk && $ds ne $disk;
6170
6171         my $volid = $drive->{file};
6172         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6173
6174         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6175         $drive->{file} = $voliddst;
6176         $conf->{$ds} = print_drive($vmid, $drive);
6177         update_config_nolock($vmid, $conf, 1);
6178     });
6179 }
6180
6181 sub is_template {
6182     my ($conf) = @_;
6183
6184     return 1 if defined $conf->{template} && $conf->{template} == 1;
6185 }
6186
6187 sub qemu_img_convert {
6188     my ($src_volid, $dst_volid, $size, $snapname) = @_;
6189
6190     my $storecfg = PVE::Storage::config();
6191     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6192     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6193
6194     if ($src_storeid && $dst_storeid) {
6195
6196         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6197
6198         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6199         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6200
6201         my $src_format = qemu_img_format($src_scfg, $src_volname);
6202         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6203
6204         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6205         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6206
6207         my $cmd = [];
6208         push @$cmd, '/usr/bin/qemu-img', 'convert', '-t', 'writeback', '-p', '-n';
6209         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
6210         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path, $dst_path;
6211
6212         my $parser = sub {
6213             my $line = shift;
6214             if($line =~ m/\((\S+)\/100\%\)/){
6215                 my $percent = $1;
6216                 my $transferred = int($size * $percent / 100);
6217                 my $remaining = $size - $transferred;
6218
6219                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6220             }
6221
6222         };
6223
6224         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6225         my $err = $@;
6226         die "copy failed: $err" if $err;
6227     }
6228 }
6229
6230 sub qemu_img_format {
6231     my ($scfg, $volname) = @_;
6232
6233     if ($scfg->{path} && $volname =~ m/\.(raw|cow|qcow|qcow2|qed|vmdk|cloop)$/) {
6234         return $1;
6235     } else {
6236         return "raw";
6237     }
6238 }
6239
6240 sub qemu_drive_mirror {
6241     my ($vmid, $drive, $dst_volid, $vmiddst) = @_;
6242
6243     my $storecfg = PVE::Storage::config();
6244     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6245
6246     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6247
6248     my $format = qemu_img_format($dst_scfg, $dst_volname);
6249
6250     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6251
6252     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $dst_path };
6253     $opts->{format} = $format if $format;
6254
6255     print "drive mirror is starting (scanning bitmap) : this step can take some minutes/hours, depend of disk size and storage speed\n";
6256
6257     eval {
6258     vm_mon_cmd($vmid, "drive-mirror", %$opts);
6259         while (1) {
6260             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6261             my $stat = @$stats[0];
6262             die "mirroring job seem to have die. Maybe do you have bad sectors?" if !$stat;
6263             die "error job is not mirroring" if $stat->{type} ne "mirror";
6264
6265             my $busy = $stat->{busy};
6266             my $ready = $stat->{ready};
6267
6268             if (my $total = $stat->{len}) {
6269                 my $transferred = $stat->{offset} || 0;
6270                 my $remaining = $total - $transferred;
6271                 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6272
6273                 print "transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6274             }
6275
6276
6277             if ($stat->{ready} eq 'true') {
6278
6279                 last if $vmiddst != $vmid;
6280
6281                 # try to switch the disk if source and destination are on the same guest
6282                 eval { vm_mon_cmd($vmid, "block-job-complete", device => "drive-$drive") };
6283                 last if !$@;
6284                 die $@ if $@ !~ m/cannot be completed/;
6285             }
6286             sleep 1;
6287         }
6288
6289
6290     };
6291     my $err = $@;
6292
6293     my $cancel_job = sub {
6294         vm_mon_cmd($vmid, "block-job-cancel", device => "drive-$drive");
6295         while (1) {
6296             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6297             my $stat = @$stats[0];
6298             last if !$stat;
6299             sleep 1;
6300         }
6301     };
6302
6303     if ($err) {
6304         eval { &$cancel_job(); };
6305         die "mirroring error: $err";
6306     }
6307
6308     if ($vmiddst != $vmid) {
6309         # if we clone a disk for a new target vm, we don't switch the disk
6310         &$cancel_job(); # so we call block-job-cancel
6311     }
6312 }
6313
6314 sub clone_disk {
6315     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6316         $newvmid, $storage, $format, $full, $newvollist) = @_;
6317
6318     my $newvolid;
6319
6320     if (!$full) {
6321         print "create linked clone of drive $drivename ($drive->{file})\n";
6322         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6323         push @$newvollist, $newvolid;
6324     } else {
6325         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6326         $storeid = $storage if $storage;
6327
6328         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6329         if (!$format) {
6330             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6331             $format = qemu_img_format($scfg, $volname);
6332         }
6333
6334         # test if requested format is supported - else use default
6335         my $supported = grep { $_ eq $format } @$validFormats;
6336         $format = $defFormat if !$supported;
6337
6338         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6339
6340         print "create full clone of drive $drivename ($drive->{file})\n";
6341         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $format, undef, ($size/1024));
6342         push @$newvollist, $newvolid;
6343
6344         PVE::Storage::activate_volumes($storecfg, $newvollist);
6345
6346         if (!$running || $snapname) {
6347             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname);
6348         } else {
6349             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid);
6350         }
6351     }
6352
6353     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6354
6355     my $disk = $drive;
6356     $disk->{format} = undef;
6357     $disk->{file} = $newvolid;
6358     $disk->{size} = $size;
6359
6360     return $disk;
6361 }
6362
6363 # this only works if VM is running
6364 sub get_current_qemu_machine {
6365     my ($vmid) = @_;
6366
6367     my $cmd = { execute => 'query-machines', arguments => {} };
6368     my $res = vm_qmp_command($vmid, $cmd);
6369
6370     my ($current, $default);
6371     foreach my $e (@$res) {
6372         $default = $e->{name} if $e->{'is-default'};
6373         $current = $e->{name} if $e->{'is-current'};
6374     }
6375
6376     # fallback to the default machine if current is not supported by qemu
6377     return $current || $default || 'pc';
6378 }
6379
6380 sub qemu_machine_feature_enabled {
6381     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6382
6383     my $current_major;
6384     my $current_minor;
6385
6386     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6387
6388         $current_major = $3;
6389         $current_minor = $4;
6390
6391     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6392
6393         $current_major = $1;
6394         $current_minor = $2;
6395     }
6396
6397     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
6398
6399
6400 }
6401
6402 sub qemu_machine_pxe {
6403     my ($vmid, $conf, $machine) = @_;
6404
6405     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6406
6407     foreach my $opt (keys %$conf) {
6408         next if $opt !~ m/^net(\d+)$/;
6409         my $net = PVE::QemuServer::parse_net($conf->{$opt});
6410         next if !$net;
6411         my $romfile = PVE::QemuServer::vm_mon_cmd_nocheck($vmid, 'qom-get', path => $opt, property => 'romfile');
6412         return $machine.".pxe" if $romfile =~ m/pxe/;
6413         last;
6414     }
6415
6416 }
6417
6418 sub qemu_use_old_bios_files {
6419     my ($machine_type) = @_;
6420
6421     return if !$machine_type;
6422
6423     my $use_old_bios_files = undef;
6424
6425     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6426         $machine_type = $1;
6427         $use_old_bios_files = 1;
6428     } else {
6429         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6430         # load new efi bios files on migration. So this hack is required to allow
6431         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6432         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6433         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, undef, 2, 4);
6434     }
6435
6436     return ($use_old_bios_files, $machine_type);
6437 }
6438
6439 sub lspci {
6440
6441     my $devices = {};
6442
6443     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6444             my (undef, $id, $function) = @_;
6445             my $res = { id => $id, function => $function};
6446             push @{$devices->{$id}}, $res;
6447     });
6448
6449     return $devices;
6450 }
6451
6452 sub vm_iothreads_list {
6453     my ($vmid) = @_;
6454
6455     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6456
6457     my $iothreads = {};
6458     foreach my $iothread (@$res) {
6459         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6460     }
6461
6462     return $iothreads;
6463 }
6464
6465 sub scsihw_infos {
6466     my ($conf, $drive) = @_;
6467
6468     my $maxdev = 0;
6469
6470     if ($conf->{scsihw} && ($conf->{scsihw} =~ m/^lsi/)) {
6471         $maxdev = 7;
6472     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6473         $maxdev = 1;
6474     } else {
6475         $maxdev = 256;
6476     }
6477
6478     my $controller = int($drive->{index} / $maxdev);
6479     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6480
6481     return ($maxdev, $controller, $controller_prefix);
6482 }
6483
6484 # bash completion helper
6485
6486 sub complete_backup_archives {
6487     my ($cmdname, $pname, $cvalue) = @_;
6488
6489     my $cfg = PVE::Storage::config();
6490
6491     my $storeid;
6492
6493     if ($cvalue =~ m/^([^:]+):/) {
6494         $storeid = $1;
6495     }
6496
6497     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6498
6499     my $res = [];
6500     foreach my $id (keys %$data) {
6501         foreach my $item (@{$data->{$id}}) {
6502             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6503             push @$res, $item->{volid} if defined($item->{volid});
6504         }
6505     }
6506
6507     return $res;
6508 }
6509
6510 my $complete_vmid_full = sub {
6511     my ($running) = @_;
6512
6513     my $idlist = vmstatus();
6514
6515     my $res = [];
6516
6517     foreach my $id (keys %$idlist) {
6518         my $d = $idlist->{$id};
6519         if (defined($running)) {
6520             next if $d->{template};
6521             next if $running && $d->{status} ne 'running';
6522             next if !$running && $d->{status} eq 'running';
6523         }
6524         push @$res, $id;
6525
6526     }
6527     return $res;
6528 };
6529
6530 sub complete_vmid {
6531     return &$complete_vmid_full();
6532 }
6533
6534 sub complete_vmid_stopped {
6535     return &$complete_vmid_full(0);
6536 }
6537
6538 sub complete_vmid_running {
6539     return &$complete_vmid_full(1);
6540 }
6541
6542 sub complete_storage {
6543
6544     my $cfg = PVE::Storage::config();
6545     my $ids = $cfg->{ids};
6546
6547     my $res = [];
6548     foreach my $sid (keys %$ids) {
6549         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6550         next if !$ids->{$sid}->{content}->{images};
6551         push @$res, $sid;
6552     }
6553
6554     return $res;
6555 }
6556
6557 1;