X-Git-Url: https://git.proxmox.com/?a=blobdiff_plain;f=PVE%2FQemuServer.pm;h=81f05843efdbbf681bb4be474d445bb5bc92269f;hb=1f720e28c739bcb0fa8ce8c61a066d7d49edbee6;hp=f4fca34300e0e237dd0bc7c124b0bef00841ff89;hpb=c6f773b818bee4ecbddc5981f578203c4ee9c03c;p=qemu-server.git

diff --git a/PVE/QemuServer.pm b/PVE/QemuServer.pm
index f4fca343..81f05843 100644
--- a/PVE/QemuServer.pm
+++ b/PVE/QemuServer.pm
@@ -373,7 +373,7 @@ EODESCR
     machine => {
 	description => "Specific the Qemu machine type.",
 	type => 'string',
-	pattern => '(pc|pc(-i440fx)?-\d+\.\d+|q35|pc-q35-\d+\.\d+)',
+	pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
 	maxLength => 40,
 	optional => 1,
     },
@@ -384,6 +384,12 @@ EODESCR
 	maxLength => 256,
 	optional => 1,
     },
+    protection => {
+	optional => 1,
+	type => 'boolean',
+	description => "Sets the protection flag of the VM. This will prevent the remove operation.",
+	default => 0,
+    },
 };
 
 # what about other qemu settings ?
@@ -475,7 +481,7 @@ my $drivename_hash;
 my $idedesc = {
     optional => 1,
     type => 'string', format => 'pve-qm-drive',
-    typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads] [,discard=ignore|on]',
+    typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads] [,discard=ignore|on] [,serial=serial][,model=model]',
     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
 };
 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
@@ -483,7 +489,7 @@ PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
 my $scsidesc = {
     optional => 1,
     type => 'string', format => 'pve-qm-drive',
-    typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads] [,discard=ignore|on] [,iothread=on] [,queues=<nbqueues>]',
+    typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads] [,discard=ignore|on] [,iothread=on] [,queues=<nbqueues>] [,serial=serial]',
     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
 };
 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
@@ -491,7 +497,7 @@ PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 my $satadesc = {
     optional => 1,
     type => 'string', format => 'pve-qm-drive',
-    typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads]  [,discard=ignore|on]',
+    typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads]  [,discard=ignore|on] [,serial=serial]',
     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
 };
 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
@@ -499,7 +505,7 @@ PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
 my $virtiodesc = {
     optional => 1,
     type => 'string', format => 'pve-qm-drive',
-    typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads]  [,discard=ignore|on] [,iothread=on]',
+    typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads]  [,discard=ignore|on] [,iothread=on] [,serial=serial]',
     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
 };
 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
@@ -528,7 +534,7 @@ PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
 my $hostpcidesc = {
         optional => 1,
         type => 'string', format => 'pve-qm-hostpci',
-        typetext => "[host=]HOSTPCIDEVICE [,driver=kvm|vfio] [,rombar=on|off] [,pcie=0|1] [,x-vga=on|off]",
+        typetext => "[host=]HOSTPCIDEVICE [,rombar=on|off] [,pcie=0|1] [,x-vga=on|off]",
         description => <<EODESCR,
 Map host pci devices. HOSTPCIDEVICE syntax is:
 
@@ -552,7 +558,7 @@ my $serialdesc = {
 	description =>  <<EODESCR,
 Create a serial device inside the VM (n is 0 to 3), and pass through a host serial device (i.e. /dev/ttyS0), or create a unix socket on the host side (use 'qm terminal' to open a terminal connection).
 
-Note: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
+Note: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
 
 Experimental: user reported problems with this option.
 EODESCR
@@ -645,7 +651,7 @@ sub kvm_user_version {
 
     my $tmp = `kvm -help 2>/dev/null`;
 
-    if ($tmp =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)[,\s]/) {
+    if ($tmp =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
 	$kvm_user_version = $2;
     }
 
@@ -869,6 +875,7 @@ my $format_size = sub {
 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
 #        [,aio=native|threads][,discard=ignore|on][,iothread=on]
+#        [,serial=serial][,model=model]
 
 sub parse_drive {
     my ($key, $data) = @_;
@@ -889,7 +896,7 @@ sub parse_drive {
     foreach my $p (split (/,/, $data)) {
 	next if $p =~ m/^\s*$/;
 
-	if ($p =~ m/^(file|volume|cyls|heads|secs|trans|media|snapshot|cache|format|rerror|werror|backup|aio|bps|mbps|mbps_max|bps_rd|mbps_rd|mbps_rd_max|bps_wr|mbps_wr|mbps_wr_max|iops|iops_max|iops_rd|iops_rd_max|iops_wr|iops_wr_max|size|discard|iothread|queues)=(.+)$/) {
+	if ($p =~ m/^(file|volume|cyls|heads|secs|trans|media|snapshot|cache|format|rerror|werror|backup|aio|bps|mbps|mbps_max|bps_rd|mbps_rd|mbps_rd_max|bps_wr|mbps_wr|mbps_wr_max|iops|iops_max|iops_rd|iops_rd_max|iops_wr|iops_wr_max|size|discard|iothread|queues|serial|model)=(.+)$/) {
 	    my ($k, $v) = ($1, $2);
 
 	    $k = 'file' if $k eq 'volume';
@@ -913,10 +920,6 @@ sub parse_drive {
 
     return undef if !$res->{file};
 
-    if($res->{file} =~ m/\.(raw|cow|qcow|qcow2|vmdk|cloop)$/){
-	$res->{format} = $1;
-    }
-
     return undef if $res->{cache} &&
 	$res->{cache} !~ m/^(off|none|writethrough|writeback|unsafe|directsync)$/;
     return undef if $res->{snapshot} && $res->{snapshot} !~ m/^(on|off)$/;
@@ -925,7 +928,7 @@ sub parse_drive {
     return undef if $res->{secs} && $res->{secs} !~ m/^\d+$/;
     return undef if $res->{media} && $res->{media} !~ m/^(disk|cdrom)$/;
     return undef if $res->{trans} && $res->{trans} !~ m/^(none|lba|auto)$/;
-    return undef if $res->{format} && $res->{format} !~ m/^(raw|cow|qcow|qcow2|vmdk|cloop)$/;
+    return undef if $res->{format} && $res->{format} !~ m/^(raw|cow|qcow|qed|qcow2|vmdk|cloop)$/;
     return undef if $res->{rerror} && $res->{rerror} !~ m/^(ignore|report|stop)$/;
     return undef if $res->{werror} && $res->{werror} !~ m/^(enospc|ignore|report|stop)$/;
     return undef if $res->{backup} && $res->{backup} !~ m/^(yes|no)$/;
@@ -955,7 +958,6 @@ sub parse_drive {
     return undef if $res->{iops_wr} && $res->{iops_wr} !~ m/^\d+$/;
     return undef if $res->{iops_wr_max} && $res->{iops_wr_max} !~ m/^\d+$/;
 
-
     if ($res->{size}) {
 	return undef if !defined($res->{size} = &$parse_size($res->{size}));
     }
@@ -974,7 +976,7 @@ sub parse_drive {
     return $res;
 }
 
-my @qemu_drive_options = qw(heads secs cyls trans media format cache snapshot rerror werror aio discard iops iops_rd iops_wr iops_max iops_rd_max iops_wr_max);
+my @qemu_drive_options = qw(heads secs cyls trans media format cache snapshot rerror werror aio discard iops iops_rd iops_wr iops_max iops_rd_max iops_wr_max serial);
 
 sub print_drive {
     my ($vmid, $drive) = @_;
@@ -988,6 +990,10 @@ sub print_drive {
 	$opts .= ",size=" . &$format_size($drive->{size});
     }
 
+    if (my $model = $drive->{model}) {
+	$opts .= ",model=$model";
+    }
+
     return "$drive->{file}$opts";
 }
 
@@ -1120,6 +1126,9 @@ sub print_drivedevice_full {
 	my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
 
 	$device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
+	if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
+	    $device .= ",model=$model";
+	}
     } elsif ($drive->{interface} eq 'sata'){
 	my $controller = int($drive->{index} / $MAX_SATA_DISKS);
 	my $unit = $drive->{index} % $MAX_SATA_DISKS;
@@ -1153,41 +1162,54 @@ sub get_initiator_name {
 sub print_drive_full {
     my ($storecfg, $vmid, $drive) = @_;
 
+    my $path;
+    my $volid = $drive->{file};
+    my $format;
+    
+    if (drive_is_cdrom($drive)) {
+	$path = get_iso_path($storecfg, $vmid, $volid);
+    } else {
+	my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
+	if ($storeid) {
+	    $path = PVE::Storage::path($storecfg, $volid);
+	    my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
+	    $format = qemu_img_format($scfg, $volname);
+	} else {
+	    $path = $volid;
+	}
+   }
+
     my $opts = '';
     foreach my $o (@qemu_drive_options) {
 	next if $o eq 'bootindex';
 	$opts .= ",$o=$drive->{$o}" if $drive->{$o};
     }
 
+    $opts .= ",format=$format" if $format && !$drive->{format};
+
     foreach my $o (qw(bps bps_rd bps_wr)) {
 	my $v = $drive->{"m$o"};
 	$opts .= ",$o=" . int($v*1024*1024) if $v;
     }
 
+    my $cache_direct = 0;
+
+    if (my $cache = $drive->{cache}) {
+	$cache_direct = $cache =~ /^(?:off|none|directsync)$/;
+    } elsif (!drive_is_cdrom($drive)) {
+	$opts .= ",cache=none";
+	$cache_direct = 1;
+    }
+
     # aio native works only with O_DIRECT
     if (!$drive->{aio}) {
-	if(!$drive->{cache} || $drive->{cache} eq 'none' || $drive->{cache} eq 'directsync') {
+	if($cache_direct) {
 	    $opts .= ",aio=native";
 	} else {
 	    $opts .= ",aio=threads";
 	}
     }
 
-
-    my $path;
-    my $volid = $drive->{file};
-    if (drive_is_cdrom($drive)) {
-	$path = get_iso_path($storecfg, $vmid, $volid);
-    } else {
-	if ($volid =~ m|^/|) {
-	    $path = $volid;
-	} else {
-	    $path = PVE::Storage::path($storecfg, $volid);
-	}
-    }
-
-    $opts .= ",cache=none" if !$drive->{cache} && !drive_is_cdrom($drive);
-
     my $detectzeroes = $drive->{discard} ? "unmap" : "on";
     $opts .= ",detect-zeroes=$detectzeroes" if !drive_is_cdrom($drive);
 
@@ -1197,7 +1219,7 @@ sub print_drive_full {
 }
 
 sub print_netdevice_full {
-    my ($vmid, $conf, $net, $netid, $bridges) = @_;
+    my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
 
     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
 
@@ -1214,11 +1236,28 @@ sub print_netdevice_full {
 	$tmpstr .= ",vectors=$vectors,mq=on";
     }
     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
+
+    if ($use_old_bios_files) {
+	my $romfile;
+	if ($device eq 'virtio-net-pci') {
+	    $romfile = 'pxe-virtio.rom';
+	} elsif ($device eq 'e1000') {
+	    $romfile = 'pxe-e1000.rom';
+	} elsif ($device eq 'ne2k') {
+	    $romfile = 'pxe-ne2k_pci.rom';
+	} elsif ($device eq 'pcnet') {
+	    $romfile = 'pxe-pcnet.rom';
+	} elsif ($device eq 'rtl8139') {
+	    $romfile = 'pxe-rtl8139.rom';
+	}
+	$tmpstr .= ",romfile=$romfile" if $romfile;
+    }
+
     return $tmpstr;
 }
 
 sub print_netdev_full {
-    my ($vmid, $conf, $net, $netid) = @_;
+    my ($vmid, $conf, $net, $netid, $hotplug) = @_;
 
     my $i = '';
     if ($netid =~ m/^net(\d+)$/) {
@@ -1239,9 +1278,10 @@ sub print_netdev_full {
     my $vmname = $conf->{name} || "vm$vmid";
 
     my $netdev = "";
+    my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
 
     if ($net->{bridge}) {
-        $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/pve-bridge,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
+        $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
     } else {
         $netdev = "type=user,id=$netid,hostname=$vmname";
     }
@@ -1304,8 +1344,6 @@ sub parse_hostpci {
 		my $pcidevices = lspci($2);
 	        $res->{pciid} = $pcidevices->{$2};
 	    }
-	} elsif ($kv =~ m/^driver=(kvm|vfio)$/) {
-	    $res->{driver} = $1;
 	} elsif ($kv =~ m/^rombar=(on|off)$/) {
 	    $res->{rombar} = $1;
 	} elsif ($kv =~ m/^x-vga=(on|off)$/) {
@@ -1418,29 +1456,35 @@ sub vm_is_volid_owner {
     return undef;
 }
 
+sub split_flagged_list {
+    my $text = shift || '';
+    $text =~ s/[,;]/ /g;
+    $text =~ s/^\s+//;
+    return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
+}
+
+sub join_flagged_list {
+    my ($how, $lst) = @_;
+    join $how, map { $lst->{$_} . $_ } keys %$lst;
+}
+
 sub vmconfig_delete_pending_option {
-    my ($conf, $key) = @_;
+    my ($conf, $key, $force) = @_;
 
     delete $conf->{pending}->{$key};
-    my $pending_delete_hash = { $key => 1 };
-    foreach my $opt (PVE::Tools::split_list($conf->{pending}->{delete})) {
-	$pending_delete_hash->{$opt} = 1;
-    }
-    $conf->{pending}->{delete} = join(',', keys %$pending_delete_hash);
+    my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
+    $pending_delete_hash->{$key} = $force ? '!' : '';
+    $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
 }
 
 sub vmconfig_undelete_pending_option {
     my ($conf, $key) = @_;
 
-    my $pending_delete_hash = {};
-    foreach my $opt (PVE::Tools::split_list($conf->{pending}->{delete})) {
-	$pending_delete_hash->{$opt} = 1;
-    }
+    my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
     delete $pending_delete_hash->{$key};
 
-    my @keylist = keys %$pending_delete_hash;
-    if (scalar(@keylist)) {
-	$conf->{pending}->{delete} = join(',', @keylist);
+    if (%$pending_delete_hash) {
+	$conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
     } else {
 	delete $conf->{pending}->{delete};
     }
@@ -1469,19 +1513,18 @@ sub vmconfig_cleanup_pending {
 	}
     }
 
-    # remove delete if option is not set
+    my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
     my $pending_delete_hash = {};
-    foreach my $opt (PVE::Tools::split_list($conf->{pending}->{delete})) {
+    while (my ($opt, $force) = each %$current_delete_hash) {
 	if (defined($conf->{$opt})) {
-	    $pending_delete_hash->{$opt} = 1;
+	    $pending_delete_hash->{$opt} = $force;
 	} else {
 	    $changes = 1;
 	}
     }
 
-    my @keylist = keys %$pending_delete_hash;
-    if (scalar(@keylist)) {
-	$conf->{pending}->{delete} = join(',', @keylist);
+    if (%$pending_delete_hash) {
+	$conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
     } else {
 	delete $conf->{pending}->{delete};
     }
@@ -1867,7 +1910,7 @@ sub parse_vm_config {
     my $vmid = $1;
 
     my $conf = $res;
-    my $descr = '';
+    my $descr;
     my $section = '';
 
     my @lines = split(/\n/, $raw);
@@ -1876,25 +1919,33 @@ sub parse_vm_config {
 
 	if ($line =~ m/^\[PENDING\]\s*$/i) {
 	    $section = 'pending';
-	    $conf->{description} = $descr if $descr;
-	    $descr = '';
+	    if (defined($descr)) {
+		$descr =~ s/\s+$//;
+		$conf->{description} = $descr;
+	    }
+	    $descr = undef;
 	    $conf = $res->{$section} = {};
 	    next;
 
 	} elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
 	    $section = $1;
-	    $conf->{description} = $descr if $descr;
-	    $descr = '';
+	    if (defined($descr)) {
+		$descr =~ s/\s+$//;
+		$conf->{description} = $descr;
+	    }
+	    $descr = undef;
 	    $conf = $res->{snapshots}->{$section} = {};
 	    next;
 	}
 
 	if ($line =~ m/^\#(.*)\s*$/) {
+	    $descr = '' if !defined($descr);
 	    $descr .= PVE::Tools::decode_text($1) . "\n";
 	    next;
 	}
 
 	if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
+	    $descr = '' if !defined($descr);
 	    $descr .= PVE::Tools::decode_text($2);
 	} elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
 	    $conf->{snapstate} = $1;
@@ -1937,8 +1988,10 @@ sub parse_vm_config {
 	}
     }
 
-    $conf->{description} = $descr if $descr;
-
+    if (defined($descr)) {
+	$descr =~ s/\s+$//;
+	$conf->{description} = $descr;
+    }
     delete $res->{snapstate}; # just to be sure
 
     return $res;
@@ -2009,14 +2062,19 @@ sub write_vm_config {
     }
 
     my $generate_raw_config = sub {
-	my ($conf) = @_;
+	my ($conf, $pending) = @_;
 
 	my $raw = '';
 
 	# add description as comment to top of file
-	my $descr = $conf->{description} || '';
-	foreach my $cl (split(/\n/, $descr)) {
-	    $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
+	if (defined(my $descr = $conf->{description})) {
+	    if ($descr) {
+		foreach my $cl (split(/\n/, $descr)) {
+		    $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
+		}
+	    } else {
+		$raw .= "#\n" if $pending;
+	    }
 	}
 
 	foreach my $key (sort keys %$conf) {
@@ -2030,7 +2088,7 @@ sub write_vm_config {
 
     if (scalar(keys %{$conf->{pending}})){
 	$raw .= "\n[PENDING]\n";
-	$raw .= &$generate_raw_config($conf->{pending});
+	$raw .= &$generate_raw_config($conf->{pending}, 1);
     }
 
     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
@@ -2100,6 +2158,8 @@ sub check_local_resources {
 
     foreach my $k (keys %$conf) {
 	next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
+	# sockets are safe: they will recreated be on the target side post-migrate
+	next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
 	$loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
     }
 
@@ -2338,6 +2398,12 @@ sub vmstatus {
 
 	$d->{netout} += $netdev->{$dev}->{receive};
 	$d->{netin} += $netdev->{$dev}->{transmit};
+
+	if ($full) {
+	    $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
+	    $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
+	}
+
     }
 
     my $ctime = gettimeofday;
@@ -2406,6 +2472,7 @@ sub vmstatus {
 	    $d->{freemem} = $info->{free_mem};
 	}
 
+	$d->{ballooninfo} = $info;
     };
 
     my $blockstatscb = sub {
@@ -2413,9 +2480,13 @@ sub vmstatus {
 	my $data = $resp->{'return'} || [];
 	my $totalrdbytes = 0;
 	my $totalwrbytes = 0;
+
 	for my $blockstat (@$data) {
 	    $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
 	    $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
+
+	    $blockstat->{device} =~ s/drive-//;
+	    $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
 	}
 	$res->{$vmid}->{diskread} = $totalrdbytes;
 	$res->{$vmid}->{diskwrite} = $totalwrbytes;
@@ -2475,6 +2546,27 @@ sub foreach_dimm {
     }
 }
 
+sub foreach_reverse_dimm {
+    my ($conf, $vmid, $memory, $sockets, $func) = @_;
+
+    my $dimm_id = 253;
+    my $current_size = 4177920;
+    my $dimm_size = 65536;
+    return if $current_size == $memory;
+
+    for (my $j = 0; $j < 8; $j++) {
+	for (my $i = 0; $i < 32; $i++) {
+ 	    my $name = "dimm${dimm_id}";
+ 	    $dimm_id--;
+ 	    my $numanode = $i % $sockets;
+ 	    $current_size -= $dimm_size;
+ 	    &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
+	    return  $current_size if $current_size <= $memory;
+	}
+	$dimm_size /= 2;
+    }
+}
+
 sub foreach_drive {
     my ($conf, $func) = @_;
 
@@ -2554,6 +2646,8 @@ sub config_to_command {
     my $q35 = machine_type_is_q35($conf);
     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
     my $machine_type = $forcemachine || $conf->{machine};
+    my $use_old_bios_files = undef;
+    ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
 
     my $cpuunits = defined($conf->{cpuunits}) ?
             $conf->{cpuunits} : $defaults->{cpuunits};
@@ -2562,6 +2656,9 @@ sub config_to_command {
     push @$cmd, '--scope';
     push @$cmd, '--slice', "qemu";
     push @$cmd, '--unit', $vmid;
+    # set KillMode=none, so that systemd don't kill those scopes
+    # at shutdown (pve-manager service should stop the VMs instead)
+    push @$cmd, '-p', "KillMode=none";
     push @$cmd, '-p', "CPUShares=$cpuunits";
     if ($conf->{cpulimit}) {
 	my $cpulimit = int($conf->{cpulimit} * 100);
@@ -2647,13 +2744,11 @@ sub config_to_command {
 	}
 
 	my $rombar = $d->{rombar} && $d->{rombar} eq 'off' ? ",rombar=0" : "";
-	my $driver = $d->{driver} && $d->{driver} eq 'vfio' ? "vfio-pci" : "pci-assign";
 	my $xvga = $d->{'x-vga'} && $d->{'x-vga'} eq 'on' ? ",x-vga=on" : "";
 	if ($xvga && $xvga ne '') {
 	    push @$cpuFlags, 'kvm=off';
 	    $vga = 'none';
 	}
-	$driver = "vfio-pci" if $xvga ne '';
 	my $pcidevices = $d->{pciid};
 	my $multifunction = 1 if @$pcidevices > 1;
 
@@ -2664,7 +2759,7 @@ sub config_to_command {
 	    $id .= ".$j" if $multifunction;
 	    my $addr = $pciaddr;
 	    $addr .= ".$j" if $multifunction;
-	    my $devicestr = "$driver,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
+	    my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
 
 	    if($j == 0){
 		$devicestr .= "$rombar$xvga";
@@ -2732,7 +2827,7 @@ sub config_to_command {
 
     my $allowed_vcpus = $cpuinfo->{cpus};
 
-    die "MAX $maxcpus vcpus allowed per VM on this node\n"
+    die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
 	if ($allowed_vcpus < $maxcpus);
 
     push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
@@ -2816,21 +2911,24 @@ sub config_to_command {
 
     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
 
-    push @$cpuFlags , '+x2apic' if !$nokvm && $conf->{ostype} ne 'solaris';
-
-    push @$cpuFlags , '-x2apic' if $conf->{ostype} eq 'solaris';
+    push @$cpuFlags , '-x2apic'
+	if $conf->{ostype} && $conf->{ostype} eq 'solaris';
 
     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
 
+    push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
+
     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
 
 	push @$cpuFlags , '+kvm_pv_unhalt' if !$nokvm;
 	push @$cpuFlags , '+kvm_pv_eoi' if !$nokvm;
     }
 
+    push @$cpuFlags, 'enforce' if $cpu ne 'host' && !$nokvm;
+
     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
 
-    push @$cmd, '-cpu', "$cpu,enforce";
+    push @$cmd, '-cpu', $cpu;
 
     my $memory = $conf->{memory} || $defaults->{memory};
     my $static_memory = 0;
@@ -3079,7 +3177,7 @@ sub config_to_command {
          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
          push @$devices, '-netdev', $netdevfull;
 
-         my $netdevicefull = print_netdevice_full($vmid,$conf,$d,"net$i",$bridges);
+         my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
          push @$devices, '-device', $netdevicefull;
     }
 
@@ -3244,7 +3342,12 @@ sub vm_deviceplug {
     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
 
         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
-        my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid);
+
+        my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf); 
+        my $use_old_bios_files = undef;
+        ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
+
+        my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
         qemu_deviceadd($vmid, $netdevicefull);
         eval { qemu_deviceaddverify($vmid, $deviceid); };
 	if (my $err = $@) {
@@ -3501,7 +3604,7 @@ sub qemu_set_link_status {
 sub qemu_netdevadd {
     my ($vmid, $conf, $device, $deviceid) = @_;
 
-    my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid);
+    my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
     my %options =  split(/[=,]/, $netdev);
 
     vm_mon_cmd($vmid, "netdev_add",  %options);
@@ -3554,33 +3657,77 @@ sub qemu_memory_hotplug {
     my $dimm_memory = $memory - $static_memory;
 
     die "memory can't be lower than $static_memory MB" if $value < $static_memory;
-    die "memory unplug is not yet available" if $value < $memory;
     die "you cannot add more memory than $MAX_MEM MB!\n" if $memory > $MAX_MEM;
 
 
     my $sockets = 1;
     $sockets = $conf->{sockets} if $conf->{sockets};
 
-    foreach_dimm($conf, $vmid, $value, $sockets, sub {
-	my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
+    if($value > $memory) {
 
-	    return if $current_size <= $conf->{memory};
+    	foreach_dimm($conf, $vmid, $value, $sockets, sub {
+	    my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
 
-	    eval { vm_mon_cmd($vmid, "object-add", 'qom-type' => "memory-backend-ram", id => "mem-$name", props => { size => int($dimm_size*1024*1024) } ) };
-	    if (my $err = $@) {
-	        eval { qemu_objectdel($vmid, "mem-$name"); };
-	        die $err;
-	    }
+		return if $current_size <= $conf->{memory};
 
-	    eval { vm_mon_cmd($vmid, "device_add", driver => "pc-dimm", id => "$name", memdev => "mem-$name", node => $numanode) };
-	    if (my $err = $@) {
-	        eval { qemu_objectdel($vmid, "mem-$name"); };
-	        die $err;
-	    }
-	    #update conf after each succesful module hotplug
-	    $conf->{memory} = $current_size;
-	    update_config_nolock($vmid, $conf, 1);
-    });
+		eval { vm_mon_cmd($vmid, "object-add", 'qom-type' => "memory-backend-ram", id => "mem-$name", props => { size => int($dimm_size*1024*1024) } ) };
+		if (my $err = $@) {
+		    eval { qemu_objectdel($vmid, "mem-$name"); };
+		    die $err;
+		}
+
+		eval { vm_mon_cmd($vmid, "device_add", driver => "pc-dimm", id => "$name", memdev => "mem-$name", node => $numanode) };
+		if (my $err = $@) {
+		    eval { qemu_objectdel($vmid, "mem-$name"); };
+		    die $err;
+		}
+		#update conf after each succesful module hotplug
+		$conf->{memory} = $current_size;
+		update_config_nolock($vmid, $conf, 1);
+	});
+
+    } else {
+
+    	foreach_reverse_dimm($conf, $vmid, $value, $sockets, sub {
+	    my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
+
+		return if $current_size >= $conf->{memory};
+		print "try to unplug memory dimm $name\n";
+
+		my $retry = 0;
+	        while (1) {
+		    eval { qemu_devicedel($vmid, $name) };
+		    sleep 3;
+		    my $dimm_list = qemu_dimm_list($vmid);
+		    last if !$dimm_list->{$name};
+		    raise_param_exc({ $name => "error unplug memory module" }) if $retry > 5;
+		    $retry++;
+		}
+
+		#update conf after each succesful module unplug
+		$conf->{memory} = $current_size;
+
+		eval { qemu_objectdel($vmid, "mem-$name"); };
+		update_config_nolock($vmid, $conf, 1);
+	});
+    }
+}
+
+sub qemu_dimm_list {
+    my ($vmid) = @_;
+
+    my $dimmarray = vm_mon_cmd_nocheck($vmid, "query-memory-devices");
+    my $dimms = {};
+
+    foreach my $dimm (@$dimmarray) {
+
+        $dimms->{$dimm->{data}->{id}}->{id} = $dimm->{data}->{id};
+        $dimms->{$dimm->{data}->{id}}->{node} = $dimm->{data}->{node};
+        $dimms->{$dimm->{data}->{id}}->{addr} = $dimm->{data}->{addr};
+        $dimms->{$dimm->{data}->{id}}->{size} = $dimm->{data}->{size};
+        $dimms->{$dimm->{data}->{id}}->{slot} = $dimm->{data}->{slot};
+    }
+    return $dimms;
 }
 
 sub qemu_block_set_io_throttle {
@@ -3744,9 +3891,10 @@ sub set_migration_caps {
 
     my $enabled_cap = {
 	"auto-converge" => 1,
-	"xbzrle" => 0,
+	"xbzrle" => 1,
 	"x-rdma-pin-all" => 0,
 	"zero-blocks" => 0,
+	"compress" => 0
     };
 
     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
@@ -3767,6 +3915,7 @@ my $fast_plug_option = {
     'onboot' => 1,
     'shares' => 1,
     'startup' => 1,
+    'description' => 1,
 };
 
 # hotplug changes in [PENDING]
@@ -3803,8 +3952,8 @@ sub vmconfig_hotplug_pending {
 
     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
 
-    my @delete = PVE::Tools::split_list($conf->{pending}->{delete});
-    foreach my $opt (@delete) {
+    my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
+    while (my ($opt, $force) = each %$pending_delete_hash) {
 	next if $selection && !$selection->{$opt};
 	eval {
 	    if ($opt eq 'hotplug') {
@@ -3830,7 +3979,7 @@ sub vmconfig_hotplug_pending {
 	    } elsif (valid_drivename($opt)) {
 		die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
 		vm_deviceunplug($vmid, $conf, $opt);
-		vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}));
+		vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
 	    } elsif ($opt =~ m/^memory$/) {
 		die "skip\n" if !$hotplug_features->{memory};
 		qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
@@ -3912,20 +4061,61 @@ sub vmconfig_hotplug_pending {
     }
 }
 
+sub try_deallocate_drive {
+    my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
+
+    if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
+	my $volid = $drive->{file};
+	if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
+	    my $sid = PVE::Storage::parse_volume_id($volid);
+	    $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
+
+	    # check if the disk is really unused
+	    my $used_paths = PVE::QemuServer::get_used_paths($vmid, $storecfg, $conf, 1, $key);
+	    my $path = PVE::Storage::path($storecfg, $volid);
+	    die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
+		   if $used_paths->{$path};
+	    PVE::Storage::vdisk_free($storecfg, $volid);
+	    return 1;
+	} else {
+	    # If vm is not owner of this disk remove from config
+	    return 1;
+	}
+    }
+
+    return undef;
+}
+
+sub vmconfig_delete_or_detach_drive {
+    my ($vmid, $storecfg, $conf, $opt, $force) = @_;
+
+    my $drive = parse_drive($opt, $conf->{$opt});
+
+    my $rpcenv = PVE::RPCEnvironment::get();
+    my $authuser = $rpcenv->get_user();
+
+    if ($force) {
+	$rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
+	try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
+    } else {
+	vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
+    }
+}
+
 sub vmconfig_apply_pending {
     my ($vmid, $conf, $storecfg) = @_;
 
     # cold plug
 
-    my @delete = PVE::Tools::split_list($conf->{pending}->{delete});
-    foreach my $opt (@delete) { # delete
+    my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
+    while (my ($opt, $force) = each %$pending_delete_hash) {
 	die "internal error" if $opt =~ m/^unused/;
 	$conf = load_config($vmid); # update/reload
 	if (!defined($conf->{$opt})) {
 	    vmconfig_undelete_pending_option($conf, $opt);
 	    update_config_nolock($vmid, $conf, 1);
 	} elsif (valid_drivename($opt)) {
-	    vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}));
+	    vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
 	    vmconfig_undelete_pending_option($conf, $opt);
 	    delete $conf->{$opt};
 	    update_config_nolock($vmid, $conf, 1);
@@ -4113,7 +4303,8 @@ sub vmconfig_update_disk {
 }
 
 sub vm_start {
-    my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused, $forcemachine, $spice_ticket) = @_;
+    my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
+	$forcemachine, $spice_ticket) = @_;
 
     lock_config($vmid, sub {
 	my $conf = load_config($vmid, $migratedfrom);
@@ -4145,10 +4336,11 @@ sub vm_start {
 		my $nodename = PVE::INotify::nodename();
 		if ($datacenterconf->{migration_unsecure}) {
 			$localip = PVE::Cluster::remote_node_ip($nodename, 1);
+			$localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
 		}
 		my $pfamily = PVE::Tools::get_host_address_family($nodename);
 		$migrate_port = PVE::Tools::next_migrate_port($pfamily);
-		$migrate_uri = "tcp:[${localip}]:${migrate_port}";
+		$migrate_uri = "tcp:${localip}:${migrate_port}";
 		push @$cmd, '-incoming', $migrate_uri;
 		push @$cmd, '-S';
 	    } else {
@@ -4169,13 +4361,7 @@ sub vm_start {
 		my $info = pci_device_info("0000:$pciid");
 		die "IOMMU not present\n" if !check_iommu_support();
 		die "no pci device info for device '$pciid'\n" if !$info;
-
-		if ($d->{driver} && $d->{driver} eq "vfio") {
-		    die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
-		} else {
-		    die "can't unbind/bind to stub pci device '$pciid'\n" if !pci_dev_bind_to_stub($info);
-		}
-
+		die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
 		die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
 	  }
         }
@@ -4380,7 +4566,7 @@ sub vm_stop {
 	    $conf = load_config($vmid);
 	    check_lock($conf) if !$skiplock;
 	    if (!defined($timeout) && $shutdown && $conf->{startup}) {
-		my $opts = parse_startup($conf->{startup});
+		my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
 		$timeout = $opts->{down} if $opts->{down};
 	    }
 	}
@@ -4460,15 +4646,21 @@ sub vm_suspend {
 }
 
 sub vm_resume {
-    my ($vmid, $skiplock) = @_;
+    my ($vmid, $skiplock, $nocheck) = @_;
 
     lock_config($vmid, sub {
 
-	my $conf = load_config($vmid);
+	if (!$nocheck) {
 
-	check_lock($conf) if !($skiplock || ($conf->{lock} && $conf->{lock} eq 'backup'));
+	    my $conf = load_config($vmid);
+
+	    check_lock($conf) if !($skiplock || ($conf->{lock} && $conf->{lock} eq 'backup'));
 
-	vm_mon_cmd($vmid, "cont");
+	    vm_mon_cmd($vmid, "cont");
+
+	} else {
+	    vm_mon_cmd_nocheck($vmid, "cont");
+	}
     });
 }
 
@@ -4558,30 +4750,6 @@ sub pci_dev_reset {
     return file_write($fn, "1");
 }
 
-sub pci_dev_bind_to_stub {
-    my ($dev) = @_;
-
-    my $name = $dev->{name};
-
-    my $testdir = "$pcisysfs/drivers/pci-stub/$name";
-    return 1 if -d $testdir;
-
-    my $data = "$dev->{vendor} $dev->{product}";
-    return undef if !file_write("$pcisysfs/drivers/pci-stub/new_id", $data);
-
-    my $fn = "$pcisysfs/devices/$name/driver/unbind";
-    if (!file_write($fn, $name)) {
-	return undef if -f $fn;
-    }
-
-    $fn = "$pcisysfs/drivers/pci-stub/bind";
-    if (! -d $testdir) {
-	return undef if !file_write($fn, $name);
-    }
-
-    return -d $testdir;
-}
-
 sub pci_dev_bind_to_vfio {
     my ($dev) = @_;
 
@@ -5207,6 +5375,8 @@ sub restore_vma_archive {
 	    $d->{volid} = $volid;
 	    my $path = PVE::Storage::path($cfg, $volid);
 
+	    PVE::Storage::activate_volumes($cfg,[$volid]);
+
 	    my $write_zeros = 1;
 	    # fixme: what other storages types initialize volumes with zero?
 	    if ($scfg->{type} eq 'dir' || $scfg->{type} eq 'nfs' || $scfg->{type} eq 'glusterfs' ||
@@ -5272,13 +5442,21 @@ sub restore_vma_archive {
 
     alarm($oldtimeout) if $oldtimeout;
 
+    my $vollist = [];
+    foreach my $devname (keys %$devinfo) {
+	my $volid = $devinfo->{$devname}->{volid};
+	push @$vollist, $volid if $volid;
+    }
+
+    my $cfg = cfs_read_file('storage.cfg');
+    PVE::Storage::deactivate_volumes($cfg, $vollist);
+
     unlink $mapfifo;
 
     if ($err) {
 	rmtree $tmpdir;
 	unlink $tmpfn;
 
-	my $cfg = cfs_read_file('storage.cfg');
 	foreach my $devname (keys %$devinfo) {
 	    my $volid = $devinfo->{$devname}->{volid};
 	    next if !$volid;
@@ -5725,7 +5903,8 @@ sub do_snapshots_with_qemu {
 
     my $storage_name = PVE::Storage::parse_volume_id($volid);
 
-    if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}} ){
+    if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}} 
+	&& !$storecfg->{ids}->{$storage_name}->{krbd}){
 	return 1;
     }
 
@@ -5980,6 +6159,9 @@ sub qemu_img_convert {
     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
 
     if ($src_storeid && $dst_storeid) {
+
+	PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
+
 	my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
 	my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
 
@@ -6015,10 +6197,8 @@ sub qemu_img_convert {
 sub qemu_img_format {
     my ($scfg, $volname) = @_;
 
-    if ($scfg->{path} && $volname =~ m/\.(raw|qcow2|qed|vmdk)$/) {
+    if ($scfg->{path} && $volname =~ m/\.(raw|cow|qcow|qcow2|qed|vmdk|cloop)$/) {
 	return $1;
-    } elsif ($scfg->{type} eq 'iscsi') {
-	return "host_device";
     } else {
 	return "raw";
     }
@@ -6032,24 +6212,17 @@ sub qemu_drive_mirror {
 
     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
 
-    my $format;
-    if ($dst_volname =~ m/\.(raw|qcow2)$/){
-	$format = $1;
-    }
+    my $format = qemu_img_format($dst_scfg, $dst_volname);
 
     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
 
-    #drive-mirror is doing lseek on source image before starting, and this can take a lot of time for big nfs volume
-    #during this time, qmp socket is hanging
-    #http://lists.nongnu.org/archive/html/qemu-devel/2015-05/msg01838.html
-    #so we need to setup a big timeout
-    my $opts = { timeout => 14400, device => "drive-$drive", mode => "existing", sync => "full", target => $dst_path };
+    my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $dst_path };
     $opts->{format} = $format if $format;
 
-    print "drive mirror is starting : this step can take some minutes/hours, depend of disk size and storage speed\n";
+    print "drive mirror is starting (scanning bitmap) : this step can take some minutes/hours, depend of disk size and storage speed\n";
 
-    vm_mon_cmd($vmid, "drive-mirror", %$opts);
     eval {
+    vm_mon_cmd($vmid, "drive-mirror", %$opts);
 	while (1) {
 	    my $stats = vm_mon_cmd($vmid, "query-block-jobs");
 	    my $stat = @$stats[0];
@@ -6121,7 +6294,8 @@ sub clone_disk {
 
 	my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
 	if (!$format) {
-	    $format = $drive->{format} || $defFormat;
+	    my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
+	    $format = qemu_img_format($scfg, $volname);
 	}
 
 	# test if requested format is supported - else use default
@@ -6134,6 +6308,8 @@ sub clone_disk {
 	$newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $format, undef, ($size/1024));
 	push @$newvollist, $newvolid;
 
+	PVE::Storage::activate_volumes($storecfg, $newvollist);
+
 	if (!$running || $snapname) {
 	    qemu_img_convert($drive->{file}, $newvolid, $size, $snapname);
 	} else {
@@ -6190,6 +6366,43 @@ sub qemu_machine_feature_enabled {
 
 }
 
+sub qemu_machine_pxe {
+    my ($vmid, $conf, $machine) = @_;
+
+    $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
+
+    foreach my $opt (keys %$conf) {
+	next if $opt !~ m/^net(\d+)$/;
+	my $net = PVE::QemuServer::parse_net($conf->{$opt});
+	next if !$net;
+	my $romfile = PVE::QemuServer::vm_mon_cmd_nocheck($vmid, 'qom-get', path => $opt, property => 'romfile');
+	return $machine.".pxe" if $romfile =~ m/pxe/;
+	last;
+    }
+
+}
+
+sub qemu_use_old_bios_files {
+    my ($machine_type) = @_;
+
+    return if !$machine_type;
+
+    my $use_old_bios_files = undef;
+
+    if ($machine_type =~ m/^(\S+)\.pxe$/) {
+        $machine_type = $1;
+        $use_old_bios_files = 1;
+    } else {
+        # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
+        # load new efi bios files on migration. So this hack is required to allow
+        # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
+        # updrading from proxmox-ve-3.X to proxmox-ve 4.0
+        $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, undef, 2, 4);
+    }
+
+    return ($use_old_bios_files, $machine_type);
+}
+
 sub lspci {
 
     my $devices = {};
@@ -6235,4 +6448,77 @@ sub scsihw_infos {
     return ($maxdev, $controller, $controller_prefix);
 }
 
+# bash completion helper
+
+sub complete_backup_archives {
+    my ($cmdname, $pname, $cvalue) = @_;
+
+    my $cfg = PVE::Storage::config();
+
+    my $storeid;
+
+    if ($cvalue =~ m/^([^:]+):/) {
+	$storeid = $1;
+    }
+
+    my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
+
+    my $res = [];
+    foreach my $id (keys %$data) {
+	foreach my $item (@{$data->{$id}}) {
+	    next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
+	    push @$res, $item->{volid} if defined($item->{volid});
+	}
+    }
+
+    return $res;
+}
+
+my $complete_vmid_full = sub {
+    my ($running) = @_;
+
+    my $idlist = vmstatus();
+
+    my $res = [];
+
+    foreach my $id (keys %$idlist) {
+	my $d = $idlist->{$id};
+	if (defined($running)) {
+	    next if $d->{template};
+	    next if $running && $d->{status} ne 'running';
+	    next if !$running && $d->{status} eq 'running';
+	}
+	push @$res, $id;
+
+    }
+    return $res;
+};
+
+sub complete_vmid {
+    return &$complete_vmid_full();
+}
+
+sub complete_vmid_stopped {
+    return &$complete_vmid_full(0);
+}
+
+sub complete_vmid_running {
+    return &$complete_vmid_full(1);
+}
+
+sub complete_storage {
+
+    my $cfg = PVE::Storage::config();
+    my $ids = $cfg->{ids};
+
+    my $res = [];
+    foreach my $sid (keys %$ids) {
+	next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
+	next if !$ids->{$sid}->{content}->{images};
+	push @$res, $sid;
+    }
+
+    return $res;
+}
+
 1;