X-Git-Url: https://git.proxmox.com/?a=blobdiff_plain;f=PVE%2FQemuServer.pm;h=8f95bf5341bee7ac1ceb1f806690c48ba66ba10c;hb=6aa4651b8928255f2982a810eb16eb7e2c7435df;hp=0501343f9804d0a68ffe61e2bb144dfc523d6624;hpb=ce332eeb583e73a2f284d0a0a5e4d6330c250a53;p=qemu-server.git

diff --git a/PVE/QemuServer.pm b/PVE/QemuServer.pm
index 0501343f..8f95bf53 100644
--- a/PVE/QemuServer.pm
+++ b/PVE/QemuServer.pm
@@ -11,19 +11,22 @@ use File::Basename;
 use File::Path;
 use File::stat;
 use Getopt::Long;
-use Digest::SHA1;
+use Digest::SHA;
 use Fcntl ':flock';
 use Cwd 'abs_path';
 use IPC::Open3;
+use JSON;
 use Fcntl;
 use PVE::SafeSyslog;
 use Storable qw(dclone);
 use PVE::Exception qw(raise raise_param_exc);
 use PVE::Storage;
 use PVE::Tools qw(run_command lock_file file_read_firstline);
+use PVE::JSONSchema qw(get_standard_option);
 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
 use PVE::INotify;
 use PVE::ProcFSTools;
+use PVE::QMPClient;
 use Time::HiRes qw(gettimeofday);
 
 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
@@ -31,11 +34,11 @@ my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
 # Note about locking: we use flock on the config file protect
 # against concurent actions.
 # Aditionaly, we have a 'lock' setting in the config file. This
-# can be set to 'migrate' or 'backup'. Most actions are not
+# can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
 # allowed when such lock is set. But you can ignore this kind of
 # lock with the --skiplock flag.
 
-cfs_register_file('/qemu-server/', 
+cfs_register_file('/qemu-server/',
 		  \&parse_vm_config,
 		  \&write_vm_config);
 
@@ -52,6 +55,12 @@ PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
     optional => 1,
 });
 
+PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
+    description => "The name of the snapshot.",
+    type => 'string', format => 'pve-configid',
+    maxLength => 40,
+});
+
 #no warnings 'redefine';
 
 unless(defined(&_VZSYSCALLS_H_)) {
@@ -169,7 +178,7 @@ my $confdesc = {
 	optional => 1,
 	type => 'string',
 	description => "Lock/unlock the VM.",
-	enum => [qw(migrate backup)],
+	enum => [qw(migrate backup snapshot rollback)],
     },
     cpulimit => {
 	optional => 1,
@@ -208,13 +217,20 @@ my $confdesc = {
     },
     name => {
 	optional => 1,
-	type => 'string',
+	type => 'string', format => 'dns-name',
 	description => "Set a name for the VM. Only used on the configuration web interface.",
     },
+    scsihw => {
+	optional => 1,
+	type => 'string',
+	description => "scsi controller model",
+	enum => [qw(lsi virtio-scsi-pci megasas)],
+	default => 'lsi',
+    },
     description => {
 	optional => 1,
 	type => 'string',
-	description => "Description for the VM. Only used on the configuration web interface.",
+	description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
     },
     ostype => {
 	optional => 1,
@@ -249,7 +265,7 @@ EODESC
 	optional => 1,
 	type => 'string', format => 'pve-qm-bootdisk',
 	description => "Enable booting from specified disk.",
-	pattern => '(ide|scsi|virtio)\d+',
+	pattern => '(ide|sata|scsi|virtio)\d+',
     },
     smp => {
 	optional => 1,
@@ -278,6 +294,12 @@ EODESC
 	description => "Enable/disable ACPI.",
 	default => 1,
     },
+    agent => {
+	optional => 1,
+	type => 'boolean',
+	description => "Enable/disable Qemu GuestAgent.",
+	default => 0,
+    },
     kvm => {
 	optional => 1,
 	type => 'boolean',
@@ -320,6 +342,12 @@ EODESC
 	pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 	default => 'now',
     },
+    startup => {
+	optional => 1,
+	type => 'string', format => 'pve-qm-startup',
+	typetext => '[[order=]\d+] [,up=\d+] [,down=\d+] ',
+	description => "Startup and shutdown behavior. Order is a non-negative number defining the general startup order. Shutdown in done with reverse ordering. Additionally you can set the 'up' or 'down' delay in seconds, which specifies a delay to wait before the next VM is started or stopped.",
+    },
     args => {
 	optional => 1,
 	type => 'string',
@@ -362,6 +390,21 @@ EODESCR
 	enum => [ qw(486 athlon pentium pentium2 pentium3 coreduo core2duo kvm32 kvm64 qemu32 qemu64 phenom cpu64-rhel6 cpu64-rhel5 Conroe Penryn Nehalem Westmere Opteron_G1 Opteron_G2 Opteron_G3 host) ],
 	default => 'qemu64',
     },
+    parent => get_standard_option('pve-snapshot-name', {
+	optional => 1,
+	description => "Parent snapshot name. This is used internally, and should not be modified.",
+    }),
+    snaptime => {
+	optional => 1,
+	description => "Timestamp for snapshots.",
+	type => 'integer',
+	minimum => 0,
+    },
+    vmstate => {
+	optional => 1,
+	type => 'string', format => 'pve-volume-id',
+	description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
+    },
 };
 
 # what about other qemu settings ?
@@ -387,10 +430,10 @@ while (my ($k, $v) = each %$confdesc) {
 
 my $MAX_IDE_DISKS = 4;
 my $MAX_SCSI_DISKS = 14;
-my $MAX_VIRTIO_DISKS = 6;
+my $MAX_VIRTIO_DISKS = 16;
 my $MAX_SATA_DISKS = 6;
 my $MAX_USB_DEVICES = 5;
-my $MAX_NETS = 6;
+my $MAX_NETS = 32;
 my $MAX_UNUSED_DISKS = 8;
 my $MAX_HOSTPCI_DEVICES = 2;
 my $MAX_SERIAL_PORTS = 4;
@@ -404,7 +447,7 @@ my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 my $netdesc = {
     optional => 1,
     type => 'string', format => 'pve-qm-net',
-    typetext => "MODEL=XX:XX:XX:XX:XX:XX [,bridge=<dev>][,rate=<mbps>]",
+    typetext => "MODEL=XX:XX:XX:XX:XX:XX [,bridge=<dev>][,rate=<mbps>][,tag=<vlanid>]",
     description => <<EODESCR,
 Specify network devices.
 
@@ -438,32 +481,32 @@ my $drivename_hash;
 my $idedesc = {
     optional => 1,
     type => 'string', format => 'pve-qm-drive',
-    typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe] [,format=f] [,backup=yes|no] [,aio=native|threads]',
-    description => "Use volume as IDE hard disk or CD-ROM (n is 0 to 3).",
+    typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads]',
+    description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
 };
 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
 
 my $scsidesc = {
     optional => 1,
     type => 'string', format => 'pve-qm-drive',
-    typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe] [,format=f] [,backup=yes|no] [,aio=native|threads]',
-    description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to 13).",
+    typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads]',
+    description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
 };
 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 
 my $satadesc = {
     optional => 1,
     type => 'string', format => 'pve-qm-drive',
-    typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe] [,format=f] [,backup=yes|no] [,aio=native|threads]',
-    description => "Use volume as SATA hard disk or CD-ROM (n is 0 to 5).",
+    typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads]',
+    description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
 };
 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
 
 my $virtiodesc = {
     optional => 1,
     type => 'string', format => 'pve-qm-drive',
-    typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe] [,format=f] [,backup=yes|no] [,aio=native|threads]',
-    description => "Use volume as VIRTIO hard disk (n is 0 to 5).",
+    typetext => '[volume=]volume,] [,media=cdrom|disk] [,cyls=c,heads=h,secs=s[,trans=t]] [,snapshot=on|off] [,cache=none|writethrough|writeback|unsafe|directsync] [,format=f] [,backup=yes|no] [,rerror=ignore|report|stop] [,werror=enospc|ignore|report|stop] [,aio=native|threads]',
+    description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
 };
 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
 
@@ -652,45 +695,6 @@ sub os_list_description {
     };
 }
 
-sub disk_devive_info {
-    my $dev = shift;
-
-    die "unknown disk device format '$dev'" if $dev !~ m/^(ide|scsi|virtio)(\d+)$/;
-
-    my $bus = $1;
-    my $index = $2;
-    my $maxdev = 1024;
-
-    if ($bus eq 'ide') {
-	$maxdev = 2;
-    } elsif ($bus eq 'scsi') {
-	$maxdev = 7;
-    }
-
-    my $controller = int($index / $maxdev);
-    my $unit = $index % $maxdev;
-
-
-    return { bus => $bus, desc => uc($bus) . " $controller:$unit",
-	     controller => $controller, unit => $unit, index => $index };
-
-}
-
-sub qemu_drive_name {
-    my ($dev, $media) = @_;
-
-    my $info = disk_devive_info($dev);
-    my $mediastr = '';
-
-    if (($info->{bus} eq 'ide') || ($info->{bus} eq 'scsi')) {
-	$mediastr = ($media eq 'cdrom') ? "-cd" : "-hd";
-	return sprintf("%s%i%s%i", $info->{bus}, $info->{controller},
-		       $mediastr, $info->{unit});
-    } else {
-	return sprintf("%s%i", $info->{bus}, $info->{index});
-    }
-}
-
 my $cdrom_path;
 
 sub get_cdrom_path {
@@ -742,7 +746,7 @@ sub verify_media_type {
 
     my $etype;
     if ($media eq 'disk') {
-	$etype = 'image';
+	$etype = 'images';
     } elsif ($media eq 'cdrom') {
 	$etype = 'iso';
     } else {
@@ -798,8 +802,43 @@ sub create_conf_nolock {
     PVE::Tools::file_set_contents($filename, $data);
 }
 
+my $parse_size = sub {
+    my ($value) = @_;
+
+    return undef if $value !~ m/^(\d+(\.\d+)?)([KMG])?$/;
+    my ($size, $unit) = ($1, $3);
+    if ($unit) {
+	if ($unit eq 'K') {
+	    $size = $size * 1024;
+	} elsif ($unit eq 'M') {
+	    $size = $size * 1024 * 1024;
+	} elsif ($unit eq 'G') {
+	    $size = $size * 1024 * 1024 * 1024;
+	}
+    }
+    return int($size);
+};
+
+my $format_size = sub {
+    my ($size) = @_;
+
+    $size = int($size);
+
+    my $kb = int($size/1024);
+    return $size if $kb*1024 != $size;
+
+    my $mb = int($kb/1024);
+    return "${kb}K" if $mb*1024 != $kb;
+
+    my $gb = int($mb/1024);
+    return "${mb}M" if $gb*1024 != $mb;
+
+    return "${gb}G";
+};
+
 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
+#        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
 #        [,aio=native|threads]
 
 sub parse_drive {
@@ -821,13 +860,18 @@ sub parse_drive {
     foreach my $p (split (/,/, $data)) {
 	next if $p =~ m/^\s*$/;
 
-	if ($p =~ m/^(file|volume|cyls|heads|secs|trans|media|snapshot|cache|format|rerror|werror|backup|aio)=(.+)$/) {
+	if ($p =~ m/^(file|volume|cyls|heads|secs|trans|media|snapshot|cache|format|rerror|werror|backup|aio|bps|mbps|bps_rd|mbps_rd|bps_wr|mbps_wr|iops|iops_rd|iops_wr|size)=(.+)$/) {
 	    my ($k, $v) = ($1, $2);
 
 	    $k = 'file' if $k eq 'volume';
 
 	    return undef if defined $res->{$k};
 
+	    if ($k eq 'bps' || $k eq 'bps_rd' || $k eq 'bps_wr') {
+		return undef if !$v || $v !~ m/^\d+/;
+		$k = "m$k";
+		$v = sprintf("%.3f", $v / (1024*1024));
+	    }
 	    $res->{$k} = $v;
 	} else {
 	    if (!$res->{file} && $p !~ m/=/) {
@@ -841,7 +885,7 @@ sub parse_drive {
     return undef if !$res->{file};
 
     return undef if $res->{cache} &&
-	$res->{cache} !~ m/^(off|none|writethrough|writeback|unsafe)$/;
+	$res->{cache} !~ m/^(off|none|writethrough|writeback|unsafe|directsync)$/;
     return undef if $res->{snapshot} && $res->{snapshot} !~ m/^(on|off)$/;
     return undef if $res->{cyls} && $res->{cyls} !~ m/^\d+$/;
     return undef if $res->{heads} && $res->{heads} !~ m/^\d+$/;
@@ -854,6 +898,25 @@ sub parse_drive {
     return undef if $res->{backup} && $res->{backup} !~ m/^(yes|no)$/;
     return undef if $res->{aio} && $res->{aio} !~ m/^(native|threads)$/;
 
+    
+    return undef if $res->{mbps_rd} && $res->{mbps};
+    return undef if $res->{mbps_wr} && $res->{mbps};
+
+    return undef if $res->{mbps} && $res->{mbps} !~ m/^\d+(\.\d+)?$/;
+    return undef if $res->{mbps_rd} && $res->{mbps_rd} !~ m/^\d+(\.\d+)?$/;
+    return undef if $res->{mbps_wr} && $res->{mbps_wr} !~ m/^\d+(\.\d+)?$/;
+
+    return undef if $res->{iops_rd} && $res->{iops};
+    return undef if $res->{iops_wr} && $res->{iops};
+    return undef if $res->{iops} && $res->{iops} !~ m/^\d+$/;
+    return undef if $res->{iops_rd} && $res->{iops_rd} !~ m/^\d+$/;
+    return undef if $res->{iops_wr} && $res->{iops_wr} !~ m/^\d+$/;
+
+
+    if ($res->{size}) {
+	return undef if !defined($res->{size} = &$parse_size($res->{size})); 
+    }
+
     if ($res->{media} && ($res->{media} eq 'cdrom')) {
 	return undef if $res->{snapshot} || $res->{trans} || $res->{format};
 	return undef if $res->{heads} || $res->{secs} || $res->{cyls};
@@ -868,30 +931,92 @@ sub parse_drive {
     return $res;
 }
 
-my @qemu_drive_options = qw(heads secs cyls trans media format cache snapshot rerror werror aio);
+my @qemu_drive_options = qw(heads secs cyls trans media format cache snapshot rerror werror aio iops iops_rd iops_wr);
 
 sub print_drive {
     my ($vmid, $drive) = @_;
 
     my $opts = '';
-    foreach my $o (@qemu_drive_options, 'backup') {
+    foreach my $o (@qemu_drive_options, 'mbps', 'mbps_rd', 'mbps_wr', 'backup') {
 	$opts .= ",$o=$drive->{$o}" if $drive->{$o};
     }
 
+    if ($drive->{size}) {
+	$opts .= ",size=" . &$format_size($drive->{size});
+    }
+
     return "$drive->{file}$opts";
 }
 
+sub scsi_inquiry {
+    my($fh, $noerr) = @_;
+
+    my $SG_IO = 0x2285;
+    my $SG_GET_VERSION_NUM = 0x2282;
+
+    my $versionbuf = "\x00" x 8;
+    my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
+    if (!$ret) {
+	die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
+	return undef;
+    }
+    my $version = unpack("I", $versionbuf);
+    if ($version < 30000) {
+	die "scsi generic interface too old\n"  if !$noerr;
+	return undef;
+    }
+
+    my $buf = "\x00" x 36;
+    my $sensebuf = "\x00" x 8;
+    my $cmd = pack("C x3 C x11", 0x12, 36);
+
+    # see /usr/include/scsi/sg.h
+    my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
+
+    my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
+		      length($sensebuf), 0, length($buf), $buf,
+		      $cmd, $sensebuf, 6000);
+
+    $ret = ioctl($fh, $SG_IO, $packet);
+    if (!$ret) {
+	die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
+	return undef;
+    }
+
+    my @res = unpack($sg_io_hdr_t, $packet);
+    if ($res[17] || $res[18]) {
+	die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
+	return undef;
+    }
+
+    my $res = {};
+    ($res->{device}, $res->{removable}, $res->{venodor},
+     $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
+
+    return $res;
+}
+
+sub path_is_scsi {
+    my ($path) = @_;
+
+    my $fh = IO::File->new("+<$path") || return undef;
+    my $res = scsi_inquiry($fh, 1);
+    close($fh);
+
+    return $res;
+}
+
 sub print_drivedevice_full {
-    my ($storecfg, $vmid, $drive) = @_;
+    my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
 
     my $device = '';
     my $maxdev = 0;
 
     if ($drive->{interface} eq 'virtio') {
-	my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}");
+	my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
 	$device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
     } elsif ($drive->{interface} eq 'scsi') {
-	$maxdev = 7;
+	$maxdev = ($conf->{scsihw} && $conf->{scsihw} ne 'lsi') ? 256 : 7;
 	my $controller = int($drive->{index} / $maxdev);
 	my $unit = $drive->{index} % $maxdev;
 	my $devicetype = 'hd';
@@ -904,12 +1029,21 @@ sub print_drivedevice_full {
               } else {
                   $path = PVE::Storage::path($storecfg, $drive->{file});
               }
-              if ($path =~ m|^/dev/| ) {
-                  $devicetype = 'block';
-              }
+
+	      if($path =~ m/^iscsi\:\/\//){
+		 $devicetype = 'generic';
+	      }
+	      else {
+		 $devicetype = 'block' if path_is_scsi($path);
+	      }
          }
 
-	$device = "scsi-$devicetype,bus=lsi$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
+        if (!$conf->{scsihw} || $conf->{scsihw} eq 'lsi'){
+            $device = "scsi-$devicetype,bus=scsihw$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}" if !$conf->{scsihw} || $conf->{scsihw} eq 'lsi';
+        } else {
+            $device = "scsi-$devicetype,bus=scsihw$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
+        }
+
     } elsif ($drive->{interface} eq 'ide'){
 	$maxdev = 2;
 	my $controller = int($drive->{index} / $maxdev);
@@ -942,6 +1076,11 @@ sub print_drive_full {
 	$opts .= ",$o=$drive->{$o}" if $drive->{$o};
     }
 
+    foreach my $o (qw(bps bps_rd bps_wr)) {
+	my $v = $drive->{"m$o"};
+	$opts .= ",$o=" . int($v*1024*1024) if $v;
+    }
+
     # use linux-aio by default (qemu default is threads)
     $opts .= ",aio=native" if !$drive->{aio};
 
@@ -966,7 +1105,7 @@ sub print_drive_full {
 }
 
 sub print_netdevice_full {
-    my ($vmid, $conf, $net, $netid) = @_;
+    my ($vmid, $conf, $net, $netid, $bridges) = @_;
 
     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
 
@@ -978,7 +1117,7 @@ sub print_netdevice_full {
     # qemu > 0.15 always try to boot from network - we disable that by
     # not loading the pxe rom file
     my $extra = ($bootorder !~ m/n/) ? "romfile=," : '';
-    my $pciaddr = print_pci_addr("$netid");
+    my $pciaddr = print_pci_addr("$netid", $bridges);
     my $tmpstr = "$device,${extra}mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
     return $tmpstr;
@@ -1052,6 +1191,8 @@ sub parse_net {
 	    $res->{bridge} = $1;
 	} elsif ($kvp =~ m/^rate=(\d+(\.\d+)?)$/) {
 	    $res->{rate} = $1;
+        } elsif ($kvp =~ m/^tag=(\d+)$/) {
+            $res->{tag} = $1;
 	} else {
 	    return undef;
 	}
@@ -1070,6 +1211,7 @@ sub print_net {
     $res .= "=$net->{macaddr}" if $net->{macaddr};
     $res .= ",bridge=$net->{bridge}" if $net->{bridge};
     $res .= ",rate=$net->{rate}" if $net->{rate};
+    $res .= ",tag=$net->{tag}" if $net->{tag};
 
     return $res;
 }
@@ -1099,7 +1241,7 @@ sub add_unused_volume {
     }
 
     die "To many unused volume - please delete them first.\n" if !$key;
-    
+
     $config->{$key} = $volid;
 
     return $key;
@@ -1184,6 +1326,41 @@ sub parse_watchdog {
     return $res;
 }
 
+PVE::JSONSchema::register_format('pve-qm-startup', \&verify_startup);
+sub verify_startup {
+    my ($value, $noerr) = @_;
+
+    return $value if parse_startup($value);
+
+    return undef if $noerr;
+
+    die "unable to parse startup options\n";
+}
+
+sub parse_startup {
+    my ($value) = @_;
+
+    return undef if !$value;
+
+    my $res = {};
+
+    foreach my $p (split(/,/, $value)) {
+	next if $p =~ m/^\s*$/;
+
+	if ($p =~ m/^(order=)?(\d+)$/) {
+	    $res->{order} = $2;
+	} elsif ($p =~ m/^up=(\d+)$/) {
+	    $res->{up} = $1;
+	} elsif ($p =~ m/^down=(\d+)$/) {
+	    $res->{down} = $1;
+	} else {
+	    return undef;
+	}
+    }
+
+    return $res;
+}
+
 sub parse_usb_device {
     my ($value) = @_;
 
@@ -1194,10 +1371,10 @@ sub parse_usb_device {
 
     my $res = {};
     foreach my $v (@dl) {
-	if ($v =~ m/^host=([0-9A-Fa-f]{4}):([0-9A-Fa-f]{4})$/) {
+	if ($v =~ m/^host=(0x)?([0-9A-Fa-f]{4}):(0x)?([0-9A-Fa-f]{4})$/) {
 	    $found = 1;
-	    $res->{vendorid} = $1;
-	    $res->{productid} = $2;
+	    $res->{vendorid} = $2;
+	    $res->{productid} = $4;
 	} elsif ($v =~ m/^host=(\d+)\-(\d+(\.\d+)*)$/) {
 	    $found = 1;
 	    $res->{hostbus} = $1;
@@ -1227,6 +1404,7 @@ sub json_config_properties {
     my $prop = shift;
 
     foreach my $opt (keys %$confdesc) {
+	next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
 	$prop->{$opt} = $confdesc->{$opt};
     }
 
@@ -1273,18 +1451,24 @@ sub check_type {
     }
 }
 
-sub lock_config {
-    my ($vmid, $code, @param) = @_;
+sub lock_config_full {
+    my ($vmid, $timeout, $code, @param) = @_;
 
     my $filename = config_file_lock($vmid);
 
-    my $res = lock_file($filename, 10, $code, @param);
+    my $res = lock_file($filename, $timeout, $code, @param);
 
     die $@ if $@;
 
     return $res;
 }
 
+sub lock_config {
+    my ($vmid, $code, @param) = @_;
+
+    return lock_config_full($vmid, 10, $code, @param);
+}
+
 sub cfs_config_path {
     my ($vmid, $node) = @_;
 
@@ -1367,52 +1551,10 @@ sub destroy_vm {
     warn $@ if $@;
 }
 
-# fixme: remove?
-sub load_diskinfo_old {
-    my ($storecfg, $vmid, $conf) = @_;
-
-    my $info = {};
-    my $res = {};
-    my $vollist;
-
-    foreach_drive($conf, sub {
-	my ($ds, $di) = @_;
-
-	$res->{$ds} = $di;
-
-	return if drive_is_cdrom($di);
-
-	if ($di->{file} =~ m|^/dev/.+|) {
-	    $info->{$di->{file}}->{size} = PVE::Storage::file_size_info($di->{file});
-	} else {
-	    push @$vollist, $di->{file};
-	}
-    });
-
-    eval {
-	my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid, $vollist);
-
-	PVE::Storage::foreach_volid($dl, sub {
-	    my ($volid, $sid, $volname, $d) = @_;
-	    $info->{$volid} = $d;
-	});
-    };
-    warn $@ if $@;
-
-    foreach my $ds (keys %$res) {
-	my $di = $res->{$ds};
-
-	$res->{$ds}->{disksize} = $info->{$di->{file}} ?
-	    $info->{$di->{file}}->{size} / (1024*1024) : 0;
-    }
-
-    return $res;
-}
-
 sub load_config {
-    my ($vmid) = @_;
+    my ($vmid, $node) = @_;
 
-    my $cfspath = cfs_config_path($vmid);
+    my $cfspath = cfs_config_path($vmid, $node);
 
     my $conf = PVE::Cluster::cfs_read_file($cfspath);
 
@@ -1427,7 +1569,8 @@ sub parse_vm_config {
     return undef if !defined($raw);
 
     my $res = {
-	digest => Digest::SHA1::sha1_hex($raw),
+	digest => Digest::SHA::sha1_hex($raw),
+	snapshots => {},
     };
 
     $filename =~ m|/qemu-server/(\d+)\.conf$|
@@ -1435,21 +1578,34 @@ sub parse_vm_config {
 
     my $vmid = $1;
 
-    while ($raw && $raw =~ s/^(.*?)(\n|$)//) {
-	my $line = $1;
-
-	next if $line =~ m/^\#/;
+    my $conf = $res;
+    my $descr = '';
 
+    my @lines = split(/\n/, $raw);
+    foreach my $line (@lines) {
 	next if $line =~ m/^\s*$/;
+	
+	if ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
+	    my $snapname = $1;
+	    $conf->{description} = $descr if $descr;
+	    $descr = '';
+	    $conf = $res->{snapshots}->{$snapname} = {}; 
+	    next;
+	}
+
+	if ($line =~ m/^\#(.*)\s*$/) {
+	    $descr .= PVE::Tools::decode_text($1) . "\n";
+	    next;
+	}
 
 	if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
-	    my $key = $1;
-	    my $value = PVE::Tools::decode_text($2);
-	    $res->{$key} = $value;
+	    $descr .= PVE::Tools::decode_text($2);
+	} elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
+	    $conf->{snapstate} = $1;
 	} elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
 	    my $key = $1;
 	    my $value = $2;
-	    $res->{$key} = $value;
+	    $conf->{$key} = $value;
 	} elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
 	    my $key = $1;
 	    my $value = $2;
@@ -1470,19 +1626,17 @@ sub parse_vm_config {
 		}
 
 		if ($key eq 'cdrom') {
-		    $res->{ide2} = $value;
+		    $conf->{ide2} = $value;
 		} else {
-		    $res->{$key} = $value;
+		    $conf->{$key} = $value;
 		}
 	    }
 	}
     }
 
-    # convert old smp to sockets
-    if ($res->{smp} && !$res->{sockets}) {
-	$res->{sockets} = $res->{smp};
-    }
-    delete $res->{smp};
+    $conf->{description} = $descr if $descr;
+
+    delete $res->{snapstate}; # just to be sure
 
     return $res;
 }
@@ -1490,6 +1644,8 @@ sub parse_vm_config {
 sub write_vm_config {
     my ($filename, $conf) = @_;
 
+    delete $conf->{snapstate}; # just to be sure
+
     if ($conf->{cdrom}) {
 	die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
 	$conf->{ide2} = $conf->{cdrom};
@@ -1505,37 +1661,62 @@ sub write_vm_config {
 	delete $conf->{smp};
     }
 
-    my $new_volids = {};
-    foreach my $key (keys %$conf) {
-	next if $key eq 'digest';
-	my $value = $conf->{$key};
-	if ($key eq 'description') {
-	    $value = PVE::Tools::encode_text($value);
-	}
-	eval { $value = check_type($key, $value); };
-	die "unable to parse value of '$key' - $@" if $@;
+    my $used_volids = {};
 
-	$conf->{$key} = $value;
+    my $cleanup_config = sub {
+	my ($cref) = @_;
 
-	if (valid_drivename($key)) {
-	    my $drive = PVE::QemuServer::parse_drive($key, $value);
-	    $new_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
+	foreach my $key (keys %$cref) {
+	    next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
+		$key eq 'snapstate';
+	    my $value = $cref->{$key};
+	    eval { $value = check_type($key, $value); };
+	    die "unable to parse value of '$key' - $@" if $@;
+
+	    $cref->{$key} = $value;
+
+	    if (valid_drivename($key)) {
+		my $drive = PVE::QemuServer::parse_drive($key, $value);
+		$used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
+	    }
 	}
+    };
+
+    &$cleanup_config($conf);
+    foreach my $snapname (keys %{$conf->{snapshots}}) {
+	&$cleanup_config($conf->{snapshots}->{$snapname});
     }
 
     # remove 'unusedX' settings if we re-add a volume
     foreach my $key (keys %$conf) {
 	my $value = $conf->{$key};
-	if ($key =~ m/^unused/ && $new_volids->{$value}) {
+	if ($key =~ m/^unused/ && $used_volids->{$value}) {
 	    delete $conf->{$key};
 	}
     }
+  
+    my $generate_raw_config = sub {
+	my ($conf) = @_;
+
+	my $raw = '';
+
+	# add description as comment to top of file
+	my $descr = $conf->{description} || '';
+	foreach my $cl (split(/\n/, $descr)) {
+	    $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
+	}
+
+	foreach my $key (sort keys %$conf) {
+	    next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots';
+	    $raw .= "$key: $conf->{$key}\n";
+	}
+	return $raw;
+    };
 
-    # gererate RAW data
-    my $raw = '';
-    foreach my $key (sort keys %$conf) {
-	next if $key eq 'digest';
-	$raw .= "$key: $conf->{$key}\n";
+    my $raw = &$generate_raw_config($conf);
+    foreach my $snapname (sort keys %{$conf->{snapshots}}) {
+	$raw .= "\n[$snapname]\n";
+	$raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
     }
 
     return $raw;
@@ -1545,7 +1726,7 @@ sub update_config_nolock {
     my ($vmid, $conf, $skiplock) = @_;
 
     check_lock($conf) if !$skiplock;
-    
+
     my $cfspath = cfs_config_path($vmid);
 
     PVE::Cluster::cfs_write_file($cfspath, $conf);
@@ -1607,6 +1788,25 @@ sub check_local_resources {
     return $loc_res;
 }
 
+# check is used storages are available on all nodes (use by migrate)
+sub check_storage_availability {
+    my ($storecfg, $conf, $node) = @_;
+
+    foreach_drive($conf, sub {
+	my ($ds, $drive) = @_;
+
+	my $volid = $drive->{file};
+	return if !$volid;
+
+	my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
+	return if !$sid;
+
+	# check if storage is available on both nodes
+	my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
+	PVE::Storage::storage_check_node($storecfg, $sid, $node);
+   });
+}
+
 sub check_lock {
     my ($conf) = @_;
 
@@ -1640,9 +1840,9 @@ sub check_cmdline {
 }
 
 sub check_running {
-    my ($vmid, $nocheck) = @_;
+    my ($vmid, $nocheck, $node) = @_;
 
-    my $filename = config_file($vmid);
+    my $filename = config_file($vmid, $node);
 
     die "unable to find configuration file for VM $vmid - no such machine\n"
 	if !$nocheck && ! -f $filename;
@@ -1690,8 +1890,6 @@ sub vzlist {
     return $vzlist;
 }
 
-my $storage_timeout_hash = {};
-
 sub disksize {
     my ($storecfg, $conf) = @_;
 
@@ -1709,49 +1907,16 @@ sub disksize {
     my $volid = $drive->{file};
     return undef if !$volid;
 
-    my $path;
-    my $storeid;
-    my $timeoutid;
-
-    if ($volid =~ m|^/|) {
-	$path = $timeoutid = $volid;
-    } else {
-	eval {
-	    $storeid = $timeoutid = PVE::Storage::parse_volume_id($volid);
-	    $path = PVE::Storage::path($storecfg, $volid);
-	};
-	if (my $err = $@) {
-	    warn $err;
-	    return undef;
-	}
-    }
-
-    my $last_timeout = $storage_timeout_hash->{$timeoutid};
-    if ($last_timeout) {
-	if ((time() - $last_timeout) < 30) {
-	    # skip storage with errors
-	    return undef ;
-	}
-	delete $storage_timeout_hash->{$timeoutid};
-    }
-
-    my ($size, $format, $used);
-
-    ($size, $format, $used) = PVE::Storage::file_size_info($path, 1);
-
-    if (!defined($format)) {
-	# got timeout
-	$storage_timeout_hash->{$timeoutid} = time();
-	return undef;
-    }
-
-    return wantarray ? ($size, $used) : $size;
+    return $drive->{size};
 }
 
 my $last_proc_pid_stat;
 
+# get VM status information
+# This must be fast and should not block ($full == false)
+# We only query KVM using QMP if $full == true (this can be slow)
 sub vmstatus {
-    my ($opt_vmid) = @_;
+    my ($opt_vmid, $full) = @_;
 
     my $res = {};
 
@@ -1774,9 +1939,9 @@ sub vmstatus {
 	# fixme: better status?
 	$d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
 
-	my ($size, $used) = disksize($storecfg, $conf);
-	if (defined($size) && defined($used)) {
-	    $d->{disk} = $used;
+	my $size = disksize($storecfg, $conf);
+	if (defined($size)) {
+	    $d->{disk} = 0; # no info available
 	    $d->{maxdisk} = $size;
 	} else {
 	    $d->{disk} = 0;
@@ -1821,18 +1986,6 @@ sub vmstatus {
 	my $pid = $d->{pid};
 	next if !$pid;
 
-	if (my $fh = IO::File->new("/proc/$pid/io", "r")) {
-	    my $data = {};
-	    while (defined(my $line = <$fh>)) {
-		if ($line =~ m/^([rw]char):\s+(\d+)$/) {
-		    $data->{$1} = $2;
-		}
-	    }
-	    close($fh);
-	    $d->{diskread} = $data->{rchar} || 0;
-	    $d->{diskwrite} = $data->{wchar} || 0;
-	}
-
 	my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
 	next if !$pstat; # not running
 
@@ -1870,6 +2023,49 @@ sub vmstatus {
 	}
     }
 
+    return $res if !$full;
+
+    my $qmpclient = PVE::QMPClient->new();
+
+    my $blockstatscb = sub {
+	my ($vmid, $resp) = @_;
+	my $data = $resp->{'return'} || [];
+	my $totalrdbytes = 0;
+	my $totalwrbytes = 0;
+	for my $blockstat (@$data) {
+	    $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
+	    $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
+	}
+	$res->{$vmid}->{diskread} = $totalrdbytes;
+	$res->{$vmid}->{diskwrite} = $totalwrbytes;
+    };
+
+    my $statuscb = sub {
+	my ($vmid, $resp) = @_;
+	$qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
+
+	my $status = 'unknown';
+	if (!defined($status = $resp->{'return'}->{status})) {
+	    warn "unable to get VM status\n";
+	    return;
+	}
+
+	$res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
+    };
+
+    foreach my $vmid (keys %$list) {
+	next if $opt_vmid && ($vmid ne $opt_vmid);
+	next if !$res->{$vmid}->{pid}; # not running
+	$qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
+    }
+
+    $qmpclient->queue_execute();
+
+    foreach my $vmid (keys %$list) {
+	next if $opt_vmid && ($vmid ne $opt_vmid);
+	$res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
+    }
+
     return $res;
 }
 
@@ -1890,7 +2086,9 @@ sub config_to_command {
     my ($storecfg, $vmid, $conf, $defaults, $migrate_uri) = @_;
 
     my $cmd = [];
+    my $devices = [];
     my $pciaddr = '';
+    my $bridges = {};
     my $kvmver = kvm_user_version();
     my $vernum = 0; # unknown
     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
@@ -1909,11 +2107,11 @@ sub config_to_command {
 
     my $use_virtio = 0;
 
-    my $socket = monitor_socket($vmid);
-    push @$cmd, '-chardev', "socket,id=monitor,path=$socket,server,nowait";
-    push @$cmd, '-mon', "chardev=monitor,mode=readline";
+    my $qmpsocket = qmp_socket($vmid);
+    push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
+    push @$cmd, '-mon', "chardev=qmp,mode=control";
 
-    $socket = vnc_socket($vmid);
+    my $socket = vnc_socket($vmid);
     push @$cmd,  '-vnc', "unix:$socket,x509,password";
 
     push @$cmd, '-pidfile' , pidfile_name($vmid);
@@ -1922,21 +2120,23 @@ sub config_to_command {
 
     push @$cmd, '-incoming', $migrate_uri if $migrate_uri;
 
+    push @$cmd, '-S' if $migrate_uri;
+
     my $use_usb2 = 0;
     for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
 	next if !$conf->{"usb$i"};
 	$use_usb2 = 1;
     }
     # include usb device config
-    push @$cmd, '-readconfig', '/usr/share/qemu-server/pve-usb.cfg' if $use_usb2;
+    push @$devices, '-readconfig', '/usr/share/qemu-server/pve-usb.cfg' if $use_usb2;
 
     # enable absolute mouse coordinates (needed by vnc)
     my $tablet = defined($conf->{tablet}) ? $conf->{tablet} : $defaults->{tablet};
     if ($tablet) {
 	if ($use_usb2) {
-	    push @$cmd, '-device', 'usb-tablet,bus=ehci.0,port=6';
+	    push @$devices, '-device', 'usb-tablet,bus=ehci.0,port=6';
 	} else {
-	    push @$cmd, '-usbdevice', 'tablet';
+	    push @$devices, '-usbdevice', 'tablet';
 	}
     }
 
@@ -1944,8 +2144,8 @@ sub config_to_command {
     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
           my $d = parse_hostpci($conf->{"hostpci$i"});
           next if !$d;
-	  $pciaddr = print_pci_addr("hostpci$i");
-          push @$cmd, '-device', "pci-assign,host=$d->{pciid},id=hostpci$i$pciaddr";
+	  $pciaddr = print_pci_addr("hostpci$i", $bridges);
+          push @$devices, '-device', "pci-assign,host=$d->{pciid},id=hostpci$i$pciaddr";
     }
 
     # usb devices
@@ -1953,9 +2153,9 @@ sub config_to_command {
 	my $d = parse_usb_device($conf->{"usb$i"});
 	next if !$d;
 	if ($d->{vendorid} && $d->{productid}) {
-	    push @$cmd, '-device', "usb-host,vendorid=$d->{vendorid},productid=$d->{productid}";
+	    push @$devices, '-device', "usb-host,vendorid=0x$d->{vendorid},productid=0x$d->{productid}";
 	} elsif (defined($d->{hostbus}) && defined($d->{hostport})) {
-	    push @$cmd, '-device', "usb-host,hostbus=$d->{hostbus},hostport=$d->{hostport}";
+	    push @$devices, '-device', "usb-host,hostbus=$d->{hostbus},hostport=$d->{hostport}";
 	}
     }
 
@@ -1963,8 +2163,8 @@ sub config_to_command {
     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
 	if (my $path = $conf->{"serial$i"}) {
 	    die "no such serial device\n" if ! -c $path;
-	    push @$cmd, '-chardev', "tty,id=serial$i,path=$path";
-	    push @$cmd, '-device', "isa-serial,chardev=serial$i";
+	    push @$devices, '-chardev', "tty,id=serial$i,path=$path";
+	    push @$devices, '-device', "isa-serial,chardev=serial$i";
 	}
     }
 
@@ -1972,8 +2172,8 @@ sub config_to_command {
     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
 	if (my $path = $conf->{"parallel$i"}) {
 	    die "no such parallel device\n" if ! -c $path;
-	    push @$cmd, '-chardev', "parport,id=parallel$i,path=$path";
-	    push @$cmd, '-device', "isa-parallel,chardev=parallel$i";
+	    push @$devices, '-chardev', "parport,id=parallel$i,path=$path";
+	    push @$devices, '-device', "isa-parallel,chardev=parallel$i";
 	}
     }
 
@@ -2038,6 +2238,11 @@ sub config_to_command {
 	    }
 	}
 
+	if ($ost eq 'win7' || $ost eq 'w2k8' || $ost eq 'wvista') {
+	    push @$cmd, '-no-kvm-pit-reinjection';
+	    push @$cmd, '-no-hpet';
+	}
+
 	# -tdf ?
 	# -no-acpi
 	# -no-kvm
@@ -2064,20 +2269,30 @@ sub config_to_command {
     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
     #push @$cmd, '-soundhw', 'es1370';
     #push @$cmd, '-soundhw', $soundhw if $soundhw;
-    $pciaddr = print_pci_addr("balloon0");
-    push @$cmd, '-device', "virtio-balloon-pci,id=balloon0$pciaddr" if $conf->{balloon};
+
+    if($conf->{agent}) {
+	my $qgasocket = qga_socket($vmid);
+	my $pciaddr = print_pci_addr("qga0", $bridges);
+	push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
+	push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
+	push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
+    }
+
+    $pciaddr = print_pci_addr("balloon0", $bridges);
+    push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr" if $conf->{balloon};
 
     if ($conf->{watchdog}) {
 	my $wdopts = parse_watchdog($conf->{watchdog});
-	$pciaddr = print_pci_addr("watchdog");
+	$pciaddr = print_pci_addr("watchdog", $bridges);
 	my $watchdog = $wdopts->{model} || 'i6300esb';
-	push @$cmd, '-device', "$watchdog$pciaddr";
-	push @$cmd, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
+	push @$devices, '-device', "$watchdog$pciaddr";
+	push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
     }
 
     my $vollist = [];
     my $scsicontroller = {};
     my $ahcicontroller = {};
+    my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
 
     foreach_drive($conf, sub {
 	my ($ds, $drive) = @_;
@@ -2101,22 +2316,23 @@ sub config_to_command {
 	}
 
         if ($drive->{interface} eq 'scsi') {
-           my $maxdev = 7;
-           my $controller = int($drive->{index} / $maxdev);
-           $pciaddr = print_pci_addr("lsi$controller");
-           push @$cmd, '-device', "lsi,id=lsi$controller$pciaddr" if !$scsicontroller->{$controller};
-           $scsicontroller->{$controller}=1;
+
+	    my $maxdev = ($scsihw ne 'lsi') ? 256 : 7;
+	    my $controller = int($drive->{index} / $maxdev);
+	    $pciaddr = print_pci_addr("scsihw$controller", $bridges);
+	    push @$devices, '-device', "$scsihw,id=scsihw$controller$pciaddr" if !$scsicontroller->{$controller};
+	    $scsicontroller->{$controller}=1;
         }
 
         if ($drive->{interface} eq 'sata') {
            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
-           $pciaddr = print_pci_addr("ahci$controller");
-           push @$cmd, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
+           $pciaddr = print_pci_addr("ahci$controller", $bridges);
+           push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
            $ahcicontroller->{$controller}=1;
         }
 
-	push @$cmd, '-drive',print_drive_full($storecfg, $vmid, $drive);
-	push @$cmd, '-device',print_drivedevice_full($storecfg,$vmid, $drive);
+	push @$devices, '-drive',print_drive_full($storecfg, $vmid, $drive);
+	push @$devices, '-device',print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
     });
 
     push @$cmd, '-m', $conf->{memory} || $defaults->{memory};
@@ -2134,10 +2350,16 @@ sub config_to_command {
          }
 
          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
-         push @$cmd, '-netdev', $netdevfull;
+         push @$devices, '-netdev', $netdevfull;
+
+         my $netdevicefull = print_netdevice_full($vmid,$conf,$d,"net$i",$bridges);
+         push @$devices, '-device', $netdevicefull;
+    }
 
-         my $netdevicefull = print_netdevice_full($vmid,$conf,$d,"net$i");
-         push @$cmd, '-device', $netdevicefull;
+    #bridges
+    while (my ($k, $v) = each %$bridges) {
+	$pciaddr = print_pci_addr("pci.$k");
+	unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
     }
 
 
@@ -2160,6 +2382,7 @@ sub config_to_command {
 	push @$cmd, @$aa;
     }
 
+    push @$cmd, @$devices;
     return wantarray ? ($cmd, $vollist) : $cmd;
 }
 
@@ -2168,9 +2391,14 @@ sub vnc_socket {
     return "${var_run_tmpdir}/$vmid.vnc";
 }
 
-sub monitor_socket {
+sub qmp_socket {
+    my ($vmid) = @_;
+    return "${var_run_tmpdir}/$vmid.qmp";
+}
+
+sub qga_socket {
     my ($vmid) = @_;
-    return "${var_run_tmpdir}/$vmid.mon";
+    return "${var_run_tmpdir}/$vmid.qga";
 }
 
 sub pidfile_name {
@@ -2200,24 +2428,13 @@ sub next_migrate_port {
 sub vm_devices_list {
     my ($vmid) = @_;
 
-    my $res = vm_monitor_command ($vmid, "info pci");
+    my $res = vm_mon_cmd($vmid, 'query-pci');
 
-    my @lines = split ("\n", $res);
-    my $devices;
-    my $bus;
-    my $addr;
-    my $id;
-
-    foreach my $line (@lines) {
-	$line =~ s/^\s+//;
-	if ($line =~ m/^Bus  (\d+), device   (\d+), function (\d+):$/) {
-	    $bus=$1;
-	    $addr=$2;
-	}
-	if ($line =~ m/^id "([a-z][a-z_\-]*\d*)"$/) {
-            $id=$1;
-            $devices->{$id}->{bus}=$bus;
-            $devices->{$id}->{addr}=$addr;
+    my $devices = {};
+    foreach my $pcibus (@$res) {
+	foreach my $device (@{$pcibus->{devices}}) {
+	    next if !$device->{'qdev_id'};
+	    $devices->{$device->{'qdev_id'}} = $device;
 	}
     }
 
@@ -2232,9 +2449,11 @@ sub vm_deviceplug {
     my $devices_list = vm_devices_list($vmid);
     return 1 if defined($devices_list->{$deviceid});
 
+    qemu_bridgeadd($storecfg, $conf, $vmid, $deviceid); #add bridge if we need it for the device
+
     if ($deviceid =~ m/^(virtio)(\d+)$/) {
         return undef if !qemu_driveadd($storecfg, $vmid, $device);
-        my $devicefull = print_drivedevice_full($storecfg, $vmid, $device);
+        my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
         qemu_deviceadd($vmid, $devicefull);
         if(!qemu_deviceaddverify($vmid, $deviceid)) {
            qemu_drivedel($vmid, $deviceid);
@@ -2242,17 +2461,19 @@ sub vm_deviceplug {
         }
     }
 
-    if ($deviceid =~ m/^(lsi)(\d+)$/) {
+    if ($deviceid =~ m/^(scsihw)(\d+)$/) {
+        my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
         my $pciaddr = print_pci_addr($deviceid);
-        my $devicefull = "lsi,id=$deviceid$pciaddr";
+        my $devicefull = "$scsihw,id=$deviceid$pciaddr";
         qemu_deviceadd($vmid, $devicefull);
         return undef if(!qemu_deviceaddverify($vmid, $deviceid));
     }
 
     if ($deviceid =~ m/^(scsi)(\d+)$/) {
-        return undef if !qemu_findorcreatelsi($storecfg,$conf, $vmid, $device);
+        return 1 if ($conf->{scsihw} && $conf->{scsihw} ne 'lsi'); #virtio-scsi not yet support hotplug
+        return undef if !qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
         return undef if !qemu_driveadd($storecfg, $vmid, $device);
-        my $devicefull = print_drivedevice_full($storecfg, $vmid, $device);
+        my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
         if(!qemu_deviceadd($vmid, $devicefull)) {
            qemu_drivedel($vmid, $deviceid);
            return undef;
@@ -2269,7 +2490,15 @@ sub vm_deviceplug {
         }
     }
 
-    return 1;
+    if ($deviceid =~ m/^(pci\.)(\d+)$/) {
+	my $bridgeid = $2;
+	my $pciaddr = print_pci_addr($deviceid);
+	my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
+	qemu_deviceadd($vmid, $devicefull);
+	return undef if !qemu_deviceaddverify($vmid, $deviceid);
+    }
+
+    return 1;
 }
 
 sub vm_deviceunplug {
@@ -2309,7 +2538,7 @@ sub vm_deviceunplug {
 sub qemu_deviceadd {
     my ($vmid, $devicefull) = @_;
 
-    my $ret = vm_monitor_command($vmid, "device_add $devicefull");
+    my $ret = vm_human_monitor_command($vmid, "device_add $devicefull");
     $ret =~ s/^\s+//;
     # Otherwise, if the command succeeds, no output is sent. So any non-empty string shows an error
     return 1 if $ret eq "";
@@ -2321,7 +2550,7 @@ sub qemu_deviceadd {
 sub qemu_devicedel {
     my($vmid, $deviceid) = @_;
 
-    my $ret = vm_monitor_command($vmid, "device_del $deviceid");
+    my $ret = vm_human_monitor_command($vmid, "device_del $deviceid");
     $ret =~ s/^\s+//;
     return 1 if $ret eq "";
     syslog("err", "detaching device $deviceid failed : $ret");
@@ -2332,7 +2561,7 @@ sub qemu_driveadd {
     my($storecfg, $vmid, $device) = @_;
 
     my $drive = print_drive_full($storecfg, $vmid, $device);
-    my $ret = vm_monitor_command($vmid, "drive_add auto $drive");
+    my $ret = vm_human_monitor_command($vmid, "drive_add auto $drive");
     # If the command succeeds qemu prints: "OK"
     if ($ret !~ m/OK/s) {
         syslog("err", "adding drive failed: $ret");
@@ -2344,7 +2573,7 @@ sub qemu_driveadd {
 sub qemu_drivedel {
     my($vmid, $deviceid) = @_;
 
-    my $ret = vm_monitor_command($vmid, "drive_del drive-$deviceid");
+    my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
     $ret =~ s/^\s+//;
     if ($ret =~ m/Device \'.*?\' not found/s) {
         # NB: device not found errors mean the drive was auto-deleted and we ignore the error
@@ -2382,16 +2611,36 @@ sub qemu_devicedelverify {
     return undef;
 }
 
-sub qemu_findorcreatelsi {
+sub qemu_findorcreatescsihw {
     my ($storecfg, $conf, $vmid, $device) = @_;
 
-    my $maxdev = 7;
+    my $maxdev = ($conf->{scsihw} && $conf->{scsihw} ne 'lsi') ? 256 : 7;
     my $controller = int($device->{index} / $maxdev);
-    my $lsiid="lsi$controller";
+    my $scsihwid="scsihw$controller";
+    my $devices_list = vm_devices_list($vmid);
+
+    if(!defined($devices_list->{$scsihwid})) {
+       return undef if !vm_deviceplug($storecfg, $conf, $vmid, $scsihwid);
+    }
+    return 1;
+}
+
+sub qemu_bridgeadd {
+    my ($storecfg, $conf, $vmid, $device) = @_;
+
+    my $bridges = {};
+    my $bridgeid = undef;
+    print_pci_addr($device, $bridges);
+
+    while (my ($k, $v) = each %$bridges) {
+	$bridgeid = $k;
+    }
+    return if $bridgeid < 1;
+    my $bridge = "pci.$bridgeid";
     my $devices_list = vm_devices_list($vmid);
 
-    if(!defined($devices_list->{$lsiid})) {
-       return undef if !vm_deviceplug($storecfg, $conf, $vmid, $lsiid);
+    if(!defined($devices_list->{$bridge})) {
+	return undef if !vm_deviceplug($storecfg, $conf, $vmid, $bridge);
     }
     return 1;
 }
@@ -2400,10 +2649,10 @@ sub qemu_netdevadd {
     my ($vmid, $conf, $device, $deviceid) = @_;
 
     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid);
-    my $ret = vm_monitor_command($vmid, "netdev_add $netdev");
+    my $ret = vm_human_monitor_command($vmid, "netdev_add $netdev");
     $ret =~ s/^\s+//;
 
-    #if the command succeeds, no output is sent. So any non-empty string shows an error 
+    #if the command succeeds, no output is sent. So any non-empty string shows an error
     return 1 if $ret eq "";
     syslog("err", "adding netdev failed: $ret");
     return undef;
@@ -2412,95 +2661,31 @@ sub qemu_netdevadd {
 sub qemu_netdevdel {
     my ($vmid, $deviceid) = @_;
 
-    my $ret = vm_monitor_command($vmid, "netdev_del $deviceid");
+    my $ret = vm_human_monitor_command($vmid, "netdev_del $deviceid");
     $ret =~ s/^\s+//;
-    #if the command succeeds, no output is sent. So any non-empty string shows an error 
+    #if the command succeeds, no output is sent. So any non-empty string shows an error
     return 1 if $ret eq "";
     syslog("err", "deleting netdev failed: $ret");
     return undef;
 }
 
-sub vm_start {
-    my ($storecfg, $vmid, $statefile, $skiplock) = @_;
-
-    lock_config($vmid, sub {
-	my $conf = load_config($vmid);
-
-	check_lock($conf) if !$skiplock;
-
-	die "VM $vmid already running\n" if check_running($vmid);
-
-	my $migrate_uri;
-	my $migrate_port = 0;
-
-	if ($statefile) {
-	    if ($statefile eq 'tcp') {
-		$migrate_port = next_migrate_port();
-		$migrate_uri = "tcp:localhost:${migrate_port}";
-	    } else {
-		if (-f $statefile) {
-		    $migrate_uri = "exec:cat $statefile";
-		} else {
-		    warn "state file '$statefile' does not exist - doing normal startup\n";
-		}
-	    }
-	}
-
-	my $defaults = load_defaults();
-
-	my ($cmd, $vollist) = config_to_command($storecfg, $vmid, $conf, $defaults, $migrate_uri);
-	# host pci devices
-        for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
-          my $d = parse_hostpci($conf->{"hostpci$i"});
-          next if !$d;
-          my $info = pci_device_info("0000:$d->{pciid}");
-          die "IOMMU not present\n" if !check_iommu_support();
-          die "no pci device info for device '$d->{pciid}'\n" if !$info;
-          die "can't unbind pci device '$d->{pciid}'\n" if !pci_dev_bind_to_stub($info);
-          die "can't reset pci device '$d->{pciid}'\n" if !pci_dev_reset($info);
-        }
+sub qemu_block_set_io_throttle {
+    my ($vmid, $deviceid, $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr) = @_;
 
-	PVE::Storage::activate_volumes($storecfg, $vollist);
+    return if !check_running($vmid) ;
 
-	eval  { run_command($cmd, timeout => $migrate_uri ? undef : 30); };
-	my $err = $@;
-	die "start failed: $err" if $err;
+    $bps = 0 if !$bps;
+    $bps_rd = 0 if !$bps_rd;
+    $bps_wr = 0 if !$bps_wr;
+    $iops = 0 if !$iops;
+    $iops_rd = 0 if !$iops_rd;
+    $iops_wr = 0 if !$iops_wr;
 
-	if ($statefile) {
+    vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid, bps => int($bps), bps_rd => int($bps_rd), bps_wr => int($bps_wr), iops => int($iops), iops_rd => int($iops_rd), iops_wr => int($iops_wr));
 
-	    if ($statefile eq 'tcp') {
-		print "migration listens on port $migrate_port\n";
-	    } else {
-		unlink $statefile;
-		# fixme: send resume - is that necessary ?
-		eval { vm_monitor_command($vmid, "cont"); };
-	    }
-	}
-
-	# always set migrate speed (overwrite kvm default of 32m)
-	# we set a very hight default of 8192m which is basically unlimited
-	my $migrate_speed = $defaults->{migrate_speed} || 8192;
-	$migrate_speed = $conf->{migrate_speed} || $migrate_speed;
-	eval {
-	    my $cmd = "migrate_set_speed ${migrate_speed}m";
-	    vm_monitor_command($vmid, $cmd);
-	};
-
-	if (my $migrate_downtime =
-	    $conf->{migrate_downtime} || $defaults->{migrate_downtime}) {
-	    my $cmd = "migrate_set_downtime ${migrate_downtime}";
-	    eval { vm_monitor_command($vmid, $cmd); };
-	}
-
-	vm_balloonset($vmid, $conf->{balloon}) if $conf->{balloon};
-
-	my $tablet = defined($conf->{tablet}) ? $conf->{tablet} : $defaults->{tablet};
-	PVE::QemuServer::vm_monitor_command($vmid, "mouse_set 1", 0) if $tablet; #ensure usb tablet is activated
-
-
-    });
 }
 
+# old code, only used to shutdown old VM after update
 sub __read_avail {
     my ($fh, $timeout) = @_;
 
@@ -2533,6 +2718,7 @@ sub __read_avail {
     return $res;
 }
 
+# old code, only used to shutdown old VM after update
 sub vm_monitor_command {
     my ($vmid, $cmdstr, $nocheck) = @_;
 
@@ -2541,7 +2727,7 @@ sub vm_monitor_command {
     eval {
 	die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
 
-	my $sname = monitor_socket($vmid);
+	my $sname = "${var_run_tmpdir}/$vmid.mon";
 
 	my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
 	    die "unable to connect to VM $vmid socket - $!\n";
@@ -2607,6 +2793,206 @@ sub vm_monitor_command {
     return $res;
 }
 
+sub qemu_block_resize {
+    my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
+
+    my $running = PVE::QemuServer::check_running($vmid);
+
+    return if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
+
+    return if !$running;
+
+    vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
+
+}
+
+sub qemu_volume_snapshot {
+    my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
+
+    my $running = PVE::QemuServer::check_running($vmid);
+
+    return if !PVE::Storage::volume_snapshot($storecfg, $volid, $snap, $running);
+
+    return if !$running;
+
+    vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
+
+}
+
+sub qemu_volume_snapshot_delete {
+    my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
+
+    my $running = PVE::QemuServer::check_running($vmid);
+
+    return if !PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
+
+    return if !$running;
+
+    vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
+}
+
+sub qga_freezefs {
+    my ($vmid) = @_;
+
+    #need to impplement call to qemu-ga
+}
+
+sub qga_unfreezefs {
+    my ($vmid) = @_;
+
+    #need to impplement call to qemu-ga
+}
+
+sub vm_start {
+    my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom) = @_;
+
+    lock_config($vmid, sub {
+	my $conf = load_config($vmid, $migratedfrom);
+
+	check_lock($conf) if !$skiplock;
+
+	die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
+
+	my $migrate_uri;
+	my $migrate_port = 0;
+
+	if ($statefile) {
+	    if ($statefile eq 'tcp') {
+		$migrate_port = next_migrate_port();
+		$migrate_uri = "tcp:localhost:${migrate_port}";
+	    } else {
+		if (-f $statefile) {
+		    $migrate_uri = "exec:cat $statefile";
+		} else {
+		    warn "state file '$statefile' does not exist - doing normal startup\n";
+		}
+	    }
+	}
+
+	my $defaults = load_defaults();
+
+	# set environment variable useful inside network script
+	$ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
+
+	my ($cmd, $vollist) = config_to_command($storecfg, $vmid, $conf, $defaults, $migrate_uri);
+	# host pci devices
+        for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
+          my $d = parse_hostpci($conf->{"hostpci$i"});
+          next if !$d;
+          my $info = pci_device_info("0000:$d->{pciid}");
+          die "IOMMU not present\n" if !check_iommu_support();
+          die "no pci device info for device '$d->{pciid}'\n" if !$info;
+          die "can't unbind pci device '$d->{pciid}'\n" if !pci_dev_bind_to_stub($info);
+          die "can't reset pci device '$d->{pciid}'\n" if !pci_dev_reset($info);
+        }
+
+	PVE::Storage::activate_volumes($storecfg, $vollist);
+
+	eval  { run_command($cmd, timeout => $migrate_uri ? undef : 30); };
+	my $err = $@;
+	die "start failed: $err" if $err;
+
+	if ($statefile) {
+
+	    if ($statefile eq 'tcp') {
+		print "migration listens on port $migrate_port\n";
+	    } else {
+		if ($migratedfrom) {
+		    unlink $statefile;
+		    # fixme: send resume - is that necessary ?
+		    eval { vm_mon_cmd($vmid, "cont"); };
+		}
+	    }
+	}
+
+	# always set migrate speed (overwrite kvm default of 32m)
+	# we set a very hight default of 8192m which is basically unlimited
+	my $migrate_speed = $defaults->{migrate_speed} || 8192;
+	$migrate_speed = $conf->{migrate_speed} || $migrate_speed;
+	$migrate_speed = $migrate_speed * 1048576;
+	eval {
+	    vm_mon_cmd($vmid, "migrate_set_speed", value => $migrate_speed);
+	};
+
+	my $migrate_downtime = $defaults->{migrate_downtime};
+	$migrate_downtime = $conf->{migrate_downtime} if defined($conf->{migrate_downtime});
+	if (defined($migrate_downtime)) {
+	    eval { vm_mon_cmd($vmid, "migrate_set_downtime", value => $migrate_downtime); };
+	}
+
+	if($migratedfrom) {
+	    my $capabilities = {};
+	    $capabilities->{capability} =  "xbzrle";
+	    $capabilities->{state} = JSON::true;
+	    eval { PVE::QemuServer::vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => [$capabilities]); };
+	}
+
+	vm_balloonset($vmid, $conf->{balloon}) if $conf->{balloon};
+
+    });
+}
+
+sub vm_mon_cmd {
+    my ($vmid, $execute, %params) = @_;
+
+    my $cmd = { execute => $execute, arguments => \%params };
+    vm_qmp_command($vmid, $cmd);
+}
+
+sub vm_mon_cmd_nocheck {
+    my ($vmid, $execute, %params) = @_;
+
+    my $cmd = { execute => $execute, arguments => \%params };
+    vm_qmp_command($vmid, $cmd, 1);
+}
+
+sub vm_qmp_command {
+    my ($vmid, $cmd, $nocheck) = @_;
+
+    my $res;
+
+    my $timeout;
+    if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
+	$timeout = $cmd->{arguments}->{timeout};
+	delete $cmd->{arguments}->{timeout};
+    }
+ 
+    eval {
+	die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
+	my $sname = PVE::QemuServer::qmp_socket($vmid);
+	if (-e $sname) {
+	    my $qmpclient = PVE::QMPClient->new();
+
+	    $res = $qmpclient->cmd($vmid, $cmd, $timeout);
+	} elsif (-e "${var_run_tmpdir}/$vmid.mon") {
+	    die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
+		if scalar(%{$cmd->{arguments}});
+	    vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
+	} else {
+	    die "unable to open monitor socket\n";
+	}
+    };
+    if (my $err = $@) {
+	syslog("err", "VM $vmid qmp command failed - $err");
+	die $err;
+    }
+
+    return $res;
+}
+
+sub vm_human_monitor_command {
+    my ($vmid, $cmdline) = @_;
+
+    my $res;
+
+    my $cmd = {
+	execute => 'human-monitor-command',
+	arguments => { 'command-line' => $cmdline},
+    };
+
+    return vm_qmp_command($vmid, $cmd);
+}
+
 sub vm_commandline {
     my ($storecfg, $vmid) = @_;
 
@@ -2628,7 +3014,7 @@ sub vm_reset {
 
 	check_lock($conf) if !$skiplock;
 
-	vm_monitor_command($vmid, "system_reset");
+	vm_mon_cmd($vmid, "system_reset");
     });
 }
 
@@ -2661,6 +3047,10 @@ sub vm_stop_cleanup {
 	    my $vollist = get_vm_volumes($conf);
 	    PVE::Storage::deactivate_volumes($storecfg, $vollist);
 	}
+
+	foreach my $ext (qw(mon qmp pid vnc qga)) {
+	    unlink "/var/run/qemu-server/${vmid}.$ext";
+	}
     };
     warn $@ if $@; # avoid errors - just warn
 }
@@ -2669,12 +3059,18 @@ sub vm_stop_cleanup {
 # We need that when migration VMs to other nodes (files already moved)
 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
 sub vm_stop {
-    my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
-
-    $timeout = 60 if !defined($timeout);
+    my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
 
     $force = 1 if !defined($force) && !$shutdown;
 
+    if ($migratedfrom){
+	my $pid = check_running($vmid, $nocheck, $migratedfrom);
+	kill 15, $pid if $pid;
+	my $conf = load_config($vmid, $migratedfrom);
+	vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive);
+	return;
+    }
+
     lock_config($vmid, sub {
 
 	my $pid = check_running($vmid, $nocheck);
@@ -2684,13 +3080,20 @@ sub vm_stop {
 	if (!$nocheck) {
 	    $conf = load_config($vmid);
 	    check_lock($conf) if !$skiplock;
+	    if (!defined($timeout) && $shutdown && $conf->{startup}) {
+		my $opts = parse_startup($conf->{startup});
+		$timeout = $opts->{down} if $opts->{down};
+	    }
 	}
 
+	$timeout = 60 if !defined($timeout);
+
 	eval {
 	    if ($shutdown) {
-		vm_monitor_command($vmid, "system_powerdown", $nocheck);
+		$nocheck ? vm_mon_cmd_nocheck($vmid, "system_powerdown") : vm_mon_cmd($vmid, "system_powerdown");
+
 	    } else {
-		vm_monitor_command($vmid, "quit", $nocheck);
+		$nocheck ? vm_mon_cmd_nocheck($vmid, "quit") : vm_mon_cmd($vmid, "quit");
 	    }
 	};
 	my $err = $@;
@@ -2750,7 +3153,7 @@ sub vm_suspend {
 
 	check_lock($conf) if !$skiplock;
 
-	vm_monitor_command($vmid, "stop");
+	vm_mon_cmd($vmid, "stop");
     });
 }
 
@@ -2763,7 +3166,7 @@ sub vm_resume {
 
 	check_lock($conf) if !$skiplock;
 
-	vm_monitor_command($vmid, "cont");
+	vm_mon_cmd($vmid, "cont");
     });
 }
 
@@ -2774,7 +3177,8 @@ sub vm_sendkey {
 
 	my $conf = load_config($vmid);
 
-	vm_monitor_command($vmid, "sendkey $key");
+	# there is no qmp command, so we use the human monitor command
+	vm_human_monitor_command($vmid, "sendkey $key");
     });
 }
 
@@ -2796,105 +3200,6 @@ sub vm_destroy {
     });
 }
 
-sub vm_stopall {
-    my ($storecfg, $timeout) = @_;
-
-    $timeout = 3*60 if !$timeout;
-
-    my $cleanuphash = {};
-
-    my $vzlist = vzlist();
-    my $count = 0;
-    foreach my $vmid (keys %$vzlist) {
-	next if !$vzlist->{$vmid}->{pid};
-	$count++;
-	$cleanuphash->{$vmid} = 1;
-    }
-
-    return if !$count;
-
-    my $msg = "Stopping Qemu Server - sending shutdown requests to all VMs\n";
-    syslog('info', $msg);
-    warn $msg;
-
-    foreach my $vmid (keys %$vzlist) {
-	next if !$vzlist->{$vmid}->{pid};
-	eval { vm_monitor_command($vmid, "system_powerdown"); };
-	warn $@ if $@;
-    }
-
-    my $wt = 5;
-    my $maxtries = int(($timeout + $wt -1)/$wt);
-    my $try = 0;
-    while (($try < $maxtries) && $count) {
-	$try++;
-	sleep $wt;
-
-	$vzlist = vzlist();
-	$count = 0;
-	foreach my $vmid (keys %$vzlist) {
-	    next if !$vzlist->{$vmid}->{pid};
-	    $count++;
-	}
-	last if !$count;
-    }
-
-    if ($count) {
-
-	foreach my $vmid (keys %$vzlist) {
-	    next if !$vzlist->{$vmid}->{pid};
-
-	    warn "VM $vmid still running - sending stop now\n";
-	    eval { vm_monitor_command($vmid, "quit"); };
-	    warn $@ if $@;
-	}
-
-	$timeout = 30;
-	$maxtries = int(($timeout + $wt -1)/$wt);
-	$try = 0;
-	while (($try < $maxtries) && $count) {
-	    $try++;
-	    sleep $wt;
-
-	    $vzlist = vzlist();
-	    $count = 0;
-	    foreach my $vmid (keys %$vzlist) {
-		next if !$vzlist->{$vmid}->{pid};
-		$count++;
-	    }
-	    last if !$count;
-	}
-
-	if ($count) {
-
-	    foreach my $vmid (keys %$vzlist) {
-		next if !$vzlist->{$vmid}->{pid};
-
-		warn "VM $vmid still running - terminating now with SIGTERM\n";
-		kill 15, $vzlist->{$vmid}->{pid};
-	    }
-	    sleep 1;
-	}
-
-	# this is called by system shotdown scripts, so remaining
-	# processes gets killed anyways (no need to send kill -9 here)
-    }
-
-    $vzlist = vzlist();
-    foreach my $vmid (keys %$cleanuphash) {
-	next if $vzlist->{$vmid}->{pid};
-	eval {
-	    my $conf = load_config($vmid);
-	    vm_stop_cleanup($storecfg, $vmid, $conf);
-	};
-	warn $@ if $@;
-    }
-
-    $msg = "Qemu Server stopped\n";
-    syslog('info', $msg);
-    print $msg;
-}
-
 # pci helpers
 
 sub file_write {
@@ -2977,7 +3282,7 @@ sub pci_dev_bind_to_stub {
 }
 
 sub print_pci_addr {
-    my ($id) = @_;
+    my ($id, $bridges) = @_;
 
     my $res = '';
     my $devices = {
@@ -2985,9 +3290,10 @@ sub print_pci_addr {
 	#addr2 : first videocard
 	balloon0 => { bus => 0, addr => 3 },
 	watchdog => { bus => 0, addr => 4 },
-	lsi0 => { bus => 0, addr => 5 },
-	lsi1 => { bus => 0, addr => 6 },
+	scsihw0 => { bus => 0, addr => 5 },
+	scsihw1 => { bus => 0, addr => 6 },
 	ahci0 => { bus => 0, addr => 7 },
+	qga0 => { bus => 0, addr => 8 },
 	virtio0 => { bus => 0, addr => 10 },
 	virtio1 => { bus => 0, addr => 11 },
 	virtio2 => { bus => 0, addr => 12 },
@@ -3003,11 +3309,51 @@ sub print_pci_addr {
 	net4 => { bus => 0, addr => 22 },
 	net5 => { bus => 0, addr => 23 },
 	#addr29 : usb-host (pve-usb.cfg)
+	'pci.1' => { bus => 0, addr => 30 },
+	'pci.2' => { bus => 0, addr => 31 },
+	'net6' => { bus => 1, addr => 1 },
+	'net7' => { bus => 1, addr => 2 },
+	'net8' => { bus => 1, addr => 3 },
+	'net9' => { bus => 1, addr => 4 },
+	'net10' => { bus => 1, addr => 5 },
+	'net11' => { bus => 1, addr => 6 },
+	'net12' => { bus => 1, addr => 7 },
+	'net13' => { bus => 1, addr => 8 },
+	'net14' => { bus => 1, addr => 9 },
+	'net15' => { bus => 1, addr => 10 },
+	'net16' => { bus => 1, addr => 11 },
+	'net17' => { bus => 1, addr => 12 },
+	'net18' => { bus => 1, addr => 13 },
+	'net19' => { bus => 1, addr => 14 },
+	'net20' => { bus => 1, addr => 15 },
+	'net21' => { bus => 1, addr => 16 },
+	'net22' => { bus => 1, addr => 17 },
+	'net23' => { bus => 1, addr => 18 },
+	'net24' => { bus => 1, addr => 19 },
+	'net25' => { bus => 1, addr => 20 },
+	'net26' => { bus => 1, addr => 21 },
+	'net27' => { bus => 1, addr => 22 },
+	'net28' => { bus => 1, addr => 23 },
+	'net29' => { bus => 1, addr => 24 },
+	'net30' => { bus => 1, addr => 25 },
+	'net31' => { bus => 1, addr => 26 },
+	'virtio6' => { bus => 2, addr => 1 },
+	'virtio7' => { bus => 2, addr => 2 },
+	'virtio8' => { bus => 2, addr => 3 },
+	'virtio9' => { bus => 2, addr => 4 },
+	'virtio10' => { bus => 2, addr => 5 },
+	'virtio11' => { bus => 2, addr => 6 },
+	'virtio12' => { bus => 2, addr => 7 },
+	'virtio13' => { bus => 2, addr => 8 },
+	'virtio14' => { bus => 2, addr => 9 },
+	'virtio15' => { bus => 2, addr => 10 },
     };
 
     if (defined($devices->{$id}->{bus}) && defined($devices->{$id}->{addr})) {
 	   my $addr = sprintf("0x%x", $devices->{$id}->{addr});
-	   $res = ",bus=pci.$devices->{$id}->{bus},addr=$addr";
+	   my $bus = $devices->{$id}->{bus};
+	   $res = ",bus=pci.$bus,addr=$addr";
+	   $bridges->{$bus} = 1 if $bridges;
     }
     return $res;
 
@@ -3016,7 +3362,7 @@ sub print_pci_addr {
 sub vm_balloonset {
     my ($vmid, $value) = @_;
 
-    vm_monitor_command($vmid, "balloon $value");
+    vm_mon_cmd($vmid, "balloon", value => $value);
 }
 
 # vzdump restore implementaion
@@ -3206,4 +3552,431 @@ sub restore_archive {
 	die "unable to commit configuration file '$conffile'\n";
 };
 
+
+# Internal snapshots
+
+# NOTE: Snapshot create/delete involves several non-atomic
+# action, and can take a long time.
+# So we try to avoid locking the file and use 'lock' variable
+# inside the config file instead.
+
+my $snapshot_copy_config = sub {
+    my ($source, $dest) = @_;
+
+    foreach my $k (keys %$source) {
+	next if $k eq 'snapshots';
+	next if $k eq 'snapstate';
+	next if $k eq 'snaptime';
+	next if $k eq 'vmstate';
+	next if $k eq 'lock';
+	next if $k eq 'digest';
+	next if $k eq 'description';
+	next if $k =~ m/^unused\d+$/;
+		
+	$dest->{$k} = $source->{$k};
+    }
+};
+
+my $snapshot_apply_config = sub {
+    my ($conf, $snap) = @_;
+
+    # copy snapshot list
+    my $newconf = {
+	snapshots => $conf->{snapshots},
+    };
+
+    # keep description and list of unused disks
+    foreach my $k (keys %$conf) {
+	next if !($k =~ m/^unused\d+$/ || $k eq 'description');
+	$newconf->{$k} = $conf->{$k};
+    }
+
+    &$snapshot_copy_config($snap, $newconf);
+
+    return $newconf;
+};
+
+sub foreach_writable_storage {
+    my ($conf, $func) = @_;
+
+    my $sidhash = {};
+
+    foreach my $ds (keys %$conf) {
+	next if !valid_drivename($ds);
+
+	my $drive = parse_drive($ds, $conf->{$ds});
+	next if !$drive;
+	next if drive_is_cdrom($drive);
+
+	my $volid = $drive->{file};
+
+	my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
+	$sidhash->{$sid} = $sid if $sid;	
+    }
+
+    foreach my $sid (sort keys %$sidhash) {
+	&$func($sid);
+    }
+}
+
+my $alloc_vmstate_volid = sub {
+    my ($storecfg, $vmid, $conf, $snapname) = @_;
+    
+    # Note: we try to be smart when selecting a $target storage
+
+    my $target;
+
+    # search shared storage first
+    foreach_writable_storage($conf, sub {
+	my ($sid) = @_;
+	my $scfg = PVE::Storage::storage_config($storecfg, $sid);
+	return if !$scfg->{shared};
+
+	$target = $sid if !$target || $scfg->{path}; # prefer file based storage
+    });
+
+    if (!$target) {
+	# now search local storage
+	foreach_writable_storage($conf, sub {
+	    my ($sid) = @_;
+	    my $scfg = PVE::Storage::storage_config($storecfg, $sid);
+	    return if $scfg->{shared};
+
+	    $target = $sid if !$target || $scfg->{path}; # prefer file based storage;
+	});
+    }
+
+    $target = 'local' if !$target;
+
+    my $driver_state_size = 32; # assume 32MB is enough to safe all driver state;
+    my $size = $conf->{memory} + $driver_state_size;
+
+    my $name = "vm-$vmid-state-$snapname";
+    my $scfg = PVE::Storage::storage_config($storecfg, $target);
+    $name .= ".raw" if $scfg->{path}; # add filename extension for file base storage
+    my $volid = PVE::Storage::vdisk_alloc($storecfg, $target, $vmid, 'raw', $name, $size*1024);
+
+    return $volid;
+};
+
+my $snapshot_prepare = sub {
+    my ($vmid, $snapname, $save_vmstate, $comment) = @_;
+
+    my $snap;
+
+    my $updatefn =  sub {
+
+	my $conf = load_config($vmid);
+
+	check_lock($conf);
+
+	$conf->{lock} = 'snapshot';
+
+	die "snapshot name '$snapname' already used\n" 
+	    if defined($conf->{snapshots}->{$snapname}); 
+
+	my $storecfg = PVE::Storage::config();
+
+	foreach_drive($conf, sub {
+	    my ($ds, $drive) = @_;
+
+	    return if drive_is_cdrom($drive);
+	    my $volid = $drive->{file};
+
+	    my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
+	    if ($storeid) {
+		my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
+		die "can't snapshot volume '$volid'\n"		
+		    if !(($scfg->{path} && $volname =~ m/\.qcow2$/) ||
+			 ($scfg->{type} eq 'rbd') || 
+			 ($scfg->{type} eq 'sheepdog'));
+	    } elsif ($volid =~ m|^(/.+)$| && -e $volid) {
+		die "snapshot device '$volid' is not possible\n";
+	    } else {
+		die "can't snapshot volume '$volid'\n";
+	    }
+	});
+
+
+	$snap = $conf->{snapshots}->{$snapname} = {};
+
+	if ($save_vmstate && check_running($vmid)) {
+	    $snap->{vmstate} = &$alloc_vmstate_volid($storecfg, $vmid, $conf, $snapname);
+	}
+
+	&$snapshot_copy_config($conf, $snap);
+
+	$snap->{snapstate} = "prepare";
+	$snap->{snaptime} = time();
+	$snap->{description} = $comment if $comment;
+
+	update_config_nolock($vmid, $conf, 1);
+    };
+
+    lock_config($vmid, $updatefn);
+
+    return $snap;
+};
+
+my $snapshot_commit = sub {
+    my ($vmid, $snapname) = @_;
+
+    my $updatefn = sub {
+
+	my $conf = load_config($vmid);
+
+	die "missing snapshot lock\n" 
+	    if !($conf->{lock} && $conf->{lock} eq 'snapshot'); 
+
+	my $snap = $conf->{snapshots}->{$snapname};
+
+	die "snapshot '$snapname' does not exist\n" if !defined($snap); 
+
+	die "wrong snapshot state\n" 
+	    if !($snap->{snapstate} && $snap->{snapstate} eq "prepare"); 
+	
+	delete $snap->{snapstate};
+	delete $conf->{lock};
+
+	my $newconf = &$snapshot_apply_config($conf, $snap);
+
+	$newconf->{parent} = $snapname;
+
+	update_config_nolock($vmid, $newconf, 1);
+    };
+
+    lock_config($vmid, $updatefn);
+};
+
+sub snapshot_rollback {
+    my ($vmid, $snapname) = @_;
+
+    my $snap;
+
+    my $prepare = 1;
+
+    my $storecfg = PVE::Storage::config();
+ 
+    my $updatefn = sub {
+
+	my $conf = load_config($vmid);
+
+	if ($prepare) {
+	    check_lock($conf);
+	    vm_stop($storecfg, $vmid, undef, undef, 5, undef, undef);
+	}
+
+	die "unable to rollback vm $vmid: vm is running\n"
+	    if check_running($vmid);
+
+	if ($prepare) {
+	    $conf->{lock} = 'rollback';
+	} else {
+	    die "got wrong lock\n" if !($conf->{lock} && $conf->{lock} eq 'rollback');
+	    delete $conf->{lock};
+	}
+
+	$snap = $conf->{snapshots}->{$snapname};
+
+	die "snapshot '$snapname' does not exist\n" if !defined($snap); 
+
+	die "unable to rollback to incomplete snapshot (snapstate = $snap->{snapstate})\n" 
+	    if $snap->{snapstate};
+
+	if (!$prepare) {
+	    # copy snapshot config to current config
+	    $conf = &$snapshot_apply_config($conf, $snap);
+	    $conf->{parent} = $snapname;
+	}
+
+ 	update_config_nolock($vmid, $conf, 1);
+
+	if (!$prepare && $snap->{vmstate}) {
+	    my $statefile = PVE::Storage::path($storecfg, $snap->{vmstate});
+	    # fixme: this only forws for files currently
+	    vm_start($storecfg, $vmid, $statefile);
+	}
+
+    };
+
+    lock_config($vmid, $updatefn);
+    
+    foreach_drive($snap, sub {
+	my ($ds, $drive) = @_;
+
+	return if drive_is_cdrom($drive);
+
+	my $volid = $drive->{file};
+	my $device = "drive-$ds";
+
+	PVE::Storage::volume_snapshot_rollback($storecfg, $volid, $snapname);
+    });
+
+    $prepare = 0;
+    lock_config($vmid, $updatefn);
+}
+
+sub snapshot_create {
+    my ($vmid, $snapname, $save_vmstate, $freezefs, $comment) = @_;
+
+    my $snap = &$snapshot_prepare($vmid, $snapname, $save_vmstate, $comment);
+
+    $freezefs = $save_vmstate = 0 if !$snap->{vmstate}; # vm is not running
+
+    my $drivehash = {};
+
+    my $running = check_running($vmid);
+
+    eval {
+	# create internal snapshots of all drives
+
+	my $storecfg = PVE::Storage::config();
+
+	if ($running) {
+	    if ($snap->{vmstate}) {
+		my $path = PVE::Storage::path($storecfg, $snap->{vmstate});	
+		vm_mon_cmd($vmid, "snapshot-start", statefile => $path);
+	    } else {
+		vm_mon_cmd($vmid, "snapshot-start");
+ 	    }
+	};
+
+	qga_freezefs($vmid) if $running && $freezefs;
+ 
+	foreach_drive($snap, sub {
+	    my ($ds, $drive) = @_;
+
+	    return if drive_is_cdrom($drive);
+
+	    my $volid = $drive->{file};
+	    my $device = "drive-$ds";
+
+	    qemu_volume_snapshot($vmid, $device, $storecfg, $volid, $snapname);
+	    $drivehash->{$ds} = 1;
+       });
+    };
+    my $err = $@;
+
+    eval { gqa_unfreezefs($vmid) if $running && $freezefs; };
+    warn $@ if $@;
+
+    eval { vm_mon_cmd($vmid, "snapshot-end") if $running; };
+    warn $@ if $@;
+
+    if ($err) {
+	warn "snapshot create failed: starting cleanup\n";
+	eval { snapshot_delete($vmid, $snapname, 0, $drivehash); };
+	warn $@ if $@;
+	die $err;
+    }
+
+    &$snapshot_commit($vmid, $snapname);
+}
+
+# Note: $drivehash is only set when called from snapshot_create.
+sub snapshot_delete {
+    my ($vmid, $snapname, $force, $drivehash) = @_;
+
+    my $prepare = 1;
+
+    my $snap;
+    my $unused = [];
+
+    my $unlink_parent = sub {
+	my ($confref, $new_parent) = @_;
+
+	if ($confref->{parent} && $confref->{parent} eq $snapname) {
+	    if ($new_parent) {
+		$confref->{parent} = $new_parent;
+	    } else {
+		delete $confref->{parent};
+	    }
+	}
+    };
+ 
+    my $updatefn =  sub {
+	my ($remove_drive) = @_;
+
+	my $conf = load_config($vmid);
+
+	check_lock($conf) if !$drivehash;
+
+	$snap = $conf->{snapshots}->{$snapname};
+
+	die "snapshot '$snapname' does not exist\n" if !defined($snap); 
+
+	# remove parent refs
+	&$unlink_parent($conf, $snap->{parent});
+	foreach my $sn (keys %{$conf->{snapshots}}) {
+	    next if $sn eq $snapname;
+	    &$unlink_parent($conf->{snapshots}->{$sn}, $snap->{parent});
+	}
+
+	if ($remove_drive) {
+	    if ($remove_drive eq 'vmstate') {
+		delete $snap->{$remove_drive};
+	    } else {
+		my $drive = parse_drive($remove_drive, $snap->{$remove_drive});
+		my $volid = $drive->{file};
+		delete $snap->{$remove_drive};
+		add_unused_volume($conf, $volid);
+	    }
+	}
+
+	if ($prepare) {
+	    $snap->{snapstate} = 'delete';
+	} else {
+	    delete $conf->{snapshots}->{$snapname};
+	    delete $conf->{lock} if $drivehash;
+	    foreach my $volid (@$unused) {
+		add_unused_volume($conf, $volid);
+	    }
+	}
+
+	update_config_nolock($vmid, $conf, 1);
+    };
+
+    lock_config($vmid, $updatefn);
+
+    # now remove vmstate file
+
+    my $storecfg = PVE::Storage::config();
+
+    if ($snap->{vmstate}) {
+	eval {  PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
+	if (my $err = $@) {
+	    die $err if !$force;
+	    warn $err;
+	}
+	# save changes (remove vmstate from snapshot)
+	lock_config($vmid, $updatefn, 'vmstate') if !$force;
+    };
+
+    # now remove all internal snapshots
+    foreach_drive($snap, sub {
+	my ($ds, $drive) = @_;
+
+	return if drive_is_cdrom($drive);
+
+	my $volid = $drive->{file};
+	my $device = "drive-$ds";
+
+	if (!$drivehash || $drivehash->{$ds}) {
+	    eval { qemu_volume_snapshot_delete($vmid, $device, $storecfg, $volid, $snapname); };
+	    if (my $err = $@) {
+		die $err if !$force;
+		warn $err;
+	    }
+	}
+
+	# save changes (remove drive fron snapshot)
+	lock_config($vmid, $updatefn, $ds) if !$force;
+	push @$unused, $volid;
+    });
+
+    # now cleanup config
+    $prepare = 0;
+    lock_config($vmid, $updatefn);
+}
+
 1;