X-Git-Url: https://git.proxmox.com/?a=blobdiff_plain;f=PVE%2FQemuServer.pm;h=53254a0e82eaae06638035aae6d3b0a995da908c;hb=4fc262bd50abf4b98e247f502564ea85b7b5d79f;hp=2e822f37dae88991aae3fb4509edff9e5bd898f2;hpb=44c2a647ff8fbfb3da1287393eb95633836f6360;p=qemu-server.git

diff --git a/PVE/QemuServer.pm b/PVE/QemuServer.pm
index 2e822f3..53254a0 100644
--- a/PVE/QemuServer.pm
+++ b/PVE/QemuServer.pm
@@ -22,7 +22,7 @@ use PVE::SafeSyslog;
 use Storable qw(dclone);
 use PVE::Exception qw(raise raise_param_exc);
 use PVE::Storage;
-use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
+use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach $IPV6RE);
 use PVE::JSONSchema qw(get_standard_option);
 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
 use PVE::INotify;
@@ -33,17 +33,30 @@ use PVE::RPCEnvironment;
 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
 use PVE::QemuServer::Memory;
 use PVE::QemuServer::USB qw(parse_usb_device);
+use PVE::QemuServer::Cloudinit;
+use PVE::Systemd;
 use Time::HiRes qw(gettimeofday);
 use File::Copy qw(copy);
 use URI::Escape;
 
-my $OVMF_CODE = '/usr/share/kvm/OVMF_CODE-pure-efi.fd';
-my $OVMF_VARS = '/usr/share/kvm/OVMF_VARS-pure-efi.fd';
+my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
+my $OVMF = {
+    x86_64 => [
+	"$EDK2_FW_BASE/OVMF_CODE.fd",
+	"$EDK2_FW_BASE/OVMF_VARS.fd"
+    ],
+    aarch64 => [
+	"$EDK2_FW_BASE/AAVMF_CODE.fd",
+	"$EDK2_FW_BASE/AAVMF_VARS.fd"
+    ],
+};
 
 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
 
 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
 
+my $QEMU_FORMAT_RE = qr/raw|cow|qcow|qcow2|qed|vmdk|cloop/;
+
 # Note about locking: we use flock on the config file protect
 # against concurent actions.
 # Aditionaly, we have a 'lock' setting in the config file. This
@@ -75,6 +88,14 @@ PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
     optional => 1,
 });
 
+PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
+	description => "Specifies the Qemu machine type.",
+	type => 'string',
+	pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?|virt(?:-\d+\.\d+)?)',
+	maxLength => 40,
+	optional => 1,
+});
+
 #no warnings 'redefine';
 
 sub cgroups_write {
@@ -108,17 +129,28 @@ my $cpu_vendor_list = {
     coreduo => 'GenuineIntel',
     core2duo => 'GenuineIntel',
     Conroe  => 'GenuineIntel',
-    Penryn  => 'GenuineIntel', 
+    Penryn  => 'GenuineIntel',
     Nehalem  => 'GenuineIntel',
+    'Nehalem-IBRS'  => 'GenuineIntel',
     Westmere => 'GenuineIntel',
+    'Westmere-IBRS' => 'GenuineIntel',
     SandyBridge => 'GenuineIntel',
+    'SandyBridge-IBRS' => 'GenuineIntel',
     IvyBridge => 'GenuineIntel',
+    'IvyBridge-IBRS' => 'GenuineIntel',
     Haswell => 'GenuineIntel',
+    'Haswell-IBRS' => 'GenuineIntel',
     'Haswell-noTSX' => 'GenuineIntel',
+    'Haswell-noTSX-IBRS' => 'GenuineIntel',
     Broadwell => 'GenuineIntel',
+    'Broadwell-IBRS' => 'GenuineIntel',
     'Broadwell-noTSX' => 'GenuineIntel',
+    'Broadwell-noTSX-IBRS' => 'GenuineIntel',
     'Skylake-Client' => 'GenuineIntel',
-    
+    'Skylake-Client-IBRS' => 'GenuineIntel',
+    'Skylake-Server' => 'GenuineIntel',
+    'Skylake-Server-IBRS' => 'GenuineIntel',
+
     # AMD CPUs
     athlon => 'AuthenticAMD',
     phenom  => 'AuthenticAMD',
@@ -127,6 +159,8 @@ my $cpu_vendor_list = {
     Opteron_G3  => 'AuthenticAMD',
     Opteron_G4  => 'AuthenticAMD',
     Opteron_G5  => 'AuthenticAMD',
+    EPYC => 'AuthenticAMD',
+    'EPYC-IBPB' => 'AuthenticAMD',
 
     # generic types, use vendor from host node
     host => 'default',
@@ -134,8 +168,11 @@ my $cpu_vendor_list = {
     kvm64 => 'default',
     qemu32 => 'default',
     qemu64 => 'default',
+    max => 'default',
 };
 
+my $cpu_flag = qr/[+-](pcid|spec-ctrl|ibpb|ssbd|virt-ssbd|amd-ssbd|amd-no-ssb|pdpe1gb)/;
+
 my $cpu_fmt = {
     cputype => {
 	description => "Emulated CPU type.",
@@ -150,6 +187,15 @@ my $cpu_fmt = {
 	optional => 1,
 	default => 0
     },
+    flags => {
+	description => "List of additional CPU flags separated by ';'."
+		     . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
+		     . " Currently supported flags: 'pcid', 'spec-ctrl', 'ibpb', 'ssbd', 'virt-ssbd', 'amd-ssbd', 'amd-no-ssb', 'pdpe1gb'.",
+	format_description => '+FLAG[;-FLAG...]',
+	type => 'string',
+	pattern => qr/$cpu_flag(;$cpu_flag)*/,
+	optional => 1,
+    },
 };
 
 my $watchdog_fmt = {
@@ -170,6 +216,39 @@ my $watchdog_fmt = {
 };
 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 
+my $agent_fmt = {
+    enabled => {
+	description => "Enable/disable Qemu GuestAgent.",
+	type => 'boolean',
+	default => 0,
+	default_key => 1,
+    },
+    fstrim_cloned_disks => {
+	description => "Run fstrim after cloning/moving a disk.",
+	type => 'boolean',
+	optional => 1,
+	default => 0
+    },
+};
+
+my $vga_fmt = {
+    type => {
+	description => "Select the VGA type.",
+	type => 'string',
+	default => 'std',
+	optional => 1,
+	default_key => 1,
+	enum => [qw(cirrus qxl qxl2 qxl3 qxl4 serial0 serial1 serial2 serial3 std virtio vmware)],
+    },
+    memory => {
+	description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
+	type => 'integer',
+	optional => 1,
+	minimum => 4,
+	maximum => 512,
+    },
+};
+
 my $confdesc = {
     onboot => {
 	optional => 1,
@@ -235,7 +314,7 @@ my $confdesc = {
     shares => {
         optional => 1,
         type => 'integer',
-        description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
+        description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 	minimum => 0,
 	maximum => 50000,
 	default => 1000,
@@ -243,9 +322,10 @@ my $confdesc = {
     keyboard => {
 	optional => 1,
 	type => 'string',
-	description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.conf' configuration file.",
+	description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
+		       "It should not be necessary to set it.",
 	enum => PVE::Tools::kvmkeymaplist(),
-	default => 'en-us',
+	default => undef,
     },
     name => {
 	optional => 1,
@@ -349,9 +429,9 @@ EODESC
     },
     agent => {
 	optional => 1,
-	type => 'boolean',
-	description => "Enable/disable Qemu GuestAgent.",
-	default => 0,
+	description => "Enable/disable Qemu GuestAgent and its properties.",
+	type => 'string',
+	format => $agent_fmt,
     },
     kvm => {
 	optional => 1,
@@ -377,17 +457,16 @@ EODESC
     },
     vga => {
 	optional => 1,
-	type => 'string',
-	description => "Select the VGA type.",
-        verbose_description => "Select the VGA type. If you want to use high resolution" .
-	    " modes (>= 1280x1024x16) then you should use the options " .
-	    "'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and " .
-	    "'cirrus' for other OS types. The 'qxl' option enables the SPICE " .
-	    "display sever. For win* OS you can select how many independent " .
-	    "displays you want, Linux guests can add displays them self. " .
-	    "You can also run without any graphic card, using a serial device" .
-	    " as terminal.",
-	enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
+	type => 'string', format => $vga_fmt,
+	description => "Configure the VGA hardware.",
+	verbose_description => "Configure the VGA Hardware. If you want to use ".
+	    "high resolution modes (>= 1280x1024x16) you may need to increase " .
+	    "the vga memory option. Since QEMU 2.9 the default VGA display type " .
+	    "is 'std' for all OS types besides some Windows versions (XP and " .
+	    "older) which use 'cirrus'. The 'qxl' option enables the SPICE " .
+	    "display server. For win* OS you can select how many independent " .
+	    "displays you want, Linux guests can add displays them self.\n".
+	    "You can also run without any graphic card, using a serial device as terminal.",
     },
     watchdog => {
 	optional => 1,
@@ -482,12 +561,15 @@ EODESCR
 	description => "Default storage for VM state volumes/files.",
 	optional => 1,
     }),
-    machine => {
-	description => "Specific the Qemu machine type.",
-	type => 'string',
-	pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
-	maxLength => 40,
+    runningmachine => get_standard_option('pve-qemu-machine', {
+	description => "Specifies the Qemu machine type of the running vm. This is used internally for snapshots.",
+    }),
+    machine => get_standard_option('pve-qemu-machine'),
+    arch => {
+	description => "Virtual processor architecture. Defaults to the host.",
 	optional => 1,
+	type => 'string',
+	enum => [qw(x86_64 aarch64)],
     },
     smbios1 => {
 	description => "Specify SMBIOS type 1 fields.",
@@ -508,6 +590,59 @@ EODESCR
 	description => "Select BIOS implementation.",
 	default => 'seabios',
     },
+    vmgenid => {
+	type => 'string',
+	pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
+	format_description => 'UUID',
+	description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0' to disable explicitly.",
+	verbose_description => "The VM generation ID (vmgenid) device exposes a".
+	    " 128-bit integer value identifier to the guest OS. This allows to".
+	    " notify the guest operating system when the virtual machine is".
+	    " executed with a different configuration (e.g. snapshot execution".
+	    " or creation from a template). The guest operating system notices".
+	    " the change, and is then able to react as appropriate by marking".
+	    " its copies of distributed databases as dirty, re-initializing its".
+	    " random number generator, etc.\n".
+	    "Note that auto-creation only works when done throug API/CLI create".
+	    " or update methods, but not when manually editing the config file.",
+	default => "1 (autogenerated)",
+	optional => 1,
+    },
+};
+
+my $confdesc_cloudinit = {
+    citype => {
+	optional => 1,
+	type => 'string',
+	description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
+	enum => ['configdrive2', 'nocloud'],
+    },
+    ciuser => {
+	optional => 1,
+	type => 'string',
+	description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
+    },
+    cipassword => {
+	optional => 1,
+	type => 'string',
+	description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
+    },
+    searchdomain => {
+	optional => 1,
+	type => 'string',
+	description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
+    },
+    nameserver => {
+	optional => 1,
+	type => 'string', format => 'address-list',
+	description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
+    },
+    sshkeys => {
+	optional => 1,
+	type => 'string',
+	format => 'urlencoded',
+	description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
+    },
 };
 
 # what about other qemu settings ?
@@ -537,7 +672,7 @@ my $MAX_VIRTIO_DISKS = 16;
 my $MAX_SATA_DISKS = 6;
 my $MAX_USB_DEVICES = 5;
 my $MAX_NETS = 32;
-my $MAX_UNUSED_DISKS = 8;
+my $MAX_UNUSED_DISKS = 256;
 my $MAX_HOSTPCI_DEVICES = 4;
 my $MAX_SERIAL_PORTS = 4;
 my $MAX_PARALLEL_PORTS = 3;
@@ -667,8 +802,64 @@ my $netdesc = {
 
 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 
+my $ipconfig_fmt = {
+    ip => {
+	type => 'string',
+	format => 'pve-ipv4-config',
+	format_description => 'IPv4Format/CIDR',
+	description => 'IPv4 address in CIDR format.',
+	optional => 1,
+	default => 'dhcp',
+    },
+    gw => {
+	type => 'string',
+	format => 'ipv4',
+	format_description => 'GatewayIPv4',
+	description => 'Default gateway for IPv4 traffic.',
+	optional => 1,
+	requires => 'ip',
+    },
+    ip6 => {
+	type => 'string',
+	format => 'pve-ipv6-config',
+	format_description => 'IPv6Format/CIDR',
+	description => 'IPv6 address in CIDR format.',
+	optional => 1,
+	default => 'dhcp',
+    },
+    gw6 => {
+	type => 'string',
+	format => 'ipv6',
+	format_description => 'GatewayIPv6',
+	description => 'Default gateway for IPv6 traffic.',
+	optional => 1,
+	requires => 'ip6',
+    },
+};
+PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
+my $ipconfigdesc = {
+    optional => 1,
+    type => 'string', format => 'pve-qm-ipconfig',
+    description => <<'EODESCR',
+cloud-init: Specify IP addresses and gateways for the corresponding interface.
+
+IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
+
+The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
+For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
+
+If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
+EODESCR
+};
+PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
+
 for (my $i = 0; $i < $MAX_NETS; $i++)  {
     $confdesc->{"net$i"} = $netdesc;
+    $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
+}
+
+foreach my $key (keys %$confdesc_cloudinit) {
+    $confdesc->{$key} = $confdesc_cloudinit->{$key};
 }
 
 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
@@ -729,7 +920,9 @@ my %drivedesc_base = (
     },
     snapshot => {
 	type => 'boolean',
-	description => "Whether the drive should be included when making snapshots.",
+	description => "Controls qemu's snapshot mode feature."
+	    . " If activated, changes made to the disk are temporary and will"
+	    . " be discarded when the VM is shutdown.",
 	optional => 1,
     },
     cache => {
@@ -793,6 +986,13 @@ my %drivedesc_base = (
 	maxLength => 20*3, # *3 since it's %xx url enoded
 	description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
 	optional => 1,
+    },
+    shared => {
+	type => 'boolean',
+	description => 'Mark this locally-managed volume as available on all nodes',
+	verbose_description => "Mark this locally-managed volume as available on all nodes.\n\nWARNING: This option does not share the volume automatically, it assumes it is shared already!",
+	optional => 1,
+	default => 0,
     }
 );
 
@@ -831,6 +1031,14 @@ my %scsiblock_fmt = (
     },
 );
 
+my %ssd_fmt = (
+    ssd => {
+	type => 'boolean',
+	description => "Whether to expose this drive as an SSD, rather than a rotational hard disk.",
+	optional => 1,
+    },
+);
+
 my $add_throttle_desc = sub {
     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
     my $d = {
@@ -878,6 +1086,7 @@ $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
 my $ide_fmt = {
     %drivedesc_base,
     %model_fmt,
+    %ssd_fmt,
 };
 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
 
@@ -893,6 +1102,7 @@ my $scsi_fmt = {
     %iothread_fmt,
     %queues_fmt,
     %scsiblock_fmt,
+    %ssd_fmt,
 };
 my $scsidesc = {
     optional => 1,
@@ -903,6 +1113,7 @@ PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 
 my $sata_fmt = {
     %drivedesc_base,
+    %ssd_fmt,
 };
 my $satadesc = {
     optional => 1,
@@ -928,6 +1139,7 @@ my $alldrive_fmt = {
     %model_fmt,
     %queues_fmt,
     %scsiblock_fmt,
+    %ssd_fmt,
 };
 
 my $efidisk_fmt = {
@@ -1133,19 +1345,15 @@ for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
 my $kvm_api_version = 0;
 
 sub kvm_version {
-
     return $kvm_api_version if $kvm_api_version;
 
-    my $fh = IO::File->new("</dev/kvm") ||
-	return 0;
+    open my $fh, '<', '/dev/kvm'
+	or return undef;
 
-    if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
-	$kvm_api_version = $v;
-    }
+    # 0xae00 => KVM_GET_API_VERSION
+    $kvm_api_version = ioctl($fh, 0xae00, 0);
 
-    $fh->close();
-
-    return  $kvm_api_version;
+    return $kvm_api_version;
 }
 
 my $kvm_user_version;
@@ -1242,7 +1450,7 @@ sub get_iso_path {
 sub filename_to_volume_id {
     my ($vmid, $file, $media) = @_;
 
-    if (!($file eq 'none' || $file eq 'cdrom' ||
+     if (!($file eq 'none' || $file eq 'cdrom' ||
 	  $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
 
 	return undef if $file =~ m|/|;
@@ -1535,21 +1743,33 @@ sub print_drivedevice_full {
 	    $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
 	}
 
-    } elsif ($drive->{interface} eq 'ide'){
-	$maxdev = 2;
+	if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
+	    $device .= ",rotation_rate=1";
+	}
+
+    } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
+	my $maxdev = ($drive->{interface} eq 'sata') ? $MAX_SATA_DISKS : 2;
 	my $controller = int($drive->{index} / $maxdev);
 	my $unit = $drive->{index} % $maxdev;
 	my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
 
-	$device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
-	if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
-	    $model = URI::Escape::uri_unescape($model);
-	    $device .= ",model=$model";
+	$device = "ide-$devicetype";
+	if ($drive->{interface} eq 'ide') {
+	    $device .= ",bus=ide.$controller,unit=$unit";
+	} else {
+	    $device .= ",bus=ahci$controller.$unit";
+	}
+	$device .= ",drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
+
+	if ($devicetype eq 'hd') {
+	    if (my $model = $drive->{model}) {
+		$model = URI::Escape::uri_unescape($model);
+		$device .= ",model=$model";
+	    }
+	    if ($drive->{ssd}) {
+		$device .= ",rotation_rate=1";
+	    }
 	}
-    } elsif ($drive->{interface} eq 'sata'){
-	my $controller = int($drive->{index} / $MAX_SATA_DISKS);
-	my $unit = $drive->{index} % $MAX_SATA_DISKS;
-	$device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
     } elsif ($drive->{interface} eq 'usb') {
 	die "implement me";
 	#  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
@@ -1559,6 +1779,12 @@ sub print_drivedevice_full {
 
     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
 
+    if (my $serial = $drive->{serial}) {
+	$serial = URI::Escape::uri_unescape($serial);
+	$device .= ",serial=$serial";
+    }
+
+
     return $device;
 }
 
@@ -1598,10 +1824,17 @@ sub print_drive_full {
    }
 
     my $opts = '';
-    my @qemu_drive_options = qw(heads secs cyls trans media format cache snapshot rerror werror aio discard);
+    my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
     foreach my $o (@qemu_drive_options) {
-	$opts .= ",$o=$drive->{$o}" if $drive->{$o};
+	$opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
+    }
+
+    # snapshot only accepts on|off
+    if (defined($drive->{snapshot})) {
+	my $v = $drive->{snapshot} ? 'on' : 'off';
+	$opts .= ",snapshot=$v";
     }
+
     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
 	my ($dir, $qmpname) = @$type;
 	if (my $v = $drive->{"mbps$dir"}) {
@@ -1624,11 +1857,6 @@ sub print_drive_full {
 	}
     }
 
-    if (my $serial = $drive->{serial}) {
-	$serial = URI::Escape::uri_unescape($serial);
-	$opts .= ",serial=$serial";
-    }
-
     $opts .= ",format=$format" if $format && !$drive->{format};
 
     my $cache_direct = 0;
@@ -1760,8 +1988,64 @@ sub print_cpu_device {
     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
 }
 
-sub drive_is_cdrom {
+my $vga_map = {
+    'cirrus' => 'cirrus-vga',
+    'std' => 'VGA',
+    'vmware' => 'vmware-svga',
+    'virtio' => 'virtio-vga',
+};
+
+sub print_vga_device {
+    my ($conf, $vga, $id, $qxlnum, $bridges) = @_;
+
+    my $type = $vga_map->{$vga->{type}};
+    my $vgamem_mb = $vga->{memory};
+    if ($qxlnum) {
+	$type = $id ? 'qxl' : 'qxl-vga';
+    }
+    die "no devicetype for $vga->{type}\n" if !$type;
+
+    my $memory = "";
+    if ($vgamem_mb) {
+	if ($vga->{type} eq 'virtio') {
+	    my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
+	    $memory = ",max_hostmem=$bytes";
+	} elsif ($qxlnum) {
+	    # from https://www.spice-space.org/multiple-monitors.html
+	    $memory = ",vgamem_mb=$vga->{memory}";
+	    my $ram = $vgamem_mb * 4;
+	    my $vram = $vgamem_mb * 2;
+	    $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
+	} else {
+	    $memory = ",vgamem_mb=$vga->{memory}";
+	}
+    } elsif ($qxlnum && $id) {
+	$memory = ",ram_size=67108864,vram_size=33554432";
+    }
+
+    my $q35 = machine_type_is_q35($conf);
+    my $vgaid = "vga" . ($id // '');
+    my $pciaddr;
+
+    if ($q35 && $vgaid eq 'vga') {
+	# the first display uses pcie.0 bus on q35 machines
+	$pciaddr = print_pcie_addr($vgaid, $bridges);
+    } else {
+	$pciaddr = print_pci_addr($vgaid, $bridges);
+    }
+
+    return "$type,id=${vgaid}${memory}${pciaddr}";
+}
+
+sub drive_is_cloudinit {
     my ($drive) = @_;
+    return $drive->{file} =~ m@[:/]vm-\d+-cloudinit(?:\.$QEMU_FORMAT_RE)?$@;
+}
+
+sub drive_is_cdrom {
+    my ($drive, $exclude_cloudinit) = @_;
+
+    return 0 if $exclude_cloudinit && drive_is_cloudinit($drive);
 
     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
 
@@ -1831,6 +2115,41 @@ sub parse_net {
     return $res;
 }
 
+# ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
+sub parse_ipconfig {
+    my ($data) = @_;
+
+    my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
+    if ($@) {
+	warn $@;
+	return undef;
+    }
+
+    if ($res->{gw} && !$res->{ip}) {
+	warn 'gateway specified without specifying an IP address';
+	return undef;
+    }
+    if ($res->{gw6} && !$res->{ip6}) {
+	warn 'IPv6 gateway specified without specifying an IPv6 address';
+	return undef;
+    }
+    if ($res->{gw} && $res->{ip} eq 'dhcp') {
+	warn 'gateway specified together with DHCP';
+	return undef;
+    }
+    if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
+	# gw6 + auto/dhcp
+	warn "IPv6 gateway specified together with $res->{ip6} address";
+	return undef;
+    }
+
+    if (!$res->{ip} && !$res->{ip6}) {
+	return { ip => 'dhcp', ip6 => 'dhcp' };
+    }
+
+    return $res;
+}
+
 sub print_net {
     my $net = shift;
 
@@ -1899,7 +2218,10 @@ sub vmconfig_undelete_pending_option {
 sub vmconfig_register_unused_drive {
     my ($storecfg, $vmid, $conf, $drive) = @_;
 
-    if (!drive_is_cdrom($drive)) {
+    if (drive_is_cloudinit($drive)) {
+	eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
+	warn $@ if $@;
+    } elsif (!drive_is_cdrom($drive)) {
 	my $volid = $drive->{file};
 	if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
 	    PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
@@ -2027,6 +2349,28 @@ sub parse_watchdog {
     return $res;
 }
 
+sub parse_guest_agent {
+    my ($value) = @_;
+
+    return {} if !defined($value->{agent});
+
+    my $res = eval { PVE::JSONSchema::parse_property_string($agent_fmt, $value->{agent}) };
+    warn $@ if $@;
+
+    # if the agent is disabled ignore the other potentially set properties
+    return {} if !$res->{enabled};
+    return $res;
+}
+
+sub parse_vga {
+    my ($value) = @_;
+
+    return {} if !$value;
+    my $res = eval { PVE::JSONSchema::parse_property_string($vga_fmt, $value) };
+    warn $@ if $@;
+    return $res;
+}
+
 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
 sub verify_usb_device {
     my ($value, $noerr) = @_;
@@ -2043,13 +2387,19 @@ sub json_config_properties {
     my $prop = shift;
 
     foreach my $opt (keys %$confdesc) {
-	next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
+	next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' || $opt eq 'runningmachine';
 	$prop->{$opt} = $confdesc->{$opt};
     }
 
     return $prop;
 }
 
+# return copy of $confdesc_cloudinit to generate documentation
+sub cloudinit_config_properties {
+
+    return dclone($confdesc_cloudinit);
+}
+
 sub check_type {
     my ($key, $value) = @_;
 
@@ -2133,7 +2483,7 @@ sub destroy_vm {
     foreach_drive($conf, sub {
 	my ($ds, $drive) = @_;
 
- 	return if drive_is_cdrom($drive);
+ 	return if drive_is_cdrom($drive, 1);
 
 	my $volid = $drive->{file};
 
@@ -2238,7 +2588,7 @@ sub parse_vm_config {
 	    } else {
 		warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
 	    }
-	} elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
+	} elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
 	    my $key = $1;
 	    my $value = $2;
 	    eval { $value = check_type($key, $value); };
@@ -2385,9 +2735,6 @@ sub load_defaults {
 	}
     }
 
-    my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
-    $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
-
     return $res;
 }
 
@@ -2491,7 +2838,7 @@ sub check_cmdline {
 	my @param = split(/\0/, $line);
 
 	my $cmd = $param[0];
-	return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
+	return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m@(?:^|/)qemu-system-[^/]+$@);
 
 	for (my $i = 0; $i < scalar (@param); $i++) {
 	    my $p = $param[$i];
@@ -2577,6 +2924,53 @@ sub disksize {
     return $drive->{size};
 }
 
+our $vmstatus_return_properties = {
+    vmid => get_standard_option('pve-vmid'),
+    status => {
+	description => "Qemu process status.",
+	type => 'string',
+	enum => ['stopped', 'running'],
+    },
+    maxmem => {
+	description => "Maximum memory in bytes.",
+	type => 'integer',
+	optional => 1,
+	renderer => 'bytes',
+    },
+    maxdisk => {
+	description => "Root disk size in bytes.",
+	type => 'integer',
+	optional => 1,
+	renderer => 'bytes',
+    },
+    name => {
+	description => "VM name.",
+	type => 'string',
+	optional => 1,
+    },
+    qmpstatus => {
+	description => "Qemu QMP agent status.",
+	type => 'string',
+	optional => 1,
+    },
+    pid => {
+	description => "PID of running qemu process.",
+	type => 'integer',
+	optional => 1,
+    },
+    uptime => {
+	description => "Uptime.",
+	type => 'integer',
+	optional => 1,
+	renderer => 'duration',
+    },
+    cpus => {
+	description => "Maximum usable CPUs.",
+	type => 'number',
+	optional => 1,
+    },
+};
+
 my $last_proc_pid_stat;
 
 # get VM status information
@@ -2590,6 +2984,8 @@ sub vmstatus {
     my $storecfg = PVE::Storage::config();
 
     my $list = vzlist();
+    my $defaults = load_defaults();
+
     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
 
     my $cpucount = $cpuinfo->{cpus} || 1;
@@ -2600,7 +2996,7 @@ sub vmstatus {
 	my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
 	my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
 
-	my $d = {};
+	my $d = { vmid => $vmid };
 	$d->{pid} = $list->{$vmid}->{pid};
 
 	# fixme: better status?
@@ -2615,16 +3011,19 @@ sub vmstatus {
 	    $d->{maxdisk} = 0;
 	}
 
-	$d->{cpus} = ($conf->{sockets} || 1) * ($conf->{cores} || 1);
+	$d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
+	    * ($conf->{cores} || $defaults->{cores});
 	$d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
 	$d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
 
 	$d->{name} = $conf->{name} || "VM $vmid";
-	$d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024) : 0;
+	$d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
+	    : $defaults->{memory}*(1024*1024);
 
 	if ($conf->{balloon}) {
 	    $d->{balloon_min} = $conf->{balloon}*(1024*1024);
-	    $d->{shares} = defined($conf->{shares}) ? $conf->{shares} : 1000;
+	    $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
+		: $defaults->{shares};
 	}
 
 	$d->{uptime} = 0;
@@ -2639,6 +3038,8 @@ sub vmstatus {
 
         $d->{template} = PVE::QemuConfig->is_template($conf);
 
+	$d->{serial} = 1 if conf_has_serial($conf);
+
 	$res->{$vmid} = $d;
     }
 
@@ -2797,7 +3198,7 @@ sub foreach_volid {
     my $volhash = {};
 
     my $test_volid = sub {
-	my ($volid, $is_cdrom, $replicate, $snapname) = @_;
+	my ($volid, $is_cdrom, $replicate, $shared, $snapname) = @_;
 
 	return if !$volid;
 
@@ -2807,6 +3208,9 @@ sub foreach_volid {
 	$volhash->{$volid}->{replicate} //= 0;
 	$volhash->{$volid}->{replicate} = 1 if $replicate;
 
+	$volhash->{$volid}->{shared} //= 0;
+	$volhash->{$volid}->{shared} = 1 if $shared;
+
 	$volhash->{$volid}->{referenced_in_config} //= 0;
 	$volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
 
@@ -2816,7 +3220,7 @@ sub foreach_volid {
 
     foreach_drive($conf, sub {
 	my ($ds, $drive) = @_;
-	$test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, undef);
+	$test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, undef);
     });
 
     foreach my $snapname (keys %{$conf->{snapshots}}) {
@@ -2824,7 +3228,7 @@ sub foreach_volid {
 	$test_volid->($snap->{vmstate}, 0, 1, $snapname);
 	foreach_drive($snap, sub {
 	    my ($ds, $drive) = @_;
-	    $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $snapname);
+	    $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, $snapname);
         });
     }
 
@@ -2833,14 +3237,125 @@ sub foreach_volid {
     }
 }
 
+sub conf_has_serial {
+    my ($conf) = @_;
+
+    for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
+	if ($conf->{"serial$i"}) {
+	    return 1;
+	}
+    }
+
+    return 0;
+}
+
 sub vga_conf_has_spice {
     my ($vga) = @_;
 
-    return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
+    my $vgaconf = parse_vga($vga);
+    my $vgatype = $vgaconf->{type};
+    return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
 
     return $1 || 1;
 }
 
+my $host_arch; # FIXME: fix PVE::Tools::get_host_arch
+sub get_host_arch() {
+    $host_arch = (POSIX::uname())[4] if !$host_arch;
+    return $host_arch;
+}
+
+sub is_native($) {
+    my ($arch) = @_;
+    return get_host_arch() eq $arch;
+}
+
+my $default_machines = {
+    x86_64 => 'pc',
+    aarch64 => 'virt',
+};
+
+sub get_basic_machine_info {
+    my ($conf, $forcemachine) = @_;
+
+    my $arch = $conf->{arch} // get_host_arch();
+    my $machine = $forcemachine || $conf->{machine} || $default_machines->{$arch};
+    return ($arch, $machine);
+}
+
+sub get_ovmf_files($) {
+    my ($arch) = @_;
+
+    my $ovmf = $OVMF->{$arch}
+	or die "no OVMF images known for architecture '$arch'\n";
+
+    return @$ovmf;
+}
+
+my $Arch2Qemu = {
+    aarch64 => '/usr/bin/qemu-system-aarch64',
+    x86_64 => '/usr/bin/qemu-system-x86_64',
+};
+sub get_command_for_arch($) {
+    my ($arch) = @_;
+    return '/usr/bin/kvm' if is_native($arch);
+
+    my $cmd = $Arch2Qemu->{$arch}
+	or die "don't know how to emulate architecture '$arch'\n";
+    return $cmd;
+}
+
+sub get_cpu_options {
+    my ($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough) = @_;
+
+    my $cpuFlags = [];
+    my $ostype = $conf->{ostype};
+
+    my $cpu = $kvm ? "kvm64" : "qemu64";
+    if (my $cputype = $conf->{cpu}) {
+	my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
+	    or die "Cannot parse cpu description: $cputype\n";
+	$cpu = $cpuconf->{cputype};
+	$kvm_off = 1 if $cpuconf->{hidden};
+
+	if (defined(my $flags = $cpuconf->{flags})) {
+	    push @$cpuFlags, split(";", $flags);
+	}
+    }
+
+    push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
+
+    push @$cpuFlags , '-x2apic'
+	if $conf->{ostype} && $conf->{ostype} eq 'solaris';
+
+    push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
+
+    push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
+
+    if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
+
+	push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
+	push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
+    }
+
+    add_hyperv_enlightenments($cpuFlags, $winversion, $machine_type, $kvmver, $conf->{bios}, $gpu_passthrough) if $kvm;
+
+    push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm;
+
+    push @$cpuFlags, 'kvm=off' if $kvm_off;
+
+    if (my $cpu_vendor = $cpu_vendor_list->{$cpu}) {
+	push @$cpuFlags, "vendor=${cpu_vendor}"
+	    if $cpu_vendor ne 'default';
+    } elsif ($arch ne 'aarch64') {
+	die "internal error"; # should not happen
+    }
+
+    $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
+
+    return ('-cpu', $cpu);
+}
+
 sub config_to_command {
     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
 
@@ -2848,7 +3363,6 @@ sub config_to_command {
     my $globalFlags = [];
     my $machineFlags = [];
     my $rtcFlags = [];
-    my $cpuFlags = [];
     my $devices = [];
     my $pciaddr = '';
     my $bridges = {};
@@ -2856,9 +3370,15 @@ sub config_to_command {
     my $vernum = 0; # unknown
     my $ostype = $conf->{ostype};
     my $winversion = windows_version($ostype);
-    my $kvm = $conf->{kvm} // 1;
+    my $kvm = $conf->{kvm};
 
-    die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n" if (!$cpuinfo->{hvm} && $kvm);
+    my ($arch, $machine_type) = get_basic_machine_info($conf, $forcemachine);
+    $kvm //= 1 if is_native($arch);
+
+    if ($kvm) {
+	die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n"
+	    if !defined kvm_version();
+    }
 
     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
 	$vernum = $1*1000000+$2*1000;
@@ -2872,23 +3392,31 @@ sub config_to_command {
 
     my $q35 = machine_type_is_q35($conf);
     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
-    my $machine_type = $forcemachine || $conf->{machine};
     my $use_old_bios_files = undef;
     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
 
     my $cpuunits = defined($conf->{cpuunits}) ?
             $conf->{cpuunits} : $defaults->{cpuunits};
 
-    push @$cmd, '/usr/bin/kvm';
+    push @$cmd, get_command_for_arch($arch);
 
     push @$cmd, '-id', $vmid;
 
+    my $vmname = $conf->{name} || "vm$vmid";
+
+    push @$cmd, '-name', $vmname;
+
     my $use_virtio = 0;
 
     my $qmpsocket = qmp_socket($vmid);
     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
     push @$cmd, '-mon', "chardev=qmp,mode=control";
 
+    if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 12)) {
+	my $eventsocket = qmp_socket($vmid, 0, 'event');
+	push @$cmd, '-chardev', "socket,id=qmp-event,path=$eventsocket,server,nowait";
+	push @$cmd, '-mon', "chardev=qmp-event,mode=control";
+    }
 
     push @$cmd, '-pidfile' , pidfile_name($vmid);
 
@@ -2898,27 +3426,39 @@ sub config_to_command {
 	push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
     }
 
+    if ($conf->{vmgenid}) {
+	push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
+    }
+
+    my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
-	die "uefi base image not found\n" if ! -f $OVMF_CODE;
+	die "uefi base image not found\n" if ! -f $ovmf_code;
 
 	my $path;
-	my $format = 'raw';
+	my $format;
 	if (my $efidisk = $conf->{efidisk0}) {
 	    my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
 	    my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
+	    $format = $d->{format};
 	    if ($storeid) {
 		$path = PVE::Storage::path($storecfg, $d->{file});
+		if (!defined($format)) {
+		    my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
+		    $format = qemu_img_format($scfg, $volname);
+		}
 	    } else {
 		$path = $d->{file};
+		die "efidisk format must be specified\n"
+		    if !defined($format);
 	    }
-	    $format = $d->{format} if $d->{format};
 	} else {
 	    warn "no efidisk configured! Using temporary efivars disk.\n";
 	    $path = "/tmp/$vmid-ovmf.fd";
-	    PVE::Tools::file_copy($OVMF_VARS, $path, -s $OVMF_VARS);
+	    PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
+	    $format = 'raw';
 	}
 
-	push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$OVMF_CODE";
+	push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
 	push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
     }
 
@@ -2926,16 +3466,16 @@ sub config_to_command {
     # add usb controllers
     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $q35, $usbdesc->{format}, $MAX_USB_DEVICES);
     push @$devices, @usbcontrollers if @usbcontrollers;
-    my $vga = $conf->{vga};
+    my $vga = parse_vga($conf->{vga});
 
-    my $qxlnum = vga_conf_has_spice($vga);
-    $vga = 'qxl' if $qxlnum;
+    my $qxlnum = vga_conf_has_spice($conf->{vga});
+    $vga->{type} = 'qxl' if $qxlnum;
 
-    if (!$vga) {
+    if (!$vga->{type}) {
 	if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 9)) {
-	    $vga = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
+	    $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
 	} else {
-	    $vga = ($winversion >= 6) ? 'std' : 'cirrus';
+	    $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
 	}
     }
 
@@ -2946,7 +3486,7 @@ sub config_to_command {
     } else {
 	$tablet = $defaults->{tablet};
 	$tablet = 0 if $qxlnum; # disable for spice because it is not needed
-	$tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
+	$tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
     }
 
     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
@@ -2974,7 +3514,7 @@ sub config_to_command {
 	if ($d->{'x-vga'}) {
 	    $xvga = ',x-vga=on';
 	    $kvm_off = 1;
-	    $vga = 'none';
+	    $vga->{type} = 'none';
 	    $gpu_passthrough = 1;
 
 	    if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
@@ -3032,9 +3572,6 @@ sub config_to_command {
 	}
     }
 
-    my $vmname = $conf->{name} || "vm$vmid";
-
-    push @$cmd, '-name', $vmname;
 
     my $sockets = 1;
     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
@@ -3080,12 +3617,12 @@ sub config_to_command {
 
     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
 
-    push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
-
-    if ($vga && $vga !~ m/^serial\d+$/ && $vga ne 'none'){
+    if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
+	push @$devices, '-device', print_vga_device($conf, $vga, undef, $qxlnum, $bridges);
 	my $socket = vnc_socket($vmid);
 	push @$cmd,  '-vnc', "unix:$socket,x509,password";
     } else {
+	push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
 	push @$cmd, '-nographic';
     }
 
@@ -3124,59 +3661,20 @@ sub config_to_command {
 	push @$rtcFlags, 'base=localtime';
     }
 
-    my $cpu = $kvm ? "kvm64" : "qemu64";
-    if (my $cputype = $conf->{cpu}) {
-	my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
-	    or die "Cannot parse cpu description: $cputype\n";
-	$cpu = $cpuconf->{cputype};
-	$kvm_off = 1 if $cpuconf->{hidden};
-    }
-
-    push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
-
-    push @$cpuFlags , '-x2apic'
-	if $conf->{ostype} && $conf->{ostype} eq 'solaris';
-
-    push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
-
-    push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
-
-    if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
-
-	push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
-	push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
-    }
-
-    add_hyperv_enlightenments($cpuFlags, $winversion, $machine_type, $kvmver, $conf->{bios}, $gpu_passthrough) if $kvm;
-
-    push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm;
-
-    push @$cpuFlags, 'kvm=off' if $kvm_off;
-
-    my $cpu_vendor = $cpu_vendor_list->{$cpu} ||
-	die "internal error"; # should not happen
-
-    push @$cpuFlags, "vendor=${cpu_vendor}"
-	if $cpu_vendor ne 'default';
-
-    $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
-
-    push @$cmd, '-cpu', $cpu;
+    push @$cmd, get_cpu_options($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough);
 
     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
     
     push @$cmd, '-S' if $conf->{freeze};
 
-    # set keyboard layout
-    my $kb = $conf->{keyboard} || $defaults->{keyboard};
-    push @$cmd, '-k', $kb if $kb;
+    push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
 
     # enable sound
     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
     #push @$cmd, '-soundhw', 'es1370';
     #push @$cmd, '-soundhw', $soundhw if $soundhw;
 
-    if($conf->{agent}) {
+    if (parse_guest_agent($conf)->{enabled}) {
 	my $qgasocket = qmp_socket($vmid, 1);
 	my $pciaddr = print_pci_addr("qga0", $bridges);
 	push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
@@ -3190,13 +3688,17 @@ sub config_to_command {
 	if ($qxlnum > 1) {
 	    if ($winversion){
 		for(my $i = 1; $i < $qxlnum; $i++){
-		    my $pciaddr = print_pci_addr("vga$i", $bridges);
-		    push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
+		    push @$devices, '-device', print_vga_device($conf, $vga, $i, $qxlnum, $bridges);
 		}
 	    } else {
 		# assume other OS works like Linux
-		push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
-		push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
+		my ($ram, $vram) = ("134217728", "67108864");
+		if ($vga->{memory}) {
+		    $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
+		    $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
+		}
+		push @$cmd, '-global', "qxl-vga.ram_size=$ram";
+		push @$cmd, '-global', "qxl-vga.vram_size=$vram";
 	    }
 	}
 
@@ -3369,9 +3871,10 @@ sub spice_port {
 }
 
 sub qmp_socket {
-    my ($vmid, $qga) = @_;
+    my ($vmid, $qga, $name) = @_;
     my $sockettype = $qga ? 'qga' : 'qmp';
-    return "${var_run_tmpdir}/$vmid.$sockettype";
+    my $ext = $name ? '-'.$name : '';
+    return "${var_run_tmpdir}/$vmid$ext.$sockettype";
 }
 
 sub pidfile_name {
@@ -3383,21 +3886,22 @@ sub vm_devices_list {
     my ($vmid) = @_;
 
     my $res = vm_mon_cmd($vmid, 'query-pci');
+    my $devices_to_check = [];
     my $devices = {};
     foreach my $pcibus (@$res) {
-	foreach my $device (@{$pcibus->{devices}}) {
-	    next if !$device->{'qdev_id'};
-	    if ($device->{'pci_bridge'}) {
-		$devices->{$device->{'qdev_id'}} = 1;
-		foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
-		    next if !$bridge_device->{'qdev_id'};
-		    $devices->{$bridge_device->{'qdev_id'}} = 1;
-		    $devices->{$device->{'qdev_id'}}++;
-		}
-	    } else {
-		$devices->{$device->{'qdev_id'}} = 1;
-	    }
+	push @$devices_to_check, @{$pcibus->{devices}},
+    }
+
+    while (@$devices_to_check) {
+	my $to_check = [];
+	for my $d (@$devices_to_check) {
+	    $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
+	    next if !$d->{'pci_bridge'};
+
+	    $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
+	    push @$to_check, @{$d->{'pci_bridge'}->{devices}};
 	}
+	$devices_to_check = $to_check;
     }
 
     my $resblock = vm_mon_cmd($vmid, 'query-block');
@@ -3569,10 +4073,6 @@ sub vm_deviceunplug {
 
     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
 
-	#qemu 2.3 segfault on drive_del with virtioscsi + iothread
-	my $device = parse_drive($deviceid, $conf->{$deviceid});
-	die "virtioscsi with iothread is not hot-unplugglable currently" if $device->{iothread};
-
         qemu_devicedel($vmid, $deviceid);
         qemu_drivedel($vmid, $deviceid);
 	qemu_deletescsihw($conf, $vmid, $deviceid);
@@ -3951,81 +4451,6 @@ sub __read_avail {
     return $res;
 }
 
-# old code, only used to shutdown old VM after update
-sub vm_monitor_command {
-    my ($vmid, $cmdstr, $nocheck) = @_;
-
-    my $res;
-
-    eval {
-	die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
-
-	my $sname = "${var_run_tmpdir}/$vmid.mon";
-
-	my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
-	    die "unable to connect to VM $vmid socket - $!\n";
-
-	my $timeout = 3;
-
-	# hack: migrate sometime blocks the monitor (when migrate_downtime
-	# is set)
-	if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
-	    $timeout = 60*60; # 1 hour
-	}
-
-	# read banner;
-	my $data = __read_avail($sock, $timeout);
-
-	if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
-	    die "got unexpected qemu monitor banner\n";
-	}
-
-	my $sel = new IO::Select;
-	$sel->add($sock);
-
-	if (!scalar(my @ready = $sel->can_write($timeout))) {
-	    die "monitor write error - timeout";
-	}
-
-	my $fullcmd = "$cmdstr\r";
-
-	# syslog('info', "VM $vmid monitor command: $cmdstr");
-
-	my $b;
-	if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
-	    die "monitor write error - $!";
-	}
-
-	return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
-
-	$timeout = 20;
-
-	if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
-	    $timeout = 60*60; # 1 hour
-	} elsif ($cmdstr =~ m/^(eject|change)/) {
-	    $timeout = 60; # note: cdrom mount command is slow
-	}
-	if ($res = __read_avail($sock, $timeout)) {
-
-	    my @lines = split("\r?\n", $res);
-
-	    shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
-
-	    $res = join("\n", @lines);
-	    $res .= "\n";
-	}
-    };
-
-    my $err = $@;
-
-    if ($err) {
-	syslog("err", "VM $vmid monitor command failed - $err");
-	die $err;
-    }
-
-    return $res;
-}
-
 sub qemu_block_resize {
     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
 
@@ -4045,7 +4470,7 @@ sub qemu_volume_snapshot {
     my $running = check_running($vmid);
 
     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
-	vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
+	vm_mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
     } else {
 	PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
     }
@@ -4056,8 +4481,18 @@ sub qemu_volume_snapshot_delete {
 
     my $running = check_running($vmid);
 
+    if($running) {
+
+	$running = undef;
+	my $conf = PVE::QemuConfig->load_config($vmid);
+	foreach_drive($conf, sub {
+	    my ($ds, $drive) = @_;
+	    $running = 1 if $drive->{file} eq $volid;
+	});
+    }
+
     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
-	vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
+	vm_mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
     } else {
 	PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
     }
@@ -4157,7 +4592,10 @@ sub vmconfig_hotplug_pending {
 		qemu_cpu_hotplug($vmid, $conf, undef);
             } elsif ($opt eq 'balloon') {
 		# enable balloon device is not hotpluggable
-		die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
+		die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
+		# here we reset the ballooning value to memory
+		my $balloon = $conf->{memory} || $defaults->{memory};
+		vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
 	    } elsif ($fast_plug_option->{$opt}) {
 		# do nothing
 	    } elsif ($opt =~ m/^net(\d+)$/) {
@@ -4189,6 +4627,22 @@ sub vmconfig_hotplug_pending {
 	}
     }
 
+    my $apply_pending_cloudinit;
+    $apply_pending_cloudinit = sub {
+	my ($key, $value) = @_;
+	$apply_pending_cloudinit = sub {}; # once is enough
+
+	my @cloudinit_opts = keys %$confdesc_cloudinit;
+	foreach my $opt (keys %{$conf->{pending}}) {
+	    next if !grep { $_ eq $opt } @cloudinit_opts;
+	    $conf->{$opt} = delete $conf->{pending}->{$opt};
+	}
+
+	my $new_conf = { %$conf };
+	$new_conf->{$key} = $value;
+	PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
+    };
+
     foreach my $opt (keys %{$conf->{pending}}) {
 	next if $selection && !$selection->{$opt};
 	my $value = $conf->{pending}->{$opt};
@@ -4230,6 +4684,10 @@ sub vmconfig_hotplug_pending {
 				    $vmid, $opt, $value);
 	    } elsif (is_valid_drivename($opt)) {
 		# some changes can be done without hotplug
+		my $drive = parse_drive($opt, $value);
+		if (drive_is_cloudinit($drive)) {
+		    &$apply_pending_cloudinit($opt, $value);
+		}
 		vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
 				     $vmid, $opt, $value, 1);
 	    } elsif ($opt =~ m/^memory$/) { #dimms
@@ -4492,6 +4950,9 @@ sub vmconfig_update_disk {
 
 		if ($drive->{file} eq 'none') {
 		    vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
+		    if (drive_is_cloudinit($old_drive)) {
+			vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
+		    }
 		} else {
 		    my $path = get_iso_path($storecfg, $vmid, $drive->{file});
 		    vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
@@ -4527,6 +4988,8 @@ sub vm_start {
 	    $conf = PVE::QemuConfig->load_config($vmid); # update/reload
 	}
 
+	PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
+
 	my $defaults = load_defaults();
 
 	# set environment variable useful inside network script
@@ -4652,16 +5115,18 @@ sub vm_start {
 
 	PVE::Storage::activate_volumes($storecfg, $vollist);
 
-	if (!check_running($vmid, 1) && -d "/sys/fs/cgroup/systemd/qemu.slice/$vmid.scope") {
-	    my $cmd = [];
-	    push @$cmd, '/bin/systemctl', 'stop', "$vmid.scope";
-	    eval  { run_command($cmd); };
+	if (!check_running($vmid, 1)) {
+	    eval {
+		run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
+		    outfunc => sub {}, errfunc => sub {});
+	    };
 	}
 
 	my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
 	                                          : $defaults->{cpuunits};
 
-	my %run_params = (timeout => $statefile ? undef : 30, umask => 0077);
+	my $start_timeout = $conf->{hugepages} ? 300 : 30;
+	my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077);
 
 	my %properties = (
 	    Slice => 'qemu.slice',
@@ -4674,6 +5139,13 @@ sub vm_start {
 	}
 	$properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
 
+	my $run_qemu = sub {
+	    PVE::Tools::run_fork sub {
+		PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
+		run_command($cmd, %run_params);
+	    };
+	};
+
 	if ($conf->{hugepages}) {
 
 	    my $code = sub {
@@ -4683,11 +5155,7 @@ sub vm_start {
 		PVE::QemuServer::Memory::hugepages_mount();
 		PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
 
-		eval  {
-		    PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
-		    run_command($cmd, %run_params);
-		};
-
+		eval { $run_qemu->() };
 		if (my $err = $@) {
 		    PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
 		    die $err;
@@ -4698,10 +5166,7 @@ sub vm_start {
 	    eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
 
 	} else {
-	    eval  {
-		PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
-		run_command($cmd, %run_params);
-	    };
+	    eval { $run_qemu->() };
 	}
 
 	if (my $err = $@) {
@@ -4752,10 +5217,8 @@ sub vm_start {
 	    }
 
 	} else {
-	    if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
-		vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
-		    if $conf->{balloon};
-	    }
+	    vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
+		if !$statefile && $conf->{balloon};
 
 	    foreach my $opt (keys %$conf) {
 		next if $opt !~  m/^net\d+$/;
@@ -4804,10 +5267,6 @@ sub vm_qmp_command {
 	    my $qmpclient = PVE::QMPClient->new();
 
 	    $res = $qmpclient->cmd($vmid, $cmd, $timeout);
-	} elsif (-e "${var_run_tmpdir}/$vmid.mon") {
-	    die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
-		if scalar(%{$cmd->{arguments}});
-	    vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
 	} else {
 	    die "unable to open monitor socket\n";
 	}
@@ -4930,7 +5389,7 @@ sub vm_stop {
 
 	eval {
 	    if ($shutdown) {
-		if (defined($conf) && $conf->{agent}) {
+		if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
 		    vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
 		} else {
 		    vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
@@ -5006,6 +5465,13 @@ sub vm_resume {
 
     PVE::QemuConfig->lock_config($vmid, sub {
 
+	my $res = vm_mon_cmd($vmid, 'query-status');
+	my $resume_cmd = 'cont';
+
+	if ($res->{status} && $res->{status} eq 'suspended') {
+	    $resume_cmd = 'system_wakeup';
+	}
+
 	if (!$nocheck) {
 
 	    my $conf = PVE::QemuConfig->load_config($vmid);
@@ -5013,10 +5479,10 @@ sub vm_resume {
 	    PVE::QemuConfig->check_lock($conf)
 		if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
 
-	    vm_mon_cmd($vmid, "cont");
+	    vm_mon_cmd($vmid, $resume_cmd);
 
 	} else {
-	    vm_mon_cmd_nocheck($vmid, "cont");
+	    vm_mon_cmd_nocheck($vmid, $resume_cmd);
 	}
     });
 }
@@ -5292,6 +5758,20 @@ sub restore_update_config_line {
 	} else {
 	    print $outfd $line;
 	}
+    } elsif (($line =~ m/^vmgenid: (.*)/)) {
+	my $vmgenid = $1;
+	if ($vmgenid ne '0') {
+	    # always generate a new vmgenid if there was a valid one setup
+	    $vmgenid = generate_uuid();
+	}
+	print $outfd "vmgenid: $vmgenid\n";
+    } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
+	my ($uuid, $uuid_str);
+	UUID::generate($uuid);
+	UUID::unparse($uuid, $uuid_str);
+	my $smbios1 = parse_smbios1($2);
+	$smbios1->{uuid} = $uuid_str;
+	print $outfd $1.print_smbios1($smbios1)."\n";
     } else {
 	print $outfd $line;
     }
@@ -5359,14 +5839,16 @@ sub update_disksize {
     my ($vmid, $conf, $volid_hash) = @_;
 
     my $changes;
+    my $prefix = "VM $vmid:";
 
-    my $used = {};
+    # used and unused disks
+    my $referenced = {};
 
     # Note: it is allowed to define multiple storages with same path (alias), so
     # we need to check both 'volid' and real 'path' (two different volid can point
     # to the same path).
 
-    my $usedpath = {};
+    my $referencedpath = {};
 
     # update size info
     foreach my $opt (keys %$conf) {
@@ -5375,10 +5857,10 @@ sub update_disksize {
 	    my $volid = $drive->{file};
 	    next if !$volid;
 
-	    $used->{$volid} = 1;
+	    $referenced->{$volid} = 1;
 	    if ($volid_hash->{$volid} &&
 		(my $path = $volid_hash->{$volid}->{path})) {
-		$usedpath->{$path} = 1;
+		$referencedpath->{$path} = 1;
 	    }
 
 	    next if drive_is_cdrom($drive);
@@ -5389,6 +5871,7 @@ sub update_disksize {
 	    if ($new ne $conf->{$opt}) {
 		$changes = 1;
 		$conf->{$opt} = $new;
+		print "$prefix update disk '$opt' information.\n";
 	    }
 	}
     }
@@ -5398,31 +5881,43 @@ sub update_disksize {
 	next if $opt !~ m/^unused\d+$/;
 	my $volid = $conf->{$opt};
 	my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
-	if ($used->{$volid} || ($path && $usedpath->{$path})) {
+	if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
+	    print "$prefix remove entry '$opt', its volume '$volid' is in use.\n";
 	    $changes = 1;
 	    delete $conf->{$opt};
 	}
+
+	$referenced->{$volid} = 1;
+	$referencedpath->{$path} = 1 if $path;
     }
 
     foreach my $volid (sort keys %$volid_hash) {
 	next if $volid =~ m/vm-$vmid-state-/;
-	next if $used->{$volid};
+	next if $referenced->{$volid};
 	my $path = $volid_hash->{$volid}->{path};
 	next if !$path; # just to be sure
-	next if $usedpath->{$path};
+	next if $referencedpath->{$path};
 	$changes = 1;
-	PVE::QemuConfig->add_unused_volume($conf, $volid);
-	$usedpath->{$path} = 1; # avoid to add more than once (aliases)
+	my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
+	print "$prefix add unreferenced volume '$volid' as '$key' to config.\n";
+	$referencedpath->{$path} = 1; # avoid to add more than once (aliases)
     }
 
     return $changes;
 }
 
 sub rescan {
-    my ($vmid, $nolock) = @_;
+    my ($vmid, $nolock, $dryrun) = @_;
 
     my $cfg = PVE::Storage::config();
 
+    # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
+    # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
+    foreach my $stor (keys %{$cfg->{ids}}) {
+	delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
+    }
+
+    print "rescan volumes...\n";
     my $volid_hash = scan_volids($cfg, $vmid);
 
     my $updatefn =  sub {
@@ -5440,7 +5935,7 @@ sub rescan {
 
 	my $changes = update_disksize($vmid, $conf, $vm_volids);
 
-	PVE::QemuConfig->write_config($vmid, $conf) if $changes;
+	PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
     };
 
     if (defined($vmid)) {
@@ -5464,21 +5959,49 @@ sub rescan {
 sub restore_vma_archive {
     my ($archive, $vmid, $user, $opts, $comp) = @_;
 
-    my $input = $archive eq '-' ? "<&STDIN" : undef;
     my $readfrom = $archive;
 
-    my $uncomp = '';
-    if ($comp) {
+    my $cfg = PVE::Storage::config();
+    my $commands = [];
+    my $bwlimit = $opts->{bwlimit};
+
+    my $dbg_cmdstring = '';
+    my $add_pipe = sub {
+	my ($cmd) = @_;
+	push @$commands, $cmd;
+	$dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
+	$dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
 	$readfrom = '-';
-	my $qarchive = PVE::Tools::shellquote($archive);
+    };
+
+    my $input = undef;
+    if ($archive eq '-') {
+	$input = '<&STDIN';
+    } else {
+	# If we use a backup from a PVE defined storage we also consider that
+	# storage's rate limit:
+	my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
+	if (defined($volid)) {
+	    my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
+	    my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
+	    if ($readlimit) {
+		print STDERR "applying read rate limit: $readlimit\n";
+		my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
+		$add_pipe->($cstream);
+	    }
+	}
+    }
+
+    if ($comp) {
+	my $cmd;
 	if ($comp eq 'gzip') {
-	    $uncomp = "zcat $qarchive|";
+	    $cmd = ['zcat', $readfrom];
 	} elsif ($comp eq 'lzop') {
-	    $uncomp = "lzop -d -c $qarchive|";
+	    $cmd = ['lzop', '-d', '-c', $readfrom];
 	} else {
 	    die "unknown compression method '$comp'\n";
 	}
-
+	$add_pipe->($cmd);
     }
 
     my $tmpdir = "/var/tmp/vzdumptmp$$";
@@ -5498,7 +6021,7 @@ sub restore_vma_archive {
 	open($fifofh, '>', $mapfifo) || die $!;
     };
 
-    my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
+    $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
 
     my $oldtimeout;
     my $timeout = 5;
@@ -5514,6 +6037,8 @@ sub restore_vma_archive {
     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
 
+    my %storage_limits;
+
     my $print_devmap = sub {
 	my $virtdev_hash = {};
 
@@ -5552,17 +6077,24 @@ sub restore_vma_archive {
 		    $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
 		}
 
+		$storage_limits{$storeid} = $bwlimit;
+
 		$virtdev_hash->{$virtdev} = $devinfo->{$devname};
 	    }
 	}
 
+	foreach my $key (keys %storage_limits) {
+	    my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
+	    next if !$limit;
+	    print STDERR "rate limit for storage $key: $limit KiB/s\n";
+	    $storage_limits{$key} = $limit * 1024;
+	}
+
 	foreach my $devname (keys %$devinfo) {
 	    die "found no device mapping information for device '$devname'\n"
 		if !$devinfo->{$devname}->{virtdev};
 	}
 
-	my $cfg = PVE::Storage::config();
-
 	# create empty/temp config
 	if ($oldconf) {
 	    PVE::Tools::file_set_contents($conffile, "memory: 128\n");
@@ -5605,14 +6137,20 @@ sub restore_vma_archive {
 	foreach my $virtdev (sort keys %$virtdev_hash) {
 	    my $d = $virtdev_hash->{$virtdev};
 	    my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
-	    my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
+	    my $storeid = $d->{storeid};
+	    my $scfg = PVE::Storage::storage_config($cfg, $storeid);
+
+	    my $map_opts = '';
+	    if (my $limit = $storage_limits{$storeid}) {
+		$map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
+	    }
 
 	    # test if requested format is supported
-	    my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
+	    my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $storeid);
 	    my $supported = grep { $_ eq $d->{format} } @$validFormats;
 	    $d->{format} = $defFormat if !$supported;
 
-	    my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
+	    my $volid = PVE::Storage::vdisk_alloc($cfg, $storeid, $vmid,
 						  $d->{format}, undef, $alloc_size);
 	    print STDERR "new volume ID is '$volid'\n";
 	    $d->{volid} = $volid;
@@ -5625,7 +6163,7 @@ sub restore_vma_archive {
 		$write_zeros = 0;
 	    }
 
-	    print $fifofh "format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
+	    print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
 
 	    print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
 	    $map->{$virtdev} = $volid;
@@ -5678,8 +6216,8 @@ sub restore_vma_archive {
 	    }
 	};
 
-	print "restore vma archive: $cmd\n";
-	run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
+	print "restore vma archive: $dbg_cmdstring\n";
+	run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
     };
     my $err = $@;
 
@@ -5691,7 +6229,6 @@ sub restore_vma_archive {
 	push @$vollist, $volid if $volid;
     }
 
-    my $cfg = PVE::Storage::config();
     PVE::Storage::deactivate_volumes($cfg, $vollist);
 
     unlink $mapfifo;
@@ -5878,11 +6415,11 @@ sub do_snapshots_with_qemu {
 }
 
 sub qga_check_running {
-    my ($vmid) = @_;
+    my ($vmid, $nowarn) = @_;
 
     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
     if ($@) {
-	warn "Qemu Guest Agent are not running - $@";
+	warn "Qemu Guest Agent is not running - $@" if !$nowarn;
 	return 0;
     }
     return 1;
@@ -5931,7 +6468,9 @@ sub qemu_img_convert {
 
 	my $cmd = [];
 	push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
-	push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
+	push @$cmd, '-l', "snapshot.name=$snapname" if($snapname && $src_format eq "qcow2");
+	push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
+	push @$cmd, '-T', 'none' if $src_scfg->{type} eq 'zfspool';
 	push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path;
 	if ($is_zero_initialized) {
 	    push @$cmd, "zeroinit:$dst_path";
@@ -5960,7 +6499,7 @@ sub qemu_img_convert {
 sub qemu_img_format {
     my ($scfg, $volname) = @_;
 
-    if ($scfg->{path} && $volname =~ m/\.(raw|cow|qcow|qcow2|qed|vmdk|cloop)$/) {
+    if ($scfg->{path} && $volname =~ m/\.($QEMU_FORMAT_RE)$/) {
 	return $1;
     } else {
 	return "raw";
@@ -5976,32 +6515,9 @@ sub qemu_drive_mirror {
     my $format;
     $jobs->{"drive-$drive"} = {};
 
-    if ($dst_volid =~ /^nbd:(localhost|[\d\.]+|\[[\d\.:a-fA-F]+\]):(\d+):exportname=(\S+)/) {
-	my $server = $1;
-	my $port = $2;
-	my $exportname = $3;
-
+    if ($dst_volid =~ /^nbd:/) {
+	$qemu_target = $dst_volid;
 	$format = "nbd";
-	my $unixsocket = "/run/qemu-server/$vmid.mirror-drive-$drive";
-	$qemu_target = "nbd+unix:///$exportname?socket=$unixsocket";
-	my $cmd = ['socat', '-T30', "UNIX-LISTEN:$unixsocket,fork", "TCP:$server:$2,connect-timeout=5"];
-
-	my $pid = fork();
-	if (!defined($pid)) {
-	    die "forking socat tunnel failed\n";
-	} elsif ($pid == 0) {
-	    exec(@$cmd);
-	    warn "exec failed: $!\n";
-	    POSIX::_exit(-1);
-	}
-	$jobs->{"drive-$drive"}->{pid} = $pid;
-
-	my $timeout = 0;
-	while (!-S $unixsocket) {
-	    die "nbd connection helper timed out\n"
-		if $timeout++ > 5;
-	    sleep 1;
-	}
     } else {
 	my $storecfg = PVE::Storage::config();
 	my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
@@ -6079,7 +6595,8 @@ sub qemu_drive_mirror_monitor {
 		last if $skipcomplete; #do the complete later
 
 		if ($vmiddst && $vmiddst != $vmid) {
-		    if ($qga) {
+		    my $agent_running = $qga && qga_check_running($vmid);
+		    if ($agent_running) {
 			print "freeze filesystem\n";
 			eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
 		    } else {
@@ -6090,7 +6607,7 @@ sub qemu_drive_mirror_monitor {
 		    # if we clone a disk for a new target vm, we don't switch the disk
 		    PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
 
-		    if ($qga) {
+		    if ($agent_running) {
 			print "unfreeze filesystem\n";
 			eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
 		    } else {
@@ -6112,7 +6629,6 @@ sub qemu_drive_mirror_monitor {
 			}else {
 			    print "$job: Completed successfully.\n";
 			    $jobs->{$job}->{complete} = 1;
-			    eval { qemu_blockjobs_finish_tunnel($vmid, $job, $jobs->{$job}->{pid}) } ;
 			}
 		    }
 		}
@@ -6150,7 +6666,6 @@ sub qemu_blockjobs_cancel {
 
 	    if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
 		print "$job: Done.\n";
-		eval { qemu_blockjobs_finish_tunnel($vmid, $job, $jobs->{$job}->{pid}) } ;
 		delete $jobs->{$job};
 	    }
 	}
@@ -6161,25 +6676,6 @@ sub qemu_blockjobs_cancel {
     }
 }
 
-sub qemu_blockjobs_finish_tunnel {
-   my ($vmid, $job, $cpid) = @_;
-
-   return if !$cpid;
-
-   for (my $i = 1; $i < 20; $i++) {
-	my $waitpid = waitpid($cpid, WNOHANG);
-	last if (defined($waitpid) && ($waitpid == $cpid));
- 
-	if ($i == 10) {
-	    kill(15, $cpid);
-	} elsif ($i >= 15) {
-	    kill(9, $cpid);
-	}
-	sleep (1);
-    }
-    unlink "/run/qemu-server/$vmid.mirror-$job";
-}
-
 sub clone_disk {
     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
 	$newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga) = @_;
@@ -6199,7 +6695,17 @@ sub clone_disk {
 	my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
 
 	print "create full clone of drive $drivename ($drive->{file})\n";
-	$newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, undef, ($size/1024));
+	my $name = undef;
+	if (drive_is_cloudinit($drive)) {
+	    $name = "vm-$newvmid-cloudinit";
+	    # cloudinit only supports raw and qcow2 atm:
+	    if ($dst_format eq 'qcow2') {
+		$name .= '.qcow2';
+	    } elsif ($dst_format ne 'raw') {
+		die "clone: unhandled format for cloudinit image\n";
+	    }
+	}
+	$newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
 	push @$newvollist, $newvolid;
 
 	PVE::Storage::activate_volumes($storecfg, [$newvolid]);
@@ -6259,7 +6765,7 @@ sub qemu_machine_feature_enabled {
     my $current_major;
     my $current_minor;
 
-    if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
+    if ($machine && $machine =~ m/^((?:pc(-i440fx|-q35)?|virt)-(\d+)\.(\d+))/) {
 
 	$current_major = $3;
 	$current_minor = $4;
@@ -6270,9 +6776,9 @@ sub qemu_machine_feature_enabled {
 	$current_minor = $2;
     }
 
-    return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
-
-
+    return 1 if $current_major > $version_major ||
+                ($current_major == $version_major &&
+                 $current_minor >= $version_minor);
 }
 
 sub qemu_machine_pxe {
@@ -6280,13 +6786,8 @@ sub qemu_machine_pxe {
 
     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
 
-    foreach my $opt (keys %$conf) {
-	next if $opt !~ m/^net(\d+)$/;
-	my $net = PVE::QemuServer::parse_net($conf->{$opt});
-	next if !$net;
-	my $romfile = PVE::QemuServer::vm_mon_cmd_nocheck($vmid, 'qom-get', path => $opt, property => 'romfile');
-	return $machine.".pxe" if $romfile =~ m/pxe/;
-	last;
+    if ($conf->{machine} && $conf->{machine} =~ m/\.pxe$/) {
+	$machine .= '.pxe';
     }
 
     return $machine;
@@ -6314,18 +6815,19 @@ sub qemu_use_old_bios_files {
     return ($use_old_bios_files, $machine_type);
 }
 
-sub create_efidisk {
-    my ($storecfg, $storeid, $vmid, $fmt) = @_;
+sub create_efidisk($$$$$) {
+    my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
 
-    die "EFI vars default image not found\n" if ! -f $OVMF_VARS;
+    my (undef, $ovmf_vars) = get_ovmf_files($arch);
+    die "EFI vars default image not found\n" if ! -f $ovmf_vars;
 
-    my $vars_size = PVE::Tools::convert_size(-s $OVMF_VARS, 'b' => 'kb');
+    my $vars_size = PVE::Tools::convert_size(-s $ovmf_vars, 'b' => 'kb');
     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
     PVE::Storage::activate_volumes($storecfg, [$volid]);
 
     my $path = PVE::Storage::path($storecfg, $volid);
     eval {
-	run_command(['/usr/bin/qemu-img', 'convert', '-n', '-f', 'raw', '-O', $fmt, $OVMF_VARS, $path]);
+	run_command(['/usr/bin/qemu-img', 'convert', '-n', '-f', 'raw', '-O', $fmt, $ovmf_vars, $path]);
     };
     die "Copying EFI vars image failed: $@" if $@;
 
@@ -6407,6 +6909,11 @@ sub add_hyperv_enlightenments {
 
     if ($winversion >= 7) {
 	push @$cpuFlags , 'hv_relaxed';
+
+	if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 12)) {
+	    push @$cpuFlags , 'hv_synic';
+	    push @$cpuFlags , 'hv_stimer';
+	}
     }
 }
 
@@ -6461,11 +6968,21 @@ sub resolve_first_disk {
     return $firstdisk;
 }
 
-sub generate_smbios1_uuid {
+sub generate_uuid {
     my ($uuid, $uuid_str);
     UUID::generate($uuid);
     UUID::unparse($uuid, $uuid_str);
-    return "uuid=$uuid_str";
+    return $uuid_str;
+}
+
+sub generate_smbios1_uuid {
+    return "uuid=".generate_uuid();
+}
+
+sub nbd_stop {
+    my ($vmid) = @_;
+
+    vm_mon_cmd($vmid, 'nbd-server-stop');
 }
 
 # bash completion helper
@@ -6541,10 +7058,4 @@ sub complete_storage {
     return $res;
 }
 
-sub nbd_stop {
-    my ($vmid) = @_;
-
-    vm_mon_cmd($vmid, 'nbd-server-stop');
-}
-
 1;