X-Git-Url: https://git.proxmox.com/?p=qemu-server.git;a=blobdiff_plain;f=PVE%2FQemuServer.pm;h=035d8b7084145d50283b34cb992bdde68a8b7350;hp=ba5511c68083c5be88b29d6484d5b9802b104be0;hb=b4496b9ed5994cb624806db4edf5dbc24576bd9e;hpb=646f2df43cd0ca92887ed612580569340216217b

diff --git a/PVE/QemuServer.pm b/PVE/QemuServer.pm
index ba5511c..035d8b7 100644
--- a/PVE/QemuServer.pm
+++ b/PVE/QemuServer.pm
@@ -40,8 +40,16 @@ use File::Copy qw(copy);
 use URI::Escape;
 
 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
-my $OVMF_CODE = "$EDK2_FW_BASE/OVMF_CODE.fd";
-my $OVMF_VARS = "$EDK2_FW_BASE/OVMF_VARS.fd";
+my $OVMF = {
+    x86_64 => [
+	"$EDK2_FW_BASE/OVMF_CODE.fd",
+	"$EDK2_FW_BASE/OVMF_VARS.fd"
+    ],
+    aarch64 => [
+	"$EDK2_FW_BASE/AAVMF_CODE.fd",
+	"$EDK2_FW_BASE/AAVMF_VARS.fd"
+    ],
+};
 
 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
 
@@ -83,7 +91,7 @@ PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
 	description => "Specifies the Qemu machine type.",
 	type => 'string',
-	pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
+	pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?|virt(?:-\d+\.\d+)?)',
 	maxLength => 40,
 	optional => 1,
 });
@@ -557,6 +565,12 @@ EODESCR
 	description => "Specifies the Qemu machine type of the running vm. This is used internally for snapshots.",
     }),
     machine => get_standard_option('pve-qemu-machine'),
+    arch => {
+	description => "Virtual processor architecture. Defaults to the host.",
+	optional => 1,
+	type => 'string',
+	enum => [qw(x86_64 aarch64)],
+    },
     smbios1 => {
 	description => "Specify SMBIOS type 1 fields.",
 	type => 'string', format => 'pve-qm-smbios1',
@@ -1676,24 +1690,37 @@ sub machine_type_is_q35 {
 }
 
 sub print_tabletdevice_full {
-    my ($conf) = @_;
+    my ($conf, $arch) = @_;
 
     my $q35 = machine_type_is_q35($conf);
 
     # we use uhci for old VMs because tablet driver was buggy in older qemu
-    my $usbbus = $q35 ? "ehci" : "uhci";
+    my $usbbus;
+    if (machine_type_is_q35($conf) || $arch eq 'aarch64') {
+	$usbbus = 'ehci';
+    } else {
+	$usbbus = 'uhci';
+    }
 
     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
 }
 
+sub print_keyboarddevice_full {
+    my ($conf, $arch, $machine) = @_;
+
+    return undef if $arch ne 'aarch64';
+
+    return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
+}
+
 sub print_drivedevice_full {
-    my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
+    my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
 
     my $device = '';
     my $maxdev = 0;
 
     if ($drive->{interface} eq 'virtio') {
-	my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
+	my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges, $arch, $machine_type);
 	$device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
 	$device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
     } elsif ($drive->{interface} eq 'scsi') {
@@ -1882,7 +1909,7 @@ sub print_drive_full {
 }
 
 sub print_netdevice_full {
-    my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
+    my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
 
     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
 
@@ -1891,7 +1918,7 @@ sub print_netdevice_full {
          $device = 'virtio-net-pci';
      };
 
-    my $pciaddr = print_pci_addr("$netid", $bridges);
+    my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
 	#Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
@@ -1920,7 +1947,7 @@ sub print_netdevice_full {
 }
 
 sub print_netdev_full {
-    my ($vmid, $conf, $net, $netid, $hotplug) = @_;
+    my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
 
     my $i = '';
     if ($netid =~ m/^net(\d+)$/) {
@@ -1936,7 +1963,9 @@ sub print_netdev_full {
         if length($ifname) >= 16;
 
     my $vhostparam = '';
-    $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
+    if (is_native($arch)) {
+	$vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
+    }
 
     my $vmname = $conf->{name} || "vm$vmid";
 
@@ -1982,9 +2011,12 @@ my $vga_map = {
 };
 
 sub print_vga_device {
-    my ($conf, $vga, $id, $qxlnum, $bridges) = @_;
+    my ($conf, $vga, $arch, $machine, $id, $qxlnum, $bridges) = @_;
 
     my $type = $vga_map->{$vga->{type}};
+    if ($type eq 'virtio-vga' && $arch eq 'aarch64') {
+	$type = 'virtio-gpu';
+    }
     my $vgamem_mb = $vga->{memory};
     if ($qxlnum) {
 	$type = $id ? 'qxl' : 'qxl-vga';
@@ -2015,9 +2047,9 @@ sub print_vga_device {
 
     if ($q35 && $vgaid eq 'vga') {
 	# the first display uses pcie.0 bus on q35 machines
-	$pciaddr = print_pcie_addr($vgaid, $bridges);
+	$pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
     } else {
-	$pciaddr = print_pci_addr($vgaid, $bridges);
+	$pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
     }
 
     return "$type,id=${vgaid}${memory}${pciaddr}";
@@ -2824,7 +2856,7 @@ sub check_cmdline {
 	my @param = split(/\0/, $line);
 
 	my $cmd = $param[0];
-	return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
+	return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m@(?:^|/)qemu-system-[^/]+$@);
 
 	for (my $i = 0; $i < scalar (@param); $i++) {
 	    my $p = $param[$i];
@@ -3245,6 +3277,106 @@ sub vga_conf_has_spice {
     return $1 || 1;
 }
 
+my $host_arch; # FIXME: fix PVE::Tools::get_host_arch
+sub get_host_arch() {
+    $host_arch = (POSIX::uname())[4] if !$host_arch;
+    return $host_arch;
+}
+
+sub is_native($) {
+    my ($arch) = @_;
+    return get_host_arch() eq $arch;
+}
+
+my $default_machines = {
+    x86_64 => 'pc',
+    aarch64 => 'virt',
+};
+
+sub get_basic_machine_info {
+    my ($conf, $forcemachine) = @_;
+
+    my $arch = $conf->{arch} // get_host_arch();
+    my $machine = $forcemachine || $conf->{machine} || $default_machines->{$arch};
+    return ($arch, $machine);
+}
+
+sub get_ovmf_files($) {
+    my ($arch) = @_;
+
+    my $ovmf = $OVMF->{$arch}
+	or die "no OVMF images known for architecture '$arch'\n";
+
+    return @$ovmf;
+}
+
+my $Arch2Qemu = {
+    aarch64 => '/usr/bin/qemu-system-aarch64',
+    x86_64 => '/usr/bin/qemu-system-x86_64',
+};
+sub get_command_for_arch($) {
+    my ($arch) = @_;
+    return '/usr/bin/kvm' if is_native($arch);
+
+    my $cmd = $Arch2Qemu->{$arch}
+	or die "don't know how to emulate architecture '$arch'\n";
+    return $cmd;
+}
+
+sub get_cpu_options {
+    my ($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough) = @_;
+
+    my $cpuFlags = [];
+    my $ostype = $conf->{ostype};
+
+    my $cpu = $kvm ? "kvm64" : "qemu64";
+    if ($arch eq 'aarch64') {
+	$cpu = 'cortex-a57';
+    }
+    if (my $cputype = $conf->{cpu}) {
+	my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
+	    or die "Cannot parse cpu description: $cputype\n";
+	$cpu = $cpuconf->{cputype};
+	$kvm_off = 1 if $cpuconf->{hidden};
+
+	if (defined(my $flags = $cpuconf->{flags})) {
+	    push @$cpuFlags, split(";", $flags);
+	}
+    }
+
+    push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64' && $arch eq 'x86_64';
+
+    push @$cpuFlags , '-x2apic'
+	if $conf->{ostype} && $conf->{ostype} eq 'solaris';
+
+    push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
+
+    push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
+
+    if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3) && $arch eq 'x86_64') {
+
+	push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
+	push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
+    }
+
+    add_hyperv_enlightenments($cpuFlags, $winversion, $machine_type, $kvmver, $conf->{bios}, $gpu_passthrough) if $kvm;
+
+    push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm && $arch eq 'x86_64';
+
+    push @$cpuFlags, 'kvm=off' if $kvm_off;
+
+    if (my $cpu_vendor = $cpu_vendor_list->{$cpu}) {
+	push @$cpuFlags, "vendor=${cpu_vendor}"
+	    if $cpu_vendor ne 'default';
+    } elsif ($arch ne 'aarch64') {
+	die "internal error"; # should not happen
+    }
+
+    $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
+
+    return ('-cpu', $cpu);
+}
+
 sub config_to_command {
     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
 
@@ -3252,7 +3384,6 @@ sub config_to_command {
     my $globalFlags = [];
     my $machineFlags = [];
     my $rtcFlags = [];
-    my $cpuFlags = [];
     my $devices = [];
     my $pciaddr = '';
     my $bridges = {};
@@ -3260,9 +3391,15 @@ sub config_to_command {
     my $vernum = 0; # unknown
     my $ostype = $conf->{ostype};
     my $winversion = windows_version($ostype);
-    my $kvm = $conf->{kvm} // 1;
+    my $kvm = $conf->{kvm};
+
+    my ($arch, $machine_type) = get_basic_machine_info($conf, $forcemachine);
+    $kvm //= 1 if is_native($arch);
 
-    die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n" if (!$cpuinfo->{hvm} && $kvm);
+    if ($kvm) {
+	die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n"
+	    if !defined kvm_version();
+    }
 
     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
 	$vernum = $1*1000000+$2*1000;
@@ -3276,14 +3413,13 @@ sub config_to_command {
 
     my $q35 = machine_type_is_q35($conf);
     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
-    my $machine_type = $forcemachine || $conf->{machine};
     my $use_old_bios_files = undef;
     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
 
     my $cpuunits = defined($conf->{cpuunits}) ?
             $conf->{cpuunits} : $defaults->{cpuunits};
 
-    push @$cmd, '/usr/bin/kvm';
+    push @$cmd, get_command_for_arch($arch);
 
     push @$cmd, '-id', $vmid;
 
@@ -3298,8 +3434,7 @@ sub config_to_command {
     push @$cmd, '-mon', "chardev=qmp,mode=control";
 
     if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 12)) {
-	my $eventsocket = qmp_socket($vmid, 0, 'event');
-	push @$cmd, '-chardev', "socket,id=qmp-event,path=$eventsocket,server,nowait";
+	push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
 	push @$cmd, '-mon', "chardev=qmp-event,mode=control";
     }
 
@@ -3315,8 +3450,9 @@ sub config_to_command {
 	push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
     }
 
+    my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
-	die "uefi base image not found\n" if ! -f $OVMF_CODE;
+	die "uefi base image not found\n" if ! -f $ovmf_code;
 
 	my $path;
 	my $format;
@@ -3338,17 +3474,17 @@ sub config_to_command {
 	} else {
 	    warn "no efidisk configured! Using temporary efivars disk.\n";
 	    $path = "/tmp/$vmid-ovmf.fd";
-	    PVE::Tools::file_copy($OVMF_VARS, $path, -s $OVMF_VARS);
+	    PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
 	    $format = 'raw';
 	}
 
-	push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$OVMF_CODE";
+	push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
 	push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
     }
 
 
     # add usb controllers
-    my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $q35, $usbdesc->{format}, $MAX_USB_DEVICES);
+    my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
     push @$devices, @usbcontrollers if @usbcontrollers;
     my $vga = parse_vga($conf->{vga});
 
@@ -3356,7 +3492,9 @@ sub config_to_command {
     $vga->{type} = 'qxl' if $qxlnum;
 
     if (!$vga->{type}) {
-	if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 9)) {
+	if ($arch eq 'aarch64') {
+	    $vga->{type} = 'virtio';
+	} elsif (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 9)) {
 	    $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
 	} else {
 	    $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
@@ -3373,7 +3511,11 @@ sub config_to_command {
 	$tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
     }
 
-    push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
+    if ($tablet) {
+	push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
+	my $kbd = print_keyboarddevice_full($conf, $arch);
+	push @$devices, '-device', $kbd if defined($kbd);
+    }
 
     my $kvm_off = 0;
     my $gpu_passthrough;
@@ -3388,7 +3530,7 @@ sub config_to_command {
 	    die "q35 machine model is not enabled" if !$q35;
 	    $pciaddr = print_pcie_addr("hostpci$i");
 	}else{
-	    $pciaddr = print_pci_addr("hostpci$i", $bridges);
+	    $pciaddr = print_pci_addr("hostpci$i", $bridges, $arch, $machine_type);
 	}
 
 	my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
@@ -3437,7 +3579,14 @@ sub config_to_command {
 	    if ($path eq 'socket') {
 		my $socket = "/var/run/qemu-server/${vmid}.serial$i";
 		push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
-		push @$devices, '-device', "isa-serial,chardev=serial$i";
+		# On aarch64, serial0 is the UART device. Qemu only allows
+		# connecting UART devices via the '-serial' command line, as
+		# the device has a fixed slot on the hardware...
+		if ($arch eq 'aarch64' && $i == 0) {
+		    push @$devices, '-serial', "chardev:serial$i";
+		} else {
+		    push @$devices, '-device', "isa-serial,chardev=serial$i";
+		}
 	    } else {
 		die "no such serial device\n" if ! -c $path;
 		push @$devices, '-chardev', "tty,id=serial$i,path=$path";
@@ -3502,7 +3651,7 @@ sub config_to_command {
     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
 
     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
-	push @$devices, '-device', print_vga_device($conf, $vga, undef, $qxlnum, $bridges);
+	push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_type, undef, $qxlnum, $bridges);
 	my $socket = vnc_socket($vmid);
 	push @$cmd,  '-vnc', "unix:$socket,x509,password";
     } else {
@@ -3545,48 +3694,7 @@ sub config_to_command {
 	push @$rtcFlags, 'base=localtime';
     }
 
-    my $cpu = $kvm ? "kvm64" : "qemu64";
-    if (my $cputype = $conf->{cpu}) {
-	my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
-	    or die "Cannot parse cpu description: $cputype\n";
-	$cpu = $cpuconf->{cputype};
-	$kvm_off = 1 if $cpuconf->{hidden};
-
-	if (defined(my $flags = $cpuconf->{flags})) {
-	    push @$cpuFlags, split(";", $flags);
-	}
-    }
-
-    push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
-
-    push @$cpuFlags , '-x2apic'
-	if $conf->{ostype} && $conf->{ostype} eq 'solaris';
-
-    push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
-
-    push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
-
-    if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
-
-	push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
-	push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
-    }
-
-    add_hyperv_enlightenments($cpuFlags, $winversion, $machine_type, $kvmver, $conf->{bios}, $gpu_passthrough) if $kvm;
-
-    push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm;
-
-    push @$cpuFlags, 'kvm=off' if $kvm_off;
-
-    my $cpu_vendor = $cpu_vendor_list->{$cpu} ||
-	die "internal error"; # should not happen
-
-    push @$cpuFlags, "vendor=${cpu_vendor}"
-	if $cpu_vendor ne 'default';
-
-    $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
-
-    push @$cmd, '-cpu', $cpu;
+    push @$cmd, get_cpu_options($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough);
 
     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
     
@@ -3601,7 +3709,7 @@ sub config_to_command {
 
     if (parse_guest_agent($conf)->{enabled}) {
 	my $qgasocket = qmp_socket($vmid, 1);
-	my $pciaddr = print_pci_addr("qga0", $bridges);
+	my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
 	push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
 	push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
 	push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
@@ -3613,7 +3721,7 @@ sub config_to_command {
 	if ($qxlnum > 1) {
 	    if ($winversion){
 		for(my $i = 1; $i < $qxlnum; $i++){
-		    push @$devices, '-device', print_vga_device($conf, $vga, $i, $qxlnum, $bridges);
+		    push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_type, $i, $qxlnum, $bridges);
 		}
 	    } else {
 		# assume other OS works like Linux
@@ -3627,7 +3735,7 @@ sub config_to_command {
 	    }
 	}
 
-	my $pciaddr = print_pci_addr("spice", $bridges);
+	my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
 
 	my $nodename = PVE::INotify::nodename();
 	my $pfamily = PVE::Tools::get_host_address_family($nodename);
@@ -3645,13 +3753,13 @@ sub config_to_command {
 
     # enable balloon by default, unless explicitly disabled
     if (!defined($conf->{balloon}) || $conf->{balloon}) {
-	$pciaddr = print_pci_addr("balloon0", $bridges);
+	$pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
 	push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
     }
 
     if ($conf->{watchdog}) {
 	my $wdopts = parse_watchdog($conf->{watchdog});
-	$pciaddr = print_pci_addr("watchdog", $bridges);
+	$pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
 	my $watchdog = $wdopts->{model} || 'i6300esb';
 	push @$devices, '-device', "$watchdog$pciaddr";
 	push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
@@ -3699,7 +3807,7 @@ sub config_to_command {
 
 	    my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
 
-	    $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
+	    $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
 	    my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
 
 	    my $iothread = '';
@@ -3721,14 +3829,14 @@ sub config_to_command {
 
         if ($drive->{interface} eq 'sata') {
            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
-           $pciaddr = print_pci_addr("ahci$controller", $bridges);
+           $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
            $ahcicontroller->{$controller}=1;
         }
 
 	my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
 	push @$devices, '-drive',$drive_cmd;
-	push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
+	push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
     });
 
     for (my $i = 0; $i < $MAX_NETS; $i++) {
@@ -3743,10 +3851,10 @@ sub config_to_command {
             $bootindex_hash->{n} += 1;
          }
 
-         my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
+         my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, "net$i");
          push @$devices, '-netdev', $netdevfull;
 
-         my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
+         my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files, $arch, $machine_type);
          push @$devices, '-device', $netdevicefull;
     }
 
@@ -3760,7 +3868,7 @@ sub config_to_command {
 	$bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
 
 	while (my ($k, $v) = each %$bridges) {
-	    $pciaddr = print_pci_addr("pci.$k");
+	    $pciaddr = print_pci_addr("pci.$k", undef, $arch, $machine_type);
 	    unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
 	}
     }
@@ -3858,18 +3966,22 @@ sub vm_devices_list {
 }
 
 sub vm_deviceplug {
-    my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
+    my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
 
     my $q35 = machine_type_is_q35($conf);
 
     my $devices_list = vm_devices_list($vmid);
     return 1 if defined($devices_list->{$deviceid});
 
-    qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
+    qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type); # add PCI bridge if we need it for the device
 
     if ($deviceid eq 'tablet') {
 
-	qemu_deviceadd($vmid, print_tabletdevice_full($conf));
+	qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
+
+    } elsif ($deviceid eq 'keyboard') {
+
+	qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
 
     } elsif ($deviceid =~ m/^usb(\d+)$/) {
 
@@ -3884,7 +3996,7 @@ sub vm_deviceplug {
 	qemu_iothread_add($vmid, $deviceid, $device);
 
         qemu_driveadd($storecfg, $vmid, $device);
-        my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
+        my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
 
         qemu_deviceadd($vmid, $devicefull);
 	eval { qemu_deviceaddverify($vmid, $deviceid); };
@@ -3898,7 +4010,7 @@ sub vm_deviceplug {
 
 
         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
-        my $pciaddr = print_pci_addr($deviceid);
+        my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
 	my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
 
         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
@@ -3917,10 +4029,10 @@ sub vm_deviceplug {
 
     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
 
-        qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
+        qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
         qemu_driveadd($storecfg, $vmid, $device);
 
-	my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
+	my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
 	eval { qemu_deviceadd($vmid, $devicefull); };
 	if (my $err = $@) {
 	    eval { qemu_drivedel($vmid, $deviceid); };
@@ -3930,13 +4042,13 @@ sub vm_deviceplug {
 
     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
 
-        return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
+        return undef if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
 
         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf); 
         my $use_old_bios_files = undef;
         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
 
-        my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
+        my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
         qemu_deviceadd($vmid, $netdevicefull);
         eval { qemu_deviceaddverify($vmid, $deviceid); };
 	if (my $err = $@) {
@@ -3948,7 +4060,7 @@ sub vm_deviceplug {
     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
 
 	my $bridgeid = $2;
-	my $pciaddr = print_pci_addr($deviceid);
+	my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
 	my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
 
 	qemu_deviceadd($vmid, $devicefull);
@@ -3970,7 +4082,7 @@ sub vm_deviceunplug {
 
     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
 
-    if ($deviceid eq 'tablet') {
+    if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
 
 	qemu_devicedel($vmid, $deviceid);
 
@@ -4121,7 +4233,7 @@ sub qemu_devicedelverify {
 }
 
 sub qemu_findorcreatescsihw {
-    my ($storecfg, $conf, $vmid, $device) = @_;
+    my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
 
     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
 
@@ -4129,7 +4241,7 @@ sub qemu_findorcreatescsihw {
     my $devices_list = vm_devices_list($vmid);
 
     if(!defined($devices_list->{$scsihwid})) {
-	vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
+	vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
     }
 
     return 1;
@@ -4165,13 +4277,13 @@ sub qemu_deletescsihw {
 }
 
 sub qemu_add_pci_bridge {
-    my ($storecfg, $conf, $vmid, $device) = @_;
+    my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
 
     my $bridges = {};
 
     my $bridgeid;
 
-    print_pci_addr($device, $bridges);
+    print_pci_addr($device, $bridges, $arch, $machine_type);
 
     while (my ($k, $v) = each %$bridges) {
 	$bridgeid = $k;
@@ -4182,7 +4294,7 @@ sub qemu_add_pci_bridge {
     my $devices_list = vm_devices_list($vmid);
 
     if (!defined($devices_list->{$bridge})) {
-	vm_deviceplug($storecfg, $conf, $vmid, $bridge);
+	vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
     }
 
     return 1;
@@ -4196,9 +4308,9 @@ sub qemu_set_link_status {
 }
 
 sub qemu_netdevadd {
-    my ($vmid, $conf, $device, $deviceid) = @_;
+    my ($vmid, $conf, $arch, $device, $deviceid) = @_;
 
-    my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
+    my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
     my %options =  split(/[=,]/, $netdev);
 
     vm_mon_cmd($vmid, "netdev_add",  %options);
@@ -4212,7 +4324,7 @@ sub qemu_netdevdel {
 }
 
 sub qemu_usb_hotplug {
-    my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
+    my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
 
     return if !$device;
 
@@ -4225,7 +4337,7 @@ sub qemu_usb_hotplug {
 	my $devicelist = vm_devices_list($vmid);
 
 	if (!$devicelist->{xhci}) {
-	    my $pciaddr = print_pci_addr("xhci");
+	    my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
 	    qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
 	}
     }
@@ -4233,7 +4345,7 @@ sub qemu_usb_hotplug {
     $d->{usb3} = $device->{usb3};
 
     # add the new one
-    vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d);
+    vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
 }
 
 sub qemu_cpu_hotplug {
@@ -4467,6 +4579,7 @@ sub vmconfig_hotplug_pending {
     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
 
     my $defaults = load_defaults();
+    my ($arch, $machine_type) = get_basic_machine_info($conf, undef);
 
     # commit values which do not have any impact on running VM first
     # Note: those option cannot raise errors, we we do not care about
@@ -4502,9 +4615,12 @@ sub vmconfig_hotplug_pending {
 	    } elsif ($opt eq 'tablet') {
 		die "skip\n" if !$hotplug_features->{usb};
 		if ($defaults->{tablet}) {
-		    vm_deviceplug($storecfg, $conf, $vmid, $opt);
+		    vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
+		    vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
+			if $arch eq 'aarch64';
 		} else {
-		    vm_deviceunplug($vmid, $conf, $opt);
+		    vm_deviceunplug($vmid, $conf, 'tablet');
+		    vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
 		}
 	    } elsif ($opt =~ m/^usb\d+/) {
 		die "skip\n";
@@ -4577,9 +4693,12 @@ sub vmconfig_hotplug_pending {
 	    } elsif ($opt eq 'tablet') {
 		die "skip\n" if !$hotplug_features->{usb};
 		if ($value == 1) {
-		    vm_deviceplug($storecfg, $conf, $vmid, $opt);
+		    vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
+		    vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
+			if $arch eq 'aarch64';
 		} elsif ($value == 0) {
-		    vm_deviceunplug($vmid, $conf, $opt);
+		    vm_deviceunplug($vmid, $conf, 'tablet');
+		    vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
 		}
 	    } elsif ($opt =~ m/^usb\d+$/) {
 		die "skip\n";
@@ -4588,7 +4707,7 @@ sub vmconfig_hotplug_pending {
 		die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
 		my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
 		die "skip\n" if !$d;
-		qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d);
+		qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
 	    } elsif ($opt eq 'vcpus') {
 		die "skip\n" if !$hotplug_features->{cpu};
 		qemu_cpu_hotplug($vmid, $conf, $value);
@@ -4606,7 +4725,7 @@ sub vmconfig_hotplug_pending {
 	    } elsif ($opt =~ m/^net(\d+)$/) {
 		# some changes can be done without hotplug
 		vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
-				    $vmid, $opt, $value);
+				    $vmid, $opt, $value, $arch, $machine_type);
 	    } elsif (is_valid_drivename($opt)) {
 		# some changes can be done without hotplug
 		my $drive = parse_drive($opt, $value);
@@ -4614,7 +4733,7 @@ sub vmconfig_hotplug_pending {
 		    &$apply_pending_cloudinit($opt, $value);
 		}
 		vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
-				     $vmid, $opt, $value, 1);
+				     $vmid, $opt, $value, 1, $arch, $machine_type);
 	    } elsif ($opt =~ m/^memory$/) { #dimms
 		die "skip\n" if !$hotplug_features->{memory};
 		$value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
@@ -4743,7 +4862,7 @@ my $safe_string_ne = sub {
 };
 
 sub vmconfig_update_net {
-    my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
+    my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
 
     my $newnet = parse_net($value);
 
@@ -4784,14 +4903,14 @@ sub vmconfig_update_net {
     }
 
     if ($hotplug) {
-	vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
+	vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
     } else {
 	die "skip\n";
     }
 }
 
 sub vmconfig_update_disk {
-    my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
+    my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force, $arch, $machine_type) = @_;
 
     # fixme: do we need force?
 
@@ -4892,7 +5011,7 @@ sub vmconfig_update_disk {
     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
     # hotplug new disks
     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
-    vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
+    vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
 }
 
 sub vm_start {
@@ -6690,7 +6809,7 @@ sub qemu_machine_feature_enabled {
     my $current_major;
     my $current_minor;
 
-    if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
+    if ($machine && $machine =~ m/^((?:pc(-i440fx|-q35)?|virt)-(\d+)\.(\d+))/) {
 
 	$current_major = $3;
 	$current_minor = $4;
@@ -6740,18 +6859,19 @@ sub qemu_use_old_bios_files {
     return ($use_old_bios_files, $machine_type);
 }
 
-sub create_efidisk {
-    my ($storecfg, $storeid, $vmid, $fmt) = @_;
+sub create_efidisk($$$$$) {
+    my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
 
-    die "EFI vars default image not found\n" if ! -f $OVMF_VARS;
+    my (undef, $ovmf_vars) = get_ovmf_files($arch);
+    die "EFI vars default image not found\n" if ! -f $ovmf_vars;
 
-    my $vars_size = PVE::Tools::convert_size(-s $OVMF_VARS, 'b' => 'kb');
+    my $vars_size = PVE::Tools::convert_size(-s $ovmf_vars, 'b' => 'kb');
     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
     PVE::Storage::activate_volumes($storecfg, [$volid]);
 
     my $path = PVE::Storage::path($storecfg, $volid);
     eval {
-	run_command(['/usr/bin/qemu-img', 'convert', '-n', '-f', 'raw', '-O', $fmt, $OVMF_VARS, $path]);
+	run_command(['/usr/bin/qemu-img', 'convert', '-n', '-f', 'raw', '-O', $fmt, $ovmf_vars, $path]);
     };
     die "Copying EFI vars image failed: $@" if $@;