From: Fiona Ebner Date: Fri, 3 May 2024 12:01:35 +0000 (+0200) Subject: schema: fix description of migrate_downtime parameter X-Git-Url: https://git.proxmox.com/?p=qemu-server.git;a=commitdiff_plain;h=HEAD;hp=edae17185b11ba8c9e88b10c29eacadbf87d6456 schema: fix description of migrate_downtime parameter Since commit 865ef132 ("implement dynamic migration_downtime") the migration downtime will be automatically increased when migration cannot converge at the very end. Update the description to reflect reality. Signed-off-by: Fiona Ebner --- diff --git a/.gitignore b/.gitignore index e48cf98..3658c85 100644 --- a/.gitignore +++ b/.gitignore @@ -1,11 +1,13 @@ -*.deb -*.buildinfo -*.changes *.1.gz *.1.pod *.5.gz *.5.pod -sparsecp -vmtar +*.buildinfo +*.changes +*.deb +/.vscode/ +/qemu-server-[0-9]*/ build qm.bash-completion +sparsecp +vmtar diff --git a/Makefile b/Makefile index 457eaef..133468d 100644 --- a/Makefile +++ b/Makefile @@ -1,39 +1,36 @@ -include /usr/share/dpkg/pkg-info.mk -include /usr/share/dpkg/architecture.mk +include /usr/share/dpkg/default.mk PACKAGE=qemu-server -BUILDDIR ?= ${PACKAGE}-${DEB_VERSION_UPSTREAM} +BUILDDIR ?= $(PACKAGE)-$(DEB_VERSION_UPSTREAM) DESTDIR= PREFIX=/usr -SBINDIR=${PREFIX}/sbin -LIBDIR=${PREFIX}/lib/${PACKAGE} -MANDIR=${PREFIX}/share/man -DOCDIR=${PREFIX}/share/doc -MAN1DIR=${MANDIR}/man1/ -MAN5DIR=${MANDIR}/man5/ -BASHCOMPLDIR=${PREFIX}/share/bash-completion/completions/ -ZSHCOMPLDIR=${PREFIX}/share/zsh/vendor-completions/ -export PERLDIR=${PREFIX}/share/perl5 -PERLINCDIR=${PERLDIR}/asm-x86_64 +SBINDIR=$(PREFIX)/sbin +LIBDIR=$(PREFIX)/lib/$(PACKAGE) +MANDIR=$(PREFIX)/share/man +DOCDIR=$(PREFIX)/share/doc +MAN1DIR=$(MANDIR)/man1/ +MAN5DIR=$(MANDIR)/man5/ +BASHCOMPLDIR=$(PREFIX)/share/bash-completion/completions/ +ZSHCOMPLDIR=$(PREFIX)/share/zsh/vendor-completions/ +export PERLDIR=$(PREFIX)/share/perl5 +PERLINCDIR=$(PERLDIR)/asm-x86_64 GITVERSION:=$(shell git rev-parse HEAD) -DEB=${PACKAGE}_${DEB_VERSION_UPSTREAM_REVISION}_${DEB_BUILD_ARCH}.deb -DBG_DEB=${PACKAGE}-dbgsym_${DEB_VERSION_UPSTREAM_REVISION}_${DEB_BUILD_ARCH}.deb -DSC=${PACKAGE}_${DEB_VERSION_UPSTREAM_REVISION}.dsc +DEB=$(PACKAGE)_$(DEB_VERSION_UPSTREAM_REVISION)_$(DEB_BUILD_ARCH).deb +DBG_DEB=$(PACKAGE)-dbgsym_$(DEB_VERSION_UPSTREAM_REVISION)_$(DEB_BUILD_ARCH).deb +DSC=$(PACKAGE)_$(DEB_VERSION_UPSTREAM_REVISION).dsc -DEBS=${DEB} ${DBG_DEB} +DEBS=$(DEB) $(DBG_DEB) -# this requires package pve-doc-generator -export NOVIEW=1 -include /usr/share/pve-doc-generator/pve-doc-generator.mk +-include /usr/share/pve-doc-generator/pve-doc-generator.mk all: .PHONY: dinstall dinstall: deb - dpkg -i ${DEB} + dpkg -i $(DEB) qm.bash-completion: PVE_GENERATING_DOCS=1 perl -I. -T -e "use PVE::CLI::qm; PVE::CLI::qm->generate_bash_completions();" >$@.tmp @@ -55,48 +52,52 @@ PKGSOURCES=qm qm.1 qmrestore qmrestore.1 qmextract qm.conf.5 qm.bash-completion qm.zsh-completion qmrestore.zsh-completion cpu-models.conf.5 .PHONY: install -install: ${PKGSOURCES} - install -d ${DESTDIR}/${SBINDIR} - install -d ${DESTDIR}${LIBDIR} - install -d ${DESTDIR}/${MAN1DIR} - install -d ${DESTDIR}/${MAN5DIR} - install -d ${DESTDIR}/usr/share/${PACKAGE} - install -m 0644 -D qm.bash-completion ${DESTDIR}/${BASHCOMPLDIR}/qm - install -m 0644 -D qmrestore.bash-completion ${DESTDIR}/${BASHCOMPLDIR}/qmrestore - install -m 0644 -D qm.zsh-completion ${DESTDIR}/${ZSHCOMPLDIR}/_qm - install -m 0644 -D qmrestore.zsh-completion ${DESTDIR}/${ZSHCOMPLDIR}/_qmrestore - install -m 0644 -D bootsplash.jpg ${DESTDIR}/usr/share/${PACKAGE} +install: $(PKGSOURCES) + install -d $(DESTDIR)/$(SBINDIR) + install -d $(DESTDIR)$(LIBDIR) + install -d $(DESTDIR)/$(MAN1DIR) + install -d $(DESTDIR)/$(MAN5DIR) + install -d $(DESTDIR)/usr/share/$(PACKAGE) + install -m 0644 -D qm.bash-completion $(DESTDIR)/$(BASHCOMPLDIR)/qm + install -m 0644 -D qmrestore.bash-completion $(DESTDIR)/$(BASHCOMPLDIR)/qmrestore + install -m 0644 -D qm.zsh-completion $(DESTDIR)/$(ZSHCOMPLDIR)/_qm + install -m 0644 -D qmrestore.zsh-completion $(DESTDIR)/$(ZSHCOMPLDIR)/_qmrestore + install -m 0644 -D bootsplash.jpg $(DESTDIR)/usr/share/$(PACKAGE) $(MAKE) -C PVE install $(MAKE) -C qmeventd install $(MAKE) -C qemu-configs install $(MAKE) -C vm-network-scripts install - install -m 0755 qm ${DESTDIR}${SBINDIR} - install -m 0755 qmrestore ${DESTDIR}${SBINDIR} - install -D -m 0644 modules-load.conf ${DESTDIR}/etc/modules-load.d/qemu-server.conf - install -m 0755 qmextract ${DESTDIR}${LIBDIR} - install -m 0644 qm.1 ${DESTDIR}/${MAN1DIR} - install -m 0644 qmrestore.1 ${DESTDIR}/${MAN1DIR} - install -m 0644 cpu-models.conf.5 ${DESTDIR}/${MAN5DIR} - install -m 0644 qm.conf.5 ${DESTDIR}/${MAN5DIR} - cd ${DESTDIR}/${MAN5DIR}; ln -s -f qm.conf.5.gz vm.conf.5.gz - -${BUILDDIR}: - rm -rf $(BUILDDIR) - rsync -a * $(BUILDDIR) - echo "git clone git://git.proxmox.com/git/qemu-server.git\\ngit checkout $(GITVERSION)" > $(BUILDDIR)/debian/SOURCE + install -m 0755 qm $(DESTDIR)$(SBINDIR) + install -m 0755 qmrestore $(DESTDIR)$(SBINDIR) + install -D -m 0644 modules-load.conf $(DESTDIR)/etc/modules-load.d/qemu-server.conf + install -m 0755 qmextract $(DESTDIR)$(LIBDIR) + install -m 0644 qm.1 $(DESTDIR)/$(MAN1DIR) + install -m 0644 qmrestore.1 $(DESTDIR)/$(MAN1DIR) + install -m 0644 cpu-models.conf.5 $(DESTDIR)/$(MAN5DIR) + install -m 0644 qm.conf.5 $(DESTDIR)/$(MAN5DIR) + cd $(DESTDIR)/$(MAN5DIR); ln -s -f qm.conf.5.gz vm.conf.5.gz + +$(BUILDDIR): + rm -rf $(BUILDDIR) $(BUILDDIR).tmp + rsync -a * $(BUILDDIR).tmp + echo "git clone git://git.proxmox.com/git/qemu-server.git\\ngit checkout $(GITVERSION)" > $(BUILDDIR).tmp/debian/SOURCE + mv $(BUILDDIR).tmp $(BUILDDIR) .PHONY: deb -deb: ${DEBS} -${DBG_DEB}: ${DEB} -${DEB}: $(BUILDDIR) +deb: $(DEBS) +$(DBG_DEB): $(DEB) +$(DEB): $(BUILDDIR) cd $(BUILDDIR); dpkg-buildpackage -b -us -uc - lintian ${DEBS} + lintian $(DEBS) .PHONY: dsc -dsc: ${DSC} -${DSC}: ${BUILDDIR} - cd ${BUILDDIR}; dpkg-buildpackage -S -us -uc -d - lintian ${DSC} +dsc: $(DSC) +$(DSC): $(BUILDDIR) + cd $(BUILDDIR); dpkg-buildpackage -S -us -uc -d + lintian $(DSC) + +sbuild: $(DSC) + sbuild $(DSC) .PHONY: test test: @@ -104,14 +105,15 @@ test: $(MAKE) -C test .PHONY: upload -upload: ${DEB} - tar cf - ${DEBS} | ssh -X repoman@repo.proxmox.com upload --product pve --dist buster +upload: UPLOAD_DIST ?= $(DEB_DISTRIBUTION) +upload: $(DEB) + tar cf - $(DEBS) | ssh -X repoman@repo.proxmox.com upload --product pve --dist $(UPLOAD_DIST) .PHONY: clean clean: - rm -rf $(PACKAGE)-*/ *.deb *.buildinfo *.changes *.dsc $(PACKAGE)_*.tar.gz - $(MAKE) cleanup-docgen - find . -name '*~' -exec rm {} ';' + $(MAKE) -C test $@ + rm -rf $(PACKAGE)-*/ *.deb *.build *.buildinfo *.changes *.dsc $(PACKAGE)_*.tar.?z + rm -f *.xml.tmp *.1 *.5 *.8 *{synopsis,opts}.adoc docinfo.xml .PHONY: distclean diff --git a/PVE/API2/Qemu.pm b/PVE/API2/Qemu.pm index e8de4ea..2a349c8 100644 --- a/PVE/API2/Qemu.pm +++ b/PVE/API2/Qemu.pm @@ -4,11 +4,16 @@ use strict; use warnings; use Cwd 'abs_path'; use Net::SSLeay; -use POSIX; use IO::Socket::IP; +use IO::Socket::UNIX; +use IPC::Open3; +use JSON; use URI::Escape; use Crypt::OpenSSL::Random; +use Socket qw(SOCK_STREAM); +use PVE::APIClient::LWP; +use PVE::CGroup; use PVE::Cluster qw (cfs_read_file cfs_write_file);; use PVE::RRD; use PVE::SafeSyslog; @@ -18,12 +23,18 @@ use PVE::Storage; use PVE::JSONSchema qw(get_standard_option); use PVE::RESTHandler; use PVE::ReplicationConfig; -use PVE::GuestHelpers; +use PVE::GuestHelpers qw(assert_tag_permissions); use PVE::QemuConfig; use PVE::QemuServer; -use PVE::QemuServer::Drive; +use PVE::QemuServer::Cloudinit; use PVE::QemuServer::CPUConfig; +use PVE::QemuServer::Drive; +use PVE::QemuServer::ImportDisk; use PVE::QemuServer::Monitor qw(mon_cmd); +use PVE::QemuServer::Machine; +use PVE::QemuServer::Memory qw(get_current_memory); +use PVE::QemuServer::PCI; +use PVE::QemuServer::USB; use PVE::QemuMigrate; use PVE::RPCEnvironment; use PVE::AccessControl; @@ -35,6 +46,9 @@ use PVE::API2::Qemu::Agent; use PVE::VZDump::Plugin; use PVE::DataCenterConfig; use PVE::SSHInfo; +use PVE::Replication; +use PVE::StorageTunnel; +use PVE::RESTEnvironment qw(log_warn); BEGIN { if (!$ENV{PVE_GENERATING_DOCS}) { @@ -45,8 +59,6 @@ BEGIN { } } -use Data::Dumper; # fixme: remove - use base qw(PVE::RESTHandler); my $opt_force_description = "Force physical removal. Without this, we simple remove the disk from the config file and create an additional configuration entry called 'unused[n]', which contains the volume ID. Unlink of unused[n] always cause physical removal."; @@ -61,11 +73,63 @@ my $resolve_cdrom_alias = sub { } }; -my $NEW_DISK_RE = qr!^(([^/:\s]+):)?(\d+(\.\d+)?)$!; +# Used in import-enabled API endpoints. Parses drives using the extended '_with_alloc' schema. +my $foreach_volume_with_alloc = sub { + my ($param, $func) = @_; + + for my $opt (sort keys $param->%*) { + next if !PVE::QemuServer::is_valid_drivename($opt); + + my $drive = PVE::QemuServer::Drive::parse_drive($opt, $param->{$opt}, 1); + next if !$drive; + + $func->($opt, $drive); + } +}; + +my $check_drive_param = sub { + my ($param, $storecfg, $extra_checks) = @_; + + for my $opt (sort keys $param->%*) { + next if !PVE::QemuServer::is_valid_drivename($opt); + + my $drive = PVE::QemuServer::parse_drive($opt, $param->{$opt}, 1); + raise_param_exc({ $opt => "unable to parse drive options" }) if !$drive; + + if ($drive->{'import-from'}) { + if ($drive->{file} !~ $PVE::QemuServer::Drive::NEW_DISK_RE || $3 != 0) { + raise_param_exc({ + $opt => "'import-from' requires special syntax - ". + "use :0,import-from=", + }); + } + + if ($opt eq 'efidisk0') { + for my $required (qw(efitype pre-enrolled-keys)) { + if (!defined($drive->{$required})) { + raise_param_exc({ + $opt => "need to specify '$required' when using 'import-from'", + }); + } + } + } elsif ($opt eq 'tpmstate0') { + raise_param_exc({ $opt => "need to specify 'version' when using 'import-from'" }) + if !defined($drive->{version}); + } + } + + PVE::QemuServer::cleanup_drive_path($opt, $storecfg, $drive); + + $extra_checks->($drive) if $extra_checks; + + $param->{$opt} = PVE::QemuServer::print_drive($drive, 1); + } +}; + my $check_storage_access = sub { my ($rpcenv, $authuser, $storecfg, $vmid, $settings, $default_storage) = @_; - PVE::QemuConfig->foreach_volume($settings, sub { + $foreach_volume_with_alloc->($settings, sub { my ($ds, $drive) = @_; my $isCDROM = PVE::QemuServer::drive_is_cdrom($drive); @@ -77,7 +141,7 @@ my $check_storage_access = sub { # nothing to check } elsif ($isCDROM && ($volid eq 'cdrom')) { $rpcenv->check($authuser, "/", ['Sys.Console']); - } elsif (!$isCDROM && ($volid =~ $NEW_DISK_RE)) { + } elsif (!$isCDROM && ($volid =~ $PVE::QemuServer::Drive::NEW_DISK_RE)) { my ($storeid, $size) = ($2 || $default_storage, $3); die "no storage ID specified (and no default storage)\n" if !$storeid; $rpcenv->check($authuser, "/storage/$storeid", ['Datastore.AllocateSpace']); @@ -86,6 +150,26 @@ my $check_storage_access = sub { if !$scfg->{content}->{images}; } else { PVE::Storage::check_volume_access($rpcenv, $authuser, $storecfg, $vmid, $volid); + if ($storeid) { + my ($vtype) = PVE::Storage::parse_volname($storecfg, $volid); + raise_param_exc({ $ds => "content type needs to be 'images' or 'iso'" }) + if $vtype ne 'images' && $vtype ne 'iso'; + } + } + + if (my $src_image = $drive->{'import-from'}) { + my $src_vmid; + if (PVE::Storage::parse_volume_id($src_image, 1)) { # PVE-managed volume + (my $vtype, undef, $src_vmid) = PVE::Storage::parse_volname($storecfg, $src_image); + raise_param_exc({ $ds => "$src_image has wrong type '$vtype' - not an image" }) + if $vtype ne 'images'; + } + + if ($src_vmid) { # might be actively used by VM and will be copied via clone_disk() + $rpcenv->check($authuser, "/vms/${src_vmid}", ['VM.Clone']); + } else { + PVE::Storage::check_volume_access($rpcenv, $authuser, $storecfg, $vmid, $src_image); + } } }); @@ -133,15 +217,125 @@ my $check_storage_access_clone = sub { return $sharedvm; }; +my $check_storage_access_migrate = sub { + my ($rpcenv, $authuser, $storecfg, $storage, $node) = @_; + + PVE::Storage::storage_check_enabled($storecfg, $storage, $node); + + $rpcenv->check($authuser, "/storage/$storage", ['Datastore.AllocateSpace']); + + my $scfg = PVE::Storage::storage_config($storecfg, $storage); + die "storage '$storage' does not support vm images\n" + if !$scfg->{content}->{images}; +}; + +my $import_from_volid = sub { + my ($storecfg, $src_volid, $dest_info, $vollist) = @_; + + die "could not get size of $src_volid\n" + if !PVE::Storage::volume_size_info($storecfg, $src_volid, 10); + + die "cannot import from cloudinit disk\n" + if PVE::QemuServer::Drive::drive_is_cloudinit({ file => $src_volid }); + + my $src_vmid = (PVE::Storage::parse_volname($storecfg, $src_volid))[2]; + + my $src_vm_state = sub { + my $exists = $src_vmid && PVE::Cluster::get_vmlist()->{ids}->{$src_vmid} ? 1 : 0; + + my $runs = 0; + if ($exists) { + eval { PVE::QemuConfig::assert_config_exists_on_node($src_vmid); }; + die "owner VM $src_vmid not on local node\n" if $@; + $runs = PVE::QemuServer::Helpers::vm_running_locally($src_vmid) || 0; + } + + return ($exists, $runs); + }; + + my ($src_vm_exists, $running) = $src_vm_state->(); + + die "cannot import from '$src_volid' - full clone feature is not supported\n" + if !PVE::Storage::volume_has_feature($storecfg, 'copy', $src_volid, undef, $running); + + my $clonefn = sub { + my ($src_vm_exists_now, $running_now) = $src_vm_state->(); + + die "owner VM $src_vmid changed state unexpectedly\n" + if $src_vm_exists_now != $src_vm_exists || $running_now != $running; + + my $src_conf = $src_vm_exists_now ? PVE::QemuConfig->load_config($src_vmid) : {}; + + my $src_drive = { file => $src_volid }; + my $src_drivename; + PVE::QemuConfig->foreach_volume($src_conf, sub { + my ($ds, $drive) = @_; + + return if $src_drivename; + + if ($drive->{file} eq $src_volid) { + $src_drive = $drive; + $src_drivename = $ds; + } + }); + + my $source_info = { + vmid => $src_vmid, + running => $running_now, + drivename => $src_drivename, + drive => $src_drive, + snapname => undef, + }; + + my ($src_storeid) = PVE::Storage::parse_volume_id($src_volid); + + return PVE::QemuServer::clone_disk( + $storecfg, + $source_info, + $dest_info, + 1, + $vollist, + undef, + undef, + $src_conf->{agent}, + PVE::Storage::get_bandwidth_limit('clone', [$src_storeid, $dest_info->{storage}]), + ); + }; + + my $cloned; + if ($running) { + $cloned = PVE::QemuConfig->lock_config_full($src_vmid, 30, $clonefn); + } elsif ($src_vmid) { + $cloned = PVE::QemuConfig->lock_config_shared($src_vmid, 30, $clonefn); + } else { + $cloned = $clonefn->(); + } + + return $cloned->@{qw(file size)}; +}; + # Note: $pool is only needed when creating a VM, because pool permissions # are automatically inherited if VM already exists inside a pool. -my $create_disks = sub { - my ($rpcenv, $authuser, $conf, $arch, $storecfg, $vmid, $pool, $settings, $default_storage) = @_; +my sub create_disks : prototype($$$$$$$$$$) { + my ( + $rpcenv, + $authuser, + $conf, + $arch, + $storecfg, + $vmid, + $pool, + $settings, + $default_storage, + $is_live_import, + ) = @_; my $vollist = []; my $res = {}; + my $live_import_mapping = {}; + my $code = sub { my ($ds, $disk) = @_; @@ -154,6 +348,15 @@ my $create_disks = sub { } elsif (defined($volname) && $volname eq 'cloudinit') { $storeid = $storeid // $default_storage; die "no storage ID specified (and no default storage)\n" if !$storeid; + + if ( + my $ci_key = PVE::QemuConfig->has_cloudinit($conf, $ds) + || PVE::QemuConfig->has_cloudinit($conf->{pending} || {}, $ds) + || PVE::QemuConfig->has_cloudinit($res, $ds) + ) { + die "$ds - cloud-init drive is already attached at '$ci_key'\n"; + } + my $scfg = PVE::Storage::storage_config($storecfg, $storeid); my $name = "vm-$vmid-cloudinit"; @@ -173,52 +376,136 @@ my $create_disks = sub { push @$vollist, $volid; delete $disk->{format}; # no longer needed $res->{$ds} = PVE::QemuServer::print_drive($disk); - } elsif ($volid =~ $NEW_DISK_RE) { + print "$ds: successfully created disk '$res->{$ds}'\n"; + } elsif ($volid =~ $PVE::QemuServer::Drive::NEW_DISK_RE) { my ($storeid, $size) = ($2 || $default_storage, $3); die "no storage ID specified (and no default storage)\n" if !$storeid; - my $defformat = PVE::Storage::storage_default_format($storecfg, $storeid); - my $fmt = $disk->{format} || $defformat; $size = PVE::Tools::convert_size($size, 'gb' => 'kb'); # vdisk_alloc uses kb - my $volid; - if ($ds eq 'efidisk0') { - ($volid, $size) = PVE::QemuServer::create_efidisk($storecfg, $storeid, $vmid, $fmt, $arch); - } else { - $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $size); - } - push @$vollist, $volid; - $disk->{file} = $volid; - $disk->{size} = PVE::Tools::convert_size($size, 'kb' => 'b'); - delete $disk->{format}; # no longer needed - $res->{$ds} = PVE::QemuServer::print_drive($disk); - } else { - - PVE::Storage::check_volume_access($rpcenv, $authuser, $storecfg, $vmid, $volid); - - my $volid_is_new = 1; + my $live_import = $is_live_import && $ds ne 'efidisk0'; + my $needs_creation = 1; + + if (my $source = delete $disk->{'import-from'}) { + my $dst_volid; + + $needs_creation = $live_import; + + if (PVE::Storage::parse_volume_id($source, 1)) { # PVE-managed volume + if ($live_import && $ds ne 'efidisk0') { + my $path = PVE::Storage::path($storecfg, $source) + or die "failed to get a path for '$source'\n"; + $source = $path; + ($size, my $source_format) = PVE::Storage::file_size_info($source); + die "could not get file size of $source\n" if !$size; + $live_import_mapping->{$ds} = { + path => $source, + format => $source_format, + }; + } else { + my $dest_info = { + vmid => $vmid, + drivename => $ds, + storage => $storeid, + format => $disk->{format}, + }; + + $dest_info->{efisize} = PVE::QemuServer::get_efivars_size($conf, $disk) + if $ds eq 'efidisk0'; + + ($dst_volid, $size) = eval { + $import_from_volid->($storecfg, $source, $dest_info, $vollist); + }; + die "cannot import from '$source' - $@" if $@; + } + } else { + $source = PVE::Storage::abs_filesystem_path($storecfg, $source, 1); + ($size, my $source_format) = PVE::Storage::file_size_info($source); + die "could not get file size of $source\n" if !$size; + + if ($live_import && $ds ne 'efidisk0') { + $live_import_mapping->{$ds} = { + path => $source, + format => $source_format, + }; + } else { + (undef, $dst_volid) = PVE::QemuServer::ImportDisk::do_import( + $source, + $vmid, + $storeid, + { + drive_name => $ds, + format => $disk->{format}, + 'skip-config-update' => 1, + }, + ); + push @$vollist, $dst_volid; + } + } - if ($conf->{$ds}) { - my $olddrive = PVE::QemuServer::parse_drive($ds, $conf->{$ds}); - $volid_is_new = undef if $olddrive->{file} && $olddrive->{file} eq $volid; + if ($needs_creation) { + $size = PVE::Tools::convert_size($size, 'b' => 'kb'); # vdisk_alloc uses kb + } else { + $disk->{file} = $dst_volid; + $disk->{size} = $size; + delete $disk->{format}; # no longer needed + $res->{$ds} = PVE::QemuServer::print_drive($disk); + } } - if ($volid_is_new) { - - PVE::Storage::activate_volumes($storecfg, [ $volid ]) if $storeid; + if ($needs_creation) { + my $defformat = PVE::Storage::storage_default_format($storecfg, $storeid); + my $fmt = $disk->{format} || $defformat; + + my $volid; + if ($ds eq 'efidisk0') { + my $smm = PVE::QemuServer::Machine::machine_type_is_q35($conf); + ($volid, $size) = PVE::QemuServer::create_efidisk( + $storecfg, $storeid, $vmid, $fmt, $arch, $disk, $smm); + } elsif ($ds eq 'tpmstate0') { + # swtpm can only use raw volumes, and uses a fixed size + $size = PVE::Tools::convert_size(PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE, 'b' => 'kb'); + $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, "raw", undef, $size); + } else { + $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $size); + } + push @$vollist, $volid; + $disk->{file} = $volid; + $disk->{size} = PVE::Tools::convert_size($size, 'kb' => 'b'); + delete $disk->{format}; # no longer needed + $res->{$ds} = PVE::QemuServer::print_drive($disk); + } - my $size = PVE::Storage::volume_size_info($storecfg, $volid); + print "$ds: successfully created disk '$res->{$ds}'\n"; + } else { + PVE::Storage::check_volume_access($rpcenv, $authuser, $storecfg, $vmid, $volid); + if ($storeid) { + my ($vtype) = PVE::Storage::parse_volname($storecfg, $volid); + die "cannot use volume $volid - content type needs to be 'images' or 'iso'" + if $vtype ne 'images' && $vtype ne 'iso'; + + if (PVE::QemuServer::Drive::drive_is_cloudinit($disk)) { + if ( + my $ci_key = PVE::QemuConfig->has_cloudinit($conf, $ds) + || PVE::QemuConfig->has_cloudinit($conf->{pending} || {}, $ds) + || PVE::QemuConfig->has_cloudinit($res, $ds) + ) { + die "$ds - cloud-init drive is already attached at '$ci_key'\n"; + } + } + } - die "volume $volid does not exist\n" if !$size; + PVE::Storage::activate_volumes($storecfg, [ $volid ]) if $storeid; - $disk->{size} = $size; - } + my $size = PVE::Storage::volume_size_info($storecfg, $volid); + die "volume $volid does not exist\n" if !$size; + $disk->{size} = $size; $res->{$ds} = PVE::QemuServer::print_drive($disk); } }; - eval { PVE::QemuConfig->foreach_volume($settings, $code); }; + eval { $foreach_volume_with_alloc->($settings, $code); }; # free allocated images on error if (my $err = $@) { @@ -230,12 +517,10 @@ my $create_disks = sub { die $err; } - # modify vm config if everything went well - foreach my $ds (keys %$res) { - $conf->{$ds} = $res->{$ds}; - } + # don't return empty import mappings + $live_import_mapping = undef if !%$live_import_mapping; - return $vollist; + return ($vollist, $res, $live_import_mapping); }; my $check_cpu_model_access = sub { @@ -306,7 +591,6 @@ my $generaloptions = { 'startup' => 1, 'tdf' => 1, 'template' => 1, - 'tags' => 1, }; my $vmpoweroptions = { @@ -324,11 +608,93 @@ my $cloudinitoptions = { cipassword => 1, citype => 1, ciuser => 1, + ciupgrade => 1, nameserver => 1, searchdomain => 1, sshkeys => 1, }; +my $check_vm_create_serial_perm = sub { + my ($rpcenv, $authuser, $vmid, $pool, $param) = @_; + + return 1 if $authuser eq 'root@pam'; + + foreach my $opt (keys %{$param}) { + next if $opt !~ m/^serial\d+$/; + + if ($param->{$opt} eq 'socket') { + $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.HWType']); + } else { + die "only root can set '$opt' config for real devices\n"; + } + } + + return 1; +}; + +my sub check_usb_perm { + my ($rpcenv, $authuser, $vmid, $pool, $opt, $value) = @_; + + return 1 if $authuser eq 'root@pam'; + + $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.HWType']); + + my $device = PVE::JSONSchema::parse_property_string('pve-qm-usb', $value); + if ($device->{host} && $device->{host} !~ m/^spice$/i) { + die "only root can set '$opt' config for real devices\n"; + } elsif ($device->{mapping}) { + $rpcenv->check_full($authuser, "/mapping/usb/$device->{mapping}", ['Mapping.Use']); + } else { + die "either 'host' or 'mapping' must be set.\n"; + } + + return 1; +} + +my sub check_vm_create_usb_perm { + my ($rpcenv, $authuser, $vmid, $pool, $param) = @_; + + return 1 if $authuser eq 'root@pam'; + + foreach my $opt (keys %{$param}) { + next if $opt !~ m/^usb\d+$/; + check_usb_perm($rpcenv, $authuser, $vmid, $pool, $opt, $param->{$opt}); + } + + return 1; +}; + +my sub check_hostpci_perm { + my ($rpcenv, $authuser, $vmid, $pool, $opt, $value) = @_; + + return 1 if $authuser eq 'root@pam'; + + my $device = PVE::JSONSchema::parse_property_string('pve-qm-hostpci', $value); + if ($device->{host}) { + die "only root can set '$opt' config for non-mapped devices\n"; + } elsif ($device->{mapping}) { + $rpcenv->check_full($authuser, "/mapping/pci/$device->{mapping}", ['Mapping.Use']); + $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.HWType']); + } else { + die "either 'host' or 'mapping' must be set.\n"; + } + + return 1; +} + +my sub check_vm_create_hostpci_perm { + my ($rpcenv, $authuser, $vmid, $pool, $param) = @_; + + return 1 if $authuser eq 'root@pam'; + + foreach my $opt (keys %{$param}) { + next if $opt !~ m/^hostpci\d+$/; + check_hostpci_perm($rpcenv, $authuser, $vmid, $pool, $opt, $param->{$opt}); + } + + return 1; +}; + my $check_vm_modify_config_perm = sub { my ($rpcenv, $authuser, $vmid, $pool, $key_list) = @_; @@ -339,7 +705,8 @@ my $check_vm_modify_config_perm = sub { # else, as there the permission can be value dependend next if PVE::QemuServer::is_valid_drivename($opt); next if $opt eq 'cdrom'; - next if $opt =~ m/^(?:unused|serial|usb)\d+$/; + next if $opt =~ m/^(?:unused|serial|usb|hostpci)\d+$/; + next if $opt eq 'tags'; if ($cpuoptions->{$opt} || $opt =~ m/^numa\d+$/) { @@ -358,16 +725,16 @@ my $check_vm_modify_config_perm = sub { $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.PowerMgmt']); } elsif ($diskoptions->{$opt}) { $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Disk']); - } elsif ($opt =~ m/^(?:net|ipconfig)\d+$/) { + } elsif ($opt =~ m/^net\d+$/) { $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Network']); - } elsif ($cloudinitoptions->{$opt}) { + } elsif ($cloudinitoptions->{$opt} || $opt =~ m/^ipconfig\d+$/) { $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Cloudinit', 'VM.Config.Network'], 1); } elsif ($opt eq 'vmstate') { # the user needs Disk and PowerMgmt privileges to change the vmstate # also needs privileges on the storage, that will be checked later $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Disk', 'VM.PowerMgmt' ]); } else { - # catches hostpci\d+, args, lock, etc. + # catches args, lock, etc. # new options will be checked here die "only root can set '$opt' config\n"; } @@ -376,6 +743,33 @@ my $check_vm_modify_config_perm = sub { return 1; }; +sub assert_scsi_feature_compatibility { + my ($opt, $conf, $storecfg, $drive_attributes) = @_; + + my $drive = PVE::QemuServer::Drive::parse_drive($opt, $drive_attributes, 1); + + my $machine_type = PVE::QemuServer::get_vm_machine($conf, undef, $conf->{arch}); + my $machine_version = PVE::QemuServer::Machine::extract_version( + $machine_type, PVE::QemuServer::kvm_user_version()); + my $drivetype = PVE::QemuServer::Drive::get_scsi_device_type( + $drive, $storecfg, $machine_version); + + if ($drivetype ne 'hd' && $drivetype ne 'cd') { + if ($drive->{product}) { + raise_param_exc({ + $opt => "Passing of product information is only supported for 'scsi-hd' and " + ."'scsi-cd' devices (e.g. not pass-through).", + }); + } + if ($drive->{vendor}) { + raise_param_exc({ + $opt => "Passing of vendor information is only supported for 'scsi-hd' and " + ."'scsi-cd' devices (e.g. not pass-through).", + }); + } + } +} + __PACKAGE__->register_method({ name => 'vmlist', path => '', @@ -388,7 +782,7 @@ __PACKAGE__->register_method({ proxyto => 'node', protected => 1, # qemu pid files are only readable by root parameters => { - additionalProperties => 0, + additionalProperties => 0, properties => { node => get_standard_option('pve-node'), full => { @@ -430,23 +824,22 @@ my $parse_restore_archive = sub { my ($archive_storeid, $archive_volname) = PVE::Storage::parse_volume_id($archive, 1); + my $res = {}; + if (defined($archive_storeid)) { my $scfg = PVE::Storage::storage_config($storecfg, $archive_storeid); + $res->{volid} = $archive; if ($scfg->{type} eq 'pbs') { - return { - type => 'pbs', - volid => $archive, - }; + $res->{type} = 'pbs'; + return $res; } } my $path = PVE::Storage::abs_filesystem_path($storecfg, $archive); - return { - type => 'file', - path => $path, - }; + $res->{type} = 'file'; + $res->{path} = $path; + return $res; }; - __PACKAGE__->register_method({ name => 'create_vm', path => '', @@ -455,13 +848,14 @@ __PACKAGE__->register_method({ permissions => { description => "You need 'VM.Allocate' permissions on /vms/{vmid} or on the VM pool /pool/{pool}. " . "For restore (option 'archive'), it is enough if the user has 'VM.Backup' permission and the VM already exists. " . - "If you create disks you need 'Datastore.AllocateSpace' on any used storage.", + "If you create disks you need 'Datastore.AllocateSpace' on any used storage." . + "If you use a bridge/vlan, you need 'SDN.Use' on any used bridge/vlan.", user => 'all', # check inside }, protected => 1, proxyto => 'node', parameters => { - additionalProperties => 0, + additionalProperties => 0, properties => PVE::QemuServer::json_config_properties( { node => get_standard_option('pve-node'), @@ -490,6 +884,11 @@ __PACKAGE__->register_method({ description => "Assign a unique random ethernet address.", requires => 'archive', }, + 'live-restore' => { + optional => 1, + type => 'boolean', + description => "Start the VM immediately while importing or restoring in the background.", + }, pool => { optional => 1, type => 'string', format => 'pve-poolid', @@ -508,7 +907,9 @@ __PACKAGE__->register_method({ default => 0, description => "Start VM after it was created successfully.", }, - }), + }, + 1, # with_disk_alloc + ), }, returns => { type => 'string', @@ -531,12 +932,16 @@ __PACKAGE__->register_method({ my $start_after_create = extract_param($param, 'start'); my $storage = extract_param($param, 'storage'); my $unique = extract_param($param, 'unique'); + my $live_restore = extract_param($param, 'live-restore'); if (defined(my $ssh_keys = $param->{sshkeys})) { $ssh_keys = URI::Escape::uri_unescape($ssh_keys); PVE::Tools::validate_ssh_public_keys($ssh_keys); } + $param->{cpuunits} = PVE::CGroup::clamp_cpu_shares($param->{cpuunits}) + if defined($param->{cpuunits}); # clamp value depending on cgroup version + PVE::Cluster::check_cfs_quorum(); my $filename = PVE::QemuConfig->config_file($vmid); @@ -555,44 +960,52 @@ __PACKAGE__->register_method({ # OK } elsif ($archive && $force && (-f $filename) && $rpcenv->check($authuser, "/vms/$vmid", ['VM.Backup'], 1)) { - # OK: user has VM.Backup permissions, and want to restore an existing VM + # OK: user has VM.Backup permissions and wants to restore an existing VM } else { raise_perm_exc(); } - if (!$archive) { + if ($archive) { + for my $opt (sort keys $param->%*) { + if (PVE::QemuServer::Drive::is_valid_drivename($opt)) { + raise_param_exc({ $opt => "option conflicts with option 'archive'" }); + } + } + + if ($archive eq '-') { + die "pipe requires cli environment\n" if $rpcenv->{type} ne 'cli'; + $archive = { type => 'pipe' }; + } else { + PVE::Storage::check_volume_access( + $rpcenv, + $authuser, + $storecfg, + $vmid, + $archive, + 'backup', + ); + + $archive = $parse_restore_archive->($storecfg, $archive); + } + } + + if (scalar(keys $param->%*) > 0) { &$resolve_cdrom_alias($param); &$check_storage_access($rpcenv, $authuser, $storecfg, $vmid, $param, $storage); &$check_vm_modify_config_perm($rpcenv, $authuser, $vmid, $pool, [ keys %$param]); - &$check_cpu_model_access($rpcenv, $authuser, $param); + &$check_vm_create_serial_perm($rpcenv, $authuser, $vmid, $pool, $param); + check_vm_create_usb_perm($rpcenv, $authuser, $vmid, $pool, $param); + check_vm_create_hostpci_perm($rpcenv, $authuser, $vmid, $pool, $param); - foreach my $opt (keys %$param) { - if (PVE::QemuServer::is_valid_drivename($opt)) { - my $drive = PVE::QemuServer::parse_drive($opt, $param->{$opt}); - raise_param_exc({ $opt => "unable to parse drive options" }) if !$drive; + PVE::QemuServer::check_bridge_access($rpcenv, $authuser, $param); + &$check_cpu_model_access($rpcenv, $authuser, $param); - PVE::QemuServer::cleanup_drive_path($opt, $storecfg, $drive); - $param->{$opt} = PVE::QemuServer::print_drive($drive); - } - } + $check_drive_param->($param, $storecfg); PVE::QemuServer::add_random_macs($param); - } else { - my $keystr = join(' ', keys %$param); - raise_param_exc({ archive => "option conflicts with other options ($keystr)"}) if $keystr; - - if ($archive eq '-') { - die "pipe requires cli environment\n" - if $rpcenv->{type} ne 'cli'; - $archive = { type => 'pipe' }; - } else { - PVE::Storage::check_volume_access($rpcenv, $authuser, $storecfg, $vmid, $archive); - - $archive = $parse_restore_archive->($storecfg, $archive); - } } my $emsg = $is_restore ? "unable to restore VM $vmid -" : "unable to create VM $vmid -"; @@ -600,6 +1013,7 @@ __PACKAGE__->register_method({ eval { PVE::QemuConfig->create_and_lock_config($vmid, $force) }; die "$emsg $@" if $@; + my $restored_data = 0; my $restorefn = sub { my $conf = PVE::QemuConfig->load_config($vmid); @@ -613,14 +1027,33 @@ __PACKAGE__->register_method({ pool => $pool, unique => $unique, bwlimit => $bwlimit, + live => $live_restore, + override_conf => $param, }; + if (my $volid = $archive->{volid}) { + # best effort, real check is after restoring! + my $merged = eval { + my $old_conf = PVE::Storage::extract_vzdump_config($storecfg, $volid); + PVE::QemuServer::restore_merge_config("backup/qemu-server/$vmid.conf", $old_conf, $param); + }; + if ($@) { + warn "Could not extract backed up config: $@\n"; + warn "Skipping early checks!\n"; + } else { + PVE::QemuServer::check_restore_permissions($rpcenv, $authuser, $merged); + } + } if ($archive->{type} eq 'file' || $archive->{type} eq 'pipe') { + die "live-restore is only compatible with backup images from a Proxmox Backup Server\n" + if $live_restore; PVE::QemuServer::restore_file_archive($archive->{path} // '-', $vmid, $authuser, $restore_options); } elsif ($archive->{type} eq 'pbs') { PVE::QemuServer::restore_proxmox_backup_archive($archive->{volid}, $vmid, $authuser, $restore_options); } else { die "unknown backup archive type\n"; } + $restored_data = 1; + my $restored_conf = PVE::QemuConfig->load_config($vmid); # Convert restored VM to template if backup was VM template if (PVE::QemuConfig->is_template($restored_conf)) { @@ -629,7 +1062,7 @@ __PACKAGE__->register_method({ warn $@ if $@; } - PVE::AccessControl::add_vm_to_pool($vmid, $pool) if $pool; + PVE::QemuServer::create_ifaces_ipams_ips($restored_conf, $vmid) if $unique; }; # ensure no old replication state are exists @@ -637,7 +1070,7 @@ __PACKAGE__->register_method({ PVE::QemuConfig->lock_config_full($vmid, 1, $realcmd); - if ($start_after_create) { + if ($start_after_create && !$live_restore) { print "Execute autostart\n"; eval { PVE::API2::Qemu->vm_start({ vmid => $vmid, node => $node }) }; warn $@ if $@; @@ -645,6 +1078,8 @@ __PACKAGE__->register_method({ }; my $createfn = sub { + my $live_import_mapping = {}; + # ensure no old replication state are exists PVE::ReplicationState::delete_guest_states($vmid); @@ -652,15 +1087,37 @@ __PACKAGE__->register_method({ my $conf = $param; my $arch = PVE::QemuServer::get_vm_arch($conf); + for my $opt (sort keys $param->%*) { + next if $opt !~ m/^scsi\d+$/; + assert_scsi_feature_compatibility($opt, $conf, $storecfg, $param->{$opt}); + } + + $conf->{meta} = PVE::QemuServer::new_meta_info_string(); + my $vollist = []; eval { - $vollist = &$create_disks($rpcenv, $authuser, $conf, $arch, $storecfg, $vmid, $pool, $param, $storage); + ($vollist, my $created_opts, $live_import_mapping) = create_disks( + $rpcenv, + $authuser, + $conf, + $arch, + $storecfg, + $vmid, + $pool, + $param, + $storage, + $live_restore, + ); + $conf->{$_} = $created_opts->{$_} for keys $created_opts->%*; if (!$conf->{boot}) { my $devs = PVE::QemuServer::get_default_bootdevices($conf); $conf->{boot} = PVE::QemuServer::print_bootorder($devs); } + my $vga = PVE::QemuServer::parse_vga($conf->{vga}); + PVE::QemuServer::assert_clipboard_config($vga); + # auto generate uuid if user did not specify smbios1 option if (!$conf->{smbios1}) { $conf->{smbios1} = PVE::QemuServer::generate_smbios1_uuid(); @@ -670,8 +1127,20 @@ __PACKAGE__->register_method({ $conf->{vmgenid} = PVE::QemuServer::generate_uuid(); } - PVE::QemuConfig->write_config($vmid, $conf); + my $machine_conf = PVE::QemuServer::Machine::parse_machine($conf->{machine}); + my $machine = $machine_conf->{type}; + if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) { + # always pin Windows' machine version on create, they get to easily confused + if (PVE::QemuServer::Helpers::windows_version($conf->{ostype})) { + $machine_conf->{type} = PVE::QemuServer::windows_get_pinned_machine_version($machine); + $conf->{machine} = PVE::QemuServer::Machine::print_machine($machine_conf); + } + } + PVE::QemuServer::Machine::assert_valid_machine_property($conf, $machine_conf); + $conf->{lock} = 'import' if $live_import_mapping; + + PVE::QemuConfig->write_config($vmid, $conf); }; my $err = $@; @@ -684,14 +1153,19 @@ __PACKAGE__->register_method({ } PVE::AccessControl::add_vm_to_pool($vmid, $pool) if $pool; + + PVE::QemuServer::create_ifaces_ipams_ips($conf, $vmid); }; PVE::QemuConfig->lock_config_full($vmid, 1, $realcmd); - if ($start_after_create) { + if ($start_after_create && !$live_restore) { print "Execute autostart\n"; eval { PVE::API2::Qemu->vm_start({vmid => $vmid, node => $node}) }; warn $@ if $@; + return; + } else { + return $live_import_mapping; } }; @@ -703,13 +1177,22 @@ __PACKAGE__->register_method({ if (my $err = $@) { eval { PVE::QemuConfig->remove_lock($vmid, 'create') }; warn $@ if $@; + if ($restored_data) { + warn "error after data was restored, VM disks should be OK but config may " + ."require adaptions. VM $vmid state is NOT cleaned up.\n"; + } else { + warn "error before or during data restore, some or all disks were not " + ."completely restored. VM $vmid state is NOT cleaned up.\n"; + } die $err; } }; } else { $worker_name = 'qmcreate'; $code = sub { - eval { $createfn->() }; + # If a live import was requested the create function returns + # the mapping for the startup. + my $live_import_mapping = eval { $createfn->() }; if (my $err = $@) { eval { my $conffile = PVE::QemuConfig->config_file($vmid); @@ -718,6 +1201,21 @@ __PACKAGE__->register_method({ warn $@ if $@; die $err; } + + if ($live_import_mapping) { + my $import_options = { + bwlimit => $bwlimit, + live => 1, + }; + + my $conf = PVE::QemuConfig->load_config($vmid); + PVE::QemuServer::live_import_from_files( + $live_import_mapping, + $vmid, + $conf, + $import_options, + ); + } }; } @@ -734,7 +1232,7 @@ __PACKAGE__->register_method({ user => 'all', }, parameters => { - additionalProperties => 0, + additionalProperties => 0, properties => { node => get_standard_option('pve-node'), vmid => get_standard_option('pve-vmid'), @@ -755,6 +1253,7 @@ __PACKAGE__->register_method({ my $res = [ { subdir => 'config' }, + { subdir => 'cloudinit' }, { subdir => 'pending' }, { subdir => 'status' }, { subdir => 'unlink' }, @@ -771,7 +1270,9 @@ __PACKAGE__->register_method({ { subdir => 'spiceproxy' }, { subdir => 'sendkey' }, { subdir => 'firewall' }, - ]; + { subdir => 'mtunnel' }, + { subdir => 'remote_migrate' }, + ]; return $res; }}); @@ -796,7 +1297,7 @@ __PACKAGE__->register_method({ }, description => "Read VM RRD statistics (returns PNG)", parameters => { - additionalProperties => 0, + additionalProperties => 0, properties => { node => get_standard_option('pve-node'), vmid => get_standard_option('pve-vmid'), @@ -842,7 +1343,7 @@ __PACKAGE__->register_method({ }, description => "Read VM RRD statistics", parameters => { - additionalProperties => 0, + additionalProperties => 0, properties => { node => get_standard_option('pve-node'), vmid => get_standard_option('pve-vmid'), @@ -885,7 +1386,7 @@ __PACKAGE__->register_method({ check => ['perm', '/vms/{vmid}', [ 'VM.Audit' ]], }, parameters => { - additionalProperties => 0, + additionalProperties => 0, properties => { node => get_standard_option('pve-node'), vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }), @@ -992,46 +1493,175 @@ __PACKAGE__->register_method({ return PVE::GuestHelpers::config_with_pending_array($conf, $pending_delete_hash); }}); -# POST/PUT {vmid}/config implementation -# -# The original API used PUT (idempotent) an we assumed that all operations -# are fast. But it turned out that almost any configuration change can -# involve hot-plug actions, or disk alloc/free. Such actions can take long -# time to complete and have side effects (not idempotent). -# -# The new implementation uses POST and forks a worker process. We added -# a new option 'background_delay'. If specified we wait up to -# 'background_delay' second for the worker task to complete. It returns null -# if the task is finished within that time, else we return the UPID. - -my $update_vm_api = sub { - my ($param, $sync) = @_; - - my $rpcenv = PVE::RPCEnvironment::get(); - - my $authuser = $rpcenv->get_user(); - - my $node = extract_param($param, 'node'); - - my $vmid = extract_param($param, 'vmid'); - - my $digest = extract_param($param, 'digest'); - - my $background_delay = extract_param($param, 'background_delay'); - - if (defined(my $cipassword = $param->{cipassword})) { - # Same logic as in cloud-init (but with the regex fixed...) - $param->{cipassword} = PVE::Tools::encrypt_pw($cipassword) - if $cipassword !~ /^\$(?:[156]|2[ay])(\$.+){2}/; - } - - my @paramarr = (); # used for log message - foreach my $key (sort keys %$param) { - my $value = $key eq 'cipassword' ? '' : $param->{$key}; - push @paramarr, "-$key", $value; - } - - my $skiplock = extract_param($param, 'skiplock'); +__PACKAGE__->register_method({ + name => 'cloudinit_pending', + path => '{vmid}/cloudinit', + method => 'GET', + proxyto => 'node', + description => "Get the cloudinit configuration with both current and pending values.", + permissions => { + check => ['perm', '/vms/{vmid}', [ 'VM.Audit' ]], + }, + parameters => { + additionalProperties => 0, + properties => { + node => get_standard_option('pve-node'), + vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }), + }, + }, + returns => { + type => "array", + items => { + type => "object", + properties => { + key => { + description => "Configuration option name.", + type => 'string', + }, + value => { + description => "Value as it was used to generate the current cloudinit image.", + type => 'string', + optional => 1, + }, + pending => { + description => "The new pending value.", + type => 'string', + optional => 1, + }, + delete => { + description => "Indicates a pending delete request if present and not 0. ", + type => 'integer', + minimum => 0, + maximum => 1, + optional => 1, + }, + }, + }, + }, + code => sub { + my ($param) = @_; + + my $vmid = $param->{vmid}; + my $conf = PVE::QemuConfig->load_config($vmid); + + my $ci = $conf->{cloudinit}; + + $conf->{cipassword} = '**********' if exists $conf->{cipassword}; + $ci->{cipassword} = '**********' if exists $ci->{cipassword}; + + my $res = []; + + # All the values that got added + my $added = delete($ci->{added}) // ''; + for my $key (PVE::Tools::split_list($added)) { + push @$res, { key => $key, pending => $conf->{$key} }; + } + + # All already existing values (+ their new value, if it exists) + for my $opt (keys %$cloudinitoptions) { + next if !$conf->{$opt}; + next if $added =~ m/$opt/; + my $item = { + key => $opt, + }; + + if (my $pending = $ci->{$opt}) { + $item->{value} = $pending; + $item->{pending} = $conf->{$opt}; + } else { + $item->{value} = $conf->{$opt}, + } + + push @$res, $item; + } + + # Now, we'll find the deleted ones + for my $opt (keys %$ci) { + next if $conf->{$opt}; + push @$res, { key => $opt, delete => 1 }; + } + + return $res; + }}); + +__PACKAGE__->register_method({ + name => 'cloudinit_update', + path => '{vmid}/cloudinit', + method => 'PUT', + protected => 1, + proxyto => 'node', + description => "Regenerate and change cloudinit config drive.", + permissions => { + check => ['perm', '/vms/{vmid}', ['VM.Config.Cloudinit']], + }, + parameters => { + additionalProperties => 0, + properties => { + node => get_standard_option('pve-node'), + vmid => get_standard_option('pve-vmid'), + }, + }, + returns => { type => 'null' }, + code => sub { + my ($param) = @_; + + my $rpcenv = PVE::RPCEnvironment::get(); + my $authuser = $rpcenv->get_user(); + + my $vmid = extract_param($param, 'vmid'); + + PVE::QemuConfig->lock_config($vmid, sub { + my $conf = PVE::QemuConfig->load_config($vmid); + PVE::QemuConfig->check_lock($conf); + + my $storecfg = PVE::Storage::config(); + PVE::QemuServer::vmconfig_update_cloudinit_drive($storecfg, $conf, $vmid); + }); + return; + }}); + +# POST/PUT {vmid}/config implementation +# +# The original API used PUT (idempotent) an we assumed that all operations +# are fast. But it turned out that almost any configuration change can +# involve hot-plug actions, or disk alloc/free. Such actions can take long +# time to complete and have side effects (not idempotent). +# +# The new implementation uses POST and forks a worker process. We added +# a new option 'background_delay'. If specified we wait up to +# 'background_delay' second for the worker task to complete. It returns null +# if the task is finished within that time, else we return the UPID. + +my $update_vm_api = sub { + my ($param, $sync) = @_; + + my $rpcenv = PVE::RPCEnvironment::get(); + + my $authuser = $rpcenv->get_user(); + + my $node = extract_param($param, 'node'); + + my $vmid = extract_param($param, 'vmid'); + + my $digest = extract_param($param, 'digest'); + + my $background_delay = extract_param($param, 'background_delay'); + + my $skip_cloud_init = extract_param($param, 'skip_cloud_init'); + + if (defined(my $cipassword = $param->{cipassword})) { + # Same logic as in cloud-init (but with the regex fixed...) + $param->{cipassword} = PVE::Tools::encrypt_pw($cipassword) + if $cipassword !~ /^\$(?:[156]|2[ay])(\$.+){2}/; + } + + my @paramarr = (); # used for log message + foreach my $key (sort keys %$param) { + my $value = $key eq 'cipassword' ? '' : $param->{$key}; + push @paramarr, "-$key", $value; + } + + my $skiplock = extract_param($param, 'skiplock'); raise_param_exc({ skiplock => "Only root may use this option." }) if $skiplock && $authuser ne 'root@pam'; @@ -1046,12 +1676,13 @@ my $update_vm_api = sub { PVE::Tools::validate_ssh_public_keys($ssh_keys); } + $param->{cpuunits} = PVE::CGroup::clamp_cpu_shares($param->{cpuunits}) + if defined($param->{cpuunits}); # clamp value depending on cgroup version + die "no options specified\n" if !$delete_str && !$revert_str && !scalar(keys %$param); my $storecfg = PVE::Storage::config(); - my $defaults = PVE::QemuServer::load_defaults(); - &$resolve_cdrom_alias($param); # now try to verify all parameters @@ -1104,7 +1735,7 @@ my $update_vm_api = sub { return if defined($volname) && $volname eq 'cloudinit'; my $format; - if ($volid =~ $NEW_DISK_RE) { + if ($volid =~ $PVE::QemuServer::Drive::NEW_DISK_RE) { $storeid = $2; $format = $drive->{format} || PVE::Storage::storage_default_format($storecfg, $storeid); } else { @@ -1116,15 +1747,10 @@ my $update_vm_api = sub { die "cannot add non-replicatable volume to a replicated VM\n"; }; + $check_drive_param->($param, $storecfg, $check_replication); + foreach my $opt (keys %$param) { - if (PVE::QemuServer::is_valid_drivename($opt)) { - # cleanup drive path - my $drive = PVE::QemuServer::parse_drive($opt, $param->{$opt}); - raise_param_exc({ $opt => "unable to parse drive options" }) if !$drive; - PVE::QemuServer::cleanup_drive_path($opt, $storecfg, $drive); - $check_replication->($drive); - $param->{$opt} = PVE::QemuServer::print_drive($drive); - } elsif ($opt =~ m/^net(\d+)$/) { + if ($opt =~ m/^net(\d+)$/) { # add macaddr my $net = PVE::QemuServer::parse_net($param->{$opt}); $param->{$opt} = PVE::QemuServer::print_net($net); @@ -1144,6 +1770,8 @@ my $update_vm_api = sub { &$check_storage_access($rpcenv, $authuser, $storecfg, $vmid, $param); + PVE::QemuServer::check_bridge_access($rpcenv, $authuser, $param); + my $updatefn = sub { my $conf = PVE::QemuConfig->load_config($vmid); @@ -1174,7 +1802,9 @@ my $update_vm_api = sub { } if ($param->{memory} || defined($param->{balloon})) { - my $maxmem = $param->{memory} || $conf->{pending}->{memory} || $conf->{memory} || $defaults->{memory}; + + my $memory = $param->{memory} || $conf->{pending}->{memory} || $conf->{memory}; + my $maxmem = get_current_memory($memory); my $balloon = defined($param->{balloon}) ? $param->{balloon} : $conf->{pending}->{balloon} || $conf->{balloon}; die "balloon value too large (must be smaller than assigned memory)\n" @@ -1198,6 +1828,19 @@ my $update_vm_api = sub { } my $bootorder_deleted = grep {$_ eq 'bootorder'} @delete; + my $check_drive_perms = sub { + my ($opt, $val) = @_; + my $drive = PVE::QemuServer::parse_drive($opt, $val, 1); + if (PVE::QemuServer::drive_is_cloudinit($drive)) { + $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Cloudinit', 'VM.Config.CDROM']); + } elsif (PVE::QemuServer::drive_is_cdrom($drive, 1)) { # CDROM + $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.CDROM']); + } else { + $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']); + + } + }; + foreach my $opt (@delete) { $modified->{$opt} = 1; $conf = PVE::QemuConfig->load_config($vmid); # update/reload @@ -1235,7 +1878,7 @@ my $update_vm_api = sub { } } elsif (PVE::QemuServer::is_valid_drivename($opt)) { PVE::QemuConfig->check_protection($conf, "can't remove drive '$opt'"); - $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']); + $check_drive_perms->($opt, $val); PVE::QemuServer::vmconfig_register_unused_drive($storecfg, $vmid, $conf, PVE::QemuServer::parse_drive($opt, $val)) if $is_pending_val; PVE::QemuConfig->add_to_pending_delete($conf, $opt, $force); @@ -1249,10 +1892,24 @@ my $update_vm_api = sub { PVE::QemuConfig->add_to_pending_delete($conf, $opt, $force); PVE::QemuConfig->write_config($vmid, $conf); } elsif ($opt =~ m/^usb\d+$/) { - if ($val =~ m/spice/) { - $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.HWType']); - } elsif ($authuser ne 'root@pam') { - die "only root can delete '$opt' config for real devices\n"; + check_usb_perm($rpcenv, $authuser, $vmid, undef, $opt, $val); + PVE::QemuConfig->add_to_pending_delete($conf, $opt, $force); + PVE::QemuConfig->write_config($vmid, $conf); + } elsif ($opt =~ m/^hostpci\d+$/) { + check_hostpci_perm($rpcenv, $authuser, $vmid, undef, $opt, $val); + PVE::QemuConfig->add_to_pending_delete($conf, $opt, $force); + PVE::QemuConfig->write_config($vmid, $conf); + } elsif ($opt eq 'tags') { + assert_tag_permissions($vmid, $val, '', $rpcenv, $authuser); + delete $conf->{$opt}; + PVE::QemuConfig->write_config($vmid, $conf); + } elsif ($opt =~ m/^net\d+$/) { + if ($conf->{$opt}) { + PVE::QemuServer::check_bridge_access( + $rpcenv, + $authuser, + { $opt => $conf->{$opt} }, + ); } PVE::QemuConfig->add_to_pending_delete($conf, $opt, $force); PVE::QemuConfig->write_config($vmid, $conf); @@ -1270,17 +1927,43 @@ my $update_vm_api = sub { my $arch = PVE::QemuServer::get_vm_arch($conf); if (PVE::QemuServer::is_valid_drivename($opt)) { - my $drive = PVE::QemuServer::parse_drive($opt, $param->{$opt}); - # FIXME: cloudinit: CDROM or Disk? - if (PVE::QemuServer::drive_is_cdrom($drive)) { # CDROM - $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.CDROM']); - } else { - $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']); + # old drive + if ($conf->{$opt}) { + $check_drive_perms->($opt, $conf->{$opt}); } + + # new drive + $check_drive_perms->($opt, $param->{$opt}); PVE::QemuServer::vmconfig_register_unused_drive($storecfg, $vmid, $conf, PVE::QemuServer::parse_drive($opt, $conf->{pending}->{$opt})) if defined($conf->{pending}->{$opt}); - &$create_disks($rpcenv, $authuser, $conf->{pending}, $arch, $storecfg, $vmid, undef, {$opt => $param->{$opt}}); + assert_scsi_feature_compatibility($opt, $conf, $storecfg, $param->{$opt}) + if $opt =~ m/^scsi\d+$/; + + my (undef, $created_opts) = create_disks( + $rpcenv, + $authuser, + $conf, + $arch, + $storecfg, + $vmid, + undef, + {$opt => $param->{$opt}}, + undef, + undef, + ); + $conf->{pending}->{$_} = $created_opts->{$_} for keys $created_opts->%*; + + # default legacy boot order implies all cdroms anyway + if (@bootorder) { + # append new CD drives to bootorder to mark them bootable + my $drive = PVE::QemuServer::parse_drive($opt, $param->{$opt}, 1); + if (PVE::QemuServer::drive_is_cdrom($drive, 1) && !grep(/^$opt$/, @bootorder)) { + push @bootorder, $opt; + $conf->{pending}->{boot} = PVE::QemuServer::print_bootorder(\@bootorder); + $modified->{boot} = 1; + } + } } elsif ($opt =~ m/^serial\d+/) { if ((!defined($conf->{$opt}) || $conf->{$opt} eq 'socket') && $param->{$opt} eq 'socket') { $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.HWType']); @@ -1288,12 +1971,37 @@ my $update_vm_api = sub { die "only root can modify '$opt' config for real devices\n"; } $conf->{pending}->{$opt} = $param->{$opt}; + } elsif ($opt eq 'vga') { + my $vga = PVE::QemuServer::parse_vga($param->{$opt}); + PVE::QemuServer::assert_clipboard_config($vga); + $conf->{pending}->{$opt} = $param->{$opt}; } elsif ($opt =~ m/^usb\d+/) { - if ((!defined($conf->{$opt}) || $conf->{$opt} =~ m/spice/) && $param->{$opt} =~ m/spice/) { - $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.HWType']); - } elsif ($authuser ne 'root@pam') { - die "only root can modify '$opt' config for real devices\n"; + if (my $olddevice = $conf->{$opt}) { + check_usb_perm($rpcenv, $authuser, $vmid, undef, $opt, $conf->{$opt}); + } + check_usb_perm($rpcenv, $authuser, $vmid, undef, $opt, $param->{$opt}); + $conf->{pending}->{$opt} = $param->{$opt}; + } elsif ($opt =~ m/^hostpci\d+$/) { + if (my $oldvalue = $conf->{$opt}) { + check_hostpci_perm($rpcenv, $authuser, $vmid, undef, $opt, $oldvalue); } + check_hostpci_perm($rpcenv, $authuser, $vmid, undef, $opt, $param->{$opt}); + $conf->{pending}->{$opt} = $param->{$opt}; + } elsif ($opt eq 'tags') { + assert_tag_permissions($vmid, $conf->{$opt}, $param->{$opt}, $rpcenv, $authuser); + $conf->{pending}->{$opt} = PVE::GuestHelpers::get_unique_tags($param->{$opt}); + } elsif ($opt =~ m/^net\d+$/) { + if ($conf->{$opt}) { + PVE::QemuServer::check_bridge_access( + $rpcenv, + $authuser, + { $opt => $conf->{$opt} }, + ); + } + $conf->{pending}->{$opt} = $param->{$opt}; + } elsif ($opt eq 'machine') { + my $machine_conf = PVE::QemuServer::Machine::parse_machine($param->{$opt}); + PVE::QemuServer::Machine::assert_valid_machine_property($conf, $machine_conf); $conf->{pending}->{$opt} = $param->{$opt}; } else { $conf->{pending}->{$opt} = $param->{$opt}; @@ -1303,7 +2011,7 @@ my $update_vm_api = sub { if ($new_bootcfg->{order}) { my @devs = PVE::Tools::split_list($new_bootcfg->{order}); for my $dev (@devs) { - my $exists = $conf->{$dev} || $conf->{pending}->{$dev}; + my $exists = $conf->{$dev} || $conf->{pending}->{$dev} || $param->{$dev}; my $deleted = grep {$_ eq $dev} @delete; die "invalid bootorder: device '$dev' does not exist'\n" if !$exists || $deleted; @@ -1337,7 +2045,8 @@ my $update_vm_api = sub { if ($running) { PVE::QemuServer::vmconfig_hotplug_pending($vmid, $conf, $storecfg, $modified, $errors); } else { - PVE::QemuServer::vmconfig_apply_pending($vmid, $conf, $storecfg, $running, $errors); + # cloud_init must be skipped if we are in an incoming, remote live migration + PVE::QemuServer::vmconfig_apply_pending($vmid, $conf, $storecfg, $errors, $skip_cloud_init); } raise_param_exc($errors) if scalar(keys %$errors); @@ -1368,8 +2077,8 @@ my $update_vm_api = sub { if (!$running) { my $status = PVE::Tools::upid_read_status($upid); - return if $status eq 'OK'; - die $status; + return if !PVE::Tools::upid_status_is_error($status); + die "failed to update VM $vmid: $status\n"; } } @@ -1402,7 +2111,7 @@ __PACKAGE__->register_method({ check => ['perm', '/vms/{vmid}', $vm_config_perm_list, any => 1], }, parameters => { - additionalProperties => 0, + additionalProperties => 0, properties => PVE::QemuServer::json_config_properties( { node => get_standard_option('pve-node'), @@ -1437,7 +2146,9 @@ __PACKAGE__->register_method({ maximum => 30, optional => 1, }, - }), + }, + 1, # with_disk_alloc + ), }, returns => { type => 'string', @@ -1457,7 +2168,7 @@ __PACKAGE__->register_method({ check => ['perm', '/vms/{vmid}', $vm_config_perm_list, any => 1], }, parameters => { - additionalProperties => 0, + additionalProperties => 0, properties => PVE::QemuServer::json_config_properties( { node => get_standard_option('pve-node'), @@ -1485,7 +2196,9 @@ __PACKAGE__->register_method({ maxLength => 40, optional => 1, }, - }), + }, + 1, # with_disk_alloc + ), }, returns => { type => 'null' }, code => sub { @@ -1501,20 +2214,28 @@ __PACKAGE__->register_method({ method => 'DELETE', protected => 1, proxyto => 'node', - description => "Destroy the vm (also delete all used/owned volumes).", + description => "Destroy the VM and all used/owned volumes. Removes any VM specific permissions" + ." and firewall rules", permissions => { check => [ 'perm', '/vms/{vmid}', ['VM.Allocate']], }, parameters => { - additionalProperties => 0, + additionalProperties => 0, properties => { node => get_standard_option('pve-node'), vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid_stopped }), skiplock => get_standard_option('skiplock'), purge => { type => 'boolean', - description => "Remove vmid from backup cron jobs.", + description => "Remove VMID from configurations, like backup & replication jobs and HA.", + optional => 1, + }, + 'destroy-unreferenced-disks' => { + type => 'boolean', + description => "If set, destroy additionally all disks not referenced in the config" + ." but with a matching VMID from all enabled storages.", optional => 1, + default => 0, }, }, }, @@ -1565,7 +2286,14 @@ __PACKAGE__->register_method({ # repeat, config might have changed my $ha_managed = $early_checks->(); - PVE::QemuServer::destroy_vm($storecfg, $vmid, $skiplock, { lock => 'destroyed' }); + my $purge_unreferenced = $param->{'destroy-unreferenced-disks'}; + + PVE::QemuServer::destroy_vm( + $storecfg, + $vmid, + $skiplock, { lock => 'destroyed' }, + $purge_unreferenced, + ); PVE::AccessControl::remove_vm_access($vmid); PVE::Firewall::remove_vmfw_conf($vmid); @@ -1599,7 +2327,7 @@ __PACKAGE__->register_method({ check => [ 'perm', '/vms/{vmid}', ['VM.Config.Disk']], }, parameters => { - additionalProperties => 0, + additionalProperties => 0, properties => { node => get_standard_option('pve-node'), vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }), @@ -1654,14 +2382,15 @@ __PACKAGE__->register_method({ }, description => "Creates a TCP VNC proxy connections.", parameters => { - additionalProperties => 0, + additionalProperties => 0, properties => { node => get_standard_option('pve-node'), vmid => get_standard_option('pve-vmid'), websocket => { optional => 1, type => 'boolean', - description => "starts websockify instead of vncproxy", + description => "Prepare for websocket upgrade (only required when using " + ."serial terminal, otherwise upgrade is always possible).", }, 'generate-password' => { optional => 1, @@ -1672,7 +2401,7 @@ __PACKAGE__->register_method({ }, }, returns => { - additionalProperties => 0, + additionalProperties => 0, properties => { user => { type => 'string' }, ticket => { type => 'string' }, @@ -1759,7 +2488,7 @@ __PACKAGE__->register_method({ } else { - $ENV{LC_PVE_TICKET} = $password if $websocket; # set ticket with "qm vncproxy" + $ENV{LC_PVE_TICKET} = $password; # set ticket with "qm vncproxy" $cmd = [@$remcmd, "/usr/sbin/qm", 'vncproxy', $vmid]; @@ -1912,7 +2641,7 @@ __PACKAGE__->register_method({ }, description => "Opens a weksocket for VNC traffic.", parameters => { - additionalProperties => 0, + additionalProperties => 0, properties => { node => get_standard_option('pve-node'), vmid => get_standard_option('pve-vmid'), @@ -1971,7 +2700,7 @@ __PACKAGE__->register_method({ }, description => "Returns a SPICE configuration to connect to the VM.", parameters => { - additionalProperties => 0, + additionalProperties => 0, properties => { node => get_standard_option('pve-node'), vmid => get_standard_option('pve-vmid'), @@ -2015,7 +2744,7 @@ __PACKAGE__->register_method({ user => 'all', }, parameters => { - additionalProperties => 0, + additionalProperties => 0, properties => { node => get_standard_option('pve-node'), vmid => get_standard_option('pve-vmid'), @@ -2061,7 +2790,7 @@ __PACKAGE__->register_method({ check => ['perm', '/vms/{vmid}', [ 'VM.Audit' ]], }, parameters => { - additionalProperties => 0, + additionalProperties => 0, properties => { node => get_standard_option('pve-node'), vmid => get_standard_option('pve-vmid'), @@ -2076,15 +2805,22 @@ __PACKAGE__->register_method({ type => 'object', }, spice => { - description => "Qemu VGA configuration supports spice.", + description => "QEMU VGA configuration supports spice.", type => 'boolean', optional => 1, }, agent => { - description => "Qemu GuestAgent enabled in config.", + description => "QEMU Guest Agent is enabled in config.", type => 'boolean', optional => 1, }, + clipboard => { + description => 'Enable a specific clipboard. If not set, depending on' + .' the display type the SPICE one will be added.', + type => 'string', + enum => ['vnc'], + optional => 1, + }, }, }, code => sub { @@ -2098,8 +2834,14 @@ __PACKAGE__->register_method({ $status->{ha} = PVE::HA::Config::get_service_status("vm:$param->{vmid}"); - $status->{spice} = 1 if PVE::QemuServer::vga_conf_has_spice($conf->{vga}); - $status->{agent} = 1 if (PVE::QemuServer::parse_guest_agent($conf)->{enabled}); + if ($conf->{vga}) { + my $vga = PVE::QemuServer::parse_vga($conf->{vga}); + my $spice = defined($vga->{type}) && $vga->{type} =~ /^virtio/; + $spice ||= PVE::QemuServer::vga_conf_has_spice($conf->{vga}); + $status->{spice} = 1 if $spice; + $status->{clipboard} = $vga->{clipboard}; + } + $status->{agent} = 1 if PVE::QemuServer::get_qga_key($conf, 'enabled'); return $status; }}); @@ -2115,7 +2857,7 @@ __PACKAGE__->register_method({ check => ['perm', '/vms/{vmid}', [ 'VM.PowerMgmt' ]], }, parameters => { - additionalProperties => 0, + additionalProperties => 0, properties => { node => get_standard_option('pve-node'), vmid => get_standard_option('pve-vmid', @@ -2164,9 +2906,7 @@ __PACKAGE__->register_method({ my $node = extract_param($param, 'node'); my $vmid = extract_param($param, 'vmid'); my $timeout = extract_param($param, 'timeout'); - my $machine = extract_param($param, 'machine'); - my $force_cpu = extract_param($param, 'force-cpu'); my $get_root_param = sub { my $value = extract_param($param, $_[0]); @@ -2181,6 +2921,7 @@ __PACKAGE__->register_method({ my $migration_type = $get_root_param->('migration_type'); my $migration_network = $get_root_param->('migration_network'); my $targetstorage = $get_root_param->('targetstorage'); + my $force_cpu = $get_root_param->('force-cpu'); my $storagemap; @@ -2196,6 +2937,7 @@ __PACKAGE__->register_method({ my $spice_ticket; my $nbd_protocol_version = 0; my $replicated_volumes = {}; + my $offline_volumes = {}; if ($stateuri && ($stateuri eq 'tcp' || $stateuri eq 'unix') && $migratedfrom && ($rpcenv->{type} eq 'cli')) { while (defined(my $line = )) { chomp $line; @@ -2205,9 +2947,15 @@ __PACKAGE__->register_method({ $nbd_protocol_version = $1; } elsif ($line =~ m/^replicated_volume: (.*)$/) { $replicated_volumes->{$1} = 1; - } else { + } elsif ($line =~ m/^tpmstate0: (.*)$/) { # Deprecated, use offline_volume instead + $offline_volumes->{tpmstate0} = $1; + } elsif ($line =~ m/^offline_volume: ([^:]+): (.*)$/) { + $offline_volumes->{$1} = $2; + } elsif (!$spice_ticket) { # fallback for old source node $spice_ticket = $line; + } else { + warn "unknown 'start' parameter on STDIN: '$line'\n"; } } } @@ -2244,6 +2992,7 @@ __PACKAGE__->register_method({ storagemap => $storagemap, nbd_proto_version => $nbd_protocol_version, replicated_volumes => $replicated_volumes, + offline_volumes => $offline_volumes, }; my $params = { @@ -2268,13 +3017,13 @@ __PACKAGE__->register_method({ method => 'POST', protected => 1, proxyto => 'node', - description => "Stop virtual machine. The qemu process will exit immediately. This" . - "is akin to pulling the power plug of a running computer and may damage the VM data", + description => "Stop virtual machine. The qemu process will exit immediately. This" + ." is akin to pulling the power plug of a running computer and may damage the VM data.", permissions => { check => ['perm', '/vms/{vmid}', [ 'VM.PowerMgmt' ]], }, parameters => { - additionalProperties => 0, + additionalProperties => 0, properties => { node => get_standard_option('pve-node'), vmid => get_standard_option('pve-vmid', @@ -2292,7 +3041,13 @@ __PACKAGE__->register_method({ type => 'boolean', optional => 1, default => 0, - } + }, + 'overrule-shutdown' => { + description => "Try to abort active 'qmshutdown' tasks before stopping.", + optional => 1, + type => 'boolean', + default => 0, + }, }, }, returns => { @@ -2319,10 +3074,13 @@ __PACKAGE__->register_method({ raise_param_exc({ migratedfrom => "Only root may use this option." }) if $migratedfrom && $authuser ne 'root@pam'; + my $overrule_shutdown = extract_param($param, 'overrule-shutdown'); my $storecfg = PVE::Storage::config(); if (PVE::HA::Config::vm_is_ha_managed($vmid) && ($rpcenv->{type} ne 'ha') && !defined($migratedfrom)) { + raise_param_exc({ 'overrule-shutdown' => "Not applicable for HA resources." }) + if $overrule_shutdown; my $hacmd = sub { my $upid = shift; @@ -2342,6 +3100,14 @@ __PACKAGE__->register_method({ syslog('info', "stop VM $vmid: $upid\n"); + if ($overrule_shutdown) { + my $overruled_tasks = PVE::GuestHelpers::abort_guest_tasks( + $rpcenv, 'qmshutdown', $vmid); + my $overruled_tasks_list = join(", ", $overruled_tasks->@*); + print "overruled qmshutdown tasks: $overruled_tasks_list\n" + if @$overruled_tasks; + }; + PVE::QemuServer::vm_stop($storecfg, $vmid, $skiplock, 0, $param->{timeout}, 0, 1, $keepActive, $migratedfrom); return; @@ -2362,7 +3128,7 @@ __PACKAGE__->register_method({ check => ['perm', '/vms/{vmid}', [ 'VM.PowerMgmt' ]], }, parameters => { - additionalProperties => 0, + additionalProperties => 0, properties => { node => get_standard_option('pve-node'), vmid => get_standard_option('pve-vmid', @@ -2401,29 +3167,20 @@ __PACKAGE__->register_method({ return $rpcenv->fork_worker('qmreset', $vmid, $authuser, $realcmd); }}); -my sub vm_is_paused { - my ($vmid) = @_; - my $qmpstatus = eval { - PVE::QemuConfig::assert_config_exists_on_node($vmid); - mon_cmd($vmid, "query-status"); - }; - warn "$@\n" if $@; - return $qmpstatus && $qmpstatus->{status} eq "paused"; -} - __PACKAGE__->register_method({ name => 'vm_shutdown', path => '{vmid}/status/shutdown', method => 'POST', protected => 1, proxyto => 'node', - description => "Shutdown virtual machine. This is similar to pressing the power button on a physical machine." . - "This will send an ACPI event for the guest OS, which should then proceed to a clean shutdown.", + description => "Shutdown virtual machine. This is similar to pressing the power button on a" + ." physical machine. This will send an ACPI event for the guest OS, which should then" + ." proceed to a clean shutdown.", permissions => { check => ['perm', '/vms/{vmid}', [ 'VM.PowerMgmt' ]], }, parameters => { - additionalProperties => 0, + additionalProperties => 0, properties => { node => get_standard_option('pve-node'), vmid => get_standard_option('pve-vmid', @@ -2473,13 +3230,9 @@ __PACKAGE__->register_method({ my $shutdown = 1; - # if vm is paused, do not shutdown (but stop if forceStop = 1) - # otherwise, we will infer a shutdown command, but run into the timeout, - # then when the vm is resumed, it will instantly shutdown - # - # checking the qmp status here to get feedback to the gui/cli/api - # and the status query should not take too long - if (vm_is_paused($vmid)) { + # sending a graceful shutdown command to paused VMs runs into timeouts, and even worse, when + # the VM gets resumed later, it still gets the request delivered and powers off + if (PVE::QemuServer::vm_is_paused($vmid, 1)) { if ($param->{forceStop}) { warn "VM is paused - stop instead of shutdown\n"; $shutdown = 0; @@ -2555,7 +3308,7 @@ __PACKAGE__->register_method({ my $node = extract_param($param, 'node'); my $vmid = extract_param($param, 'vmid'); - die "VM is paused - cannot shutdown\n" if vm_is_paused($vmid); + die "VM is paused - cannot shutdown\n" if PVE::QemuServer::vm_is_paused($vmid, 1); die "VM $vmid not running\n" if !PVE::QemuServer::check_running($vmid); @@ -2584,7 +3337,7 @@ __PACKAGE__->register_method({ check => ['perm', '/vms/{vmid}', [ 'VM.PowerMgmt' ]], }, parameters => { - additionalProperties => 0, + additionalProperties => 0, properties => { node => get_standard_option('pve-node'), vmid => get_standard_option('pve-vmid', @@ -2632,10 +3385,17 @@ __PACKAGE__->register_method({ # early check for storage permission, for better user feedback if ($todisk) { $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']); + my $conf = PVE::QemuConfig->load_config($vmid); + + # cannot save the state of a non-virtualized PCIe device, so resume cannot really work + for my $key (keys %$conf) { + next if $key !~ /^hostpci\d+/; + die "cannot suspend VM to disk due to passed-through PCI device(s), which lack the" + ." possibility to save/restore their internal state\n"; + } if (!$statestorage) { # get statestorage from config if none is given - my $conf = PVE::QemuConfig->load_config($vmid); my $storecfg = PVE::Storage::config(); $statestorage = PVE::QemuServer::find_vmstate_storage($conf, $storecfg); } @@ -2668,7 +3428,7 @@ __PACKAGE__->register_method({ check => ['perm', '/vms/{vmid}', [ 'VM.PowerMgmt' ]], }, parameters => { - additionalProperties => 0, + additionalProperties => 0, properties => { node => get_standard_option('pve-node'), vmid => get_standard_option('pve-vmid', @@ -2696,6 +3456,8 @@ __PACKAGE__->register_method({ raise_param_exc({ skiplock => "Only root may use this option." }) if $skiplock && $authuser ne 'root@pam'; + # nocheck is used as part of migration when config file might be still + # be on source node my $nocheck = extract_param($param, 'nocheck'); raise_param_exc({ nocheck => "Only root may use this option." }) if $nocheck && $authuser ne 'root@pam'; @@ -2740,7 +3502,7 @@ __PACKAGE__->register_method({ check => ['perm', '/vms/{vmid}', [ 'VM.Console' ]], }, parameters => { - additionalProperties => 0, + additionalProperties => 0, properties => { node => get_standard_option('pve-node'), vmid => get_standard_option('pve-vmid', @@ -2784,7 +3546,7 @@ __PACKAGE__->register_method({ check => ['perm', '/vms/{vmid}', [ 'VM.Audit' ]], }, parameters => { - additionalProperties => 0, + additionalProperties => 0, properties => { node => get_standard_option('pve-node'), vmid => get_standard_option('pve-vmid'), @@ -2849,7 +3611,7 @@ __PACKAGE__->register_method({ permissions => { description => "You need 'VM.Clone' permissions on /vms/{vmid}, and 'VM.Allocate' permissions " . "on /vms/{newid} (or on the VM pool /pool/{pool}). You also need " . - "'Datastore.AllocateSpace' on any used storage.", + "'Datastore.AllocateSpace' on any used storage and 'SDN.Use' on any used bridge/vnet", check => [ 'and', ['perm', '/vms/{vmid}', [ 'VM.Clone' ]], @@ -2860,7 +3622,7 @@ __PACKAGE__->register_method({ ] }, parameters => { - additionalProperties => 0, + additionalProperties => 0, properties => { node => get_standard_option('pve-node'), vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }), @@ -2927,7 +3689,6 @@ __PACKAGE__->register_method({ my $vmid = extract_param($param, 'vmid'); my $newid = extract_param($param, 'newid'); my $pool = extract_param($param, 'pool'); - $rpcenv->check_pool_exist($pool) if defined($pool); my $snapname = extract_param($param, 'snapname'); my $storage = extract_param($param, 'storage'); @@ -2940,28 +3701,28 @@ __PACKAGE__->register_method({ undef $target; } - PVE::Cluster::check_node_exists($target) if $target; - - my $storecfg = PVE::Storage::config(); + my $running = PVE::QemuServer::check_running($vmid) || 0; - if ($storage) { - # check if storage is enabled on local node - PVE::Storage::storage_check_enabled($storecfg, $storage); - if ($target) { - # check if storage is available on target node - PVE::Storage::storage_check_node($storecfg, $storage, $target); - # clone only works if target storage is shared - my $scfg = PVE::Storage::storage_config($storecfg, $storage); - die "can't clone to non-shared storage '$storage'\n" if !$scfg->{shared}; - } - } + my $load_and_check = sub { + $rpcenv->check_pool_exist($pool) if defined($pool); + PVE::Cluster::check_node_exists($target) if $target; - PVE::Cluster::check_cfs_quorum(); + my $storecfg = PVE::Storage::config(); - my $running = PVE::QemuServer::check_running($vmid) || 0; + if ($storage) { + # check if storage is enabled on local node + PVE::Storage::storage_check_enabled($storecfg, $storage); + if ($target) { + # check if storage is available on target node + PVE::Storage::storage_check_enabled($storecfg, $storage, $target); + # clone only works if target storage is shared + my $scfg = PVE::Storage::storage_config($storecfg, $storage); + die "can't clone to non-shared storage '$storage'\n" + if !$scfg->{shared}; + } + } - my $clonefn = sub { - # do all tests after lock but before forking worker - if possible + PVE::Cluster::check_cfs_quorum(); my $conf = PVE::QemuConfig->load_config($vmid); PVE::QemuConfig->check_lock($conf); @@ -2972,7 +3733,7 @@ __PACKAGE__->register_method({ die "snapshot '$snapname' does not exist\n" if $snapname && !defined( $conf->{snapshots}->{$snapname}); - my $full = extract_param($param, 'full') // !PVE::QemuConfig->is_template($conf); + my $full = $param->{full} // !PVE::QemuConfig->is_template($conf); die "parameter 'storage' not allowed for linked clones\n" if defined($storage) && !$full; @@ -2983,6 +3744,9 @@ __PACKAGE__->register_method({ my $oldconf = $snapname ? $conf->{snapshots}->{$snapname} : $conf; my $sharedvm = &$check_storage_access_clone($rpcenv, $authuser, $storecfg, $oldconf, $storage); + PVE::QemuServer::check_mapping_access($rpcenv, $authuser, $oldconf); + + PVE::QemuServer::check_bridge_access($rpcenv, $authuser, $oldconf); die "can't clone VM to node '$target' (VM uses local storage)\n" if $target && !$sharedvm; @@ -3006,6 +3770,9 @@ __PACKAGE__->register_method({ # no need to copy unused images, because VMID(owner) changes anyways next if $opt =~ m/^unused\d+$/; + die "cannot clone TPM state while VM is running\n" + if $full && $running && !$snapname && $opt eq 'tpmstate0'; + # always change MAC! address if ($opt =~ m/^net(\d+)$/) { my $net = PVE::QemuServer::parse_net($value); @@ -3037,7 +3804,14 @@ __PACKAGE__->register_method({ } } - # auto generate a new uuid + return ($conffile, $newconf, $oldconf, $vollist, $drives, $fullclone); + }; + + my $clonefn = sub { + my ($conffile, $newconf, $oldconf, $vollist, $drives, $fullclone) = $load_and_check->(); + my $storecfg = PVE::Storage::config(); + + # auto generate a new uuid my $smbios1 = PVE::QemuServer::parse_smbios1($newconf->{smbios1} || ''); $smbios1->{uuid} = PVE::QemuServer::generate_uuid(); $newconf->{smbios1} = PVE::QemuServer::print_smbios1($smbios1); @@ -3062,90 +3836,117 @@ __PACKAGE__->register_method({ # FIXME use PVE::QemuConfig->create_and_lock_config and adapt code PVE::Tools::file_set_contents($conffile, "# qmclone temporary file\nlock: clone\n"); - my $realcmd = sub { - my $upid = shift; + PVE::Firewall::clone_vmfw_conf($vmid, $newid); - my $newvollist = []; - my $jobs = {}; + my $newvollist = []; + my $jobs = {}; - eval { - local $SIG{INT} = - local $SIG{TERM} = - local $SIG{QUIT} = - local $SIG{HUP} = sub { die "interrupted by signal\n"; }; + eval { + local $SIG{INT} = + local $SIG{TERM} = + local $SIG{QUIT} = + local $SIG{HUP} = sub { die "interrupted by signal\n"; }; - PVE::Storage::activate_volumes($storecfg, $vollist, $snapname); + PVE::Storage::activate_volumes($storecfg, $vollist, $snapname); - my $bwlimit = extract_param($param, 'bwlimit'); + my $bwlimit = extract_param($param, 'bwlimit'); - my $total_jobs = scalar(keys %{$drives}); - my $i = 1; + my $total_jobs = scalar(keys %{$drives}); + my $i = 1; - foreach my $opt (keys %$drives) { - my $drive = $drives->{$opt}; - my $skipcomplete = ($total_jobs != $i); # finish after last drive - my $completion = $skipcomplete ? 'skip' : 'complete'; + foreach my $opt (sort keys %$drives) { + my $drive = $drives->{$opt}; + my $skipcomplete = ($total_jobs != $i); # finish after last drive + my $completion = $skipcomplete ? 'skip' : 'complete'; - my $src_sid = PVE::Storage::parse_volume_id($drive->{file}); - my $storage_list = [ $src_sid ]; - push @$storage_list, $storage if defined($storage); - my $clonelimit = PVE::Storage::get_bandwidth_limit('clone', $storage_list, $bwlimit); + my $src_sid = PVE::Storage::parse_volume_id($drive->{file}); + my $storage_list = [ $src_sid ]; + push @$storage_list, $storage if defined($storage); + my $clonelimit = PVE::Storage::get_bandwidth_limit('clone', $storage_list, $bwlimit); - my $newdrive = PVE::QemuServer::clone_disk($storecfg, $vmid, $running, $opt, $drive, $snapname, - $newid, $storage, $format, $fullclone->{$opt}, $newvollist, - $jobs, $completion, $oldconf->{agent}, $clonelimit, $oldconf); + my $source_info = { + vmid => $vmid, + running => $running, + drivename => $opt, + drive => $drive, + snapname => $snapname, + }; - $newconf->{$opt} = PVE::QemuServer::print_drive($newdrive); + my $dest_info = { + vmid => $newid, + drivename => $opt, + storage => $storage, + format => $format, + }; - PVE::QemuConfig->write_config($newid, $newconf); - $i++; - } + $dest_info->{efisize} = PVE::QemuServer::get_efivars_size($oldconf) + if $opt eq 'efidisk0'; - delete $newconf->{lock}; + my $newdrive = PVE::QemuServer::clone_disk( + $storecfg, + $source_info, + $dest_info, + $fullclone->{$opt}, + $newvollist, + $jobs, + $completion, + $oldconf->{agent}, + $clonelimit, + ); - # do not write pending changes - if (my @changes = keys %{$newconf->{pending}}) { - my $pending = join(',', @changes); - warn "found pending changes for '$pending', discarding for clone\n"; - delete $newconf->{pending}; - } + $newconf->{$opt} = PVE::QemuServer::print_drive($newdrive); PVE::QemuConfig->write_config($newid, $newconf); + $i++; + } - if ($target) { - # always deactivate volumes - avoid lvm LVs to be active on several nodes - PVE::Storage::deactivate_volumes($storecfg, $vollist, $snapname) if !$running; - PVE::Storage::deactivate_volumes($storecfg, $newvollist); + delete $newconf->{lock}; - my $newconffile = PVE::QemuConfig->config_file($newid, $target); - die "Failed to move config to node '$target' - rename failed: $!\n" - if !rename($conffile, $newconffile); - } + # do not write pending changes + if (my @changes = keys %{$newconf->{pending}}) { + my $pending = join(',', @changes); + warn "found pending changes for '$pending', discarding for clone\n"; + delete $newconf->{pending}; + } - PVE::AccessControl::add_vm_to_pool($newid, $pool) if $pool; - }; - if (my $err = $@) { - eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) }; - sleep 1; # some storage like rbd need to wait before release volume - really? + PVE::QemuConfig->write_config($newid, $newconf); - foreach my $volid (@$newvollist) { - eval { PVE::Storage::vdisk_free($storecfg, $volid); }; - warn $@ if $@; - } + PVE::QemuServer::create_ifaces_ipams_ips($newconf, $newid); - PVE::Firewall::remove_vmfw_conf($newid); + if ($target) { + if (!$running) { + # always deactivate volumes – avoids that LVM LVs are active on several nodes + eval { PVE::Storage::deactivate_volumes($storecfg, $vollist, $snapname) }; + # but only warn when that fails (e.g., parallel clones keeping them active) + log_warn($@) if $@; + } - unlink $conffile; # avoid races -> last thing before die + PVE::Storage::deactivate_volumes($storecfg, $newvollist); - die "clone failed: $err"; + my $newconffile = PVE::QemuConfig->config_file($newid, $target); + die "Failed to move config to node '$target' - rename failed: $!\n" + if !rename($conffile, $newconffile); } - return; + PVE::AccessControl::add_vm_to_pool($newid, $pool) if $pool; }; + if (my $err = $@) { + eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) }; + sleep 1; # some storage like rbd need to wait before release volume - really? - PVE::Firewall::clone_vmfw_conf($vmid, $newid); + foreach my $volid (@$newvollist) { + eval { PVE::Storage::vdisk_free($storecfg, $volid); }; + warn $@ if $@; + } + + PVE::Firewall::remove_vmfw_conf($newid); + + unlink $conffile; # avoid races -> last thing before die - return $rpcenv->fork_worker('qmclone', $vmid, $authuser, $realcmd); + die "clone failed: $err"; + } + + return; }; # Aquire exclusive lock lock for $newid @@ -3153,12 +3954,18 @@ __PACKAGE__->register_method({ return PVE::QemuConfig->lock_config_full($newid, 1, $clonefn); }; - # exclusive lock if VM is running - else shared lock is enough; - if ($running) { - return PVE::QemuConfig->lock_config_full($vmid, 1, $lock_target_vm); - } else { - return PVE::QemuConfig->lock_config_shared($vmid, 1, $lock_target_vm); - } + my $lock_source_vm = sub { + # exclusive lock if VM is running - else shared lock is enough; + if ($running) { + return PVE::QemuConfig->lock_config_full($vmid, 1, $lock_target_vm); + } else { + return PVE::QemuConfig->lock_config_shared($vmid, 1, $lock_target_vm); + } + }; + + $load_and_check->(); # early checks before forking/locking + + return $rpcenv->fork_worker('qmclone', $vmid, $authuser, $lock_source_vm); }}); __PACKAGE__->register_method({ @@ -3167,43 +3974,49 @@ __PACKAGE__->register_method({ method => 'POST', protected => 1, proxyto => 'node', - description => "Move volume to different storage.", + description => "Move volume to different storage or to a different VM.", permissions => { - description => "You need 'VM.Config.Disk' permissions on /vms/{vmid}, and 'Datastore.AllocateSpace' permissions on the storage.", - check => [ 'and', - ['perm', '/vms/{vmid}', [ 'VM.Config.Disk' ]], - ['perm', '/storage/{storage}', [ 'Datastore.AllocateSpace' ]], - ], + description => "You need 'VM.Config.Disk' permissions on /vms/{vmid}, " . + "and 'Datastore.AllocateSpace' permissions on the storage. To move ". + "a disk to another VM, you need the permissions on the target VM as well.", + check => ['perm', '/vms/{vmid}', [ 'VM.Config.Disk' ]], }, parameters => { - additionalProperties => 0, + additionalProperties => 0, properties => { node => get_standard_option('pve-node'), vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }), + 'target-vmid' => get_standard_option('pve-vmid', { + completion => \&PVE::QemuServer::complete_vmid, + optional => 1, + }), disk => { type => 'string', description => "The disk you want to move.", - enum => [PVE::QemuServer::Drive::valid_drive_names()], + enum => [PVE::QemuServer::Drive::valid_drive_names_with_unused()], }, storage => get_standard_option('pve-storage-id', { description => "Target storage.", completion => \&PVE::QemuServer::complete_storage, + optional => 1, }), - 'format' => { - type => 'string', - description => "Target Format.", - enum => [ 'raw', 'qcow2', 'vmdk' ], - optional => 1, - }, + 'format' => { + type => 'string', + description => "Target Format.", + enum => [ 'raw', 'qcow2', 'vmdk' ], + optional => 1, + }, delete => { type => 'boolean', - description => "Delete the original disk after successful copy. By default the original disk is kept as unused disk.", + description => "Delete the original disk after successful copy. By default the" + ." original disk is kept as unused disk.", optional => 1, default => 0, }, digest => { type => 'string', - description => 'Prevent changes if current configuration file has different SHA1 digest. This can be used to prevent concurrent modifications.', + description => 'Prevent changes if current configuration file has different SHA1" + ." digest. This can be used to prevent concurrent modifications.', maxLength => 40, optional => 1, }, @@ -3214,6 +4027,20 @@ __PACKAGE__->register_method({ minimum => '0', default => 'move limit from datacenter or storage config', }, + 'target-disk' => { + type => 'string', + description => "The config key the disk will be moved to on the target VM" + ." (for example, ide0 or scsi1). Default is the source disk key.", + enum => [PVE::QemuServer::Drive::valid_drive_names_with_unused()], + optional => 1, + }, + 'target-digest' => { + type => 'string', + description => 'Prevent changes if the current config file of the target VM has a" + ." different SHA1 digest. This can be used to detect concurrent modifications.', + maxLength => 40, + optional => 1, + }, }, }, returns => { @@ -3228,19 +4055,21 @@ __PACKAGE__->register_method({ my $node = extract_param($param, 'node'); my $vmid = extract_param($param, 'vmid'); + my $target_vmid = extract_param($param, 'target-vmid'); my $digest = extract_param($param, 'digest'); + my $target_digest = extract_param($param, 'target-digest'); my $disk = extract_param($param, 'disk'); + my $target_disk = extract_param($param, 'target-disk') // $disk; my $storeid = extract_param($param, 'storage'); my $format = extract_param($param, 'format'); my $storecfg = PVE::Storage::config(); - my $updatefn = sub { + my $load_and_check_move = sub { my $conf = PVE::QemuConfig->load_config($vmid); PVE::QemuConfig->check_lock($conf); - die "VM config checksum missmatch (file change by other user?)\n" - if $digest && $digest ne $conf->{digest}; + PVE::Tools::assert_if_modified($digest, $conf->{digest}); die "disk '$disk' does not exist\n" if !$conf->{$disk}; @@ -3256,81 +4085,315 @@ __PACKAGE__->register_method({ $oldfmt = $1; } - die "you can't move to the same storage with same format\n" if $oldstoreid eq $storeid && - (!$format || !$oldfmt || $oldfmt eq $format); + die "you can't move to the same storage with same format\n" + if $oldstoreid eq $storeid && (!$format || !$oldfmt || $oldfmt eq $format); # this only checks snapshots because $disk is passed! - my $snapshotted = PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $disk, $old_volid); + my $snapshotted = PVE::QemuServer::Drive::is_volume_in_use( + $storecfg, + $conf, + $disk, + $old_volid + ); die "you can't move a disk with snapshots and delete the source\n" if $snapshotted && $param->{delete}; - PVE::Cluster::log_msg('info', $authuser, "move disk VM $vmid: move --disk $disk --storage $storeid"); + return ($conf, $drive, $oldstoreid, $snapshotted); + }; + + my $move_updatefn = sub { + my ($conf, $drive, $oldstoreid, $snapshotted) = $load_and_check_move->(); + my $old_volid = $drive->{file}; + + PVE::Cluster::log_msg( + 'info', + $authuser, + "move disk VM $vmid: move --disk $disk --storage $storeid" + ); my $running = PVE::QemuServer::check_running($vmid); PVE::Storage::activate_volumes($storecfg, [ $drive->{file} ]); - my $realcmd = sub { - my $newvollist = []; + my $newvollist = []; + + eval { + local $SIG{INT} = + local $SIG{TERM} = + local $SIG{QUIT} = + local $SIG{HUP} = sub { die "interrupted by signal\n"; }; + + warn "moving disk with snapshots, snapshots will not be moved!\n" + if $snapshotted; + + my $bwlimit = extract_param($param, 'bwlimit'); + my $movelimit = PVE::Storage::get_bandwidth_limit( + 'move', + [$oldstoreid, $storeid], + $bwlimit + ); + + my $source_info = { + vmid => $vmid, + running => $running, + drivename => $disk, + drive => $drive, + snapname => undef, + }; + + my $dest_info = { + vmid => $vmid, + drivename => $disk, + storage => $storeid, + format => $format, + }; + + $dest_info->{efisize} = PVE::QemuServer::get_efivars_size($conf) + if $disk eq 'efidisk0'; + + my $newdrive = PVE::QemuServer::clone_disk( + $storecfg, + $source_info, + $dest_info, + 1, + $newvollist, + undef, + undef, + undef, + $movelimit, + ); + $conf->{$disk} = PVE::QemuServer::print_drive($newdrive); + + PVE::QemuConfig->add_unused_volume($conf, $old_volid) if !$param->{delete}; + + # convert moved disk to base if part of template + PVE::QemuServer::template_create($vmid, $conf, $disk) + if PVE::QemuConfig->is_template($conf); + + PVE::QemuConfig->write_config($vmid, $conf); + + my $do_trim = PVE::QemuServer::get_qga_key($conf, 'fstrim_cloned_disks'); + if ($running && $do_trim && PVE::QemuServer::qga_check_running($vmid)) { + eval { mon_cmd($vmid, "guest-fstrim") }; + } eval { - local $SIG{INT} = - local $SIG{TERM} = - local $SIG{QUIT} = - local $SIG{HUP} = sub { die "interrupted by signal\n"; }; + # try to deactivate volumes - avoid lvm LVs to be active on several nodes + PVE::Storage::deactivate_volumes($storecfg, [ $newdrive->{file} ]) + if !$running; + }; + warn $@ if $@; + }; + if (my $err = $@) { + foreach my $volid (@$newvollist) { + eval { PVE::Storage::vdisk_free($storecfg, $volid) }; + warn $@ if $@; + } + die "storage migration failed: $err"; + } - warn "moving disk with snapshots, snapshots will not be moved!\n" - if $snapshotted; + if ($param->{delete}) { + eval { + PVE::Storage::deactivate_volumes($storecfg, [$old_volid]); + PVE::Storage::vdisk_free($storecfg, $old_volid); + }; + warn $@ if $@; + } + }; - my $bwlimit = extract_param($param, 'bwlimit'); - my $movelimit = PVE::Storage::get_bandwidth_limit('move', [$oldstoreid, $storeid], $bwlimit); + my $load_and_check_reassign_configs = sub { + my $vmlist = PVE::Cluster::get_vmlist()->{ids}; - my $newdrive = PVE::QemuServer::clone_disk($storecfg, $vmid, $running, $disk, $drive, undef, - $vmid, $storeid, $format, 1, $newvollist, undef, undef, undef, $movelimit, $conf); + die "could not find VM ${vmid}\n" if !exists($vmlist->{$vmid}); + die "could not find target VM ${target_vmid}\n" if !exists($vmlist->{$target_vmid}); - $conf->{$disk} = PVE::QemuServer::print_drive($newdrive); + my $source_node = $vmlist->{$vmid}->{node}; + my $target_node = $vmlist->{$target_vmid}->{node}; - PVE::QemuConfig->add_unused_volume($conf, $old_volid) if !$param->{delete}; + die "Both VMs need to be on the same node ($source_node != $target_node)\n" + if $source_node ne $target_node; - # convert moved disk to base if part of template - PVE::QemuServer::template_create($vmid, $conf, $disk) - if PVE::QemuConfig->is_template($conf); + my $source_conf = PVE::QemuConfig->load_config($vmid); + PVE::QemuConfig->check_lock($source_conf); + my $target_conf = PVE::QemuConfig->load_config($target_vmid); + PVE::QemuConfig->check_lock($target_conf); - PVE::QemuConfig->write_config($vmid, $conf); + die "Can't move disks from or to template VMs\n" + if ($source_conf->{template} || $target_conf->{template}); - my $do_trim = PVE::QemuServer::parse_guest_agent($conf)->{fstrim_cloned_disks}; - if ($running && $do_trim && PVE::QemuServer::qga_check_running($vmid)) { - eval { mon_cmd($vmid, "guest-fstrim") }; - } + if ($digest) { + eval { PVE::Tools::assert_if_modified($digest, $source_conf->{digest}) }; + die "VM ${vmid}: $@" if $@; + } - eval { - # try to deactivate volumes - avoid lvm LVs to be active on several nodes - PVE::Storage::deactivate_volumes($storecfg, [ $newdrive->{file} ]) - if !$running; - }; - warn $@ if $@; - }; - if (my $err = $@) { - foreach my $volid (@$newvollist) { - eval { PVE::Storage::vdisk_free($storecfg, $volid) }; - warn $@ if $@; + if ($target_digest) { + eval { PVE::Tools::assert_if_modified($target_digest, $target_conf->{digest}) }; + die "VM ${target_vmid}: $@" if $@; + } + + die "Disk '${disk}' for VM '$vmid' does not exist\n" if !defined($source_conf->{$disk}); + + die "Target disk key '${target_disk}' is already in use for VM '$target_vmid'\n" + if $target_conf->{$target_disk}; + + my $drive = PVE::QemuServer::parse_drive( + $disk, + $source_conf->{$disk}, + ); + die "failed to parse source disk - $@\n" if !$drive; + + my $source_volid = $drive->{file}; + + die "disk '${disk}' has no associated volume\n" if !$source_volid; + die "CD drive contents can't be moved to another VM\n" + if PVE::QemuServer::drive_is_cdrom($drive, 1); + + my $storeid = PVE::Storage::parse_volume_id($source_volid, 1); + die "Volume '$source_volid' not managed by PVE\n" if !defined($storeid); + + die "Can't move disk used by a snapshot to another VM\n" + if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $source_conf, $disk, $source_volid); + die "Storage does not support moving of this disk to another VM\n" + if (!PVE::Storage::volume_has_feature($storecfg, 'rename', $source_volid)); + die "Cannot move disk to another VM while the source VM is running - detach first\n" + if PVE::QemuServer::check_running($vmid) && $disk !~ m/^unused\d+$/; + + # now re-parse using target disk slot format + if ($target_disk =~ /^unused\d+$/) { + $drive = PVE::QemuServer::parse_drive( + $target_disk, + $source_volid, + ); + } else { + $drive = PVE::QemuServer::parse_drive( + $target_disk, + $source_conf->{$disk}, + ); + } + die "failed to parse source disk for target disk format - $@\n" if !$drive; + + my $repl_conf = PVE::ReplicationConfig->new(); + if ($repl_conf->check_for_existing_jobs($target_vmid, 1)) { + my $format = (PVE::Storage::parse_volname($storecfg, $source_volid))[6]; + die "Cannot move disk to a replicated VM. Storage does not support replication!\n" + if !PVE::Storage::storage_can_replicate($storecfg, $storeid, $format); + } + + return ($source_conf, $target_conf, $drive); + }; + + my $logfunc = sub { + my ($msg) = @_; + print STDERR "$msg\n"; + }; + + my $disk_reassignfn = sub { + return PVE::QemuConfig->lock_config($vmid, sub { + return PVE::QemuConfig->lock_config($target_vmid, sub { + my ($source_conf, $target_conf, $drive) = &$load_and_check_reassign_configs(); + + my $source_volid = $drive->{file}; + + print "moving disk '$disk' from VM '$vmid' to '$target_vmid'\n"; + my ($storeid, $source_volname) = PVE::Storage::parse_volume_id($source_volid); + + my $fmt = (PVE::Storage::parse_volname($storecfg, $source_volid))[6]; + + my $new_volid = PVE::Storage::rename_volume( + $storecfg, + $source_volid, + $target_vmid, + ); + + $drive->{file} = $new_volid; + + my $boot_order = PVE::QemuServer::device_bootorder($source_conf); + if (defined(delete $boot_order->{$disk})) { + print "removing disk '$disk' from boot order config\n"; + my $boot_devs = [ sort { $boot_order->{$a} <=> $boot_order->{$b} } keys %$boot_order ]; + $source_conf->{boot} = PVE::QemuServer::print_bootorder($boot_devs); } - die "storage migration failed: $err"; - } - if ($param->{delete}) { - eval { - PVE::Storage::deactivate_volumes($storecfg, [$old_volid]); - PVE::Storage::vdisk_free($storecfg, $old_volid); - }; - warn $@ if $@; - } - }; + delete $source_conf->{$disk}; + print "removing disk '${disk}' from VM '${vmid}' config\n"; + PVE::QemuConfig->write_config($vmid, $source_conf); - return $rpcenv->fork_worker('qmmove', $vmid, $authuser, $realcmd); + my $drive_string = PVE::QemuServer::print_drive($drive); + + if ($target_disk =~ /^unused\d+$/) { + $target_conf->{$target_disk} = $drive_string; + PVE::QemuConfig->write_config($target_vmid, $target_conf); + } else { + &$update_vm_api( + { + node => $node, + vmid => $target_vmid, + digest => $target_digest, + $target_disk => $drive_string, + }, + 1, + ); + } + + # remove possible replication snapshots + if (PVE::Storage::volume_has_feature( + $storecfg, + 'replicate', + $source_volid), + ) { + eval { + PVE::Replication::prepare( + $storecfg, + [$new_volid], + undef, + 1, + undef, + $logfunc, + ) + }; + if (my $err = $@) { + print "Failed to remove replication snapshots on moved disk " . + "'$target_disk'. Manual cleanup could be necessary.\n"; + } + } + }); + }); }; - return PVE::QemuConfig->lock_config($vmid, $updatefn); + if ($target_vmid && $storeid) { + my $msg = "either set 'storage' or 'target-vmid', but not both"; + raise_param_exc({ 'target-vmid' => $msg, 'storage' => $msg }); + } elsif ($target_vmid) { + $rpcenv->check_vm_perm($authuser, $target_vmid, undef, ['VM.Config.Disk']) + if $authuser ne 'root@pam'; + + raise_param_exc({ 'target-vmid' => "must be different than source VMID to reassign disk" }) + if $vmid eq $target_vmid; + + my (undef, undef, $drive) = &$load_and_check_reassign_configs(); + my $storage = PVE::Storage::parse_volume_id($drive->{file}); + $rpcenv->check($authuser, "/storage/$storage", ['Datastore.AllocateSpace']); + + return $rpcenv->fork_worker( + 'qmmove', + "${vmid}-${disk}>${target_vmid}-${target_disk}", + $authuser, + $disk_reassignfn + ); + } elsif ($storeid) { + $rpcenv->check($authuser, "/storage/$storeid", ['Datastore.AllocateSpace']); + + $load_and_check_move->(); # early checks before forking/locking + + my $realcmd = sub { + PVE::QemuConfig->lock_config($vmid, $move_updatefn); + }; + + return $rpcenv->fork_worker('qmmove', $vmid, $authuser, $realcmd); + } else { + my $msg = "both 'storage' and 'target-vmid' missing, either needs to be set"; + raise_param_exc({ 'target-vmid' => $msg, 'storage' => $msg }); + } }}); my $check_vm_disks_local = sub { @@ -3407,7 +4470,11 @@ __PACKAGE__->register_method({ local_resources => { type => 'array', description => "List local resources e.g. pci, usb" - } + }, + 'mapped-resources' => { + type => 'array', + description => "List of mapped resources e.g. pci, usb" + }, }, }, code => sub { @@ -3436,7 +4503,16 @@ __PACKAGE__->register_method({ $res->{running} = PVE::QemuServer::check_running($vmid) ? 1:0; - # if vm is not running, return target nodes where local storage is available + my ($local_resources, $mapped_resources, $missing_mappings_by_node) = + PVE::QemuServer::check_local_resources($vmconf, 1); + delete $missing_mappings_by_node->{$localnode}; + + my $vga = PVE::QemuServer::parse_vga($vmconf->{vga}); + if ($res->{running} && $vga->{'clipboard'} && $vga->{'clipboard'} eq 'vnc') { + push $local_resources->@*, "clipboard=vnc"; + } + + # if vm is not running, return target nodes where local storage/mapped devices are available # for offline migration if (!$res->{running}) { $res->{allowed_nodes} = []; @@ -3444,7 +4520,13 @@ __PACKAGE__->register_method({ delete $checked_nodes->{$localnode}; foreach my $node (keys %$checked_nodes) { - if (!defined $checked_nodes->{$node}->{unavailable_storages}) { + my $missing_mappings = $missing_mappings_by_node->{$node}; + if (scalar($missing_mappings->@*)) { + $checked_nodes->{$node}->{'unavailable-resources'} = $missing_mappings; + next; + } + + if (!defined($checked_nodes->{$node}->{unavailable_storages})) { push @{$res->{allowed_nodes}}, $node; } @@ -3452,13 +4534,11 @@ __PACKAGE__->register_method({ $res->{not_allowed_nodes} = $checked_nodes; } - my $local_disks = &$check_vm_disks_local($storecfg, $vmconf, $vmid); $res->{local_disks} = [ values %$local_disks ];; - my $local_resources = PVE::QemuServer::check_local_resources($vmconf, 1); - $res->{local_resources} = $local_resources; + $res->{'mapped-resources'} = $mapped_resources; return $res; @@ -3476,7 +4556,7 @@ __PACKAGE__->register_method({ check => ['perm', '/vms/{vmid}', [ 'VM.Migrate' ]], }, parameters => { - additionalProperties => 0, + additionalProperties => 0, properties => { node => get_standard_option('pve-node'), vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }), @@ -3568,14 +4648,9 @@ __PACKAGE__->register_method({ my $repl_conf = PVE::ReplicationConfig->new(); my $is_replicated = $repl_conf->check_for_existing_jobs($vmid, 1); my $is_replicated_to_target = defined($repl_conf->find_local_replication_job($vmid, $target)); - if ($is_replicated && !$is_replicated_to_target) { - if ($param->{force}) { - warn "WARNING: Node '$target' is not a replication target. Existing replication " . - "jobs will fail after migration!\n"; - } else { - die "Cannot live-migrate replicated VM to node '$target' - not a replication target." . - " Use 'force' to override.\n"; - } + if (!$param->{force} && $is_replicated && !$is_replicated_to_target) { + die "Cannot live-migrate replicated VM to node '$target' - not a replication " . + "target. Use 'force' to override.\n"; } } else { warn "VM isn't running. Doing offline migration instead.\n" if $param->{online}; @@ -3583,17 +4658,7 @@ __PACKAGE__->register_method({ } my $storecfg = PVE::Storage::config(); - if (my $targetstorage = $param->{targetstorage}) { - my $check_storage = sub { - my ($target_sid) = @_; - PVE::Storage::storage_check_node($storecfg, $target_sid, $target); - $rpcenv->check($authuser, "/storage/$target_sid", ['Datastore.AllocateSpace']); - my $scfg = PVE::Storage::storage_config($storecfg, $target_sid); - raise_param_exc({ targetstorage => "storage '$target_sid' does not support vm images"}) - if !$scfg->{content}->{images}; - }; - my $storagemap = eval { PVE::JSONSchema::parse_idmap($targetstorage, 'pve-storage-id') }; raise_param_exc({ targetstorage => "failed to parse storage map: $@" }) if $@; @@ -3601,11 +4666,11 @@ __PACKAGE__->register_method({ $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']) if !defined($storagemap->{identity}); - foreach my $source (values %{$storagemap->{entries}}) { - $check_storage->($source); + foreach my $target_sid (values %{$storagemap->{entries}}) { + $check_storage_access_migrate->($rpcenv, $authuser, $storecfg, $target_sid, $target); } - $check_storage->($storagemap->{default}) + $check_storage_access_migrate->($rpcenv, $authuser, $storecfg, $storagemap->{default}, $target) if $storagemap->{default}; PVE::QemuServer::check_storage_availability($storecfg, $conf, $target) @@ -3645,19 +4710,184 @@ __PACKAGE__->register_method({ }}); +__PACKAGE__->register_method({ + name => 'remote_migrate_vm', + path => '{vmid}/remote_migrate', + method => 'POST', + protected => 1, + proxyto => 'node', + description => "Migrate virtual machine to a remote cluster. Creates a new migration task. EXPERIMENTAL feature!", + permissions => { + check => ['perm', '/vms/{vmid}', [ 'VM.Migrate' ]], + }, + parameters => { + additionalProperties => 0, + properties => { + node => get_standard_option('pve-node'), + vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }), + 'target-vmid' => get_standard_option('pve-vmid', { optional => 1 }), + 'target-endpoint' => get_standard_option('proxmox-remote', { + description => "Remote target endpoint", + }), + online => { + type => 'boolean', + description => "Use online/live migration if VM is running. Ignored if VM is stopped.", + optional => 1, + }, + delete => { + type => 'boolean', + description => "Delete the original VM and related data after successful migration. By default the original VM is kept on the source cluster in a stopped state.", + optional => 1, + default => 0, + }, + 'target-storage' => get_standard_option('pve-targetstorage', { + completion => \&PVE::QemuServer::complete_migration_storage, + optional => 0, + }), + 'target-bridge' => { + type => 'string', + description => "Mapping from source to target bridges. Providing only a single bridge ID maps all source bridges to that bridge. Providing the special value '1' will map each source bridge to itself.", + format => 'bridge-pair-list', + }, + bwlimit => { + description => "Override I/O bandwidth limit (in KiB/s).", + optional => 1, + type => 'integer', + minimum => '0', + default => 'migrate limit from datacenter or storage config', + }, + }, + }, + returns => { + type => 'string', + description => "the task ID.", + }, + code => sub { + my ($param) = @_; + + my $rpcenv = PVE::RPCEnvironment::get(); + my $authuser = $rpcenv->get_user(); + + my $source_vmid = extract_param($param, 'vmid'); + my $target_endpoint = extract_param($param, 'target-endpoint'); + my $target_vmid = extract_param($param, 'target-vmid') // $source_vmid; + + my $delete = extract_param($param, 'delete') // 0; + + PVE::Cluster::check_cfs_quorum(); + + # test if VM exists + my $conf = PVE::QemuConfig->load_config($source_vmid); + + PVE::QemuConfig->check_lock($conf); + + raise_param_exc({ vmid => "cannot migrate HA-managed VM to remote cluster" }) + if PVE::HA::Config::vm_is_ha_managed($source_vmid); + + my $remote = PVE::JSONSchema::parse_property_string('proxmox-remote', $target_endpoint); + + # TODO: move this as helper somewhere appropriate? + my $conn_args = { + protocol => 'https', + host => $remote->{host}, + port => $remote->{port} // 8006, + apitoken => $remote->{apitoken}, + }; + + my $fp; + if ($fp = $remote->{fingerprint}) { + $conn_args->{cached_fingerprints} = { uc($fp) => 1 }; + } + + print "Establishing API connection with remote at '$remote->{host}'\n"; + + my $api_client = PVE::APIClient::LWP->new(%$conn_args); + + if (!defined($fp)) { + my $cert_info = $api_client->get("/nodes/localhost/certificates/info"); + foreach my $cert (@$cert_info) { + my $filename = $cert->{filename}; + next if $filename ne 'pveproxy-ssl.pem' && $filename ne 'pve-ssl.pem'; + $fp = $cert->{fingerprint} if !$fp || $filename eq 'pveproxy-ssl.pem'; + } + $conn_args->{cached_fingerprints} = { uc($fp) => 1 } + if defined($fp); + } + + my $repl_conf = PVE::ReplicationConfig->new(); + my $is_replicated = $repl_conf->check_for_existing_jobs($source_vmid, 1); + die "cannot remote-migrate replicated VM\n" if $is_replicated; + + if (PVE::QemuServer::check_running($source_vmid)) { + die "can't migrate running VM without --online\n" if !$param->{online}; + + } else { + warn "VM isn't running. Doing offline migration instead.\n" if $param->{online}; + $param->{online} = 0; + } + + my $storecfg = PVE::Storage::config(); + my $target_storage = extract_param($param, 'target-storage'); + my $storagemap = eval { PVE::JSONSchema::parse_idmap($target_storage, 'pve-storage-id') }; + raise_param_exc({ 'target-storage' => "failed to parse storage map: $@" }) + if $@; + + my $target_bridge = extract_param($param, 'target-bridge'); + my $bridgemap = eval { PVE::JSONSchema::parse_idmap($target_bridge, 'pve-bridge-id') }; + raise_param_exc({ 'target-bridge' => "failed to parse bridge map: $@" }) + if $@; + + die "remote migration requires explicit storage mapping!\n" + if $storagemap->{identity}; + + $param->{storagemap} = $storagemap; + $param->{bridgemap} = $bridgemap; + $param->{remote} = { + conn => $conn_args, # re-use fingerprint for tunnel + client => $api_client, + vmid => $target_vmid, + }; + $param->{migration_type} = 'websocket'; + $param->{'with-local-disks'} = 1; + $param->{delete} = $delete if $delete; + + my $cluster_status = $api_client->get("/cluster/status"); + my $target_node; + foreach my $entry (@$cluster_status) { + next if $entry->{type} ne 'node'; + if ($entry->{local}) { + $target_node = $entry->{name}; + last; + } + } + + die "couldn't determine endpoint's node name\n" + if !defined($target_node); + + my $realcmd = sub { + PVE::QemuMigrate->migrate($target_node, $remote->{host}, $source_vmid, $param); + }; + + my $worker = sub { + return PVE::GuestHelpers::guest_migration_lock($source_vmid, 10, $realcmd); + }; + + return $rpcenv->fork_worker('qmigrate', $source_vmid, $authuser, $worker); + }}); + __PACKAGE__->register_method({ name => 'monitor', path => '{vmid}/monitor', method => 'POST', protected => 1, proxyto => 'node', - description => "Execute Qemu monitor commands.", + description => "Execute QEMU monitor commands.", permissions => { description => "Sys.Modify is required for (sub)commands which are not read-only ('info *' and 'help')", check => ['perm', '/vms/{vmid}', [ 'VM.Monitor' ]], }, parameters => { - additionalProperties => 0, + additionalProperties => 0, properties => { node => get_standard_option('pve-node'), vmid => get_standard_option('pve-vmid'), @@ -3707,8 +4937,8 @@ __PACKAGE__->register_method({ check => ['perm', '/vms/{vmid}', [ 'VM.Config.Disk' ]], }, parameters => { - additionalProperties => 0, - properties => { + additionalProperties => 0, + properties => { node => get_standard_option('pve-node'), vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }), skiplock => get_standard_option('skiplock'), @@ -3730,7 +4960,10 @@ __PACKAGE__->register_method({ }, }, }, - returns => { type => 'null'}, + returns => { + type => 'string', + description => "the task ID.", + }, code => sub { my ($param) = @_; @@ -3769,9 +5002,6 @@ __PACKAGE__->register_method({ my (undef, undef, undef, undef, undef, undef, $format) = PVE::Storage::parse_volname($storecfg, $drive->{file}); - die "can't resize volume: $disk if snapshot exists\n" - if %{$conf->{snapshots}} && $format eq 'qcow2'; - my $volid = $drive->{file}; die "disk '$disk' has no associated volume\n" if !$volid; @@ -3817,8 +5047,11 @@ __PACKAGE__->register_method({ PVE::QemuConfig->write_config($vmid, $conf); }; - PVE::QemuConfig->lock_config($vmid, $updatefn); - return; + my $worker = sub { + PVE::QemuConfig->lock_config($vmid, $updatefn); + }; + + return $rpcenv->fork_worker('resize', $vmid, $authuser, $worker); }}); __PACKAGE__->register_method({ @@ -3832,7 +5065,7 @@ __PACKAGE__->register_method({ proxyto => 'node', protected => 1, # qemu pid files are only readable by root parameters => { - additionalProperties => 0, + additionalProperties => 0, properties => { vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }), node => get_standard_option('pve-node'), @@ -3977,7 +5210,7 @@ __PACKAGE__->register_method({ user => 'all', }, parameters => { - additionalProperties => 0, + additionalProperties => 0, properties => { vmid => get_standard_option('pve-vmid'), node => get_standard_option('pve-node'), @@ -4114,6 +5347,13 @@ __PACKAGE__->register_method({ node => get_standard_option('pve-node'), vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }), snapname => get_standard_option('pve-snapshot-name'), + start => { + type => 'boolean', + description => "Whether the VM should get started after rolling back successfully." + . " (Note: VMs will be automatically started if the snapshot includes RAM.)", + optional => 1, + default => 0, + }, }, }, returns => { @@ -4136,6 +5376,10 @@ __PACKAGE__->register_method({ my $realcmd = sub { PVE::Cluster::log_msg('info', $authuser, "rollback snapshot VM $vmid: $snapname"); PVE::QemuConfig->snapshot_rollback($vmid, $snapname); + + if ($param->{start} && !PVE::QemuServer::Helpers::vm_running_locally($vmid)) { + PVE::API2::Qemu->vm_start({ vmid => $vmid, node => $node }); + } }; my $worker = sub { @@ -4186,11 +5430,25 @@ __PACKAGE__->register_method({ my $snapname = extract_param($param, 'snapname'); - my $realcmd = sub { + my $lock_obtained; + my $do_delete = sub { + $lock_obtained = 1; PVE::Cluster::log_msg('info', $authuser, "delete snapshot VM $vmid: $snapname"); PVE::QemuConfig->snapshot_delete($vmid, $snapname, $param->{force}); }; + my $realcmd = sub { + if ($param->{force}) { + $do_delete->(); + } else { + eval { PVE::GuestHelpers::guest_migration_lock($vmid, 10, $do_delete); }; + if (my $err = $@) { + die $err if $lock_obtained; + die "Failed to obtain guest migration lock - replication running?\n"; + } + } + }; + return $rpcenv->fork_worker('qmdelsnapshot', $vmid, $authuser, $realcmd); }}); @@ -4219,7 +5477,10 @@ __PACKAGE__->register_method({ }, }, - returns => { type => 'null'}, + returns => { + type => 'string', + description => "the task ID.", + }, code => sub { my ($param) = @_; @@ -4233,8 +5494,7 @@ __PACKAGE__->register_method({ my $disk = extract_param($param, 'disk'); - my $updatefn = sub { - + my $load_and_check = sub { my $conf = PVE::QemuConfig->load_config($vmid); PVE::QemuConfig->check_lock($conf); @@ -4248,18 +5508,23 @@ __PACKAGE__->register_method({ die "you can't convert a VM to template if VM is running\n" if PVE::QemuServer::check_running($vmid); - my $realcmd = sub { - PVE::QemuServer::template_create($vmid, $conf, $disk); - }; + return $conf; + }; - $conf->{template} = 1; - PVE::QemuConfig->write_config($vmid, $conf); + $load_and_check->(); + + my $realcmd = sub { + PVE::QemuConfig->lock_config($vmid, sub { + my $conf = $load_and_check->(); - return $rpcenv->fork_worker('qmtemplate', $vmid, $authuser, $realcmd); + $conf->{template} = 1; + PVE::QemuConfig->write_config($vmid, $conf); + + PVE::QemuServer::template_create($vmid, $conf, $disk); + }); }; - PVE::QemuConfig->lock_config($vmid, $updatefn); - return; + return $rpcenv->fork_worker('qmtemplate', $vmid, $authuser, $realcmd); }}); __PACKAGE__->register_method({ @@ -4294,4 +5559,537 @@ __PACKAGE__->register_method({ return PVE::QemuServer::Cloudinit::dump_cloudinit_config($conf, $param->{vmid}, $param->{type}); }}); +__PACKAGE__->register_method({ + name => 'mtunnel', + path => '{vmid}/mtunnel', + method => 'POST', + protected => 1, + description => 'Migration tunnel endpoint - only for internal use by VM migration.', + permissions => { + check => + [ 'and', + ['perm', '/vms/{vmid}', [ 'VM.Allocate' ]], + ['perm', '/', [ 'Sys.Incoming' ]], + ], + description => "You need 'VM.Allocate' permissions on '/vms/{vmid}' and Sys.Incoming" . + " on '/'. Further permission checks happen during the actual migration.", + }, + parameters => { + additionalProperties => 0, + properties => { + node => get_standard_option('pve-node'), + vmid => get_standard_option('pve-vmid'), + storages => { + type => 'string', + format => 'pve-storage-id-list', + optional => 1, + description => 'List of storages to check permission and availability. Will be checked again for all actually used storages during migration.', + }, + bridges => { + type => 'string', + format => 'pve-bridge-id-list', + optional => 1, + description => 'List of network bridges to check availability. Will be checked again for actually used bridges during migration.', + }, + }, + }, + returns => { + additionalProperties => 0, + properties => { + upid => { type => 'string' }, + ticket => { type => 'string' }, + socket => { type => 'string' }, + }, + }, + code => sub { + my ($param) = @_; + + my $rpcenv = PVE::RPCEnvironment::get(); + my $authuser = $rpcenv->get_user(); + + my $node = extract_param($param, 'node'); + my $vmid = extract_param($param, 'vmid'); + + my $storages = extract_param($param, 'storages'); + my $bridges = extract_param($param, 'bridges'); + + my $nodename = PVE::INotify::nodename(); + + raise_param_exc({ node => "node needs to be 'localhost' or local hostname '$nodename'" }) + if $node ne 'localhost' && $node ne $nodename; + + $node = $nodename; + + my $storecfg = PVE::Storage::config(); + foreach my $storeid (PVE::Tools::split_list($storages)) { + $check_storage_access_migrate->($rpcenv, $authuser, $storecfg, $storeid, $node); + } + + foreach my $bridge (PVE::Tools::split_list($bridges)) { + PVE::Network::read_bridge_mtu($bridge); + } + + PVE::Cluster::check_cfs_quorum(); + + my $lock = 'create'; + eval { PVE::QemuConfig->create_and_lock_config($vmid, 0, $lock); }; + + raise_param_exc({ vmid => "unable to create empty VM config - $@"}) + if $@; + + my $realcmd = sub { + my $state = { + storecfg => PVE::Storage::config(), + lock => $lock, + vmid => $vmid, + }; + + my $run_locked = sub { + my ($code, $params) = @_; + return PVE::QemuConfig->lock_config($state->{vmid}, sub { + my $conf = PVE::QemuConfig->load_config($state->{vmid}); + + $state->{conf} = $conf; + + die "Encountered wrong lock - aborting mtunnel command handling.\n" + if $state->{lock} && !PVE::QemuConfig->has_lock($conf, $state->{lock}); + + return $code->($params); + }); + }; + + my $cmd_desc = { + config => { + conf => { + type => 'string', + description => 'Full VM config, adapted for target cluster/node', + }, + 'firewall-config' => { + type => 'string', + description => 'VM firewall config', + optional => 1, + }, + }, + disk => { + format => PVE::JSONSchema::get_standard_option('pve-qm-image-format'), + storage => { + type => 'string', + format => 'pve-storage-id', + }, + drive => { + type => 'object', + description => 'parsed drive information without volid and format', + }, + }, + start => { + start_params => { + type => 'object', + description => 'params passed to vm_start_nolock', + }, + migrate_opts => { + type => 'object', + description => 'migrate_opts passed to vm_start_nolock', + }, + }, + ticket => { + path => { + type => 'string', + description => 'socket path for which the ticket should be valid. must be known to current mtunnel instance.', + }, + }, + quit => { + cleanup => { + type => 'boolean', + description => 'remove VM config and disks, aborting migration', + default => 0, + }, + }, + 'disk-import' => $PVE::StorageTunnel::cmd_schema->{'disk-import'}, + 'query-disk-import' => $PVE::StorageTunnel::cmd_schema->{'query-disk-import'}, + bwlimit => $PVE::StorageTunnel::cmd_schema->{bwlimit}, + }; + + my $cmd_handlers = { + 'version' => sub { + # compared against other end's version + # bump/reset for breaking changes + # bump/bump for opt-in changes + return { + api => $PVE::QemuMigrate::WS_TUNNEL_VERSION, + age => 0, + }; + }, + 'config' => sub { + my ($params) = @_; + + # parse and write out VM FW config if given + if (my $fw_conf = $params->{'firewall-config'}) { + my ($path, $fh) = PVE::Tools::tempfile_contents($fw_conf, 700); + + my $empty_conf = { + rules => [], + options => {}, + aliases => {}, + ipset => {} , + ipset_comments => {}, + }; + my $cluster_fw_conf = PVE::Firewall::load_clusterfw_conf(); + + # TODO: add flag for strict parsing? + # TODO: add import sub that does all this given raw content? + my $vmfw_conf = PVE::Firewall::generic_fw_config_parser($path, $cluster_fw_conf, $empty_conf, 'vm'); + $vmfw_conf->{vmid} = $state->{vmid}; + PVE::Firewall::save_vmfw_conf($state->{vmid}, $vmfw_conf); + + $state->{cleanup}->{fw} = 1; + } + + my $conf_fn = "incoming/qemu-server/$state->{vmid}.conf"; + my $new_conf = PVE::QemuServer::parse_vm_config($conf_fn, $params->{conf}, 1); + delete $new_conf->{lock}; + delete $new_conf->{digest}; + + # TODO handle properly? + delete $new_conf->{snapshots}; + delete $new_conf->{parent}; + delete $new_conf->{pending}; + + # not handled by update_vm_api + my $vmgenid = delete $new_conf->{vmgenid}; + my $meta = delete $new_conf->{meta}; + my $cloudinit = delete $new_conf->{cloudinit}; # this is informational only + $new_conf->{skip_cloud_init} = 1; # re-use image from source side + + $new_conf->{vmid} = $state->{vmid}; + $new_conf->{node} = $node; + + PVE::QemuConfig->remove_lock($state->{vmid}, 'create'); + + eval { + $update_vm_api->($new_conf, 1); + }; + if (my $err = $@) { + # revert to locked previous config + my $conf = PVE::QemuConfig->load_config($state->{vmid}); + $conf->{lock} = 'create'; + PVE::QemuConfig->write_config($state->{vmid}, $conf); + + die $err; + } + + my $conf = PVE::QemuConfig->load_config($state->{vmid}); + $conf->{lock} = 'migrate'; + $conf->{vmgenid} = $vmgenid if defined($vmgenid); + $conf->{meta} = $meta if defined($meta); + $conf->{cloudinit} = $cloudinit if defined($cloudinit); + PVE::QemuConfig->write_config($state->{vmid}, $conf); + + $state->{lock} = 'migrate'; + + return; + }, + 'bwlimit' => sub { + my ($params) = @_; + return PVE::StorageTunnel::handle_bwlimit($params); + }, + 'disk' => sub { + my ($params) = @_; + + my $format = $params->{format}; + my $storeid = $params->{storage}; + my $drive = $params->{drive}; + + $check_storage_access_migrate->($rpcenv, $authuser, $state->{storecfg}, $storeid, $node); + + my $storagemap = { + default => $storeid, + }; + + my $source_volumes = { + 'disk' => [ + undef, + $storeid, + $drive, + 0, + $format, + ], + }; + + my $res = PVE::QemuServer::vm_migrate_alloc_nbd_disks($state->{storecfg}, $state->{vmid}, $source_volumes, $storagemap); + if (defined($res->{disk})) { + $state->{cleanup}->{volumes}->{$res->{disk}->{volid}} = 1; + return $res->{disk}; + } else { + die "failed to allocate NBD disk..\n"; + } + }, + 'disk-import' => sub { + my ($params) = @_; + + $check_storage_access_migrate->( + $rpcenv, + $authuser, + $state->{storecfg}, + $params->{storage}, + $node + ); + + $params->{unix} = "/run/qemu-server/$state->{vmid}.storage"; + + return PVE::StorageTunnel::handle_disk_import($state, $params); + }, + 'query-disk-import' => sub { + my ($params) = @_; + + return PVE::StorageTunnel::handle_query_disk_import($state, $params); + }, + 'start' => sub { + my ($params) = @_; + + my $info = PVE::QemuServer::vm_start_nolock( + $state->{storecfg}, + $state->{vmid}, + $state->{conf}, + $params->{start_params}, + $params->{migrate_opts}, + ); + + + if ($info->{migrate}->{proto} ne 'unix') { + PVE::QemuServer::vm_stop(undef, $state->{vmid}, 1, 1); + die "migration over non-UNIX sockets not possible\n"; + } + + my $socket = $info->{migrate}->{addr}; + chown $state->{socket_uid}, -1, $socket; + $state->{sockets}->{$socket} = 1; + + my $unix_sockets = $info->{migrate}->{unix_sockets}; + foreach my $socket (@$unix_sockets) { + chown $state->{socket_uid}, -1, $socket; + $state->{sockets}->{$socket} = 1; + } + return $info; + }, + 'fstrim' => sub { + if (PVE::QemuServer::qga_check_running($state->{vmid})) { + eval { mon_cmd($state->{vmid}, "guest-fstrim") }; + warn "fstrim failed: $@\n" if $@; + } + return; + }, + 'stop' => sub { + PVE::QemuServer::vm_stop(undef, $state->{vmid}, 1, 1); + return; + }, + 'nbdstop' => sub { + PVE::QemuServer::nbd_stop($state->{vmid}); + return; + }, + 'resume' => sub { + if (PVE::QemuServer::Helpers::vm_running_locally($state->{vmid})) { + PVE::QemuServer::vm_resume($state->{vmid}, 1, 1); + } else { + die "VM $state->{vmid} not running\n"; + } + return; + }, + 'unlock' => sub { + PVE::QemuConfig->remove_lock($state->{vmid}, $state->{lock}); + delete $state->{lock}; + return; + }, + 'ticket' => sub { + my ($params) = @_; + + my $path = $params->{path}; + + die "Not allowed to generate ticket for unknown socket '$path'\n" + if !defined($state->{sockets}->{$path}); + + return { ticket => PVE::AccessControl::assemble_tunnel_ticket($authuser, "/socket/$path") }; + }, + 'quit' => sub { + my ($params) = @_; + + if ($params->{cleanup}) { + if ($state->{cleanup}->{fw}) { + PVE::Firewall::remove_vmfw_conf($state->{vmid}); + } + + for my $volid (keys $state->{cleanup}->{volumes}->%*) { + print "freeing volume '$volid' as part of cleanup\n"; + eval { PVE::Storage::vdisk_free($state->{storecfg}, $volid) }; + warn $@ if $@; + } + + PVE::QemuServer::destroy_vm($state->{storecfg}, $state->{vmid}, 1); + } + + print "switching to exit-mode, waiting for client to disconnect\n"; + $state->{exit} = 1; + return; + }, + }; + + $run_locked->(sub { + my $socket_addr = "/run/qemu-server/$state->{vmid}.mtunnel"; + unlink $socket_addr; + + $state->{socket} = IO::Socket::UNIX->new( + Type => SOCK_STREAM(), + Local => $socket_addr, + Listen => 1, + ); + + $state->{socket_uid} = getpwnam('www-data') + or die "Failed to resolve user 'www-data' to numeric UID\n"; + chown $state->{socket_uid}, -1, $socket_addr; + }); + + print "mtunnel started\n"; + + my $conn = eval { PVE::Tools::run_with_timeout(300, sub { $state->{socket}->accept() }) }; + if ($@) { + warn "Failed to accept tunnel connection - $@\n"; + + warn "Removing tunnel socket..\n"; + unlink $state->{socket}; + + warn "Removing temporary VM config..\n"; + $run_locked->(sub { + PVE::QemuServer::destroy_vm($state->{storecfg}, $state->{vmid}, 1); + }); + + die "Exiting mtunnel\n"; + } + + $state->{conn} = $conn; + + my $reply_err = sub { + my ($msg) = @_; + + my $reply = JSON::encode_json({ + success => JSON::false, + msg => $msg, + }); + $conn->print("$reply\n"); + $conn->flush(); + }; + + my $reply_ok = sub { + my ($res) = @_; + + $res->{success} = JSON::true; + my $reply = JSON::encode_json($res); + $conn->print("$reply\n"); + $conn->flush(); + }; + + while (my $line = <$conn>) { + chomp $line; + + # untaint, we validate below if needed + ($line) = $line =~ /^(.*)$/; + my $parsed = eval { JSON::decode_json($line) }; + if ($@) { + $reply_err->("failed to parse command - $@"); + next; + } + + my $cmd = delete $parsed->{cmd}; + if (!defined($cmd)) { + $reply_err->("'cmd' missing"); + } elsif ($state->{exit}) { + $reply_err->("tunnel is in exit-mode, processing '$cmd' cmd not possible"); + next; + } elsif (my $handler = $cmd_handlers->{$cmd}) { + print "received command '$cmd'\n"; + eval { + if ($cmd_desc->{$cmd}) { + PVE::JSONSchema::validate($parsed, $cmd_desc->{$cmd}); + } else { + $parsed = {}; + } + my $res = $run_locked->($handler, $parsed); + $reply_ok->($res); + }; + $reply_err->("failed to handle '$cmd' command - $@") + if $@; + } else { + $reply_err->("unknown command '$cmd' given"); + } + } + + if ($state->{exit}) { + print "mtunnel exited\n"; + } else { + die "mtunnel exited unexpectedly\n"; + } + }; + + my $socket_addr = "/run/qemu-server/$vmid.mtunnel"; + my $ticket = PVE::AccessControl::assemble_tunnel_ticket($authuser, "/socket/$socket_addr"); + my $upid = $rpcenv->fork_worker('qmtunnel', $vmid, $authuser, $realcmd); + + return { + ticket => $ticket, + upid => $upid, + socket => $socket_addr, + }; + }}); + +__PACKAGE__->register_method({ + name => 'mtunnelwebsocket', + path => '{vmid}/mtunnelwebsocket', + method => 'GET', + permissions => { + description => "You need to pass a ticket valid for the selected socket. Tickets can be created via the mtunnel API call, which will check permissions accordingly.", + user => 'all', # check inside + }, + description => 'Migration tunnel endpoint for websocket upgrade - only for internal use by VM migration.', + parameters => { + additionalProperties => 0, + properties => { + node => get_standard_option('pve-node'), + vmid => get_standard_option('pve-vmid'), + socket => { + type => "string", + description => "unix socket to forward to", + }, + ticket => { + type => "string", + description => "ticket return by initial 'mtunnel' API call, or retrieved via 'ticket' tunnel command", + }, + }, + }, + returns => { + type => "object", + properties => { + port => { type => 'string', optional => 1 }, + socket => { type => 'string', optional => 1 }, + }, + }, + code => sub { + my ($param) = @_; + + my $rpcenv = PVE::RPCEnvironment::get(); + my $authuser = $rpcenv->get_user(); + + my $nodename = PVE::INotify::nodename(); + my $node = extract_param($param, 'node'); + + raise_param_exc({ node => "node needs to be 'localhost' or local hostname '$nodename'" }) + if $node ne 'localhost' && $node ne $nodename; + + my $vmid = $param->{vmid}; + # check VM exists + PVE::QemuConfig->load_config($vmid); + + my $socket = $param->{socket}; + PVE::AccessControl::verify_tunnel_ticket($param->{ticket}, $authuser, "/socket/$socket"); + + return { socket => $socket }; + }}); + 1; diff --git a/PVE/API2/Qemu/Agent.pm b/PVE/API2/Qemu/Agent.pm index 13919a4..dceee77 100644 --- a/PVE/API2/Qemu/Agent.pm +++ b/PVE/API2/Qemu/Agent.pm @@ -88,7 +88,7 @@ __PACKAGE__->register_method({ path => '', proxyto => 'node', method => 'GET', - description => "Qemu Agent command index.", + description => "QEMU Guest Agent command index.", permissions => { user => 'all', }, @@ -107,7 +107,7 @@ __PACKAGE__->register_method({ properties => {}, }, links => [ { rel => 'child', href => '{name}' } ], - description => "Returns the list of Qemu Agent commands", + description => "Returns the list of QEMU Guest Agent commands", }, code => sub { my ($param) = @_; @@ -150,7 +150,8 @@ sub register_command { properties => { node => get_standard_option('pve-node'), vmid => get_standard_option('pve-vmid', { - completion => \&PVE::QemuServer::complete_vmid_running }), + completion => \&PVE::QemuServer::complete_vmid_running, + }), command => { type => 'string', description => "The QGA command.", @@ -159,7 +160,7 @@ sub register_command { }, }; - my $description = "Execute Qemu Guest Agent commands."; + my $description = "Execute QEMU Guest Agent commands."; my $name = 'agent'; if ($command ne '') { @@ -273,10 +274,12 @@ __PACKAGE__->register_method({ vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid_running }), command => { - type => 'string', - format => 'string-alist', - description => 'The command as a list of program + arguments', - optional => 1, + type => 'array', + description => 'The command as a list of program + arguments.', + items => { + format => 'string', + description => 'A single part of the program + arguments.', + } }, 'input-data' => { type => 'string', @@ -299,10 +302,7 @@ __PACKAGE__->register_method({ my ($param) = @_; my $vmid = $param->{vmid}; - my $cmd = undef; - if ($param->{command}) { - $cmd = [PVE::Tools::split_list($param->{command})]; - } + my $cmd = $param->{command}; my $res = PVE::QemuServer::Agent::qemu_exec($vmid, $param->{'input-data'}, $cmd); return $res; @@ -470,9 +470,16 @@ __PACKAGE__->register_method({ }, content => { type => 'string', - maxLength => 60*1024, # 60k, smaller than our 64k POST limit + maxLength => 60 * 1024, # 60k, smaller than our 64k POST limit description => "The content to write into the file." - } + }, + encode => { + type => 'boolean', + description => "If set, the content will be encoded as base64 (required by QEMU)." + ."Otherwise the content needs to be encoded beforehand - defaults to true.", + optional => 1, + default => 1, + }, }, }, returns => { type => 'null' }, @@ -480,7 +487,8 @@ __PACKAGE__->register_method({ my ($param) = @_; my $vmid = $param->{vmid}; - my $buf = encode_base64($param->{content}); + + my $buf = ($param->{encode} // 1) ? encode_base64($param->{content}) : $param->{content}; my $qgafh = agent_cmd($vmid, "file-open", { path => $param->{file}, mode => 'wb' }, "can't open file"); my $write = agent_cmd($vmid, "file-write", { handle => $qgafh, 'buf-b64' => $buf }, "can't write to file"); diff --git a/PVE/API2/Qemu/Machine.pm b/PVE/API2/Qemu/Machine.pm new file mode 100644 index 0000000..afb535c --- /dev/null +++ b/PVE/API2/Qemu/Machine.pm @@ -0,0 +1,61 @@ +package PVE::API2::Qemu::Machine; + +use strict; +use warnings; + +use JSON; + +use PVE::JSONSchema qw(get_standard_option); +use PVE::RESTHandler; +use PVE::Tools qw(file_get_contents); + +use base qw(PVE::RESTHandler); + +__PACKAGE__->register_method({ + name => 'types', + path => '', + method => 'GET', + proxyto => 'node', + description => "Get available QEMU/KVM machine types.", + permissions => { + user => 'all', + }, + parameters => { + additionalProperties => 0, + properties => { + node => get_standard_option('pve-node'), + }, + }, + returns => { + type => 'array', + items => { + type => 'object', + additionalProperties => 1, + properties => { + id => { + type => 'string', + description => "Full name of machine type and version.", + }, + type => { + type => 'string', + enum => ['q35', 'i440fx'], + description => "The machine type.", + }, + version => { + type => 'string', + description => "The machine version.", + }, + }, + }, + }, + code => sub { + my $machines = eval { + my $raw = file_get_contents('/usr/share/kvm/machine-versions-x86_64.json'); + return from_json($raw, { utf8 => 1 }); + }; + die "could not load supported machine versions - $@\n" if $@; + return $machines; + } +}); + +1; diff --git a/PVE/API2/Qemu/Makefile b/PVE/API2/Qemu/Makefile index f4b7be6..bdd4762 100644 --- a/PVE/API2/Qemu/Makefile +++ b/PVE/API2/Qemu/Makefile @@ -1,4 +1,4 @@ -SOURCES=Agent.pm CPU.pm +SOURCES=Agent.pm CPU.pm Machine.pm OVF.pm .PHONY: install install: diff --git a/PVE/API2/Qemu/OVF.pm b/PVE/API2/Qemu/OVF.pm new file mode 100644 index 0000000..cc0ef2d --- /dev/null +++ b/PVE/API2/Qemu/OVF.pm @@ -0,0 +1,53 @@ +package PVE::API2::Qemu::OVF; + +use strict; +use warnings; + +use PVE::JSONSchema qw(get_standard_option); +use PVE::QemuServer::OVF; +use PVE::RESTHandler; + +use base qw(PVE::RESTHandler); + +__PACKAGE__->register_method ({ + name => 'readovf', + path => '', + method => 'GET', + proxyto => 'node', + description => "Read an .ovf manifest.", + protected => 1, + parameters => { + additionalProperties => 0, + properties => { + node => get_standard_option('pve-node'), + manifest => { + description => "Path to .ovf manifest.", + type => 'string', + }, + }, + }, + returns => { + type => 'object', + additionalProperties => 1, + properties => PVE::QemuServer::json_ovf_properties(), + description => "VM config according to .ovf manifest.", + }, + code => sub { + my ($param) = @_; + + my $manifest = $param->{manifest}; + die "check for file $manifest failed - $!\n" if !-f $manifest; + + my $parsed = PVE::QemuServer::OVF::parse_ovf($manifest); + my $result; + $result->{cores} = $parsed->{qm}->{cores}; + $result->{name} = $parsed->{qm}->{name}; + $result->{memory} = $parsed->{qm}->{memory}; + my $disks = $parsed->{disks}; + for my $disk (@$disks) { + $result->{$disk->{disk_address}} = $disk->{backing_file}; + } + return $result; + }}); + +1; diff --git a/PVE/CLI/qm.pm b/PVE/CLI/qm.pm index b3b9251..b105830 100755 --- a/PVE/CLI/qm.pm +++ b/PVE/CLI/qm.pm @@ -15,6 +15,7 @@ use POSIX qw(strftime); use Term::ReadLine; use URI::Escape; +use PVE::APIClient::LWP; use PVE::Cluster; use PVE::Exception qw(raise_param_exc); use PVE::GuestHelpers; @@ -23,7 +24,7 @@ use PVE::JSONSchema qw(get_standard_option); use PVE::Network; use PVE::RPCEnvironment; use PVE::SafeSyslog; -use PVE::Tools qw(extract_param); +use PVE::Tools qw(extract_param file_get_contents); use PVE::API2::Qemu::Agent; use PVE::API2::Qemu; @@ -42,10 +43,11 @@ use base qw(PVE::CLIHandler); my $upid_exit = sub { my $upid = shift; my $status = PVE::Tools::upid_read_status($upid); - exit($status eq 'OK' ? 0 : -1); + exit(PVE::Tools::upid_status_is_error($status) ? -1 : 0); }; my $nodename = PVE::INotify::nodename(); +my %node = (node => $nodename); sub setup_environment { PVE::RPCEnvironment->setup_default_cli_env(); @@ -100,17 +102,17 @@ sub print_recursive_hash { if (defined($key)) { print "$prefix$key:\n"; } - foreach my $itemkey (keys %$hash) { + for my $itemkey (sort keys %$hash) { print_recursive_hash("\t$prefix", $hash->{$itemkey}, $itemkey); } } elsif (ref($hash) eq 'ARRAY') { if (defined($key)) { print "$prefix$key:\n"; } - foreach my $item (@$hash) { + for my $item (@$hash) { print_recursive_hash("\t$prefix", $item); } - } elsif (!ref($hash) && defined($hash)) { + } elsif ((!ref($hash) && defined($hash)) || ref($hash) eq 'JSON::PP::Boolean') { if (defined($key)) { print "$prefix$key: $hash\n"; } else { @@ -158,6 +160,117 @@ __PACKAGE__->register_method ({ return; }}); + +__PACKAGE__->register_method({ + name => 'remote_migrate_vm', + path => 'remote_migrate_vm', + method => 'POST', + description => "Migrate virtual machine to a remote cluster. Creates a new migration task. EXPERIMENTAL feature!", + permissions => { + check => ['perm', '/vms/{vmid}', [ 'VM.Migrate' ]], + }, + parameters => { + additionalProperties => 0, + properties => { + node => get_standard_option('pve-node'), + vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }), + 'target-vmid' => get_standard_option('pve-vmid', { optional => 1 }), + 'target-endpoint' => get_standard_option('proxmox-remote', { + description => "Remote target endpoint", + }), + online => { + type => 'boolean', + description => "Use online/live migration if VM is running. Ignored if VM is stopped.", + optional => 1, + }, + delete => { + type => 'boolean', + description => "Delete the original VM and related data after successful migration. By default the original VM is kept on the source cluster in a stopped state.", + optional => 1, + default => 0, + }, + 'target-storage' => get_standard_option('pve-targetstorage', { + completion => \&PVE::QemuServer::complete_migration_storage, + optional => 0, + }), + 'target-bridge' => { + type => 'string', + description => "Mapping from source to target bridges. Providing only a single bridge ID maps all source bridges to that bridge. Providing the special value '1' will map each source bridge to itself.", + format => 'bridge-pair-list', + }, + bwlimit => { + description => "Override I/O bandwidth limit (in KiB/s).", + optional => 1, + type => 'integer', + minimum => '0', + default => 'migrate limit from datacenter or storage config', + }, + }, + }, + returns => { + type => 'string', + description => "the task ID.", + }, + code => sub { + my ($param) = @_; + + my $rpcenv = PVE::RPCEnvironment::get(); + my $authuser = $rpcenv->get_user(); + + my $source_vmid = $param->{vmid}; + my $target_endpoint = $param->{'target-endpoint'}; + my $target_vmid = $param->{'target-vmid'} // $source_vmid; + + my $remote = PVE::JSONSchema::parse_property_string('proxmox-remote', $target_endpoint); + + # TODO: move this as helper somewhere appropriate? + my $conn_args = { + protocol => 'https', + host => $remote->{host}, + port => $remote->{port} // 8006, + apitoken => $remote->{apitoken}, + }; + + $conn_args->{cached_fingerprints} = { uc($remote->{fingerprint}) => 1 } + if defined($remote->{fingerprint}); + + my $api_client = PVE::APIClient::LWP->new(%$conn_args); + my $resources = $api_client->get("/cluster/resources", { type => 'vm' }); + if (grep { defined($_->{vmid}) && $_->{vmid} eq $target_vmid } @$resources) { + raise_param_exc({ target_vmid => "Guest with ID '$target_vmid' already exists on remote cluster" }); + } + + my $storages = $api_client->get("/nodes/localhost/storage", { enabled => 1 }); + + my $storecfg = PVE::Storage::config(); + my $target_storage = $param->{'target-storage'}; + my $storagemap = eval { PVE::JSONSchema::parse_idmap($target_storage, 'pve-storage-id') }; + raise_param_exc({ 'target-storage' => "failed to parse storage map: $@" }) + if $@; + + my $check_remote_storage = sub { + my ($storage) = @_; + my $found = [ grep { $_->{storage} eq $storage } @$storages ]; + die "remote: storage '$storage' does not exist (or missing permission)!\n" + if !@$found; + + $found = @$found[0]; + + my $content_types = [ PVE::Tools::split_list($found->{content}) ]; + die "remote: storage '$storage' cannot store images\n" + if !grep { $_ eq 'images' } @$content_types; + }; + + foreach my $target_sid (values %{$storagemap->{entries}}) { + $check_remote_storage->($target_sid); + } + + $check_remote_storage->($storagemap->{default}) + if $storagemap->{default}; + + return PVE::API2::Qemu->remote_migrate_vm($param); + }}); + __PACKAGE__->register_method ({ name => 'status', path => 'status', @@ -217,12 +330,10 @@ __PACKAGE__->register_method ({ my $vnc_socket = PVE::QemuServer::Helpers::vnc_socket($vmid); if (my $ticket = $ENV{LC_PVE_TICKET}) { # NOTE: ssh on debian only pass LC_* variables - mon_cmd($vmid, "change", device => 'vnc', target => "unix:$vnc_socket,password"); mon_cmd($vmid, "set_password", protocol => 'vnc', password => $ticket); mon_cmd($vmid, "expire_password", protocol => 'vnc', time => "+30"); } else { - # FIXME: remove or allow to add tls-creds object, as x509 vnc param is removed with qemu 4?? - mon_cmd($vmid, "change", device => 'vnc', target => "unix:$vnc_socket,password"); + die "LC_PVE_TICKET not set, VNC proxy without password is forbidden\n"; } run_vnc_proxy($vnc_socket); @@ -315,6 +426,8 @@ __PACKAGE__->register_method ({ last; } elsif ($line =~ /^resume (\d+)$/) { my $vmid = $1; + # check_running and vm_resume with nocheck, since local node + # might not have processed config move/rename yet if (PVE::QemuServer::check_running($vmid, 1)) { eval { PVE::QemuServer::vm_resume($vmid, 1, 1); }; if ($@) { @@ -375,7 +488,7 @@ __PACKAGE__->register_method ({ name => 'monitor', path => 'monitor', method => 'POST', - description => "Enter Qemu Monitor interface.", + description => "Enter QEMU Monitor interface.", parameters => { additionalProperties => 0, properties => { @@ -390,7 +503,7 @@ __PACKAGE__->register_method ({ my $conf = PVE::QemuConfig->load_config ($vmid); # check if VM exists - print "Entering Qemu Monitor for VM $vmid - type 'help' for help\n"; + print "Entering QEMU Monitor for VM $vmid - type 'help' for help\n"; my $term = Term::ReadLine->new('qm'); @@ -634,6 +747,7 @@ __PACKAGE__->register_method ({ $conf->{memory} = $parsed->{qm}->{memory} if defined($parsed->{qm}->{memory}); $conf->{cores} = $parsed->{qm}->{cores} if defined($parsed->{qm}->{cores}); + my $imported_disks = []; eval { # order matters, as do_import() will load_config() internally $conf->{vmgenid} = PVE::QemuServer::generate_uuid(); @@ -642,11 +756,13 @@ __PACKAGE__->register_method ({ foreach my $disk (@{ $parsed->{disks} }) { my ($file, $drive) = ($disk->{backing_file}, $disk->{disk_address}); - PVE::QemuServer::ImportDisk::do_import($file, $vmid, $storeid, { + my ($name, $volid) = PVE::QemuServer::ImportDisk::do_import($file, $vmid, $storeid, { drive_name => $drive, format => $format, skiplock => 1, }); + # for cleanup on (later) error + push @$imported_disks, $volid; } # reload after disks entries have been created @@ -656,10 +772,13 @@ __PACKAGE__->register_method ({ PVE::QemuConfig->write_config($vmid, $conf); }; - my $err = $@; - if ($err) { + if (my $err = $@) { my $skiplock = 1; - # eval for additional safety in error path + warn "error during import, cleaning up created resources...\n"; + for my $volid (@$imported_disks) { + eval { PVE::Storage::vdisk_free($storecfg, $volid) }; + warn "cleanup of $volid failed: $@\n" if $@; + } eval { PVE::QemuServer::destroy_vm($storecfg, $vmid, $skiplock) }; warn "Could not destroy VM $vmid: $@" if "$@"; die "import failed - $err"; @@ -796,7 +915,8 @@ __PACKAGE__->register_method({ my $storecfg = PVE::Storage::config(); warn "Starting cleanup for $vmid\n"; - PVE::QemuConfig->lock_config($vmid, sub { + # mdev cleanup can take a while, so wait up to 60 seconds + PVE::QemuConfig->lock_config_full($vmid, 60, sub { my $conf = PVE::QemuConfig->load_config ($vmid); my $pid = PVE::QemuServer::check_running ($vmid); die "vm still running\n" if $pid; @@ -805,7 +925,7 @@ __PACKAGE__->register_method({ # we have to cleanup the tap devices after a crash foreach my $opt (keys %$conf) { - next if $opt !~ m/^net(\d)+$/; + next if $opt !~ m/^net(\d+)$/; my $interface = $1; PVE::Network::tap_unplug("tap${vmid}i${interface}"); } @@ -827,13 +947,109 @@ __PACKAGE__->register_method({ warn "Restarting VM $vmid\n"; PVE::API2::Qemu->vm_start({ vmid => $vmid, - node => $nodename, + %node, }); } return; }}); +__PACKAGE__->register_method({ + name => 'vm_import', + path => 'vm-import', + description => "Import a foreign virtual guest from a supported import source, such as an ESXi storage.", + parameters => { + additionalProperties => 0, + properties => PVE::QemuServer::json_config_properties({ + vmid => get_standard_option('pve-vmid', { completion => \&PVE::Cluster::complete_next_vmid }), + 'source' => { + type => 'string', + description => 'The import source volume id.', + }, + storage => get_standard_option('pve-storage-id', { + description => "Default storage.", + completion => \&PVE::QemuServer::complete_storage, + }), + 'live-import' => { + type => 'boolean', + optional => 1, + default => 0, + description => "Immediately start the VM and copy the data in the background.", + }, + 'dryrun' => { + type => 'boolean', + optional => 1, + default => 0, + description => "Show the create command and exit without doing anything.", + }, + delete => { + type => 'string', format => 'pve-configid-list', + description => "A list of settings you want to delete.", + optional => 1, + }, + format => { + type => 'string', + description => 'Target format', + enum => [ 'raw', 'qcow2', 'vmdk' ], + optional => 1, + }, + }), + }, + returns => { type => 'null' }, + code => sub { + my ($param) = @_; + + my ($vmid, $source, $storage, $format, $live_import, $dryrun, $delete) = + delete $param->@{qw(vmid source storage format live-import dryrun delete)}; + + if (defined($format)) { + $format = ",format=$format"; + } else { + $format = ''; + } + + my $storecfg = PVE::Storage::config(); + my $metadata = PVE::Storage::get_import_metadata($storecfg, $source); + + my $create_args = $metadata->{'create-args'}; + if (my $netdevs = $metadata->{net}) { + for my $net (keys $netdevs->%*) { + my $value = $netdevs->{$net}; + $create_args->{$net} = join(',', map { $_ . '=' . $value->{$_} } sort keys %$value); + } + } + if (my $disks = $metadata->{disks}) { + if (delete $disks->{efidisk0}) { + $create_args->{efidisk0} = "$storage:1$format,efitype=4m"; + } + for my $disk (keys $disks->%*) { + my $value = $disks->{$disk}->{volid}; + $create_args->{$disk} = "$storage:0${format},import-from=$value"; + } + } + + $create_args->{'live-restore'} = 1 if $live_import; + + $create_args->{$_} = $param->{$_} for keys $param->%*; + delete $create_args->{$_} for PVE::Tools::split_list($delete); + + if ($dryrun) { + print("# dry-run – the resulting create command for the import would be:\n"); + print("qm create $vmid \\\n "); + print(join(" \\\n ", map { "--$_ $create_args->{$_}" } sort keys $create_args->%*)); + print("\n"); + return; + } + + PVE::API2::Qemu->create_vm({ + %node, + vmid => $vmid, + %$create_args, + }); + return; + } +}); + my $print_agent_result = sub { my ($data) = @_; @@ -853,14 +1069,14 @@ my $print_agent_result = sub { return; } - print to_json($result, { pretty => 1, canonical => 1}); + print to_json($result, { pretty => 1, canonical => 1, utf8 => 1}); }; sub param_mapping { my ($name) = @_; my $ssh_key_map = ['sshkeys', sub { - return URI::Escape::uri_escape(PVE::Tools::file_get_contents($_[0])); + return URI::Escape::uri_escape(file_get_contents($_[0])); }]; my $cipassword_map = PVE::CLIHandler::get_standard_mapping('pve-password', { name => 'cipassword' }); my $password_map = PVE::CLIHandler::get_standard_mapping('pve-password'); @@ -874,85 +1090,66 @@ sub param_mapping { } our $cmddef = { - list => [ "PVE::API2::Qemu", 'vmlist', [], - { node => $nodename }, sub { - my $vmlist = shift; - - exit 0 if (!scalar(@$vmlist)); - - printf "%10s %-20s %-10s %-10s %12s %-10s\n", - qw(VMID NAME STATUS MEM(MB) BOOTDISK(GB) PID); - - foreach my $rec (sort { $a->{vmid} <=> $b->{vmid} } @$vmlist) { - printf "%10s %-20s %-10s %-10s %12.2f %-10s\n", $rec->{vmid}, $rec->{name}, - $rec->{qmpstatus} || $rec->{status}, - ($rec->{maxmem} || 0)/(1024*1024), - ($rec->{maxdisk} || 0)/(1024*1024*1024), - $rec->{pid}||0; - } - - - } ], - - create => [ "PVE::API2::Qemu", 'create_vm', ['vmid'], { node => $nodename }, $upid_exit ], - - destroy => [ "PVE::API2::Qemu", 'destroy_vm', ['vmid'], { node => $nodename }, $upid_exit ], - - clone => [ "PVE::API2::Qemu", 'clone_vm', ['vmid', 'newid'], { node => $nodename }, $upid_exit ], - - migrate => [ "PVE::API2::Qemu", 'migrate_vm', ['vmid', 'target'], { node => $nodename }, $upid_exit ], - - set => [ "PVE::API2::Qemu", 'update_vm', ['vmid'], { node => $nodename } ], + list=> [ "PVE::API2::Qemu", 'vmlist', [], { %node }, sub { + my $vmlist = shift; + exit 0 if (!scalar(@$vmlist)); + + printf "%10s %-20s %-10s %-10s %12s %-10s\n", + qw(VMID NAME STATUS MEM(MB) BOOTDISK(GB) PID); + + foreach my $rec (sort { $a->{vmid} <=> $b->{vmid} } @$vmlist) { + printf "%10s %-20s %-10s %-10s %12.2f %-10s\n", $rec->{vmid}, $rec->{name}, + $rec->{qmpstatus} || $rec->{status}, + ($rec->{maxmem} || 0)/(1024*1024), + ($rec->{maxdisk} || 0)/(1024*1024*1024), + $rec->{pid} || 0; + } + }], - resize => [ "PVE::API2::Qemu", 'resize_vm', ['vmid', 'disk', 'size'], { node => $nodename } ], + create => [ "PVE::API2::Qemu", 'create_vm', ['vmid'], { %node }, $upid_exit ], + destroy => [ "PVE::API2::Qemu", 'destroy_vm', ['vmid'], { %node }, $upid_exit ], + clone => [ "PVE::API2::Qemu", 'clone_vm', ['vmid', 'newid'], { %node }, $upid_exit ], - move_disk => [ "PVE::API2::Qemu", 'move_vm_disk', ['vmid', 'disk', 'storage'], { node => $nodename }, $upid_exit ], + migrate => [ "PVE::API2::Qemu", 'migrate_vm', ['vmid', 'target'], { %node }, $upid_exit ], + 'remote-migrate' => [ __PACKAGE__, 'remote_migrate_vm', ['vmid', 'target-vmid', 'target-endpoint'], { %node }, $upid_exit ], - unlink => [ "PVE::API2::Qemu", 'unlink', ['vmid'], { node => $nodename } ], + set => [ "PVE::API2::Qemu", 'update_vm', ['vmid'], { %node } ], - config => [ "PVE::API2::Qemu", 'vm_config', ['vmid'], - { node => $nodename }, sub { - my $config = shift; - foreach my $k (sort (keys %$config)) { - next if $k eq 'digest'; - my $v = $config->{$k}; - if ($k eq 'description') { - $v = PVE::Tools::encode_text($v); - } - print "$k: $v\n"; - } - }], + config => [ "PVE::API2::Qemu", 'vm_config', ['vmid'], { %node }, sub { + my $config = shift; + foreach my $k (sort (keys %$config)) { + next if $k eq 'digest'; + my $v = $config->{$k}; + if ($k eq 'description') { + $v = PVE::Tools::encode_text($v); + } + print "$k: $v\n"; + } + }], - pending => [ "PVE::API2::Qemu", 'vm_pending', ['vmid'], { node => $nodename }, \&PVE::GuestHelpers::format_pending ], + pending => [ "PVE::API2::Qemu", 'vm_pending', ['vmid'], { %node }, \&PVE::GuestHelpers::format_pending ], showcmd => [ __PACKAGE__, 'showcmd', ['vmid']], status => [ __PACKAGE__, 'status', ['vmid']], - snapshot => [ "PVE::API2::Qemu", 'snapshot', ['vmid', 'snapname'], { node => $nodename } , $upid_exit ], - - delsnapshot => [ "PVE::API2::Qemu", 'delsnapshot', ['vmid', 'snapname'], { node => $nodename } , $upid_exit ], - - listsnapshot => [ "PVE::API2::Qemu", 'snapshot_list', ['vmid'], { node => $nodename }, \&PVE::GuestHelpers::print_snapshot_tree], - - rollback => [ "PVE::API2::Qemu", 'rollback', ['vmid', 'snapname'], { node => $nodename } , $upid_exit ], - - template => [ "PVE::API2::Qemu", 'template', ['vmid'], { node => $nodename }], + # FIXME: for 8.0 move to command group snapshot { create, list, destroy, rollback } + snapshot => [ "PVE::API2::Qemu", 'snapshot', ['vmid', 'snapname'], { %node } , $upid_exit ], + delsnapshot => [ "PVE::API2::Qemu", 'delsnapshot', ['vmid', 'snapname'], { %node } , $upid_exit ], + listsnapshot => [ "PVE::API2::Qemu", 'snapshot_list', ['vmid'], { %node }, \&PVE::GuestHelpers::print_snapshot_tree], + rollback => [ "PVE::API2::Qemu", 'rollback', ['vmid', 'snapname'], { %node } , $upid_exit ], - start => [ "PVE::API2::Qemu", 'vm_start', ['vmid'], { node => $nodename } , $upid_exit ], + template => [ "PVE::API2::Qemu", 'template', ['vmid'], { %node }], - stop => [ "PVE::API2::Qemu", 'vm_stop', ['vmid'], { node => $nodename }, $upid_exit ], + # FIXME: should be in a power command group? + start => [ "PVE::API2::Qemu", 'vm_start', ['vmid'], { %node } , $upid_exit ], + stop => [ "PVE::API2::Qemu", 'vm_stop', ['vmid'], { %node }, $upid_exit ], + reset => [ "PVE::API2::Qemu", 'vm_reset', ['vmid'], { %node }, $upid_exit ], + shutdown => [ "PVE::API2::Qemu", 'vm_shutdown', ['vmid'], { %node }, $upid_exit ], + reboot => [ "PVE::API2::Qemu", 'vm_reboot', ['vmid'], { %node }, $upid_exit ], + suspend => [ "PVE::API2::Qemu", 'vm_suspend', ['vmid'], { %node }, $upid_exit ], + resume => [ "PVE::API2::Qemu", 'vm_resume', ['vmid'], { %node }, $upid_exit ], - reset => [ "PVE::API2::Qemu", 'vm_reset', ['vmid'], { node => $nodename }, $upid_exit ], - - shutdown => [ "PVE::API2::Qemu", 'vm_shutdown', ['vmid'], { node => $nodename }, $upid_exit ], - - reboot => [ "PVE::API2::Qemu", 'vm_reboot', ['vmid'], { node => $nodename }, $upid_exit ], - - suspend => [ "PVE::API2::Qemu", 'vm_suspend', ['vmid'], { node => $nodename }, $upid_exit ], - - resume => [ "PVE::API2::Qemu", 'vm_resume', ['vmid'], { node => $nodename }, $upid_exit ], - - sendkey => [ "PVE::API2::Qemu", 'vm_sendkey', ['vmid', 'key'], { node => $nodename } ], + sendkey => [ "PVE::API2::Qemu", 'vm_sendkey', ['vmid', 'key'], { %node } ], vncproxy => [ __PACKAGE__, 'vncproxy', ['vmid']], @@ -960,17 +1157,31 @@ our $cmddef = { unlock => [ __PACKAGE__, 'unlock', ['vmid']], - rescan => [ __PACKAGE__, 'rescan', []], + # TODO: evluate dropping below aliases for 8.0, if no usage is left + importdisk => { alias => 'disk import' }, + 'move-disk' => { alias => 'disk move' }, + move_disk => { alias => 'disk move' }, + rescan => { alias => 'disk rescan' }, + resize => { alias => 'disk resize' }, + unlink => { alias => 'disk unlink' }, + + disk => { + import => [ __PACKAGE__, 'importdisk', ['vmid', 'source', 'storage']], + 'move' => [ "PVE::API2::Qemu", 'move_vm_disk', ['vmid', 'disk', 'storage'], { %node }, $upid_exit ], + rescan => [ __PACKAGE__, 'rescan', []], + resize => [ "PVE::API2::Qemu", 'resize_vm', ['vmid', 'disk', 'size'], { %node } ], + unlink => [ "PVE::API2::Qemu", 'unlink', ['vmid'], { %node } ], + }, monitor => [ __PACKAGE__, 'monitor', ['vmid']], - agent => { alias => 'guest cmd' }, + agent => { alias => 'guest cmd' }, # FIXME: remove with PVE 8.0 guest => { - cmd => [ "PVE::API2::Qemu::Agent", 'agent', ['vmid', 'command'], { node => $nodename }, $print_agent_result ], - passwd => [ "PVE::API2::Qemu::Agent", 'set-user-password', [ 'vmid', 'username' ], { node => $nodename }], - exec => [ __PACKAGE__, 'exec', [ 'vmid', 'extra-args' ], { node => $nodename }, $print_agent_result], - 'exec-status' => [ "PVE::API2::Qemu::Agent", 'exec-status', [ 'vmid', 'pid' ], { node => $nodename }, $print_agent_result], + cmd => [ "PVE::API2::Qemu::Agent", 'agent', ['vmid', 'command'], { %node }, $print_agent_result ], + passwd => [ "PVE::API2::Qemu::Agent", 'set-user-password', [ 'vmid', 'username' ], { %node }], + exec => [ __PACKAGE__, 'exec', [ 'vmid', 'extra-args' ], { %node }, $print_agent_result], + 'exec-status' => [ "PVE::API2::Qemu::Agent", 'exec-status', [ 'vmid', 'pid' ], { %node }, $print_agent_result], }, mtunnel => [ __PACKAGE__, 'mtunnel', []], @@ -979,19 +1190,17 @@ our $cmddef = { terminal => [ __PACKAGE__, 'terminal', ['vmid']], - importdisk => [ __PACKAGE__, 'importdisk', ['vmid', 'source', 'storage']], - importovf => [ __PACKAGE__, 'importovf', ['vmid', 'manifest', 'storage']], - cleanup => [ __PACKAGE__, 'cleanup', ['vmid', 'clean-shutdown', 'guest-requested'], { node => $nodename }], + cleanup => [ __PACKAGE__, 'cleanup', ['vmid', 'clean-shutdown', 'guest-requested'], { %node }], cloudinit => { - dump => [ "PVE::API2::Qemu", 'cloudinit_generated_config_dump', ['vmid', 'type'], { node => $nodename }, sub { - my $data = shift; - print "$data\n"; - }], + dump => [ "PVE::API2::Qemu", 'cloudinit_generated_config_dump', ['vmid', 'type'], { %node }, sub { print "$_[0]\n"; }], + pending => [ "PVE::API2::Qemu", 'cloudinit_pending', ['vmid'], { %node }, \&PVE::GuestHelpers::format_pending ], + update => [ "PVE::API2::Qemu", 'cloudinit_update', ['vmid'], { node => $nodename }], }, + import => [ __PACKAGE__, 'vm_import', ['vmid', 'source']], }; 1; diff --git a/PVE/CLI/qmrestore.pm b/PVE/CLI/qmrestore.pm index cb5c122..a47648b 100755 --- a/PVE/CLI/qmrestore.pm +++ b/PVE/CLI/qmrestore.pm @@ -54,11 +54,16 @@ __PACKAGE__->register_method({ description => "Add the VM to the specified pool.", }, bwlimit => { - description => "Override i/o bandwidth limit (in KiB/s).", + description => "Override I/O bandwidth limit (in KiB/s).", optional => 1, type => 'number', minimum => '0', - } + }, + 'live-restore' => { + optional => 1, + type => 'boolean', + description => "Start the VM immediately from the backup and restore in background. PBS only.", + }, }, }, returns => { @@ -76,7 +81,7 @@ our $cmddef = [ __PACKAGE__, 'qmrestore', ['archive', 'vmid'], undef, sub { my $upid = shift; my $status = PVE::Tools::upid_read_status($upid); - exit($status eq 'OK' ? 0 : -1); + exit(PVE::Tools::upid_status_is_error($status) ? -1 : 0); }]; 1; diff --git a/PVE/QMPClient.pm b/PVE/QMPClient.pm index ea4dc0b..a785d9a 100644 --- a/PVE/QMPClient.pm +++ b/PVE/QMPClient.pm @@ -4,6 +4,7 @@ use strict; use warnings; use IO::Multiplex; +use IO::Socket::UNIX; use JSON; use POSIX qw(EINTR EAGAIN); use Scalar::Util qw(weaken); @@ -12,7 +13,7 @@ use Time::HiRes qw(usleep gettimeofday tv_interval); use PVE::IPCC; use PVE::QemuServer::Helpers; -# Qemu Monitor Protocol (QMP) client. +# QEMU Monitor Protocol (QMP) client. # # This implementation uses IO::Multiplex (libio-multiplex-perl) and # allows you to issue qmp and qga commands to different VMs in parallel. @@ -113,25 +114,30 @@ sub cmd { # locked state with high probability, so use an generous timeout $timeout = 60*60; # 1 hour } elsif ($cmd->{execute} eq 'guest-fsfreeze-thaw') { - # thaw has no possible long blocking actions, either it returns - # instantly or never (dead locked) - $timeout = 10; - } elsif ($cmd->{execute} eq 'savevm-start' || - $cmd->{execute} eq 'savevm-end' || - $cmd->{execute} eq 'query-backup' || - $cmd->{execute} eq 'query-block-jobs' || - $cmd->{execute} eq 'block-job-cancel' || - $cmd->{execute} eq 'block-job-complete' || - $cmd->{execute} eq 'backup-cancel' || - $cmd->{execute} eq 'query-savevm' || - $cmd->{execute} eq 'delete-drive-snapshot' || - $cmd->{execute} eq 'guest-shutdown' || - $cmd->{execute} eq 'blockdev-snapshot-internal-sync' || - $cmd->{execute} eq 'blockdev-snapshot-delete-internal-sync' || - $cmd->{execute} eq 'snapshot-drive' ) { - $timeout = 10*60; # 10 mins ? + # While it should return instantly or never (dead locked) for Linux guests, + # the variance for Windows guests can be big. And there might be hook scripts + # that are executed upon thaw, so use 3 minutes to be on the safe side. + $timeout = 3 * 60; + } elsif ( + $cmd->{execute} eq 'backup-cancel' || + $cmd->{execute} eq 'blockdev-snapshot-delete-internal-sync' || + $cmd->{execute} eq 'blockdev-snapshot-internal-sync' || + $cmd->{execute} eq 'block-job-cancel' || + $cmd->{execute} eq 'block-job-complete' || + $cmd->{execute} eq 'drive-mirror' || + $cmd->{execute} eq 'guest-fstrim' || + $cmd->{execute} eq 'guest-shutdown' || + $cmd->{execute} eq 'query-backup' || + $cmd->{execute} eq 'query-block-jobs' || + $cmd->{execute} eq 'query-savevm' || + $cmd->{execute} eq 'savevm-end' || + $cmd->{execute} eq 'savevm-start' + ) { + $timeout = 10*60; # 10 mins } else { - $timeout = 3; # default + # NOTE: if you came here as user and want to change this, try using IO-Threads first + # which move out quite some processing of the main thread, leaving more time for QMP + $timeout = 5; # default } } diff --git a/PVE/QemuConfig.pm b/PVE/QemuConfig.pm index 3f4605f..8e8a782 100644 --- a/PVE/QemuConfig.pm +++ b/PVE/QemuConfig.pm @@ -12,8 +12,10 @@ use PVE::QemuServer::Helpers; use PVE::QemuServer::Monitor qw(mon_cmd); use PVE::QemuServer; use PVE::QemuServer::Machine; +use PVE::QemuServer::Memory qw(get_current_memory); use PVE::Storage; use PVE::Tools; +use PVE::Format qw(render_bytes render_duration); use base qw(PVE::AbstractConfig); @@ -207,8 +209,7 @@ sub __snapshot_save_vmstate { $target = PVE::QemuServer::find_vmstate_storage($conf, $storecfg); } - my $defaults = PVE::QemuServer::load_defaults(); - my $mem_size = $conf->{memory} // $defaults->{memory}; + my $mem_size = get_current_memory($conf->{memory}); my $driver_state_size = 500; # assume 500MB is enough to safe all driver state; # our savevm-start does live-save of the memory until the space left in the # volume is just enough for the remaining memory content + internal state @@ -240,6 +241,25 @@ sub __snapshot_save_vmstate { return $statefile; } +sub __snapshot_activate_storages { + my ($class, $conf, $include_vmstate) = @_; + + my $storecfg = PVE::Storage::config(); + my $opts = $include_vmstate ? { 'extra_keys' => ['vmstate'] } : {}; + my $storage_hash = {}; + + $class->foreach_volume_full($conf, $opts, sub { + my ($key, $drive) = @_; + + return if PVE::QemuServer::drive_is_cdrom($drive); + + my ($storeid) = PVE::Storage::parse_volume_id($drive->{file}); + $storage_hash->{$storeid} = 1; + }); + + PVE::Storage::activate_storage_list($storecfg, [ sort keys $storage_hash->%* ]); +} + sub __snapshot_check_running { my ($class, $vmid) = @_; return PVE::QemuServer::Helpers::vm_running_locally($vmid); @@ -278,19 +298,40 @@ sub __snapshot_create_vol_snapshots_hook { if ($snap->{vmstate}) { my $path = PVE::Storage::path($storecfg, $snap->{vmstate}); PVE::Storage::activate_volumes($storecfg, [$snap->{vmstate}]); + my $state_storage_id = PVE::Storage::parse_volume_id($snap->{vmstate}); + PVE::QemuServer::set_migration_caps($vmid, 1); mon_cmd($vmid, "savevm-start", statefile => $path); + print "saving VM state and RAM using storage '$state_storage_id'\n"; + my $render_state = sub { + my ($stat) = @_; + my $b = render_bytes($stat->{bytes}); + my $t = render_duration($stat->{'total-time'} / 1000); + return ($b, $t); + }; + my $round = 0; for(;;) { + $round++; my $stat = mon_cmd($vmid, "query-savevm"); if (!$stat->{status}) { die "savevm not active\n"; } elsif ($stat->{status} eq 'active') { + if ($round < 60 || $round % 10 == 0) { + my ($b, $t) = $render_state->($stat); + print "$b in $t\n"; + } + print "reducing reporting rate to every 10s\n" if $round == 60; sleep(1); next; } elsif ($stat->{status} eq 'completed') { + my ($b, $t) = $render_state->($stat); + print "completed saving the VM state in $t, saved $b\n"; last; + } elsif ($stat->{status} eq 'failed') { + my $err = $stat->{error} || 'unknown error'; + die "unable to save VM state and RAM - $err\n"; } else { - die "query-savevm returned status '$stat->{status}'\n"; + die "query-savevm returned unexpected status '$stat->{status}'\n"; } } } else { @@ -327,6 +368,8 @@ sub __snapshot_create_vol_snapshot { my $device = "drive-$ds"; my $storecfg = PVE::Storage::config(); + print "snapshotting '$device' ($drive->{file})\n"; + PVE::QemuServer::qemu_volume_snapshot($vmid, $device, $storecfg, $volid, $snapname); } @@ -363,9 +406,8 @@ sub __snapshot_delete_vol_snapshot { return if PVE::QemuServer::drive_is_cdrom($drive); my $storecfg = PVE::Storage::config(); my $volid = $drive->{file}; - my $device = "drive-$ds"; - PVE::QemuServer::qemu_volume_snapshot_delete($vmid, $device, $storecfg, $volid, $snapname); + PVE::QemuServer::qemu_volume_snapshot_delete($vmid, $storecfg, $volid, $snapname); push @$unused, $volid; } @@ -390,7 +432,8 @@ sub __snapshot_rollback_hook { } else { # Note: old code did not store 'machine', so we try to be smart # and guess the snapshot was generated with kvm 1.4 (pc-i440fx-1.4). - $data->{forcemachine} = $conf->{machine} || 'pc-i440fx-1.4'; + my $machine_conf = PVE::QemuServer::Machine::parse_machine($conf->{machine}); + $data->{forcemachine} = $machine_conf->{type} || 'pc-i440fx-1.4'; # we remove the 'machine' configuration if not explicitly specified # in the original config. @@ -409,14 +452,14 @@ sub __snapshot_rollback_hook { } sub __snapshot_rollback_vol_possible { - my ($class, $drive, $snapname) = @_; + my ($class, $drive, $snapname, $blockers) = @_; return if PVE::QemuServer::drive_is_cdrom($drive); my $storecfg = PVE::Storage::config(); my $volid = $drive->{file}; - PVE::Storage::volume_rollback_is_possible($storecfg, $volid, $snapname); + PVE::Storage::volume_rollback_is_possible($storecfg, $volid, $snapname, $blockers); } sub __snapshot_rollback_vol_rollback { @@ -455,7 +498,7 @@ sub __snapshot_rollback_get_unused { $class->foreach_volume($conf, sub { my ($vs, $volume) = @_; - return if PVE::QemuServer::drive_is_cdrom($volume); + return if PVE::QemuServer::drive_is_cdrom($volume, 1); my $found = 0; my $volid = $volume->{file}; @@ -464,7 +507,7 @@ sub __snapshot_rollback_get_unused { my ($ds, $drive) = @_; return if $found; - return if PVE::QemuServer::drive_is_cdrom($drive); + return if PVE::QemuServer::drive_is_cdrom($drive, 1); $found = 1 if ($drive->{file} && $drive->{file} eq $volid); @@ -476,6 +519,58 @@ sub __snapshot_rollback_get_unused { return $unused; } +sub add_unused_volume { + my ($class, $config, $volid) = @_; + + if ($volid =~ m/vm-\d+-cloudinit/) { + print "found unused cloudinit disk '$volid', removing it\n"; + my $storecfg = PVE::Storage::config(); + PVE::Storage::vdisk_free($storecfg, $volid); + return undef; + } else { + return $class->SUPER::add_unused_volume($config, $volid); + } +} + +sub load_current_config { + my ($class, $vmid, $current) = @_; + + my $conf = $class->SUPER::load_current_config($vmid, $current); + delete $conf->{cloudinit}; + return $conf; +} + +sub get_derived_property { + my ($class, $conf, $name) = @_; + + my $defaults = PVE::QemuServer::load_defaults(); + + if ($name eq 'max-cpu') { + my $cpus = + ($conf->{sockets} || $defaults->{sockets}) * ($conf->{cores} || $defaults->{cores}); + return $conf->{vcpus} || $cpus; + } elsif ($name eq 'max-memory') { # current usage maximum, not maximum hotpluggable + return get_current_memory($conf->{memory}) * 1024 * 1024; + } else { + die "unknown derived property - $name\n"; + } +} + # END implemented abstract methods from PVE::AbstractConfig +sub has_cloudinit { + my ($class, $conf, $skip) = @_; + + my $found; + + $class->foreach_volume($conf, sub { + my ($key, $volume) = @_; + + return if ($skip && $skip eq $key) || $found; + $found = $key if PVE::QemuServer::Drive::drive_is_cloudinit($volume); + }); + + return $found; +} + 1; diff --git a/PVE/QemuMigrate.pm b/PVE/QemuMigrate.pm index 2f4eec3..8d9b35a 100644 --- a/PVE/QemuMigrate.pm +++ b/PVE/QemuMigrate.pm @@ -5,17 +5,20 @@ use warnings; use IO::File; use IPC::Open2; -use POSIX qw( WNOHANG ); use Time::HiRes qw( usleep ); use PVE::Cluster; +use PVE::Format qw(render_bytes); +use PVE::GuestHelpers qw(safe_boolean_ne safe_string_ne); use PVE::INotify; use PVE::RPCEnvironment; use PVE::Replication; use PVE::ReplicationConfig; use PVE::ReplicationState; use PVE::Storage; +use PVE::StorageTunnel; use PVE::Tools; +use PVE::Tunnel; use PVE::QemuConfig; use PVE::QemuServer::CPUConfig; @@ -23,185 +26,122 @@ use PVE::QemuServer::Drive; use PVE::QemuServer::Helpers qw(min_version); use PVE::QemuServer::Machine; use PVE::QemuServer::Monitor qw(mon_cmd); +use PVE::QemuServer::Memory qw(get_current_memory); use PVE::QemuServer; use PVE::AbstractMigrate; use base qw(PVE::AbstractMigrate); -sub fork_command_pipe { - my ($self, $cmd) = @_; +# compared against remote end's minimum version +our $WS_TUNNEL_VERSION = 2; - my $reader = IO::File->new(); - my $writer = IO::File->new(); - - my $orig_pid = $$; - - my $cpid; - - eval { $cpid = open2($reader, $writer, @$cmd); }; - - my $err = $@; - - # catch exec errors - if ($orig_pid != $$) { - $self->log('err', "can't fork command pipe\n"); - POSIX::_exit(1); - kill('KILL', $$); - } +sub fork_tunnel { + my ($self, $ssh_forward_info) = @_; - die $err if $err; + my $cmd = ['/usr/sbin/qm', 'mtunnel']; + my $log = sub { + my ($level, $msg) = @_; + $self->log($level, $msg); + }; - return { writer => $writer, reader => $reader, pid => $cpid }; + return PVE::Tunnel::fork_ssh_tunnel($self->{rem_ssh}, $cmd, $ssh_forward_info, $log); } -sub finish_command_pipe { - my ($self, $cmdpipe, $timeout) = @_; - - my $cpid = $cmdpipe->{pid}; - return if !defined($cpid); +sub fork_websocket_tunnel { + my ($self, $storages, $bridges) = @_; - my $writer = $cmdpipe->{writer}; - my $reader = $cmdpipe->{reader}; + my $remote = $self->{opts}->{remote}; + my $conn = $remote->{conn}; - $writer->close(); - $reader->close(); - - my $collect_child_process = sub { - my $res = waitpid($cpid, WNOHANG); - if (defined($res) && ($res == $cpid)) { - delete $cmdpipe->{cpid}; - return 1; - } else { - return 0; - } - }; - - if ($timeout) { - for (my $i = 0; $i < $timeout; $i++) { - return if &$collect_child_process(); - sleep(1); - } - } - - $self->log('info', "ssh tunnel still running - terminating now with SIGTERM\n"); - kill(15, $cpid); - - # wait again - for (my $i = 0; $i < 10; $i++) { - return if &$collect_child_process(); - sleep(1); - } - - $self->log('info', "ssh tunnel still running - terminating now with SIGKILL\n"); - kill 9, $cpid; - sleep 1; - - $self->log('err', "ssh tunnel child process (PID $cpid) couldn't be collected\n") - if !&$collect_child_process(); -} + my $log = sub { + my ($level, $msg) = @_; + $self->log($level, $msg); + }; -sub read_tunnel { - my ($self, $tunnel, $timeout) = @_; + my $websocket_url = "https://$conn->{host}:$conn->{port}/api2/json/nodes/$self->{node}/qemu/$remote->{vmid}/mtunnelwebsocket"; + my $url = "/nodes/$self->{node}/qemu/$remote->{vmid}/mtunnel"; - $timeout = 60 if !defined($timeout); + my $tunnel_params = { + url => $websocket_url, + }; - my $reader = $tunnel->{reader}; + my $storage_list = join(',', keys %$storages); + my $bridge_list = join(',', keys %$bridges); - my $output; - eval { - PVE::Tools::run_with_timeout($timeout, sub { $output = <$reader>; }); + my $req_params = { + storages => $storage_list, + bridges => $bridge_list, }; - die "reading from tunnel failed: $@\n" if $@; - chomp $output; - - return $output; + return PVE::Tunnel::fork_websocket_tunnel($conn, $url, $req_params, $tunnel_params, $log); } -sub write_tunnel { - my ($self, $tunnel, $timeout, $command) = @_; +# tunnel_info: +# proto: unix (secure) or tcp (insecure/legacy compat) +# addr: IP or UNIX socket path +# port: optional TCP port +# unix_sockets: additional UNIX socket paths to forward +sub start_remote_tunnel { + my ($self, $tunnel_info) = @_; - $timeout = 60 if !defined($timeout); + my $nodename = PVE::INotify::nodename(); + my $migration_type = $self->{opts}->{migration_type}; - my $writer = $tunnel->{writer}; + if ($migration_type eq 'secure') { - eval { - PVE::Tools::run_with_timeout($timeout, sub { - print $writer "$command\n"; - $writer->flush(); - }); - }; - die "writing to tunnel failed: $@\n" if $@; + if ($tunnel_info->{proto} eq 'unix') { + my $ssh_forward_info = []; - if ($tunnel->{version} && $tunnel->{version} >= 1) { - my $res = eval { $self->read_tunnel($tunnel, 10); }; - die "no reply to command '$command': $@\n" if $@; + my $unix_sockets = [ keys %{$tunnel_info->{unix_sockets}} ]; + push @$unix_sockets, $tunnel_info->{addr}; + for my $sock (@$unix_sockets) { + push @$ssh_forward_info, "$sock:$sock"; + unlink $sock; + } - if ($res eq 'OK') { - return; - } else { - die "tunnel replied '$res' to command '$command'\n"; - } - } -} + $self->{tunnel} = $self->fork_tunnel($ssh_forward_info); -sub fork_tunnel { - my ($self, $ssh_forward_info) = @_; + my $unix_socket_try = 0; # wait for the socket to become ready + while ($unix_socket_try <= 100) { + $unix_socket_try++; + my $available = 0; + foreach my $sock (@$unix_sockets) { + if (-S $sock) { + $available++; + } + } - my @localtunnelinfo = (); - foreach my $addr (@$ssh_forward_info) { - push @localtunnelinfo, '-L', $addr; - } + if ($available == @$unix_sockets) { + last; + } - my $cmd = [@{$self->{rem_ssh}}, '-o ExitOnForwardFailure=yes', @localtunnelinfo, '/usr/sbin/qm', 'mtunnel' ]; + usleep(50000); + } + if ($unix_socket_try > 100) { + $self->{errors} = 1; + PVE::Tunnel::finish_tunnel($self->{tunnel}); + die "Timeout, migration socket $tunnel_info->{addr} did not get ready"; + } + $self->{tunnel}->{unix_sockets} = $unix_sockets if (@$unix_sockets); - my $tunnel = $self->fork_command_pipe($cmd); + } elsif ($tunnel_info->{proto} eq 'tcp') { + my $ssh_forward_info = []; + if ($tunnel_info->{addr} eq "localhost") { + # for backwards compatibility with older qemu-server versions + my $pfamily = PVE::Tools::get_host_address_family($nodename); + my $lport = PVE::Tools::next_migrate_port($pfamily); + push @$ssh_forward_info, "$lport:localhost:$tunnel_info->{port}"; + } - eval { - my $helo = $self->read_tunnel($tunnel, 60); - die "no reply\n" if !$helo; - die "no quorum on target node\n" if $helo =~ m/^no quorum$/; - die "got strange reply from mtunnel ('$helo')\n" - if $helo !~ m/^tunnel online$/; - }; - my $err = $@; + $self->{tunnel} = $self->fork_tunnel($ssh_forward_info); - eval { - my $ver = $self->read_tunnel($tunnel, 10); - if ($ver =~ /^ver (\d+)$/) { - $tunnel->{version} = $1; - $self->log('info', "ssh tunnel $ver\n"); } else { - $err = "received invalid tunnel version string '$ver'\n" if !$err; + die "unsupported protocol in migration URI: $tunnel_info->{proto}\n"; } - }; - - if ($err) { - $self->finish_command_pipe($tunnel); - die "can't open migration tunnel - $err"; - } - return $tunnel; -} - -sub finish_tunnel { - my ($self, $tunnel) = @_; - - eval { $self->write_tunnel($tunnel, 30, 'quit'); }; - my $err = $@; - - $self->finish_command_pipe($tunnel, 30); - - if (my $unix_sockets = $tunnel->{unix_sockets}) { - # ssh does not clean up on local host - my $cmd = ['rm', '-f', @$unix_sockets]; - PVE::Tools::run_command($cmd); - - # .. and just to be sure check on remote side - unshift @{$cmd}, @{$self->{rem_ssh}}; - PVE::Tools::run_command($cmd); + } else { + #fork tunnel for insecure migration, to send faster commands like resume + $self->{tunnel} = $self->fork_tunnel(); } - - die $err if $err; } sub lock_vm { @@ -210,16 +150,51 @@ sub lock_vm { return PVE::QemuConfig->lock_config($vmid, $code, @param); } +sub target_storage_check_available { + my ($self, $storecfg, $targetsid, $volid) = @_; + + if (!$self->{opts}->{remote}) { + # check if storage is available on target node + my $target_scfg = PVE::Storage::storage_check_enabled( + $storecfg, + $targetsid, + $self->{node}, + ); + my ($vtype) = PVE::Storage::parse_volname($storecfg, $volid); + die "$volid: content type '$vtype' is not available on storage '$targetsid'\n" + if !$target_scfg->{content}->{$vtype}; + } +} + sub prepare { my ($self, $vmid) = @_; my $online = $self->{opts}->{online}; - $self->{storecfg} = PVE::Storage::config(); + my $storecfg = $self->{storecfg} = PVE::Storage::config(); # test if VM exists my $conf = $self->{vmconf} = PVE::QemuConfig->load_config($vmid); + my $version = PVE::QemuServer::Helpers::get_node_pvecfg_version($self->{node}); + my $cloudinit_config = $conf->{cloudinit}; + + if ( + PVE::QemuConfig->has_cloudinit($conf) && defined($cloudinit_config) + && scalar(keys %$cloudinit_config) > 0 + && !PVE::QemuServer::Helpers::pvecfg_min_version($version, 7, 2, 13) + ) { + die "target node is too old (manager <= 7.2-13) and doesn't support new cloudinit section\n"; + } + + my $repl_conf = PVE::ReplicationConfig->new(); + $self->{replication_jobcfg} = $repl_conf->find_local_replication_job($vmid, $self->{node}); + $self->{is_replicated} = $repl_conf->check_for_existing_jobs($vmid, 1); + + if ($self->{replication_jobcfg} && defined($self->{replication_jobcfg}->{remove_job})) { + die "refusing to migrate replicated VM whose replication job is marked for removal\n"; + } + PVE::QemuConfig->check_lock($conf); my $running = 0; @@ -227,6 +202,16 @@ sub prepare { die "can't migrate running VM without --online\n" if !$online; $running = $pid; + if ($self->{is_replicated} && !$self->{replication_jobcfg}) { + if ($self->{opts}->{force}) { + $self->log('warn', "WARNING: Node '$self->{node}' is not a replication target. Existing " . + "replication jobs will fail after migration!\n"); + } else { + die "Cannot live-migrate replicated VM to node '$self->{node}' - not a replication " . + "target. Use 'force' to override.\n"; + } + } + $self->{forcemachine} = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf); # To support custom CPU types, we keep QEMU's "-cpu" parameter intact. @@ -238,70 +223,113 @@ sub prepare { $self->{forcecpu} = PVE::QemuServer::CPUConfig::get_cpu_from_running_vm($pid); } } + + # Do not treat a suspended VM as paused, as it might wake up + # during migration and remain paused after migration finishes. + $self->{vm_was_paused} = 1 if PVE::QemuServer::vm_is_paused($vmid, 0); } - my $loc_res = PVE::QemuServer::check_local_resources($conf, 1); - if (scalar @$loc_res) { + my ($loc_res, $mapped_res, $missing_mappings_by_node) = PVE::QemuServer::check_local_resources($conf, 1); + my $blocking_resources = []; + for my $res ($loc_res->@*) { + if (!grep($res, $mapped_res->@*)) { + push $blocking_resources->@*, $res; + } + } + if (scalar($blocking_resources->@*)) { if ($self->{running} || !$self->{opts}->{force}) { - die "can't migrate VM which uses local devices: " . join(", ", @$loc_res) . "\n"; + die "can't migrate VM which uses local devices: " . join(", ", $blocking_resources->@*) . "\n"; } else { $self->log('info', "migrating VM which uses local devices"); } } + if (scalar($mapped_res->@*)) { + my $missing_mappings = $missing_mappings_by_node->{$self->{node}}; + if ($running) { + die "can't migrate running VM which uses mapped devices: " . join(", ", $mapped_res->@*) . "\n"; + } elsif (scalar($missing_mappings->@*)) { + die "can't migrate to '$self->{node}': missing mapped devices " . join(", ", $missing_mappings->@*) . "\n"; + } else { + $self->log('info', "migrating VM which uses mapped local devices"); + } + } + + my $vga = PVE::QemuServer::parse_vga($conf->{vga}); + if ($running && $vga->{'clipboard'} && $vga->{'clipboard'} eq 'vnc') { + die "VMs with 'clipboard' set to 'vnc' are not live migratable!\n"; + } + my $vollist = PVE::QemuServer::get_vm_volumes($conf); - my $need_activate = []; + my $storages = {}; foreach my $volid (@$vollist) { my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1); - # check if storage is available on both nodes - my $targetsid = PVE::QemuServer::map_storage($self->{opts}->{storagemap}, $sid); + # check if storage is available on source node + my $scfg = PVE::Storage::storage_check_enabled($storecfg, $sid); + + my $targetsid = $sid; + # NOTE: local ignores shared mappings, remote maps them + if (!$scfg->{shared} || $self->{opts}->{remote}) { + $targetsid = PVE::JSONSchema::map_id($self->{opts}->{storagemap}, $sid); + } + + $storages->{$targetsid} = 1; - my $scfg = PVE::Storage::storage_check_node($self->{storecfg}, $sid); - PVE::Storage::storage_check_node($self->{storecfg}, $targetsid, $self->{node}); + $self->target_storage_check_available($storecfg, $targetsid, $volid); if ($scfg->{shared}) { # PVE::Storage::activate_storage checks this for non-shared storages my $plugin = PVE::Storage::Plugin->lookup($scfg->{type}); warn "Used shared storage '$sid' is not online on source node!\n" if !$plugin->check_connection($sid, $scfg); - } else { - # only activate if not shared - next if ($volid =~ m/vm-\d+-cloudinit/); - push @$need_activate, $volid; } } - # activate volumes - PVE::Storage::activate_volumes($self->{storecfg}, $need_activate); - - # test ssh connection - my $cmd = [ @{$self->{rem_ssh}}, '/bin/true' ]; - eval { $self->cmd_quiet($cmd); }; - die "Can't connect to destination address using public key\n" if $@; + if ($self->{opts}->{remote}) { + # test & establish websocket connection + my $bridges = map_bridges($conf, $self->{opts}->{bridgemap}, 1); + my $tunnel = $self->fork_websocket_tunnel($storages, $bridges); + my $min_version = $tunnel->{version} - $tunnel->{age}; + $self->log('info', "local WS tunnel version: $WS_TUNNEL_VERSION"); + $self->log('info', "remote WS tunnel version: $tunnel->{version}"); + $self->log('info', "minimum required WS tunnel version: $min_version"); + die "Remote tunnel endpoint not compatible, upgrade required\n" + if $WS_TUNNEL_VERSION < $min_version; + die "Remote tunnel endpoint too old, upgrade required\n" + if $WS_TUNNEL_VERSION > $tunnel->{version}; + + print "websocket tunnel started\n"; + $self->{tunnel} = $tunnel; + } else { + # test ssh connection + my $cmd = [ @{$self->{rem_ssh}}, '/bin/true' ]; + eval { $self->cmd_quiet($cmd); }; + die "Can't connect to destination address using public key\n" if $@; + } return $running; } -sub sync_disks { +sub scan_local_volumes { my ($self, $vmid) = @_; my $conf = $self->{vmconf}; # local volumes which have been copied # and their old_id => new_id pairs - $self->{volumes} = []; $self->{volume_map} = {}; + $self->{local_volumes} = {}; my $storecfg = $self->{storecfg}; eval { - # found local volumes and their origin - my $local_volumes = {}; + my $local_volumes = $self->{local_volumes}; my $local_volumes_errors = {}; my $other_errors = []; my $abort = 0; + my $path_to_volid = {}; my $log_error = sub { my ($msg, $volid) = @_; @@ -314,45 +342,11 @@ sub sync_disks { $abort = 1; }; - my @sids = PVE::Storage::storage_ids($storecfg); - foreach my $storeid (@sids) { - my $scfg = PVE::Storage::storage_config($storecfg, $storeid); - next if $scfg->{shared}; - next if !PVE::Storage::storage_check_enabled($storecfg, $storeid, undef, 1); - - # get list from PVE::Storage (for unused volumes) - my $dl = PVE::Storage::vdisk_list($storecfg, $storeid, $vmid); - - next if @{$dl->{$storeid}} == 0; - - my $targetsid = PVE::QemuServer::map_storage($self->{opts}->{storagemap}, $storeid); - # check if storage is available on target node - PVE::Storage::storage_check_node($storecfg, $targetsid, $self->{node}); - - # grandfather in existing mismatches - if ($targetsid ne $storeid) { - my $target_scfg = PVE::Storage::storage_config($storecfg, $targetsid); - die "content type 'images' is not available on storage '$targetsid'\n" - if !$target_scfg->{content}->{images}; - } - - PVE::Storage::foreach_volid($dl, sub { - my ($volid, $sid, $volinfo) = @_; - - $local_volumes->{$volid}->{ref} = 'storage'; - - # If with_snapshots is not set for storage migrate, it tries to use - # a raw+size stream, but on-the-fly conversion from qcow2 to raw+size - # back to qcow2 is currently not possible. - $local_volumes->{$volid}->{snapshots} = ($volinfo->{format} =~ /^(?:qcow2|vmdk)$/); - $local_volumes->{$volid}->{format} = $volinfo->{format}; - }); - } - - my $rep_cfg = PVE::ReplicationConfig->new(); - my $replication_jobcfg = $rep_cfg->find_local_replication_job($vmid, $self->{node}); - my $replicatable_volumes = !$replication_jobcfg ? {} + my $replicatable_volumes = !$self->{replication_jobcfg} ? {} : PVE::QemuConfig->get_replicatable_volumes($storecfg, $vmid, $conf, 0, 1); + foreach my $volid (keys %{$replicatable_volumes}) { + $local_volumes->{$volid}->{replicated} = 1; + } my $test_volid = sub { my ($volid, $attr) = @_; @@ -368,7 +362,7 @@ sub sync_disks { if ($attr->{cdrom}) { if ($volid eq 'cdrom') { my $msg = "can't migrate local cdrom drive"; - if (defined($snaprefs) && !$attr->{referenced_in_config}) { + if (defined($snaprefs) && !$attr->{is_attached}) { my $snapnames = join(', ', sort keys %$snaprefs); $msg .= " (referenced in snapshot - $snapnames)"; } @@ -380,18 +374,39 @@ sub sync_disks { my ($sid, $volname) = PVE::Storage::parse_volume_id($volid); - my $targetsid = PVE::QemuServer::map_storage($self->{opts}->{storagemap}, $sid); # check if storage is available on both nodes - my $scfg = PVE::Storage::storage_check_node($storecfg, $sid); - PVE::Storage::storage_check_node($storecfg, $targetsid, $self->{node}); + my $scfg = PVE::Storage::storage_check_enabled($storecfg, $sid); + + my $targetsid = $sid; + # NOTE: local ignores shared mappings, remote maps them + if (!$scfg->{shared} || $self->{opts}->{remote}) { + $targetsid = PVE::JSONSchema::map_id($self->{opts}->{storagemap}, $sid); + } + + $self->target_storage_check_available($storecfg, $targetsid, $volid); + return if $scfg->{shared} && !$self->{opts}->{remote}; + + $local_volumes->{$volid}->{ref} = 'pending' if $attr->{referenced_in_pending}; + $local_volumes->{$volid}->{ref} = 'snapshot' if $attr->{referenced_in_snapshot}; + $local_volumes->{$volid}->{ref} = 'unused' if $attr->{is_unused}; + $local_volumes->{$volid}->{ref} = 'attached' if $attr->{is_attached}; + $local_volumes->{$volid}->{ref} = 'generated' if $attr->{is_tpmstate}; - return if $scfg->{shared}; + $local_volumes->{$volid}->{bwlimit} = $self->get_bwlimit($sid, $targetsid); + $local_volumes->{$volid}->{targetsid} = $targetsid; - $local_volumes->{$volid}->{ref} = $attr->{referenced_in_config} ? 'config' : 'snapshot'; - $local_volumes->{$volid}->{ref} = 'storage' if $attr->{is_unused}; + $local_volumes->{$volid}->@{qw(size format)} = PVE::Storage::volume_size_info($storecfg, $volid); $local_volumes->{$volid}->{is_vmstate} = $attr->{is_vmstate} ? 1 : 0; + $local_volumes->{$volid}->{drivename} = $attr->{drivename} + if $attr->{drivename}; + + # If with_snapshots is not set for storage migrate, it tries to use + # a raw+size stream, but on-the-fly conversion from qcow2 to raw+size + # back to qcow2 is currently not possible. + $local_volumes->{$volid}->{snapshots} = ($local_volumes->{$volid}->{format} =~ /^(?:qcow2|vmdk)$/); + if ($attr->{cdrom}) { if ($volid =~ /vm-\d+-cloudinit/) { $local_volumes->{$volid}->{ref} = 'generated'; @@ -405,6 +420,8 @@ sub sync_disks { die "owned by other VM (owner = VM $owner)\n" if !$owner || ($owner != $vmid); + $path_to_volid->{$path}->{$volid} = 1; + return if $attr->{is_vmstate}; if (defined($snaprefs)) { @@ -413,8 +430,15 @@ sub sync_disks { # we cannot migrate shapshots on local storage # exceptions: 'zfspool' or 'qcow2' files (on directory storage) - die "online storage migration not possible if snapshot exists\n" if $self->{running}; - if (!($scfg->{type} eq 'zfspool' || $local_volumes->{$volid}->{format} eq 'qcow2')) { + die "online storage migration not possible if non-replicated snapshot exists\n" + if $self->{running} && !$local_volumes->{$volid}->{replicated}; + + die "remote migration with snapshots not supported yet\n" if $self->{opts}->{remote}; + + if (!($scfg->{type} eq 'zfspool' + || ($scfg->{type} eq 'btrfs' && $local_volumes->{$volid}->{format} eq 'raw') + || $local_volumes->{$volid}->{format} eq 'qcow2' + )) { die "non-migratable snapshot exists\n"; } } @@ -431,17 +455,25 @@ sub sync_disks { } }); + for my $path (keys %$path_to_volid) { + my @volids = keys $path_to_volid->{$path}->%*; + die "detected not supported aliased volumes: '" . join("', '", @volids) . "'\n" + if (scalar(@volids) > 1); + } + foreach my $vol (sort keys %$local_volumes) { my $type = $replicatable_volumes->{$vol} ? 'local, replicated' : 'local'; my $ref = $local_volumes->{$vol}->{ref}; - if ($ref eq 'storage') { - $self->log('info', "found $type disk '$vol' (via storage)\n"); - } elsif ($ref eq 'config') { + if ($ref eq 'attached') { &$log_error("can't live migrate attached local disks without with-local-disks option\n", $vol) if $self->{running} && !$self->{opts}->{"with-local-disks"}; - $self->log('info', "found $type disk '$vol' (in current VM config)\n"); + $self->log('info', "found $type disk '$vol' (attached)\n"); + } elsif ($ref eq 'unused') { + $self->log('info', "found $type disk '$vol' (unused)\n"); } elsif ($ref eq 'snapshot') { $self->log('info', "found $type disk '$vol' (referenced by snapshot(s))\n"); + } elsif ($ref eq 'pending') { + $self->log('info', "found $type disk '$vol' (pending change)\n"); } elsif ($ref eq 'generated') { $self->log('info', "found generated disk '$vol' (in current VM config)\n"); } else { @@ -465,7 +497,10 @@ sub sync_disks { my ($sid, $volname) = PVE::Storage::parse_volume_id($volid); my $scfg = PVE::Storage::storage_config($storecfg, $sid); - my $migratable = $scfg->{type} =~ /^(?:dir|zfspool|lvmthin|lvm)$/; + my $migratable = $scfg->{type} =~ /^(?:dir|btrfs|zfspool|lvmthin|lvm)$/; + + # TODO: what is this even here for? + $migratable = 1 if $self->{opts}->{remote}; die "can't migrate '$volid' - storage type '$scfg->{type}' not supported\n" if !$migratable; @@ -476,126 +511,207 @@ sub sync_disks { } } - if ($replication_jobcfg) { - if ($self->{running}) { + foreach my $volid (sort keys %$local_volumes) { + my $ref = $local_volumes->{$volid}->{ref}; + if ($self->{running} && $ref eq 'attached') { + $local_volumes->{$volid}->{migration_mode} = 'online'; + } elsif ($self->{running} && $ref eq 'generated') { + # offline migrate the cloud-init ISO and don't regenerate on VM start + # + # tpmstate will also be offline migrated first, and in case of + # live migration then updated by QEMU/swtpm if necessary + $local_volumes->{$volid}->{migration_mode} = 'offline'; + } else { + $local_volumes->{$volid}->{migration_mode} = 'offline'; + } + } + }; + die "Problem found while scanning volumes - $@" if $@; +} + +sub handle_replication { + my ($self, $vmid) = @_; - my $version = PVE::QemuServer::kvm_user_version(); - if (!min_version($version, 4, 2)) { - die "can't live migrate VM with replicated volumes, pve-qemu to old (< 4.2)!\n" - } + my $conf = $self->{vmconf}; + my $local_volumes = $self->{local_volumes}; - my $live_replicatable_volumes = {}; - PVE::QemuConfig->foreach_volume($conf, sub { - my ($ds, $drive) = @_; + return if !$self->{replication_jobcfg}; - my $volid = $drive->{file}; - $live_replicatable_volumes->{$ds} = $volid - if defined($replicatable_volumes->{$volid}); - }); - foreach my $drive (keys %$live_replicatable_volumes) { - my $volid = $live_replicatable_volumes->{$drive}; + die "can't migrate VM with replicated volumes to remote cluster/node\n" + if $self->{opts}->{remote}; - my $bitmap = "repl_$drive"; + if ($self->{running}) { - # start tracking before replication to get full delta + a few duplicates - $self->log('info', "$drive: start tracking writes using block-dirty-bitmap '$bitmap'"); - mon_cmd($vmid, 'block-dirty-bitmap-add', node => "drive-$drive", name => $bitmap); + my $version = PVE::QemuServer::kvm_user_version(); + if (!min_version($version, 4, 2)) { + die "can't live migrate VM with replicated volumes, pve-qemu to old (< 4.2)!\n" + } - # other info comes from target node in phase 2 - $self->{target_drive}->{$drive}->{bitmap} = $bitmap; - } - } - $self->log('info', "replicating disk images"); + my @live_replicatable_volumes = $self->filter_local_volumes('online', 1); + foreach my $volid (@live_replicatable_volumes) { + my $drive = $local_volumes->{$volid}->{drivename}; + die "internal error - no drive for '$volid'\n" if !defined($drive); + + my $bitmap = "repl_$drive"; - my $start_time = time(); - my $logfunc = sub { $self->log('info', shift) }; - $self->{replicated_volumes} = PVE::Replication::run_replication( - 'PVE::QemuConfig', $replication_jobcfg, $start_time, $start_time, $logfunc); + # start tracking before replication to get full delta + a few duplicates + $self->log('info', "$drive: start tracking writes using block-dirty-bitmap '$bitmap'"); + mon_cmd($vmid, 'block-dirty-bitmap-add', node => "drive-$drive", name => $bitmap); + + # other info comes from target node in phase 2 + $self->{target_drive}->{$drive}->{bitmap} = $bitmap; } + } + $self->log('info', "replicating disk images"); + + my $start_time = time(); + my $logfunc = sub { $self->log('info', shift) }; + my $actual_replicated_volumes = PVE::Replication::run_replication( + 'PVE::QemuConfig', $self->{replication_jobcfg}, $start_time, $start_time, $logfunc); + + # extra safety check + my @replicated_volumes = $self->filter_local_volumes(undef, 1); + foreach my $volid (@replicated_volumes) { + die "expected volume '$volid' to get replicated, but it wasn't\n" + if !$actual_replicated_volumes->{$volid}; + } +} - # sizes in config have to be accurate for remote node to correctly - # allocate disks, rescan to be sure - my $volid_hash = PVE::QemuServer::scan_volids($storecfg, $vmid); - PVE::QemuConfig->foreach_volume($conf, sub { - my ($key, $drive) = @_; - return if $key eq 'efidisk0'; # skip efidisk, will be handled later - - my $volid = $drive->{file}; - return if !defined($local_volumes->{$volid}); # only update sizes for local volumes - return if !defined($volid_hash->{$volid}); - - my ($updated, $msg) = PVE::QemuServer::Drive::update_disksize($drive, $volid_hash->{$volid}->{size}); - if (defined($updated)) { - $conf->{$key} = PVE::QemuServer::print_drive($updated); - $self->log('info', "drive '$key': $msg"); - } - }); +sub config_update_local_disksizes { + my ($self) = @_; - # we want to set the efidisk size in the config to the size of the - # real OVMF_VARS.fd image, else we can create a too big image, which does not work - if (defined($conf->{efidisk0})) { - PVE::QemuServer::update_efidisk_size($conf); + my $conf = $self->{vmconf}; + my $local_volumes = $self->{local_volumes}; + + PVE::QemuConfig->foreach_volume($conf, sub { + my ($key, $drive) = @_; + # skip special disks, will be handled later + return if $key eq 'efidisk0'; + return if $key eq 'tpmstate0'; + + my $volid = $drive->{file}; + return if !defined($local_volumes->{$volid}); # only update sizes for local volumes + + my ($updated, $msg) = PVE::QemuServer::Drive::update_disksize($drive, $local_volumes->{$volid}->{size}); + if (defined($updated)) { + $conf->{$key} = PVE::QemuServer::print_drive($updated); + $self->log('info', "drive '$key': $msg"); } + }); - $self->log('info', "copying local disk images") if scalar(%$local_volumes); + # we want to set the efidisk size in the config to the size of the + # real OVMF_VARS.fd image, else we can create a too big image, which does not work + if (defined($conf->{efidisk0})) { + PVE::QemuServer::update_efidisk_size($conf); + } - foreach my $volid (keys %$local_volumes) { - my ($sid, $volname) = PVE::Storage::parse_volume_id($volid); - my $targetsid = PVE::QemuServer::map_storage($self->{opts}->{storagemap}, $sid); - my $ref = $local_volumes->{$volid}->{ref}; - if ($self->{running} && $ref eq 'config') { - push @{$self->{online_local_volumes}}, $volid; - } elsif ($ref eq 'generated') { - die "can't live migrate VM with local cloudinit disk. use a shared storage instead\n" if $self->{running}; - # skip all generated volumes but queue them for deletion in phase3_cleanup - push @{$self->{volumes}}, $volid; - next; - } else { - next if $self->{replicated_volumes}->{$volid}; - push @{$self->{volumes}}, $volid; - my $opts = $self->{opts}; - # use 'migrate' limit for transfer to other node - my $bwlimit = PVE::Storage::get_bandwidth_limit('migration', [$targetsid, $sid], $opts->{bwlimit}); - # JSONSchema and get_bandwidth_limit use kbps - storage_migrate bps - $bwlimit = $bwlimit * 1024 if defined($bwlimit); - - my $storage_migrate_opts = { - 'ratelimit_bps' => $bwlimit, - 'insecure' => $opts->{migration_type} eq 'insecure', - 'with_snapshots' => $local_volumes->{$volid}->{snapshots}, - 'allow_rename' => !$local_volumes->{$volid}->{is_vmstate}, - }; + # TPM state might have an irregular filesize, to avoid problems on transfer + # we always assume the static size of 4M to allocate on the target + if (defined($conf->{tpmstate0})) { + PVE::QemuServer::update_tpmstate_size($conf); + } +} - my $logfunc = sub { $self->log('info', $_[0]); }; - my $new_volid = eval { - PVE::Storage::storage_migrate($storecfg, $volid, $self->{ssh_info}, - $targetsid, $storage_migrate_opts, $logfunc); - }; - if (my $err = $@) { - die "storage migration for '$volid' to storage '$targetsid' failed - $err\n"; - } +sub filter_local_volumes { + my ($self, $migration_mode, $replicated) = @_; + + my $volumes = $self->{local_volumes}; + my @filtered_volids; + + foreach my $volid (sort keys %{$volumes}) { + next if defined($migration_mode) && safe_string_ne($volumes->{$volid}->{migration_mode}, $migration_mode); + next if defined($replicated) && safe_boolean_ne($volumes->{$volid}->{replicated}, $replicated); + push @filtered_volids, $volid; + } + + return @filtered_volids; +} - $self->{volume_map}->{$volid} = $new_volid; - $self->log('info', "volume '$volid' is '$new_volid' on the target\n"); +sub sync_offline_local_volumes { + my ($self) = @_; + + my $local_volumes = $self->{local_volumes}; + my @volids = $self->filter_local_volumes('offline', 0); + + my $storecfg = $self->{storecfg}; + my $opts = $self->{opts}; + + $self->log('info', "copying local disk images") if scalar(@volids); + + foreach my $volid (@volids) { + my $new_volid; + + my $opts = $self->{opts}; + if ($opts->{remote}) { + my $log = sub { + my ($level, $msg) = @_; + $self->log($level, $msg); + }; + + $new_volid = PVE::StorageTunnel::storage_migrate( + $self->{tunnel}, + $storecfg, + $volid, + $self->{vmid}, + $opts->{remote}->{vmid}, + $local_volumes->{$volid}, + $log, + ); + } else { + my $targetsid = $local_volumes->{$volid}->{targetsid}; + + my $bwlimit = $local_volumes->{$volid}->{bwlimit}; + $bwlimit = $bwlimit * 1024 if defined($bwlimit); # storage_migrate uses bps + + my $storage_migrate_opts = { + 'ratelimit_bps' => $bwlimit, + 'insecure' => $opts->{migration_type} eq 'insecure', + 'with_snapshots' => $local_volumes->{$volid}->{snapshots}, + 'allow_rename' => !$local_volumes->{$volid}->{is_vmstate}, + }; + + my $logfunc = sub { $self->log('info', $_[0]); }; + $new_volid = eval { + PVE::Storage::storage_migrate( + $storecfg, + $volid, + $self->{ssh_info}, + $targetsid, + $storage_migrate_opts, + $logfunc, + ); + }; + if (my $err = $@) { + die "storage migration for '$volid' to storage '$targetsid' failed - $err\n"; } } - }; - die "Failed to sync data - $@" if $@; + + $self->{volume_map}->{$volid} = $new_volid; + $self->log('info', "volume '$volid' is '$new_volid' on the target\n"); + + eval { PVE::Storage::deactivate_volumes($storecfg, [$volid]); }; + if (my $err = $@) { + $self->log('warn', $err); + } + } } sub cleanup_remotedisks { my ($self) = @_; - foreach my $target_drive (keys %{$self->{target_drive}}) { - my $drivestr = $self->{target_drive}->{$target_drive}->{drivestr}; - next if !defined($drivestr); + if ($self->{opts}->{remote}) { + PVE::Tunnel::finish_tunnel($self->{tunnel}, 1); + delete $self->{tunnel}; + return; + } - my $drive = PVE::QemuServer::parse_drive($target_drive, $drivestr); + my $local_volumes = $self->{local_volumes}; + foreach my $volid (values %{$self->{volume_map}}) { # don't clean up replicated disks! - next if defined($self->{replicated_volumes}->{$drive->{file}}); + next if $local_volumes->{$volid}->{replicated}; - my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}); + my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid); my $cmd = [@{$self->{rem_ssh}}, 'pvesm', 'free', "$storeid:$volname"]; @@ -628,13 +744,110 @@ sub phase1 { $conf->{lock} = 'migrate'; PVE::QemuConfig->write_config($vmid, $conf); - sync_disks($self, $vmid); + $self->scan_local_volumes($vmid); - # sync_disks fixes disk sizes to match their actual size, write changes so - # target allocates correct volumes + # fix disk sizes to match their actual size and write changes, + # so that the target allocates the correct volumes + $self->config_update_local_disksizes(); PVE::QemuConfig->write_config($vmid, $conf); + + $self->handle_replication($vmid); + + $self->sync_offline_local_volumes(); + $self->phase1_remote($vmid) if $self->{opts}->{remote}; }; +sub map_bridges { + my ($conf, $map, $scan_only) = @_; + + my $bridges = {}; + + foreach my $opt (keys %$conf) { + next if $opt !~ m/^net\d+$/; + + next if !$conf->{$opt}; + my $d = PVE::QemuServer::parse_net($conf->{$opt}); + next if !$d || !$d->{bridge}; + + my $target_bridge = PVE::JSONSchema::map_id($map, $d->{bridge}); + $bridges->{$target_bridge}->{$opt} = $d->{bridge}; + + next if $scan_only; + + $d->{bridge} = $target_bridge; + $conf->{$opt} = PVE::QemuServer::print_net($d); + } + + return $bridges; +} + +sub phase1_remote { + my ($self, $vmid) = @_; + + my $remote_conf = PVE::QemuConfig->load_config($vmid); + PVE::QemuConfig->update_volume_ids($remote_conf, $self->{volume_map}); + + my $bridges = map_bridges($remote_conf, $self->{opts}->{bridgemap}); + for my $target (keys $bridges->%*) { + for my $nic (keys $bridges->{$target}->%*) { + $self->log('info', "mapped: $nic from $bridges->{$target}->{$nic} to $target"); + } + } + + my @online_local_volumes = $self->filter_local_volumes('online'); + + my $storage_map = $self->{opts}->{storagemap}; + $self->{nbd} = {}; + PVE::QemuConfig->foreach_volume($remote_conf, sub { + my ($ds, $drive) = @_; + + # TODO eject CDROM? + return if PVE::QemuServer::drive_is_cdrom($drive); + + my $volid = $drive->{file}; + return if !$volid; + + return if !grep { $_ eq $volid} @online_local_volumes; + + my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid); + my $scfg = PVE::Storage::storage_config($self->{storecfg}, $storeid); + my $source_format = PVE::QemuServer::qemu_img_format($scfg, $volname); + + # set by target cluster + my $oldvolid = delete $drive->{file}; + delete $drive->{format}; + + my $targetsid = PVE::JSONSchema::map_id($storage_map, $storeid); + + my $params = { + format => $source_format, + storage => $targetsid, + drive => $drive, + }; + + $self->log('info', "Allocating volume for drive '$ds' on remote storage '$targetsid'.."); + my $res = PVE::Tunnel::write_tunnel($self->{tunnel}, 600, 'disk', $params); + + $self->log('info', "volume '$oldvolid' is '$res->{volid}' on the target\n"); + $remote_conf->{$ds} = $res->{drivestr}; + $self->{nbd}->{$ds} = $res; + }); + + my $conf_str = PVE::QemuServer::write_vm_config("remote", $remote_conf); + + # TODO expose in PVE::Firewall? + my $vm_fw_conf_path = "/etc/pve/firewall/$vmid.fw"; + my $fw_conf_str; + $fw_conf_str = PVE::Tools::file_get_contents($vm_fw_conf_path) + if -e $vm_fw_conf_path; + my $params = { + conf => $conf_str, + 'firewall-config' => $fw_conf_str, + }; + + PVE::Tunnel::write_tunnel($self->{tunnel}, 10, 'config', $params); +} + sub phase1_cleanup { my ($self, $vmid, $err) = @_; @@ -647,87 +860,103 @@ sub phase1_cleanup { $self->log('err', $err); } - if ($self->{volumes}) { - foreach my $volid (@{$self->{volumes}}) { - $self->log('err', "found stale volume copy '$volid' on node '$self->{node}'"); - # fixme: try to remove ? - } + eval { $self->cleanup_remotedisks() }; + if (my $err = $@) { + $self->log('err', $err); } eval { $self->cleanup_bitmaps() }; if (my $err =$@) { $self->log('err', $err); } - } -sub phase2 { - my ($self, $vmid) = @_; +sub phase2_start_local_cluster { + my ($self, $vmid, $params) = @_; my $conf = $self->{vmconf}; + my $local_volumes = $self->{local_volumes}; + my @online_local_volumes = $self->filter_local_volumes('online'); + + my $start = $params->{start_params}; + my $migrate = $params->{migrate_opts}; $self->log('info', "starting VM $vmid on remote node '$self->{node}'"); - my $raddr; - my $rport; - my $ruri; # the whole migration dst. URI (protocol:address[:port]) - my $nodename = PVE::INotify::nodename(); + my $tunnel_info = {}; ## start on remote node my $cmd = [@{$self->{rem_ssh}}]; - my $spice_ticket; - if (PVE::QemuServer::vga_conf_has_spice($conf->{vga})) { - my $res = mon_cmd($vmid, 'query-spice'); - $spice_ticket = $res->{ticket}; - } + push @$cmd, 'qm', 'start', $vmid; - push @$cmd , 'qm', 'start', $vmid, '--skiplock', '--migratedfrom', $nodename; + if ($start->{skiplock}) { + push @$cmd, '--skiplock'; + } - my $migration_type = $self->{opts}->{migration_type}; + push @$cmd, '--migratedfrom', $migrate->{migratedfrom}; - push @$cmd, '--migration_type', $migration_type; + push @$cmd, '--migration_type', $migrate->{type}; - push @$cmd, '--migration_network', $self->{opts}->{migration_network} - if $self->{opts}->{migration_network}; + push @$cmd, '--migration_network', $migrate->{network} + if $migrate->{network}; - if ($migration_type eq 'insecure') { - push @$cmd, '--stateuri', 'tcp'; - } else { - push @$cmd, '--stateuri', 'unix'; - } + push @$cmd, '--stateuri', $start->{statefile}; - if ($self->{forcemachine}) { - push @$cmd, '--machine', $self->{forcemachine}; + if ($start->{forcemachine}) { + push @$cmd, '--machine', $start->{forcemachine}; } - if ($self->{forcecpu}) { - push @$cmd, '--force-cpu', $self->{forcecpu}; + if ($start->{forcecpu}) { + push @$cmd, '--force-cpu', $start->{forcecpu}; } - if ($self->{online_local_volumes}) { + if ($self->{storage_migration}) { push @$cmd, '--targetstorage', ($self->{opts}->{targetstorage} // '1'); } my $spice_port; - my $unix_socket_info = {}; - # version > 0 for unix socket support - my $nbd_protocol_version = 1; - # TODO change to 'spice_ticket: \n' in 7.0 - my $input = $spice_ticket ? "$spice_ticket\n" : "\n"; - $input .= "nbd_protocol_version: $nbd_protocol_version\n"; - - my $number_of_online_replicated_volumes = 0; - - # prevent auto-vivification - if ($self->{online_local_volumes}) { - foreach my $volid (@{$self->{online_local_volumes}}) { - next if !$self->{replicated_volumes}->{$volid}; - $number_of_online_replicated_volumes++; - $input .= "replicated_volume: $volid\n"; + my $input = "nbd_protocol_version: $migrate->{nbd_proto_version}\n"; + + my @offline_local_volumes = $self->filter_local_volumes('offline'); + for my $volid (@offline_local_volumes) { + my $drivename = $local_volumes->{$volid}->{drivename}; + next if !$drivename || !$conf->{$drivename}; + + my $new_volid = $self->{volume_map}->{$volid}; + next if !$new_volid || $volid eq $new_volid; + + # FIXME PVE 8.x only use offline_volume variant once all targets can handle it + if ($drivename eq 'tpmstate0') { + $input .= "$drivename: $new_volid\n" + } else { + $input .= "offline_volume: $drivename: $new_volid\n" } } + $input .= "spice_ticket: $migrate->{spice_ticket}\n" if $migrate->{spice_ticket}; + + my @online_replicated_volumes = $self->filter_local_volumes('online', 1); + foreach my $volid (@online_replicated_volumes) { + $input .= "replicated_volume: $volid\n"; + } + + my $handle_storage_migration_listens = sub { + my ($drive_key, $drivestr, $nbd_uri) = @_; + + $self->{stopnbd} = 1; + $self->{target_drive}->{$drive_key}->{drivestr} = $drivestr; + $self->{target_drive}->{$drive_key}->{nbd_uri} = $nbd_uri; + + my $source_drive = PVE::QemuServer::parse_drive($drive_key, $conf->{$drive_key}); + my $target_drive = PVE::QemuServer::parse_drive($drive_key, $drivestr); + my $source_volid = $source_drive->{file}; + my $target_volid = $target_drive->{file}; + + $self->{volume_map}->{$source_volid} = $target_volid; + $self->log('info', "volume '$source_volid' is '$target_volid' on the target\n"); + }; + my $target_replicated_volumes = {}; # Note: We try to keep $spice_ticket secret (do not pass via command line parameter) @@ -735,20 +964,20 @@ sub phase2 { my $exitcode = PVE::Tools::run_command($cmd, input => $input, outfunc => sub { my $line = shift; - if ($line =~ m/^migration listens on tcp:(localhost|[\d\.]+|\[[\d\.:a-fA-F]+\]):(\d+)$/) { - $raddr = $1; - $rport = int($2); - $ruri = "tcp:$raddr:$rport"; + if ($line =~ m/^migration listens on (tcp):(localhost|[\d\.]+|\[[\d\.:a-fA-F]+\]):(\d+)$/) { + $tunnel_info->{addr} = $2; + $tunnel_info->{port} = int($3); + $tunnel_info->{proto} = $1; } - elsif ($line =~ m!^migration listens on unix:(/run/qemu-server/(\d+)\.migrate)$!) { - $raddr = $1; - die "Destination UNIX sockets VMID does not match source VMID" if $vmid ne $2; - $ruri = "unix:$raddr"; + elsif ($line =~ m!^migration listens on (unix):(/run/qemu-server/(\d+)\.migrate)$!) { + $tunnel_info->{addr} = $2; + die "Destination UNIX sockets VMID does not match source VMID" if $vmid ne $3; + $tunnel_info->{proto} = $1; } elsif ($line =~ m/^migration listens on port (\d+)$/) { - $raddr = "localhost"; - $rport = int($1); - $ruri = "tcp:$raddr:$rport"; + $tunnel_info->{addr} = "localhost"; + $tunnel_info->{port} = int($1); + $tunnel_info->{proto} = "tcp"; } elsif ($line =~ m/^spice listens on port (\d+)$/) { $spice_port = int($1); @@ -759,9 +988,7 @@ sub phase2 { my $targetdrive = $3; $targetdrive =~ s/drive-//g; - $self->{stopnbd} = 1; - $self->{target_drive}->{$targetdrive}->{drivestr} = $drivestr; - $self->{target_drive}->{$targetdrive}->{nbd_uri} = $nbd_uri; + $handle_storage_migration_listens->($targetdrive, $drivestr, $nbd_uri); } elsif ($line =~ m!^storage migration listens on nbd:unix:(/run/qemu-server/(\d+)_nbd\.migrate):exportname=(\S+) volume:(\S+)$!) { my $drivestr = $4; die "Destination UNIX socket's VMID does not match source VMID" if $vmid ne $2; @@ -770,10 +997,8 @@ sub phase2 { my $targetdrive = $3; $targetdrive =~ s/drive-//g; - $self->{stopnbd} = 1; - $self->{target_drive}->{$targetdrive}->{drivestr} = $drivestr; - $self->{target_drive}->{$targetdrive}->{nbd_uri} = $nbd_uri; - $unix_socket_info->{$nbd_unix_addr} = 1; + $handle_storage_migration_listens->($targetdrive, $drivestr, $nbd_uri); + $tunnel_info->{unix_sockets}->{$nbd_unix_addr} = 1; } elsif ($line =~ m/^re-using replicated volume: (\S+) - (.*)$/) { my $drive = $1; my $volid = $2; @@ -788,134 +1013,170 @@ sub phase2 { die "remote command failed with exit code $exitcode\n" if $exitcode; - die "unable to detect remote migration address\n" if !$raddr; + die "unable to detect remote migration address\n" if !$tunnel_info->{addr} || !$tunnel_info->{proto}; - if (scalar(keys %$target_replicated_volumes) != $number_of_online_replicated_volumes) { + if (scalar(keys %$target_replicated_volumes) != scalar(@online_replicated_volumes)) { die "number of replicated disks on source and target node do not match - target node too old?\n" } - $self->log('info', "start remote tunnel"); + return ($tunnel_info, $spice_port); +} - if ($migration_type eq 'secure') { +sub phase2_start_remote_cluster { + my ($self, $vmid, $params) = @_; - if ($ruri =~ /^unix:/) { - my $ssh_forward_info = ["$raddr:$raddr"]; - $unix_socket_info->{$raddr} = 1; + die "insecure migration to remote cluster not implemented\n" + if $params->{migrate_opts}->{type} ne 'websocket'; - my $unix_sockets = [ keys %$unix_socket_info ]; - for my $sock (@$unix_sockets) { - push @$ssh_forward_info, "$sock:$sock"; - unlink $sock; - } + my $remote_vmid = $self->{opts}->{remote}->{vmid}; - $self->{tunnel} = $self->fork_tunnel($ssh_forward_info); + # like regular start but with some overhead accounted for + my $memory = get_current_memory($self->{vmconf}->{memory}); + my $timeout = PVE::QemuServer::Helpers::config_aware_timeout($self->{vmconf}, $memory) + 10; - my $unix_socket_try = 0; # wait for the socket to become ready - while ($unix_socket_try <= 100) { - $unix_socket_try++; - my $available = 0; - foreach my $sock (@$unix_sockets) { - if (-S $sock) { - $available++; - } - } + my $res = PVE::Tunnel::write_tunnel($self->{tunnel}, $timeout, "start", $params); - if ($available == @$unix_sockets) { - last; - } + foreach my $drive (keys %{$res->{drives}}) { + $self->{stopnbd} = 1; + $self->{target_drive}->{$drive}->{drivestr} = $res->{drives}->{$drive}->{drivestr}; + my $nbd_uri = $res->{drives}->{$drive}->{nbd_uri}; + die "unexpected NBD uri for '$drive': $nbd_uri\n" + if $nbd_uri !~ s!/run/qemu-server/$remote_vmid\_!/run/qemu-server/$vmid\_!; - usleep(50000); - } - if ($unix_socket_try > 100) { - $self->{errors} = 1; - $self->finish_tunnel($self->{tunnel}); - die "Timeout, migration socket $ruri did not get ready"; - } - $self->{tunnel}->{unix_sockets} = $unix_sockets if (@$unix_sockets); + $self->{target_drive}->{$drive}->{nbd_uri} = $nbd_uri; + } - } elsif ($ruri =~ /^tcp:/) { - my $ssh_forward_info = []; - if ($raddr eq "localhost") { - # for backwards compatibility with older qemu-server versions - my $pfamily = PVE::Tools::get_host_address_family($nodename); - my $lport = PVE::Tools::next_migrate_port($pfamily); - push @$ssh_forward_info, "$lport:localhost:$rport"; - } + return ($res->{migrate}, $res->{spice_port}); +} - $self->{tunnel} = $self->fork_tunnel($ssh_forward_info); +sub phase2 { + my ($self, $vmid) = @_; - } else { - die "unsupported protocol in migration URI: $ruri\n"; + my $conf = $self->{vmconf}; + my $local_volumes = $self->{local_volumes}; + + # version > 0 for unix socket support + my $nbd_protocol_version = 1; + + my $spice_ticket; + if (PVE::QemuServer::vga_conf_has_spice($conf->{vga})) { + my $res = mon_cmd($vmid, 'query-spice'); + $spice_ticket = $res->{ticket}; + } + + my $migration_type = $self->{opts}->{migration_type}; + my $state_uri = $migration_type eq 'insecure' ? 'tcp' : 'unix'; + + my $params = { + start_params => { + statefile => $state_uri, + forcemachine => $self->{forcemachine}, + forcecpu => $self->{forcecpu}, + skiplock => 1, + }, + migrate_opts => { + spice_ticket => $spice_ticket, + type => $migration_type, + network => $self->{opts}->{migration_network}, + storagemap => $self->{opts}->{storagemap}, + migratedfrom => PVE::INotify::nodename(), + nbd_proto_version => $nbd_protocol_version, + nbd => $self->{nbd}, + }, + }; + + my ($tunnel_info, $spice_port); + + my @online_local_volumes = $self->filter_local_volumes('online'); + $self->{storage_migration} = 1 if scalar(@online_local_volumes); + + if (my $remote = $self->{opts}->{remote}) { + my $remote_vmid = $remote->{vmid}; + $params->{migrate_opts}->{remote_node} = $self->{node}; + ($tunnel_info, $spice_port) = $self->phase2_start_remote_cluster($vmid, $params); + die "only UNIX sockets are supported for remote migration\n" + if $tunnel_info->{proto} ne 'unix'; + + my $remote_socket = $tunnel_info->{addr}; + my $local_socket = $remote_socket; + $local_socket =~ s/$remote_vmid/$vmid/g; + $tunnel_info->{addr} = $local_socket; + + $self->log('info', "Setting up tunnel for '$local_socket'"); + PVE::Tunnel::forward_unix_socket($self->{tunnel}, $local_socket, $remote_socket); + + foreach my $remote_socket (@{$tunnel_info->{unix_sockets}}) { + my $local_socket = $remote_socket; + $local_socket =~ s/$remote_vmid/$vmid/g; + next if $self->{tunnel}->{forwarded}->{$local_socket}; + $self->log('info', "Setting up tunnel for '$local_socket'"); + PVE::Tunnel::forward_unix_socket($self->{tunnel}, $local_socket, $remote_socket); } } else { - #fork tunnel for insecure migration, to send faster commands like resume - $self->{tunnel} = $self->fork_tunnel(); + ($tunnel_info, $spice_port) = $self->phase2_start_local_cluster($vmid, $params); + + $self->log('info', "start remote tunnel"); + $self->start_remote_tunnel($tunnel_info); } - my $start = time(); - my $opt_bwlimit = $self->{opts}->{bwlimit}; + my $migrate_uri = "$tunnel_info->{proto}:$tunnel_info->{addr}"; + $migrate_uri .= ":$tunnel_info->{port}" + if defined($tunnel_info->{port}); - if (defined($self->{online_local_volumes})) { - $self->{storage_migration} = 1; + if ($self->{storage_migration}) { $self->{storage_migration_jobs} = {}; $self->log('info', "starting storage migration"); die "The number of local disks does not match between the source and the destination.\n" - if (scalar(keys %{$self->{target_drive}}) != scalar @{$self->{online_local_volumes}}); + if (scalar(keys %{$self->{target_drive}}) != scalar(@online_local_volumes)); foreach my $drive (keys %{$self->{target_drive}}){ my $target = $self->{target_drive}->{$drive}; my $nbd_uri = $target->{nbd_uri}; my $source_drive = PVE::QemuServer::parse_drive($drive, $conf->{$drive}); - my $target_drive = PVE::QemuServer::parse_drive($drive, $target->{drivestr}); - my $source_volid = $source_drive->{file}; - my $target_volid = $target_drive->{file}; - my $source_sid = PVE::Storage::Plugin::parse_volume_id($source_volid); - my $target_sid = PVE::Storage::Plugin::parse_volume_id($target_volid); - - my $bwlimit = PVE::Storage::get_bandwidth_limit('migration', [$source_sid, $target_sid], $opt_bwlimit); + my $bwlimit = $self->{local_volumes}->{$source_volid}->{bwlimit}; my $bitmap = $target->{bitmap}; $self->log('info', "$drive: start migration to $nbd_uri"); PVE::QemuServer::qemu_drive_mirror($vmid, $drive, $nbd_uri, $vmid, undef, $self->{storage_migration_jobs}, 'skip', undef, $bwlimit, $bitmap); - - $self->{volume_map}->{$source_volid} = $target_volid; - $self->log('info', "volume '$source_volid' is '$target_volid' on the target\n"); } } - $self->log('info', "starting online/live migration on $ruri"); + $self->log('info', "starting online/live migration on $migrate_uri"); $self->{livemigration} = 1; # load_defaults my $defaults = PVE::QemuServer::load_defaults(); - $self->log('info', "set migration_caps"); - eval { - PVE::QemuServer::set_migration_caps($vmid); - }; + $self->log('info', "set migration capabilities"); + eval { PVE::QemuServer::set_migration_caps($vmid) }; warn $@ if $@; my $qemu_migrate_params = {}; # migrate speed can be set via bwlimit (datacenter.cfg and API) and via the # migrate_speed parameter in qm.conf - take the lower of the two. - my $bwlimit = PVE::Storage::get_bandwidth_limit('migration', undef, $opt_bwlimit) // 0; - my $migrate_speed = $conf->{migrate_speed} // $bwlimit; - # migrate_speed is in MB/s, bwlimit in KB/s - $migrate_speed *= 1024; + my $bwlimit = $self->get_bwlimit(); - $migrate_speed = ($bwlimit < $migrate_speed) ? $bwlimit : $migrate_speed; + my $migrate_speed = $conf->{migrate_speed} // 0; + $migrate_speed *= 1024; # migrate_speed is in MB/s, bwlimit in KB/s - # always set migrate speed (overwrite kvm default of 32m) we set a very high - # default of 8192m which is basically unlimited - $migrate_speed ||= ($defaults->{migrate_speed} || 8192) * 1024; + if ($bwlimit && $migrate_speed) { + $migrate_speed = ($bwlimit < $migrate_speed) ? $bwlimit : $migrate_speed; + } else { + $migrate_speed ||= $bwlimit; + } + $migrate_speed ||= ($defaults->{migrate_speed} || 0) * 1024; - # qmp takes migrate_speed in B/s. - $migrate_speed *= 1024; - $self->log('info', "migration speed limit: $migrate_speed B/s"); + if ($migrate_speed) { + $migrate_speed *= 1024; # qmp takes migrate_speed in B/s. + $self->log('info', "migration speed limit: ". render_bytes($migrate_speed, 1) ."/s"); + } else { + # always set migrate speed as QEMU default to 128 MiBps == 1 Gbps, use 16 GiBps == 128 Gbps + $migrate_speed = (16 << 30); + } $qemu_migrate_params->{'max-bandwidth'} = int($migrate_speed); my $migrate_downtime = $defaults->{migrate_downtime}; @@ -926,11 +1187,11 @@ sub phase2 { $qemu_migrate_params->{'downtime-limit'} = int($migrate_downtime); # set cachesize to 10% of the total memory - my $memory = $conf->{memory} || $defaults->{memory}; + my $memory = get_current_memory($conf->{memory}); my $cachesize = int($memory * 1048576 / 10); $cachesize = round_powerof2($cachesize); - $self->log('info', "migration cachesize: $cachesize B"); + $self->log('info', "migration cachesize: " . render_bytes($cachesize, 1)); $qemu_migrate_params->{'xbzrle-cache-size'} = int($cachesize); $self->log('info', "set migration parameters"); @@ -939,7 +1200,7 @@ sub phase2 { }; $self->log('info', "migrate-set-parameters error: $@") if $@; - if (PVE::QemuServer::vga_conf_has_spice($conf->{vga})) { + if (PVE::QemuServer::vga_conf_has_spice($conf->{vga}) && !$self->{opts}->{remote}) { my $rpcenv = PVE::RPCEnvironment::get(); my $authuser = $rpcenv->get_user(); @@ -959,14 +1220,16 @@ sub phase2 { } - $self->log('info', "start migrate command to $ruri"); + my $start = time(); + + $self->log('info', "start migrate command to $migrate_uri"); eval { - mon_cmd($vmid, "migrate", uri => $ruri); + mon_cmd($vmid, "migrate", uri => $migrate_uri); }; my $merr = $@; - $self->log('info', "migrate uri => $ruri failed: $merr") if $merr; + $self->log('info', "migrate uri => $migrate_uri failed: $merr") if $merr; - my $lstat = 0; + my $last_mem_transferred = 0; my $usleep = 1000000; my $i = 0; my $err_count = 0; @@ -974,95 +1237,127 @@ sub phase2 { my $downtimecounter = 0; while (1) { $i++; - my $avglstat = $lstat ? $lstat / $i : 0; + my $avglstat = $last_mem_transferred ? $last_mem_transferred / $i : 0; usleep($usleep); - my $stat; - eval { - $stat = mon_cmd($vmid, "query-migrate"); - }; + + my $stat = eval { mon_cmd($vmid, "query-migrate") }; if (my $err = $@) { $err_count++; warn "query migrate failed: $err\n"; $self->log('info', "query migrate failed: $err"); if ($err_count <= 5) { - usleep(1000000); + usleep(1_000_000); next; } die "too many query migrate failures - aborting\n"; } - if (defined($stat->{status}) && $stat->{status} =~ m/^(setup)$/im) { - sleep(1); - next; - } - - if (defined($stat->{status}) && $stat->{status} =~ m/^(active|completed|failed|cancelled)$/im) { - $merr = undef; - $err_count = 0; - if ($stat->{status} eq 'completed') { - my $delay = time() - $start; - if ($delay > 0) { - my $mbps = sprintf "%.2f", $memory / $delay; - my $downtime = $stat->{downtime} || 0; - $self->log('info', "migration speed: $mbps MB/s - downtime $downtime ms"); - } - } + my $status = $stat->{status}; + if (defined($status) && $status =~ m/^(setup)$/im) { + sleep(1); + next; + } - if ($stat->{status} eq 'failed' || $stat->{status} eq 'cancelled') { - $self->log('info', "migration status error: $stat->{status}"); - die "aborting\n" + if (!defined($status) || $status !~ m/^(active|completed|failed|cancelled)$/im) { + die $merr if $merr; + die "unable to parse migration status '$status' - aborting\n"; + } + $merr = undef; + $err_count = 0; + + my $memstat = $stat->{ram}; + + if ($status eq 'completed') { + my $delay = time() - $start; + if ($delay > 0) { + my $total = $memstat->{total} || 0; + my $avg_speed = render_bytes($total / $delay, 1); + my $downtime = $stat->{downtime} || 0; + $self->log('info', "average migration speed: $avg_speed/s - downtime $downtime ms"); } + } + + if ($status eq 'failed' || $status eq 'cancelled') { + my $message = $stat->{'error-desc'} ? "$status - $stat->{'error-desc'}" : $status; + $self->log('info', "migration status error: $message"); + die "aborting\n" + } + + if ($status ne 'active') { + $self->log('info', "migration status: $status"); + last; + } + + if ($memstat->{transferred} ne $last_mem_transferred) { + my $trans = $memstat->{transferred} || 0; + my $rem = $memstat->{remaining} || 0; + my $total = $memstat->{total} || 0; + my $speed = ($memstat->{'pages-per-second'} // 0) * ($memstat->{'page-size'} // 0); + my $dirty_rate = ($memstat->{'dirty-pages-rate'} // 0) * ($memstat->{'page-size'} // 0); + + # reduce sleep if remainig memory is lower than the average transfer speed + $usleep = 100_000 if $avglstat && $rem < $avglstat; + + # also reduce loggin if we poll more frequent + my $should_log = $usleep > 100_000 ? 1 : ($i % 10) == 0; + + my $total_h = render_bytes($total, 1); + my $transferred_h = render_bytes($trans, 1); + my $speed_h = render_bytes($speed, 1); - if ($stat->{status} ne 'active') { - $self->log('info', "migration status: $stat->{status}"); - last; + my $progress = "transferred $transferred_h of $total_h VM-state, ${speed_h}/s"; + + if ($dirty_rate > $speed) { + my $dirty_rate_h = render_bytes($dirty_rate, 1); + $progress .= ", VM dirties lots of memory: $dirty_rate_h/s"; } - if ($stat->{ram}->{transferred} ne $lstat) { - my $trans = $stat->{ram}->{transferred} || 0; - my $rem = $stat->{ram}->{remaining} || 0; - my $total = $stat->{ram}->{total} || 0; - my $xbzrlecachesize = $stat->{"xbzrle-cache"}->{"cache-size"} || 0; - my $xbzrlebytes = $stat->{"xbzrle-cache"}->{"bytes"} || 0; - my $xbzrlepages = $stat->{"xbzrle-cache"}->{"pages"} || 0; - my $xbzrlecachemiss = $stat->{"xbzrle-cache"}->{"cache-miss"} || 0; - my $xbzrleoverflow = $stat->{"xbzrle-cache"}->{"overflow"} || 0; - # reduce sleep if remainig memory is lower than the average transfer speed - $usleep = 100000 if $avglstat && $rem < $avglstat; - - $self->log('info', "migration status: $stat->{status} (transferred ${trans}, " . - "remaining ${rem}), total ${total})"); - - if (${xbzrlecachesize}) { - $self->log('info', "migration xbzrle cachesize: ${xbzrlecachesize} transferred ${xbzrlebytes} pages ${xbzrlepages} cachemiss ${xbzrlecachemiss} overflow ${xbzrleoverflow}"); - } + $self->log('info', "migration $status, $progress") if $should_log; - if (($lastrem && $rem > $lastrem ) || ($rem == 0)) { - $downtimecounter++; - } - $lastrem = $rem; - - if ($downtimecounter > 5) { - $downtimecounter = 0; - $migrate_downtime *= 2; - $self->log('info', "auto-increased downtime to continue migration: $migrate_downtime ms"); - eval { - # migrate-set-parameters does not touch values not - # specified, so this only changes downtime-limit - mon_cmd($vmid, "migrate-set-parameters", 'downtime-limit' => int($migrate_downtime)); - }; - $self->log('info', "migrate-set-parameters error: $@") if $@; - } + my $xbzrle = $stat->{"xbzrle-cache"} || {}; + my ($xbzrlebytes, $xbzrlepages) = $xbzrle->@{'bytes', 'pages'}; + if ($xbzrlebytes || $xbzrlepages) { + my $bytes_h = render_bytes($xbzrlebytes, 1); + + my $msg = "send updates to $xbzrlepages pages in $bytes_h encoded memory"; + $msg .= sprintf(", cache-miss %.2f%%", $xbzrle->{'cache-miss-rate'} * 100) + if $xbzrle->{'cache-miss-rate'}; + + $msg .= ", overflow $xbzrle->{overflow}" if $xbzrle->{overflow}; + + $self->log('info', "xbzrle: $msg") if $should_log; } + if (($lastrem && $rem > $lastrem) || ($rem == 0)) { + $downtimecounter++; + } + $lastrem = $rem; + + if ($downtimecounter > 5) { + $downtimecounter = 0; + $migrate_downtime *= 2; + $self->log('info', "auto-increased downtime to continue migration: $migrate_downtime ms"); + eval { + # migrate-set-parameters does not touch values not + # specified, so this only changes downtime-limit + mon_cmd($vmid, "migrate-set-parameters", 'downtime-limit' => int($migrate_downtime)); + }; + $self->log('info', "migrate-set-parameters error: $@") if $@; + } + } - $lstat = $stat->{ram}->{transferred}; + $last_mem_transferred = $memstat->{transferred}; + } - } else { - die $merr if $merr; - die "unable to parse migration status '$stat->{status}' - aborting\n"; + if ($self->{storage_migration}) { + # finish block-job with block-job-cancel, to disconnect source VM from NBD + # to avoid it trying to re-establish it. We are in blockjob ready state, + # thus, this command changes to it to blockjob complete (see qapi docs) + eval { PVE::QemuServer::qemu_drive_mirror_monitor($vmid, undef, $self->{storage_migration_jobs}, 'cancel'); }; + if (my $err = $@) { + die "Failed to complete storage migration: $err\n"; } } } @@ -1081,6 +1376,22 @@ sub phase2_cleanup { }; $self->log('info', "migrate_cancel error: $@") if $@; + my $vm_status = eval { + mon_cmd($vmid, 'query-status')->{status} or die "no 'status' in result\n"; + }; + $self->log('err', "query-status error: $@") if $@; + + # Can end up in POSTMIGRATE state if failure occurred after convergence. Try going back to + # original state. Unfortunately, direct transition from POSTMIGRATE to PAUSED is not possible. + if ($vm_status && $vm_status eq 'postmigrate') { + if (!$self->{vm_was_paused}) { + eval { mon_cmd($vmid, 'cont'); }; + $self->log('err', "resuming VM failed: $@") if $@; + } else { + $self->log('err', "VM was paused, but ended in postmigrate state"); + } + } + my $conf = $self->{vmconf}; delete $conf->{lock}; eval { PVE::QemuConfig->write_config($vmid, $conf) }; @@ -1103,11 +1414,15 @@ sub phase2_cleanup { my $nodename = PVE::INotify::nodename(); - my $cmd = [@{$self->{rem_ssh}}, 'qm', 'stop', $vmid, '--skiplock', '--migratedfrom', $nodename]; - eval{ PVE::Tools::run_command($cmd, outfunc => sub {}, errfunc => sub {}) }; - if (my $err = $@) { - $self->log('err', $err); - $self->{errors} = 1; + if ($self->{tunnel} && $self->{tunnel}->{version} >= 2) { + PVE::Tunnel::write_tunnel($self->{tunnel}, 10, 'stop'); + } else { + my $cmd = [@{$self->{rem_ssh}}, 'qm', 'stop', $vmid, '--skiplock', '--migratedfrom', $nodename]; + eval{ PVE::Tools::run_command($cmd, outfunc => sub {}, errfunc => sub {}) }; + if (my $err = $@) { + $self->log('err', $err); + $self->{errors} = 1; + } } # cleanup after stopping, otherwise disks might be in-use by target VM! @@ -1118,7 +1433,7 @@ sub phase2_cleanup { if ($self->{tunnel}) { - eval { finish_tunnel($self, $self->{tunnel}); }; + eval { PVE::Tunnel::finish_tunnel($self->{tunnel}); }; if (my $err = $@) { $self->log('err', $err); $self->{errors} = 1; @@ -1129,18 +1444,7 @@ sub phase2_cleanup { sub phase3 { my ($self, $vmid) = @_; - my $volids = $self->{volumes}; - return if $self->{phase2errors}; - - # destroy local copies - foreach my $volid (@$volids) { - eval { PVE::Storage::vdisk_free($self->{storecfg}, $volid); }; - if (my $err = $@) { - $self->log('err', "removing local copy of '$volid' failed - $err"); - $self->{errors} = 1; - last if $err =~ /^interrupted by signal$/; - } - } + return; } sub phase3_cleanup { @@ -1151,20 +1455,9 @@ sub phase3_cleanup { my $tunnel = $self->{tunnel}; - if ($self->{storage_migration}) { - # finish block-job with block-job-cancel, to disconnect source VM from NBD - # to avoid it trying to re-establish it. We are in blockjob ready state, - # thus, this command changes to it to blockjob complete (see qapi docs) - eval { PVE::QemuServer::qemu_drive_mirror_monitor($vmid, undef, $self->{storage_migration_jobs}, 'cancel'); }; + my $sourcevollist = PVE::QemuServer::get_vm_volumes($conf); - if (my $err = $@) { - eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $self->{storage_migration_jobs}) }; - eval { PVE::QemuMigrate::cleanup_remotedisks($self) }; - die "Failed to complete storage migration: $err\n"; - } - } - - if ($self->{volume_map}) { + if ($self->{volume_map} && !$self->{opts}->{remote}) { my $target_drives = $self->{target_drive}; # FIXME: for NBD storage migration we now only update the volid, and @@ -1180,65 +1473,96 @@ sub phase3_cleanup { } # transfer replication state before move config - $self->transfer_replication_state() if $self->{replicated_volumes}; - - # move config to remote node - my $conffile = PVE::QemuConfig->config_file($vmid); - my $newconffile = PVE::QemuConfig->config_file($vmid, $self->{node}); - - die "Failed to move config to node '$self->{node}' - rename failed: $!\n" - if !rename($conffile, $newconffile); - - $self->switch_replication_job_target() if $self->{replicated_volumes}; + if (!$self->{opts}->{remote}) { + $self->transfer_replication_state() if $self->{is_replicated}; + PVE::QemuConfig->move_config_to_node($vmid, $self->{node}); + $self->switch_replication_job_target() if $self->{is_replicated}; + } if ($self->{livemigration}) { if ($self->{stopnbd}) { $self->log('info', "stopping NBD storage migration server on target."); # stop nbd server on remote vm - requirement for resume since 2.9 - my $cmd = [@{$self->{rem_ssh}}, 'qm', 'nbdstop', $vmid]; + if ($tunnel && $tunnel->{version} && $tunnel->{version} >= 2) { + eval { + PVE::Tunnel::write_tunnel($tunnel, 30, 'nbdstop'); + }; + if (my $err = $@) { + $self->log('err', $err); + $self->{errors} = 1; + } + } else { + my $cmd = [@{$self->{rem_ssh}}, 'qm', 'nbdstop', $vmid]; - eval{ PVE::Tools::run_command($cmd, outfunc => sub {}, errfunc => sub {}) }; - if (my $err = $@) { - $self->log('err', $err); - $self->{errors} = 1; + eval{ PVE::Tools::run_command($cmd, outfunc => sub {}, errfunc => sub {}) }; + if (my $err = $@) { + $self->log('err', $err); + $self->{errors} = 1; + } } } - # config moved and nbd server stopped - now we can resume vm on target - if ($tunnel && $tunnel->{version} && $tunnel->{version} >= 1) { - eval { - $self->write_tunnel($tunnel, 30, "resume $vmid"); - }; - if (my $err = $@) { - $self->log('err', $err); - $self->{errors} = 1; - } - } else { - my $cmd = [@{$self->{rem_ssh}}, 'qm', 'resume', $vmid, '--skiplock', '--nocheck']; - my $logf = sub { - my $line = shift; - $self->log('err', $line); - }; - eval { PVE::Tools::run_command($cmd, outfunc => sub {}, errfunc => $logf); }; - if (my $err = $@) { - $self->log('err', $err); - $self->{errors} = 1; + # deletes local FDB entries if learning is disabled, they'll be re-added on target on resume + PVE::QemuServer::del_nets_bridge_fdb($conf, $vmid); + + if (!$self->{vm_was_paused}) { + # config moved and nbd server stopped - now we can resume vm on target + if ($tunnel && $tunnel->{version} && $tunnel->{version} >= 1) { + my $cmd = $tunnel->{version} == 1 ? "resume $vmid" : "resume"; + eval { + PVE::Tunnel::write_tunnel($tunnel, 30, $cmd); + }; + if (my $err = $@) { + $self->log('err', $err); + $self->{errors} = 1; + } + } else { + # nocheck in case target node hasn't processed the config move/rename yet + my $cmd = [@{$self->{rem_ssh}}, 'qm', 'resume', $vmid, '--skiplock', '--nocheck']; + my $logf = sub { + my $line = shift; + $self->log('err', $line); + }; + eval { PVE::Tools::run_command($cmd, outfunc => sub {}, errfunc => $logf); }; + if (my $err = $@) { + $self->log('err', $err); + $self->{errors} = 1; + } } } - if ($self->{storage_migration} && PVE::QemuServer::parse_guest_agent($conf)->{fstrim_cloned_disks} && $self->{running}) { - my $cmd = [@{$self->{rem_ssh}}, 'qm', 'guest', 'cmd', $vmid, 'fstrim']; - eval{ PVE::Tools::run_command($cmd, outfunc => sub {}, errfunc => sub {}) }; + if ( + $self->{storage_migration} + && PVE::QemuServer::parse_guest_agent($conf)->{fstrim_cloned_disks} + && $self->{running} + ) { + if (!$self->{vm_was_paused}) { + $self->log('info', "issuing guest fstrim"); + if ($self->{opts}->{remote}) { + PVE::Tunnel::write_tunnel($self->{tunnel}, 600, 'fstrim'); + } else { + my $cmd = [@{$self->{rem_ssh}}, 'qm', 'guest', 'cmd', $vmid, 'fstrim']; + eval{ PVE::Tools::run_command($cmd, outfunc => sub {}, errfunc => sub {}) }; + if (my $err = $@) { + $self->log('err', "fstrim failed - $err"); + $self->{errors} = 1; + } + } + } else { + $self->log('info', "skipping guest fstrim, because VM is paused"); + } } } # close tunnel on successful migration, on error phase2_cleanup closed it - if ($tunnel) { - eval { finish_tunnel($self, $tunnel); }; + if ($tunnel && $tunnel->{version} == 1) { + eval { PVE::Tunnel::finish_tunnel($tunnel); }; if (my $err = $@) { $self->log('err', $err); $self->{errors} = 1; } + $tunnel = undef; + delete $self->{tunnel}; } eval { @@ -1255,7 +1579,7 @@ sub phase3_cleanup { } }; - # always stop local VM + # always stop local VM with nocheck, since config is moved already eval { PVE::QemuServer::vm_stop($self->{storecfg}, $vmid, 1, 1); }; if (my $err = $@) { $self->log('err', "stopping vm failed - $err"); @@ -1264,35 +1588,42 @@ sub phase3_cleanup { # always deactivate volumes - avoid lvm LVs to be active on several nodes eval { - my $vollist = PVE::QemuServer::get_vm_volumes($conf); - PVE::Storage::deactivate_volumes($self->{storecfg}, $vollist); + PVE::Storage::deactivate_volumes($self->{storecfg}, $sourcevollist); }; if (my $err = $@) { $self->log('err', $err); $self->{errors} = 1; } - if($self->{storage_migration}) { - # destroy local copies - my $volids = $self->{online_local_volumes}; + my @not_replicated_volumes = $self->filter_local_volumes(undef, 0); - foreach my $volid (@$volids) { - # keep replicated volumes! - next if $self->{replicated_volumes}->{$volid}; + # destroy local copies + foreach my $volid (@not_replicated_volumes) { + # remote is cleaned up below + next if $self->{opts}->{remote}; - eval { PVE::Storage::vdisk_free($self->{storecfg}, $volid); }; - if (my $err = $@) { - $self->log('err', "removing local copy of '$volid' failed - $err"); - $self->{errors} = 1; - last if $err =~ /^interrupted by signal$/; - } + eval { PVE::Storage::vdisk_free($self->{storecfg}, $volid); }; + if (my $err = $@) { + $self->log('err', "removing local copy of '$volid' failed - $err"); + $self->{errors} = 1; + last if $err =~ /^interrupted by signal$/; } - } # clear migrate lock - my $cmd = [ @{$self->{rem_ssh}}, 'qm', 'unlock', $vmid ]; - $self->cmd_logerr($cmd, errmsg => "failed to clear migrate lock"); + if ($tunnel && $tunnel->{version} >= 2) { + PVE::Tunnel::write_tunnel($tunnel, 10, "unlock"); + + PVE::Tunnel::finish_tunnel($tunnel); + } else { + my $cmd = [ @{$self->{rem_ssh}}, 'qm', 'unlock', $vmid ]; + $self->cmd_logerr($cmd, errmsg => "failed to clear migrate lock"); + } + + if ($self->{opts}->{remote} && $self->{opts}->{delete}) { + eval { PVE::QemuServer::destroy_vm($self->{storecfg}, $vmid, 1, undef, 0) }; + warn "Failed to remove source VM - $@\n" if $@; + } } sub final_cleanup { diff --git a/PVE/QemuServer.pm b/PVE/QemuServer.pm index a282449..9032d29 100644 --- a/PVE/QemuServer.pm +++ b/PVE/QemuServer.pm @@ -19,20 +19,29 @@ use IO::Select; use IO::Socket::UNIX; use IPC::Open3; use JSON; +use List::Util qw(first); use MIME::Base64; use POSIX; use Storable qw(dclone); -use Time::HiRes qw(gettimeofday); +use Time::HiRes qw(gettimeofday usleep); use URI::Escape; use UUID; use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file); +use PVE::CGroup; +use PVE::CpuSet; use PVE::DataCenterConfig; use PVE::Exception qw(raise raise_param_exc); +use PVE::Format qw(render_duration render_bytes); use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne); +use PVE::HA::Config; +use PVE::Mapping::PCI; +use PVE::Mapping::USB; use PVE::INotify; use PVE::JSONSchema qw(get_standard_option parse_property_string); use PVE::ProcFSTools; +use PVE::PBSClient; +use PVE::RESTEnvironment qw(log_warn); use PVE::RPCEnvironment; use PVE::Storage; use PVE::SysFSTools; @@ -41,46 +50,72 @@ use PVE::Tools qw(run_command file_read_firstline file_get_contents dir_glob_for use PVE::QMPClient; use PVE::QemuConfig; -use PVE::QemuServer::Helpers qw(min_version config_aware_timeout); +use PVE::QemuServer::Helpers qw(config_aware_timeout min_version windows_version); use PVE::QemuServer::Cloudinit; -use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options); -use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom parse_drive print_drive); +use PVE::QemuServer::CGroup; +use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options get_cpu_bitness is_native_arch); +use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom drive_is_read_only parse_drive print_drive); use PVE::QemuServer::Machine; -use PVE::QemuServer::Memory; +use PVE::QemuServer::Memory qw(get_current_memory); use PVE::QemuServer::Monitor qw(mon_cmd); use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci); -use PVE::QemuServer::USB qw(parse_usb_device); +use PVE::QemuServer::QMPHelpers qw(qemu_deviceadd qemu_devicedel qemu_objectadd qemu_objectdel); +use PVE::QemuServer::USB; my $have_sdn; eval { require PVE::Network::SDN::Zones; + require PVE::Network::SDN::Vnets; $have_sdn = 1; }; my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/'; my $OVMF = { - x86_64 => [ - "$EDK2_FW_BASE/OVMF_CODE.fd", - "$EDK2_FW_BASE/OVMF_VARS.fd" - ], - aarch64 => [ - "$EDK2_FW_BASE/AAVMF_CODE.fd", - "$EDK2_FW_BASE/AAVMF_VARS.fd" - ], + x86_64 => { + '4m-no-smm' => [ + "$EDK2_FW_BASE/OVMF_CODE_4M.fd", + "$EDK2_FW_BASE/OVMF_VARS_4M.fd", + ], + '4m-no-smm-ms' => [ + "$EDK2_FW_BASE/OVMF_CODE_4M.fd", + "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd", + ], + '4m' => [ + "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd", + "$EDK2_FW_BASE/OVMF_VARS_4M.fd", + ], + '4m-ms' => [ + "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd", + "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd", + ], + # FIXME: These are legacy 2MB-sized images that modern OVMF doesn't supports to build + # anymore. how can we deperacate this sanely without breaking existing instances, or using + # older backups and snapshot? + default => [ + "$EDK2_FW_BASE/OVMF_CODE.fd", + "$EDK2_FW_BASE/OVMF_VARS.fd", + ], + }, + aarch64 => { + default => [ + "$EDK2_FW_BASE/AAVMF_CODE.fd", + "$EDK2_FW_BASE/AAVMF_VARS.fd", + ], + }, }; my $cpuinfo = PVE::ProcFSTools::read_cpuinfo(); -# Note about locking: we use flock on the config file protect -# against concurent actions. -# Aditionaly, we have a 'lock' setting in the config file. This -# can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not -# allowed when such lock is set. But you can ignore this kind of -# lock with the --skiplock flag. +# Note about locking: we use flock on the config file protect against concurent actions. +# Aditionaly, we have a 'lock' setting in the config file. This can be set to 'migrate', +# 'backup', 'snapshot' or 'rollback'. Most actions are not allowed when such lock is set. +# But you can ignore this kind of lock with the --skiplock flag. -cfs_register_file('/qemu-server/', - \&parse_vm_config, - \&write_vm_config); +cfs_register_file( + '/qemu-server/', + \&parse_vm_config, + \&write_vm_config +); PVE::JSONSchema::register_standard_option('pve-qm-stateuri', { description => "Some command save/restore state from this location.", @@ -89,46 +124,7 @@ PVE::JSONSchema::register_standard_option('pve-qm-stateuri', { optional => 1, }); -PVE::JSONSchema::register_standard_option('pve-qemu-machine', { - description => "Specifies the Qemu machine type.", - type => 'string', - pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)', - maxLength => 40, - optional => 1, -}); - - -sub map_storage { - my ($map, $source) = @_; - - return $source if !defined($map); - - return $map->{entries}->{$source} - if $map->{entries} && defined($map->{entries}->{$source}); - - return $map->{default} if $map->{default}; - - # identity (fallback) - return $source; -} - -PVE::JSONSchema::register_standard_option('pve-targetstorage', { - description => "Mapping from source to target storages. Providing only a single storage ID maps all source storages to that storage. Providing the special value '1' will map each source storage to itself.", - type => 'string', - format => 'storagepair-list', - optional => 1, -}); - -#no warnings 'redefine'; - -sub cgroups_write { - my ($controller, $vmid, $option, $value) = @_; - - my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option"; - PVE::ProcFSTools::write_proc_entry($path, $value); - -} - +# FIXME: remove in favor of just using the INotify one, it's cached there exactly the same way my $nodename_cache; sub nodename { $nodename_cache //= PVE::INotify::nodename(); @@ -155,16 +151,22 @@ PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt); my $agent_fmt = { enabled => { - description => "Enable/disable Qemu GuestAgent.", + description => "Enable/disable communication with a QEMU Guest Agent (QGA) running in the VM.", type => 'boolean', default => 0, default_key => 1, }, fstrim_cloned_disks => { - description => "Run fstrim after cloning/moving a disk.", + description => "Run fstrim after moving a disk or migrating the VM.", + type => 'boolean', + optional => 1, + default => 0, + }, + 'freeze-fs-on-backup' => { + description => "Freeze/thaw guest filesystems on backup for consistency.", type => 'boolean', optional => 1, - default => 0 + default => 1, }, type => { description => "Select the agent type", @@ -182,7 +184,7 @@ my $vga_fmt = { default => 'std', optional => 1, default_key => 1, - enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)], + enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio virtio-gl vmware)], }, memory => { description => "Sets the VGA memory (in MiB). Has no effect with serial display.", @@ -191,6 +193,13 @@ my $vga_fmt = { minimum => 4, maximum => 512, }, + clipboard => { + description => 'Enable a specific clipboard. If not set, depending on the display type the' + .' SPICE one will be added. Migration with VNC clipboard is not yet supported!', + type => 'string', + enum => ['vnc'], + optional => 1, + }, }; my $ivshmem_fmt = { @@ -216,7 +225,7 @@ my $audio_fmt = { }, driver => { type => 'string', - enum => ['spice'], + enum => ['spice', 'none'], default => 'spice', optional => 1, description => "Driver backend for the audio device." @@ -244,38 +253,49 @@ my $rng_fmt = { type => 'string', enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'], default_key => 1, - description => "The file on the host to gather entropy from. In most" - . " cases /dev/urandom should be preferred over /dev/random" - . " to avoid entropy-starvation issues on the host. Using" - . " urandom does *not* decrease security in any meaningful" - . " way, as it's still seeded from real entropy, and the" - . " bytes provided will most likely be mixed with real" - . " entropy on the guest as well. /dev/hwrng can be used" - . " to pass through a hardware RNG from the host.", + description => "The file on the host to gather entropy from. In most cases '/dev/urandom'" + ." should be preferred over '/dev/random' to avoid entropy-starvation issues on the" + ." host. Using urandom does *not* decrease security in any meaningful way, as it's" + ." still seeded from real entropy, and the bytes provided will most likely be mixed" + ." with real entropy on the guest as well. '/dev/hwrng' can be used to pass through" + ." a hardware RNG from the host.", }, max_bytes => { type => 'integer', - description => "Maximum bytes of entropy injected into the guest every" - . " 'period' milliseconds. Prefer a lower value when using" - . " /dev/random as source. Use 0 to disable limiting" - . " (potentially dangerous!).", + description => "Maximum bytes of entropy allowed to get injected into the guest every" + ." 'period' milliseconds. Prefer a lower value when using '/dev/random' as source. Use" + ." `0` to disable limiting (potentially dangerous!).", optional => 1, - # default is 1 KiB/s, provides enough entropy to the guest to avoid - # boot-starvation issues (e.g. systemd etc...) while allowing no chance - # of overwhelming the host, provided we're reading from /dev/urandom + # default is 1 KiB/s, provides enough entropy to the guest to avoid boot-starvation issues + # (e.g. systemd etc...) while allowing no chance of overwhelming the host, provided we're + # reading from /dev/urandom default => 1024, }, period => { type => 'integer', - description => "Every 'period' milliseconds the entropy-injection quota" - . " is reset, allowing the guest to retrieve another" - . " 'max_bytes' of entropy.", + description => "Every 'period' milliseconds the entropy-injection quota is reset, allowing" + ." the guest to retrieve another 'max_bytes' of entropy.", optional => 1, default => 1000, }, }; +my $meta_info_fmt = { + 'ctime' => { + type => 'integer', + description => "The guest creation timestamp as UNIX epoch time", + minimum => 0, + optional => 1, + }, + 'creation-qemu' => { + type => 'string', + description => "The QEMU (machine) version from the time this VM was created.", + pattern => '\d+(\.\d+)+', + optional => 1, + }, +}; + my $confdesc = { onboot => { optional => 1, @@ -290,9 +310,13 @@ my $confdesc = { default => 0, }, hotplug => { - optional => 1, - type => 'string', format => 'pve-hotplug-features', - description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.", + optional => 1, + type => 'string', format => 'pve-hotplug-features', + description => "Selectively enable hotplug features. This is a comma separated list of" + ." hotplug features: 'network', 'disk', 'cpu', 'memory', 'usb' and 'cloudinit'. Use '0' to disable" + ." hotplug completely. Using '1' as value is an alias for the default `network,disk,usb`." + ." USB hotplugging is possible for guests with machine version >= 7.1 and ostype l26 or" + ." windows > 7.", default => 'network,disk,usb', }, reboot => { @@ -311,37 +335,41 @@ my $confdesc = { optional => 1, type => 'number', description => "Limit of CPU usage.", - verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.", + verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has" + ." total of '2' CPU time. Value '0' indicates no CPU limit.", minimum => 0, maximum => 128, - default => 0, + default => 0, }, cpuunits => { optional => 1, type => 'integer', - description => "CPU weight for a VM.", - verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.", - minimum => 2, + description => "CPU weight for a VM, will be clamped to [1, 10000] in cgroup v2.", + verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler." + ." The larger the number is, the more CPU time this VM gets. Number is relative to" + ." weights of all the other running VMs.", + minimum => 1, maximum => 262144, - default => 1024, + default => 'cgroup v1: 1024, cgroup v2: 100', }, memory => { optional => 1, - type => 'integer', - description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.", - minimum => 16, - default => 512, + type => 'string', + description => "Memory properties.", + format => $PVE::QemuServer::Memory::memory_fmt }, balloon => { - optional => 1, - type => 'integer', - description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.", + optional => 1, + type => 'integer', + description => "Amount of target RAM for the VM in MiB. Using zero disables the ballon driver.", minimum => 0, }, shares => { - optional => 1, - type => 'integer', - description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.", + optional => 1, + type => 'integer', + description => "Amount of memory shares for auto-ballooning. The larger the number is, the" + ." more memory this VM gets. Number is relative to weights of all other running VMs." + ." Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.", minimum => 0, maximum => 50000, default => 1000, @@ -349,8 +377,8 @@ my $confdesc = { keyboard => { optional => 1, type => 'string', - description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.". - "It should not be necessary to set it.", + description => "Keyboard layout for VNC server. This option is generally not required and" + ." is often better handled from within the guest OS.", enum => PVE::Tools::kvmkeymaplist(), default => undef, }, @@ -369,12 +397,15 @@ my $confdesc = { description => { optional => 1, type => 'string', - description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.", + description => "Description for the VM. Shown in the web-interface VM's summary." + ." This is saved as comment inside the configuration file.", + maxLength => 1024 * 8, }, ostype => { optional => 1, type => 'string', - enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)], + # NOTE: When extending, also consider extending `%guest_types` in `Import/ESXi.pm`. + enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 win11 l24 l26 solaris)], description => "Specify guest operating system.", verbose_description => < { optional => 1, type => 'string', format => 'pve-qm-boot', - description => "Specify guest boot order. Use with 'order=', usage with" - . " no key or 'legacy=' is deprecated.", + description => "Specify guest boot order. Use the 'order=' sub-property as usage with no" + ." key or 'legacy=' is deprecated.", }, bootdisk => { optional => 1, @@ -462,7 +494,7 @@ EODESC }, agent => { optional => 1, - description => "Enable/disable Qemu GuestAgent and its properties.", + description => "Enable/disable communication with the QEMU Guest Agent and its properties.", type => 'string', format => $agent_fmt, }, @@ -481,8 +513,8 @@ EODESC localtime => { optional => 1, type => 'boolean', - description => "Set the real time clock to local time. This is enabled by default if ostype" - ." indicates a Microsoft OS.", + description => "Set the real time clock (RTC) to local time. This is enabled by default if" + ." the `ostype` indicates a Microsoft Windows OS.", }, freeze => { optional => 1, @@ -532,7 +564,7 @@ EODESC verbose_description => < { optional => 1, type => 'number', - description => "Set maximum tolerated downtime (in seconds) for migrations.", + description => "Set maximum tolerated downtime (in seconds) for migrations. Should the" + ." migration not be able to converge in the very end, because too much newly dirtied" + ." RAM needs to be transferred, the limit will be increased automatically step-by-step" + ." until migration can converge.", minimum => 0, default => 0.1, }, @@ -686,6 +721,17 @@ EODESCR description => "Configure a VirtIO-based Random Number Generator.", optional => 1, }, + meta => { + type => 'string', + format => $meta_info_fmt, + description => "Some (read-only) meta-information about this guest.", + optional => 1, + }, + affinity => { + type => 'string', format => 'pve-cpuset', + description => "List of host cores used to execute guest processes, for example: 0,5,8-11", + optional => 1, + }, }; my $cicustom_fmt = { @@ -700,22 +746,28 @@ my $cicustom_fmt = { network => { type => 'string', optional => 1, - description => 'Specify a custom file containing all network data passed to the VM via' - .' cloud-init.', + description => 'To pass a custom file containing all network data to the VM via cloud-init.', format => 'pve-volume-id', format_description => 'volume', }, user => { type => 'string', optional => 1, - description => 'Specify a custom file containing all user data passed to the VM via' - .' cloud-init.', + description => 'To pass a custom file containing all user data to the VM via cloud-init.', + format => 'pve-volume-id', + format_description => 'volume', + }, + vendor => { + type => 'string', + optional => 1, + description => 'To pass a custom file containing all vendor data to the VM via cloud-init.', format => 'pve-volume-id', format_description => 'volume', }, }; PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt); +# any new option might need to be added to $cloudinitoptions in PVE::API2::Qemu my $confdesc_cloudinit = { citype => { optional => 1, @@ -723,7 +775,7 @@ my $confdesc_cloudinit = { description => 'Specifies the cloud-init configuration format. The default depends on the' .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,' .' and `configdrive2` for windows.', - enum => ['configdrive2', 'nocloud'], + enum => ['configdrive2', 'nocloud', 'opennebula'], }, ciuser => { optional => 1, @@ -738,6 +790,12 @@ my $confdesc_cloudinit = { .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not' .' support hashed passwords.', }, + ciupgrade => { + optional => 1, + type => 'boolean', + description => 'cloud-init: do an automatic package upgrade after the first boot.', + default => 1, + }, cicustom => { optional => 1, type => 'string', @@ -748,16 +806,16 @@ my $confdesc_cloudinit = { searchdomain => { optional => 1, type => 'string', - description => "cloud-init: Sets DNS search domains for a container. Create will' + description => 'cloud-init: Sets DNS search domains for a container. Create will' .' automatically use the setting from the host if neither searchdomain nor nameserver' - .' are set.", + .' are set.', }, nameserver => { optional => 1, type => 'string', format => 'address-list', - description => "cloud-init: Sets DNS server IP address for a container. Create will' + description => 'cloud-init: Sets DNS server IP address for a container. Create will' .' automatically use the setting from the host if neither searchdomain nor nameserver' - .' are set.", + .' are set.', }, sshkeys => { optional => 1, @@ -788,53 +846,30 @@ while (my ($k, $v) = each %$confdesc) { PVE::JSONSchema::register_standard_option("pve-qm-$k", $v); } -my $MAX_USB_DEVICES = 5; my $MAX_NETS = 32; my $MAX_SERIAL_PORTS = 4; my $MAX_PARALLEL_PORTS = 3; -my $MAX_NUMA = 8; -my $numa_fmt = { - cpus => { - type => "string", - pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/, - description => "CPUs accessing this NUMA node.", - format_description => "id[-id];...", - }, - memory => { - type => "number", - description => "Amount of memory this NUMA node provides.", - optional => 1, - }, - hostnodes => { - type => "string", - pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/, - description => "Host NUMA nodes to use.", - format_description => "id[-id];...", - optional => 1, - }, - policy => { - type => 'string', - enum => [qw(preferred bind interleave)], - description => "NUMA allocation policy.", - optional => 1, - }, -}; -PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt); -my $numadesc = { - optional => 1, - type => 'string', format => $numa_fmt, - description => "NUMA topology.", -}; -PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc); - -for (my $i = 0; $i < $MAX_NUMA; $i++) { - $confdesc->{"numa$i"} = $numadesc; -} - -my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000', 'pcnet', 'virtio', - 'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3', - 'e1000-82540em', 'e1000-82544gc', 'e1000-82545em']; +for (my $i = 0; $i < $PVE::QemuServer::Memory::MAX_NUMA; $i++) { + $confdesc->{"numa$i"} = $PVE::QemuServer::Memory::numadesc; +} + +my $nic_model_list = [ + 'e1000', + 'e1000-82540em', + 'e1000-82544gc', + 'e1000-82545em', + 'e1000e', + 'i82551', + 'i82557b', + 'i82559er', + 'ne2k_isa', + 'ne2k_pci', + 'pcnet', + 'rtl8139', + 'virtio', + 'vmxnet3', +]; my $nic_model_list_txt = join(' ', sort @$nic_model_list); my $net_fmt_bridge_descr = <<__EOD__; @@ -866,16 +901,13 @@ my $net_fmt = { default_key => 1, }, (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list), - bridge => { - type => 'string', + bridge => get_standard_option('pve-bridge-id', { description => $net_fmt_bridge_descr, - format_description => 'bridge', - pattern => '[-_.\w\d]+', optional => 1, - }, + }), queues => { type => 'integer', - minimum => 0, maximum => 16, + minimum => 0, maximum => 64, description => 'Number of packet queues to be used on the device.', optional => 1, }, @@ -969,7 +1001,8 @@ IP addresses use CIDR notation, gateways are optional but need an IP of the same The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided. -For IPv6 the special string 'auto' can be used to use stateless autoconfiguration. +For IPv6 the special string 'auto' can be used to use stateless autoconfiguration. This requires +cloud-init 19.4 or newer. If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4. @@ -986,57 +1019,42 @@ foreach my $key (keys %$confdesc_cloudinit) { $confdesc->{$key} = $confdesc_cloudinit->{$key}; } -PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path); -sub verify_volume_id_or_qm_path { - my ($volid, $noerr) = @_; +PVE::JSONSchema::register_format('pve-cpuset', \&pve_verify_cpuset); +sub pve_verify_cpuset { + my ($set_text, $noerr) = @_; - if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) { - return $volid; - } + my ($count, $members) = eval { PVE::CpuSet::parse_cpuset($set_text) }; - # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id - $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') }; if ($@) { return if $noerr; - die $@; + die "unable to parse cpuset option\n"; } - return $volid; + + return PVE::CpuSet->new($members)->short_string(); } -my $usb_fmt = { - host => { - default_key => 1, - type => 'string', format => 'pve-qm-usb-device', - format_description => 'HOSTUSBDEVICE|spice', - description => < { - optional => 1, - type => 'boolean', - description => "Specifies whether if given host option is a USB3 device or port.", - default => 0, - }, -}; + return $volid if $volid =~ m|^/|; -my $usbdesc = { - optional => 1, - type => 'string', format => $usb_fmt, - description => "Configure an USB device (n is 0 to 4).", -}; -PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc); + $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') }; + if ($@) { + return if $noerr; + die $@; + } + return $volid; +} my $serialdesc = { optional => 1, @@ -1086,8 +1104,8 @@ for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) { $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key}; } -for (my $i = 0; $i < $MAX_USB_DEVICES; $i++) { - $confdesc->{"usb$i"} = $usbdesc; +for (my $i = 0; $i < $PVE::QemuServer::USB::MAX_USB_DEVICES; $i++) { + $confdesc->{"usb$i"} = $PVE::QemuServer::USB::usbdesc; } my $boot_fmt = { @@ -1129,7 +1147,8 @@ PVE::JSONSchema::register_format('pve-qm-bootdev', \&verify_bootdev); sub verify_bootdev { my ($dev, $noerr) = @_; - return $dev if PVE::QemuServer::Drive::is_valid_drivename($dev) && $dev !~ m/^efidisk/; + my $special = $dev =~ m/^efidisk/ || $dev =~ m/^tpmstate/; + return $dev if PVE::QemuServer::Drive::is_valid_drivename($dev) && !$special; my $check = sub { my ($base) = @_; @@ -1199,7 +1218,7 @@ sub kvm_user_version { my sub extract_version { my ($machine_type, $version) = @_; $version = kvm_user_version() if !defined($version); - PVE::QemuServer::Machine::extract_version($machine_type, $version) + return PVE::QemuServer::Machine::extract_version($machine_type, $version) } sub kernel_has_vhost_net { @@ -1214,11 +1233,16 @@ sub option_exists { my $cdrom_path; sub get_cdrom_path { - return $cdrom_path if $cdrom_path; + return $cdrom_path if defined($cdrom_path); - return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom"; - return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1"; - return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2"; + $cdrom_path = first { -l $_ } map { "/dev/cdrom$_" } ('', '1', '2'); + + if (!defined($cdrom_path)) { + log_warn("no physical CD-ROM available, ignoring"); + $cdrom_path = ''; + } + + return $cdrom_path; } sub get_iso_path { @@ -1303,7 +1327,7 @@ sub parse_hotplug_features { $data = $confdesc->{hotplug}->{default} if $data eq '1'; foreach my $feature (PVE::Tools::split_list($data)) { - if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) { + if ($feature =~ m/^(network|disk|cpu|memory|usb|cloudinit)$/) { $res->{$1} = 1; } else { die "invalid hotplug feature '$feature'\n"; @@ -1323,65 +1347,19 @@ sub pve_verify_hotplug_features { die "unable to parse hotplug option\n"; } -sub scsi_inquiry { - my($fh, $noerr) = @_; - - my $SG_IO = 0x2285; - my $SG_GET_VERSION_NUM = 0x2282; - - my $versionbuf = "\x00" x 8; - my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf); - if (!$ret) { - die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr; - return; - } - my $version = unpack("I", $versionbuf); - if ($version < 30000) { - die "scsi generic interface too old\n" if !$noerr; - return; - } - - my $buf = "\x00" x 36; - my $sensebuf = "\x00" x 8; - my $cmd = pack("C x3 C x1", 0x12, 36); - - # see /usr/include/scsi/sg.h - my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I"; - - my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd), - length($sensebuf), 0, length($buf), $buf, - $cmd, $sensebuf, 6000); +sub assert_clipboard_config { + my ($vga) = @_; - $ret = ioctl($fh, $SG_IO, $packet); - if (!$ret) { - die "scsi ioctl SG_IO failed - $!\n" if !$noerr; - return; - } + my $clipboard_regex = qr/^(std|cirrus|vmware|virtio|qxl)/; - my @res = unpack($sg_io_hdr_t, $packet); - if ($res[17] || $res[18]) { - die "scsi ioctl SG_IO status error - $!\n" if !$noerr; - return; + if ( + $vga->{'clipboard'} + && $vga->{'clipboard'} eq 'vnc' + && $vga->{type} + && $vga->{type} !~ $clipboard_regex + ) { + die "vga type $vga->{type} is not compatible with VNC clipboard\n"; } - - my $res = {}; - (my $byte0, my $byte1, $res->{vendor}, - $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf); - - $res->{removable} = $byte1 & 128 ? 1 : 0; - $res->{type} = $byte0 & 31; - - return $res; -} - -sub path_is_scsi { - my ($path) = @_; - - my $fh = IO::File->new("+<$path") || return; - my $res = scsi_inquiry($fh, 1); - close($fh); - - return $res; } sub print_tabletdevice_full { @@ -1391,7 +1369,7 @@ sub print_tabletdevice_full { # we use uhci for old VMs because tablet driver was buggy in older qemu my $usbbus; - if (PVE::QemuServer::Machine::machine_type_is_q35($conf) || $arch eq 'aarch64') { + if ($q35 || $arch eq 'aarch64') { $usbbus = 'ehci'; } else { $usbbus = 'uhci'; @@ -1401,20 +1379,25 @@ sub print_tabletdevice_full { } sub print_keyboarddevice_full { - my ($conf, $arch, $machine) = @_; + my ($conf, $arch) = @_; return if $arch ne 'aarch64'; return "usb-kbd,id=keyboard,bus=ehci.0,port=2"; } +my sub get_drive_id { + my ($drive) = @_; + return "$drive->{interface}$drive->{index}"; +} + sub print_drivedevice_full { my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_; my $device = ''; my $maxdev = 0; - my $drive_id = "$drive->{interface}$drive->{index}"; + my $drive_id = get_drive_id($drive); if ($drive->{interface} eq 'virtio') { my $pciaddr = print_pci_addr("$drive_id", $bridges, $arch, $machine_type); $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}"; @@ -1423,52 +1406,52 @@ sub print_drivedevice_full { my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive); my $unit = $drive->{index} % $maxdev; - my $devicetype = 'hd'; - my $path = ''; - if (drive_is_cdrom($drive)) { - $devicetype = 'cd'; - } else { - if ($drive->{file} =~ m|^/|) { - $path = $drive->{file}; - if (my $info = path_is_scsi($path)) { - if ($info->{type} == 0 && $drive->{scsiblock}) { - $devicetype = 'block'; - } elsif ($info->{type} == 1) { # tape - $devicetype = 'generic'; - } - } - } else { - $path = PVE::Storage::path($storecfg, $drive->{file}); - } - # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380) - my $version = extract_version($machine_type, kvm_user_version()); - if ($path =~ m/^iscsi\:\/\// && - !min_version($version, 4, 1)) { - $devicetype = 'generic'; - } - } + my $machine_version = extract_version($machine_type, kvm_user_version()); + my $device_type = PVE::QemuServer::Drive::get_scsi_device_type( + $drive, $storecfg, $machine_version); - if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){ - $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit"; + if (!$conf->{scsihw} || $conf->{scsihw} =~ m/^lsi/ || $conf->{scsihw} eq 'pvscsi') { + $device = "scsi-$device_type,bus=$controller_prefix$controller.0,scsi-id=$unit"; } else { - $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0" + $device = "scsi-$device_type,bus=$controller_prefix$controller.0,channel=0,scsi-id=0" .",lun=$drive->{index}"; } $device .= ",drive=drive-$drive_id,id=$drive_id"; - if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) { + if ($drive->{ssd} && ($device_type eq 'block' || $device_type eq 'hd')) { $device .= ",rotation_rate=1"; } $device .= ",wwn=$drive->{wwn}" if $drive->{wwn}; + # only scsi-hd and scsi-cd support passing vendor and product information + if ($device_type eq 'hd' || $device_type eq 'cd') { + if (my $vendor = $drive->{vendor}) { + $device .= ",vendor=$vendor"; + } + if (my $product = $drive->{product}) { + $device .= ",product=$product"; + } + } + } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') { my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2; my $controller = int($drive->{index} / $maxdev); my $unit = $drive->{index} % $maxdev; - my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd"; - $device = "ide-$devicetype"; + # machine type q35 only supports unit=0 for IDE rather than 2 units. This wasn't handled + # correctly before, so e.g. index=2 was mapped to controller=1,unit=0 rather than + # controller=2,unit=0. Note that odd indices never worked, as they would be mapped to + # unit=1, so to keep backwards compat for migration, it suffices to keep even ones as they + # were before. Move odd ones up by 2 where they don't clash. + if (PVE::QemuServer::Machine::machine_type_is_q35($conf) && $drive->{interface} eq 'ide') { + $controller += 2 * ($unit % 2); + $unit = 0; + } + + my $device_type = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd"; + + $device = "ide-$device_type"; if ($drive->{interface} eq 'ide') { $device .= ",bus=ide.$controller,unit=$unit"; } else { @@ -1476,7 +1459,7 @@ sub print_drivedevice_full { } $device .= ",drive=drive-$drive_id,id=$drive_id"; - if ($devicetype eq 'hd') { + if ($device_type eq 'hd') { if (my $model = $drive->{model}) { $model = URI::Escape::uri_unescape($model); $device .= ",model=$model"; @@ -1518,29 +1501,65 @@ sub get_initiator_name { return $initiator; } +my sub storage_allows_io_uring_default { + my ($scfg, $cache_direct) = @_; + + # io_uring with cache mode writeback or writethrough on krbd will hang... + return if $scfg && $scfg->{type} eq 'rbd' && $scfg->{krbd} && !$cache_direct; + + # io_uring with cache mode writeback or writethrough on LVM will hang, without cache only + # sometimes, just plain disable... + return if $scfg && $scfg->{type} eq 'lvm'; + + # io_uring causes problems when used with CIFS since kernel 5.15 + # Some discussion: https://www.spinics.net/lists/linux-cifs/msg26734.html + return if $scfg && $scfg->{type} eq 'cifs'; + + return 1; +} + +my sub drive_uses_cache_direct { + my ($drive, $scfg) = @_; + + my $cache_direct = 0; + + if (my $cache = $drive->{cache}) { + $cache_direct = $cache =~ /^(?:off|none|directsync)$/; + } elsif (!drive_is_cdrom($drive) && !($scfg && $scfg->{type} eq 'btrfs' && !$scfg->{nocow})) { + $cache_direct = 1; + } + + return $cache_direct; +} + sub print_drive_commandline_full { - my ($storecfg, $vmid, $drive) = @_; + my ($storecfg, $vmid, $drive, $live_restore_name, $io_uring) = @_; my $path; my $volid = $drive->{file}; - my $format; + my $format = $drive->{format}; + my $drive_id = get_drive_id($drive); + + my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1); + my $scfg = $storeid ? PVE::Storage::storage_config($storecfg, $storeid) : undef; if (drive_is_cdrom($drive)) { $path = get_iso_path($storecfg, $vmid, $volid); + die "$drive_id: cannot back cdrom drive with a live restore image\n" if $live_restore_name; } else { - my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1); if ($storeid) { $path = PVE::Storage::path($storecfg, $volid); - my $scfg = PVE::Storage::storage_config($storecfg, $storeid); - $format = qemu_img_format($scfg, $volname); + $format //= qemu_img_format($scfg, $volname); } else { $path = $volid; - $format = "raw"; + $format //= "raw"; } } + my $is_rbd = $path =~ m/^rbd:/; + my $opts = ''; - my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard); + my @qemu_drive_options = qw(heads secs cyls trans media cache rerror werror aio discard); foreach my $o (@qemu_drive_options) { $opts .= ",$o=$drive->{$o}" if defined($drive->{$o}); } @@ -1551,6 +1570,10 @@ sub print_drive_commandline_full { $opts .= ",snapshot=$v"; } + if (defined($drive->{ro})) { # ro maps to QEMUs `readonly`, which accepts `on` or `off` only + $opts .= ",readonly=" . ($drive->{ro} ? 'on' : 'off'); + } + foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) { my ($dir, $qmpname) = @$type; if (my $v = $drive->{"mbps$dir"}) { @@ -1573,23 +1596,30 @@ sub print_drive_commandline_full { } } - $opts .= ",format=$format" if $format && !$drive->{format}; + if ($live_restore_name) { + $format = "rbd" if $is_rbd; + die "$drive_id: Proxmox Backup Server backed drive cannot auto-detect the format\n" + if !$format; + $opts .= ",format=alloc-track,file.driver=$format"; + } elsif ($format) { + $opts .= ",format=$format"; + } - my $cache_direct = 0; + my $cache_direct = drive_uses_cache_direct($drive, $scfg); - if (my $cache = $drive->{cache}) { - $cache_direct = $cache =~ /^(?:off|none|directsync)$/; - } elsif (!drive_is_cdrom($drive)) { - $opts .= ",cache=none"; - $cache_direct = 1; - } + $opts .= ",cache=none" if !$drive->{cache} && $cache_direct; - # aio native works only with O_DIRECT if (!$drive->{aio}) { - if($cache_direct) { - $opts .= ",aio=native"; + if ($io_uring && storage_allows_io_uring_default($scfg, $cache_direct)) { + # io_uring supports all cache modes + $opts .= ",aio=io_uring"; } else { - $opts .= ",aio=threads"; + # aio native works only with O_DIRECT + if($cache_direct) { + $opts .= ",aio=native"; + } else { + $opts .= ",aio=threads"; + } } } @@ -1603,16 +1633,44 @@ sub print_drive_commandline_full { # This used to be our default with discard not being specified: $detectzeroes = 'on'; } - $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes; + + # note: 'detect-zeroes' works per blockdev and we want it to persist + # after the alloc-track is removed, so put it on 'file' directly + my $dz_param = $live_restore_name ? "file.detect-zeroes" : "detect-zeroes"; + $opts .= ",$dz_param=$detectzeroes" if $detectzeroes; + } + + if ($live_restore_name) { + $opts .= ",backing=$live_restore_name"; + $opts .= ",auto-remove=on"; } - my $pathinfo = $path ? "file=$path," : ''; + # my $file_param = $live_restore_name ? "file.file.filename" : "file"; + my $file_param = "file"; + if ($live_restore_name) { + # non-rbd drivers require the underlying file to be a seperate block + # node, so add a second .file indirection + $file_param .= ".file" if !$is_rbd; + $file_param .= ".filename"; + } + my $pathinfo = $path ? "$file_param=$path," : ''; return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts"; } +sub print_pbs_blockdev { + my ($pbs_conf, $pbs_name) = @_; + my $blockdev = "driver=pbs,node-name=$pbs_name,read-only=on"; + $blockdev .= ",repository=$pbs_conf->{repository}"; + $blockdev .= ",namespace=$pbs_conf->{namespace}" if $pbs_conf->{namespace}; + $blockdev .= ",snapshot=$pbs_conf->{snapshot}"; + $blockdev .= ",archive=$pbs_conf->{archive}"; + $blockdev .= ",keyfile=$pbs_conf->{keyfile}" if $pbs_conf->{keyfile}; + return $blockdev; +} + sub print_netdevice_full { - my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_; + my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type, $machine_version) = @_; my $device = $net->{model}; if ($net->{model} eq 'virtio') { @@ -1626,7 +1684,15 @@ sub print_netdevice_full { # and out of each queue plus one config interrupt and control vector queue my $vectors = $net->{queues} * 2 + 2; $tmpstr .= ",vectors=$vectors,mq=on"; + if (min_version($machine_version, 7, 1)) { + $tmpstr .= ",packed=on"; + } + } + + if (min_version($machine_version, 7, 1) && $net->{model} eq 'virtio'){ + $tmpstr .= ",rx_queue_size=1024,tx_queue_size=256"; } + $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ; if (my $mtu = $net->{mtu}) { @@ -1651,6 +1717,8 @@ sub print_netdevice_full { $romfile = 'pxe-virtio.rom'; } elsif ($device eq 'e1000') { $romfile = 'pxe-e1000.rom'; + } elsif ($device eq 'e1000e') { + $romfile = 'pxe-e1000e.rom'; } elsif ($device eq 'ne2k') { $romfile = 'pxe-ne2k_pci.rom'; } elsif ($device eq 'pcnet') { @@ -1681,7 +1749,7 @@ sub print_netdev_full { if length($ifname) >= 16; my $vhostparam = ''; - if (is_native($arch)) { + if (is_native_arch($arch)) { $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio'; } @@ -1707,6 +1775,7 @@ my $vga_map = { 'std' => 'VGA', 'vmware' => 'vmware-svga', 'virtio' => 'virtio-vga', + 'virtio-gl' => 'virtio-vga-gl', }; sub print_vga_device { @@ -1730,11 +1799,11 @@ sub print_vga_device { } } - die "no devicetype for $vga->{type}\n" if !$type; + die "no device-type for $vga->{type}\n" if !$type; my $memory = ""; if ($vgamem_mb) { - if ($vga->{type} eq 'virtio') { + if ($vga->{type} =~ /^virtio/) { my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b"); $memory = ",max_hostmem=$bytes"; } elsif ($qxlnum) { @@ -1758,7 +1827,6 @@ sub print_vga_device { my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf); my $vgaid = "vga" . ($id // ''); my $pciaddr; - if ($q35 && $vgaid eq 'vga') { # the first display uses pcie.0 bus on q35 machines $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine); @@ -1766,42 +1834,28 @@ sub print_vga_device { $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine); } - return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}"; -} + if ($vga->{type} eq 'virtio-gl') { + my $base = '/usr/lib/x86_64-linux-gnu/lib'; + die "missing libraries for '$vga->{type}' detected! Please install 'libgl1' and 'libegl1'\n" + if !-e "${base}EGL.so.1" || !-e "${base}GL.so.1"; -sub parse_number_sets { - my ($set) = @_; - my $res = []; - foreach my $part (split(/;/, $set)) { - if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) { - die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1; - push @$res, [ $1, $2 ]; - } else { - die "invalid range: $part\n"; - } + die "no DRM render node detected (/dev/dri/renderD*), no GPU? - needed for '$vga->{type}' display\n" + if !PVE::Tools::dir_glob_regex('/dev/dri/', "renderD.*"); } - return $res; -} -sub parse_numa { - my ($data) = @_; - - my $res = parse_property_string($numa_fmt, $data); - $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus}); - $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes}); - return $res; + return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}"; } # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate= sub parse_net { - my ($data) = @_; + my ($data, $disable_mac_autogen) = @_; my $res = eval { parse_property_string($net_fmt, $data) }; if ($@) { warn $@; return; } - if (!defined($res->{macaddr})) { + if (!defined($res->{macaddr}) && !$disable_mac_autogen) { my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg'); $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}); } @@ -1880,6 +1934,7 @@ sub vmconfig_register_unused_drive { if (drive_is_cloudinit($drive)) { eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) }; warn $@ if $@; + delete $conf->{cloudinit}; } elsif (!drive_is_cdrom($drive)) { my $volid = $drive->{file}; if (vm_is_volid_owner($storecfg, $vmid, $volid)) { @@ -1972,11 +2027,11 @@ sub parse_watchdog { } sub parse_guest_agent { - my ($value) = @_; + my ($conf) = @_; - return {} if !defined($value->{agent}); + return {} if !defined($conf->{agent}); - my $res = eval { parse_property_string($agent_fmt, $value->{agent}) }; + my $res = eval { parse_property_string($agent_fmt, $conf->{agent}) }; warn $@ if $@; # if the agent is disabled ignore the other potentially set properties @@ -1984,6 +2039,14 @@ sub parse_guest_agent { return $res; } +sub get_qga_key { + my ($conf, $key) = @_; + return undef if !defined($conf->{agent}); + + my $agent = parse_guest_agent($conf); + return $agent->{$key}; +} + sub parse_vga { my ($value) = @_; @@ -2003,44 +2066,135 @@ sub parse_rng { return $res; } -PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device); -sub verify_usb_device { - my ($value, $noerr) = @_; +sub parse_meta_info { + my ($value) = @_; - return $value if parse_usb_device($value); + return if !$value; - return if $noerr; + my $res = eval { parse_property_string($meta_info_fmt, $value) }; + warn $@ if $@; + return $res; +} + +sub new_meta_info_string { + my () = @_; # for now do not allow to override any value + + return PVE::JSONSchema::print_property_string( + { + 'creation-qemu' => kvm_user_version(), + ctime => "". int(time()), + }, + $meta_info_fmt + ); +} + +sub qemu_created_version_fixups { + my ($conf, $forcemachine, $kvmver) = @_; - die "unable to parse usb device\n"; + my $meta = parse_meta_info($conf->{meta}) // {}; + my $forced_vers = PVE::QemuServer::Machine::extract_version($forcemachine); + + # check if we need to apply some handling for VMs that always use the latest machine version but + # had a machine version transition happen that affected HW such that, e.g., an OS config change + # would be required (we do not want to pin machine version for non-windows OS type) + my $machine_conf = PVE::QemuServer::Machine::parse_machine($conf->{machine}); + if ( + (!defined($machine_conf->{type}) || $machine_conf->{type} =~ m/^(?:pc|q35|virt)$/) # non-versioned machine + && (!defined($meta->{'creation-qemu'}) || !min_version($meta->{'creation-qemu'}, 6, 1)) # created before 6.1 + && (!$forced_vers || min_version($forced_vers, 6, 1)) # handle snapshot-rollback/migrations + && min_version($kvmver, 6, 1) # only need to apply the change since 6.1 + ) { + my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf); + if ($q35 && $conf->{ostype} && $conf->{ostype} eq 'l26') { + # this changed to default-on in Q 6.1 for q35 machines, it will mess with PCI slot view + # and thus with the predictable interface naming of systemd + return ['-global', 'ICH9-LPC.acpi-pci-hotplug-with-bridge-support=off']; + } + } + return; } # add JSON properties for create and set function sub json_config_properties { - my $prop = shift; + my ($prop, $with_disk_alloc) = @_; + + my $skip_json_config_opts = { + parent => 1, + snaptime => 1, + vmstate => 1, + runningmachine => 1, + runningcpu => 1, + meta => 1, + }; foreach my $opt (keys %$confdesc) { - next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' || - $opt eq 'runningmachine' || $opt eq 'runningcpu'; - $prop->{$opt} = $confdesc->{$opt}; + next if $skip_json_config_opts->{$opt}; + + if ($with_disk_alloc && is_valid_drivename($opt)) { + $prop->{$opt} = $PVE::QemuServer::Drive::drivedesc_hash_with_alloc->{$opt}; + } else { + $prop->{$opt} = $confdesc->{$opt}; + } } return $prop; } -# return copy of $confdesc_cloudinit to generate documentation -sub cloudinit_config_properties { - - return dclone($confdesc_cloudinit); -} - -sub check_type { - my ($key, $value) = @_; - - die "unknown setting '$key'\n" if !$confdesc->{$key}; +# Properties that we can read from an OVF file +sub json_ovf_properties { + my $prop = {}; - my $type = $confdesc->{$key}->{type}; + for my $device (PVE::QemuServer::Drive::valid_drive_names()) { + $prop->{$device} = { + type => 'string', + format => 'pve-volume-id-or-absolute-path', + description => "Disk image that gets imported to $device", + optional => 1, + }; + } - if (!defined($value)) { + $prop->{cores} = { + type => 'integer', + description => "The number of CPU cores.", + optional => 1, + }; + $prop->{memory} = { + type => 'integer', + description => "Amount of RAM for the VM in MB.", + optional => 1, + }; + $prop->{name} = { + type => 'string', + description => "Name of the VM.", + optional => 1, + }; + + return $prop; +} + +# return copy of $confdesc_cloudinit to generate documentation +sub cloudinit_config_properties { + + return dclone($confdesc_cloudinit); +} + +sub cloudinit_pending_properties { + my $p = { + map { $_ => 1 } keys $confdesc_cloudinit->%*, + name => 1, + }; + $p->{"net$_"} = 1 for 0..($MAX_NETS-1); + return $p; +} + +sub check_type { + my ($key, $value) = @_; + + die "unknown setting '$key'\n" if !$confdesc->{$key}; + + my $type = $confdesc->{$key}->{type}; + + if (!defined($value)) { die "got undefined value\n"; } @@ -2071,15 +2225,17 @@ sub check_type { } sub destroy_vm { - my ($storecfg, $vmid, $skiplock, $replacement_conf) = @_; + my ($storecfg, $vmid, $skiplock, $replacement_conf, $purge_unreferenced) = @_; my $conf = PVE::QemuConfig->load_config($vmid); - PVE::QemuConfig->check_lock($conf) if !$skiplock; + if (!$skiplock && !PVE::QemuConfig->has_lock($conf, 'suspended')) { + PVE::QemuConfig->check_lock($conf); + } if ($conf->{template}) { # check if any base image is still used by a linked clone - PVE::QemuConfig->foreach_volume($conf, sub { + PVE::QemuConfig->foreach_volume_full($conf, { include_unused => 1 }, sub { my ($ds, $drive) = @_; return if drive_is_cdrom($drive); @@ -2092,28 +2248,50 @@ sub destroy_vm { }); } - # only remove disks owned by this VM - PVE::QemuConfig->foreach_volume($conf, sub { + my $volids = {}; + my $remove_owned_drive = sub { my ($ds, $drive) = @_; return if drive_is_cdrom($drive, 1); my $volid = $drive->{file}; return if !$volid || $volid =~ m|^/|; + return if $volids->{$volid}; my ($path, $owner) = PVE::Storage::path($storecfg, $volid); return if !$path || !$owner || ($owner != $vmid); + $volids->{$volid} = 1; eval { PVE::Storage::vdisk_free($storecfg, $volid) }; warn "Could not remove disk '$volid', check manually: $@" if $@; - }); + }; - # also remove unused disk - my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid); - PVE::Storage::foreach_volid($vmdisks, sub { - my ($volid, $sid, $volname, $d) = @_; - eval { PVE::Storage::vdisk_free($storecfg, $volid) }; - warn $@ if $@; - }); + # only remove disks owned by this VM (referenced in the config) + my $include_opts = { + include_unused => 1, + extra_keys => ['vmstate'], + }; + PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $remove_owned_drive); + + for my $snap (values %{$conf->{snapshots}}) { + next if !defined($snap->{vmstate}); + my $drive = PVE::QemuConfig->parse_volume('vmstate', $snap->{vmstate}, 1); + next if !defined($drive); + $remove_owned_drive->('vmstate', $drive); + } + + PVE::QemuConfig->foreach_volume_full($conf->{pending}, $include_opts, $remove_owned_drive); + + if ($purge_unreferenced) { # also remove unreferenced disk + my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid, undef, 'images'); + PVE::Storage::foreach_volid($vmdisks, sub { + my ($volid, $sid, $volname, $d) = @_; + eval { PVE::Storage::vdisk_free($storecfg, $volid) }; + warn $@ if $@; + }); + } + + eval { delete_ifaces_ipams_ips($conf, $vmid)}; + warn $@ if $@; if (defined $replacement_conf) { PVE::QemuConfig->write_config($vmid, $replacement_conf); @@ -2123,7 +2301,7 @@ sub destroy_vm { } sub parse_vm_config { - my ($filename, $raw) = @_; + my ($filename, $raw, $strict) = @_; return if !defined($raw); @@ -2131,6 +2309,17 @@ sub parse_vm_config { digest => Digest::SHA::sha1_hex($raw), snapshots => {}, pending => {}, + cloudinit => {}, + }; + + my $handle_error = sub { + my ($msg) = @_; + + if ($strict) { + die $msg; + } else { + warn $msg; + } }; $filename =~ m|/qemu-server/(\d+)\.conf$| @@ -2140,6 +2329,13 @@ sub parse_vm_config { my $conf = $res; my $descr; + my $finish_description = sub { + if (defined($descr)) { + $descr =~ s/\s+$//; + $conf->{description} = $descr; + } + $descr = undef; + }; my $section = ''; my @lines = split(/\n/, $raw); @@ -2148,26 +2344,23 @@ sub parse_vm_config { if ($line =~ m/^\[PENDING\]\s*$/i) { $section = 'pending'; - if (defined($descr)) { - $descr =~ s/\s+$//; - $conf->{description} = $descr; - } - $descr = undef; + $finish_description->(); + $conf = $res->{$section} = {}; + next; + } elsif ($line =~ m/^\[special:cloudinit\]\s*$/i) { + $section = 'cloudinit'; + $finish_description->(); $conf = $res->{$section} = {}; next; } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) { $section = $1; - if (defined($descr)) { - $descr =~ s/\s+$//; - $conf->{description} = $descr; - } - $descr = undef; + $finish_description->(); $conf = $res->{snapshots}->{$section} = {}; next; } - if ($line =~ m/^\#(.*)\s*$/) { + if ($line =~ m/^\#(.*)$/) { $descr = '' if !defined($descr); $descr .= PVE::Tools::decode_text($1) . "\n"; next; @@ -2187,14 +2380,19 @@ sub parse_vm_config { if ($section eq 'pending') { $conf->{delete} = $value; # we parse this later } else { - warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n"; + $handle_error->("vm $vmid - property 'delete' is only allowed in [PENDING]\n"); } } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) { my $key = $1; my $value = $2; + if ($section eq 'cloudinit') { + # ignore validation only used for informative purpose + $conf->{$key} = $value; + next; + } eval { $value = check_type($key, $value); }; if ($@) { - warn "vm $vmid - unable to parse value of '$key' - $@"; + $handle_error->("vm $vmid - unable to parse value of '$key' - $@"); } else { $key = 'ide2' if $key eq 'cdrom'; my $fmt = $confdesc->{$key}->{format}; @@ -2204,20 +2402,19 @@ sub parse_vm_config { $v->{file} = $volid; $value = print_drive($v); } else { - warn "vm $vmid - unable to parse value of '$key'\n"; + $handle_error->("vm $vmid - unable to parse value of '$key'\n"); next; } } $conf->{$key} = $value; } + } else { + $handle_error->("vm $vmid - unable to parse config: $line\n"); } } - if (defined($descr)) { - $descr =~ s/\s+$//; - $conf->{description} = $descr; - } + $finish_description->(); delete $res->{snapstate}; # just to be sure return $res; @@ -2250,7 +2447,7 @@ sub write_vm_config { foreach my $key (keys %$cref) { next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' || - $key eq 'snapstate' || $key eq 'pending'; + $key eq 'snapstate' || $key eq 'pending' || $key eq 'cloudinit'; my $value = $cref->{$key}; if ($key eq 'delete') { die "propertry 'delete' is only allowed in [PENDING]\n" @@ -2304,7 +2501,7 @@ sub write_vm_config { } foreach my $key (sort keys %$conf) { - next if $key =~ /^(digest|description|pending|snapshots)$/; + next if $key =~ /^(digest|description|pending|cloudinit|snapshots)$/; $raw .= "$key: $conf->{$key}\n"; } return $raw; @@ -2317,6 +2514,11 @@ sub write_vm_config { $raw .= &$generate_raw_config($conf->{pending}, 1); } + if (scalar(keys %{$conf->{cloudinit}}) && PVE::QemuConfig->has_cloudinit($conf)){ + $raw .= "\n[special:cloudinit]\n"; + $raw .= &$generate_raw_config($conf->{cloudinit}); + } + foreach my $snapname (sort keys %{$conf->{snapshots}}) { $raw .= "\n[$snapname]\n"; $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname}); @@ -2360,6 +2562,28 @@ sub check_local_resources { my ($conf, $noerr) = @_; my @loc_res = (); + my $mapped_res = []; + + my $nodelist = PVE::Cluster::get_nodelist(); + my $pci_map = PVE::Mapping::PCI::config(); + my $usb_map = PVE::Mapping::USB::config(); + + my $missing_mappings_by_node = { map { $_ => [] } @$nodelist }; + + my $add_missing_mapping = sub { + my ($type, $key, $id) = @_; + for my $node (@$nodelist) { + my $entry; + if ($type eq 'pci') { + $entry = PVE::Mapping::PCI::get_node_mapping($pci_map, $id, $node); + } elsif ($type eq 'usb') { + $entry = PVE::Mapping::USB::get_node_mapping($usb_map, $id, $node); + } + if (!scalar($entry->@*)) { + push @{$missing_mappings_by_node->{$node}}, $key; + } + } + }; push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax @@ -2367,7 +2591,21 @@ sub check_local_resources { push @loc_res, "ivshmem" if $conf->{ivshmem}; foreach my $k (keys %$conf) { - next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/); + if ($k =~ m/^usb/) { + my $entry = parse_property_string('pve-qm-usb', $conf->{$k}); + next if $entry->{host} && $entry->{host} =~ m/^spice$/i; + if ($entry->{mapping}) { + $add_missing_mapping->('usb', $k, $entry->{mapping}); + push @$mapped_res, $k; + } + } + if ($k =~ m/^hostpci/) { + my $entry = parse_property_string('pve-qm-hostpci', $conf->{$k}); + if ($entry->{mapping}) { + $add_missing_mapping->('pci', $k, $entry->{mapping}); + push @$mapped_res, $k; + } + } # sockets are safe: they will recreated be on the target side post-migrate next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket'); push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/; @@ -2375,7 +2613,7 @@ sub check_local_resources { die "VM uses local resources\n" if scalar @loc_res && !$noerr; - return \@loc_res; + return wantarray ? (\@loc_res, $mapped_res, $missing_mappings_by_node) : \@loc_res; } # check if used storages are available on all nodes (use by migrate) @@ -2392,8 +2630,13 @@ sub check_storage_availability { return if !$sid; # check if storage is available on both nodes - my $scfg = PVE::Storage::storage_check_node($storecfg, $sid); - PVE::Storage::storage_check_node($storecfg, $sid, $node); + my $scfg = PVE::Storage::storage_check_enabled($storecfg, $sid); + PVE::Storage::storage_check_enabled($storecfg, $sid, $node); + + my ($vtype) = PVE::Storage::parse_volname($storecfg, $volid); + + die "$volid: content type '$vtype' is not available on storage '$sid'\n" + if !$scfg->{content}->{$vtype}; }); } @@ -2474,6 +2717,12 @@ sub check_local_storage_availability { sub check_running { my ($vmid, $nocheck, $node) = @_; + # $nocheck is set when called during a migration, in which case the config + # file might still or already reside on the *other* node + # - because rename has already happened, and current node is source + # - because rename hasn't happened yet, and current node is target + # - because rename has happened, current node is target, but hasn't yet + # processed it yet PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck; return PVE::QemuServer::Helpers::vm_running_locally($vmid); } @@ -2499,7 +2748,7 @@ sub vzlist { our $vmstatus_return_properties = { vmid => get_standard_option('pve-vmid'), status => { - description => "Qemu process status.", + description => "QEMU process status.", type => 'string', enum => ['stopped', 'running'], }, @@ -2521,7 +2770,7 @@ our $vmstatus_return_properties = { optional => 1, }, qmpstatus => { - description => "Qemu QMP agent status.", + description => "VM run state from the 'query-status' QMP monitor command.", type => 'string', optional => 1, }, @@ -2551,6 +2800,16 @@ our $vmstatus_return_properties = { type => 'string', optional => 1, }, + 'running-machine' => { + description => "The currently running machine type (if running).", + type => 'string', + optional => 1, + }, + 'running-qemu' => { + description => "The currently running QEMU version (if running).", + type => 'string', + optional => 1, + }, }; my $last_proc_pid_stat; @@ -2577,8 +2836,8 @@ sub vmstatus { my $conf = PVE::QemuConfig->load_config($vmid); - my $d = { vmid => $vmid }; - $d->{pid} = $list->{$vmid}->{pid}; + my $d = { vmid => int($vmid) }; + $d->{pid} = int($list->{$vmid}->{pid}) if $list->{$vmid}->{pid}; # fixme: better status? $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped'; @@ -2598,8 +2857,7 @@ sub vmstatus { $d->{cpus} = $conf->{vcpus} if $conf->{vcpus}; $d->{name} = $conf->{name} || "VM $vmid"; - $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024) - : $defaults->{memory}*(1024*1024); + $d->{maxmem} = get_current_memory($conf->{memory})*(1024*1024); if ($conf->{balloon}) { $d->{balloon_min} = $conf->{balloon}*(1024*1024); @@ -2617,7 +2875,7 @@ sub vmstatus { $d->{diskread} = 0; $d->{diskwrite} = 0; - $d->{template} = PVE::QemuConfig->is_template($conf); + $d->{template} = 1 if PVE::QemuConfig->is_template($conf); $d->{serial} = 1 if conf_has_serial($conf); $d->{lock} = $conf->{lock} if $conf->{lock}; @@ -2637,8 +2895,8 @@ sub vmstatus { $d->{netin} += $netdev->{$dev}->{transmit}; if ($full) { - $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive}; - $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit}; + $d->{nics}->{$dev}->{netout} = int($netdev->{$dev}->{receive}); + $d->{nics}->{$dev}->{netin} = int($netdev->{$dev}->{transmit}); } } @@ -2729,10 +2987,32 @@ sub vmstatus { $res->{$vmid}->{diskwrite} = $totalwrbytes; }; + my $machinecb = sub { + my ($vmid, $resp) = @_; + my $data = $resp->{'return'} || []; + + $res->{$vmid}->{'running-machine'} = + PVE::QemuServer::Machine::current_from_query_machines($data); + }; + + my $versioncb = sub { + my ($vmid, $resp) = @_; + my $data = $resp->{'return'} // {}; + my $version = 'unknown'; + + if (my $v = $data->{qemu}) { + $version = $v->{major} . "." . $v->{minor} . "." . $v->{micro}; + } + + $res->{$vmid}->{'running-qemu'} = $version; + }; + my $statuscb = sub { my ($vmid, $resp) = @_; $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats'); + $qmpclient->queue_cmd($vmid, $machinecb, 'query-machines'); + $qmpclient->queue_cmd($vmid, $versioncb, 'query-version'); # this fails if ballon driver is not loaded, so this must be # the last commnand (following command are aborted if this fails). $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon'); @@ -2754,6 +3034,16 @@ sub vmstatus { $qmpclient->queue_execute(undef, 2); + foreach my $vmid (keys %$list) { + next if $opt_vmid && ($vmid ne $opt_vmid); + next if !$res->{$vmid}->{pid}; #not running + + # we can't use the $qmpclient since it might have already aborted on + # 'query-balloon', but this might also fail for older versions... + my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") }; + $res->{$vmid}->{'proxmox-support'} = $qemu_support // {}; + } + foreach my $vmid (keys %$list) { next if $opt_vmid && ($vmid ne $opt_vmid); $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus}; @@ -2818,6 +3108,101 @@ sub audio_devs { return $devs; } +sub get_tpm_paths { + my ($vmid) = @_; + return { + socket => "/var/run/qemu-server/$vmid.swtpm", + pid => "/var/run/qemu-server/$vmid.swtpm.pid", + }; +} + +sub add_tpm_device { + my ($vmid, $devices, $conf) = @_; + + return if !$conf->{tpmstate0}; + + my $paths = get_tpm_paths($vmid); + + push @$devices, "-chardev", "socket,id=tpmchar,path=$paths->{socket}"; + push @$devices, "-tpmdev", "emulator,id=tpmdev,chardev=tpmchar"; + push @$devices, "-device", "tpm-tis,tpmdev=tpmdev"; +} + +sub start_swtpm { + my ($storecfg, $vmid, $tpmdrive, $migration) = @_; + + return if !$tpmdrive; + + my $state; + my $tpm = parse_drive("tpmstate0", $tpmdrive); + my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1); + if ($storeid) { + $state = PVE::Storage::map_volume($storecfg, $tpm->{file}); + } else { + $state = $tpm->{file}; + } + + my $paths = get_tpm_paths($vmid); + + # during migration, we will get state from remote + # + if (!$migration) { + # run swtpm_setup to create a new TPM state if it doesn't exist yet + my $setup_cmd = [ + "swtpm_setup", + "--tpmstate", + "file://$state", + "--createek", + "--create-ek-cert", + "--create-platform-cert", + "--lock-nvram", + "--config", + "/etc/swtpm_setup.conf", # do not use XDG configs + "--runas", + "0", # force creation as root, error if not possible + "--not-overwrite", # ignore existing state, do not modify + ]; + + push @$setup_cmd, "--tpm2" if $tpm->{version} eq 'v2.0'; + # TPM 2.0 supports ECC crypto, use if possible + push @$setup_cmd, "--ecc" if $tpm->{version} eq 'v2.0'; + + run_command($setup_cmd, outfunc => sub { + print "swtpm_setup: $1\n"; + }); + } + + # Used to distinguish different invocations in the log. + my $log_prefix = "[id=" . int(time()) . "] "; + + my $emulator_cmd = [ + "swtpm", + "socket", + "--tpmstate", + "backend-uri=file://$state,mode=0600", + "--ctrl", + "type=unixio,path=$paths->{socket},mode=0600", + "--pid", + "file=$paths->{pid}", + "--terminate", # terminate on QEMU disconnect + "--daemon", + "--log", + "file=/run/qemu-server/$vmid-swtpm.log,level=1,prefix=$log_prefix", + ]; + push @$emulator_cmd, "--tpm2" if $tpm->{version} eq 'v2.0'; + run_command($emulator_cmd, outfunc => sub { print $1; }); + + my $tries = 100; # swtpm may take a bit to start before daemonizing, wait up to 5s for pid + while (! -e $paths->{pid}) { + die "failed to start swtpm: pid file '$paths->{pid}' wasn't created.\n" if --$tries == 0; + usleep(50_000); + } + + # return untainted PID of swtpm daemon so it can be killed on error + file_read_firstline($paths->{pid}) =~ m/(\d+)/; + return $1; +} + sub vga_conf_has_spice { my ($vga) = @_; @@ -2828,11 +3213,6 @@ sub vga_conf_has_spice { return $1 || 1; } -sub is_native($) { - my ($arch) = @_; - return get_host_arch() eq $arch; -} - sub get_vm_arch { my ($conf) = @_; return $conf->{arch} // get_host_arch(); @@ -2843,37 +3223,92 @@ my $default_machines = { aarch64 => 'virt', }; +sub get_installed_machine_version { + my ($kvmversion) = @_; + $kvmversion = kvm_user_version() if !defined($kvmversion); + $kvmversion =~ m/^(\d+\.\d+)/; + return $1; +} + +sub windows_get_pinned_machine_version { + my ($machine, $base_version, $kvmversion) = @_; + + my $pin_version = $base_version; + if (!defined($base_version) || + !PVE::QemuServer::Machine::can_run_pve_machine_version($base_version, $kvmversion) + ) { + $pin_version = get_installed_machine_version($kvmversion); + } + if (!$machine || $machine eq 'pc') { + $machine = "pc-i440fx-$pin_version"; + } elsif ($machine eq 'q35') { + $machine = "pc-q35-$pin_version"; + } elsif ($machine eq 'virt') { + $machine = "virt-$pin_version"; + } else { + warn "unknown machine type '$machine', not touching that!\n"; + } + + return $machine; +} + sub get_vm_machine { my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_; - my $machine = $forcemachine || $conf->{machine}; + my $machine_conf = PVE::QemuServer::Machine::parse_machine($conf->{machine}); + my $machine = $forcemachine || $machine_conf->{type}; if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) { + $kvmversion //= kvm_user_version(); + # we must pin Windows VMs without a specific version to 5.1, as 5.2 fixed a bug in ACPI + # layout which confuses windows quite a bit and may result in various regressions.. + # see: https://lists.gnu.org/archive/html/qemu-devel/2021-02/msg08484.html + if (windows_version($conf->{ostype})) { + $machine = windows_get_pinned_machine_version($machine, '5.1', $kvmversion); + } $arch //= 'x86_64'; $machine ||= $default_machines->{$arch}; if ($add_pve_version) { - $kvmversion //= kvm_user_version(); my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion); $machine .= "+pve$pvever"; } } - if ($add_pve_version && $machine !~ m/\+pve\d+$/) { + if ($add_pve_version && $machine !~ m/\+pve\d+?(?:\.pxe)?$/) { + my $is_pxe = $machine =~ m/^(.*?)\.pxe$/; + $machine = $1 if $is_pxe; + # for version-pinned machines that do not include a pve-version (e.g. # pc-q35-4.1), we assume 0 to keep them stable in case we bump $machine .= '+pve0'; + + $machine .= '.pxe' if $is_pxe; } return $machine; } -sub get_ovmf_files($) { - my ($arch) = @_; +sub get_ovmf_files($$$) { + my ($arch, $efidisk, $smm) = @_; - my $ovmf = $OVMF->{$arch} + my $types = $OVMF->{$arch} or die "no OVMF images known for architecture '$arch'\n"; - return @$ovmf; + my $type = 'default'; + if ($arch eq 'x86_64') { + if (defined($efidisk->{efitype}) && $efidisk->{efitype} eq '4m') { + $type = $smm ? "4m" : "4m-no-smm"; + $type .= '-ms' if $efidisk->{'pre-enrolled-keys'}; + } else { + # TODO: log_warn about use of legacy images for x86_64 with Promxox VE 9 + } + } + + my ($ovmf_code, $ovmf_vars) = $types->{$type}->@*; + die "EFI base image '$ovmf_code' not found\n" if ! -f $ovmf_code; + die "EFI vars image '$ovmf_vars' not found\n" if ! -f $ovmf_vars; + + return ($ovmf_code, $ovmf_vars); } my $Arch2Qemu = { @@ -2882,7 +3317,7 @@ my $Arch2Qemu = { }; sub get_command_for_arch($) { my ($arch) = @_; - return '/usr/bin/kvm' if is_native($arch); + return '/usr/bin/kvm' if is_native_arch($arch); my $cmd = $Arch2Qemu->{$arch} or die "don't know how to emulate architecture '$arch'\n"; @@ -2936,7 +3371,7 @@ sub query_supported_cpu_flags { $qemu_cmd, '-machine', $default_machine, '-display', 'none', - '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server,nowait", + '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server=on,wait=off", '-mon', 'chardev=qmp,mode=control', '-pidfile', $pidfile, '-S', '-daemonize' @@ -2969,8 +3404,7 @@ sub query_supported_cpu_flags { }; my $err = $@; - # force stop with 10 sec timeout and 'nocheck' - # always stop, even if QMP failed + # force stop with 10 sec timeout and 'nocheck', always stop, even if QMP failed vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1); die $err if $err; @@ -3008,21 +3442,71 @@ sub query_understood_cpu_flags { return \@flags; } +# Since commit 277d33454f77ec1d1e0bc04e37621e4dd2424b67 in pve-qemu, smm is not off by default +# anymore. But smm=off seems to be required when using SeaBIOS and serial display. +my sub should_disable_smm { + my ($conf, $vga, $machine) = @_; + + return if $machine =~ m/^virt/; # there is no smm flag that could be disabled + + return (!defined($conf->{bios}) || $conf->{bios} eq 'seabios') && + $vga->{type} && $vga->{type} =~ m/^(serial\d+|none)$/; +} + +my sub print_ovmf_drive_commandlines { + my ($conf, $storecfg, $vmid, $arch, $q35, $version_guard) = @_; + + my $d = $conf->{efidisk0} ? parse_drive('efidisk0', $conf->{efidisk0}) : undef; + + my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch, $d, $q35); + + my $var_drive_str = "if=pflash,unit=1,id=drive-efidisk0"; + if ($d) { + my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1); + my ($path, $format) = $d->@{'file', 'format'}; + if ($storeid) { + $path = PVE::Storage::path($storecfg, $d->{file}); + if (!defined($format)) { + my $scfg = PVE::Storage::storage_config($storecfg, $storeid); + $format = qemu_img_format($scfg, $volname); + } + } elsif (!defined($format)) { + die "efidisk format must be specified\n"; + } + # SPI flash does lots of read-modify-write OPs, without writeback this gets really slow #3329 + if ($path =~ m/^rbd:/) { + $var_drive_str .= ',cache=writeback'; + $path .= ':rbd_cache_policy=writeback'; # avoid write-around, we *need* to cache writes too + } + $var_drive_str .= ",format=$format,file=$path"; + + $var_drive_str .= ",size=" . (-s $ovmf_vars) if $format eq 'raw' && $version_guard->(4, 1, 2); + $var_drive_str .= ',readonly=on' if drive_is_read_only($conf, $d); + } else { + log_warn("no efidisk configured! Using temporary efivars disk."); + my $path = "/tmp/$vmid-ovmf.fd"; + PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars); + $var_drive_str .= ",format=raw,file=$path"; + $var_drive_str .= ",size=" . (-s $ovmf_vars) if $version_guard->(4, 1, 2); + } + + return ("if=pflash,unit=0,format=raw,readonly=on,file=$ovmf_code", $var_drive_str); +} + sub config_to_command { - my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu) = @_; + my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu, + $live_restore_backing) = @_; - my $cmd = []; - my $globalFlags = []; - my $machineFlags = []; - my $rtcFlags = []; + my ($globalFlags, $machineFlags, $rtcFlags) = ([], [], []); my $devices = []; - my $pciaddr = ''; my $bridges = {}; my $ostype = $conf->{ostype}; my $winversion = windows_version($ostype); my $kvm = $conf->{kvm}; my $nodename = nodename(); + my $machine_conf = PVE::QemuServer::Machine::parse_machine($conf->{machine}); + my $arch = get_vm_arch($conf); my $kvm_binary = get_command_for_arch($arch); my $kvmver = kvm_user_version($kvm_binary); @@ -3036,7 +3520,7 @@ sub config_to_command { my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version); my $machine_version = extract_version($machine_type, $kvmver); - $kvm //= 1 if is_native($arch); + $kvm //= 1 if is_native_arch($arch); $machine_version =~ m/(\d+)\.(\d+)/; my ($machine_major, $machine_minor) = ($1, $2); @@ -3076,8 +3560,10 @@ sub config_to_command { my $use_old_bios_files = undef; ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type); - my $cpuunits = defined($conf->{cpuunits}) ? - $conf->{cpuunits} : $defaults->{cpuunits}; + my $cmd = []; + if ($conf->{affinity}) { + push @$cmd, '/usr/bin/taskset', '--cpu-list', '--all-tasks', $conf->{affinity}; + } push @$cmd, $kvm_binary; @@ -3085,12 +3571,14 @@ sub config_to_command { my $vmname = $conf->{name} || "vm$vmid"; - push @$cmd, '-name', $vmname; + push @$cmd, '-name', "$vmname,debug-threads=on"; + + push @$cmd, '-no-shutdown'; my $use_virtio = 0; my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid); - push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait"; + push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server=on,wait=off"; push @$cmd, '-mon', "chardev=qmp,mode=control"; if (min_version($machine_version, 2, 12)) { @@ -3126,44 +3614,16 @@ sub config_to_command { } if ($conf->{bios} && $conf->{bios} eq 'ovmf') { - my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch); - die "uefi base image '$ovmf_code' not found\n" if ! -f $ovmf_code; - - my ($path, $format); - if (my $efidisk = $conf->{efidisk0}) { - my $d = parse_drive('efidisk0', $efidisk); - my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1); - $format = $d->{format}; - if ($storeid) { - $path = PVE::Storage::path($storecfg, $d->{file}); - if (!defined($format)) { - my $scfg = PVE::Storage::storage_config($storecfg, $storeid); - $format = qemu_img_format($scfg, $volname); - } - } else { - $path = $d->{file}; - die "efidisk format must be specified\n" - if !defined($format); - } - } else { - warn "no efidisk configured! Using temporary efivars disk.\n"; - $path = "/tmp/$vmid-ovmf.fd"; - PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars); - $format = 'raw'; - } + die "OVMF (UEFI) BIOS is not supported on 32-bit CPU types\n" + if !$forcecpu && get_cpu_bitness($conf->{cpu}, $arch) == 32; - my $size_str = ""; - - if ($format eq 'raw' && $version_guard->(4, 1, 2)) { - $size_str = ",size=" . (-s $ovmf_vars); - } - - push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code"; - push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0$size_str,file=$path"; + my ($code_drive_str, $var_drive_str) = + print_ovmf_drive_commandlines($conf, $storecfg, $vmid, $arch, $q35, $version_guard); + push $cmd->@*, '-drive', $code_drive_str; + push $cmd->@*, '-drive', $var_drive_str; } - # load q35 config - if ($q35) { + if ($q35) { # tell QEMU to load q35 config early # we use different pcie-port hardware for qemu >= 4.0 for passthrough if (min_version($machine_version, 4, 0)) { push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg'; @@ -3172,13 +3632,17 @@ sub config_to_command { } } + if (defined(my $fixups = qemu_created_version_fixups($conf, $forcemachine, $kvmver))) { + push @$cmd, $fixups->@*; + } + if ($conf->{vmgenid}) { push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid}; } # add usb controllers my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers( - $conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES); + $conf, $bridges, $arch, $machine_type, $machine_version); push @$devices, @usbcontrollers if @usbcontrollers; my $vga = parse_vga($conf->{vga}); @@ -3196,10 +3660,8 @@ sub config_to_command { } # enable absolute mouse coordinates (needed by vnc) - my $tablet; - if (defined($conf->{tablet})) { - $tablet = $conf->{tablet}; - } else { + my $tablet = $conf->{tablet}; + if (!defined($tablet)) { $tablet = $defaults->{tablet}; $tablet = 0 if $qxlnum; # disable for spice because it is not needed $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card) @@ -3214,36 +3676,35 @@ sub config_to_command { my $bootorder = device_bootorder($conf); # host pci device passthrough - my ($kvm_off, $gpu_passthrough, $legacy_igd) = PVE::QemuServer::PCI::print_hostpci_devices( - $vmid, $conf, $devices, $vga, $winversion, $q35, $bridges, $arch, $machine_type, $bootorder); + my ($kvm_off, $gpu_passthrough, $legacy_igd, $pci_devices) = PVE::QemuServer::PCI::print_hostpci_devices( + $vmid, $conf, $devices, $vga, $winversion, $bridges, $arch, $machine_type, $bootorder); # usb devices my $usb_dev_features = {}; $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0); my @usbdevices = PVE::QemuServer::USB::get_usb_devices( - $conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features, $bootorder); + $conf, $usb_dev_features, $bootorder, $machine_version); push @$devices, @usbdevices if @usbdevices; # serial devices for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) { - if (my $path = $conf->{"serial$i"}) { - if ($path eq 'socket') { - my $socket = "/var/run/qemu-server/${vmid}.serial$i"; - push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait"; - # On aarch64, serial0 is the UART device. Qemu only allows - # connecting UART devices via the '-serial' command line, as - # the device has a fixed slot on the hardware... - if ($arch eq 'aarch64' && $i == 0) { - push @$devices, '-serial', "chardev:serial$i"; - } else { - push @$devices, '-device', "isa-serial,chardev=serial$i"; - } + my $path = $conf->{"serial$i"} or next; + if ($path eq 'socket') { + my $socket = "/var/run/qemu-server/${vmid}.serial$i"; + push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server=on,wait=off"; + # On aarch64, serial0 is the UART device. QEMU only allows + # connecting UART devices via the '-serial' command line, as + # the device has a fixed slot on the hardware... + if ($arch eq 'aarch64' && $i == 0) { + push @$devices, '-serial', "chardev:serial$i"; } else { - die "no such serial device\n" if ! -c $path; - push @$devices, '-chardev', "tty,id=serial$i,path=$path"; push @$devices, '-device', "isa-serial,chardev=serial$i"; } + } else { + die "no such serial device\n" if ! -c $path; + push @$devices, '-chardev', "serial,id=serial$i,path=$path"; + push @$devices, '-device', "isa-serial,chardev=serial$i"; } } @@ -3251,7 +3712,7 @@ sub config_to_command { for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++) { if (my $path = $conf->{"parallel$i"}) { die "no such parallel device\n" if ! -c $path; - my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport'; + my $devtype = $path =~ m!^/dev/usb/lp! ? 'serial' : 'parallel'; push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path"; push @$devices, '-device', "isa-parallel,chardev=parallel$i"; } @@ -3263,6 +3724,10 @@ sub config_to_command { push @$devices, @$audio_devs; } + # Add a TPM only if the VM is not a template, + # to support backing up template VMs even if the TPM disk is write-protected. + add_tpm_device($vmid, $devices, $conf) if (!PVE::QemuConfig->is_template($conf)); + my $sockets = 1; $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused $sockets = $conf->{sockets} if $conf->{sockets}; @@ -3275,14 +3740,12 @@ sub config_to_command { my $allowed_vcpus = $cpuinfo->{cpus}; - die "MAX $allowed_vcpus vcpus allowed per VM on this node\n" - if ($allowed_vcpus < $maxcpus); - - if($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) { + die "MAX $allowed_vcpus vcpus allowed per VM on this node\n" if ($allowed_vcpus < $maxcpus); + if ($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) { push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus"; for (my $i = 2; $i <= $vcpus; $i++) { - my $cpustr = print_cpu_device($conf,$i); + my $cpustr = print_cpu_device($conf, $arch, $i); push @$cmd, '-device', $cpustr; } @@ -3294,15 +3757,18 @@ sub config_to_command { push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg"; - push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0; + push $machineFlags->@*, 'acpi=off' if defined($conf->{acpi}) && $conf->{acpi} == 0; push @$cmd, '-no-reboot' if defined($conf->{reboot}) && $conf->{reboot} == 0; if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){ push @$devices, '-device', print_vga_device( $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges); + + push @$cmd, '-display', 'egl-headless,gl=core' if $vga->{type} eq 'virtio-gl'; # VIRGL + my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid); - push @$cmd, '-vnc', "unix:$socket,password"; + push @$cmd, '-vnc', "unix:$socket,password=on"; } else { push @$cmd, '-vga', 'none' if $vga->{type} eq 'none'; push @$cmd, '-nographic'; @@ -3323,7 +3789,7 @@ sub config_to_command { if ($winversion >= 6) { push @$globalFlags, 'kvm-pit.lost_tick_policy=discard'; - push @$cmd, '-no-hpet'; + push @$machineFlags, 'hpet=off'; } push @$rtcFlags, 'driftfix=slew' if $tdf; @@ -3340,7 +3806,8 @@ sub config_to_command { push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough); } - PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd); + PVE::QemuServer::Memory::config( + $conf, $vmid, $sockets, $cores, $hotplug_features->{memory}, $cmd); push @$cmd, '-S' if $conf->{freeze}; @@ -3350,7 +3817,7 @@ sub config_to_command { if ($guest_agent->{enabled}) { my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1); - push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0"; + push @$devices, '-chardev', "socket,path=$qgasocket,server=on,wait=off,id=qga0"; if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') { my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type); @@ -3379,7 +3846,10 @@ sub config_to_command { my $spice_port; - if ($qxlnum) { + assert_clipboard_config($vga); + my $is_spice = $qxlnum || $vga->{type} =~ /^virtio/; + + if ($is_spice || ($vga->{'clipboard'} && $vga->{'clipboard'} eq 'vnc')) { if ($qxlnum > 1) { if ($winversion){ for (my $i = 1; $i < $qxlnum; $i++){ @@ -3400,40 +3870,47 @@ sub config_to_command { my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type); - my $pfamily = PVE::Tools::get_host_address_family($nodename); - my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily); - die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs; - push @$devices, '-device', "virtio-serial,id=spice$pciaddr"; - push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent"; + if ($vga->{'clipboard'} && $vga->{'clipboard'} eq 'vnc') { + push @$devices, '-chardev', 'qemu-vdagent,id=vdagent,name=vdagent,clipboard=on'; + } else { + push @$devices, '-chardev', 'spicevmc,id=vdagent,name=vdagent'; + } push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0"; - my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr}); - $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost); + if ($is_spice) { + my $pfamily = PVE::Tools::get_host_address_family($nodename); + my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily); + die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs; - my $spice_enhancement_str = $conf->{spice_enhancements} // ''; - my $spice_enhancement = parse_property_string($spice_enhancements_fmt, $spice_enhancement_str); - if ($spice_enhancement->{foldersharing}) { - push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0"; - push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0"; - } + my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr}); + $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost); - my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on"; - $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}" - if $spice_enhancement->{videostreaming}; + my $spice_enhancement_str = $conf->{spice_enhancements} // ''; + my $spice_enhancement = parse_property_string($spice_enhancements_fmt, $spice_enhancement_str); + if ($spice_enhancement->{foldersharing}) { + push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0"; + push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0"; + } - push @$devices, '-spice', "$spice_opts"; + my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on"; + $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}" + if $spice_enhancement->{videostreaming}; + push @$devices, '-spice', "$spice_opts"; + } } # enable balloon by default, unless explicitly disabled if (!defined($conf->{balloon}) || $conf->{balloon}) { - $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type); - push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr"; + my $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type); + my $ballooncmd = "virtio-balloon-pci,id=balloon0$pciaddr"; + $ballooncmd .= ",free-page-reporting=on" if min_version($machine_version, 6, 2); + push @$devices, '-device', $ballooncmd; } if ($conf->{watchdog}) { my $wdopts = parse_watchdog($conf->{watchdog}); - $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type); + my $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type); my $watchdog = $wdopts->{model} || 'i6300esb'; push @$devices, '-device', "$watchdog$pciaddr"; push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action}; @@ -3453,11 +3930,14 @@ sub config_to_command { my ($ds, $drive) = @_; if (PVE::Storage::parse_volume_id($drive->{file}, 1)) { + check_volume_storage_type($storecfg, $drive->{file}); push @$vollist, $drive->{file}; } # ignore efidisk here, already added in bios/fw handling code above return if $drive->{interface} eq 'efidisk'; + # similar for TPM + return if $drive->{interface} eq 'tpmstate'; $use_virtio = 1 if $ds =~ m/^virtio/; @@ -3474,7 +3954,7 @@ sub config_to_command { die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n" if $drive->{index} > 13 && !&$version_guard(4, 1, 2); - $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type); + my $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type); my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw; my $iothread = ''; @@ -3482,7 +3962,9 @@ sub config_to_command { $iothread .= ",iothread=iothread-$controller_prefix$controller"; push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller"; } elsif ($drive->{iothread}) { - warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n"; + log_warn( + "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n" + ); } my $queues = ''; @@ -3497,14 +3979,24 @@ sub config_to_command { if ($drive->{interface} eq 'sata') { my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS); - $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type); + my $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type); push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller}; $ahcicontroller->{$controller}=1; } - my $drive_cmd = print_drive_commandline_full($storecfg, $vmid, $drive); - $drive_cmd .= ',readonly' if PVE::QemuConfig->is_template($conf); + my $live_restore = $live_restore_backing->{$ds}; + my $live_blockdev_name = undef; + if ($live_restore) { + $live_blockdev_name = $live_restore->{name}; + push @$devices, '-blockdev', $live_restore->{blockdev}; + } + + my $drive_cmd = print_drive_commandline_full( + $storecfg, $vmid, $drive, $live_blockdev_name, min_version($kvmver, 6, 0)); + + # extra protection for templates, but SATA and IDE don't support it.. + $drive_cmd .= ',readonly=on' if drive_is_read_only($conf, $drive); push @$devices, '-drive',$drive_cmd; push @$devices, '-device', print_drivedevice_full( @@ -3517,6 +4009,7 @@ sub config_to_command { next if !$conf->{$netname}; my $d = parse_net($conf->{$netname}); next if !$d; + # save the MAC addr here (could be auto-gen. in some odd setups) for FDB registering later? $use_virtio = 1 if $d->{model} eq 'virtio'; @@ -3526,7 +4019,7 @@ sub config_to_command { push @$devices, '-netdev', $netdevfull; my $netdevicefull = print_netdevice_full( - $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type); + $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type, $machine_version); push @$devices, '-device', $netdevicefull; } @@ -3552,15 +4045,12 @@ sub config_to_command { # pci.4 is nested in pci.1 $bridges->{1} = 1 if $bridges->{4}; - if (!$q35) { - # add pci bridges - if (min_version($machine_version, 2, 3)) { + if (!$q35) { # add pci bridges + if (min_version($machine_version, 2, 3)) { $bridges->{1} = 1; $bridges->{2} = 1; } - $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/; - } for my $k (sort {$b cmp $a} keys %$bridges) { @@ -3570,11 +4060,10 @@ sub config_to_command { if ($k == 2 && $legacy_igd) { $k_name = "$k-igd"; } - $pciaddr = print_pci_addr("pci.$k_name", undef, $arch, $machine_type); - + my $pciaddr = print_pci_addr("pci.$k_name", undef, $arch, $machine_type); my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr"; - if ($q35) { - # add after -readconfig pve-q35.cfg + + if ($q35) { # add after -readconfig pve-q35.cfg splice @$devices, 2, 0, '-device', $devstr; } else { unshift @$devices, '-device', $devstr if $k > 0; @@ -3585,6 +4074,8 @@ sub config_to_command { push @$machineFlags, 'accel=tcg'; } + push @$machineFlags, 'smm=off' if should_disable_smm($conf, $vga, $machine_type); + my $machine_type_min = $machine_type; if ($add_pve_version) { $machine_type_min =~ s/\+pve\d+$//; @@ -3592,6 +4083,17 @@ sub config_to_command { } push @$machineFlags, "type=${machine_type_min}"; + PVE::QemuServer::Machine::assert_valid_machine_property($conf, $machine_conf); + + if (my $viommu = $machine_conf->{viommu}) { + if ($viommu eq 'intel') { + unshift @$devices, '-device', 'intel-iommu,intremap=on,caching-mode=on'; + push @$machineFlags, 'kernel-irqchip=split'; + } elsif ($viommu eq 'virtio') { + push @$devices, '-device', 'virtio-iommu-pci'; + } + } + push @$cmd, @$devices; push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags); push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags); @@ -3604,13 +4106,18 @@ sub config_to_command { print "activating and using '$vmstate' as vmstate\n"; } + if (PVE::QemuConfig->is_template($conf)) { + # needed to workaround base volumes being read-only + push @$cmd, '-snapshot'; + } + # add custom args if ($conf->{args}) { my $aa = PVE::Tools::split_args($conf->{args}); push @$cmd, @$aa; } - return wantarray ? ($cmd, $vollist, $spice_port) : $cmd; + return wantarray ? ($cmd, $vollist, $spice_port, $pci_devices) : $cmd; } sub check_rng_source { @@ -3652,7 +4159,7 @@ sub vm_devices_list { my $to_check = []; for my $d (@$devices_to_check) { $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'}; - next if !$d->{'pci_bridge'}; + next if !$d->{'pci_bridge'} || !$d->{'pci_bridge'}->{devices}; $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}}); push @$to_check, @{$d->{'pci_bridge'}->{devices}}; @@ -3680,7 +4187,7 @@ sub vm_devices_list { # qom-list path=/machine/peripheral my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral'); foreach my $per (@$resperipheral) { - if ($per->{name} =~ m/^usb\d+$/) { + if ($per->{name} =~ m/^usb(?:redirdev)?\d+$/) { $devices->{$per->{name}} = 1; } } @@ -3700,43 +4207,34 @@ sub vm_deviceplug { qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type); if ($deviceid eq 'tablet') { - qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch)); - } elsif ($deviceid eq 'keyboard') { - qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch)); - + } elsif ($deviceid =~ m/^usbredirdev(\d+)$/) { + my $id = $1; + qemu_spice_usbredir_chardev_add($vmid, "usbredirchardev$id"); + qemu_deviceadd($vmid, PVE::QemuServer::USB::print_spice_usbdevice($id, "xhci", $id + 1)); } elsif ($deviceid =~ m/^usb(\d+)$/) { - - die "usb hotplug currently not reliable\n"; - # since we can't reliably hot unplug all added usb devices and usb - # passthrough breaks live migration we disable usb hotplugging for now - #qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device)); - + qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device, {}, $1 + 1)); } elsif ($deviceid =~ m/^(virtio)(\d+)$/) { - qemu_iothread_add($vmid, $deviceid, $device); - qemu_driveadd($storecfg, $vmid, $device); - my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type); + qemu_driveadd($storecfg, $vmid, $device); + my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type); - qemu_deviceadd($vmid, $devicefull); + qemu_deviceadd($vmid, $devicefull); eval { qemu_deviceaddverify($vmid, $deviceid); }; if (my $err = $@) { eval { qemu_drivedel($vmid, $deviceid); }; warn $@ if $@; die $err; } - } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) { - - - my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi"; - my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type); + my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi"; + my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type); my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw; - my $devicefull = "$scsihw_type,id=$deviceid$pciaddr"; + my $devicefull = "$scsihw_type,id=$deviceid$pciaddr"; if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) { qemu_iothread_add($vmid, $deviceid, $device); @@ -3747,32 +4245,29 @@ sub vm_deviceplug { $devicefull .= ",num_queues=$device->{queues}"; } - qemu_deviceadd($vmid, $devicefull); - qemu_deviceaddverify($vmid, $deviceid); - + qemu_deviceadd($vmid, $devicefull); + qemu_deviceaddverify($vmid, $deviceid); } elsif ($deviceid =~ m/^(scsi)(\d+)$/) { - qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type); qemu_driveadd($storecfg, $vmid, $device); - my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type); + my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type); eval { qemu_deviceadd($vmid, $devicefull); }; if (my $err = $@) { eval { qemu_drivedel($vmid, $deviceid); }; warn $@ if $@; die $err; } - } elsif ($deviceid =~ m/^(net)(\d+)$/) { - return if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid); my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf); + my $machine_version = PVE::QemuServer::Machine::extract_version($machine_type); my $use_old_bios_files = undef; ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type); my $netdevicefull = print_netdevice_full( - $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type); + $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type, $machine_version); qemu_deviceadd($vmid, $netdevicefull); eval { qemu_deviceaddverify($vmid, $deviceid); @@ -3783,16 +4278,13 @@ sub vm_deviceplug { warn $@ if $@; die $err; } - } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) { - my $bridgeid = $2; my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type); my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr"; qemu_deviceadd($vmid, $devicefull); qemu_deviceaddverify($vmid, $deviceid); - } else { die "can't hotplug device '$deviceid'\n"; } @@ -3810,43 +4302,38 @@ sub vm_deviceunplug { my $bootdisks = PVE::QemuServer::Drive::get_bootdisks($conf); die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks; - if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') { - + if ($deviceid eq 'tablet' || $deviceid eq 'keyboard' || $deviceid eq 'xhci') { qemu_devicedel($vmid, $deviceid); - + } elsif ($deviceid =~ m/^usbredirdev\d+$/) { + qemu_devicedel($vmid, $deviceid); + qemu_devicedelverify($vmid, $deviceid); } elsif ($deviceid =~ m/^usb\d+$/) { - - die "usb hotplug currently not reliable\n"; - # when unplugging usb devices this way, there may be remaining usb - # controllers/hubs so we disable it for now - #qemu_devicedel($vmid, $deviceid); - #qemu_devicedelverify($vmid, $deviceid); - + qemu_devicedel($vmid, $deviceid); + qemu_devicedelverify($vmid, $deviceid); } elsif ($deviceid =~ m/^(virtio)(\d+)$/) { + my $device = parse_drive($deviceid, $conf->{$deviceid}); - qemu_devicedel($vmid, $deviceid); - qemu_devicedelverify($vmid, $deviceid); - qemu_drivedel($vmid, $deviceid); - qemu_iothread_del($conf, $vmid, $deviceid); - + qemu_devicedel($vmid, $deviceid); + qemu_devicedelverify($vmid, $deviceid); + qemu_drivedel($vmid, $deviceid); + qemu_iothread_del($vmid, $deviceid, $device); } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) { - qemu_devicedel($vmid, $deviceid); qemu_devicedelverify($vmid, $deviceid); - qemu_iothread_del($conf, $vmid, $deviceid); - } elsif ($deviceid =~ m/^(scsi)(\d+)$/) { + my $device = parse_drive($deviceid, $conf->{$deviceid}); - qemu_devicedel($vmid, $deviceid); - qemu_drivedel($vmid, $deviceid); + qemu_devicedel($vmid, $deviceid); + qemu_devicedelverify($vmid, $deviceid); + qemu_drivedel($vmid, $deviceid); qemu_deletescsihw($conf, $vmid, $deviceid); + qemu_iothread_del($vmid, "virtioscsi$device->{index}", $device) + if $conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single'); } elsif ($deviceid =~ m/^(net)(\d+)$/) { - - qemu_devicedel($vmid, $deviceid); - qemu_devicedelverify($vmid, $deviceid); - qemu_netdevdel($vmid, $deviceid); - + qemu_devicedel($vmid, $deviceid); + qemu_devicedelverify($vmid, $deviceid); + qemu_netdevdel($vmid, $deviceid); } else { die "can't unplug device '$deviceid'\n"; } @@ -3854,23 +4341,22 @@ sub vm_deviceunplug { return 1; } -sub qemu_deviceadd { - my ($vmid, $devicefull) = @_; - - $devicefull = "driver=".$devicefull; - my %options = split(/[=,]/, $devicefull); - - mon_cmd($vmid, "device_add" , %options); -} - -sub qemu_devicedel { - my ($vmid, $deviceid) = @_; +sub qemu_spice_usbredir_chardev_add { + my ($vmid, $id) = @_; - my $ret = mon_cmd($vmid, "device_del", id => $deviceid); + mon_cmd($vmid, "chardev-add" , ( + id => $id, + backend => { + type => 'spicevmc', + data => { + type => "usbredir", + }, + }, + )); } sub qemu_iothread_add { - my($vmid, $deviceid, $device) = @_; + my ($vmid, $deviceid, $device) = @_; if ($device->{iothread}) { my $iothreads = vm_iothreads_list($vmid); @@ -3879,39 +4365,20 @@ sub qemu_iothread_add { } sub qemu_iothread_del { - my($conf, $vmid, $deviceid) = @_; + my ($vmid, $deviceid, $device) = @_; - my $confid = $deviceid; - if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) { - $confid = 'scsi' . $1; - } - my $device = parse_drive($confid, $conf->{$confid}); if ($device->{iothread}) { my $iothreads = vm_iothreads_list($vmid); qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"}; } } -sub qemu_objectadd { - my($vmid, $objectid, $qomtype) = @_; - - mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype); - - return 1; -} - -sub qemu_objectdel { - my($vmid, $objectid) = @_; - - mon_cmd($vmid, "object-del", id => $objectid); - - return 1; -} - sub qemu_driveadd { my ($storecfg, $vmid, $device) = @_; - my $drive = print_drive_commandline_full($storecfg, $vmid, $device); + my $kvmver = get_running_qemu_version($vmid); + my $io_uring = min_version($kvmver, 6, 0); + my $drive = print_drive_commandline_full($storecfg, $vmid, $device, undef, $io_uring); $drive =~ s/\\/\\\\/g; my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\""); @@ -3922,7 +4389,7 @@ sub qemu_driveadd { } sub qemu_drivedel { - my($vmid, $deviceid) = @_; + my ($vmid, $deviceid) = @_; my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid"); $ret =~ s/^\s+//; @@ -3971,7 +4438,7 @@ sub qemu_findorcreatescsihw { my $scsihwid="$controller_prefix$controller"; my $devices_list = vm_devices_list($vmid); - if(!defined($devices_list->{$scsihwid})) { + if (!defined($devices_list->{$scsihwid})) { vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type); } @@ -3994,7 +4461,7 @@ sub qemu_deletescsihw { foreach my $opt (keys %{$devices_list}) { if (is_valid_drivename($opt)) { my $drive = parse_drive($opt, $conf->{$opt}); - if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) { + if ($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) { return 1; } } @@ -4071,20 +4538,15 @@ sub qemu_usb_hotplug { vm_deviceunplug($vmid, $conf, $deviceid); # check if xhci controller is necessary and available - if ($device->{usb3}) { - - my $devicelist = vm_devices_list($vmid); + my $devicelist = vm_devices_list($vmid); - if (!$devicelist->{xhci}) { - my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type); - qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr"); - } + if (!$devicelist->{xhci}) { + my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type); + qemu_deviceadd($vmid, PVE::QemuServer::USB::print_qemu_xhci_controller($pciaddr)); } - my $d = parse_usb_device($device->{host}); - $d->{usb3} = $device->{usb3}; # add the new one - vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type); + vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type); } sub qemu_cpu_hotplug { @@ -4136,9 +4598,10 @@ sub qemu_cpu_hotplug { if scalar(@{$currentrunningvcpus}) != $currentvcpus; if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) { + my $arch = get_vm_arch($conf); for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) { - my $cpustr = print_cpu_device($conf, $i); + my $cpustr = print_cpu_device($conf, $arch, $i); qemu_deviceadd($vmid, $cpustr); my $retry = 0; @@ -4199,15 +4662,20 @@ sub qemu_block_resize { my $running = check_running($vmid); - $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running); + PVE::Storage::volume_resize($storecfg, $volid, $size, $running); return if !$running; my $padding = (1024 - $size % 1024) % 1024; $size = $size + $padding; - mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size)); - + mon_cmd( + $vmid, + "block_resize", + device => $deviceid, + size => int($size), + timeout => 60, + ); } sub qemu_volume_snapshot { @@ -4215,7 +4683,7 @@ sub qemu_volume_snapshot { my $running = check_running($vmid); - if ($running && do_snapshots_with_qemu($storecfg, $volid)){ + if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) { mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap); } else { PVE::Storage::volume_snapshot($storecfg, $volid, $snap); @@ -4223,32 +4691,40 @@ sub qemu_volume_snapshot { } sub qemu_volume_snapshot_delete { - my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_; + my ($vmid, $storecfg, $volid, $snap) = @_; my $running = check_running($vmid); + my $attached_deviceid; - if($running) { - - $running = undef; + if ($running) { my $conf = PVE::QemuConfig->load_config($vmid); PVE::QemuConfig->foreach_volume($conf, sub { my ($ds, $drive) = @_; - $running = 1 if $drive->{file} eq $volid; + $attached_deviceid = "drive-$ds" if $drive->{file} eq $volid; }); } - if ($running && do_snapshots_with_qemu($storecfg, $volid)){ - mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap); + if ($attached_deviceid && do_snapshots_with_qemu($storecfg, $volid, $attached_deviceid)) { + mon_cmd( + $vmid, + 'blockdev-snapshot-delete-internal-sync', + device => $attached_deviceid, + name => $snap, + ); } else { - PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running); + PVE::Storage::volume_snapshot_delete( + $storecfg, $volid, $snap, $attached_deviceid ? 1 : undef); } } sub set_migration_caps { - my ($vmid) = @_; + my ($vmid, $savevm) = @_; my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") }; + my $bitmap_prop = $savevm ? 'pbs-dirty-bitmap-savevm' : 'pbs-dirty-bitmap-migration'; + my $dirty_bitmaps = $qemu_support->{$bitmap_prop} ? 1 : 0; + my $cap_ref = []; my $enabled_cap = { @@ -4257,7 +4733,7 @@ sub set_migration_caps { "x-rdma-pin-all" => 0, "zero-blocks" => 0, "compress" => 0, - "dirty-bitmaps" => $qemu_support->{'pbs-dirty-bitmap-migration'} ? 1 : 0, + "dirty-bitmaps" => $dirty_bitmaps, }; my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities"); @@ -4278,7 +4754,7 @@ sub foreach_volid { my $volhash = {}; my $test_volid = sub { - my ($key, $drive, $snapname) = @_; + my ($key, $drive, $snapname, $pending) = @_; my $volid = $drive->{file}; return if !$volid; @@ -4293,20 +4769,28 @@ sub foreach_volid { $volhash->{$volid}->{shared} //= 0; $volhash->{$volid}->{shared} = 1 if $drive->{shared}; - $volhash->{$volid}->{referenced_in_config} //= 0; - $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname); + $volhash->{$volid}->{is_unused} //= 0; + $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/; + + $volhash->{$volid}->{is_attached} //= 0; + $volhash->{$volid}->{is_attached} = 1 + if !$volhash->{$volid}->{is_unused} && !defined($snapname) && !$pending; $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1 if defined($snapname); + $volhash->{$volid}->{referenced_in_pending} = 1 if $pending; + my $size = $drive->{size}; $volhash->{$volid}->{size} //= $size if $size; $volhash->{$volid}->{is_vmstate} //= 0; $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate'; - $volhash->{$volid}->{is_unused} //= 0; - $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/; + $volhash->{$volid}->{is_tpmstate} //= 0; + $volhash->{$volid}->{is_tpmstate} = 1 if $key eq 'tpmstate0'; + + $volhash->{$volid}->{drivename} = $key if is_valid_drivename($key); }; my $include_opts = { @@ -4315,6 +4799,10 @@ sub foreach_volid { }; PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid); + + PVE::QemuConfig->foreach_volume_full($conf->{pending}, $include_opts, $test_volid, undef, 1) + if defined($conf->{pending}) && $conf->{pending}->%*; + foreach my $snapname (keys %{$conf->{snapshots}}) { my $snap = $conf->{snapshots}->{$snapname}; PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname); @@ -4326,16 +4814,22 @@ sub foreach_volid { } my $fast_plug_option = { + 'description' => 1, + 'hookscript' => 1, 'lock' => 1, + 'migrate_downtime' => 1, + 'migrate_speed' => 1, 'name' => 1, 'onboot' => 1, + 'protection' => 1, 'shares' => 1, 'startup' => 1, - 'description' => 1, - 'protection' => 1, - 'vmstatestorage' => 1, - 'hookscript' => 1, 'tags' => 1, + 'vmstatestorage' => 1, +}; + +for my $opt (keys %$confdesc_cloudinit) { + $fast_plug_option->{$opt} = 1; }; # hotplug changes in [PENDING] @@ -4358,11 +4852,78 @@ sub vmconfig_hotplug_pending { $errors->{$opt} = "hotplug problem - $msg"; }; + my $cloudinit_pending_properties = PVE::QemuServer::cloudinit_pending_properties(); + + my $cloudinit_record_changed = sub { + my ($conf, $opt, $old, $new) = @_; + return if !$cloudinit_pending_properties->{$opt}; + + my $ci = ($conf->{cloudinit} //= {}); + + my $recorded = $ci->{$opt}; + my %added = map { $_ => 1 } PVE::Tools::split_list(delete($ci->{added}) // ''); + + if (defined($new)) { + if (defined($old)) { + # an existing value is being modified + if (defined($recorded)) { + # the value was already not in sync + if ($new eq $recorded) { + # a value is being reverted to the cloud-init state: + delete $ci->{$opt}; + delete $added{$opt}; + } else { + # the value was changed multiple times, do nothing + } + } elsif ($added{$opt}) { + # the value had been marked as added and is being changed, do nothing + } else { + # the value is new, record it: + $ci->{$opt} = $old; + } + } else { + # a new value is being added + if (defined($recorded)) { + # it was already not in sync + if ($new eq $recorded) { + # a value is being reverted to the cloud-init state: + delete $ci->{$opt}; + delete $added{$opt}; + } else { + # the value had temporarily been removed, do nothing + } + } elsif ($added{$opt}) { + # the value had been marked as added already, do nothing + } else { + # the value is new, add it + $added{$opt} = 1; + } + } + } elsif (!defined($old)) { + # a non-existent value is being removed? ignore... + } else { + # a value is being deleted + if (defined($recorded)) { + # a value was already recorded, just keep it + } elsif ($added{$opt}) { + # the value was marked as added, remove it + delete $added{$opt}; + } else { + # a previously unrecorded value is being removed, record the old value: + $ci->{$opt} = $old; + } + } + + my $added = join(',', sort keys %added); + $ci->{added} = $added if length($added); + }; + my $changes = 0; foreach my $opt (keys %{$conf->{pending}}) { # add/change if ($fast_plug_option->{$opt}) { - $conf->{$opt} = $conf->{pending}->{$opt}; - delete $conf->{pending}->{$opt}; + my $new = delete $conf->{pending}->{$opt}; + $cloudinit_record_changed->($conf, $opt, $conf->{$opt}, $new); + $conf->{$opt} = $new; $changes = 1; } } @@ -4371,15 +4932,22 @@ sub vmconfig_hotplug_pending { PVE::QemuConfig->write_config($vmid, $conf); } + my $ostype = $conf->{ostype}; + my $version = extract_version($machine_type, get_running_qemu_version($vmid)); my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1'); + my $usb_hotplug = $hotplug_features->{usb} + && min_version($version, 7, 1) + && defined($ostype) && ($ostype eq 'l26' || windows_version($ostype) > 7); + my $cgroup = PVE::QemuServer::CGroup->new($vmid); my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete}); + foreach my $opt (sort keys %$pending_delete_hash) { next if $selection && !$selection->{$opt}; my $force = $pending_delete_hash->{$opt}->{force}; eval { if ($opt eq 'hotplug') { - die "skip\n" if ($conf->{hotplug} =~ /memory/); + die "skip\n" if ($conf->{hotplug} =~ /(cpu|memory)/); } elsif ($opt eq 'tablet') { die "skip\n" if !$hotplug_features->{usb}; if ($defaults->{tablet}) { @@ -4390,11 +4958,11 @@ sub vmconfig_hotplug_pending { vm_deviceunplug($vmid, $conf, 'tablet'); vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64'; } - } elsif ($opt =~ m/^usb\d+/) { - die "skip\n"; - # since we cannot reliably hot unplug usb devices we are disabling it - #die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i; - #vm_deviceunplug($vmid, $conf, $opt); + } elsif ($opt =~ m/^usb(\d+)$/) { + my $index = $1; + die "skip\n" if !$usb_hotplug; + vm_deviceunplug($vmid, $conf, "usbredirdev$index"); # if it's a spice port + vm_deviceunplug($vmid, $conf, $opt); } elsif ($opt eq 'vcpus') { die "skip\n" if !$hotplug_features->{cpu}; qemu_cpu_hotplug($vmid, $conf, undef); @@ -4402,24 +4970,28 @@ sub vmconfig_hotplug_pending { # enable balloon device is not hotpluggable die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0; # here we reset the ballooning value to memory - my $balloon = $conf->{memory} || $defaults->{memory}; + my $balloon = get_current_memory($conf->{memory}); mon_cmd($vmid, "balloon", value => $balloon*1024*1024); } elsif ($fast_plug_option->{$opt}) { # do nothing } elsif ($opt =~ m/^net(\d+)$/) { die "skip\n" if !$hotplug_features->{network}; vm_deviceunplug($vmid, $conf, $opt); + if($have_sdn) { + my $net = PVE::QemuServer::parse_net($conf->{$opt}); + PVE::Network::SDN::Vnets::del_ips_from_mac($net->{bridge}, $net->{macaddr}, $conf->{name}); + } } elsif (is_valid_drivename($opt)) { die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/; vm_deviceunplug($vmid, $conf, $opt); vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force); } elsif ($opt =~ m/^memory$/) { die "skip\n" if !$hotplug_features->{memory}; - PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt); + PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf); } elsif ($opt eq 'cpuunits') { - cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits}); + $cgroup->change_cpu_shares(undef); } elsif ($opt eq 'cpulimit') { - cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1); + $cgroup->change_cpu_quota(undef, undef); # reset, cgroup module can better decide values } else { die "skip\n"; } @@ -4427,35 +4999,20 @@ sub vmconfig_hotplug_pending { if (my $err = $@) { &$add_error($opt, $err) if $err ne "skip\n"; } else { - delete $conf->{$opt}; + my $old = delete $conf->{$opt}; + $cloudinit_record_changed->($conf, $opt, $old, undef); PVE::QemuConfig->remove_from_pending_delete($conf, $opt); } } - my ($apply_pending_cloudinit, $apply_pending_cloudinit_done); - $apply_pending_cloudinit = sub { - return if $apply_pending_cloudinit_done; # once is enough - $apply_pending_cloudinit_done = 1; # once is enough - - my ($key, $value) = @_; - - my @cloudinit_opts = keys %$confdesc_cloudinit; - foreach my $opt (keys %{$conf->{pending}}) { - next if !grep { $_ eq $opt } @cloudinit_opts; - $conf->{$opt} = delete $conf->{pending}->{$opt}; - } - - my $new_conf = { %$conf }; - $new_conf->{$key} = $value; - PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid); - }; - + my $cloudinit_opt; foreach my $opt (keys %{$conf->{pending}}) { next if $selection && !$selection->{$opt}; my $value = $conf->{pending}->{$opt}; eval { if ($opt eq 'hotplug') { die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/); + die "skip\n" if ($value =~ /cpu/) || ($value !~ /cpu/ && $conf->{hotplug} =~ /cpu/); } elsif ($opt eq 'tablet') { die "skip\n" if !$hotplug_features->{usb}; if ($value == 1) { @@ -4466,13 +5023,15 @@ sub vmconfig_hotplug_pending { vm_deviceunplug($vmid, $conf, 'tablet'); vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64'; } - } elsif ($opt =~ m/^usb\d+$/) { - die "skip\n"; - # since we cannot reliably hot unplug usb devices we disable it for now - #die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i; - #my $d = eval { parse_property_string($usbdesc->{format}, $value) }; - #die "skip\n" if !$d; - #qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type); + } elsif ($opt =~ m/^usb(\d+)$/) { + my $index = $1; + die "skip\n" if !$usb_hotplug; + my $d = eval { parse_property_string('pve-qm-usb', $value) }; + my $id = $opt; + if ($d->{host} =~ m/^spice$/i) { + $id = "usbredirdev$index"; + } + qemu_usb_hotplug($storecfg, $conf, $vmid, $id, $d, $arch, $machine_type); } elsif ($opt eq 'vcpus') { die "skip\n" if !$hotplug_features->{cpu}; qemu_cpu_hotplug($vmid, $conf, $value); @@ -4484,7 +5043,8 @@ sub vmconfig_hotplug_pending { # allow manual ballooning if shares is set to zero if ((defined($conf->{shares}) && ($conf->{shares} == 0))) { - my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory}; + my $memory = get_current_memory($conf->{memory}); + my $balloon = $conf->{pending}->{balloon} || $memory; mon_cmd($vmid, "balloon", value => $balloon*1024*1024); } } elsif ($opt =~ m/^net(\d+)$/) { @@ -4492,22 +5052,27 @@ sub vmconfig_hotplug_pending { vmconfig_update_net($storecfg, $conf, $hotplug_features->{network}, $vmid, $opt, $value, $arch, $machine_type); } elsif (is_valid_drivename($opt)) { - die "skip\n" if $opt eq 'efidisk0'; + die "skip\n" if $opt eq 'efidisk0' || $opt eq 'tpmstate0'; # some changes can be done without hotplug my $drive = parse_drive($opt, $value); if (drive_is_cloudinit($drive)) { - &$apply_pending_cloudinit($opt, $value); + $cloudinit_opt = [$opt, $drive]; + # apply all the other changes first, then generate the cloudinit disk + die "skip\n"; } vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk}, $vmid, $opt, $value, $arch, $machine_type); } elsif ($opt =~ m/^memory$/) { #dimms die "skip\n" if !$hotplug_features->{memory}; - $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value); + $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $value); } elsif ($opt eq 'cpuunits') { - cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt}); + my $new_cpuunits = PVE::CGroup::clamp_cpu_shares($conf->{pending}->{$opt}); #clamp + $cgroup->change_cpu_shares($new_cpuunits); } elsif ($opt eq 'cpulimit') { my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000); - cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit); + $cgroup->change_cpu_quota($cpulimit, 100000); + } elsif ($opt eq 'agent') { + vmconfig_update_agent($conf, $opt, $value); } else { die "skip\n"; # skip non-hot-pluggable options } @@ -4515,12 +5080,47 @@ sub vmconfig_hotplug_pending { if (my $err = $@) { &$add_error($opt, $err) if $err ne "skip\n"; } else { + $cloudinit_record_changed->($conf, $opt, $conf->{$opt}, $value); $conf->{$opt} = $value; delete $conf->{pending}->{$opt}; } } + if (defined($cloudinit_opt)) { + my ($opt, $drive) = @$cloudinit_opt; + my $value = $conf->{pending}->{$opt}; + eval { + my $temp = {%$conf, $opt => $value}; + PVE::QemuServer::Cloudinit::apply_cloudinit_config($temp, $vmid); + vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk}, + $vmid, $opt, $value, $arch, $machine_type); + }; + if (my $err = $@) { + &$add_error($opt, $err) if $err ne "skip\n"; + } else { + $conf->{$opt} = $value; + delete $conf->{pending}->{$opt}; + } + } + + # unplug xhci controller if no usb device is left + if ($usb_hotplug) { + my $has_usb = 0; + for (my $i = 0; $i < $PVE::QemuServer::USB::MAX_USB_DEVICES; $i++) { + next if !defined($conf->{"usb$i"}); + $has_usb = 1; + last; + } + if (!$has_usb) { + vm_deviceunplug($vmid, $conf, 'xhci'); + } + } + PVE::QemuConfig->write_config($vmid, $conf); + + if ($hotplug_features->{cloudinit} && PVE::QemuServer::Cloudinit::has_changes($conf)) { + PVE::QemuServer::vmconfig_update_cloudinit_drive($storecfg, $conf, $vmid); + } } sub try_deallocate_drive { @@ -4565,7 +5165,9 @@ sub vmconfig_delete_or_detach_drive { sub vmconfig_apply_pending { - my ($vmid, $conf, $storecfg, $errors) = @_; + my ($vmid, $conf, $storecfg, $errors, $skip_cloud_init) = @_; + + return if !scalar(keys %{$conf->{pending}}); my $add_apply_error = sub { my ($opt, $msg) = @_; @@ -4584,6 +5186,12 @@ sub vmconfig_apply_pending { die "internal error"; } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) { vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force); + } elsif (defined($conf->{$opt}) && $opt =~ m/^net\d+$/) { + if($have_sdn) { + my $net = PVE::QemuServer::parse_net($conf->{$opt}); + eval { PVE::Network::SDN::Vnets::del_ips_from_mac($net->{bridge}, $net->{macaddr}, $conf->{name}) }; + warn if $@; + } } }; if (my $err = $@) { @@ -4596,22 +5204,53 @@ sub vmconfig_apply_pending { PVE::QemuConfig->cleanup_pending($conf); + my $generate_cloudinit = $skip_cloud_init ? 0 : undef; + foreach my $opt (keys %{$conf->{pending}}) { # add/change next if $opt eq 'delete'; # just to be sure eval { if (defined($conf->{$opt}) && is_valid_drivename($opt)) { vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt})) + } elsif (defined($conf->{pending}->{$opt}) && $opt =~ m/^net\d+$/) { + return if !$have_sdn; # return from eval if SDN is not available + + my $new_net = PVE::QemuServer::parse_net($conf->{pending}->{$opt}); + if ($conf->{$opt}) { + my $old_net = PVE::QemuServer::parse_net($conf->{$opt}); + + if (defined($old_net->{bridge}) && defined($old_net->{macaddr}) && ( + safe_string_ne($old_net->{bridge}, $new_net->{bridge}) || + safe_string_ne($old_net->{macaddr}, $new_net->{macaddr}) + )) { + PVE::Network::SDN::Vnets::del_ips_from_mac($old_net->{bridge}, $old_net->{macaddr}, $conf->{name}); + } + } + #fixme: reuse ip if mac change && same bridge + PVE::Network::SDN::Vnets::add_next_free_cidr($new_net->{bridge}, $conf->{name}, $new_net->{macaddr}, $vmid, undef, 1); } }; if (my $err = $@) { $add_apply_error->($opt, $err); } else { + + if (is_valid_drivename($opt)) { + my $drive = parse_drive($opt, $conf->{pending}->{$opt}); + $generate_cloudinit //= 1 if drive_is_cloudinit($drive); + } + $conf->{$opt} = delete $conf->{pending}->{$opt}; } } # write all changes at once to avoid unnecessary i/o PVE::QemuConfig->write_config($vmid, $conf); + if ($generate_cloudinit) { + if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) { + # After successful generation and if there were changes to be applied, update the + # config to drop the {cloudinit} entry. + PVE::QemuConfig->write_config($vmid, $conf); + } + } } sub vmconfig_update_net { @@ -4625,11 +5264,18 @@ sub vmconfig_update_net { if (safe_string_ne($oldnet->{model}, $newnet->{model}) || safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) || safe_num_ne($oldnet->{queues}, $newnet->{queues}) || - !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change + safe_num_ne($oldnet->{mtu}, $newnet->{mtu}) || + !($newnet->{bridge} && $oldnet->{bridge}) + ) { # bridge/nat mode change # for non online change, we try to hot-unplug die "skip\n" if !$hotplug; vm_deviceunplug($vmid, $conf, $opt); + + if ($have_sdn) { + PVE::Network::SDN::Vnets::del_ips_from_mac($oldnet->{bridge}, $oldnet->{macaddr}, $conf->{name}); + } + } else { die "internal error" if $opt !~ m/net(\d+)/; @@ -4638,14 +5284,37 @@ sub vmconfig_update_net { if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) || safe_num_ne($oldnet->{tag}, $newnet->{tag}) || safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) || - safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) { + safe_num_ne($oldnet->{firewall}, $newnet->{firewall}) + ) { PVE::Network::tap_unplug($iface); + #set link_down in guest if bridge or vlan change to notify guest (dhcp renew for example) + if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) || + safe_num_ne($oldnet->{tag}, $newnet->{tag}) + ) { + qemu_set_link_status($vmid, $opt, 0); + } + + if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge})) { + if ($have_sdn) { + PVE::Network::SDN::Vnets::del_ips_from_mac($oldnet->{bridge}, $oldnet->{macaddr}, $conf->{name}); + PVE::Network::SDN::Vnets::add_next_free_cidr($newnet->{bridge}, $conf->{name}, $newnet->{macaddr}, $vmid, undef, 1); + } + } + if ($have_sdn) { PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate}); } else { PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate}); } + + #set link_up in guest if bridge or vlan change to notify guest (dhcp renew for example) + if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) || + safe_num_ne($oldnet->{tag}, $newnet->{tag}) + ) { + qemu_set_link_status($vmid, $opt, 1); + } + } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) { # Rate can be applied on its own but any change above needs to # include the rate in tap_plug since OVS resets everything. @@ -4661,12 +5330,39 @@ sub vmconfig_update_net { } if ($hotplug) { + if ($have_sdn) { + PVE::Network::SDN::Vnets::add_next_free_cidr($newnet->{bridge}, $conf->{name}, $newnet->{macaddr}, $vmid, undef, 1); + PVE::Network::SDN::Vnets::add_dhcp_mapping($newnet->{bridge}, $newnet->{macaddr}, $vmid, $conf->{name}); + } vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type); } else { die "skip\n"; } } +sub vmconfig_update_agent { + my ($conf, $opt, $value) = @_; + + die "skip\n" if !$conf->{$opt}; + + my $hotplug_options = { fstrim_cloned_disks => 1 }; + + my $old_agent = parse_guest_agent($conf); + my $agent = parse_guest_agent({$opt => $value}); + + for my $option (keys %$agent) { # added/changed options + next if defined($hotplug_options->{$option}); + die "skip\n" if safe_string_ne($agent->{$option}, $old_agent->{$option}); + } + + for my $option (keys %$old_agent) { # removed options + next if defined($hotplug_options->{$option}); + die "skip\n" if safe_string_ne($old_agent->{$option}, $agent->{$option}); + } + + return; # either no actual change (e.g., format string reordered) or just hotpluggable changes +} + sub vmconfig_update_disk { my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_; @@ -4691,11 +5387,15 @@ sub vmconfig_update_disk { # update existing disk # skip non hotpluggable value - if (safe_string_ne($drive->{discard}, $old_drive->{discard}) || + if (safe_string_ne($drive->{aio}, $old_drive->{aio}) || + safe_string_ne($drive->{discard}, $old_drive->{discard}) || safe_string_ne($drive->{iothread}, $old_drive->{iothread}) || safe_string_ne($drive->{queues}, $old_drive->{queues}) || + safe_string_ne($drive->{product}, $old_drive->{product}) || safe_string_ne($drive->{cache}, $old_drive->{cache}) || - safe_string_ne($drive->{ssd}, $old_drive->{ssd})) { + safe_string_ne($drive->{ssd}, $old_drive->{ssd}) || + safe_string_ne($drive->{vendor}, $old_drive->{vendor}) || + safe_string_ne($drive->{ro}, $old_drive->{ro})) { die "skip\n"; } @@ -4775,6 +5475,37 @@ sub vmconfig_update_disk { vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type); } +sub vmconfig_update_cloudinit_drive { + my ($storecfg, $conf, $vmid) = @_; + + my $cloudinit_ds = undef; + my $cloudinit_drive = undef; + + PVE::QemuConfig->foreach_volume($conf, sub { + my ($ds, $drive) = @_; + if (PVE::QemuServer::drive_is_cloudinit($drive)) { + $cloudinit_ds = $ds; + $cloudinit_drive = $drive; + } + }); + + return if !$cloudinit_drive; + + if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) { + PVE::QemuConfig->write_config($vmid, $conf); + } + + my $running = PVE::QemuServer::check_running($vmid); + + if ($running) { + my $path = PVE::Storage::path($storecfg, $cloudinit_drive->{file}); + if ($path) { + mon_cmd($vmid, "eject", force => JSON::true, id => "$cloudinit_ds"); + mon_cmd($vmid, "blockdev-change-medium", id => "$cloudinit_ds", filename => "$path"); + } + } +} + # called in locked context by incoming migration sub vm_migrate_get_nbd_disks { my ($storecfg, $conf, $replicated_volumes) = @_; @@ -4784,6 +5515,7 @@ sub vm_migrate_get_nbd_disks { my ($ds, $drive) = @_; return if drive_is_cdrom($drive); + return if $ds eq 'tpmstate0'; my $volid = $drive->{file}; @@ -4794,9 +5526,11 @@ sub vm_migrate_get_nbd_disks { my $scfg = PVE::Storage::storage_config($storecfg, $storeid); return if $scfg->{shared}; + my $format = qemu_img_format($scfg, $volname); + # replicated disks re-use existing state via bitmap my $use_existing = $replicated_volumes->{$volid} ? 1 : 0; - $local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing]; + $local_volumes->{$ds} = [$volid, $storeid, $drive, $use_existing, $format]; }); return $local_volumes; } @@ -4805,11 +5539,9 @@ sub vm_migrate_get_nbd_disks { sub vm_migrate_alloc_nbd_disks { my ($storecfg, $vmid, $source_volumes, $storagemap) = @_; - my $format = undef; - my $nbd = {}; foreach my $opt (sort keys %$source_volumes) { - my ($volid, $storeid, $volname, $drive, $use_existing) = @{$source_volumes->{$opt}}; + my ($volid, $storeid, $drive, $use_existing, $format) = @{$source_volumes->{$opt}}; if ($use_existing) { $nbd->{$opt}->{drivestr} = print_drive($drive); @@ -4818,19 +5550,13 @@ sub vm_migrate_alloc_nbd_disks { next; } - # If a remote storage is specified and the format of the original - # volume is not available there, fall back to the default format. - # Otherwise use the same format as the original. - if (!$storagemap->{identity}) { - $storeid = map_storage($storagemap, $storeid); - my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid); - my $scfg = PVE::Storage::storage_config($storecfg, $storeid); - my $fileFormat = qemu_img_format($scfg, $volname); - $format = (grep {$fileFormat eq $_} @{$validFormats}) ? $fileFormat : $defFormat; - } else { - my $scfg = PVE::Storage::storage_config($storecfg, $storeid); - $format = qemu_img_format($scfg, $volname); - } + $storeid = PVE::JSONSchema::map_id($storagemap, $storeid); + + # order of precedence, filtered by whether storage supports it: + # 1. explicit requested format + # 2. default format of storage + my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid); + $format = $defFormat if !$format || !grep { $format eq $_ } $validFormats->@*; my $size = $drive->{size} / 1024; my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, $size); @@ -4858,13 +5584,22 @@ sub vm_start { if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf); my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended'); + my $has_backup_lock = PVE::QemuConfig->has_lock($conf, 'backup'); + + my $running = check_running($vmid, undef, $migrate_opts->{migratedfrom}); + + if ($has_backup_lock && $running) { + # a backup is currently running, attempt to start the guest in the + # existing QEMU instance + return vm_resume($vmid); + } PVE::QemuConfig->check_lock($conf) if !($params->{skiplock} || $has_suspended_lock); $params->{resume} = $has_suspended_lock || defined($conf->{vmstate}); - die "VM $vmid already running\n" if check_running($vmid, undef, $migrate_opts->{migratedfrom}); + die "VM $vmid already running\n" if $running; if (my $storagemap = $migrate_opts->{storagemap}) { my $replicated = $migrate_opts->{replicated_volumes}; @@ -4885,11 +5620,18 @@ sub vm_start { # statefile => 'tcp', 'unix' for migration or path/volid for RAM state # skiplock => 0/1, skip checking for config lock # skiptemplate => 0/1, skip checking whether VM is template -# forcemachine => to force Qemu machine (rollback/migration) +# forcemachine => to force QEMU machine (rollback/migration) # forcecpu => a QEMU '-cpu' argument string to override get_cpu_options # timeout => in seconds # paused => start VM in paused state (backup) # resume => resume from hibernation +# live-restore-backing => { +# sata0 => { +# name => blockdev-name, +# blockdev => "arg to the -blockdev command instantiating device named 'name'", +# }, +# virtio2 => ... +# } # migrate_opts: # nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks) # migratedfrom => source node @@ -4897,7 +5639,9 @@ sub vm_start { # network => CIDR of migration network # type => secure/insecure - tunnel over encrypted connection or plain-text # nbd_proto_version => int, 0 for TCP, 1 for UNIX -# replicated_volumes = which volids should be re-used with bitmaps for nbd migration +# replicated_volumes => which volids should be re-used with bitmaps for nbd migration +# offline_volumes => new volids of offline migrated disks like tpmstate and cloudinit, not yet +# contained in config sub vm_start_nolock { my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_; @@ -4918,12 +5662,33 @@ sub vm_start_nolock { $conf = PVE::QemuConfig->load_config($vmid); # update/reload } - PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid); + # don't regenerate the ISO if the VM is started as part of a live migration + # this way we can reuse the old ISO with the correct config + if (!$migratedfrom) { + if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) { + # FIXME: apply_cloudinit_config updates $conf in this case, and it would only drop + # $conf->{cloudinit}, so we could just not do this? + # But we do it above, so for now let's be consistent. + $conf = PVE::QemuConfig->load_config($vmid); # update/reload + } + } + + # override offline migrated volumes, conf is out of date still + if (my $offline_volumes = $migrate_opts->{offline_volumes}) { + for my $key (sort keys $offline_volumes->%*) { + my $parsed = parse_drive($key, $conf->{$key}); + $parsed->{file} = $offline_volumes->{$key}; + $conf->{$key} = print_drive($parsed); + } + } my $defaults = load_defaults(); # set environment variable useful inside network script - $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom; + # for remote migration the config is available on the target node! + if (!$migrate_opts->{remote_node}) { + $ENV{PVE_MIGRATED_FROM} = $migratedfrom; + } PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1); @@ -4936,8 +5701,8 @@ sub vm_start_nolock { print "Resuming suspended VM\n"; } - my ($cmd, $vollist, $spice_port) = - config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu); + my ($cmd, $vollist, $spice_port, $pci_devices) = config_to_command($storecfg, $vmid, + $conf, $defaults, $forcemachine, $forcecpu, $params->{'live-restore-backing'}); my $migration_ip; my $get_migration_ip = sub { @@ -4970,10 +5735,10 @@ sub vm_start_nolock { return $migration_ip; }; - my $migrate_uri; if ($statefile) { if ($statefile eq 'tcp') { - my $localip = "localhost"; + my $migrate = $res->{migrate} = { proto => 'tcp' }; + $migrate->{addr} = "localhost"; my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg'); my $nodename = nodename(); @@ -4986,26 +5751,25 @@ sub vm_start_nolock { } if ($migration_type eq 'insecure') { - $localip = $get_migration_ip->($nodename); - $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip); + $migrate->{addr} = $get_migration_ip->($nodename); + $migrate->{addr} = "[$migrate->{addr}]" if Net::IP::ip_is_ipv6($migrate->{addr}); } - my $pfamily = PVE::Tools::get_host_address_family($nodename); - my $migrate_port = PVE::Tools::next_migrate_port($pfamily); - $migrate_uri = "tcp:${localip}:${migrate_port}"; - push @$cmd, '-incoming', $migrate_uri; + # see #4501: port reservation should be done close to usage - tell QEMU where to listen + # via QMP later + push @$cmd, '-incoming', 'defer'; push @$cmd, '-S'; } elsif ($statefile eq 'unix') { # should be default for secure migrations as a ssh TCP forward # tunnel is not deterministic reliable ready and fails regurarly # to set up in time, so use UNIX socket forwards - my $socket_addr = "/run/qemu-server/$vmid.migrate"; - unlink $socket_addr; - - $migrate_uri = "unix:$socket_addr"; + my $migrate = $res->{migrate} = { proto => 'unix' }; + $migrate->{addr} = "/run/qemu-server/$vmid.migrate"; + unlink $migrate->{addr}; - push @$cmd, '-incoming', $migrate_uri; + $migrate->{uri} = "unix:$migrate->{addr}"; + push @$cmd, '-incoming', $migrate->{uri}; push @$cmd, '-S'; } elsif (-e $statefile) { @@ -5019,44 +5783,68 @@ sub vm_start_nolock { push @$cmd, '-S'; } - # host pci devices - for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++) { - my $d = parse_hostpci($conf->{"hostpci$i"}); - next if !$d; - my $pcidevices = $d->{pciid}; - foreach my $pcidevice (@$pcidevices) { - my $pciid = $pcidevice->{id}; + my $memory = get_current_memory($conf->{memory}); + my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $memory, $resume); - my $info = PVE::SysFSTools::pci_device_info("$pciid"); - die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support(); - die "no pci device info for device '$pciid'\n" if !$info; + my $pci_reserve_list = []; + for my $device (values $pci_devices->%*) { + next if $device->{mdev}; # we don't reserve for mdev devices + push $pci_reserve_list->@*, map { $_->{id} } $device->{ids}->@*; + } - if ($d->{mdev}) { - my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i); - PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev}); - } else { - die "can't unbind/bind PCI group to VFIO '$pciid'\n" - if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid); - die "can't reset PCI device '$pciid'\n" - if $info->{has_fl_reset} && !PVE::SysFSTools::pci_dev_reset($info); + # reserve all PCI IDs before actually doing anything with them + PVE::QemuServer::PCI::reserve_pci_usage($pci_reserve_list, $vmid, $start_timeout); + + eval { + my $uuid; + for my $id (sort keys %$pci_devices) { + my $d = $pci_devices->{$id}; + my ($index) = ($id =~ m/^hostpci(\d+)$/); + + my $chosen_mdev; + for my $dev ($d->{ids}->@*) { + my $info = eval { PVE::QemuServer::PCI::prepare_pci_device($vmid, $dev->{id}, $index, $d->{mdev}) }; + if ($d->{mdev}) { + warn $@ if $@; + $chosen_mdev = $info; + last if $chosen_mdev; # if successful, we're done + } else { + die $@ if $@; + } } - } + + next if !$d->{mdev}; + die "could not create mediated device\n" if !defined($chosen_mdev); + + # nvidia grid needs the uuid of the mdev as qemu parameter + if (!defined($uuid) && $chosen_mdev->{vendor} =~ m/^(0x)?10de$/) { + if (defined($conf->{smbios1})) { + my $smbios_conf = parse_smbios1($conf->{smbios1}); + $uuid = $smbios_conf->{uuid} if defined($smbios_conf->{uuid}); + } + $uuid = PVE::QemuServer::PCI::generate_mdev_uuid($vmid, $index) if !defined($uuid); + } + } + push @$cmd, '-uuid', $uuid if defined($uuid); + }; + if (my $err = $@) { + eval { cleanup_pci_devices($vmid, $conf) }; + warn $@ if $@; + die $err; } PVE::Storage::activate_volumes($storecfg, $vollist); - eval { - run_command(['/bin/systemctl', 'stop', "$vmid.scope"], - outfunc => sub {}, errfunc => sub {}); - }; + + my %silence_std_outs = (outfunc => sub {}, errfunc => sub {}); + eval { run_command(['/bin/systemctl', 'reset-failed', "$vmid.scope"], %silence_std_outs) }; + eval { run_command(['/bin/systemctl', 'stop', "$vmid.scope"], %silence_std_outs) }; # Issues with the above 'stop' not being fully completed are extremely rare, a very low # timeout should be more than enough here... - PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5); + PVE::Systemd::wait_for_unit_removed("$vmid.scope", 20); - my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits} - : $defaults->{cpuunits}; + my $cpuunits = PVE::CGroup::clamp_cpu_shares($conf->{cpuunits}); - my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $resume); my %run_params = ( timeout => $statefile ? undef : $start_timeout, umask => 0077, @@ -5070,30 +5858,53 @@ sub vm_start_nolock { $run_params{logfunc} = sub { print "QEMU: $_[0]\n" }; } - my %properties = ( + my %systemd_properties = ( Slice => 'qemu.slice', - KillMode => 'none', - CPUShares => $cpuunits + KillMode => 'process', + SendSIGKILL => 0, + TimeoutStopUSec => ULONG_MAX, # infinity ); + if (PVE::CGroup::cgroup_mode() == 2) { + $systemd_properties{CPUWeight} = $cpuunits; + } else { + $systemd_properties{CPUShares} = $cpuunits; + } + if (my $cpulimit = $conf->{cpulimit}) { - $properties{CPUQuota} = int($cpulimit * 100); + $systemd_properties{CPUQuota} = int($cpulimit * 100); } - $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick + $systemd_properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick my $run_qemu = sub { PVE::Tools::run_fork sub { - PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties); + PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %systemd_properties); + + my $tpmpid; + if ((my $tpm = $conf->{tpmstate0}) && !PVE::QemuConfig->is_template($conf)) { + # start the TPM emulator so QEMU can connect on start + $tpmpid = start_swtpm($storecfg, $vmid, $tpm, $migratedfrom); + } my $exitcode = run_command($cmd, %run_params); - die "QEMU exited with code $exitcode\n" if $exitcode; + if ($exitcode) { + if ($tpmpid) { + warn "stopping swtpm instance (pid $tpmpid) due to QEMU startup error\n"; + kill 'TERM', $tpmpid; + } + die "QEMU exited with code $exitcode\n"; + } }; }; if ($conf->{hugepages}) { my $code = sub { - my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf); + my $hotplug_features = + parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1'); + my $hugepages_topology = + PVE::QemuServer::Memory::hugepages_topology($conf, $hotplug_features->{memory}); + my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology(); PVE::QemuServer::Memory::hugepages_mount(); @@ -5118,13 +5929,28 @@ sub vm_start_nolock { if (my $err = $@) { # deactivate volumes if start fails eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); }; + warn $@ if $@; + eval { cleanup_pci_devices($vmid, $conf) }; + warn $@ if $@; + die "start failed: $err"; } - print "migration listens on $migrate_uri\n" if $migrate_uri; - $res->{migrate_uri} = $migrate_uri; + # re-reserve all PCI IDs now that we can know the actual VM PID + my $pid = PVE::QemuServer::Helpers::vm_running_locally($vmid); + eval { PVE::QemuServer::PCI::reserve_pci_usage($pci_reserve_list, $vmid, undef, $pid) }; + warn $@ if $@; - if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix') { + if (defined(my $migrate = $res->{migrate})) { + if ($migrate->{proto} eq 'tcp') { + my $nodename = nodename(); + my $pfamily = PVE::Tools::get_host_address_family($nodename); + $migrate->{port} = PVE::Tools::next_migrate_port($pfamily); + $migrate->{uri} = "tcp:$migrate->{addr}:$migrate->{port}"; + mon_cmd($vmid, "migrate-incoming", uri => $migrate->{uri}); + } + print "migration listens on $migrate->{uri}\n"; + } elsif ($statefile) { eval { mon_cmd($vmid, "cont"); }; warn $@ if $@; } @@ -5135,10 +5961,11 @@ sub vm_start_nolock { my $migrate_storage_uri; # nbd_protocol_version > 0 for unix socket support - if ($nbd_protocol_version > 0 && $migration_type eq 'secure') { + if ($nbd_protocol_version > 0 && ($migration_type eq 'secure' || $migration_type eq 'websocket')) { my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate"; mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } ); $migrate_storage_uri = "nbd:unix:$socket_path"; + $res->{migrate}->{unix_sockets} = [$socket_path]; } else { my $nodename = nodename(); my $localip = $get_migration_ip->($nodename); @@ -5156,12 +5983,25 @@ sub vm_start_nolock { $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}"; } - $res->{migrate_storage_uri} = $migrate_storage_uri; + my $block_info = mon_cmd($vmid, "query-block"); + $block_info = { map { $_->{device} => $_ } $block_info->@* }; foreach my $opt (sort keys %$nbd) { my $drivestr = $nbd->{$opt}->{drivestr}; my $volid = $nbd->{$opt}->{volid}; - mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true ); + + my $block_node = $block_info->{"drive-$opt"}->{inserted}->{'node-name'}; + + mon_cmd( + $vmid, + "block-export-add", + id => "drive-$opt", + 'node-name' => $block_node, + writable => JSON::true, + type => "nbd", + name => "drive-$opt", # NBD export name + ); + my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt"; print "storage migration listens on $nbd_uri volume:$drivestr\n"; print "re-using replicated volume: $opt - $volid\n" @@ -5197,12 +6037,21 @@ sub vm_start_nolock { my $nicconf = parse_net($conf->{$opt}); qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down}; } + add_nets_bridge_fdb($conf, $vmid); } - mon_cmd($vmid, 'qom-set', + if (!defined($conf->{balloon}) || $conf->{balloon}) { + eval { + mon_cmd( + $vmid, + 'qom-set', path => "machine/peripheral/balloon0", property => "guest-stats-polling-interval", - value => 2) if (!defined($conf->{balloon}) || $conf->{balloon}); + value => 2 + ); + }; + log_warn("could not set polling interval for ballooning - $@") if $@; + } if ($resume) { print "Resumed VM, removing state\n"; @@ -5216,6 +6065,15 @@ sub vm_start_nolock { PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start'); + my ($current_machine, $is_deprecated) = + PVE::QemuServer::Machine::get_current_qemu_machine($vmid); + if ($is_deprecated) { + log_warn( + "current machine version '$current_machine' is deprecated - see the documentation and ". + "change to a newer one", + ); + } + return $res; } @@ -5223,9 +6081,8 @@ sub vm_commandline { my ($storecfg, $vmid, $snapname) = @_; my $conf = PVE::QemuConfig->load_config($vmid); - my $forcemachine; - my $forcecpu; + my ($forcemachine, $forcecpu); if ($snapname) { my $snapshot = $conf->{snapshots}->{$snapname}; die "snapshot '$snapname' does not exist\n" if !defined($snapshot); @@ -5241,8 +6098,7 @@ sub vm_commandline { my $defaults = load_defaults(); - my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults, - $forcemachine, $forcecpu); + my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu); return PVE::Tools::cmd2string($cmd); } @@ -5278,6 +6134,39 @@ sub get_vm_volumes { return $vollist; } +sub cleanup_pci_devices { + my ($vmid, $conf) = @_; + + foreach my $key (keys %$conf) { + next if $key !~ m/^hostpci(\d+)$/; + my $hostpciindex = $1; + my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex); + my $d = parse_hostpci($conf->{$key}); + if ($d->{mdev}) { + # NOTE: avoid PVE::SysFSTools::pci_cleanup_mdev_device as it requires PCI ID and we + # don't want to break ABI just for this two liner + my $dev_sysfs_dir = "/sys/bus/mdev/devices/$uuid"; + + # some nvidia vgpu driver versions want to clean the mdevs up themselves, and error + # out when we do it first. so wait for up to 10 seconds and then try it manually + if ($d->{ids}->[0]->[0]->{vendor} =~ m/^(0x)?10de$/ && -e $dev_sysfs_dir) { + my $count = 0; + while (-e $dev_sysfs_dir && $count < 10) { + sleep 1; + $count++; + } + print "waited $count seconds for mediated device driver finishing clean up\n"; + } + + if (-e $dev_sysfs_dir) { + print "actively clean up mediated device with UUID $uuid\n"; + PVE::SysFSTools::file_write("$dev_sysfs_dir/remove", "1"); + } + } + } + PVE::QemuServer::PCI::remove_pci_reservation($vmid); +} + sub vm_stop_cleanup { my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_; @@ -5286,6 +6175,14 @@ sub vm_stop_cleanup { if (!$keepActive) { my $vollist = get_vm_volumes($conf); PVE::Storage::deactivate_volumes($storecfg, $vollist); + + if (my $tpmdrive = $conf->{tpmstate0}) { + my $tpm = parse_drive("tpmstate0", $tpmdrive); + my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1); + if ($storeid) { + PVE::Storage::unmap_volume($storecfg, $tpm->{file}); + } + } } foreach my $ext (qw(mon qmp pid vnc qga)) { @@ -5301,17 +6198,7 @@ sub vm_stop_cleanup { unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid); } - foreach my $key (keys %$conf) { - next if $key !~ m/^hostpci(\d+)$/; - my $hostpciindex = $1; - my $d = parse_hostpci($conf->{$key}); - my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex); - - foreach my $pci (@{$d->{pciid}}) { - my $pciid = $pci->{id}; - PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid); - } - } + cleanup_pci_devices($vmid, $conf); vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes; }; @@ -5338,7 +6225,7 @@ sub _do_vm_stop { eval { if ($shutdown) { - if (defined($conf) && parse_guest_agent($conf)->{enabled}) { + if (defined($conf) && get_qga_key($conf, 'enabled')) { mon_cmd($vmid, "guest-shutdown", timeout => $timeout); } else { mon_cmd($vmid, "system_powerdown"); @@ -5493,6 +6380,7 @@ sub vm_suspend { PVE::Storage::activate_volumes($storecfg, [$vmstate]); eval { + set_migration_caps($vmid, 1); mon_cmd($vmid, "savevm-start", statefile => $path); for(;;) { my $state = mon_cmd($vmid, "query-savevm"); @@ -5538,25 +6426,53 @@ sub vm_suspend { } } +# $nocheck is set when called as part of a migration - in this context the +# location of the config file (source or target node) is not deterministic, +# since migration cannot wait for pmxcfs to process the rename sub vm_resume { my ($vmid, $skiplock, $nocheck) = @_; PVE::QemuConfig->lock_config($vmid, sub { my $res = mon_cmd($vmid, 'query-status'); my $resume_cmd = 'cont'; + my $reset = 0; + my $conf; + if ($nocheck) { + $conf = eval { PVE::QemuConfig->load_config($vmid) }; # try on target node + if ($@) { + my $vmlist = PVE::Cluster::get_vmlist(); + if (exists($vmlist->{ids}->{$vmid})) { + my $node = $vmlist->{ids}->{$vmid}->{node}; + $conf = eval { PVE::QemuConfig->load_config($vmid, $node) }; # try on source node + } + if (!$conf) { + PVE::Cluster::cfs_update(); # vmlist was wrong, invalidate cache + $conf = PVE::QemuConfig->load_config($vmid); # last try on target node again + } + } + } else { + $conf = PVE::QemuConfig->load_config($vmid); + } - if ($res->{status} && $res->{status} eq 'suspended') { - $resume_cmd = 'system_wakeup'; + if ($res->{status}) { + return if $res->{status} eq 'running'; # job done, go home + $resume_cmd = 'system_wakeup' if $res->{status} eq 'suspended'; + $reset = 1 if $res->{status} eq 'shutdown'; } if (!$nocheck) { - - my $conf = PVE::QemuConfig->load_config($vmid); - PVE::QemuConfig->check_lock($conf) if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup')); } + if ($reset) { + # required if a VM shuts down during a backup and we get a resume + # request before the backup finishes for example + mon_cmd($vmid, "system_reset"); + } + + add_nets_bridge_fdb($conf, $vmid) if $resume_cmd eq 'cont'; + mon_cmd($vmid, $resume_cmd); }); } @@ -5574,6 +6490,53 @@ sub vm_sendkey { }); } +sub check_bridge_access { + my ($rpcenv, $authuser, $conf) = @_; + + return 1 if $authuser eq 'root@pam'; + + for my $opt (sort keys $conf->%*) { + next if $opt !~ m/^net\d+$/; + my $net = parse_net($conf->{$opt}); + my ($bridge, $tag, $trunks) = $net->@{'bridge', 'tag', 'trunks'}; + PVE::GuestHelpers::check_vnet_access($rpcenv, $authuser, $bridge, $tag, $trunks); + } + return 1; +}; + +sub check_mapping_access { + my ($rpcenv, $user, $conf) = @_; + + for my $opt (keys $conf->%*) { + if ($opt =~ m/^usb\d+$/) { + my $device = PVE::JSONSchema::parse_property_string('pve-qm-usb', $conf->{$opt}); + if (my $host = $device->{host}) { + die "only root can set '$opt' config for real devices\n" + if $host !~ m/^spice$/i && $user ne 'root@pam'; + } elsif ($device->{mapping}) { + $rpcenv->check_full($user, "/mapping/usb/$device->{mapping}", ['Mapping.Use']); + } else { + die "either 'host' or 'mapping' must be set.\n"; + } + } elsif ($opt =~ m/^hostpci\d+$/) { + my $device = PVE::JSONSchema::parse_property_string('pve-qm-hostpci', $conf->{$opt}); + if ($device->{host}) { + die "only root can set '$opt' config for non-mapped devices\n" if $user ne 'root@pam'; + } elsif ($device->{mapping}) { + $rpcenv->check_full($user, "/mapping/pci/$device->{mapping}", ['Mapping.Use']); + } else { + die "either 'host' or 'mapping' must be set.\n"; + } + } + } +}; + +sub check_restore_permissions { + my ($rpcenv, $user, $conf) = @_; + + check_bridge_access($rpcenv, $user, $conf); + check_mapping_access($rpcenv, $user, $conf); +} # vzdump restore implementaion sub tar_archive_read_firstfile { @@ -5642,6 +6605,8 @@ sub restore_file_archive { my $restore_cleanup_oldconf = sub { my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_; + my $kept_disks = {}; + PVE::QemuConfig->foreach_volume($oldconf, sub { my ($ds, $drive) = @_; @@ -5660,11 +6625,13 @@ my $restore_cleanup_oldconf = sub { if (my $err = $@) { warn $err; } + } else { + $kept_disks->{$volid} = 1; } }); - # delete vmstate files, after the restore we have no snapshots anymore - foreach my $snapname (keys %{$oldconf->{snapshots}}) { + # after the restore we have no snapshots anymore + for my $snapname (keys $oldconf->{snapshots}->%*) { my $snap = $oldconf->{snapshots}->{$snapname}; if ($snap->{vmstate}) { eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); }; @@ -5672,6 +6639,11 @@ my $restore_cleanup_oldconf = sub { warn $err; } } + + for my $volid (keys $kept_disks->%*) { + eval { PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snapname); }; + warn $@ if $@; + } } }; @@ -5688,8 +6660,15 @@ my $restore_cleanup_oldconf = sub { my $parse_backup_hints = sub { my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_; - my $virtdev_hash = {}; + my $check_storage = sub { # assert if an image can be allocate + my ($storeid, $scfg) = @_; + die "Content type 'images' is not available on storage '$storeid'\n" + if !$scfg->{content}->{images}; + $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']) + if $user ne 'root@pam'; + }; + my $virtdev_hash = {}; while (defined(my $line = <$fh>)) { if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) { my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4); @@ -5707,22 +6686,22 @@ my $parse_backup_hints = sub { $devinfo->{$devname}->{format} = $format; $devinfo->{$devname}->{storeid} = $storeid; - # check permission on storage - my $pool = $options->{pool}; # todo: do we need that? - if ($user ne 'root@pam') { - $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']); - } + my $scfg = PVE::Storage::storage_config($storecfg, $storeid); + $check_storage->($storeid, $scfg); # permission and content type check $virtdev_hash->{$virtdev} = $devinfo->{$devname}; } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) { my $virtdev = $1; my $drive = parse_drive($virtdev, $2); + if (drive_is_cloudinit($drive)) { my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}); $storeid = $options->{storage} if defined ($options->{storage}); my $scfg = PVE::Storage::storage_config($storecfg, $storeid); my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback + $check_storage->($storeid, $scfg); # permission and content type check + $virtdev_hash->{$virtdev} = { format => $format, storeid => $storeid, @@ -5760,7 +6739,10 @@ my $restore_allocate_devices = sub { my $name; if ($d->{is_cloudinit}) { $name = "vm-$vmid-cloudinit"; - $name .= ".$d->{format}" if $d->{format} ne 'raw'; + my $scfg = PVE::Storage::storage_config($storecfg, $storeid); + if ($scfg->{path}) { + $name .= ".$d->{format}"; + } } my $volid = PVE::Storage::vdisk_alloc( @@ -5777,14 +6759,16 @@ my $restore_allocate_devices = sub { return $map; }; -my $restore_update_config_line = sub { - my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_; +sub restore_update_config_line { + my ($cookie, $map, $line, $unique) = @_; + + return '' if $line =~ m/^\#qmdump\#/; + return '' if $line =~ m/^\#vzdump\#/; + return '' if $line =~ m/^lock:/; + return '' if $line =~ m/^unused\d+:/; + return '' if $line =~ m/^parent:/; - return if $line =~ m/^\#qmdump\#/; - return if $line =~ m/^\#vzdump\#/; - return if $line =~ m/^lock:/; - return if $line =~ m/^unused\d+:/; - return if $line =~ m/^parent:/; + my $res = ''; my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg'); if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) { @@ -5800,7 +6784,7 @@ my $restore_update_config_line = sub { }; my $netstr = print_net($net); - print $outfd "net$cookie->{netcount}: $netstr\n"; + $res .= "net$cookie->{netcount}: $netstr\n"; $cookie->{netcount}++; } } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) { @@ -5808,20 +6792,20 @@ my $restore_update_config_line = sub { my $net = parse_net($netstr); $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr}; $netstr = print_net($net); - print $outfd "$id: $netstr\n"; - } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) { + $res .= "$id: $netstr\n"; + } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk|tpmstate)\d+):\s*(\S+)\s*$/) { my $virtdev = $1; my $value = $3; my $di = parse_drive($virtdev, $value); if (defined($di->{backup}) && !$di->{backup}) { - print $outfd "#$line"; + $res .= "#$line"; } elsif ($map->{$virtdev}) { delete $di->{format}; # format can change on restore $di->{file} = $map->{$virtdev}; $value = print_drive($di); - print $outfd "$virtdev: $value\n"; + $res .= "$virtdev: $value\n"; } else { - print $outfd $line; + $res .= $line; } } elsif (($line =~ m/^vmgenid: (.*)/)) { my $vmgenid = $1; @@ -5829,53 +6813,61 @@ my $restore_update_config_line = sub { # always generate a new vmgenid if there was a valid one setup $vmgenid = generate_uuid(); } - print $outfd "vmgenid: $vmgenid\n"; + $res .= "vmgenid: $vmgenid\n"; } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) { my ($uuid, $uuid_str); UUID::generate($uuid); UUID::unparse($uuid, $uuid_str); my $smbios1 = parse_smbios1($2); $smbios1->{uuid} = $uuid_str; - print $outfd $1.print_smbios1($smbios1)."\n"; + $res .= $1.print_smbios1($smbios1)."\n"; } else { - print $outfd $line; + $res .= $line; } -}; + + return $res; +} my $restore_deactivate_volumes = sub { - my ($storecfg, $devinfo) = @_; + my ($storecfg, $virtdev_hash) = @_; my $vollist = []; - foreach my $devname (keys %$devinfo) { - my $volid = $devinfo->{$devname}->{volid}; - push @$vollist, $volid if $volid; + for my $dev (values $virtdev_hash->%*) { + push $vollist->@*, $dev->{volid} if $dev->{volid}; } - PVE::Storage::deactivate_volumes($storecfg, $vollist); + eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); }; + print STDERR $@ if $@; }; my $restore_destroy_volumes = sub { - my ($storecfg, $devinfo) = @_; + my ($storecfg, $virtdev_hash) = @_; - foreach my $devname (keys %$devinfo) { - my $volid = $devinfo->{$devname}->{volid}; - next if !$volid; + for my $dev (values $virtdev_hash->%*) { + my $volid = $dev->{volid} or next; eval { - if ($volid =~ m|^/|) { - unlink $volid || die 'unlink failed\n'; - } else { - PVE::Storage::vdisk_free($storecfg, $volid); - } + PVE::Storage::vdisk_free($storecfg, $volid); print STDERR "temporary volume '$volid' sucessfuly removed\n"; }; print STDERR "unable to cleanup '$volid' - $@" if $@; } }; +sub restore_merge_config { + my ($filename, $backup_conf_raw, $override_conf) = @_; + + my $backup_conf = parse_vm_config($filename, $backup_conf_raw); + for my $key (keys $override_conf->%*) { + $backup_conf->{$key} = $override_conf->{$key}; + } + + return $backup_conf; +} + sub scan_volids { my ($cfg, $vmid) = @_; - my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid); + my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid, undef, 'images'); my $volid_hash = {}; foreach my $storeid (keys %$info) { @@ -5968,12 +6960,6 @@ sub rescan { my $cfg = PVE::Storage::config(); - # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage - # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html - foreach my $stor (keys %{$cfg->{ids}}) { - delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images}; - } - print "rescan volumes...\n"; my $volid_hash = scan_volids($cfg, $vmid); @@ -6021,15 +7007,13 @@ sub restore_proxmox_backup_archive { my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive); my $scfg = PVE::Storage::storage_config($storecfg, $storeid); - my $server = $scfg->{server}; - my $datastore = $scfg->{datastore}; - my $username = $scfg->{username} // 'root@pam'; my $fingerprint = $scfg->{fingerprint}; my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($storecfg, $storeid); - my $repo = "$username\@$server:$datastore"; + my $repo = PVE::PBSClient::get_repository($scfg); + my $namespace = $scfg->{namespace}; - # This is only used for `pbs-restore`! + # This is only used for `pbs-restore` and the QEMU PBS driver (live-restore) my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid); local $ENV{PBS_PASSWORD} = $password; local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint); @@ -6046,7 +7030,6 @@ sub restore_proxmox_backup_archive { mkpath $tmpdir; my $conffile = PVE::QemuConfig->config_file($vmid); - my $tmpfn = "$conffile.$$.tmp"; # disable interrupts (always do cleanups) local $SIG{INT} = local $SIG{TERM} = @@ -6056,9 +7039,11 @@ sub restore_proxmox_backup_archive { # Note: $oldconf is undef if VM does not exists my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid); my $oldconf = PVE::Cluster::cfs_read_file($cfs_path); + my $new_conf_raw = ''; my $rpcenv = PVE::RPCEnvironment::get(); - my $devinfo = {}; + my $devinfo = {}; # info about drives included in backup + my $virtdev_hash = {}; # info about allocated drives eval { # enable interrupts @@ -6079,7 +7064,6 @@ sub restore_proxmox_backup_archive { my $index = PVE::Tools::file_get_contents($index_fn); $index = decode_json($index); - # print Dumper($index); foreach my $info (@{$index->{files}}) { if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) { my $devname = $1; @@ -6114,7 +7098,7 @@ sub restore_proxmox_backup_archive { my $fh = IO::File->new($cfgfn, "r") || die "unable to read qemu-server.conf - $!\n"; - my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options); + $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options); # fixme: rate limit? @@ -6130,14 +7114,22 @@ sub restore_proxmox_backup_archive { my $d = $virtdev_hash->{$virtdev}; next if $d->{is_cloudinit}; # no need to restore cloudinit + # this fails if storage is unavailable my $volid = $d->{volid}; - my $path = PVE::Storage::path($storecfg, $volid); - # This is the ONLY user of the PBS_ env vars set on top of this function! + # for live-restore we only want to preload the efidisk and TPM state + next if $options->{live} && $virtdev ne 'efidisk0' && $virtdev ne 'tpmstate0'; + + my @ns_arg; + if (defined(my $ns = $scfg->{namespace})) { + @ns_arg = ('--ns', $ns); + } + my $pbs_restore_cmd = [ '/usr/bin/pbs-restore', '--repository', $repo, + @ns_arg, $pbs_backup_name, "$d->{devname}.img.fidx", $path, @@ -6158,35 +7150,235 @@ sub restore_proxmox_backup_archive { $fh->seek(0, 0) || die "seek failed - $!\n"; - my $outfd = IO::File->new($tmpfn, "w") || die "unable to write config for VM $vmid\n"; - my $cookie = { netcount => 0 }; while (defined(my $line = <$fh>)) { - $restore_update_config_line->($outfd, $cookie, $vmid, $map, $line, $options->{unique}); + $new_conf_raw .= restore_update_config_line( + $cookie, + $map, + $line, + $options->{unique}, + ); } $fh->close(); - $outfd->close(); }; my $err = $@; - $restore_deactivate_volumes->($storecfg, $devinfo); + if ($err || !$options->{live}) { + $restore_deactivate_volumes->($storecfg, $virtdev_hash); + } rmtree $tmpdir; if ($err) { - unlink $tmpfn; - $restore_destroy_volumes->($storecfg, $devinfo); + $restore_destroy_volumes->($storecfg, $virtdev_hash); die $err; } - rename($tmpfn, $conffile) || - die "unable to commit configuration file '$conffile'\n"; + if ($options->{live}) { + # keep lock during live-restore + $new_conf_raw .= "\nlock: create"; + } - PVE::Cluster::cfs_update(); # make sure we read new file + my $new_conf = restore_merge_config($conffile, $new_conf_raw, $options->{override_conf}); + check_restore_permissions($rpcenv, $user, $new_conf); + PVE::QemuConfig->write_config($vmid, $new_conf); eval { rescan($vmid, 1); }; warn $@ if $@; + + PVE::AccessControl::add_vm_to_pool($vmid, $options->{pool}) if $options->{pool}; + + if ($options->{live}) { + # enable interrupts + local $SIG{INT} = + local $SIG{TERM} = + local $SIG{QUIT} = + local $SIG{HUP} = + local $SIG{PIPE} = sub { die "got signal ($!) - abort\n"; }; + + my $conf = PVE::QemuConfig->load_config($vmid); + die "cannot do live-restore for template\n" if PVE::QemuConfig->is_template($conf); + + # these special drives are already restored before start + delete $devinfo->{'drive-efidisk0'}; + delete $devinfo->{'drive-tpmstate0-backup'}; + + my $pbs_opts = { + repo => $repo, + keyfile => $keyfile, + snapshot => $pbs_backup_name, + namespace => $namespace, + }; + pbs_live_restore($vmid, $conf, $storecfg, $devinfo, $pbs_opts); + + PVE::QemuConfig->remove_lock($vmid, "create"); + } +} + +sub pbs_live_restore { + my ($vmid, $conf, $storecfg, $restored_disks, $opts) = @_; + + print "starting VM for live-restore\n"; + print "repository: '$opts->{repo}', snapshot: '$opts->{snapshot}'\n"; + + my $live_restore_backing = {}; + for my $ds (keys %$restored_disks) { + $ds =~ m/^drive-(.*)$/; + my $confname = $1; + my $pbs_conf = {}; + $pbs_conf = { + repository => $opts->{repo}, + snapshot => $opts->{snapshot}, + archive => "$ds.img.fidx", + }; + $pbs_conf->{keyfile} = $opts->{keyfile} if -e $opts->{keyfile}; + $pbs_conf->{namespace} = $opts->{namespace} if defined($opts->{namespace}); + + my $drive = parse_drive($confname, $conf->{$confname}); + print "restoring '$ds' to '$drive->{file}'\n"; + + my $pbs_name = "drive-${confname}-pbs"; + $live_restore_backing->{$confname} = { + name => $pbs_name, + blockdev => print_pbs_blockdev($pbs_conf, $pbs_name), + }; + } + + my $drives_streamed = 0; + eval { + # make sure HA doesn't interrupt our restore by stopping the VM + if (PVE::HA::Config::vm_is_ha_managed($vmid)) { + run_command(['ha-manager', 'set', "vm:$vmid", '--state', 'started']); + } + + # start VM with backing chain pointing to PBS backup, environment vars for PBS driver + # in QEMU (PBS_PASSWORD and PBS_FINGERPRINT) are already set by our caller + vm_start_nolock($storecfg, $vmid, $conf, {paused => 1, 'live-restore-backing' => $live_restore_backing}, {}); + + my $qmeventd_fd = register_qmeventd_handle($vmid); + + # begin streaming, i.e. data copy from PBS to target disk for every vol, + # this will effectively collapse the backing image chain consisting of + # [target <- alloc-track -> PBS snapshot] to just [target] (alloc-track + # removes itself once all backing images vanish with 'auto-remove=on') + my $jobs = {}; + for my $ds (sort keys %$restored_disks) { + my $job_id = "restore-$ds"; + mon_cmd($vmid, 'block-stream', + 'job-id' => $job_id, + device => "$ds", + ); + $jobs->{$job_id} = {}; + } + + mon_cmd($vmid, 'cont'); + qemu_drive_mirror_monitor($vmid, undef, $jobs, 'auto', 0, 'stream'); + + print "restore-drive jobs finished successfully, removing all tracking block devices" + ." to disconnect from Proxmox Backup Server\n"; + + for my $ds (sort keys %$restored_disks) { + mon_cmd($vmid, 'blockdev-del', 'node-name' => "$ds-pbs"); + } + + close($qmeventd_fd); + }; + + my $err = $@; + + if ($err) { + warn "An error occurred during live-restore: $err\n"; + _do_vm_stop($storecfg, $vmid, 1, 1, 10, 0, 1); + die "live-restore failed\n"; + } +} + +# Inspired by pbs live-restore, this restores with the disks being available as files. +# Theoretically this can also be used to quick-start a full-clone vm if the +# disks are all available as files. +# +# The mapping should provide a path by config entry, such as +# `{ scsi0 => { format => , path => "/path/to/file", sata1 => ... } }` +# +# This is used when doing a `create` call with the `--live-import` parameter, +# where the disks get an `import-from=` property. The non-live part is +# therefore already handled in the `$create_disks()` call happening in the +# `create` api call +sub live_import_from_files { + my ($mapping, $vmid, $conf, $restore_options) = @_; + + my $live_restore_backing = {}; + for my $dev (keys %$mapping) { + die "disk not support for live-restoring: '$dev'\n" + if !is_valid_drivename($dev) || $dev =~ /^(?:efidisk|tpmstate)/; + + die "mapping contains disk '$dev' which does not exist in the config\n" + if !exists($conf->{$dev}); + + my $info = $mapping->{$dev}; + my ($format, $path) = $info->@{qw(format path)}; + die "missing path for '$dev' mapping\n" if !$path; + die "missing format for '$dev' mapping\n" if !$format; + die "invalid format '$format' for '$dev' mapping\n" + if !grep { $format eq $_ } qw(raw qcow2 vmdk); + + $live_restore_backing->{$dev} = { + name => "drive-$dev-restore", + blockdev => "driver=$format,node-name=drive-$dev-restore" + . ",read-only=on" + . ",file.driver=file,file.filename=$path" + }; + }; + + my $storecfg = PVE::Storage::config(); + eval { + + # make sure HA doesn't interrupt our restore by stopping the VM + if (PVE::HA::Config::vm_is_ha_managed($vmid)) { + run_command(['ha-manager', 'set', "vm:$vmid", '--state', 'started']); + } + + vm_start_nolock($storecfg, $vmid, $conf, {paused => 1, 'live-restore-backing' => $live_restore_backing}, {}); + + # prevent shutdowns from qmeventd when the VM powers off from the inside + my $qmeventd_fd = register_qmeventd_handle($vmid); + + # begin streaming, i.e. data copy from PBS to target disk for every vol, + # this will effectively collapse the backing image chain consisting of + # [target <- alloc-track -> PBS snapshot] to just [target] (alloc-track + # removes itself once all backing images vanish with 'auto-remove=on') + my $jobs = {}; + for my $ds (sort keys %$live_restore_backing) { + my $job_id = "restore-$ds"; + mon_cmd($vmid, 'block-stream', + 'job-id' => $job_id, + device => "drive-$ds", + ); + $jobs->{$job_id} = {}; + } + + mon_cmd($vmid, 'cont'); + qemu_drive_mirror_monitor($vmid, undef, $jobs, 'auto', 0, 'stream'); + + print "restore-drive jobs finished successfully, removing all tracking block devices\n"; + + for my $ds (sort keys %$live_restore_backing) { + mon_cmd($vmid, 'blockdev-del', 'node-name' => "drive-$ds-restore"); + } + + close($qmeventd_fd); + }; + + my $err = $@; + + if ($err) { + warn "An error occurred during live-restore: $err\n"; + _do_vm_stop($storecfg, $vmid, 1, 1, 10, 0, 1); + die "live-restore failed\n"; + } + + PVE::QemuConfig->remove_lock($vmid, "import"); } sub restore_vma_archive { @@ -6248,19 +7440,17 @@ sub restore_vma_archive { $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]); - my $oldtimeout; - my $timeout = 5; - - my $devinfo = {}; + my $devinfo = {}; # info about drives included in backup + my $virtdev_hash = {}; # info about allocated drives my $rpcenv = PVE::RPCEnvironment::get(); my $conffile = PVE::QemuConfig->config_file($vmid); - my $tmpfn = "$conffile.$$.tmp"; # Note: $oldconf is undef if VM does not exist my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid); my $oldconf = PVE::Cluster::cfs_read_file($cfs_path); + my $new_conf_raw = ''; my %storage_limits; @@ -6278,13 +7468,15 @@ sub restore_vma_archive { PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw"); } - my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts); + $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts); + + foreach my $info (values %{$virtdev_hash}) { + my $storeid = $info->{storeid}; + next if defined($storage_limits{$storeid}); - foreach my $key (keys %storage_limits) { - my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit); - next if !$limit; - print STDERR "rate limit for storage $key: $limit KiB/s\n"; - $storage_limits{$key} = $limit * 1024; + my $limit = PVE::Storage::get_bandwidth_limit('restore', [$storeid], $bwlimit) // 0; + print STDERR "rate limit for storage $storeid: $limit KiB/s\n" if $limit; + $storage_limits{$storeid} = $limit * 1024; } foreach my $devname (keys %$devinfo) { @@ -6328,17 +7520,21 @@ sub restore_vma_archive { $fh->seek(0, 0) || die "seek failed - $!\n"; - my $outfd = IO::File->new($tmpfn, "w") || die "unable to write config for VM $vmid\n"; - my $cookie = { netcount => 0 }; while (defined(my $line = <$fh>)) { - $restore_update_config_line->($outfd, $cookie, $vmid, $map, $line, $opts->{unique}); + $new_conf_raw .= restore_update_config_line( + $cookie, + $map, + $line, + $opts->{unique}, + ); } $fh->close(); - $outfd->close(); }; + my $oldtimeout; + eval { # enable interrupts local $SIG{INT} = @@ -6348,7 +7544,7 @@ sub restore_vma_archive { local $SIG{PIPE} = sub { die "interrupted by signal\n"; }; local $SIG{ALRM} = sub { die "got timeout\n"; }; - $oldtimeout = alarm($timeout); + $oldtimeout = alarm(5); # for reading the VMA header - might hang with a corrupted one my $parser = sub { my $line = shift; @@ -6360,14 +7556,11 @@ sub restore_vma_archive { $devinfo->{$devname} = { size => $size, dev_id => $dev_id }; } elsif ($line =~ m/^CTIME: /) { # we correctly received the vma config, so we can disable - # the timeout now for disk allocation (set to 10 minutes, so - # that we always timeout if something goes wrong) - alarm(600); + # the timeout now for disk allocation + alarm($oldtimeout || 0); + $oldtimeout = undef; &$print_devmap(); print $fifofh "done\n"; - my $tmp = $oldtimeout || 0; - $oldtimeout = undef; - alarm($tmp); close($fifofh); $fifofh = undef; } @@ -6380,30 +7573,35 @@ sub restore_vma_archive { alarm($oldtimeout) if $oldtimeout; - $restore_deactivate_volumes->($cfg, $devinfo); + $restore_deactivate_volumes->($cfg, $virtdev_hash); close($fifofh) if $fifofh; unlink $mapfifo; rmtree $tmpdir; if ($err) { - unlink $tmpfn; - $restore_destroy_volumes->($cfg, $devinfo); + $restore_destroy_volumes->($cfg, $virtdev_hash); die $err; } - rename($tmpfn, $conffile) || - die "unable to commit configuration file '$conffile'\n"; - - PVE::Cluster::cfs_update(); # make sure we read new file + my $new_conf = restore_merge_config($conffile, $new_conf_raw, $opts->{override_conf}); + check_restore_permissions($rpcenv, $user, $new_conf); + PVE::QemuConfig->write_config($vmid, $new_conf); eval { rescan($vmid, 1); }; warn $@ if $@; + + PVE::AccessControl::add_vm_to_pool($vmid, $opts->{pool}) if $opts->{pool}; } sub restore_tar_archive { my ($archive, $vmid, $user, $opts) = @_; + if (scalar(keys $opts->{override_conf}->%*) > 0) { + my $keystring = join(' ', keys $opts->{override_conf}->%*); + die "cannot pass along options ($keystring) when restoring from tar archive\n"; + } + if ($archive ne '-') { my $firstfile = tar_archive_read_firstfile($archive); die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n" @@ -6438,7 +7636,7 @@ sub restore_tar_archive { local $ENV{VZDUMP_USER} = $user; my $conffile = PVE::QemuConfig->config_file($vmid); - my $tmpfn = "$conffile.$$.tmp"; + my $new_conf_raw = ''; # disable interrupts (always do cleanups) local $SIG{INT} = @@ -6482,26 +7680,26 @@ sub restore_tar_archive { my $srcfd = IO::File->new($confsrc, "r") || die "unable to open file '$confsrc'\n"; - my $outfd = IO::File->new($tmpfn, "w") || die "unable to write config for VM $vmid\n"; - my $cookie = { netcount => 0 }; while (defined (my $line = <$srcfd>)) { - $restore_update_config_line->($outfd, $cookie, $vmid, $map, $line, $opts->{unique}); + $new_conf_raw .= restore_update_config_line( + $cookie, + $map, + $line, + $opts->{unique}, + ); } $srcfd->close(); - $outfd->close(); }; if (my $err = $@) { - unlink $tmpfn; tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info}; die $err; } rmtree $tmpdir; - rename $tmpfn, $conffile || - die "unable to commit configuration file '$conffile'\n"; + PVE::Tools::file_set_contents($conffile, $new_conf_raw); PVE::Cluster::cfs_update(); # make sure we read new file @@ -6533,7 +7731,9 @@ my $qemu_snap_storage = { rbd => 1, }; sub do_snapshots_with_qemu { - my ($storecfg, $volid) = @_; + my ($storecfg, $volid, $deviceid) = @_; + + return if $deviceid =~ m/tpmstate0/; my $storage_name = PVE::Storage::parse_volume_id($volid); my $scfg = $storecfg->{ids}->{$storage_name}; @@ -6555,7 +7755,7 @@ sub qga_check_running { eval { mon_cmd($vmid, "guest-ping", timeout => 3); }; if ($@) { - warn "Qemu Guest Agent is not running - $@" if !$nowarn; + warn "QEMU Guest Agent is not running - $@" if !$nowarn; return 0; } return 1; @@ -6600,7 +7800,7 @@ sub convert_iscsi_path { } sub qemu_img_convert { - my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_; + my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized, $bwlimit) = @_; my $storecfg = PVE::Storage::config(); my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1); @@ -6620,7 +7820,7 @@ sub qemu_img_convert { $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname); $src_is_iscsi = ($src_path =~ m|^iscsi://|); $cachemode = 'none' if $src_scfg->{type} eq 'zfspool'; - } elsif (-f $src_volid) { + } elsif (-f $src_volid || -b $src_volid) { $src_path = $src_volid; if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) { $src_format = $1; @@ -6640,6 +7840,7 @@ sub qemu_img_convert { if $snapname && $src_format && $src_format eq "qcow2"; push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool'; push @$cmd, '-T', $cachemode if defined($cachemode); + push @$cmd, '-r', "${bwlimit}K" if defined($bwlimit); if ($src_is_iscsi) { push @$cmd, '--image-opts'; @@ -6668,9 +7869,10 @@ sub qemu_img_convert { if($line =~ m/\((\S+)\/100\%\)/){ my $percent = $1; my $transferred = int($size * $percent / 100); - my $remaining = $size - $transferred; + my $total_h = render_bytes($size, 1); + my $transferred_h = render_bytes($transferred, 1); - print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n"; + print "transferred $transferred_h of $total_h ($percent%)\n"; } }; @@ -6683,7 +7885,11 @@ sub qemu_img_convert { sub qemu_img_format { my ($scfg, $volname) = @_; - if ($scfg->{path} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) { + # FIXME: this entire function is kind of weird given that `parse_volname` + # also already gives us a format? + my $is_path_storage = $scfg->{path} || $scfg->{type} eq 'esxi'; + + if ($is_path_storage && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) { return $1; } else { return "raw"; @@ -6746,64 +7952,92 @@ sub qemu_drive_mirror { # 'complete': wait until all jobs are ready, block-job-complete them (default) # 'cancel': wait until all jobs are ready, block-job-cancel them # 'skip': wait until all jobs are ready, return with block jobs in ready state +# 'auto': wait until all jobs disappear, only use for jobs which complete automatically sub qemu_drive_mirror_monitor { - my ($vmid, $vmiddst, $jobs, $completion, $qga) = @_; + my ($vmid, $vmiddst, $jobs, $completion, $qga, $op) = @_; $completion //= 'complete'; + $op //= "mirror"; eval { my $err_complete = 0; + my $starttime = time (); while (1) { - die "storage migration timed out\n" if $err_complete > 300; + die "block job ('$op') timed out\n" if $err_complete > 300; my $stats = mon_cmd($vmid, "query-block-jobs"); + my $ctime = time(); - my $running_mirror_jobs = {}; - foreach my $stat (@$stats) { - next if $stat->{type} ne 'mirror'; - $running_mirror_jobs->{$stat->{device}} = $stat; + my $running_jobs = {}; + for my $stat (@$stats) { + next if $stat->{type} ne $op; + $running_jobs->{$stat->{device}} = $stat; } my $readycounter = 0; - foreach my $job (keys %$jobs) { + for my $job_id (sort keys %$jobs) { + my $job = $running_jobs->{$job_id}; - if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) { - print "$job : finished\n"; - delete $jobs->{$job}; + my $vanished = !defined($job); + my $complete = defined($jobs->{$job_id}->{complete}) && $vanished; + if($complete || ($vanished && $completion eq 'auto')) { + print "$job_id: $op-job finished\n"; + delete $jobs->{$job_id}; next; } - die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job}); + die "$job_id: '$op' has been cancelled\n" if !defined($job); - my $busy = $running_mirror_jobs->{$job}->{busy}; - my $ready = $running_mirror_jobs->{$job}->{ready}; - if (my $total = $running_mirror_jobs->{$job}->{len}) { - my $transferred = $running_mirror_jobs->{$job}->{offset} || 0; + my $busy = $job->{busy}; + my $ready = $job->{ready}; + if (my $total = $job->{len}) { + my $transferred = $job->{offset} || 0; my $remaining = $total - $transferred; my $percent = sprintf "%.2f", ($transferred * 100 / $total); - print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n"; + my $duration = $ctime - $starttime; + my $total_h = render_bytes($total, 1); + my $transferred_h = render_bytes($transferred, 1); + + my $status = sprintf( + "transferred $transferred_h of $total_h ($percent%%) in %s", + render_duration($duration), + ); + + if ($ready) { + if ($busy) { + $status .= ", still busy"; # shouldn't even happen? but mirror is weird + } else { + $status .= ", ready"; + } + } + print "$job_id: $status\n" if !$jobs->{$job_id}->{ready}; + $jobs->{$job_id}->{ready} = $ready; } - $readycounter++ if $running_mirror_jobs->{$job}->{ready}; + $readycounter++ if $job->{ready}; } last if scalar(keys %$jobs) == 0; if ($readycounter == scalar(keys %$jobs)) { - print "all mirroring jobs are ready \n"; - last if $completion eq 'skip'; #do the complete later + print "all '$op' jobs are ready\n"; + + # do the complete later (or has already been done) + last if $completion eq 'skip' || $completion eq 'auto'; if ($vmiddst && $vmiddst != $vmid) { my $agent_running = $qga && qga_check_running($vmid); if ($agent_running) { print "freeze filesystem\n"; eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); }; + warn $@ if $@; } else { print "suspend vm\n"; eval { PVE::QemuServer::vm_suspend($vmid, 1); }; + warn $@ if $@; } # if we clone a disk for a new target vm, we don't switch the disk @@ -6812,17 +8046,19 @@ sub qemu_drive_mirror_monitor { if ($agent_running) { print "unfreeze filesystem\n"; eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); }; + warn $@ if $@; } else { print "resume vm\n"; - eval { PVE::QemuServer::vm_resume($vmid, 1, 1); }; + eval { PVE::QemuServer::vm_resume($vmid, 1, 1); }; + warn $@ if $@; } last; } else { - foreach my $job (keys %$jobs) { + for my $job_id (sort keys %$jobs) { # try to switch the disk if source and destination are on the same guest - print "$job: Completing block job...\n"; + print "$job_id: Completing block job_id...\n"; my $op; if ($completion eq 'complete') { @@ -6832,13 +8068,13 @@ sub qemu_drive_mirror_monitor { } else { die "invalid completion value: $completion\n"; } - eval { mon_cmd($vmid, $op, device => $job) }; + eval { mon_cmd($vmid, $op, device => $job_id) }; if ($@ =~ m/cannot be completed/) { - print "$job: Block job cannot be completed, try again.\n"; + print "$job_id: block job cannot be completed, trying again.\n"; $err_complete++; }else { - print "$job: Completed successfully.\n"; - $jobs->{$job}->{complete} = 1; + print "$job_id: Completed successfully.\n"; + $jobs->{$job_id}->{complete} = 1; } } } @@ -6850,9 +8086,8 @@ sub qemu_drive_mirror_monitor { if ($err) { eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) }; - die "mirroring error: $err"; + die "block job ($op) error: $err"; } - } sub qemu_blockjobs_cancel { @@ -6886,80 +8121,150 @@ sub qemu_blockjobs_cancel { } } +# Check for bug #4525: drive-mirror will open the target drive with the same aio setting as the +# source, but some storages have problems with io_uring, sometimes even leading to crashes. +my sub clone_disk_check_io_uring { + my ($src_drive, $storecfg, $src_storeid, $dst_storeid, $use_drive_mirror) = @_; + + return if !$use_drive_mirror; + + # Don't complain when not changing storage. + # Assume if it works for the source, it'll work for the target too. + return if $src_storeid eq $dst_storeid; + + my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid); + my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid); + + my $cache_direct = drive_uses_cache_direct($src_drive); + + my $src_uses_io_uring; + if ($src_drive->{aio}) { + $src_uses_io_uring = $src_drive->{aio} eq 'io_uring'; + } else { + $src_uses_io_uring = storage_allows_io_uring_default($src_scfg, $cache_direct); + } + + die "target storage is known to cause issues with aio=io_uring (used by current drive)\n" + if $src_uses_io_uring && !storage_allows_io_uring_default($dst_scfg, $cache_direct); +} + sub clone_disk { - my ($storecfg, $vmid, $running, $drivename, $drive, $snapname, - $newvmid, $storage, $format, $full, $newvollist, $jobs, $completion, $qga, $bwlimit, $conf) = @_; + my ($storecfg, $source, $dest, $full, $newvollist, $jobs, $completion, $qga, $bwlimit) = @_; + + my ($vmid, $running) = $source->@{qw(vmid running)}; + my ($src_drivename, $drive, $snapname) = $source->@{qw(drivename drive snapname)}; + + my ($newvmid, $dst_drivename, $efisize) = $dest->@{qw(vmid drivename efisize)}; + my ($storage, $format) = $dest->@{qw(storage format)}; + + my $unused = defined($src_drivename) && $src_drivename =~ /^unused/; + my $use_drive_mirror = $full && $running && $src_drivename && !$snapname && !$unused; + + if ($src_drivename && $dst_drivename && $src_drivename ne $dst_drivename) { + die "cloning from/to EFI disk requires EFI disk\n" + if $src_drivename eq 'efidisk0' || $dst_drivename eq 'efidisk0'; + die "cloning from/to TPM state requires TPM state\n" + if $src_drivename eq 'tpmstate0' || $dst_drivename eq 'tpmstate0'; + + # This would lead to two device nodes in QEMU pointing to the same backing image! + die "cannot change drive name when cloning disk from/to the same VM\n" + if $use_drive_mirror && $vmid == $newvmid; + } + + die "cannot move TPM state while VM is running\n" + if $use_drive_mirror && $src_drivename eq 'tpmstate0'; my $newvolid; + print "create " . ($full ? 'full' : 'linked') . " clone of drive "; + print "$src_drivename " if $src_drivename; + print "($drive->{file})\n"; + if (!$full) { - print "create linked clone of drive $drivename ($drive->{file})\n"; $newvolid = PVE::Storage::vdisk_clone($storecfg, $drive->{file}, $newvmid, $snapname); push @$newvollist, $newvolid; } else { - - my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}); - $storeid = $storage if $storage; + my ($src_storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}); + my $storeid = $storage || $src_storeid; my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format); - print "create full clone of drive $drivename ($drive->{file})\n"; my $name = undef; my $size = undef; if (drive_is_cloudinit($drive)) { $name = "vm-$newvmid-cloudinit"; - $name .= ".$dst_format" if $dst_format ne 'raw'; + my $scfg = PVE::Storage::storage_config($storecfg, $storeid); + if ($scfg->{path}) { + $name .= ".$dst_format"; + } $snapname = undef; $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE; - } elsif ($drivename eq 'efidisk0') { - $size = get_efivars_size($conf); + } elsif ($dst_drivename eq 'efidisk0') { + $size = $efisize or die "internal error - need to specify EFI disk size\n"; + } elsif ($dst_drivename eq 'tpmstate0') { + $dst_format = 'raw'; + $size = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE; } else { - ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3); + clone_disk_check_io_uring($drive, $storecfg, $src_storeid, $storeid, $use_drive_mirror); + + $size = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 10); } - $size /= 1024; - $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, $size); + $newvolid = PVE::Storage::vdisk_alloc( + $storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024) + ); push @$newvollist, $newvolid; PVE::Storage::activate_volumes($storecfg, [$newvolid]); if (drive_is_cloudinit($drive)) { + # when cloning multiple disks (e.g. during clone_vm) it might be the last disk + # if this is the case, we have to complete any block-jobs still there from + # previous drive-mirrors + if (($completion eq 'complete') && (scalar(keys %$jobs) > 0)) { + qemu_drive_mirror_monitor($vmid, $newvmid, $jobs, $completion, $qga); + } goto no_data_clone; } my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid); - if (!$running || $snapname) { - # TODO: handle bwlimits - if ($drivename eq 'efidisk0') { + if ($use_drive_mirror) { + qemu_drive_mirror($vmid, $src_drivename, $newvolid, $newvmid, $sparseinit, $jobs, + $completion, $qga, $bwlimit); + } else { + if ($dst_drivename eq 'efidisk0') { # the relevant data on the efidisk may be smaller than the source # e.g. on RBD/ZFS, so we use dd to copy only the amount # that is given by the OVMF_VARS.fd - my $src_path = PVE::Storage::path($storecfg, $drive->{file}); + my $src_path = PVE::Storage::path($storecfg, $drive->{file}, $snapname); my $dst_path = PVE::Storage::path($storecfg, $newvolid); - run_command(['qemu-img', 'dd', '-n', '-O', $dst_format, "bs=1", "count=$size", - "if=$src_path", "of=$dst_path"]); - } else { - qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit); - } - } else { - my $kvmver = get_running_qemu_version ($vmid); - if (!min_version($kvmver, 2, 7)) { - die "drive-mirror with iothread requires qemu version 2.7 or higher\n" - if $drive->{iothread}; - } + my $src_format = (PVE::Storage::parse_volname($storecfg, $drive->{file}))[6]; - qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, - $completion, $qga, $bwlimit); + # better for Ceph if block size is not too small, see bug #3324 + my $bs = 1024*1024; + + my $cmd = ['qemu-img', 'dd', '-n', '-O', $dst_format]; + + if ($src_format eq 'qcow2' && $snapname) { + die "cannot clone qcow2 EFI disk snapshot - requires QEMU >= 6.2\n" + if !min_version(kvm_user_version(), 6, 2); + push $cmd->@*, '-l', $snapname; + } + push $cmd->@*, "bs=$bs", "osize=$size", "if=$src_path", "of=$dst_path"; + run_command($cmd); + } else { + qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit, $bwlimit); + } } } no_data_clone: - my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3); + my $size = eval { PVE::Storage::volume_size_info($storecfg, $newvolid, 10) }; - my $disk = $drive; - $disk->{format} = undef; + my $disk = dclone($drive); + delete $disk->{format}; $disk->{file} = $newvolid; - $disk->{size} = $size; + $disk->{size} = $size if defined($size) && !$unused; return $disk; } @@ -6993,10 +8298,12 @@ sub qemu_use_old_bios_files { } sub get_efivars_size { - my ($conf) = @_; + my ($conf, $efidisk) = @_; + my $arch = get_vm_arch($conf); - my (undef, $ovmf_vars) = get_ovmf_files($arch); - die "uefi vars image '$ovmf_vars' not found\n" if ! -f $ovmf_vars; + $efidisk //= $conf->{efidisk0} ? parse_drive('efidisk0', $conf->{efidisk0}) : undef; + my $smm = PVE::QemuServer::Machine::machine_type_is_q35($conf); + my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm); return -s $ovmf_vars; } @@ -7012,11 +8319,18 @@ sub update_efidisk_size { return; } -sub create_efidisk($$$$$) { - my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_; +sub update_tpmstate_size { + my ($conf) = @_; + + my $disk = PVE::QemuServer::parse_drive('tpmstate0', $conf->{tpmstate0}); + $disk->{size} = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE; + $conf->{tpmstate0} = print_drive($disk); +} + +sub create_efidisk($$$$$$$) { + my ($storecfg, $storeid, $vmid, $fmt, $arch, $efidisk, $smm) = @_; - my (undef, $ovmf_vars) = get_ovmf_files($arch); - die "EFI vars default image not found\n" if ! -f $ovmf_vars; + my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm); my $vars_size_b = -s $ovmf_vars; my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb'); @@ -7024,7 +8338,7 @@ sub create_efidisk($$$$$) { PVE::Storage::activate_volumes($storecfg, [$volid]); qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0); - my ($size) = PVE::Storage::volume_size_info($storecfg, $volid, 3); + my $size = PVE::Storage::volume_size_info($storecfg, $volid, 3); return ($volid, $size/1024); } @@ -7063,24 +8377,6 @@ sub scsihw_infos { return ($maxdev, $controller, $controller_prefix); } -sub windows_version { - my ($ostype) = @_; - - return 0 if !$ostype; - - my $winversion = 0; - - if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') { - $winversion = 5; - } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') { - $winversion = 6; - } elsif ($ostype =~ m/^win(\d+)$/) { - $winversion = $1; - } - - return $winversion; -} - sub resolve_dst_disk_format { my ($storecfg, $storeid, $src_volname, $format) = @_; my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid); @@ -7139,7 +8435,7 @@ sub generate_smbios1_uuid { sub nbd_stop { my ($vmid) = @_; - mon_cmd($vmid, 'nbd-server-stop'); + mon_cmd($vmid, 'nbd-server-stop', timeout => 25); } sub create_reboot_request { @@ -7249,6 +8545,34 @@ sub device_bootorder { return $bootorder; } +sub register_qmeventd_handle { + my ($vmid) = @_; + + my $fh; + my $peer = "/var/run/qmeventd.sock"; + my $count = 0; + + for (;;) { + $count++; + $fh = IO::Socket::UNIX->new(Peer => $peer, Blocking => 0, Timeout => 1); + last if $fh; + if ($! != EINTR && $! != EAGAIN) { + die "unable to connect to qmeventd socket (vmid: $vmid) - $!\n"; + } + if ($count > 4) { + die "unable to connect to qmeventd socket (vmid: $vmid) - timeout " + . "after $count retries\n"; + } + usleep(25000); + } + + # send handshake to mark VM as backing up + print $fh to_json({vzdump => {vmid => "$vmid"}}); + + # return handle to be closed later when inhibit is no longer required + return $fh; +} + # bash completion helper sub complete_backup_archives { @@ -7340,4 +8664,108 @@ sub complete_migration_storage { return $res; } +sub vm_is_paused { + my ($vmid, $include_suspended) = @_; + my $qmpstatus = eval { + PVE::QemuConfig::assert_config_exists_on_node($vmid); + mon_cmd($vmid, "query-status"); + }; + warn "$@\n" if $@; + return $qmpstatus && ( + $qmpstatus->{status} eq "paused" || + $qmpstatus->{status} eq "prelaunch" || + ($include_suspended && $qmpstatus->{status} eq "suspended") + ); +} + +sub check_volume_storage_type { + my ($storecfg, $vol) = @_; + + my ($storeid, $volname) = PVE::Storage::parse_volume_id($vol); + my $scfg = PVE::Storage::storage_config($storecfg, $storeid); + my ($vtype) = PVE::Storage::parse_volname($storecfg, $vol); + + die "storage '$storeid' does not support content-type '$vtype'\n" + if !$scfg->{content}->{$vtype}; + + return 1; +} + +sub add_nets_bridge_fdb { + my ($conf, $vmid) = @_; + + for my $opt (keys %$conf) { + next if $opt !~ m/^net(\d+)$/; + my $iface = "tap${vmid}i$1"; + # NOTE: expect setups with learning off to *not* use auto-random-generation of MAC on start + my $net = parse_net($conf->{$opt}, 1) or next; + + my $mac = $net->{macaddr}; + if (!$mac) { + log_warn("MAC learning disabled, but vNIC '$iface' has no static MAC to add to forwarding DB!") + if !file_read_firstline("/sys/class/net/$iface/brport/learning"); + next; + } + + my $bridge = $net->{bridge}; + if (!$bridge) { + log_warn("Interface '$iface' not attached to any bridge."); + next; + } + if ($have_sdn) { + PVE::Network::SDN::Zones::add_bridge_fdb($iface, $mac, $bridge); + } elsif (-d "/sys/class/net/$bridge/bridge") { # avoid fdb management with OVS for now + PVE::Network::add_bridge_fdb($iface, $mac); + } + } +} + +sub del_nets_bridge_fdb { + my ($conf, $vmid) = @_; + + for my $opt (keys %$conf) { + next if $opt !~ m/^net(\d+)$/; + my $iface = "tap${vmid}i$1"; + + my $net = parse_net($conf->{$opt}) or next; + my $mac = $net->{macaddr} or next; + + my $bridge = $net->{bridge}; + if ($have_sdn) { + PVE::Network::SDN::Zones::del_bridge_fdb($iface, $mac, $bridge); + } elsif (-d "/sys/class/net/$bridge/bridge") { # avoid fdb management with OVS for now + PVE::Network::del_bridge_fdb($iface, $mac); + } + } +} + +sub create_ifaces_ipams_ips { + my ($conf, $vmid) = @_; + + return if !$have_sdn; + + foreach my $opt (keys %$conf) { + if ($opt =~ m/^net(\d+)$/) { + my $value = $conf->{$opt}; + my $net = PVE::QemuServer::parse_net($value); + eval { PVE::Network::SDN::Vnets::add_next_free_cidr($net->{bridge}, $conf->{name}, $net->{macaddr}, $vmid, undef, 1) }; + warn $@ if $@; + } + } +} + +sub delete_ifaces_ipams_ips { + my ($conf, $vmid) = @_; + + return if !$have_sdn; + + foreach my $opt (keys %$conf) { + if ($opt =~ m/^net(\d+)$/) { + my $net = PVE::QemuServer::parse_net($conf->{$opt}); + eval { PVE::Network::SDN::Vnets::del_ips_from_mac($net->{bridge}, $net->{macaddr}, $conf->{name}) }; + warn $@ if $@; + } + } +} + 1; diff --git a/PVE/QemuServer/CGroup.pm b/PVE/QemuServer/CGroup.pm new file mode 100644 index 0000000..479991e --- /dev/null +++ b/PVE/QemuServer/CGroup.pm @@ -0,0 +1,14 @@ +package PVE::QemuServer::CGroup; + +use strict; +use warnings; +use PVE::CGroup; +use base('PVE::CGroup'); + +sub get_subdir { + my ($self, $controller, $limiting) = @_; + my $vmid = $self->{vmid}; + return "qemu.slice/$vmid.scope/"; +} + +1; diff --git a/PVE/QemuServer/CPUConfig.pm b/PVE/QemuServer/CPUConfig.pm index 32192f2..33f7524 100644 --- a/PVE/QemuServer/CPUConfig.pm +++ b/PVE/QemuServer/CPUConfig.pm @@ -5,6 +5,7 @@ use warnings; use PVE::JSONSchema; use PVE::Cluster qw(cfs_register_file cfs_read_file); +use PVE::Tools qw(get_host_arch); use PVE::QemuServer::Helpers qw(min_version); use base qw(PVE::SectionConfig Exporter); @@ -12,6 +13,8 @@ use base qw(PVE::SectionConfig Exporter); our @EXPORT_OK = qw( print_cpu_device get_cpu_options +get_cpu_bitness +is_native_arch ); # under certain race-conditions, this module might be loaded before pve-cluster @@ -21,14 +24,53 @@ if (PVE::Cluster::check_cfs_is_mounted(1)) { } my $default_filename = "virtual-guest/cpu-models.conf"; -cfs_register_file($default_filename, - sub { PVE::QemuServer::CPUConfig->parse_config(@_); }, - sub { PVE::QemuServer::CPUConfig->write_config(@_); }); +cfs_register_file( + $default_filename, + sub { PVE::QemuServer::CPUConfig->parse_config(@_); }, + sub { PVE::QemuServer::CPUConfig->write_config(@_); }, +); sub load_custom_model_conf { return cfs_read_file($default_filename); } +#builtin models : reported-model is mandatory +my $builtin_models = { + 'x86-64-v2' => { + 'reported-model' => 'qemu64', + flags => "+popcnt;+pni;+sse4.1;+sse4.2;+ssse3", + }, + 'x86-64-v2-AES' => { + 'reported-model' => 'qemu64', + flags => "+aes;+popcnt;+pni;+sse4.1;+sse4.2;+ssse3", + }, + 'x86-64-v3' => { + 'reported-model' => 'qemu64', + flags => "+aes;+popcnt;+pni;+sse4.1;+sse4.2;+ssse3;+avx;+avx2;+bmi1;+bmi2;+f16c;+fma;+abm;+movbe;+xsave", + }, + 'x86-64-v4' => { + 'reported-model' => 'qemu64', + flags => "+aes;+popcnt;+pni;+sse4.1;+sse4.2;+ssse3;+avx;+avx2;+bmi1;+bmi2;+f16c;+fma;+abm;+movbe;+xsave;+avx512f;+avx512bw;+avx512cd;+avx512dq;+avx512vl", + }, +}; + +my $depreacated_cpu_map = { + # there never was such a client CPU, so map it to the server one for backward compat + 'Icelake-Client' => 'Icelake-Server', + 'Icelake-Client-noTSX' => 'Icelake-Server-noTSX', +}; + +my $cputypes_32bit = { + '486' => 1, + 'pentium' => 1, + 'pentium2' => 1, + 'pentium3' => 1, + 'coreduo' => 1, + 'athlon' => 1, + 'kvm32' => 1, + 'qemu32' => 1, +}; + my $cpu_vendor_list = { # Intel CPUs 486 => 'GenuineIntel', @@ -58,16 +100,31 @@ my $cpu_vendor_list = { 'Skylake-Client' => 'GenuineIntel', 'Skylake-Client-IBRS' => 'GenuineIntel', 'Skylake-Client-noTSX-IBRS' => 'GenuineIntel', + 'Skylake-Client-v4' => 'GenuineIntel', 'Skylake-Server' => 'GenuineIntel', 'Skylake-Server-IBRS' => 'GenuineIntel', 'Skylake-Server-noTSX-IBRS' => 'GenuineIntel', + 'Skylake-Server-v4' => 'GenuineIntel', + 'Skylake-Server-v5' => 'GenuineIntel', 'Cascadelake-Server' => 'GenuineIntel', + 'Cascadelake-Server-v2' => 'GenuineIntel', 'Cascadelake-Server-noTSX' => 'GenuineIntel', + 'Cascadelake-Server-v4' => 'GenuineIntel', + 'Cascadelake-Server-v5' => 'GenuineIntel', + 'Cooperlake' => 'GenuineIntel', + 'Cooperlake-v2' => 'GenuineIntel', KnightsMill => 'GenuineIntel', - 'Icelake-Client' => 'GenuineIntel', - 'Icelake-Client-noTSX' => 'GenuineIntel', + 'Icelake-Client' => 'GenuineIntel', # depreacated, removed with QEMU 7.1 + 'Icelake-Client-noTSX' => 'GenuineIntel', # depreacated, removed with QEMU 7.1 'Icelake-Server' => 'GenuineIntel', 'Icelake-Server-noTSX' => 'GenuineIntel', + 'Icelake-Server-v3' => 'GenuineIntel', + 'Icelake-Server-v4' => 'GenuineIntel', + 'Icelake-Server-v5' => 'GenuineIntel', + 'Icelake-Server-v6' => 'GenuineIntel', + 'SapphireRapids' => 'GenuineIntel', + 'SapphireRapids-v2' => 'GenuineIntel', + 'GraniteRapids' => 'GenuineIntel', # AMD CPUs athlon => 'AuthenticAMD', @@ -79,7 +136,15 @@ my $cpu_vendor_list = { Opteron_G5 => 'AuthenticAMD', EPYC => 'AuthenticAMD', 'EPYC-IBPB' => 'AuthenticAMD', + 'EPYC-v3' => 'AuthenticAMD', + 'EPYC-v4' => 'AuthenticAMD', 'EPYC-Rome' => 'AuthenticAMD', + 'EPYC-Rome-v2' => 'AuthenticAMD', + 'EPYC-Rome-v3' => 'AuthenticAMD', + 'EPYC-Rome-v4' => 'AuthenticAMD', + 'EPYC-Milan' => 'AuthenticAMD', + 'EPYC-Milan-v2' => 'AuthenticAMD', + 'EPYC-Genoa' => 'AuthenticAMD', # generic types, use vendor from host node host => 'default', @@ -120,7 +185,7 @@ my $cpu_fmt = { }, 'reported-model' => { description => "CPU model and vendor to report to the guest. Must be a QEMU/KVM supported model." - . " Only valid for custom CPU model definitions, default models will always report themselves to the guest OS.", + ." Only valid for custom CPU model definitions, default models will always report themselves to the guest OS.", type => 'string', enum => [ sort { lc("$a") cmp lc("$b") } keys %$cpu_vendor_list ], default => 'kvm64', @@ -140,11 +205,10 @@ my $cpu_fmt = { optional => 1, }, flags => { - description => "List of additional CPU flags separated by ';'." - . " Use '+FLAG' to enable, '-FLAG' to disable a flag." - . " Custom CPU models can specify any flag supported by" - . " QEMU/KVM, VM-specific flags must be from the following" - . " set for security reasons: @{[join(', ', @supported_cpu_flags)]}.", + description => "List of additional CPU flags separated by ';'. Use '+FLAG' to enable," + ." '-FLAG' to disable a flag. Custom CPU models can specify any flag supported by" + ." QEMU/KVM, VM-specific flags must be from the following set for security reasons: " + . join(', ', @supported_cpu_flags), format_description => '+FLAG[;-FLAG...]', type => 'string', pattern => qr/$cpu_flag_any_re(;$cpu_flag_any_re)*/, @@ -154,10 +218,9 @@ my $cpu_fmt = { type => 'string', format => 'pve-phys-bits', format_description => '8-64|host', - description => "The physical memory address bits that are reported to" - . " the guest OS. Should be smaller or equal to the host's." - . " Set to 'host' to use value from host CPU, but note that" - . " doing so will break live migration to CPUs with other values.", + description => "The physical memory address bits that are reported to the guest OS. Should" + ." be smaller or equal to the host's. Set to 'host' to use value from host CPU, but" + ." note that doing so will break live migration to CPUs with other values.", optional => 1, }, }; @@ -187,11 +250,8 @@ sub parse_phys_bits { PVE::JSONSchema::register_format('pve-cpu-conf', $cpu_fmt, \&validate_cpu_conf); sub validate_cpu_conf { my ($cpu) = @_; - - # required, but can't be forced in schema since it's encoded in section - # header for custom models + # required, but can't be forced in schema since it's encoded in section header for custom models die "CPU is missing cputype\n" if !$cpu->{cputype}; - return $cpu; } PVE::JSONSchema::register_format('pve-vm-cpu-conf', $cpu_fmt, \&validate_vm_cpu_conf); @@ -208,13 +268,13 @@ sub validate_vm_cpu_conf { get_custom_model($cputype); } else { die "Built-in cputype '$cputype' is not defined (missing 'custom-' prefix?)\n" - if !defined($cpu_vendor_list->{$cputype}); + if !defined($cpu_vendor_list->{$cputype}) && !defined($builtin_models->{$cputype}); } # in a VM-specific config, certain properties are limited/forbidden die "VM-specific CPU flags must be a subset of: @{[join(', ', @supported_cpu_flags)]}\n" - if ($cpu->{flags} && $cpu->{flags} !~ m/$cpu_flag_supported_re(;$cpu_flag_supported_re)*/); + if ($cpu->{flags} && $cpu->{flags} !~ m/^$cpu_flag_supported_re(;$cpu_flag_supported_re)*$/); die "Property 'reported-model' not allowed in VM-specific CPU config.\n" if defined($cpu->{'reported-model'}); @@ -298,6 +358,16 @@ sub get_cpu_models { }; } + for my $model (keys %{$builtin_models}) { + my $reported_model = $builtin_models->{$model}->{'reported-model'}; + my $vendor = $cpu_vendor_list->{$reported_model}; + push @$models, { + name => $model, + custom => 0, + vendor => $vendor, + }; + } + return $models if !$include_custom; my $conf = load_custom_model_conf(); @@ -346,20 +416,27 @@ sub get_custom_model { # Print a QEMU device node for a given VM configuration for hotplugging CPUs sub print_cpu_device { - my ($conf, $id) = @_; + my ($conf, $arch, $id) = @_; + + # FIXME: hot plugging other architectures like our unofficial aarch64 support? + die "Hotplug of non x86_64 CPU not yet supported" if $arch ne 'x86_64'; - my $kvm = $conf->{kvm} // 1; - my $cpu = $kvm ? "kvm64" : "qemu64"; + my $kvm = $conf->{kvm} // is_native_arch($arch); + my $cpu = get_default_cpu_type('x86_64', $kvm); if (my $cputype = $conf->{cpu}) { my $cpuconf = PVE::JSONSchema::parse_property_string('pve-vm-cpu-conf', $cputype) or die "Cannot parse cpu description: $cputype\n"; $cpu = $cpuconf->{cputype}; - if (is_custom_model($cpu)) { + if (my $model = $builtin_models->{$cpu}) { + $cpu = $model->{'reported-model'}; + } elsif (is_custom_model($cputype)) { my $custom_cpu = get_custom_model($cpu); - $cpu = $custom_cpu->{'reported-model'} // - $cpu_fmt->{'reported-model'}->{default}; + $cpu = $custom_cpu->{'reported-model'} // $cpu_fmt->{'reported-model'}->{default}; + } + if (my $replacement_type = $depreacated_cpu_map->{$cpu}) { + $cpu = $replacement_type; } } @@ -442,7 +519,7 @@ sub parse_cpuflag_list { return $res if !$flaglist; foreach my $flag (split(";", $flaglist)) { - if ($flag =~ $re) { + if ($flag =~ m/^$re$/) { $res->{$2} = { op => $1, reason => $reason }; } } @@ -454,35 +531,35 @@ sub parse_cpuflag_list { sub get_cpu_options { my ($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough) = @_; - my $cputype = $kvm ? "kvm64" : "qemu64"; - if ($arch eq 'aarch64') { - $cputype = 'cortex-a57'; - } + my $cputype = get_default_cpu_type($arch, $kvm); my $cpu = {}; my $custom_cpu; + my $builtin_cpu; my $hv_vendor_id; if (my $cpu_prop_str = $conf->{cpu}) { $cpu = PVE::JSONSchema::parse_property_string('pve-vm-cpu-conf', $cpu_prop_str) or die "Cannot parse cpu description: $cpu_prop_str\n"; $cputype = $cpu->{cputype}; - - if (is_custom_model($cputype)) { + if (my $model = $builtin_models->{$cputype}) { + $cputype = $model->{'reported-model'}; + $builtin_cpu->{flags} = $model->{'flags'}; + } elsif (is_custom_model($cputype)) { $custom_cpu = get_custom_model($cputype); - $cputype = $custom_cpu->{'reported-model'} // - $cpu_fmt->{'reported-model'}->{default}; - $kvm_off = $custom_cpu->{hidden} - if defined($custom_cpu->{hidden}); + $cputype = $custom_cpu->{'reported-model'} // $cpu_fmt->{'reported-model'}->{default}; + $kvm_off = $custom_cpu->{hidden} if defined($custom_cpu->{hidden}); $hv_vendor_id = $custom_cpu->{'hv-vendor-id'}; } + if (my $replacement_type = $depreacated_cpu_map->{$cputype}) { + $cputype = $replacement_type; + } + # VM-specific settings override custom CPU config - $kvm_off = $cpu->{hidden} - if defined($cpu->{hidden}); - $hv_vendor_id = $cpu->{'hv-vendor-id'} - if defined($cpu->{'hv-vendor-id'}); + $kvm_off = $cpu->{hidden} if defined($cpu->{hidden}); + $hv_vendor_id = $cpu->{'hv-vendor-id'} if defined($cpu->{'hv-vendor-id'}); } my $pve_flags = get_pve_cpu_flags($conf, $kvm, $cputype, $arch, $machine_version); @@ -497,11 +574,14 @@ sub get_cpu_options { ) : undef; - my $custom_cputype_flags = parse_cpuflag_list($cpu_flag_any_re, - "set by custom CPU model", $custom_cpu->{flags}); + my $builtin_cputype_flags = parse_cpuflag_list( + $cpu_flag_any_re, "set by builtin CPU model", $builtin_cpu->{flags}); - my $vm_flags = parse_cpuflag_list($cpu_flag_supported_re, - "manually set for VM", $cpu->{flags}); + my $custom_cputype_flags = parse_cpuflag_list( + $cpu_flag_any_re, "set by custom CPU model", $custom_cpu->{flags}); + + my $vm_flags = parse_cpuflag_list( + $cpu_flag_supported_re, "manually set for VM", $cpu->{flags}); my $pve_forced_flags = {}; $pve_forced_flags->{'enforce'} = { @@ -526,8 +606,8 @@ sub get_cpu_options { my $cpu_str = $cputype; # will be resolved in parameter order - $cpu_str .= resolve_cpu_flags($pve_flags, $hv_flags, $custom_cputype_flags, - $vm_flags, $pve_forced_flags); + $cpu_str .= resolve_cpu_flags( + $pve_flags, $hv_flags, $builtin_cputype_flags, $custom_cputype_flags, $vm_flags, $pve_forced_flags); my $phys_bits = ''; foreach my $conf ($custom_cpu, $cpu) { @@ -652,6 +732,47 @@ sub get_cpu_from_running_vm { return $1; } +sub get_default_cpu_type { + my ($arch, $kvm) = @_; + + my $cputype = $kvm ? 'kvm64' : 'qemu64'; + $cputype = 'cortex-a57' if $arch eq 'aarch64'; + + return $cputype; +} + +sub is_native_arch($) { + my ($arch) = @_; + return get_host_arch() eq $arch; +} + +sub get_cpu_bitness { + my ($cpu_prop_str, $arch) = @_; + + $arch //= get_host_arch(); + + my $cputype = get_default_cpu_type($arch, 0); + + if ($cpu_prop_str) { + my $cpu = PVE::JSONSchema::parse_property_string('pve-vm-cpu-conf', $cpu_prop_str) + or die "Cannot parse cpu description: $cpu_prop_str\n"; + + $cputype = $cpu->{cputype}; + + if (my $model = $builtin_models->{$cputype}) { + $cputype = $model->{'reported-model'}; + } elsif (is_custom_model($cputype)) { + my $custom_cpu = get_custom_model($cputype); + $cputype = $custom_cpu->{'reported-model'} // $cpu_fmt->{'reported-model'}->{default}; + } + } + + return $cputypes_32bit->{$cputype} ? 32 : 64 if $arch eq 'x86_64'; + return 64 if $arch eq 'aarch64'; + + die "unsupported architecture '$arch'\n"; +} + __PACKAGE__->register(); __PACKAGE__->init(); diff --git a/PVE/QemuServer/Cloudinit.pm b/PVE/QemuServer/Cloudinit.pm index 52a4203..abc6b14 100644 --- a/PVE/QemuServer/Cloudinit.pm +++ b/PVE/QemuServer/Cloudinit.pm @@ -6,10 +6,13 @@ use warnings; use File::Path; use Digest::SHA; use URI::Escape; +use MIME::Base64 qw(encode_base64); +use Storable qw(dclone); use PVE::Tools qw(run_command file_set_contents); use PVE::Storage; use PVE::QemuServer; +use PVE::QemuServer::Helpers; use constant CLOUDINIT_DISK_SIZE => 4 * 1024 * 1024; # 4MiB in bytes @@ -69,7 +72,7 @@ sub get_cloudinit_format { # the new predicatble network device naming scheme. if (defined(my $ostype = $conf->{ostype})) { return 'configdrive2' - if PVE::QemuServer::windows_version($ostype); + if PVE::QemuServer::Helpers::windows_version($ostype); } return 'nocloud'; @@ -84,6 +87,8 @@ sub get_hostname_fqdn { $hostname =~ s/\..*$//; } elsif (my $search = $conf->{searchdomain}) { $fqdn = "$hostname.$search"; + } else { + $fqdn = $hostname; } return ($hostname, $fqdn); } @@ -117,7 +122,7 @@ sub cloudinit_userdata { $content .= "hostname: $hostname\n"; $content .= "manage_etc_hosts: true\n"; - $content .= "fqdn: $fqdn\n" if defined($fqdn); + $content .= "fqdn: $fqdn\n"; my $username = $conf->{ciuser}; my $password = $conf->{cipassword}; @@ -143,7 +148,7 @@ sub cloudinit_userdata { $content .= " - default\n"; } - $content .= "package_upgrade: true\n"; + $content .= "package_upgrade: true\n" if !defined($conf->{ciupgrade}) || $conf->{ciupgrade}; return $content; } @@ -226,21 +231,102 @@ EOF sub generate_configdrive2 { my ($conf, $vmid, $drive, $volname, $storeid) = @_; - my ($user_data, $network_data, $meta_data) = get_custom_cloudinit_files($conf); + my ($user_data, $network_data, $meta_data, $vendor_data) = get_custom_cloudinit_files($conf); $user_data = cloudinit_userdata($conf, $vmid) if !defined($user_data); $network_data = configdrive2_network($conf) if !defined($network_data); + $vendor_data = '' if !defined($vendor_data); if (!defined($meta_data)) { $meta_data = configdrive2_gen_metadata($user_data, $network_data); } + + # we always allocate a 4MiB disk for cloudinit and with the overhead of the ISO + # make sure we always stay below it by keeping the sum of all files below 3 MiB + my $sum = length($user_data) + length($network_data) + length($meta_data) + length($vendor_data); + die "Cloud-Init sum of snippets too big (> 3 MiB)\n" if $sum > (3 * 1024 * 1024); + my $files = { '/openstack/latest/user_data' => $user_data, '/openstack/content/0000' => $network_data, - '/openstack/latest/meta_data.json' => $meta_data + '/openstack/latest/meta_data.json' => $meta_data, + '/openstack/latest/vendor_data.json' => $vendor_data }; commit_cloudinit_disk($conf, $vmid, $drive, $volname, $storeid, $files, 'config-2'); } +sub generate_opennebula { + my ($conf, $vmid, $drive, $volname, $storeid) = @_; + + my $content = ""; + + my $username = $conf->{ciuser} || "root"; + $content .= "USERNAME=$username\n" if defined($username); + + if (defined(my $password = $conf->{cipassword})) { + $content .= "CRYPTED_PASSWORD_BASE64=". encode_base64($password) ."\n"; + } + + if (defined($conf->{sshkeys})) { + my $keys = [ split(/\s*\n\s*/, URI::Escape::uri_unescape($conf->{sshkeys})) ]; + $content .= "SSH_PUBLIC_KEY=\"". join("\n", $keys->@*) ."\"\n"; + } + + my ($hostname, $fqdn) = get_hostname_fqdn($conf, $vmid); + $content .= "SET_HOSTNAME=$hostname\n"; + + my ($searchdomains, $nameservers) = get_dns_conf($conf); + $content .= 'DNS="' . join(' ', @$nameservers) ."\"\n" if $nameservers && @$nameservers; + $content .= 'SEARCH_DOMAIN="'. join(' ', @$searchdomains) ."\"\n" if $searchdomains && @$searchdomains; + + my $networkenabled = undef; + my @ifaces = grep { /^net(\d+)$/ } keys %$conf; + foreach my $iface (sort @ifaces) { + (my $id = $iface) =~ s/^net//; + my $net = PVE::QemuServer::parse_net($conf->{$iface}); + next if !$conf->{"ipconfig$id"}; + my $ipconfig = PVE::QemuServer::parse_ipconfig($conf->{"ipconfig$id"}); + my $ethid = "ETH$id"; + + my $mac = lc $net->{hwaddr}; + + if ($ipconfig->{ip}) { + $networkenabled = 1; + + if ($ipconfig->{ip} eq 'dhcp') { + $content .= "${ethid}_DHCP=YES\n"; + } else { + my ($addr, $mask) = split_ip4($ipconfig->{ip}); + $content .= "${ethid}_IP=$addr\n"; + $content .= "${ethid}_MASK=$mask\n"; + $content .= "${ethid}_MAC=$mac\n"; + $content .= "${ethid}_GATEWAY=$ipconfig->{gw}\n" if $ipconfig->{gw}; + } + $content .= "${ethid}_MTU=$net->{mtu}\n" if $net->{mtu}; + } + + if ($ipconfig->{ip6}) { + $networkenabled = 1; + if ($ipconfig->{ip6} eq 'dhcp') { + $content .= "${ethid}_DHCP6=YES\n"; + } elsif ($ipconfig->{ip6} eq 'auto') { + $content .= "${ethid}_AUTO6=YES\n"; + } else { + my ($addr, $mask) = split('/', $ipconfig->{ip6}); + $content .= "${ethid}_IP6=$addr\n"; + $content .= "${ethid}_MASK6=$mask\n"; + $content .= "${ethid}_MAC6=$mac\n"; + $content .= "${ethid}_GATEWAY6=$ipconfig->{gw6}\n" if $ipconfig->{gw6}; + } + $content .= "${ethid}_MTU=$net->{mtu}\n" if $net->{mtu}; + } + } + + $content .= "NETWORK=YES\n" if $networkenabled; + + my $files = { '/context.sh' => $content }; + commit_cloudinit_disk($conf, $vmid, $drive, $volname, $storeid, $files, 'CONTEXT'); +} + sub nocloud_network_v2 { my ($conf) = @_; @@ -358,10 +444,10 @@ sub nocloud_network { if ($ip eq 'dhcp') { $content .= "${i}- type: dhcp6\n"; } elsif ($ip eq 'auto') { - # SLAAC is not supported by cloud-init, this fallback should work with an up-to-date netplan at least - $content .= "${i}- type: dhcp6\n"; + # SLAAC is only supported by cloud-init since 19.4 + $content .= "${i}- type: ipv6_slaac\n"; } else { - $content .= "${i}- type: static\n" + $content .= "${i}- type: static6\n" . "${i} address: '$ip'\n"; if (defined(my $gw = $ipconfig->{gw6})) { $content .= "${i} gateway: '$gw'\n"; @@ -402,18 +488,25 @@ sub nocloud_gen_metadata { sub generate_nocloud { my ($conf, $vmid, $drive, $volname, $storeid) = @_; - my ($user_data, $network_data, $meta_data) = get_custom_cloudinit_files($conf); + my ($user_data, $network_data, $meta_data, $vendor_data) = get_custom_cloudinit_files($conf); $user_data = cloudinit_userdata($conf, $vmid) if !defined($user_data); $network_data = nocloud_network($conf) if !defined($network_data); + $vendor_data = '' if !defined($vendor_data); if (!defined($meta_data)) { $meta_data = nocloud_gen_metadata($user_data, $network_data); } + # we always allocate a 4MiB disk for cloudinit and with the overhead of the ISO + # make sure we always stay below it by keeping the sum of all files below 3 MiB + my $sum = length($user_data) + length($network_data) + length($meta_data) + length($vendor_data); + die "Cloud-Init sum of snippets too big (> 3 MiB)\n" if $sum > (3 * 1024 * 1024); + my $files = { '/user-data' => $user_data, '/network-config' => $network_data, - '/meta-data' => $meta_data + '/meta-data' => $meta_data, + '/vendor-data' => $vendor_data }; commit_cloudinit_disk($conf, $vmid, $drive, $volname, $storeid, $files, 'cidata'); } @@ -427,6 +520,7 @@ sub get_custom_cloudinit_files { my $network_volid = $files->{network}; my $user_volid = $files->{user}; my $meta_volid = $files->{meta}; + my $vendor_volid = $files->{vendor}; my $storage_conf = PVE::Storage::config(); @@ -445,27 +539,44 @@ sub get_custom_cloudinit_files { $meta_data = read_cloudinit_snippets_file($storage_conf, $meta_volid); } - return ($user_data, $network_data, $meta_data); + my $vendor_data; + if ($vendor_volid) { + $vendor_data = read_cloudinit_snippets_file($storage_conf, $vendor_volid); + } + + return ($user_data, $network_data, $meta_data, $vendor_data); } sub read_cloudinit_snippets_file { my ($storage_conf, $volid) = @_; - my ($full_path, undef, $type) = PVE::Storage::path($storage_conf, $volid); - die "$volid is not in the snippets directory\n" if $type ne 'snippets'; + my ($vtype, undef) = PVE::Storage::parse_volname($storage_conf, $volid); + + die "$volid is not in the snippets directory\n" if $vtype ne 'snippets'; + + my $full_path = PVE::Storage::abs_filesystem_path($storage_conf, $volid, 1); return PVE::Tools::file_get_contents($full_path, 1 * 1024 * 1024); } my $cloudinit_methods = { configdrive2 => \&generate_configdrive2, nocloud => \&generate_nocloud, + opennebula => \&generate_opennebula, }; -sub generate_cloudinitconfig { +sub has_changes { + my ($conf) = @_; + + return !!$conf->{cloudinit}->%*; +} + +sub generate_cloudinit_config { my ($conf, $vmid) = @_; my $format = get_cloudinit_format($conf); + my $has_changes = has_changes($conf); + PVE::QemuConfig->foreach_volume($conf, sub { my ($ds, $drive) = @_; @@ -478,6 +589,22 @@ sub generate_cloudinitconfig { $generator->($conf, $vmid, $drive, $volname, $storeid); }); + + return $has_changes; +} + +sub apply_cloudinit_config { + my ($conf, $vmid) = @_; + + my $has_changes = generate_cloudinit_config($conf, $vmid); + + if ($has_changes) { + delete $conf->{cloudinit}; + PVE::QemuConfig->write_config($vmid, $conf); + return 1; + } + + return $has_changes; } sub dump_cloudinit_config { diff --git a/PVE/QemuServer/Drive.pm b/PVE/QemuServer/Drive.pm index d560937..6a4fafd 100644 --- a/PVE/QemuServer/Drive.pm +++ b/PVE/QemuServer/Drive.pm @@ -3,6 +3,10 @@ package PVE::QemuServer::Drive; use strict; use warnings; +use Storable qw(dclone); + +use IO::File; + use PVE::Storage; use PVE::JSONSchema qw(get_standard_option); @@ -12,6 +16,8 @@ our @EXPORT_OK = qw( is_valid_drivename drive_is_cloudinit drive_is_cdrom +drive_is_read_only +get_scsi_devicetype parse_drive print_drive ); @@ -30,8 +36,11 @@ my $MAX_SCSI_DISKS = 31; my $MAX_VIRTIO_DISKS = 16; our $MAX_SATA_DISKS = 6; our $MAX_UNUSED_DISKS = 256; +our $NEW_DISK_RE = qr!^(([^/:\s]+):)?(\d+(\.\d+)?)$!; our $drivedesc_hash; +# Schema when disk allocation is possible. +our $drivedesc_hash_with_alloc = {}; my %drivedesc_base = ( volume => { alias => 'file' }, @@ -116,7 +125,7 @@ my %drivedesc_base = ( }, aio => { type => 'string', - enum => [qw(native threads)], + enum => [qw(native threads io_uring)], description => 'AIO type to use.', optional => 1, }, @@ -154,6 +163,26 @@ my %iothread_fmt = ( iothread => { optional => 1, }); +my %product_fmt = ( + product => { + type => 'string', + pattern => '[A-Za-z0-9\-_\s]{,16}', # QEMU (8.1) will quietly only use 16 bytes + format_description => 'product', + description => "The drive's product name, up to 16 bytes long.", + optional => 1, + }, +); + +my %vendor_fmt = ( + vendor => { + type => 'string', + pattern => '[A-Za-z0-9\-_\s]{,8}', # QEMU (8.1) will quietly only use 8 bytes + format_description => 'vendor', + description => "The drive's vendor name, up to 8 bytes long.", + optional => 1, + }, +); + my %model_fmt = ( model => { type => 'string', @@ -174,6 +203,14 @@ my %queues_fmt = ( } ); +my %readonly_fmt = ( + ro => { + type => 'boolean', + description => "Whether the drive is read-only.", + optional => 1, + }, +); + my %scsiblock_fmt = ( scsiblock => { type => 'boolean', @@ -256,16 +293,19 @@ PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt); my $idedesc = { optional => 1, type => 'string', format => $ide_fmt, - description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").", + description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS - 1) . ").", }; PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc); my $scsi_fmt = { %drivedesc_base, %iothread_fmt, + %product_fmt, %queues_fmt, + %readonly_fmt, %scsiblock_fmt, %ssd_fmt, + %vendor_fmt, %wwn_fmt, }; my $scsidesc = { @@ -290,6 +330,7 @@ PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc); my $virtio_fmt = { %drivedesc_base, %iothread_fmt, + %readonly_fmt, }; my $virtiodesc = { optional => 1, @@ -298,15 +339,25 @@ my $virtiodesc = { }; PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc); -my $alldrive_fmt = { - %drivedesc_base, - %iothread_fmt, - %model_fmt, - %queues_fmt, - %scsiblock_fmt, - %ssd_fmt, - %wwn_fmt, -}; +my %efitype_fmt = ( + efitype => { + type => 'string', + enum => [qw(2m 4m)], + description => "Size and type of the OVMF EFI vars. '4m' is newer and recommended," + . " and required for Secure Boot. For backwards compatibility, '2m' is used" + . " if not otherwise specified. Ignored for VMs with arch=aarch64 (ARM).", + optional => 1, + default => '2m', + }, + 'pre-enrolled-keys' => { + type => 'boolean', + description => "Use am EFI vars template with distribution-specific and Microsoft Standard" + ." keys enrolled, if used with 'efitype=4m'. Note that this will enable Secure Boot by" + ." default, though it can still be turned off from within the VM.", + optional => 1, + default => 0, + }, +); my $efidisk_fmt = { volume => { alias => 'file' }, @@ -325,16 +376,84 @@ my $efidisk_fmt = { description => "Disk size. This is purely informational and has no effect.", optional => 1, }, + %efitype_fmt, }; my $efidisk_desc = { optional => 1, type => 'string', format => $efidisk_fmt, - description => "Configure a Disk for storing EFI vars", + description => "Configure a disk for storing EFI vars.", }; PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc); +my %tpmversion_fmt = ( + version => { + type => 'string', + enum => [qw(v1.2 v2.0)], + description => "The TPM interface version. v2.0 is newer and should be preferred." + ." Note that this cannot be changed later on.", + optional => 1, + default => 'v2.0', + }, +); +my $tpmstate_fmt = { + volume => { alias => 'file' }, + file => { + type => 'string', + format => 'pve-volume-id-or-qm-path', + default_key => 1, + format_description => 'volume', + description => "The drive's backing volume.", + }, + size => { + type => 'string', + format => 'disk-size', + format_description => 'DiskSize', + description => "Disk size. This is purely informational and has no effect.", + optional => 1, + }, + %tpmversion_fmt, +}; +my $tpmstate_desc = { + optional => 1, + type => 'string', format => $tpmstate_fmt, + description => "Configure a Disk for storing TPM state. The format is fixed to 'raw'.", +}; +use constant TPMSTATE_DISK_SIZE => 4 * 1024 * 1024; + +my $alldrive_fmt = { + %drivedesc_base, + %iothread_fmt, + %model_fmt, + %product_fmt, + %queues_fmt, + %readonly_fmt, + %scsiblock_fmt, + %ssd_fmt, + %vendor_fmt, + %wwn_fmt, + %tpmversion_fmt, + %efitype_fmt, +}; + +my %import_from_fmt = ( + 'import-from' => { + type => 'string', + format => 'pve-volume-id-or-absolute-path', + format_description => 'source volume', + description => "Create a new disk, importing from this source (volume ID or absolute ". + "path). When an absolute path is specified, it's up to you to ensure that the source ". + "is not actively used by another process during the import!", + optional => 1, + }, +); + +my $alldrive_fmt_with_alloc = { + %$alldrive_fmt, + %import_from_fmt, +}; + my $unused_fmt = { volume => { alias => 'file' }, file => { @@ -352,26 +471,68 @@ my $unuseddesc = { description => "Reference to unused volumes. This is used internally, and should not be modified manually.", }; +my $with_alloc_desc_cache = { + unused => $unuseddesc, # Allocation for unused is not supported currently. +}; +my $desc_with_alloc = sub { + my ($type, $desc) = @_; + + return $with_alloc_desc_cache->{$type} if $with_alloc_desc_cache->{$type}; + + my $new_desc = dclone($desc); + + $new_desc->{format}->{'import-from'} = $import_from_fmt{'import-from'}; + + my $extra_note = ''; + if ($type eq 'efidisk') { + $extra_note = " Note that SIZE_IN_GiB is ignored here and that the default EFI vars are ". + "copied to the volume instead."; + } elsif ($type eq 'tpmstate') { + $extra_note = " Note that SIZE_IN_GiB is ignored here and 4 MiB will be used instead."; + } + + $new_desc->{description} .= " Use the special syntax STORAGE_ID:SIZE_IN_GiB to allocate a new ". + "volume.${extra_note} Use STORAGE_ID:0 and the 'import-from' parameter to import from an ". + "existing volume."; + + $with_alloc_desc_cache->{$type} = $new_desc; + + return $new_desc; +}; + for (my $i = 0; $i < $MAX_IDE_DISKS; $i++) { $drivedesc_hash->{"ide$i"} = $idedesc; + $drivedesc_hash_with_alloc->{"ide$i"} = $desc_with_alloc->('ide', $idedesc); } for (my $i = 0; $i < $MAX_SATA_DISKS; $i++) { $drivedesc_hash->{"sata$i"} = $satadesc; + $drivedesc_hash_with_alloc->{"sata$i"} = $desc_with_alloc->('sata', $satadesc); } for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++) { $drivedesc_hash->{"scsi$i"} = $scsidesc; + $drivedesc_hash_with_alloc->{"scsi$i"} = $desc_with_alloc->('scsi', $scsidesc); } for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++) { $drivedesc_hash->{"virtio$i"} = $virtiodesc; + $drivedesc_hash_with_alloc->{"virtio$i"} = $desc_with_alloc->('virtio', $virtiodesc); } $drivedesc_hash->{efidisk0} = $efidisk_desc; +$drivedesc_hash_with_alloc->{efidisk0} = $desc_with_alloc->('efidisk', $efidisk_desc); + +$drivedesc_hash->{tpmstate0} = $tpmstate_desc; +$drivedesc_hash_with_alloc->{tpmstate0} = $desc_with_alloc->('tpmstate', $tpmstate_desc); for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++) { $drivedesc_hash->{"unused$i"} = $unuseddesc; + $drivedesc_hash_with_alloc->{"unused$i"} = $desc_with_alloc->('unused', $unuseddesc); +} + +sub valid_drive_names_for_boot { + return grep { $_ ne 'efidisk0' && $_ ne 'tpmstate0' } valid_drive_names(); } sub valid_drive_names { @@ -380,7 +541,12 @@ sub valid_drive_names { (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))), (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))), (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))), - 'efidisk0'); + 'efidisk0', + 'tpmstate0'); +} + +sub valid_drive_names_with_unused { + return (valid_drive_names(), map {"unused$_"} (0 .. ($MAX_UNUSED_DISKS - 1))); } sub is_valid_drivename { @@ -402,7 +568,7 @@ sub verify_bootdisk { sub drive_is_cloudinit { my ($drive) = @_; - return $drive->{file} =~ m@[:/]vm-\d+-cloudinit(?:\.$QEMU_FORMAT_RE)?$@; + return $drive->{file} =~ m@[:/](?:vm-\d+-)?cloudinit(?:\.$QEMU_FORMAT_RE)?$@; } sub drive_is_cdrom { @@ -413,6 +579,15 @@ sub drive_is_cdrom { return $drive && $drive->{media} && ($drive->{media} eq 'cdrom'); } +sub drive_is_read_only { + my ($conf, $drive) = @_; + + return 0 if !PVE::QemuConfig->is_template($conf); + + # don't support being marked read-only + return $drive->{interface} ne 'sata' && $drive->{interface} ne 'ide'; +} + # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]] # [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no] # [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop] @@ -420,7 +595,7 @@ sub drive_is_cdrom { # [,iothread=on][,serial=serial][,model=model] sub parse_drive { - my ($key, $data) = @_; + my ($key, $data, $with_alloc) = @_; my ($interface, $index); @@ -431,12 +606,14 @@ sub parse_drive { return; } - if (!defined($drivedesc_hash->{$key})) { + my $desc_hash = $with_alloc ? $drivedesc_hash_with_alloc : $drivedesc_hash; + + if (!defined($desc_hash->{$key})) { warn "invalid drive key: $key\n"; return; } - my $desc = $drivedesc_hash->{$key}->{format}; + my $desc = $desc_hash->{$key}->{format}; my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) }; return if !$res; $res->{interface} = $interface; @@ -496,9 +673,10 @@ sub parse_drive { } sub print_drive { - my ($drive) = @_; + my ($drive, $with_alloc) = @_; my $skip = [ 'index', 'interface' ]; - return PVE::JSONSchema::print_property_string($drive, $alldrive_fmt, $skip); + my $fmt = $with_alloc ? $alldrive_fmt_with_alloc : $alldrive_fmt; + return PVE::JSONSchema::print_property_string($drive, $fmt, $skip); } sub get_bootdisks { @@ -522,20 +700,18 @@ sub bootdisk_size { my $bootdisks = get_bootdisks($conf); return if !@$bootdisks; - my $bootdisk = $bootdisks->[0]; - return if !is_valid_drivename($bootdisk); - - return if !$conf->{$bootdisk}; - - my $drive = parse_drive($bootdisk, $conf->{$bootdisk}); - return if !defined($drive); - - return if drive_is_cdrom($drive); - - my $volid = $drive->{file}; - return if !$volid; + for my $bootdisk (@$bootdisks) { + next if !is_valid_drivename($bootdisk); + next if !$conf->{$bootdisk}; + my $drive = parse_drive($bootdisk, $conf->{$bootdisk}); + next if !defined($drive); + next if drive_is_cdrom($drive); + my $volid = $drive->{file}; + next if !$volid; + return $drive->{size}; + } - return $drive->{size}; + return; } sub update_disksize { @@ -565,7 +741,7 @@ sub is_volume_in_use { my $path = PVE::Storage::path($storecfg, $volid); my $scan_config = sub { - my ($cref, $snapname) = @_; + my ($cref) = @_; foreach my $key (keys %$cref) { my $value = $cref->{$key}; @@ -581,7 +757,7 @@ sub is_volume_in_use { next if !$storeid; my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1); next if !$scfg; - return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname); + return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}); } } } @@ -593,8 +769,8 @@ sub is_volume_in_use { undef $skip_drive; - foreach my $snapname (keys %{$conf->{snapshots}}) { - return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname); + for my $snap (values %{$conf->{snapshots}}) { + return 1 if $scan_config->($snap); } return 0; @@ -602,7 +778,7 @@ sub is_volume_in_use { sub resolve_first_disk { my ($conf, $cdrom) = @_; - my @disks = valid_drive_names(); + my @disks = valid_drive_names_for_boot(); foreach my $ds (@disks) { next if !$conf->{$ds}; my $disk = parse_drive($ds, $conf->{$ds}); @@ -612,4 +788,97 @@ sub resolve_first_disk { return; } +sub scsi_inquiry { + my($fh, $noerr) = @_; + + my $SG_IO = 0x2285; + my $SG_GET_VERSION_NUM = 0x2282; + + my $versionbuf = "\x00" x 8; + my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf); + if (!$ret) { + die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr; + return; + } + my $version = unpack("I", $versionbuf); + if ($version < 30000) { + die "scsi generic interface too old\n" if !$noerr; + return; + } + + my $buf = "\x00" x 36; + my $sensebuf = "\x00" x 8; + my $cmd = pack("C x3 C x1", 0x12, 36); + + # see /usr/include/scsi/sg.h + my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I"; + + my $packet = pack( + $sg_io_hdr_t, ord('S'), -3, length($cmd), length($sensebuf), 0, length($buf), $buf, $cmd, $sensebuf, 6000 + ); + + $ret = ioctl($fh, $SG_IO, $packet); + if (!$ret) { + die "scsi ioctl SG_IO failed - $!\n" if !$noerr; + return; + } + + my @res = unpack($sg_io_hdr_t, $packet); + if ($res[17] || $res[18]) { + die "scsi ioctl SG_IO status error - $!\n" if !$noerr; + return; + } + + my $res = {}; + $res->@{qw(type removable vendor product revision)} = unpack("C C x6 A8 A16 A4", $buf); + + $res->{removable} = $res->{removable} & 128 ? 1 : 0; + $res->{type} &= 0x1F; + + return $res; +} + +sub path_is_scsi { + my ($path) = @_; + + my $fh = IO::File->new("+<$path") || return; + my $res = scsi_inquiry($fh, 1); + close($fh); + + return $res; +} + +sub get_scsi_device_type { + my ($drive, $storecfg, $machine_version) = @_; + + my $devicetype = 'hd'; + my $path = ''; + if (drive_is_cdrom($drive) || drive_is_cloudinit($drive)) { + $devicetype = 'cd'; + } else { + if ($drive->{file} =~ m|^/|) { + $path = $drive->{file}; + if (my $info = path_is_scsi($path)) { + if ($info->{type} == 0 && $drive->{scsiblock}) { + $devicetype = 'block'; + } elsif ($info->{type} == 1) { # tape + $devicetype = 'generic'; + } + } + } elsif ($drive->{file} =~ $NEW_DISK_RE){ + # special syntax cannot be parsed to path + return $devicetype; + } else { + $path = PVE::Storage::path($storecfg, $drive->{file}); + } + + # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380) + if ($path =~ m/^iscsi\:\/\// && + !PVE::QemuServer::Helpers::min_version($machine_version, 4, 1)) { + $devicetype = 'generic'; + } + } + + return $devicetype; +} 1; diff --git a/PVE/QemuServer/Helpers.pm b/PVE/QemuServer/Helpers.pm index c10d842..0afb631 100644 --- a/PVE/QemuServer/Helpers.pm +++ b/PVE/QemuServer/Helpers.pm @@ -4,6 +4,7 @@ use strict; use warnings; use File::stat; +use JSON; use PVE::INotify; use PVE::ProcFSTools; @@ -12,6 +13,8 @@ use base 'Exporter'; our @EXPORT_OK = qw( min_version config_aware_timeout +parse_number_sets +windows_version ); my $nodename = PVE::INotify::nodename(); @@ -141,8 +144,7 @@ sub version_cmp { } sub config_aware_timeout { - my ($config, $is_suspended) = @_; - my $memory = $config->{memory}; + my ($config, $memory, $is_suspended) = @_; my $timeout = 30; # Based on user reported startup time for vm with 512GiB @ 4-5 minutes @@ -150,6 +152,13 @@ sub config_aware_timeout { $timeout = int($memory/1024); } + # When using PCI passthrough, users reported much higher startup times, + # growing with the amount of memory configured. Constant factor chosen + # based on user reports. + if (grep(/^hostpci[0-9]+$/, keys %$config)) { + $timeout *= 4; + } + if ($is_suspended && $timeout < 300) { $timeout = 300; } @@ -161,4 +170,59 @@ sub config_aware_timeout { return $timeout; } +sub get_node_pvecfg_version { + my ($node) = @_; + + my $nodes_version_info = PVE::Cluster::get_node_kv('version-info', $node); + return if !$nodes_version_info->{$node}; + + my $version_info = decode_json($nodes_version_info->{$node}); + return $version_info->{version}; +} + +sub pvecfg_min_version { + my ($verstr, $major, $minor, $release) = @_; + + return 0 if !$verstr; + + if ($verstr =~ m/^(\d+)\.(\d+)(?:[.-](\d+))?/) { + return 1 if version_cmp($1, $major, $2, $minor, $3 // 0, $release) >= 0; + return 0; + } + + die "internal error: cannot check version of invalid string '$verstr'"; +} + +sub parse_number_sets { + my ($set) = @_; + my $res = []; + foreach my $part (split(/;/, $set)) { + if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) { + die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1; + push @$res, [ $1, $2 ]; + } else { + die "invalid range: $part\n"; + } + } + return $res; +} + +sub windows_version { + my ($ostype) = @_; + + return 0 if !$ostype; + + my $winversion = 0; + + if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') { + $winversion = 5; + } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') { + $winversion = 6; + } elsif ($ostype =~ m/^win(\d+)$/) { + $winversion = $1; + } + + return $winversion; +} + 1; diff --git a/PVE/QemuServer/ImportDisk.pm b/PVE/QemuServer/ImportDisk.pm index 51ad52e..132932a 100755 --- a/PVE/QemuServer/ImportDisk.pm +++ b/PVE/QemuServer/ImportDisk.pm @@ -11,6 +11,8 @@ use PVE::Tools qw(run_command extract_param); # and creates by default a drive entry unused[n] pointing to the created volume # $params->{drive_name} may be used to specify ide0, scsi1, etc ... # $params->{format} may be used to specify qcow2, raw, etc ... +# $params->{skiplock} may be used to skip checking for a lock in the VM config +# $params->{'skip-config-update'} may be used to import the disk without updating the VM config sub do_import { my ($src_path, $vmid, $storage_id, $params) = @_; @@ -26,6 +28,8 @@ sub do_import { # get target format, target image's path, and whether it's possible to sparseinit my $storecfg = PVE::Storage::config(); my $dst_format = PVE::QemuServer::resolve_dst_disk_format($storecfg, $storage_id, undef, $format); + warn "format '$format' is not supported by the target storage - using '$dst_format' instead\n" + if $format && $format ne $dst_format; my $dst_volid = PVE::Storage::vdisk_alloc($storecfg, $storage_id, $vmid, $dst_format, undef, $src_size / 1024); @@ -71,7 +75,7 @@ sub do_import { PVE::Storage::activate_volumes($storecfg, [$dst_volid]); PVE::QemuServer::qemu_img_convert($src_path, $dst_volid, $src_size, undef, $zeroinit); PVE::Storage::deactivate_volumes($storecfg, [$dst_volid]); - PVE::QemuConfig->lock_config($vmid, $create_drive); + PVE::QemuConfig->lock_config($vmid, $create_drive) if !$params->{'skip-config-update'}; }; if (my $err = $@) { eval { PVE::Storage::vdisk_free($storecfg, $dst_volid) }; diff --git a/PVE/QemuServer/Machine.pm b/PVE/QemuServer/Machine.pm index c168ade..cc92e7e 100644 --- a/PVE/QemuServer/Machine.pm +++ b/PVE/QemuServer/Machine.pm @@ -5,6 +5,7 @@ use warnings; use PVE::QemuServer::Helpers; use PVE::QemuServer::Monitor; +use PVE::JSONSchema qw(get_standard_option parse_property_string print_property_string); # Bump this for VM HW layout changes during a release (where the QEMU machine # version stays the same) @@ -12,29 +13,92 @@ our $PVE_MACHINE_VERSION = { '4.1' => 2, }; +my $machine_fmt = { + type => { + default_key => 1, + description => "Specifies the QEMU machine type.", + type => 'string', + pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)', + maxLength => 40, + format_description => 'machine type', + optional => 1, + }, + viommu => { + type => 'string', + description => "Enable and set guest vIOMMU variant (Intel vIOMMU needs q35 to be set as" + ." machine type).", + enum => ['intel', 'virtio'], + optional => 1, + }, +}; + +PVE::JSONSchema::register_format('pve-qemu-machine-fmt', $machine_fmt); + +PVE::JSONSchema::register_standard_option('pve-qemu-machine', { + description => "Specify the QEMU machine.", + type => 'string', + optional => 1, + format => PVE::JSONSchema::get_format('pve-qemu-machine-fmt'), +}); + +sub parse_machine { + my ($value) = @_; + + return if !$value; + + my $res = parse_property_string($machine_fmt, $value); + return $res; +} + +sub print_machine { + my ($machine_conf) = @_; + return print_property_string($machine_conf, $machine_fmt); +} + +sub assert_valid_machine_property { + my ($conf, $machine_conf) = @_; + my $q35 = $machine_conf->{type} && ($machine_conf->{type} =~ m/q35/) ? 1 : 0; + if ($machine_conf->{viommu} && $machine_conf->{viommu} eq "intel" && !$q35) { + die "to use Intel vIOMMU please set the machine type to q35\n"; + } +} + sub machine_type_is_q35 { my ($conf) = @_; - return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0; + my $machine_conf = parse_machine($conf->{machine}); + return $machine_conf->{type} && ($machine_conf->{type} =~ m/q35/) ? 1 : 0; } -# this only works if VM is running -sub get_current_qemu_machine { - my ($vmid) = @_; +# In list context, also returns whether the current machine is deprecated or not. +sub current_from_query_machines { + my ($machines) = @_; - my $res = PVE::QemuServer::Monitor::mon_cmd($vmid, 'query-machines'); + my ($current, $default); + for my $machine ($machines->@*) { + $default = $machine->{name} if $machine->{'is-default'}; - my ($current, $pve_version, $default); - foreach my $e (@$res) { - $default = $e->{name} if $e->{'is-default'}; - $current = $e->{name} if $e->{'is-current'}; - $pve_version = $e->{'pve-version'} if $e->{'pve-version'}; + if ($machine->{'is-current'}) { + $current = $machine->{name}; + # pve-version only exists for the current machine + $current .= "+$machine->{'pve-version'}" if $machine->{'pve-version'}; + return wantarray ? ($current, $machine->{deprecated} ? 1 : 0) : $current; + } } - $current .= "+$pve_version" if $current && $pve_version; + # fallback to the default machine if current is not supported by qemu - assume never deprecated + my $fallback = $default || 'pc'; + return wantarray ? ($fallback, 0) : $fallback; +} + +# This only works if VM is running. +# In list context, also returns whether the current machine is deprecated or not. +sub get_current_qemu_machine { + my ($vmid) = @_; + + my $res = PVE::QemuServer::Monitor::mon_cmd($vmid, 'query-machines'); - # fallback to the default machine if current is not supported by qemu - return $current || $default || 'pc'; + return current_from_query_machines($res); } # returns a string with major.minor+pve, patch version-part is ignored @@ -43,7 +107,9 @@ sub get_current_qemu_machine { sub extract_version { my ($machine_type, $kvmversion) = @_; - if (defined($machine_type) && $machine_type =~ m/^(?:pc(?:-i440fx|-q35)?|virt)-(\d+)\.(\d+)(?:\.(\d+))?(\+pve\d+)?/) { + if (defined($machine_type) && $machine_type =~ + m/^(?:pc(?:-i440fx|-q35)?|virt)-(\d+)\.(\d+)(?:\.(\d+))?(\+pve\d+)?(?:\.pxe)?/) + { my $versionstr = "$1.$2"; $versionstr .= $4 if $4; return $versionstr; @@ -77,7 +143,7 @@ sub get_pve_version { sub can_run_pve_machine_version { my ($machine_version, $kvmversion) = @_; - $machine_version =~ m/^(\d+)\.(\d+)(?:\+pve(\d+))$/; + $machine_version =~ m/^(\d+)\.(\d+)(?:\+pve(\d+))?(?:\.pxe)?$/; my $major = $1; my $minor = $2; my $pvever = $3; @@ -112,7 +178,8 @@ sub qemu_machine_pxe { my $machine = get_current_qemu_machine($vmid); - if ($conf->{machine} && $conf->{machine} =~ m/\.pxe$/) { + my $machine_conf = parse_machine($conf->{machine}); + if ($machine_conf->{type} && $machine_conf->{type} =~ m/\.pxe$/) { $machine .= '.pxe'; } diff --git a/PVE/QemuServer/Makefile b/PVE/QemuServer/Makefile index fd8cfbb..ac26e56 100644 --- a/PVE/QemuServer/Makefile +++ b/PVE/QemuServer/Makefile @@ -9,7 +9,9 @@ SOURCES=PCI.pm \ Monitor.pm \ Machine.pm \ CPUConfig.pm \ + CGroup.pm \ Drive.pm \ + QMPHelpers.pm .PHONY: install install: ${SOURCES} diff --git a/PVE/QemuServer/Memory.pm b/PVE/QemuServer/Memory.pm index f3e15f1..f365f2d 100644 --- a/PVE/QemuServer/Memory.pm +++ b/PVE/QemuServer/Memory.pm @@ -3,22 +3,158 @@ package PVE::QemuServer::Memory; use strict; use warnings; +use PVE::JSONSchema qw(parse_property_string); use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach); use PVE::Exception qw(raise raise_param_exc); -use PVE::QemuServer; +use PVE::QemuServer::Helpers qw(parse_number_sets); use PVE::QemuServer::Monitor qw(mon_cmd); +use PVE::QemuServer::QMPHelpers qw(qemu_devicedel qemu_objectdel); + +use base qw(Exporter); + +our @EXPORT_OK = qw( +get_current_memory +); + +our $MAX_NUMA = 8; + +my $numa_fmt = { + cpus => { + type => "string", + pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/, + description => "CPUs accessing this NUMA node.", + format_description => "id[-id];...", + }, + memory => { + type => "number", + description => "Amount of memory this NUMA node provides.", + optional => 1, + }, + hostnodes => { + type => "string", + pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/, + description => "Host NUMA nodes to use.", + format_description => "id[-id];...", + optional => 1, + }, + policy => { + type => 'string', + enum => [qw(preferred bind interleave)], + description => "NUMA allocation policy.", + optional => 1, + }, +}; +PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt); +our $numadesc = { + optional => 1, + type => 'string', format => $numa_fmt, + description => "NUMA topology.", +}; +PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc); + +sub parse_numa { + my ($data) = @_; + + my $res = parse_property_string($numa_fmt, $data); + $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus}); + $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes}); + return $res; +} -my $MAX_NUMA = 8; -my $MAX_MEM = 4194304; my $STATICMEM = 1024; +our $memory_fmt = { + current => { + description => "Current amount of online RAM for the VM in MiB. This is the maximum available memory when" + ." you use the balloon device.", + type => 'integer', + default_key => 1, + minimum => 16, + default => 512, + }, +}; + +sub print_memory { + my $memory = shift; + + return PVE::JSONSchema::print_property_string($memory, $memory_fmt); +} + +sub parse_memory { + my ($value) = @_; + + return { current => $memory_fmt->{current}->{default} } if !defined($value); + + my $res = PVE::JSONSchema::parse_property_string($memory_fmt, $value); + + return $res; +} + +my $_host_bits; +sub get_host_phys_address_bits { + return $_host_bits if defined($_host_bits); + + my $fh = IO::File->new ('/proc/cpuinfo', "r") or return; + while (defined(my $line = <$fh>)) { + # hopefully we never need to care about mixed (big.LITTLE) archs + if ($line =~ m/^address sizes\s*:\s*(\d+)\s*bits physical/i) { + $_host_bits = int($1); + $fh->close(); + return $_host_bits; + } + } + $fh->close(); + return; # undef, cannot really do anything.. +} + +my sub get_max_mem { + my ($conf) = @_; + + my $cpu = {}; + if (my $cpu_prop_str = $conf->{cpu}) { + $cpu = PVE::JSONSchema::parse_property_string('pve-vm-cpu-conf', $cpu_prop_str) + or die "Cannot parse cpu description: $cpu_prop_str\n"; + } + my $bits; + if (my $phys_bits = $cpu->{'phys-bits'}) { + if ($phys_bits eq 'host') { + $bits = get_host_phys_address_bits(); + } elsif ($phys_bits =~ /^(\d+)$/) { + $bits = int($phys_bits); + } + } + + if (!defined($bits)) { + my $host_bits = get_host_phys_address_bits() // 36; # fixme: what fallback? + if ($cpu->{cputype} && $cpu->{cputype} =~ /^(host|max)$/) { + $bits = $host_bits; + } else { + $bits = $host_bits > 40 ? 40 : $host_bits; # take the smaller one + } + } + + $bits = $bits & ~1; # round down to nearest even as limit is lower with odd bit sizes + + # heuristic: remove 20 bits to get MB and half that as QEMU needs some overhead + my $bits_to_max_mem = int(1<<($bits - 21)); + + return $bits_to_max_mem > 4*1024*1024 ? 4*1024*1024 : $bits_to_max_mem; +} + +sub get_current_memory { + my ($value) = @_; + + my $memory = parse_memory($value); + return $memory->{current}; +} + sub get_numa_node_list { my ($conf) = @_; my @numa_map; for (my $i = 0; $i < $MAX_NUMA; $i++) { my $entry = $conf->{"numa$i"} or next; - my $numa = PVE::QemuServer::parse_numa($entry) or next; + my $numa = parse_numa($entry) or next; push @numa_map, $i; } return @numa_map if @numa_map; @@ -26,13 +162,19 @@ sub get_numa_node_list { return (0..($sockets-1)); } +sub host_numanode_exists { + my ($id) = @_; + + return -d "/sys/devices/system/node/node$id/"; +} + # only valid when numa nodes map to a single host node sub get_numa_guest_to_host_map { my ($conf) = @_; my $map = {}; for (my $i = 0; $i < $MAX_NUMA; $i++) { my $entry = $conf->{"numa$i"} or next; - my $numa = PVE::QemuServer::parse_numa($entry) or next; + my $numa = parse_numa($entry) or next; $map->{$i} = print_numa_hostnodes($numa->{hostnodes}); } return $map if %$map; @@ -41,17 +183,15 @@ sub get_numa_guest_to_host_map { } sub foreach_dimm{ - my ($conf, $vmid, $memory, $sockets, $func) = @_; + my ($conf, $vmid, $memory, $static_memory, $func) = @_; my $dimm_id = 0; - my $current_size = 0; + my $current_size = $static_memory; my $dimm_size = 0; if($conf->{hugepages} && $conf->{hugepages} == 1024) { - $current_size = 1024 * $sockets; $dimm_size = 1024; } else { - $current_size = 1024; $dimm_size = 512; } @@ -72,64 +212,35 @@ sub foreach_dimm{ } } -sub foreach_reverse_dimm { - my ($conf, $vmid, $memory, $sockets, $func) = @_; - - my $dimm_id = 253; - my $current_size = 0; - my $dimm_size = 0; - - if($conf->{hugepages} && $conf->{hugepages} == 1024) { - $current_size = 8355840; - $dimm_size = 131072; - } else { - $current_size = 4177920; - $dimm_size = 65536; - } - - return if $current_size == $memory; - - my @numa_map = get_numa_node_list($conf); - - for (my $j = 0; $j < 8; $j++) { - for (my $i = 0; $i < 32; $i++) { - my $name = "dimm${dimm_id}"; - $dimm_id--; - my $numanode = $numa_map[(31-$i) % @numa_map]; - $current_size -= $dimm_size; - &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory); - return $current_size if $current_size <= $memory; - } - $dimm_size /= 2; - } -} - sub qemu_memory_hotplug { - my ($vmid, $conf, $defaults, $opt, $value) = @_; + my ($vmid, $conf, $value) = @_; + + return $value if !PVE::QemuServer::Helpers::vm_running_locally($vmid); - return $value if !PVE::QemuServer::check_running($vmid); + my $oldmem = parse_memory($conf->{memory}); + my $newmem = parse_memory($value); - my $sockets = 1; - $sockets = $conf->{sockets} if $conf->{sockets}; + return $value if $newmem->{current} == $oldmem->{current}; - my $memory = $conf->{memory} || $defaults->{memory}; - $value = $defaults->{memory} if !$value; - return $value if $value == $memory; + my $memory = $oldmem->{current}; + $value = $newmem->{current}; + my $sockets = $conf->{sockets} || 1; my $static_memory = $STATICMEM; $static_memory = $static_memory * $sockets if ($conf->{hugepages} && $conf->{hugepages} == 1024); die "memory can't be lower than $static_memory MB" if $value < $static_memory; - die "you cannot add more memory than $MAX_MEM MB!\n" if $memory > $MAX_MEM; + my $MAX_MEM = get_max_mem($conf); + die "you cannot add more memory than max mem $MAX_MEM MB!\n" if $value > $MAX_MEM; - if($value > $memory) { + if ($value > $memory) { my $numa_hostmap; - foreach_dimm($conf, $vmid, $value, $sockets, sub { + foreach_dimm($conf, $vmid, $value, $static_memory, sub { my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_; - return if $current_size <= $conf->{memory}; + return if $current_size <= get_current_memory($conf->{memory}); if ($conf->{hugepages}) { $numa_hostmap = get_numa_guest_to_host_map($conf) if !$numa_hostmap; @@ -143,8 +254,7 @@ sub qemu_memory_hotplug { my $hugepages_host_topology = hugepages_host_topology(); hugepages_allocate($hugepages_topology, $hugepages_host_topology); - eval { mon_cmd($vmid, "object-add", 'qom-type' => "memory-backend-file", id => "mem-$name", props => { - size => int($dimm_size*1024*1024), 'mem-path' => $path, share => JSON::true, prealloc => JSON::true } ); }; + eval { mon_cmd($vmid, "object-add", 'qom-type' => "memory-backend-file", id => "mem-$name", size => int($dimm_size*1024*1024), 'mem-path' => $path, share => JSON::true, prealloc => JSON::true ) }; if (my $err = $@) { hugepages_reset($hugepages_host_topology); die $err; @@ -155,59 +265,67 @@ sub qemu_memory_hotplug { eval { hugepages_update_locked($code); }; } else { - eval { mon_cmd($vmid, "object-add", 'qom-type' => "memory-backend-ram", id => "mem-$name", props => { size => int($dimm_size*1024*1024) } ) }; + eval { mon_cmd($vmid, "object-add", 'qom-type' => "memory-backend-ram", id => "mem-$name", size => int($dimm_size*1024*1024) ) }; } if (my $err = $@) { - eval { PVE::QemuServer::qemu_objectdel($vmid, "mem-$name"); }; + eval { qemu_objectdel($vmid, "mem-$name"); }; die $err; } eval { mon_cmd($vmid, "device_add", driver => "pc-dimm", id => "$name", memdev => "mem-$name", node => $numanode) }; if (my $err = $@) { - eval { PVE::QemuServer::qemu_objectdel($vmid, "mem-$name"); }; + eval { qemu_objectdel($vmid, "mem-$name"); }; die $err; } #update conf after each succesful module hotplug - $conf->{memory} = $current_size; + $newmem->{current} = $current_size; + $conf->{memory} = print_memory($newmem); PVE::QemuConfig->write_config($vmid, $conf); }); } else { - foreach_reverse_dimm($conf, $vmid, $value, $sockets, sub { - my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_; + my $dimms = qemu_memdevices_list($vmid, 'dimm'); - return if $current_size >= $conf->{memory}; - print "try to unplug memory dimm $name\n"; - - my $retry = 0; - while (1) { - eval { PVE::QemuServer::qemu_devicedel($vmid, $name) }; - sleep 3; - my $dimm_list = qemu_dimm_list($vmid); - last if !$dimm_list->{$name}; - raise_param_exc({ $name => "error unplug memory module" }) if $retry > 5; - $retry++; - } + my $current_size = $memory; + for my $name (sort { ($b =~ /^dimm(\d+)$/)[0] <=> ($a =~ /^dimm(\d+)$/)[0] } keys %$dimms) { - #update conf after each succesful module unplug - $conf->{memory} = $current_size; + my $dimm_size = $dimms->{$name}->{size} / 1024 / 1024; - eval { PVE::QemuServer::qemu_objectdel($vmid, "mem-$name"); }; - PVE::QemuConfig->write_config($vmid, $conf); - }); + last if $current_size <= $value; + + print "try to unplug memory dimm $name\n"; + + my $retry = 0; + while (1) { + eval { qemu_devicedel($vmid, $name) }; + sleep 3; + my $dimm_list = qemu_memdevices_list($vmid, 'dimm'); + last if !$dimm_list->{$name}; + raise_param_exc({ $name => "error unplug memory module" }) if $retry > 5; + $retry++; + } + $current_size -= $dimm_size; + #update conf after each succesful module unplug + $newmem->{current} = $current_size; + $conf->{memory} = print_memory($newmem); + + eval { qemu_objectdel($vmid, "mem-$name"); }; + PVE::QemuConfig->write_config($vmid, $conf); + } } + return $conf->{memory}; } -sub qemu_dimm_list { - my ($vmid) = @_; +sub qemu_memdevices_list { + my ($vmid, $type) = @_; my $dimmarray = mon_cmd($vmid, "query-memory-devices"); my $dimms = {}; foreach my $dimm (@$dimmarray) { - + next if $type && $dimm->{data}->{id} !~ /^$type(\d+)$/; $dimms->{$dimm->{data}->{id}}->{id} = $dimm->{data}->{id}; $dimms->{$dimm->{data}->{id}}->{node} = $dimm->{data}->{node}; $dimms->{$dimm->{data}->{id}}->{addr} = $dimm->{data}->{addr}; @@ -218,13 +336,14 @@ sub qemu_dimm_list { } sub config { - my ($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd) = @_; + my ($conf, $vmid, $sockets, $cores, $hotplug, $cmd) = @_; - my $memory = $conf->{memory} || $defaults->{memory}; + my $memory = get_current_memory($conf->{memory}); my $static_memory = 0; - if ($hotplug_features->{memory}) { + if ($hotplug) { die "NUMA needs to be enabled for memory hotplug\n" if !$conf->{numa}; + my $MAX_MEM = get_max_mem($conf); die "Total memory is bigger than ${MAX_MEM}MB\n" if $memory > $MAX_MEM; for (my $i = 0; $i < $MAX_NUMA; $i++) { @@ -232,8 +351,7 @@ sub config { if $conf->{"numa$i"}; } - my $sockets = 1; - $sockets = $conf->{sockets} if $conf->{sockets}; + my $sockets = $conf->{sockets} || 1; $static_memory = $STATICMEM; $static_memory = $static_memory * $sockets if ($conf->{hugepages} && $conf->{hugepages} == 1024); @@ -254,7 +372,7 @@ sub config { my $numa_totalmemory = undef; for (my $i = 0; $i < $MAX_NUMA; $i++) { next if !$conf->{"numa$i"}; - my $numa = PVE::QemuServer::parse_numa($conf->{"numa$i"}); + my $numa = parse_numa($conf->{"numa$i"}); next if !$numa; # memory die "missing NUMA node$i memory value\n" if !$numa->{memory}; @@ -297,7 +415,8 @@ sub config { my $numa_memory = ($static_memory / $sockets); for (my $i = 0; $i < $sockets; $i++) { - die "host NUMA node$i doesn't exist\n" if ! -d "/sys/devices/system/node/node$i/" && $conf->{hugepages}; + die "host NUMA node$i doesn't exist\n" + if !host_numanode_exists($i) && $conf->{hugepages}; my $mem_object = print_mem_object($conf, "ram-node$i", $numa_memory); push @$cmd, '-object', $mem_object; @@ -310,8 +429,8 @@ sub config { } } - if ($hotplug_features->{memory}) { - foreach_dimm($conf, $vmid, $memory, $sockets, sub { + if ($hotplug) { + foreach_dimm($conf, $vmid, $memory, $static_memory, sub { my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_; my $mem_object = print_mem_object($conf, "mem-$name", $dimm_size); @@ -351,7 +470,7 @@ sub print_numa_hostnodes { $hostnodes .= "-$end" if defined($end); $end //= $start; for (my $i = $start; $i <= $end; ++$i ) { - die "host NUMA node$i doesn't exist\n" if ! -d "/sys/devices/system/node/node$i/"; + die "host NUMA node$i doesn't exist\n" if !host_numanode_exists($i); } } return $hostnodes; @@ -394,21 +513,27 @@ sub hugepages_nr { return $size / $hugepages_size; } +sub hugepages_chunk_size_supported { + my ($size) = @_; + + return -d "/sys/kernel/mm/hugepages/hugepages-". ($size * 1024) ."kB"; +} + sub hugepages_size { my ($conf, $size) = @_; die "hugepages option is not enabled" if !$conf->{hugepages}; die "memory size '$size' is not a positive even integer; cannot use for hugepages\n" if $size <= 0 || $size & 1; - my $page_chunk = sub { -d "/sys/kernel/mm/hugepages/hugepages-". ($_[0] * 1024) ."kB" }; - die "your system doesn't support hugepages\n" if !$page_chunk->(2) && !$page_chunk->(1024); + die "your system doesn't support hugepages\n" + if !hugepages_chunk_size_supported(2) && !hugepages_chunk_size_supported(1024); if ($conf->{hugepages} eq 'any') { # try to use 1GB if available && memory size is matching - if ($page_chunk->(1024) && ($size & 1023) == 0) { + if (hugepages_chunk_size_supported(1024) && ($size & 1023) == 0) { return 1024; - } elsif ($page_chunk->(2)) { + } elsif (hugepages_chunk_size_supported(2)) { return 2; } else { die "host only supports 1024 GB hugepages, but requested size '$size' is not a multiple of 1024 MB\n" @@ -417,7 +542,7 @@ sub hugepages_size { my $hugepagesize = $conf->{hugepages}; - if (!$page_chunk->($hugepagesize)) { + if (!hugepages_chunk_size_supported($hugepagesize)) { die "your system doesn't support hugepages of $hugepagesize MB\n"; } elsif (($size % $hugepagesize) != 0) { die "Memory size $size is not a multiple of the requested hugepages size $hugepagesize\n"; @@ -428,22 +553,18 @@ sub hugepages_size { } sub hugepages_topology { - my ($conf) = @_; + my ($conf, $hotplug) = @_; my $hugepages_topology = {}; return if !$conf->{numa}; - my $defaults = PVE::QemuServer::load_defaults(); - my $memory = $conf->{memory} || $defaults->{memory}; + my $memory = get_current_memory($conf->{memory}); my $static_memory = 0; - my $sockets = 1; - $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused - $sockets = $conf->{sockets} if $conf->{sockets}; + my $sockets = $conf->{sockets} || 1; my $numa_custom_topology = undef; - my $hotplug_features = PVE::QemuServer::parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1'); - if ($hotplug_features->{memory}) { + if ($hotplug) { $static_memory = $STATICMEM; $static_memory = $static_memory * $sockets if ($conf->{hugepages} && $conf->{hugepages} == 1024); } else { @@ -453,7 +574,7 @@ sub hugepages_topology { #custom numa topology for (my $i = 0; $i < $MAX_NUMA; $i++) { next if !$conf->{"numa$i"}; - my $numa = PVE::QemuServer::parse_numa($conf->{"numa$i"}); + my $numa = parse_numa($conf->{"numa$i"}); next if !$numa; $numa_custom_topology = 1; @@ -479,10 +600,10 @@ sub hugepages_topology { } } - if ($hotplug_features->{memory}) { + if ($hotplug) { my $numa_hostmap = get_numa_guest_to_host_map($conf); - foreach_dimm($conf, undef, $memory, $sockets, sub { + foreach_dimm($conf, undef, $memory, $static_memory, sub { my ($conf, undef, $name, $dimm_size, $numanode, $current_size, $memory) = @_; $numanode = $numa_hostmap->{$numanode}; diff --git a/PVE/QemuServer/OVF.pm b/PVE/QemuServer/OVF.pm index c76c199..b97b052 100644 --- a/PVE/QemuServer/OVF.pm +++ b/PVE/QemuServer/OVF.pm @@ -111,7 +111,8 @@ sub parse_ovf { my $ovf_name = $xpc->findvalue($xpath_find_name); if ($ovf_name) { - ($qm->{name} = $ovf_name) =~ s/[^a-zA-Z0-9\-]//g; # PVE::QemuServer::confdesc requires a valid DNS name + # PVE::QemuServer::confdesc requires a valid DNS name + ($qm->{name} = $ovf_name) =~ s/[^a-zA-Z0-9\-\.]//g; } else { warn "warning: unable to parse the VM name in this OVF manifest, generating a default value\n"; } @@ -220,10 +221,11 @@ ovf:Item[rasd:InstanceID='%s']/rasd:ResourceType", $controller_id); die "error parsing $filepath, file seems not to exist at $backing_file_path\n"; } - my $virtual_size; - if ( !($virtual_size = PVE::Storage::file_size_info($backing_file_path)) ) { - die "error parsing $backing_file_path, size seems to be $virtual_size\n"; - } + ($backing_file_path) = $backing_file_path =~ m|^(/.*)|; # untaint + + my $virtual_size = PVE::Storage::file_size_info($backing_file_path); + die "error parsing $backing_file_path, cannot determine file size\n" + if !$virtual_size; $pve_disk = { disk_address => $pve_disk_address, diff --git a/PVE/QemuServer/PCI.pm b/PVE/QemuServer/PCI.pm index 2ee142f..1673041 100644 --- a/PVE/QemuServer/PCI.pm +++ b/PVE/QemuServer/PCI.pm @@ -4,7 +4,9 @@ use warnings; use strict; use PVE::JSONSchema; +use PVE::Mapping::PCI; use PVE::SysFSTools; +use PVE::Tools; use base 'Exporter'; @@ -17,10 +19,11 @@ parse_hostpci our $MAX_HOSTPCI_DEVICES = 16; -my $PCIRE = qr/([a-f0-9]{4}:)?[a-f0-9]{2}:[a-f0-9]{2}(?:\.[a-f0-9])?/; +my $PCIRE = qr/(?:[a-f0-9]{4,}:)?[a-f0-9]{2}:[a-f0-9]{2}(?:\.[a-f0-9])?/; my $hostpci_fmt = { host => { default_key => 1, + optional => 1, type => 'string', pattern => qr/$PCIRE(;$PCIRE)*/, format_description => 'HOSTPCIID[;HOSTPCIID2...]', @@ -31,8 +34,18 @@ of PCI virtual functions of the host. HOSTPCIID syntax is: 'bus:dev.func' (hexadecimal numbers) You can us the 'lspci' command to list existing PCI devices. + +Either this or the 'mapping' key must be set. EODESCR }, + mapping => { + optional => 1, + type => 'string', + format_description => 'mapping-id', + format => 'pve-configid', + description => "The ID of a cluster wide mapping. Either this or the default-key 'host'" + ." must be set.", + }, rombar => { type => 'boolean', description => "Specify whether or not the device's ROM will be visible in the" @@ -76,6 +89,34 @@ The type of mediated device to use. An instance of this type will be created on startup of the VM and will be cleaned up when the VM stops. EODESCR + }, + 'vendor-id' => { + type => 'string', + pattern => qr/^0x[0-9a-fA-F]{4}$/, + format_description => 'hex id', + optional => 1, + description => "Override PCI vendor ID visible to guest" + }, + 'device-id' => { + type => 'string', + pattern => qr/^0x[0-9a-fA-F]{4}$/, + format_description => 'hex id', + optional => 1, + description => "Override PCI device ID visible to guest" + }, + 'sub-vendor-id' => { + type => 'string', + pattern => qr/^0x[0-9a-fA-F]{4}$/, + format_description => 'hex id', + optional => 1, + description => "Override PCI subsystem vendor ID visible to guest" + }, + 'sub-device-id' => { + type => 'string', + pattern => qr/^0x[0-9a-fA-F]{4}$/, + format_description => 'hex id', + optional => 1, + description => "Override PCI subsystem device ID visible to guest" } }; PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt); @@ -224,6 +265,11 @@ sub get_pci_addr_map { return $pci_addr_map; } +sub generate_mdev_uuid { + my ($vmid, $index) = @_; + return sprintf("%08d-0000-0000-0000-%012d", $index, $vmid); +} + my $get_addr_mapping_from_id = sub { my ($map, $id) = @_; @@ -342,6 +388,32 @@ sub print_pcie_root_port { return $res; } +# returns the parsed pci config but parses the 'host' part into +# a list if lists into the 'id' property like this: +# +# { +# mdev => 1, +# rombar => ... +# ... +# ids => [ +# # this contains a list of alternative devices, +# [ +# # which are itself lists of ids for one multifunction device +# { +# id => "0000:00:00.0", +# vendor => "...", +# }, +# { +# id => "0000:00:00.1", +# vendor => "...", +# }, +# ], +# [ +# ... +# ], +# ... +# ], +# } sub parse_hostpci { my ($value) = @_; @@ -349,31 +421,173 @@ sub parse_hostpci { my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value); - my @idlist = split(/;/, $res->{host}); - delete $res->{host}; - foreach my $id (@idlist) { - my $devs = PVE::SysFSTools::lspci($id); - die "no PCI device found for '$id'\n" if !scalar(@$devs); - push @{$res->{pciid}}, @$devs; + my $alternatives = []; + my $host = delete $res->{host}; + my $mapping = delete $res->{mapping}; + + die "Cannot set both 'host' and 'mapping'.\n" if defined($host) && defined($mapping); + + if ($mapping) { + # we have no ordinary pci id, must be a mapping + my $devices = PVE::Mapping::PCI::find_on_current_node($mapping); + die "PCI device mapping not found for '$mapping'\n" if !$devices || !scalar($devices->@*); + + for my $device ($devices->@*) { + eval { PVE::Mapping::PCI::assert_valid($mapping, $device) }; + die "PCI device mapping invalid (hardware probably changed): $@\n" if $@; + push $alternatives->@*, [split(/;/, $device->{path})]; + } + } elsif ($host) { + push $alternatives->@*, [split(/;/, $host)]; + } else { + die "Either 'host' or 'mapping' must be set.\n"; + } + + $res->{ids} = []; + for my $alternative ($alternatives->@*) { + my $ids = []; + foreach my $id ($alternative->@*) { + my $devs = PVE::SysFSTools::lspci($id); + die "no PCI device found for '$id'\n" if !scalar($devs->@*); + push $ids->@*, @$devs; + } + if (scalar($ids->@*) > 1) { + $res->{'has-multifunction'} = 1; + die "cannot use mediated device with multifunction device\n" if $res->{mdev}; + } + push $res->{ids}->@*, $ids; } + return $res; } +# parses all hostpci devices from a config and does some sanity checks +# returns a hash like this: +# { +# hostpci0 => { +# # hash from parse_hostpci function +# }, +# hostpci1 => { ... }, +# ... +# } +sub parse_hostpci_devices { + my ($conf) = @_; + + my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf); + my $legacy_igd = 0; + + my $parsed_devices = {}; + for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++) { + my $id = "hostpci$i"; + my $d = parse_hostpci($conf->{$id}); + next if !$d; + + # check syntax + die "q35 machine model is not enabled" if !$q35 && $d->{pcie}; + + if ($d->{'legacy-igd'}) { + die "only one device can be assigned in legacy-igd mode\n" + if $legacy_igd; + $legacy_igd = 1; + + die "legacy IGD assignment requires VGA mode to be 'none'\n" + if !defined($conf->{'vga'}) || $conf->{'vga'} ne 'none'; + die "legacy IGD assignment requires rombar to be enabled\n" + if defined($d->{rombar}) && !$d->{rombar}; + die "legacy IGD assignment is not compatible with x-vga\n" + if $d->{'x-vga'}; + die "legacy IGD assignment is not compatible with mdev\n" + if $d->{mdev}; + die "legacy IGD assignment is not compatible with q35\n" + if $q35; + die "legacy IGD assignment is not compatible with multifunction devices\n" + if $d->{'has-multifunction'}; + die "legacy IGD assignment is not compatible with alternate devices\n" + if scalar($d->{ids}->@*) > 1; + # check first device for valid id + die "legacy IGD assignment only works for devices on host bus 00:02.0\n" + if $d->{ids}->[0]->[0]->{id} !~ m/02\.0$/; + } + + $parsed_devices->{$id} = $d; + } + + return $parsed_devices; +} + +# takes the hash returned by parse_hostpci_devices and for all non mdev gpus, +# selects one of the given alternatives by trying to reserve it +# +# mdev devices must be chosen later when we actually allocate it, but we +# flatten the inner list since there can only be one device per alternative anyway +my sub choose_hostpci_devices { + my ($devices, $vmid) = @_; + + my $used = {}; + + my $add_used_device = sub { + my ($devices) = @_; + for my $used_device ($devices->@*) { + my $used_id = $used_device->{id}; + die "device '$used_id' assigned more than once\n" if $used->{$used_id}; + $used->{$used_id} = 1; + } + }; + + for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++) { + my $device = $devices->{"hostpci$i"}; + next if !$device; + + if ($device->{mdev}) { + $device->{ids} = [ map { $_->[0] } $device->{ids}->@* ]; + next; + } + + if (scalar($device->{ids}->@* == 1)) { + # we only have one alternative, use that + $device->{ids} = $device->{ids}->[0]; + $add_used_device->($device->{ids}); + next; + } + + my $found = 0; + for my $alternative ($device->{ids}->@*) { + my $ids = [map { $_->{id} } @$alternative]; + + next if grep { defined($used->{$_}) } @$ids; # already used + eval { reserve_pci_usage($ids, $vmid, 10, undef) }; + next if $@; + + # found one that is not used or reserved + $add_used_device->($alternative); + $device->{ids} = $alternative; + $found = 1; + last; + } + die "could not find a free device for 'hostpci$i'\n" if !$found; + } + + return $devices; +} + sub print_hostpci_devices { - my ($vmid, $conf, $devices, $vga, $winversion, $q35, $bridges, $arch, $machine_type, $bootorder) = @_; + my ($vmid, $conf, $devices, $vga, $winversion, $bridges, $arch, $machine_type, $bootorder) = @_; my $kvm_off = 0; my $gpu_passthrough = 0; my $legacy_igd = 0; my $pciaddr; + my $pci_devices = choose_hostpci_devices(parse_hostpci_devices($conf), $vmid); + for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++) { my $id = "hostpci$i"; - my $d = parse_hostpci($conf->{$id}); + my $d = $pci_devices->{$id}; next if !$d; + $legacy_igd = 1 if $d->{'legacy-igd'}; + if (my $pcie = $d->{pcie}) { - die "q35 machine model is not enabled" if !$q35; # win7 wants to have the pcie devices directly on the pcie bus # instead of in the root port if ($winversion == 7) { @@ -391,29 +605,8 @@ sub print_hostpci_devices { $pciaddr = print_pci_addr($pci_name, $bridges, $arch, $machine_type); } - my $pcidevices = $d->{pciid}; - my $multifunction = @$pcidevices > 1; - - if ($d->{'legacy-igd'}) { - die "only one device can be assigned in legacy-igd mode\n" - if $legacy_igd; - $legacy_igd = 1; - - die "legacy IGD assignment requires VGA mode to be 'none'\n" - if !defined($conf->{'vga'}) || $conf->{'vga'} ne 'none'; - die "legacy IGD assignment requires rombar to be enabled\n" - if defined($d->{rombar}) && !$d->{rombar}; - die "legacy IGD assignment is not compatible with x-vga\n" - if $d->{'x-vga'}; - die "legacy IGD assignment is not compatible with mdev\n" - if $d->{mdev}; - die "legacy IGD assignment is not compatible with q35\n" - if $q35; - die "legacy IGD assignment is not compatible with multifunction devices\n" - if $multifunction; - die "legacy IGD assignment only works for devices on host bus 00:02.0\n" - if $pcidevices->[0]->{id} !~ m/02\.0$/; - } + my $num_devices = scalar($d->{ids}->@*); + my $multifunction = $num_devices > 1 && !$d->{mdev}; my $xvga = ''; if ($d->{'x-vga'}) { @@ -424,16 +617,13 @@ sub print_hostpci_devices { } my $sysfspath; - if ($d->{mdev} && scalar(@$pcidevices) == 1) { - my $pci_id = $pcidevices->[0]->{id}; - my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i); - $sysfspath = "/sys/bus/pci/devices/$pci_id/$uuid"; - } elsif ($d->{mdev}) { - warn "ignoring mediated device '$id' with multifunction device\n"; + if ($d->{mdev}) { + my $uuid = generate_mdev_uuid($vmid, $i); + $sysfspath = "/sys/bus/mdev/devices/$uuid"; } - my $j = 0; - foreach my $pcidevice (@$pcidevices) { + for (my $j = 0; $j < $num_devices; $j++) { + my $pcidevice = $d->{ids}->[$j]; my $devicestr = "vfio-pci"; if ($sysfspath) { @@ -451,14 +641,151 @@ sub print_hostpci_devices { $devicestr .= ",multifunction=on" if $multifunction; $devicestr .= ",romfile=/usr/share/kvm/$d->{romfile}" if $d->{romfile}; $devicestr .= ",bootindex=$bootorder->{$id}" if $bootorder->{$id}; + for my $option (qw(vendor-id device-id sub-vendor-id sub-device-id)) { + $devicestr .= ",x-pci-$option=$d->{$option}" if $d->{$option}; + } } + push @$devices, '-device', $devicestr; - $j++; + last if $d->{mdev}; + } + } + + return ($kvm_off, $gpu_passthrough, $legacy_igd, $pci_devices); +} + +sub prepare_pci_device { + my ($vmid, $pciid, $index, $mdev) = @_; + + my $info = PVE::SysFSTools::pci_device_info("$pciid"); + die "cannot prepare PCI pass-through, IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support(); + die "no pci device info for device '$pciid'\n" if !$info; + + if ($mdev) { + my $uuid = generate_mdev_uuid($vmid, $index); + PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $mdev); + } else { + die "can't unbind/bind PCI group to VFIO '$pciid'\n" + if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid); + die "can't reset PCI device '$pciid'\n" + if $info->{has_fl_reset} && !PVE::SysFSTools::pci_dev_reset($info); + } + + return $info; +} + +my $RUNDIR = '/run/qemu-server'; +my $PCIID_RESERVATION_FILE = "${RUNDIR}/pci-id-reservations"; +my $PCIID_RESERVATION_LOCK = "${PCIID_RESERVATION_FILE}.lock"; + +# a list of PCI ID to VMID reservations, the validity is protected against leakage by either a PID, +# for succesfully started VM processes, or a expiration time for the initial time window between +# reservation and actual VM process start-up. +my $parse_pci_reservation_unlocked = sub { + my $pciids = {}; + if (my $fh = IO::File->new($PCIID_RESERVATION_FILE, "r")) { + while (my $line = <$fh>) { + if ($line =~ m/^($PCIRE)\s(\d+)\s(time|pid)\:(\d+)$/) { + $pciids->{$1} = { + vmid => $2, + "$3" => $4, + }; + } } } + return $pciids; +}; + +my $write_pci_reservation_unlocked = sub { + my ($reservations) = @_; - return ($kvm_off, $gpu_passthrough, $legacy_igd); + my $data = ""; + for my $pci_id (sort keys $reservations->%*) { + my ($vmid, $pid, $time) = $reservations->{$pci_id}->@{'vmid', 'pid', 'time'}; + if (defined($pid)) { + $data .= "$pci_id $vmid pid:$pid\n"; + } else { + $data .= "$pci_id $vmid time:$time\n"; + } + } + PVE::Tools::file_set_contents($PCIID_RESERVATION_FILE, $data); +}; + +# removes all PCI device reservations held by the `vmid` +sub remove_pci_reservation { + my ($vmid) = @_; + + PVE::Tools::lock_file($PCIID_RESERVATION_LOCK, 2, sub { + my $reservation_list = $parse_pci_reservation_unlocked->(); + for my $id (keys %$reservation_list) { + my $reservation = $reservation_list->{$id}; + next if $reservation->{vmid} != $vmid; + delete $reservation_list->{$id}; + } + $write_pci_reservation_unlocked->($reservation_list); + }); + die $@ if $@; +} + +sub reserve_pci_usage { + my ($requested_ids, $vmid, $timeout, $pid) = @_; + + $requested_ids = [ $requested_ids ] if !ref($requested_ids); + return if !scalar(@$requested_ids); # do nothing for empty list + + PVE::Tools::lock_file($PCIID_RESERVATION_LOCK, 5, sub { + my $reservation_list = $parse_pci_reservation_unlocked->(); + + my $ctime = time(); + for my $id ($requested_ids->@*) { + my $reservation = $reservation_list->{$id}; + if ($reservation && $reservation->{vmid} != $vmid) { + # check time based reservation + die "PCI device '$id' is currently reserved for use by VMID '$reservation->{vmid}'\n" + if defined($reservation->{time}) && $reservation->{time} > $ctime; + + if (my $reserved_pid = $reservation->{pid}) { + # check running vm + my $running_pid = PVE::QemuServer::Helpers::vm_running_locally($reservation->{vmid}); + if (defined($running_pid) && $running_pid == $reserved_pid) { + die "PCI device '$id' already in use by VMID '$reservation->{vmid}'\n"; + } else { + warn "leftover PCI reservation found for $id, lets take it...\n"; + } + } + } elsif ($reservation) { + # already reserved by the same vmid + if (my $reserved_time = $reservation->{time}) { + if (defined($timeout)) { + # use the longer timeout + my $old_timeout = $reservation->{time} - 5 - $ctime; + $timeout = $old_timeout if $old_timeout > $timeout; + } + } elsif (my $reserved_pid = $reservation->{pid}) { + my $running_pid = PVE::QemuServer::Helpers::vm_running_locally($reservation->{vmid}); + if (defined($running_pid) && $running_pid == $reservation->{pid}) { + if (defined($pid)) { + die "PCI device '$id' already in use by running VMID '$reservation->{vmid}'\n"; + } elsif (defined($timeout)) { + # ignore timeout reservation for running vms, can happen with e.g. + # qm showcmd + return; + } + } + } + } + + $reservation_list->{$id} = { vmid => $vmid }; + if (defined($pid)) { # VM started up, we can reserve now with the actual PID + $reservation_list->{$id}->{pid} = $pid; + } elsif (defined($timeout)) { # tempoaray reserve as we don't now the PID yet + $reservation_list->{$id}->{time} = $ctime + $timeout + 5; + } + } + $write_pci_reservation_unlocked->($reservation_list); + }); + die $@ if $@; } 1; diff --git a/PVE/QemuServer/QMPHelpers.pm b/PVE/QemuServer/QMPHelpers.pm new file mode 100644 index 0000000..d3a5232 --- /dev/null +++ b/PVE/QemuServer/QMPHelpers.pm @@ -0,0 +1,48 @@ +package PVE::QemuServer::QMPHelpers; + +use warnings; +use strict; + +use PVE::QemuServer::Monitor qw(mon_cmd); + +use base 'Exporter'; + +our @EXPORT_OK = qw( +qemu_deviceadd +qemu_devicedel +qemu_objectadd +qemu_objectdel +); + +sub qemu_deviceadd { + my ($vmid, $devicefull) = @_; + + $devicefull = "driver=".$devicefull; + my %options = split(/[=,]/, $devicefull); + + mon_cmd($vmid, "device_add" , %options); +} + +sub qemu_devicedel { + my ($vmid, $deviceid) = @_; + + my $ret = mon_cmd($vmid, "device_del", id => $deviceid); +} + +sub qemu_objectadd { + my ($vmid, $objectid, $qomtype) = @_; + + mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype); + + return 1; +} + +sub qemu_objectdel { + my ($vmid, $objectid) = @_; + + mon_cmd($vmid, "object-del", id => $objectid); + + return 1; +} + +1; diff --git a/PVE/QemuServer/USB.pm b/PVE/QemuServer/USB.pm index 3c8da2c..4995744 100644 --- a/PVE/QemuServer/USB.pm +++ b/PVE/QemuServer/USB.pm @@ -3,7 +3,10 @@ package PVE::QemuServer::USB; use strict; use warnings; use PVE::QemuServer::PCI qw(print_pci_addr); +use PVE::QemuServer::Machine; +use PVE::QemuServer::Helpers qw(min_version windows_version); use PVE::JSONSchema; +use PVE::Mapping::USB; use base 'Exporter'; our @EXPORT_OK = qw( @@ -12,114 +15,227 @@ get_usb_controllers get_usb_devices ); +my $OLD_MAX_USB = 5; +our $MAX_USB_DEVICES = 14; + + +my $USB_ID_RE = qr/(0x)?([0-9A-Fa-f]{4}):(0x)?([0-9A-Fa-f]{4})/; +my $USB_PATH_RE = qr/(\d+)\-(\d+(\.\d+)*)/; + +my $usb_fmt = { + host => { + default_key => 1, + optional => 1, + type => 'string', + pattern => qr/(?:(?:$USB_ID_RE)|(?:$USB_PATH_RE)|[Ss][Pp][Ii][Cc][Ee])/, + format_description => 'HOSTUSBDEVICE|spice', + description => < { + optional => 1, + type => 'string', + format_description => 'mapping-id', + format => 'pve-configid', + description => "The ID of a cluster wide mapping. Either this or the default-key 'host'" + ." must be set.", + }, + usb3 => { + optional => 1, + type => 'boolean', + description => "Specifies whether if given host option is a USB3 device or port." + ." For modern guests (machine version >= 7.1 and ostype l26 and windows > 7), this flag" + ." is irrelevant (all devices are plugged into a xhci controller).", + default => 0, + }, +}; + +PVE::JSONSchema::register_format('pve-qm-usb', $usb_fmt); + +our $usbdesc = { + optional => 1, + type => 'string', format => $usb_fmt, + description => "Configure an USB device (n is 0 to 4, for machine version >= 7.1 and ostype" + ." l26 or windows > 7, n can be up to 14).", +}; +PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc); + sub parse_usb_device { - my ($value) = @_; + my ($value, $mapping) = @_; - return if !$value; + return if $value && $mapping; # not a valid configuration my $res = {}; - if ($value =~ m/^(0x)?([0-9A-Fa-f]{4}):(0x)?([0-9A-Fa-f]{4})$/) { - $res->{vendorid} = $2; - $res->{productid} = $4; - } elsif ($value =~ m/^(\d+)\-(\d+(\.\d+)*)$/) { - $res->{hostbus} = $1; - $res->{hostport} = $2; - } elsif ($value =~ m/^spice$/i) { - $res->{spice} = 1; - } else { - return; + if (defined($value)) { + if ($value =~ m/^$USB_ID_RE$/) { + $res->{vendorid} = $2; + $res->{productid} = $4; + } elsif ($value =~ m/^$USB_PATH_RE$/) { + $res->{hostbus} = $1; + $res->{hostport} = $2; + } elsif ($value =~ m/^spice$/i) { + $res->{spice} = 1; + } + } elsif (defined($mapping)) { + my $devices = PVE::Mapping::USB::find_on_current_node($mapping); + die "USB device mapping not found for '$mapping'\n" if !$devices || !scalar($devices->@*); + die "More than one USB mapping per host not supported\n" if scalar($devices->@*) > 1; + eval { + PVE::Mapping::USB::assert_valid($mapping, $devices->[0]); + }; + if (my $err = $@) { + die "USB Mapping invalid (hardware probably changed): $err\n"; + } + my $device = $devices->[0]; + + if ($device->{path}) { + $res = parse_usb_device($device->{path}); + } else { + $res = parse_usb_device($device->{id}); + } } return $res; } +my sub assert_usb_index_is_useable { + my ($index, $use_qemu_xhci) = @_; + + die "using usb$index is only possible with machine type >= 7.1 and ostype l26 or windows > 7\n" + if $index >= $OLD_MAX_USB && !$use_qemu_xhci; + + return undef; +} + sub get_usb_controllers { - my ($conf, $bridges, $arch, $machine, $format, $max_usb_devices) = @_; + my ($conf, $bridges, $arch, $machine, $machine_version) = @_; my $devices = []; my $pciaddr = ""; + my $ostype = $conf->{ostype}; + + my $use_qemu_xhci = min_version($machine_version, 7, 1) + && defined($ostype) && ($ostype eq 'l26' || windows_version($ostype) > 7); + my $is_q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf); + if ($arch eq 'aarch64') { $pciaddr = print_pci_addr('ehci', $bridges, $arch, $machine); push @$devices, '-device', "usb-ehci,id=ehci$pciaddr"; - } elsif ($machine !~ /q35/) { # FIXME: combine this and machine_type_is_q35 + } elsif (!$is_q35) { $pciaddr = print_pci_addr("piix3", $bridges, $arch, $machine); push @$devices, '-device', "piix3-usb-uhci,id=uhci$pciaddr.0x2"; - - my $use_usb2 = 0; - for (my $i = 0; $i < $max_usb_devices; $i++) { - next if !$conf->{"usb$i"}; - my $d = eval { PVE::JSONSchema::parse_property_string($format,$conf->{"usb$i"}) }; - next if !$d || $d->{usb3}; # do not add usb2 controller if we have only usb3 devices - $use_usb2 = 1; - } - # include usb device config - push @$devices, '-readconfig', '/usr/share/qemu-server/pve-usb.cfg' if $use_usb2; } - # add usb3 controller if needed - - my $use_usb3 = 0; - for (my $i = 0; $i < $max_usb_devices; $i++) { + my ($use_usb2, $use_usb3) = 0; + my $any_usb = 0; + for (my $i = 0; $i < $MAX_USB_DEVICES; $i++) { next if !$conf->{"usb$i"}; - my $d = eval { PVE::JSONSchema::parse_property_string($format,$conf->{"usb$i"}) }; - next if !$d || !$d->{usb3}; - $use_usb3 = 1; + assert_usb_index_is_useable($i, $use_qemu_xhci); + my $d = eval { PVE::JSONSchema::parse_property_string($usb_fmt, $conf->{"usb$i"}) } or next; + $any_usb = 1; + $use_usb3 = 1 if $d->{usb3}; + $use_usb2 = 1 if !$d->{usb3}; + } + + if (!$use_qemu_xhci && !$is_q35 && $use_usb2 && $arch ne 'aarch64') { + # include usb device config if still on x86 before-xhci machines and if USB 3 is not used + push @$devices, '-readconfig', '/usr/share/qemu-server/pve-usb.cfg'; } $pciaddr = print_pci_addr("xhci", $bridges, $arch, $machine); - push @$devices, '-device', "nec-usb-xhci,id=xhci$pciaddr" if $use_usb3; + if ($use_qemu_xhci && $any_usb) { + push @$devices, '-device', print_qemu_xhci_controller($pciaddr); + } elsif ($use_usb3) { + push @$devices, '-device', "nec-usb-xhci,id=xhci$pciaddr"; + } return @$devices; } sub get_usb_devices { - my ($conf, $format, $max_usb_devices, $features, $bootorder) = @_; + my ($conf, $features, $bootorder, $machine_version) = @_; my $devices = []; - for (my $i = 0; $i < $max_usb_devices; $i++) { + my $ostype = $conf->{ostype}; + my $use_qemu_xhci = min_version($machine_version, 7, 1) + && defined($ostype) && ($ostype eq 'l26' || windows_version($ostype) > 7); + + for (my $i = 0; $i < $MAX_USB_DEVICES; $i++) { my $devname = "usb$i"; next if !$conf->{$devname}; - my $d = eval { PVE::JSONSchema::parse_property_string($format,$conf->{$devname}) }; + assert_usb_index_is_useable($i, $use_qemu_xhci); + my $d = eval { PVE::JSONSchema::parse_property_string($usb_fmt, $conf->{$devname}) }; next if !$d; - if (defined($d->{host})) { - my $hostdevice = parse_usb_device($d->{host}); - $hostdevice->{usb3} = $d->{usb3}; - if ($hostdevice->{spice}) { - # usb redir support for spice - my $bus = 'ehci'; - $bus = 'xhci' if $hostdevice->{usb3} && $features->{spice_usb3}; - - push @$devices, '-chardev', "spicevmc,id=usbredirchardev$i,name=usbredir"; - push @$devices, '-device', "usb-redir,chardev=usbredirchardev$i,id=usbredirdev$i,bus=$bus.0"; - - warn "warning: spice usb port set as bootdevice, ignoring\n" if $bootorder->{$devname}; - } else { - push @$devices, '-device', print_usbdevice_full($conf, $devname, $hostdevice, $bootorder); - } + my $port = $use_qemu_xhci ? $i + 1 : undef; + + if ($d->{host} && $d->{host} =~ m/^spice$/) { + # usb redir support for spice + my $bus = 'ehci'; + $bus = 'xhci' if ($d->{usb3} && $features->{spice_usb3}) || $use_qemu_xhci; + + push @$devices, '-chardev', "spicevmc,id=usbredirchardev$i,name=usbredir"; + push @$devices, '-device', print_spice_usbdevice($i, $bus, $port); + + warn "warning: spice usb port set as bootdevice, ignoring\n" if $bootorder->{$devname}; + } else { + push @$devices, '-device', print_usbdevice_full($conf, $devname, $d, $bootorder, $port); } } return @$devices; } +sub print_qemu_xhci_controller { + my ($pciaddr) = @_; + return "qemu-xhci,p2=15,p3=15,id=xhci$pciaddr"; +} + +sub print_spice_usbdevice { + my ($index, $bus, $port) = @_; + my $device = "usb-redir,chardev=usbredirchardev$index,id=usbredirdev$index,bus=$bus.0"; + if (defined($port)) { + $device .= ",port=$port"; + } + return $device; +} + sub print_usbdevice_full { - my ($conf, $deviceid, $device, $bootorder) = @_; + my ($conf, $deviceid, $device, $bootorder, $port) = @_; return if !$device; my $usbdevice = "usb-host"; - # if it is a usb3 device, attach it to the xhci controller, else omit the bus option - if($device->{usb3}) { + # if it is a usb3 device or with newer qemu, attach it to the xhci controller, else omit the bus option + if ($device->{usb3} || defined($port)) { $usbdevice .= ",bus=xhci.0"; + $usbdevice .= ",port=$port" if defined($port); } - if (defined($device->{vendorid}) && defined($device->{productid})) { - $usbdevice .= ",vendorid=0x$device->{vendorid},productid=0x$device->{productid}"; - } elsif (defined($device->{hostbus}) && defined($device->{hostport})) { - $usbdevice .= ",hostbus=$device->{hostbus},hostport=$device->{hostport}"; + my $parsed = parse_usb_device($device->{host}, $device->{mapping}); + + if (defined($parsed->{vendorid}) && defined($parsed->{productid})) { + $usbdevice .= ",vendorid=0x$parsed->{vendorid},productid=0x$parsed->{productid}"; + } elsif (defined($parsed->{hostbus}) && defined($parsed->{hostport})) { + $usbdevice .= ",hostbus=$parsed->{hostbus},hostport=$parsed->{hostport}"; + } else { + die "no usb id or path given\n"; } $usbdevice .= ",id=$deviceid"; diff --git a/PVE/VZDump/QemuServer.pm b/PVE/VZDump/QemuServer.pm index 8792e76..8c97ee6 100644 --- a/PVE/VZDump/QemuServer.pm +++ b/PVE/VZDump/QemuServer.pm @@ -8,20 +8,25 @@ use File::Path; use IO::File; use IPC::Open3; use JSON; +use POSIX qw(EINTR EAGAIN); use PVE::Cluster qw(cfs_read_file); use PVE::INotify; use PVE::IPCC; use PVE::JSONSchema; +use PVE::PBSClient; +use PVE::RESTEnvironment qw(log_warn); use PVE::QMPClient; use PVE::Storage::Plugin; use PVE::Storage::PBSPlugin; use PVE::Storage; use PVE::Tools; use PVE::VZDump; +use PVE::Format qw(render_duration render_bytes); use PVE::QemuConfig; use PVE::QemuServer; +use PVE::QemuServer::Helpers; use PVE::QemuServer::Machine; use PVE::QemuServer::Monitor qw(mon_cmd); @@ -60,8 +65,13 @@ sub prepare { if defined($conf->{name}); $self->{vm_was_running} = 1; + $self->{vm_was_paused} = 0; if (!PVE::QemuServer::check_running($vmid)) { $self->{vm_was_running} = 0; + } elsif (PVE::QemuServer::vm_is_paused($vmid, 0)) { + # Do not treat a suspended VM as paused, as it would cause us to skip + # fs-freeze even if the VM wakes up before we reach qga_fs_freeze. + $self->{vm_was_paused} = 1; } $task->{hostname} = $conf->{name}; @@ -80,11 +90,10 @@ sub prepare { if (!$volume->{included}) { $self->loginfo("exclude disk '$name' '$volid' ($volume->{reason})"); next; - } elsif ($self->{vm_was_running} && $volume_config->{iothread}) { - if (!PVE::QemuServer::Machine::runs_at_least_qemu_version($vmid, 4, 0, 1)) { - die "disk '$name' '$volid' (iothread=on) can't use backup feature with running QEMU " . - "version < 4.0.1! Either set backup=no for this drive or upgrade QEMU and restart VM\n"; - } + } elsif ($self->{vm_was_running} && $volume_config->{iothread} && + !PVE::QemuServer::Machine::runs_at_least_qemu_version($vmid, 4, 0, 1)) { + die "disk '$name' '$volid' (iothread=on) can't use backup feature with running QEMU " . + "version < 4.0.1! Either set backup=no for this drive or upgrade QEMU and restart VM\n"; } else { my $log = "include disk '$name' '$volid'"; if (defined(my $size = $volume_config->{size})) { @@ -113,18 +122,37 @@ sub prepare { } next if !$path; - my ($size, $format) = eval { PVE::Storage::volume_size_info($self->{storecfg}, $volid, 5) }; - die "no such volume '$volid'\n" if $@; + my ($size, $format); + if ($storeid) { + # The call in list context can be expensive for certain plugins like RBD, just get size + $size = eval { PVE::Storage::volume_size_info($self->{storecfg}, $volid, 5) }; + die "cannot determine size of volume '$volid' - $@\n" if $@; + + my $scfg = PVE::Storage::storage_config($self->{storecfg}, $storeid); + $format = PVE::QemuServer::qemu_img_format($scfg, $volname); + } else { + ($size, $format) = eval { + PVE::Storage::volume_size_info($self->{storecfg}, $volid, 5); + }; + die "cannot determine size and format of volume '$volid' - $@\n" if $@; + } my $diskinfo = { path => $path, volid => $volid, storeid => $storeid, + size => $size, format => $format, virtdev => $ds, qmdevice => "drive-$ds", }; + if ($ds eq 'tpmstate0') { + # TPM drive only exists for backup, which is reflected in the name + $diskinfo->{qmdevice} = 'drive-tpmstate0-backup'; + $task->{tpmpath} = $path; + } + if (-b $path) { $diskinfo->{type} = 'block'; } else { @@ -174,12 +202,16 @@ sub start_vm { sub suspend_vm { my ($self, $task, $vmid) = @_; + return if $self->{vm_was_paused}; + $self->cmd ("qm suspend $vmid --skiplock"); } sub resume_vm { my ($self, $task, $vmid) = @_; + return if $self->{vm_was_paused}; + $self->cmd ("qm resume $vmid --skiplock"); } @@ -192,24 +224,25 @@ sub assemble { my $firewall_src = "/etc/pve/firewall/$vmid.fw"; my $firewall_dest = "$task->{tmpdir}/qemu-server.fw"; - my $outfd = IO::File->new (">$outfile") || - die "unable to open '$outfile'"; - my $conffd = IO::File->new ($conffile, 'r') || - die "unable open '$conffile'"; + my $outfd = IO::File->new(">$outfile") or die "unable to open '$outfile' - $!\n"; + my $conffd = IO::File->new($conffile, 'r') or die "unable to open '$conffile' - $!\n"; my $found_snapshot; my $found_pending; + my $found_cloudinit; while (defined (my $line = <$conffd>)) { next if $line =~ m/^\#vzdump\#/; # just to be sure next if $line =~ m/^\#qmdump\#/; # just to be sure if ($line =~ m/^\[(.*)\]\s*$/) { if ($1 =~ m/PENDING/i) { $found_pending = 1; + } elsif ($1 =~ m/special:cloudinit/) { + $found_cloudinit = 1; } else { $found_snapshot = 1; } } - next if $found_snapshot || $found_pending; # skip all snapshots and pending changes config data + next if $found_snapshot || $found_pending || $found_cloudinit; # skip all snapshots,pending changes and cloudinit config data if ($line =~ m/^unused\d+:\s*(\S+)\s*/) { $self->loginfo("skip unused drive '$1' (not included into backup)"); @@ -253,53 +286,13 @@ sub archive { } } -# number, [precision=1] -my $num2str = sub { - return sprintf( "%." . ( $_[1] || 1 ) . "f", $_[0] ); -}; -my sub bytes_to_human { - my ($bytes, $precission) = @_; - - return $num2str->($bytes, $precission) . ' B' if $bytes < 1024; - my $kb = $bytes/1024; - - return $num2str->($kb, $precission) . " KiB" if $kb < 1024; - my $mb = $kb/1024; - - return $num2str->($mb, $precission) . " MiB" if $mb < 1024; - my $gb = $mb/1024; - - return $num2str->($gb, $precission) . " GiB" if $gb < 1024; - my $tb = $gb/1024; - - return $num2str->($tb, $precission) . " TiB"; -} -my sub duration_to_human { - my ($seconds) = @_; - - return sprintf('%2ds', $seconds) if $seconds < 60; - my $minutes = $seconds / 60; - $seconds = $seconds % 60; - - return sprintf('%2dm %2ds', $minutes, $seconds) if $minutes < 60; - my $hours = $minutes / 60; - $minutes = $minutes % 60; - - return sprintf('%2dh %2dm %2ds', $hours, $minutes, $seconds) if $hours < 24; - my $days = $hours / 24; - $hours = $hours % 24; - - return sprintf('%2dd %2dh %2dm', $days, $hours, $minutes); -} - my $bitmap_action_to_human = sub { my ($self, $info) = @_; my $action = $info->{action}; if ($action eq "not-used") { - return "disabled (no support)" if $self->{vm_was_running}; - return "disabled (VM not running)"; + return "disabled (no support)"; } elsif ($action eq "not-used-removed") { return "disabled (old bitmap cleared)"; } elsif ($action eq "new") { @@ -308,8 +301,8 @@ my $bitmap_action_to_human = sub { if ($info->{dirty} == 0) { return "OK (drive clean)"; } else { - my $size = bytes_to_human($info->{size}); - my $dirty = bytes_to_human($info->{dirty}); + my $size = render_bytes($info->{size}, 1); + my $dirty = render_bytes($info->{dirty}, 1); return "OK ($dirty of $size dirty)"; } } elsif ($action eq "invalid") { @@ -331,7 +324,7 @@ my $query_backup_status_loop = sub { my ($mb, $delta) = @_; return "0 B/s" if $mb <= 0; my $bw = int(($mb / $delta)); - return bytes_to_human($bw) . "/s"; + return render_bytes($bw, 1) . "/s"; }; my $target = 0; @@ -353,13 +346,12 @@ my $query_backup_status_loop = sub { $last_reused += $info->{size} - $info->{dirty}; } if ($target < $total) { - my $total_h = bytes_to_human($total); - my $target_h = bytes_to_human($target); + my $total_h = render_bytes($total, 1); + my $target_h = render_bytes($target, 1); $self->loginfo("using fast incremental mode (dirty-bitmap), $target_h dirty of $total_h total"); } } - my $first_round = 1; my $last_finishing = 0; while(1) { my $status = mon_cmd($vmid, 'query-backup'); @@ -389,16 +381,11 @@ my $query_backup_status_loop = sub { my $timediff = ($ctime - $last_time) || 1; # fixme my $mbps_read = $get_mbps->($rbytes, $timediff); my $mbps_write = $get_mbps->($wbytes, $timediff); - my $target_h = bytes_to_human($target); - my $transferred_h = bytes_to_human($transferred); - - if (!$has_query_bitmap && $first_round && $target != $total) { # FIXME: remove with PVE 7.0 - my $total_h = bytes_to_human($total); - $self->loginfo("using fast incremental mode (dirty-bitmap), $target_h dirty of $total_h total"); - } + my $target_h = render_bytes($target, 1); + my $transferred_h = render_bytes($transferred, 1); my $statusline = sprintf("%3d%% ($transferred_h of $target_h) in %s" - .", read: $mbps_read, write: $mbps_write", $percent, duration_to_human($duration)); + .", read: $mbps_read, write: $mbps_write", $percent, render_duration($duration)); my $res = $status->{status} || 'unknown'; if ($res ne 'active') { @@ -431,23 +418,22 @@ my $query_backup_status_loop = sub { $last_finishing = $status->{finishing}; } sleep(1); - $first_round = 0 if $first_round; } my $duration = time() - $starttime; if ($last_zero) { my $zero_per = $last_target ? int(($last_zero * 100)/$last_target) : 0; - my $zero_h = bytes_to_human($last_zero, 2); + my $zero_h = render_bytes($last_zero); $self->loginfo("backup is sparse: $zero_h (${zero_per}%) total zero data"); } if ($reused) { - my $reused_h = bytes_to_human($reused, 2); + my $reused_h = render_bytes($reused); my $reuse_per = int($reused * 100 / $last_total); $self->loginfo("backup was done incrementally, reused $reused_h (${reuse_per}%)"); } if ($transferred) { - my $transferred_h = bytes_to_human($transferred, 2); + my $transferred_h = render_bytes($transferred); if ($duration) { my $mbps = $get_mbps->($transferred, $duration); $self->loginfo("transferred $transferred_h in $duration seconds ($mbps)"); @@ -462,6 +448,199 @@ my $query_backup_status_loop = sub { }; }; +my $attach_tpmstate_drive = sub { + my ($self, $task, $vmid) = @_; + + return if !$task->{tpmpath}; + + # unconditionally try to remove the tpmstate-named drive - it only exists + # for backing up, and avoids errors if left over from some previous event + eval { PVE::QemuServer::qemu_drivedel($vmid, "tpmstate0-backup"); }; + + $self->loginfo('attaching TPM drive to QEMU for backup'); + + my $drive = "file=$task->{tpmpath},if=none,read-only=on,id=drive-tpmstate0-backup"; + $drive =~ s/\\/\\\\/g; + my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\""); + die "attaching TPM drive failed - $ret\n" if $ret !~ m/OK/s; +}; + +my $detach_tpmstate_drive = sub { + my ($task, $vmid) = @_; + return if !$task->{tpmpath} || !PVE::QemuServer::check_running($vmid); + eval { PVE::QemuServer::qemu_drivedel($vmid, "tpmstate0-backup"); }; +}; + +my sub add_backup_performance_options { + my ($qmp_param, $perf, $qemu_support) = @_; + + return if !$perf || scalar(keys $perf->%*) == 0; + + if (!$qemu_support) { + my $settings_string = join(', ', sort keys $perf->%*); + log_warn("ignoring setting(s): $settings_string - issue checking if supported"); + return; + } + + if (defined($perf->{'max-workers'})) { + if ($qemu_support->{'backup-max-workers'}) { + $qmp_param->{'max-workers'} = int($perf->{'max-workers'}); + } else { + log_warn("ignoring 'max-workers' setting - not supported by running QEMU"); + } + } +} + +sub get_and_check_pbs_encryption_config { + my ($self) = @_; + + my $opts = $self->{vzdump}->{opts}; + my $scfg = $opts->{scfg}; + + my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($scfg, $opts->{storage}); + my $master_keyfile = PVE::Storage::PBSPlugin::pbs_master_pubkey_file_name($scfg, $opts->{storage}); + + if (-e $keyfile) { + if (-e $master_keyfile) { + $self->loginfo("enabling encryption with master key feature"); + return ($keyfile, $master_keyfile); + } elsif ($scfg->{'master-pubkey'}) { + die "master public key configured but no key file found\n"; + } else { + $self->loginfo("enabling encryption"); + return ($keyfile, undef); + } + } else { + my $encryption_fp = $scfg->{'encryption-key'}; + die "encryption configured ('$encryption_fp') but no encryption key file found!\n" + if $encryption_fp; + if (-e $master_keyfile) { + $self->log( + 'warn', + "backup target storage is configured with master-key, but no encryption key set!" + ." Ignoring master key settings and creating unencrypted backup." + ); + } + return (undef, undef); + } + die "internal error - unhandled case for getting & checking PBS encryption ($keyfile, $master_keyfile)!"; +} + +my sub cleanup_fleecing_images { + my ($self, $disks) = @_; + + for my $di ($disks->@*) { + if (my $volid = $di->{'fleece-volid'}) { + eval { PVE::Storage::vdisk_free($self->{storecfg}, $volid); }; + $self->log('warn', "error removing fleecing image '$volid' - $@") if $@; + } + } +} + +my sub allocate_fleecing_images { + my ($self, $disks, $vmid, $fleecing_storeid, $format) = @_; + + die "internal error - no fleecing storage specified\n" if !$fleecing_storeid; + + # TODO what about potential left-over images from a failed attempt? Just + # auto-remove? While unlikely, could conflict with manually created image from user... + + eval { + my $n = 0; # counter for fleecing image names + + for my $di ($disks->@*) { + next if $di->{virtdev} =~ m/^(?:tpmstate|efidisk)\d$/; # too small to be worth it + if ($di->{type} eq 'block' || $di->{type} eq 'file') { + my $scfg = PVE::Storage::storage_config($self->{storecfg}, $fleecing_storeid); + my $name = "vm-$vmid-fleece-$n"; + $name .= ".$format" if $scfg->{path}; + + my $size = PVE::Tools::convert_size($di->{size}, 'b' => 'kb'); + + $di->{'fleece-volid'} = PVE::Storage::vdisk_alloc( + $self->{storecfg}, $fleecing_storeid, $vmid, $format, $name, $size); + + $n++; + } else { + die "implement me (type '$di->{type}')"; + } + } + }; + if (my $err = $@) { + cleanup_fleecing_images($self, $disks); + die $err; + } +} + +my sub detach_fleecing_images { + my ($disks, $vmid) = @_; + + return if !PVE::QemuServer::Helpers::vm_running_locally($vmid); + + for my $di ($disks->@*) { + if (my $volid = $di->{'fleece-volid'}) { + my $devid = "$di->{qmdevice}-fleecing"; + $devid =~ s/^drive-//; # re-added by qemu_drivedel() + eval { PVE::QemuServer::qemu_drivedel($vmid, $devid) }; + } + } +} + +my sub attach_fleecing_images { + my ($self, $disks, $vmid, $format) = @_; + + # unconditionally try to remove potential left-overs from a previous backup + detach_fleecing_images($disks, $vmid); + + my $vollist = [ map { $_->{'fleece-volid'} } grep { $_->{'fleece-volid'} } $disks->@* ]; + PVE::Storage::activate_volumes($self->{storecfg}, $vollist); + + for my $di ($disks->@*) { + if (my $volid = $di->{'fleece-volid'}) { + $self->loginfo("$di->{qmdevice}: attaching fleecing image $volid to QEMU"); + + my $path = PVE::Storage::path($self->{storecfg}, $volid); + my $devid = "$di->{qmdevice}-fleecing"; + my $drive = "file=$path,if=none,id=$devid,format=$format,discard=unmap"; + # Specify size explicitly, to make it work if storage backend rounded up size for + # fleecing image when allocating. + $drive .= ",size=$di->{size}" if $format eq 'raw'; + $drive =~ s/\\/\\\\/g; + my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\""); + die "attaching fleecing image $volid failed - $ret\n" if $ret !~ m/OK/s; + } + } +} + +my sub check_and_prepare_fleecing { + my ($self, $vmid, $fleecing_opts, $disks, $is_template, $qemu_support) = @_; + + # Even if the VM was started specifically for fleecing, it's possible that the VM is resumed and + # then starts doing IO. For VMs that are not resumed the fleecing images will just stay empty, + # so there is no big cost. + + my $use_fleecing = $fleecing_opts && $fleecing_opts->{enabled} && !$is_template; + + if ($use_fleecing && !defined($qemu_support->{'backup-fleecing'})) { + $self->log( + 'warn', + "running QEMU version does not support backup fleecing - continuing without", + ); + $use_fleecing = 0; + } + + if ($use_fleecing) { + my ($default_format, $valid_formats) = PVE::Storage::storage_default_format( + $self->{storecfg}, $fleecing_opts->{storage}); + my $format = scalar(grep { $_ eq 'qcow2' } $valid_formats->@*) ? 'qcow2' : 'raw'; + + allocate_fleecing_images($self, $disks, $vmid, $fleecing_opts->{storage}, $format); + attach_fleecing_images($self, $disks, $vmid, $format); + } + + return $use_fleecing; +} + sub archive_pbs { my ($self, $task, $vmid) = @_; @@ -473,20 +652,15 @@ sub archive_pbs { my $starttime = time(); - my $server = $scfg->{server}; - my $datastore = $scfg->{datastore}; - my $username = $scfg->{username} // 'root@pam'; my $fingerprint = $scfg->{fingerprint}; - - my $repo = "$username\@$server:$datastore"; + my $repo = PVE::PBSClient::get_repository($scfg); my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $opts->{storage}); - my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($scfg, $opts->{storage}); + my ($keyfile, $master_keyfile) = $self->get_and_check_pbs_encryption_config(); my $diskcount = scalar(@{$task->{disks}}); - # proxmox-backup-client can only handle raw files and block devs - # only use it (directly) for disk-less VMs + # proxmox-backup-client can only handle raw files and block devs, so only use it (directly) for + # disk-less VMs if (!$diskcount) { - my @pathlist; $self->loginfo("backup contains no disks"); local $ENV{PBS_PASSWORD} = $password; @@ -499,6 +673,13 @@ sub archive_pbs { '--backup-id', "$vmid", '--backup-time', $task->{backup_time}, ]; + if (defined(my $ns = $scfg->{namespace})) { + push @$cmd, '--ns', $ns; + } + if (defined($keyfile)) { + push @$cmd, '--keyfile', $keyfile; + push @$cmd, '--master-pubkey-file', $master_keyfile if defined($master_keyfile); + } push @$cmd, "qemu-server.conf:$conffile"; push @$cmd, "fw.conf:$firewall" if -e $firewall; @@ -513,8 +694,10 @@ sub archive_pbs { # get list early so we die on unkown drive types before doing anything my $devlist = _get_task_devlist($task); + my $use_fleecing; $self->enforce_vm_running_for_backup($vmid); + $self->{qmeventd_fh} = PVE::QemuServer::register_qmeventd_handle($vmid); my $backup_job_uuid; eval { @@ -523,11 +706,28 @@ sub archive_pbs { }; my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") }; - if (!$qemu_support) { - die "PBS backups are not supported by the running QEMU version. Please make " - . "sure you've installed the latest version and the VM has been restarted.\n"; + my $err = $@; + if (!$qemu_support || $err) { + die "query-proxmox-support returned empty value\n" if !$err; + if ($err =~ m/The command query-proxmox-support has not been found/) { + die "PBS backups are not supported by the running QEMU version. Please make " + . "sure you've installed the latest version and the VM has been restarted.\n"; + } else { + die "QMP command query-proxmox-support failed - $err\n"; + } } + # pve-qemu supports it since 5.2.0-1 (PVE 6.4), so safe to die since PVE 8 + die "master key configured but running QEMU version does not support master keys\n" + if !defined($qemu_support->{'pbs-masterkey'}) && defined($master_keyfile); + + $attach_tpmstate_drive->($self, $task, $vmid); + + my $is_template = PVE::QemuConfig->is_template($self->{vmlist}->{$vmid}); + + $use_fleecing = check_and_prepare_fleecing( + $self, $vmid, $opts->{fleecing}, $task->{disks}, $is_template, $qemu_support); + my $fs_frozen = $self->qga_fs_freeze($task, $vmid); my $params = { @@ -539,22 +739,28 @@ sub archive_pbs { devlist => $devlist, 'config-file' => $conffile, }; + $params->{fleecing} = JSON::true if $use_fleecing; + + if (defined(my $ns = $scfg->{namespace})) { + $params->{'backup-ns'} = $ns; + } + $params->{speed} = $opts->{bwlimit}*1024 if $opts->{bwlimit}; + add_backup_performance_options($params, $opts->{performance}, $qemu_support); + $params->{fingerprint} = $fingerprint if defined($fingerprint); $params->{'firewall-file'} = $firewall if -e $firewall; - if (-e $keyfile) { - $self->loginfo("enabling encryption"); + + $params->{encrypt} = defined($keyfile) ? JSON::true : JSON::false; + if (defined($keyfile)) { $params->{keyfile} = $keyfile; - $params->{encrypt} = JSON::true; - } else { - $params->{encrypt} = JSON::false; + $params->{"master-keyfile"} = $master_keyfile if defined($master_keyfile); } - my $is_template = PVE::QemuConfig->is_template($self->{vmlist}->{$vmid}); $params->{'use-dirty-bitmap'} = JSON::true - if $qemu_support->{'pbs-dirty-bitmap'} && $self->{vm_was_running} && !$is_template; + if $qemu_support->{'pbs-dirty-bitmap'} && !$is_template; - $params->{timeout} = 60; # give some time to connect to the backup server + $params->{timeout} = 125; # give some time to connect to the backup server my $res = eval { mon_cmd($vmid, "backup", %$params) }; my $qmperr = $@; @@ -578,9 +784,15 @@ sub archive_pbs { if ($err) { $self->logerr($err); $self->mon_backup_cancel($vmid); + $self->resume_vm_after_job_start($task, $vmid); } $self->restore_vm_power_state($vmid); + if ($use_fleecing) { + detach_fleecing_images($task->{disks}, $vmid); + cleanup_fleecing_images($self, $task->{disks}); + } + die $err if $err; } @@ -640,8 +852,10 @@ sub archive_vma { $speed = $opts->{bwlimit}*1024; } + my $is_template = PVE::QemuConfig->is_template($self->{vmlist}->{$vmid}); + my $diskcount = scalar(@{$task->{disks}}); - if (PVE::QemuConfig->is_template($self->{vmlist}->{$vmid}) || !$diskcount) { + if ($is_template || !$diskcount) { my @pathlist; foreach my $di (@{$task->{disks}}) { if ($di->{type} eq 'block' || $di->{type} eq 'file') { @@ -681,8 +895,10 @@ sub archive_vma { } my $devlist = _get_task_devlist($task); + my $use_fleecing; $self->enforce_vm_running_for_backup($vmid); + $self->{qmeventd_fh} = PVE::QemuServer::register_qmeventd_handle($vmid); my $cpid; my $backup_job_uuid; @@ -692,6 +908,16 @@ sub archive_vma { die "interrupted by signal\n"; }; + # Currently, failing to determine Proxmox support is not critical here, because it's only + # used for performance settings like 'max-workers'. + my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") }; + log_warn($@) if $@; + + $attach_tpmstate_drive->($self, $task, $vmid); + + $use_fleecing = check_and_prepare_fleecing( + $self, $vmid, $opts->{fleecing}, $task->{disks}, $is_template, $qemu_support); + my $outfh; if ($opts->{stdout}) { $outfh = $opts->{stdout}; @@ -720,6 +946,8 @@ sub archive_vma { devlist => $devlist }; $params->{'firewall-file'} = $firewall if -e $firewall; + $params->{fleecing} = JSON::true if $use_fleecing; + add_backup_performance_options($params, $opts->{performance}, $qemu_support); $qmpclient->queue_cmd($vmid, $backup_cb, 'backup', %$params); }; @@ -755,10 +983,16 @@ sub archive_vma { if ($err) { $self->logerr($err); $self->mon_backup_cancel($vmid); + $self->resume_vm_after_job_start($task, $vmid); } $self->restore_vm_power_state($vmid); + if ($use_fleecing) { + detach_fleecing_images($task->{disks}, $vmid); + cleanup_fleecing_images($self, $task->{disks}); + } + if ($err) { if ($cpid) { kill(9, $cpid); @@ -795,13 +1029,19 @@ sub _get_task_devlist { sub qga_fs_freeze { my ($self, $task, $vmid) = @_; - return if !$self->{vmlist}->{$vmid}->{agent} || $task->{mode} eq 'stop' || !$self->{vm_was_running}; + return if !$self->{vmlist}->{$vmid}->{agent} || $task->{mode} eq 'stop' || !$self->{vm_was_running} || $self->{vm_was_paused}; if (!PVE::QemuServer::qga_check_running($vmid, 1)) { $self->loginfo("skipping guest-agent 'fs-freeze', agent configured but not running?"); return; } + my $freeze = PVE::QemuServer::get_qga_key($self->{vmlist}->{$vmid}, 'freeze-fs-on-backup') // 1; + if (!$freeze) { + $self->loginfo("skipping guest-agent 'fs-freeze', disabled in VM options"); + return; + } + $self->loginfo("issuing guest-agent 'fs-freeze' command"); eval { mon_cmd($vmid, "guest-fsfreeze-freeze") }; $self->logerr($@) if $@; @@ -841,11 +1081,11 @@ sub enforce_vm_running_for_backup { die $@ if $@; } -# resume VM againe once we got in a clear state (stop mode backup of running VM) +# resume VM again once in a clear state (stop mode backup of running VM) sub resume_vm_after_job_start { my ($self, $task, $vmid) = @_; - return if !$self->{vm_was_running}; + return if !$self->{vm_was_running} || $self->{vm_was_paused}; if (my $stoptime = $task->{vmstoptime}) { my $delay = time() - $task->{vmstoptime}; @@ -854,7 +1094,7 @@ sub resume_vm_after_job_start { } else { $self->loginfo("resuming VM again"); } - mon_cmd($vmid, 'cont'); + mon_cmd($vmid, 'cont', timeout => 45); } # stop again if VM was not running before @@ -894,7 +1134,11 @@ sub snapshot { sub cleanup { my ($self, $task, $vmid) = @_; - # nothing to do ? + $detach_tpmstate_drive->($task, $vmid); + + if ($self->{qmeventd_fh}) { + close($self->{qmeventd_fh}); + } } 1; diff --git a/debian/changelog b/debian/changelog index d63fd9b..2e4df79 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,1038 @@ +qemu-server (8.2.1) bookworm; urgency=medium + + * cpu config: fix get_cpu_bitness always reverting to default cpu type + + -- Proxmox Support Team Wed, 24 Apr 2024 11:49:03 +0200 + +qemu-server (8.2.0) bookworm; urgency=medium + + * qmeventd: also treat 'prelaunch' and 'suspended' states as active to avoid + issues when backing up VMs that currently are in those states. + + * OS type: add Windows Server 2025 as supported, map it to the same virtual + hardware profile as the Windows 11 one. + + -- Proxmox Support Team Tue, 23 Apr 2024 17:09:20 +0200 + +qemu-server (8.1.4) bookworm; urgency=medium + + * api: create vm: add missing import for serializing machine type to fix a + regression of version 8.1.2. + + -- Proxmox Support Team Sat, 20 Apr 2024 12:28:35 +0200 + +qemu-server (8.1.3) bookworm; urgency=medium + + * firewall: add handling for new nftables based firewall implementation, + which currently is a opt-in drop-in replacement for the older iptables- + based one. + + -- Proxmox Support Team Fri, 19 Apr 2024 20:23:39 +0200 + +qemu-server (8.1.2) bookworm; urgency=medium + + * fix #4136: backup: implement fleecing option for improved guest + stability when the backup target is slow + + * fix #4474: stop VM: add 'overrule-shutdown' parameter to prevent + currently running shutdown tasks from blocking the stop task + + * fix #1905: disk move: allow moving unused disks + + * fix #3784: add vIOMMU parameter to support passthrough of PCI devices to + nested virtual machines + + * fix #5363: cloudinit: fix regression to make creation of scsi cloudinit + disks possible again + + -- Proxmox Support Team Fri, 19 Apr 2024 16:09:18 +0200 + +qemu-server (8.1.1) bookworm; urgency=medium + + * config: pending network: avoid undef-warning on old/new comparison + + * support live-import for 'import-from' disk options on create + + * qm: add 'import' command for importing a VM through a volumeid from a + storage that provides the new 'import' content-type. + + * disk import: warn when fallback is used instead of requested format + + * cpu config: die on hotplug of non x86_64 CPUs + + -- Proxmox Support Team Thu, 14 Mar 2024 14:04:34 +0100 + +qemu-server (8.1.0) bookworm; urgency=medium + + * migration: do not allow live-migration with VNC clipboard, it's not yet + supported by QEMU's vdagent device, which gets used for this feature. + + * cpu config: add QEMU 8.1 cpu models + + * fix #4501: TCP migration: start vm: move port reservation and usage closer + together + + * fix #2258: select correct device when removing drive snapshot via QEMU, as + the device where the disk is currently attached needs to be targeted, not + the one where the disk was attached at the time the snapshot was taken + + * fix #4957: allow one to set the vendor and product information for + SCSI-Disks explicitly + + * migration: remember original volume names from the source so that they can + get deactivated even if the volume name has to change due to being already + in use on the target storage. + + * fix #4085: properly activate the storage(s) of custom cloud-init snippets + + * fix #1734: clone VM: if deactivation of source volume fails demote error + to warning. Deactivation can fail if the source is still, or again, in + use, e.g., due to parallel clones of the same template. + + * mediated device pass-through: avoid race condition for cleaning up on VM + reboot + + * prevent starting a 32-bit VM using a 64-bit OVMF BIOS + + * QMP client: increase default timeout for drive-mirror to 10 minutes like + for other block operations. + + -- Proxmox Support Team Fri, 08 Mar 2024 15:00:25 +0100 + +qemu-server (8.0.10) bookworm; urgency=medium + + * sdn: pass vmid and hostname to allow requesting a new mapping + + -- Proxmox Support Team Wed, 22 Nov 2023 14:12:46 +0100 + +qemu-server (8.0.9) bookworm; urgency=medium + + * add clipboard option to to vga config entry + + * api: add clipboard variable to return at status/current + + * recommend libpve-network-perl for SDN support + + * initial support for dhcp ip allocation in dhcp-enabled SDN zones + + -- Proxmox Support Team Tue, 21 Nov 2023 15:40:27 +0100 + +qemu-server (8.0.8) bookworm; urgency=medium + + * fix #2816: restore: remove timeout when allocating disks + + * start: increase maximal timeout if using PCI passthrough + + * fix #4522: api: vncproxy: also set environment variable for the ticket + if the websocket option is not set + + * backup, migrate: fix races with suspended VMs that can wake up + + * cpu hotplug: cannot change feature online, so keep these as pending change + + * nbd-stop: increase timeout to 25s + + * start: add warning if a deprecated machine version is configured + + -- Proxmox Support Team Sun, 12 Nov 2023 18:54:37 +0100 + +qemu-server (8.0.7) bookworm; urgency=medium + + * fix #4822: vzdump: fix PBS encryption for guests without disks + + * qmeventd: fix parsing of VMID in presence of legacy cgroup entries + + * api: check access for already configured bridge when updating vNIC + + * fix #4620: make 'ide1' and 'ide3' drive keys work for machine type q35 + + * cloudinit: fix two checks that were mistakenly restricted to root only, + one for setting the ciupgrade option and one for updating the cloudinit + drive + + * migration: improve format hint when allocating live-migrated disk on the + target to make e.g. remote-migration with qcow2 and LVM-thin target work + + * net: fix setting value for tx_queue_size + + * fix #3963: allow backup of template VM with immutable TPM drive + + -- Proxmox Support Team Mon, 21 Aug 2023 11:30:45 +0200 + +qemu-server (8.0.6) bookworm; urgency=medium + + * cloudinit: restore previous default for package upgrades + + * migration: only migrate disks used by the guest, not also those that are + owned by them (VMID in name) but not referenced in the config + + * migration: fail when aliased volume are detected, as referencing the same + volume multiple times can lead to unexpected behavior in a migration. + + * migration: fix issue with qcow2 cloudinit disk live migration + + -- Proxmox Support Team Wed, 21 Jun 2023 13:03:01 +0200 + +qemu-server (8.0.5) bookworm; urgency=medium + + * restore: extend permissions checks + + * vm start: always reset any failed-state of the VM systemd scope to avoid + failing a re-start after, e.g., a OOM kill. + + -- Proxmox Support Team Wed, 21 Jun 2023 09:17:41 +0200 + +qemu-server (8.0.4) bookworm; urgency=medium + + * vCPU config: add new x86-64-v2, x86-64-v3 and x86-64-v4 models + + * fix #4784: helpers: cope with native versions in manager version check + + * enable cluster mapped USB devices for guests + + * enable cluster mapped PCI devices for guests + + -- Proxmox Support Team Mon, 19 Jun 2023 07:24:11 +0200 + +qemu-server (8.0.3) bookworm; urgency=medium + + * qemu: fix permission check call + + -- Proxmox Support Team Fri, 09 Jun 2023 12:20:40 +0200 + +qemu-server (8.0.2) bookworm; urgency=medium + + * cfg2cmd: use actual backend names instead of removed tty and paraport + aliases + + * cfg2cmd: replace deprecated no-acpi option with acpi=off machine flag + + * cfg2cmd: replace deprecated no-hpet option with hpet=off machine flag + + * schema: avoid using deprecated -no-hpet in example for 'args' property, + instead pass thate via new machine option + + * allow setting ipconfigX with VM.Config.Cloudinit + + * fix #3428: cloudinit: add parameter for upgrade on boot + + * cloudinit: fix 'pending' api endpoint + + * fast plug options: add migrate_downtime and migrate_speed for convenience + + * fix #517: api: allow resizing qcow2 disk with snapshots + + * fix #2315: api: have resize endpoint spawn a worker task + + * cloudinit: pass through hostname via fqdn field + + * qmeventd: extract vmid from cgroup file instead of cmdline + + * config: implement method to calculate derived properties from a config + + * api: check bridge access for create, update, clone & restore + + * qm: remote migration: improve error when storage cannot be found + + -- Proxmox Support Team Fri, 09 Jun 2023 10:26:19 +0200 + +qemu-server (8.0.1) bookworm; urgency=medium + + * fix #4737: qmeventd: gracefully handle interrupted epoll_wait call + + * handle and warn about VM network interfaces not attached to any bridges + + * block resize: avoid passing zero size to QMP command + + * qmrestore: improve description of bwlimit parameter + + * api: switch agent api call to 'array' type + + * tests: fix invoking migration tests with make + + -- Proxmox Support Team Wed, 07 Jun 2023 13:50:09 +0200 + +qemu-server (8.0.0) bookworm; urgency=medium + + * never enable 'smm' flag for the 'virt' machine type (doesn't exist) + + * test: mock calls that can fail in a chroot environment + + * rebuild for Debian Bookworm based releases + + -- Proxmox Support Team Fri, 19 May 2023 15:07:45 +0200 + +qemu-server (7.4-3) bullseye; urgency=medium + + * backup prepare: fix format detection for disks without storage ID + + * backup prepare: improve error messages + + -- Proxmox Support Team Mon, 27 Mar 2023 11:17:16 +0200 + +qemu-server (7.4-2) bullseye; urgency=medium + + * avoid list context for volume_size_info calls as otherwise we + unnecessarily take a slower code path + + -- Proxmox Support Team Tue, 21 Mar 2023 16:51:01 +0100 + +qemu-server (7.4-1) bullseye; urgency=medium + + * fix #4553: nvidia vgpu: reuse smbios uuid for '-uuid' parameter + + * pci: workaround nvidia driver issue on mdev cleanup + + * memory: hotplug: sort by numerical ID rather than slot when unplugging + + * memory: use the DIMM list info from QEMU for unplug + + -- Proxmox Support Team Mon, 20 Mar 2023 17:24:45 +0100 + +qemu-server (7.3-4) bullseye; urgency=medium + + * fix #4378: standardized error for missing OVMF files + + * schema: memory: be precise that unit is binary prefix + + * close #2792: allow online migration with replicated snapshots + + * schema: OS type: note that the l26 type is also compatible with Linux 6.x + + * hotplug: disk: mark the 'aio' (async IO) as non-hotpluggable to avoid + suggesting that it already changed + + * fix #4525: clone disk: disallow mirror if it might cause problems with + io_uring using the same heuristics as for start-up + + * start: make not being able to set polling interval for ballooning + non-critical + + * swtpm: enable logging to `/run/qemu-server/$vmid-swtpm.log` + + * fix #4140: vzdump: transform the previous hardcoded behavior of issuing a + fs-freeze and fs-thaw if QGA is enabled into an overrideable option named + 'fs-freeze-on-backup' + + * update network dev: MTU is not hot-pluggable, avoid suggesting so + + * fix #4249: make image clone or conversion respect bandwidth limit + + -- Proxmox Support Team Thu, 23 Feb 2023 17:12:42 +0100 + +qemu-server (7.3-3) bullseye; urgency=medium + + * rollback: ignore auto-start request if VM is already running + + * memory hot-plug: check correct value for maximal memory check + + * fix #4435: device list: avoid error for undefined value + + * fix #4358: ignore any suspended lock when destroying a VM + + * migration: log error from query-migrate, if any, upon migration failure + + * cd rom handling: return a clearer error when there is no CD-ROM drive + + * migration: nbd export: switch away from deprecated QMP command + + -- Proxmox Support Team Mon, 16 Jan 2023 13:52:30 +0100 + +qemu-server (7.3-2) bullseye; urgency=medium + + * fix #4372: improve edge-case for config-loading on VM resume when + migrating + + * ovmf cmd assembly: re-work and re-order arguments assembly + + -- Proxmox Support Team Fri, 16 Dec 2022 12:54:53 +0100 + +qemu-server (7.3-1) bullseye; urgency=medium + + * vm resume: improve loading just recently moved config on nocheck/migrate + handling + + -- Proxmox Support Team Mon, 21 Nov 2022 13:43:59 +0100 + +qemu-server (7.2-12) bullseye; urgency=medium + + * config: only save unique tags when updating them via the API + + * api: create/update vm: fix clamping CPU units function calls + + -- Proxmox Support Team Mon, 21 Nov 2022 08:36:06 +0100 + +qemu-server (7.2-11) bullseye; urgency=medium + + * fdb: only manage FDB entries for Linux bridges, ignore OVS for now + + -- Proxmox Support Team Sun, 20 Nov 2022 16:30:28 +0100 + +qemu-server (7.2-10) bullseye; urgency=medium + + * fix #4321: properly check cloud-init drive permissions, require both + VM.Config.CDROM and VM.Config.Cloudinit, and not VM.Config.Disk, for being + able to add a cloud init drive in the first place. + + * api: config update: enforce new tag permission system when setting or + removing tags from a guest + + + * parse config: do not validate informative values in cloud init section + + * fix edge-cases on new cloudinit pending/active recording + + * mtunnel: add API endpoints + + * migrate: add foundation for remote (external cluster) migration, add + respective endpoints and qm `remote-migrate` CLI command + + * memory hotplug: make max-memory dynamically calculated from the physicall + address bits the VM will use, that is the actual one from the config, if + set, the one from the host for CPU type host and 40 bits as fallback for + everything else. Calculate the addressable memory (e.g., 40 bits = 1 TiB) + and half that for the possible max-memory a VM can use, using the previous + hard-coded 4 TiB as overall maximum for backward compat. + Admins with inhomogeneous CPUs and thus possible different bit-widths need + to take special care themselves to ensure that a VM with memory hot-plug + configured can run on other nodes, for example for live-migration. + + -- Proxmox Support Team Thu, 17 Nov 2022 17:48:03 +0100 + +qemu-server (7.2-8) bullseye; urgency=medium + + * fix #4296: virtio-net: enable packed queues for qemu 7.1 + + * virtio-net: increase defaults rx- and tx-queue-size to 1024 + + * fix #4296: virtio-net: enable packed queues for QEMU machines using 7.1 or + newer + + * net: increase max queues to 64 + + * fix #4284: add read-only to non-hotpluggable disk options + + * delay cloudinit generation in hotplug + + * record cloud-init changes in the cloudinit section + + * rework cloudint config pending handling + + -- Proxmox Support Team Wed, 16 Nov 2022 18:23:39 +0100 + +qemu-server (7.2-7) bullseye; urgency=medium + + * api: create/update vm: automatically clamp cpuunit value depending of + cgroup version + + * improve cloud init support and add cloudinit hotplug + + * vzdump: skip `special:cloudinit` section + + * fix #3890 - GUI: warn for unlikely iothread config clashes + + * fix #4228: add `start` parameter to snapshot rollback API so that one can + automatocally start the VM after rollback finished. + + * vm start/stop: cleanup passed-through pci devices in more situations + + * fix #3593: allow one to configure task set affinity for VMs + + * fix #4324: USB: use qemu-xhci for machine versions >= 7.1 + + * usb: increase max USB devices from 5 to 14 for modern 7.1 machine + and OS versions (Linux 2.6+ annd Windows 8+) + + * fix #4201: delete cloud-init disk on rollback + + * net devs: register vNIC MAC-Address manually to FDB on start/resume if + bridge has learning disabled + + -- Proxmox Support Team Sun, 13 Nov 2022 15:46:18 +0100 + +qemu-server (7.2-6) bullseye; urgency=medium + + * schema: move 'pve-targetstorage' to pve-common + + -- Proxmox Support Team Mon, 07 Nov 2022 16:22:50 +0100 + +qemu-server (7.2-5) bullseye; urgency=medium + + * qmp client: increase guest fstrim timeout to 10 minutes + + * fix #3577: prevent suspension for VMs with pci passthrough + + * cpu config: map deprecated IceLake-Client CPU type to IceLake-Server + + * snapshot: save VM state: propagate error from QEMU + + * api: create disks: avoid adding secondary cloud-init drives + + * vzdump: TPM state: escape drive string + + * qmp client: increase default fallback timeout to 5s + + * fix regex matching network devices in qm cleanup so that vNICs with double + digit IDs are covered too + + * qmeventd: rework 'forced_cleanup' handling and set timeout to 60s + + * qmeventd: send QMP 'quit' command instead of SIGTERM + + * vzdump: set max-workers QMP option when specified and supported + + * fix #4099: disable io_uring for virtual disks on CIFS storages for now + + * qm: move VM-disk related commands to own command group, keep old ones + around for backward compatibility + + -- Proxmox Support Team Mon, 07 Nov 2022 16:15:16 +0100 + +qemu-server (7.2-4) bullseye; urgency=medium + + * fix #3754: encode JSON as utf8 for CLI + + * cpuconfig: add amd epyc milan model + + * fix #4115: enable option to name QEMU threads after their main purpose + + * fix #4119: give namespace parameter to live-restore + + * automatically add 'uuid' parameter when passing through NVIDIA vGPU + + * vzdump/pbs: die with missing, but configured encryption key + + * vzdump/pbs: die with missing, but configured master key + + -- Proxmox Support Team Tue, 16 Aug 2022 13:59:20 +0200 + +qemu-server (7.2-3) bullseye; urgency=medium + + * support pbs namespaces + + -- Proxmox Support Team Thu, 12 May 2022 15:14:39 +0200 + +qemu-server (7.2-2) bullseye; urgency=medium + + * api: reassign disk: drop moved disk from boot order + + * explicitly check some prerequisites for virtio-gl display + + -- Proxmox Support Team Mon, 02 May 2022 17:26:16 +0200 + +qemu-server (7.2-1) bullseye; urgency=medium + + * migrate: add log for guest fstrim and make a failure noticable + + * migrate: resume initially running VM when failing after convergence + + * parse vm config: remove "\s*" from multi-line comment regex + + * memory: enable balloon free-page-reporting for auto-memory reclaim + + * enable spice also for virtio-gl and virtio-gpu displays and report so in + status API + + * api: create: allow overriding non-disk options during restore + + * fix #3861: migrate: fix live migration when cloud-init changes storage + + -- Proxmox Support Team Thu, 28 Apr 2022 18:35:22 +0200 + +qemu-server (7.1-5) bullseye; urgency=medium + + * avoid writing the config if there are no pending changes to apply + + * fix #3792: cloudinit: use of uninitialized value + + * pci: allow override of PCI vendor/device ids + + * drive mirror monitor: warn when suspend/resume/freeze/thaw calls fail + + * update config: allow setting boot-order and dev in one go + + * migrate: move tunnel-helpers to pve-guest-common + + * fix #3683: agent file-write: enable user to encode the content themselves + + * cpu units: lower minimum for accessing full cgroupv2 range + + * fix #3845: also clamp cpu units to cgroup dependent valid range on hotplug + + * clone disk: force raw format for TPM state + + * fix #3886: QEMU restore: verify storage allows images before writing + + * fix #3733: bump the timeout used to wait that a for backup started VM is + fully stopped (i.e., it's "$vmid.scope vanished) to 20 seconds after the + backup has finished to + + * qmp client: increase timeout for thaw to better accommodate the QGA running + in Windows VMs + + * api: vm start: 'force-cpu' is for internal migration use only, mark as + such + + * device unplug: verify that unplugging SCSI disk completed before + continuing with remaining unplug work. + + * clone disk: remove ancient check for min QEMU version 2.7 + + * clone disk: pass in efi vars size rather than config + + * clone disk: allow cloning from an unused or unreferenced disk + + * parse ovf: untaint path when getting the file's size info + + * image convert: allow block device as source + + * fix #3424: api: snapshot delete: wait for active replication + + * PCI: allow longer pci domains + + * fix #3957: spell 'occurred' correctly + + * clone disk: also clone EFI disk from snapshot + + * api: add endpoint for parsing .ovf files + + * api: support VM disk import + + * migrate: keep VM paused after migration if it was before + + * vga: add virtio-gl display type for VIRGL + + * restore: cleanup oldconf: also clean up snapshots from kept volumes + + * restore: also deactivate/destroy cloud-init disk upon error + + -- Proxmox Support Team Mon, 25 Apr 2022 20:15:59 +0200 + +qemu-server (7.1-4) bullseye; urgency=medium + + * migrate: send updated TPM state volume ID to target node on local-storage + migration + + -- Proxmox Support Team Mon, 22 Nov 2021 17:07:13 +0100 + +qemu-server (7.1-3) bullseye; urgency=medium + + * replication: do not setup NBD server on VM migrate for the TPM state, + QEMU cannot access it directly and we already migrate it via the non-QEMU + storage migration anyway. + + -- Proxmox Support Team Tue, 16 Nov 2021 14:04:45 +0100 + +qemu-server (7.1-2) bullseye; urgency=medium + + * cfg2cmd: disable SMM when display=none and SeaBIOS is both used + + * pci: do not reserve pci-ids for mediated devices, already handled by sysfs + anyway + + * exclude efidisk and tpmstate for boot disk selection heuristic + + -- Proxmox Support Team Mon, 15 Nov 2021 16:59:23 +0100 + +qemu-server (7.0-19) bullseye; urgency=medium + + * rollback: improve interaction with snapshot replication + + * cli: qm: rename 'move_disk' command to 'move-disk' with an alias for + backward compatibility + + * pi: move-disk: add possibility to reassign a disk to another VM + + * turn SMM off when SeaBIOS and a serial-display are used in combination to + avoid a possible boot loop + + -- Proxmox Support Team Thu, 11 Nov 2021 12:49:10 +0100 + +qemu-server (7.0-18) bullseye; urgency=medium + + * use non SMM ovmf code file for i440fx machines + + * fix hot-unplugging (removing) a cpulimit on a running VM + + * vm start: only print tpm-related message if there is an actual instance + + * vzdump: increase timeout for QMP 'cont' command after backup started + + * drives: expose readonly flag for SCSI/VirtIO drives as 'ro' property + + * qemu-agent: allow hotplug of the 'fstrim cloned disk' option + + * fix #2429: allow to specify cloud-init vendor snippet via 'cicustom' + + * config: add new meta property with the VM creation time + + * config: meta: also save the QEMU version installed during creation + + * cfg2cmd: switch off ACPI hotplug on bridges for q35 VMs with linux as + ostype to avoid changes in network interface naming due to systemd's + predicatble naming scheme + + -- Proxmox Support Team Thu, 04 Nov 2021 15:29:55 +0100 + +qemu-server (7.0-17) bullseye; urgency=medium + + * fix #3258: block vm start when a PCI(e) device is already in use + + * snapshot: fix TPM state with RBD + + * swtpm: wait for PID file to appear before continuing with VM start + + * OS type: add entry for Windows 11/Server 2022 + + -- Proxmox Support Team Thu, 21 Oct 2021 11:57:09 +0200 + +qemu-server (7.0-16) bullseye; urgency=medium + + * ovmf: support secure boot enabled code images + + * ovmf: support provisioning an EFI vars template with secureboot by default + on and distribution + Microsofts secure-boot key pre-enrolled + + -- Proxmox Support Team Tue, 05 Oct 2021 20:22:18 +0200 + +qemu-server (7.0-15) bullseye; urgency=medium + + * api: return task-worker UPID in create template endpoint + + * api: destroy VM: remove pending volumes as well + + * fix #3075: add TPM v1.2 and v2.0 support via swtpm~ + + -- Proxmox Support Team Tue, 05 Oct 2021 07:24:52 +0200 + +qemu-server (7.0-14) bullseye; urgency=medium + + * fix #3581: pass size via argument for memory-backend-ram QMP call + + * fix #3608: improve removal of the underlying SCSI controller when removing + last drive on it + + * migrate: do not suggest that we map shared storages to avoid that + subsequent checks could result in false negatives. + + -- Proxmox Support Team Wed, 22 Sep 2021 09:31:06 +0200 + +qemu-server (7.0-13) bullseye; urgency=medium + + * fix bootorder regression with implicit default order + + -- Proxmox Support Team Thu, 5 Aug 2021 14:03:14 +0200 + +qemu-server (7.0-12) bullseye; urgency=medium + + * fix #3371: import ovf: allow the use of dots in the VM name + + * bootorder: fix double entry on cdrom edit + + -- Proxmox Support Team Fri, 30 Jul 2021 16:53:44 +0200 + +qemu-server (7.0-11) bullseye; urgency=medium + + * nic: support the intel e1000e model + + * lvm: avoid the use of io_uring for now + + * live-restore: fail early if target storage doesn't exist + + * api: always add new CD drives to bootorder + + * fix #2563: allow live migration with local cloud-init disk + + -- Proxmox Support Team Fri, 23 Jul 2021 11:08:48 +0200 + +qemu-server (7.0-10) bullseye; urgency=medium + + * avoid using io_uring for drives backed by LVM and configured for write-back + or write-through cache + + -- Proxmox Support Team Wed, 07 Jul 2021 14:56:34 +0200 + +qemu-server (7.0-9) bullseye; urgency=medium + + * cpu weight: always clamp value to lower maximum for cgroup v2 and fix + defaults (v1 -> 1024, v2 -> 100) + + * api: improve error handling when applying pending config changes + + -- Proxmox Support Team Wed, 07 Jul 2021 12:02:13 +0200 + +qemu-server (7.0-7) bullseye; urgency=medium + + * improve #3329: ensure write-back is used over write-around for EFI disk, + as OVMF profits a lot from cached writes due to its frequent + read-modify-write operations + + -- Proxmox Support Team Mon, 05 Jul 2021 20:49:50 +0200 + +qemu-server (7.0-6) bullseye; urgency=medium + + * live-restore: preload efidisk before starting VM + + * For now do not use io_uring for drives backed by Ceph RBD, with KRBD and + write-back or write-through cache enabled, as in that case some polling/IO + may hang in QEMU 6.0. + + -- Proxmox Support Team Fri, 02 Jul 2021 09:45:06 +0200 + +qemu-server (7.0-5) bullseye; urgency=medium + + * don't default to O_DIRECT (cache=none) on btrfs without nocow + + * fix #2175: api: update VM: check old drive-config for permissions too to + ensure a valid transition when limited to CDROM changes. + + -- Proxmox Support Team Thu, 24 Jun 2021 18:58:19 +0200 + +qemu-server (7.0-4) bullseye; urgency=medium + + * enable io-uring support by default when running QEMU 6.0 or newer + + * VM start: always check if storages of volumes support correct content-type + + * use KillMode 'process' for systemd scope to cope with depreacation of + KillMode=none + + * cli, api: handle new warnings task status + + * improve backup of templates with EFI disks and with SATA and IDE + disk controllers in use + + -- Proxmox Support Team Wed, 23 Jun 2021 12:57:27 +0200 + +qemu-server (7.0-3) bullseye; urgency=medium + + * vzdump: add master key support + + * vzdump: drop legacy fallback logging for dirty-bitmap + + * vm destroy: do not remove unreferenced disks by default + + * fix #3329: turn on cache=writeback for efidisks on rbd + + * avoid setting LUN number for drives when the `pvscsi` controller is used, + as that cannot handle multiple LUNs, increase the `scsi-id` instead + + * config: limit description/comment length to 8 KiB + + * migrate: enforce that image content type is available and configured on + target storage + + -- Proxmox Support Team Mon, 21 Jun 2021 11:17:52 +0200 + +qemu-server (7.0-2) bullseye; urgency=medium + + * api: clone: sort vm disks to keep numbers consistent + + * api: VM status: make template property optional in return object + + * add compatibility for QEMU 6.0 + + * destroy VM: always remove (referenced) VM state volumes + + * destroy VM: also check if unused volumes are base images + + * live-restore: log more similar to regular restore, outputting the user the + PBS repo/snapshot and target for each drive. + + -- Proxmox Support Team Fri, 28 May 2021 12:46:36 +0200 + +qemu-server (7.0-1) pve; urgency=medium + + * re-build for Proxmox VE 7 / Debian Bullseye + + -- Proxmox Support Team Thu, 13 May 2021 19:11:18 +0200 + +qemu-server (6.4-2) pve; urgency=medium + + * fix #2862: allow sata/ide template backups + + * migration: improve speed-limits for >1G connections again + + * fix getting bootdisk size for new bootorder config scheme + + -- Proxmox Support Team Thu, 29 Apr 2021 16:16:04 +0200 + +qemu-server (6.4-1) pve; urgency=medium + + * fix the +pveN versioned machine types when PXE is used + + * migration: avoid re-scanning all volumes + + * migration: do not always set default speed limit if none is configured + + * migration: rework logging to more humand friendly format, avoiding to much + output noise + + * qmrestore: add live-restore option for CLI tool + + * live-restore: hold 'create' lock during operation + + * live-restore: don't remove VM on error, to allow an VM user to save any new + data before retrying the operation. + + * fix #3369: auto-start vm after failed stopmode backup + + -- Proxmox Support Team Fri, 23 Apr 2021 16:26:54 +0200 + +qemu-server (6.3-11) pve; urgency=medium + + * enable live-restore tech preview for Proxmox Backup Server hosted backup + snapshots. + + * drive mirror: rework periodic status reporting to be human friendlier + + * drive mirror: stop logging progress for a disk once it got ready + + * image convert: use human-readable units in progress report + + -- Proxmox Support Team Thu, 15 Apr 2021 18:32:06 +0200 + +qemu-server (6.3-10) pve; urgency=medium + + * increase timeout for block (disk) resize QMP command + + * fix #3314: cloudinit: IPv6 requires type 'static6' + + * fix #2670: cloudinit: enable SLAAC again now that client support is there + + -- Proxmox Support Team Tue, 30 Mar 2021 18:40:58 +0200 + +qemu-server (6.3-9) pve; urgency=medium + + * restore vma: fix applying storage-specific bandwidth limit + + * snapshot: set migration caps before savevm-start + + * vzdump: improve error logging for query-proxmox-support to avoid + false-positives + + -- Proxmox Support Team Fri, 26 Mar 2021 09:47:27 +0100 + +qemu-server (6.3-8) pve; urgency=medium + + * qm status: sort hash keys on verbose output + + * improve windows VM version pinning on VM creation + + -- Proxmox Support Team Fri, 12 Mar 2021 10:01:09 +0100 + +qemu-server (6.3-7) pve; urgency=medium + + * vzdump: increase Proxmox Backup Server backup QMP command timeout + + -- Proxmox Support Team Tue, 09 Mar 2021 08:21:43 +0100 + +qemu-server (6.3-6) pve; urgency=medium + + * fix #3324: clone disk: use larger blocksize for EFI disk + + * fix #3301: status: add currently running machine and QEMU version to full + status + + * api: add endpoint to list all available QEMU machine type and version + tuples + + * always pin virtual machines with Windows as ostype to a fixed QEMU machine + version by default. For existing VMs with Windows based OS-type use the 5.1 + machine version (or the next available one, for older QEMU versions) to + improve stabillity of the hardware layout from Windows point of view. Linux + and other OS types are not as sensitive to those changes, so keep the + default to the currently latest available machine versions for non-Windows + VMs. + + * update VM: check for CDROM not just drive permissions when removing a + device + + -- Proxmox Support Team Fri, 05 Mar 2021 21:42:59 +0100 + +qemu-server (6.3-5) pve; urgency=medium + + * cloudinit: add opennebula config format + + * cloudinit: remove pending delete on online regenerate image + + * snapshot/save-vm: periodically print progress and show information about + drives during snapshot + + * qmeventd: explicitly close() pidfds + + -- Proxmox Support Team Thu, 11 Feb 2021 18:05:18 +0100 + +qemu-server (6.3-4) pve; urgency=medium + + * audio: add the "none" dummy audio backend + + * fix drive-mirror completion with cloudinit + + * vm destroy: allow opt-out of purging unreferenced disks + + * fix #2788: do not resume vms after backup if they were paused before + + * anchor CPU flag regex to avoid arbitrary flag suffixes + + -- Proxmox Support Team Thu, 28 Jan 2021 17:21:07 +0100 + +qemu-server (6.3-3) pve; urgency=medium + + * api: adapt VM destroy and purge description + + * clone disk: fix regression in offline clone of efidisk + + * cloudinit: fix cloning/restoring of cloudinit disks in raw format + + -- Proxmox Support Team Tue, 15 Dec 2020 16:33:01 +0100 + +qemu-server (6.3-2) pve; urgency=medium + + * PBS: use improved method to assemble repository url, fixing issues when + using IPv6 or non-default ports + + -- Proxmox Support Team Thu, 03 Dec 2020 18:06:25 +0100 + +qemu-server (6.3-1) pve; urgency=medium + + * deactivate volumes after storage migrate + + * print query-proxmox-support result in 'full' status + + * clone disk: avoid errors after disk was moved by QEMU + + * replace cgroups_write by cgroup change_cpu_shares && change_cpu_quota + + -- Proxmox Support Team Wed, 25 Nov 2020 14:30:50 +0100 + +qemu-server (6.2-20) pve; urgency=medium + + * don't migrate replicated VM whose replication job is marked for + removal + + * ensure qmeventd service is stopped after pve-guests and pve-ha-lrm service + on shutdown + + -- Proxmox Support Team Thu, 12 Nov 2020 17:08:45 +0100 + +qemu-server (6.2-19) pve; urgency=medium + + * fix #3113: unbreak drive hotplug + + * qmeventd: add handling for -no-shutdown QEMU instances, to avoid errors if + the guest OS shuts down the VM during a backup job. + + -- Proxmox Support Team Thu, 05 Nov 2020 13:37:00 +0100 + +qemu-server (6.2-18) pve; urgency=medium + + * migrate: tell QEMU to enable dirty-bitmap migration, if supported + + * partially fix #3056: always try to cancel backups when failed to start job + + -- Proxmox Support Team Thu, 29 Oct 2020 18:23:13 +0100 + qemu-server (6.2-17) pve; urgency=medium * bootorder: don't print empty 'order=' property diff --git a/debian/compat b/debian/compat deleted file mode 100644 index b4de394..0000000 --- a/debian/compat +++ /dev/null @@ -1 +0,0 @@ -11 diff --git a/debian/control b/debian/control index fad7c8f..1301a36 100644 --- a/debian/control +++ b/debian/control @@ -2,12 +2,14 @@ Source: qemu-server Section: admin Priority: optional Maintainer: Proxmox Support Team -Build-Depends: debhelper (>= 11~), +Build-Depends: debhelper-compat (= 13), + libglib2.0-dev, libio-multiplex-perl, libjson-c-dev, + libpve-apiclient-perl, libpve-cluster-perl, - libpve-common-perl (>= 6.1-5), - libpve-guest-common-perl (>= 3.0-9), + libpve-common-perl (>= 8.0.2), + libpve-guest-common-perl (>= 5.1.0), libpve-storage-perl (>= 6.1-7), libtest-mockmodule-perl, libuuid-perl, @@ -19,8 +21,9 @@ Build-Depends: debhelper (>= 11~), pve-doc-generator (>= 6.2-5), pve-edk2-firmware, pve-firewall, - pve-qemu-kvm, -Standards-Version: 4.3.0 + pve-ha-manager , + pve-qemu-kvm (>= 7.1~), +Standards-Version: 4.5.1 Homepage: https://www.proxmox.com Package: qemu-server @@ -31,25 +34,33 @@ Depends: dbus, libjson-perl, libjson-xs-perl, libnet-ssleay-perl, - libpve-access-control (>= 5.0-7), + libpve-access-control (>= 8.0.0~), + libpve-apiclient-perl, libpve-cluster-perl, - libpve-common-perl (>= 6.1-5), - libpve-guest-common-perl (>= 3.0-9), - libpve-storage-perl (>= 6.2-2), + libpve-common-perl (>= 8.0.2), + libpve-guest-common-perl (>= 5.1.0), + libpve-storage-perl (>= 7.2-10), libterm-readline-gnu-perl, libuuid-perl, libxml-libxml-perl, perl (>= 5.10.0-19), + proxmox-websocket-tunnel, pve-cluster, - pve-edk2-firmware (>= 1.20181023-1), - pve-firewall, +# TODO: make legacy edk2 optional (suggests) for PVE 9 and warn explicitly about it + pve-edk2-firmware-legacy | pve-edk2-firmware (<< 4~), + pve-edk2-firmware-ovmf | pve-edk2-firmware (>= 3.20210831-1), + pve-firewall (>= 5.0.4), pve-ha-manager (>= 3.0-9), - pve-qemu-kvm (>= 3.0.1-62), + pve-qemu-kvm (>= 7.1~), socat, + swtpm, + swtpm-tools, ${misc:Depends}, ${perl:Depends}, ${shlibs:Depends}, -Breaks: pve-ha-manager (<= 3.0-4), - pve-manager (<= 6.0-13), +Recommends: proxmox-backup-file-restore (>= 2.1.9-2), + libpve-network-perl (>= 0.8.3), +Suggests: pve-edk2-firmware-aarch64, pve-edk2-firmware-riscv, +Breaks: pve-ha-manager (<< 4.0.1), pve-manager (<= 6.0-13), Description: Qemu Server Tools This package contains the Qemu Server tools used by Proxmox VE diff --git a/debian/copyright b/debian/copyright index 624045b..4c7a015 100644 --- a/debian/copyright +++ b/debian/copyright @@ -1,4 +1,4 @@ -Copyright (C) 2011 - 2020 Proxmox Server Solutions GmbH +Copyright (C) 2011 - 2024 Proxmox Server Solutions GmbH This software is written by Proxmox Server Solutions GmbH diff --git a/debian/source/format b/debian/source/format new file mode 100644 index 0000000..89ae9db --- /dev/null +++ b/debian/source/format @@ -0,0 +1 @@ +3.0 (native) diff --git a/qmeventd/Makefile b/qmeventd/Makefile index a68818d..9c00a36 100644 --- a/qmeventd/Makefile +++ b/qmeventd/Makefile @@ -9,11 +9,11 @@ include /usr/share/pve-doc-generator/pve-doc-generator.mk CC ?= gcc CFLAGS += -O2 -Werror -Wall -Wextra -Wpedantic -Wtype-limits -Wl,-z,relro -std=gnu11 -CFLAGS += $(shell pkg-config --cflags json-c) -LDFLAGS += $(shell pkg-config --libs json-c) +CFLAGS += $(shell pkg-config --cflags json-c glib-2.0) +LDFLAGS += $(shell pkg-config --libs json-c glib-2.0) qmeventd: qmeventd.c - $(CC) $(CFLAGS) $(LDFLAGS) -o $@ $< + $(CC) $(CFLAGS) -o $@ $< $(LDFLAGS) docs: qmeventd.8 diff --git a/qmeventd/qmeventd.c b/qmeventd/qmeventd.c index 77c6297..d8f3ee7 100644 --- a/qmeventd/qmeventd.c +++ b/qmeventd/qmeventd.c @@ -1,41 +1,21 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later /* - - Copyright (C) 2018 Proxmox Server Solutions GmbH - - Copyright: qmeventd is under GNU GPL, the GNU General Public License. - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; version 2 dated June, 1991. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA - 02111-1307, USA. + Copyright (C) 2018 - 2021 Proxmox Server Solutions GmbH Author: Dominik Csapak - - qmeventd listens on a given socket, and waits for qemu processes - to connect - - it then waits for shutdown events followed by the closing of the socket, - it then calls /usr/sbin/qm cleanup with following arguments - - /usr/sbin/qm cleanup VMID - - parameter explanation: - - graceful: - 1|0 depending if it saw a shutdown event before the socket closed - - guest: - 1|0 depending if the shutdown was requested from the guest - + Author: Stefan Reiter + + Description: + + qmeventd listens on a given socket, and waits for qemu processes to + connect. After accepting a connection qmeventd waits for shutdown events + followed by the closing of the socket. Once that happens `qm cleanup` will + be executed with following three arguments: + VMID + Where `graceful` can be `1` or `0` depending if shutdown event was observed + before the socket got closed. The second parameter `guest` is also boolean + `1` or `0` depending if the shutdown was requested from the guest OS + (i.e., the "inside"). */ #ifndef _GNU_SOURCE @@ -44,10 +24,12 @@ #include #include +#include #include #include #include #include +#include #include #include #include @@ -55,12 +37,20 @@ #include #include #include +#include #include "qmeventd.h" +#define DEFAULT_KILL_TIMEOUT 60 + static int verbose = 0; +static int kill_timeout = DEFAULT_KILL_TIMEOUT; static int epoll_fd = 0; static const char *progname; +GHashTable *vm_clients; // key=vmid (freed on remove), value=*Client (free manually) +GSList *forced_cleanups; +static int needs_cleanup = 0; + /* * Helper functions */ @@ -71,6 +61,7 @@ usage() fprintf(stderr, "Usage: %s [-f] [-v] PATH\n", progname); fprintf(stderr, " -f run in foreground (default: false)\n"); fprintf(stderr, " -v verbose (default: false)\n"); + fprintf(stderr, " -t kill timeout (default: %ds)\n", DEFAULT_KILL_TIMEOUT); fprintf(stderr, " PATH use PATH for socket\n"); } @@ -84,14 +75,13 @@ get_pid_from_fd(int fd) } /* - * reads the vmid from /proc//cmdline - * after the '-id' argument + * parses the vmid from the qemu.slice entry of /proc//cgroup */ static unsigned long get_vmid_from_pid(pid_t pid) { char filename[32] = { 0 }; - int len = snprintf(filename, sizeof(filename), "/proc/%d/cmdline", pid); + int len = snprintf(filename, sizeof(filename), "/proc/%d/cgroup", pid); if (len < 0) { fprintf(stderr, "error during snprintf for %d: %s\n", pid, strerror(errno)); @@ -108,43 +98,54 @@ get_vmid_from_pid(pid_t pid) } unsigned long vmid = 0; - ssize_t rc = 0; char *buf = NULL; size_t buflen = 0; - while ((rc = getdelim(&buf, &buflen, '\0', fp)) >= 0) { - if (!strcmp(buf, "-id")) { - break; + + while (getline(&buf, &buflen, fp) >= 0) { + char *cgroup_path = strrchr(buf, ':'); + if (!cgroup_path) { + fprintf(stderr, "unexpected cgroup entry %s\n", buf); + continue; } - } + cgroup_path++; - if (rc < 0) { - goto err; - } + if (strncmp(cgroup_path, "/qemu.slice/", 12)) { + continue; + } + + char *vmid_start = strrchr(buf, '/'); + if (!vmid_start) { + fprintf(stderr, "unexpected cgroup entry %s\n", buf); + continue; + } + vmid_start++; - if (getdelim(&buf, &buflen, '\0', fp) >= 0) { - if (buf[0] == '-' || buf[0] == '\0') { - fprintf(stderr, "invalid vmid %s\n", buf); - goto ret; + if (vmid_start[0] == '-' || vmid_start[0] == '\0') { + fprintf(stderr, "invalid vmid in cgroup entry %s\n", buf); + continue; } errno = 0; char *endptr = NULL; - vmid = strtoul(buf, &endptr, 10); - if (errno != 0) { + vmid = strtoul(vmid_start, &endptr, 10); + if (!endptr || strncmp(endptr, ".scope", 6)) { + fprintf(stderr, "unexpected cgroup entry %s\n", buf); vmid = 0; - goto err; - } else if (*endptr != '\0') { - fprintf(stderr, "invalid vmid %s\n", buf); + continue; + } + if (errno != 0) { vmid = 0; } - goto ret; + break; } -err: - fprintf(stderr, "error parsing vmid for %d: %s\n", pid, strerror(errno)); + if (errno) { + fprintf(stderr, "error parsing vmid for %d: %s\n", pid, strerror(errno)); + } else if (!vmid) { + fprintf(stderr, "error parsing vmid for %d: no matching qemu.slice cgroup entry\n", pid); + } -ret: free(buf); fclose(fp); return vmid; @@ -165,15 +166,39 @@ must_write(int fd, const char *buf, size_t len) * qmp handling functions */ +static void +send_qmp_cmd(struct Client *client, const char *buf, size_t len) +{ + if (!must_write(client->fd, buf, len - 1)) { + fprintf(stderr, "%s: cannot send QMP message\n", client->qemu.vmid); + cleanup_client(client); + } +} + void handle_qmp_handshake(struct Client *client) { - VERBOSE_PRINT("%s: got QMP handshake\n", client->vmid); - static const char qmp_answer[] = "{\"execute\":\"qmp_capabilities\"}\n"; - if (!must_write(client->fd, qmp_answer, sizeof(qmp_answer) - 1)) { - fprintf(stderr, "%s: cannot complete handshake\n", client->vmid); + VERBOSE_PRINT("pid%d: got QMP handshake, assuming QEMU client\n", client->pid); + + // extract vmid from cmdline, now that we know it's a QEMU process + unsigned long vmid = get_vmid_from_pid(client->pid); + int res = snprintf(client->qemu.vmid, sizeof(client->qemu.vmid), "%lu", vmid); + if (vmid == 0 || res < 0 || res >= (int)sizeof(client->qemu.vmid)) { + fprintf(stderr, "could not get vmid from pid %d\n", client->pid); cleanup_client(client); + return; + } + + VERBOSE_PRINT("pid%d: assigned VMID: %s\n", client->pid, client->qemu.vmid); + client->type = CLIENT_QEMU; + if(!g_hash_table_insert(vm_clients, strdup(client->qemu.vmid), client)) { + // not fatal, just means backup handling won't work + fprintf(stderr, "%s: could not insert client into VMID->client table\n", + client->qemu.vmid); } + + static const char qmp_answer[] = "{\"execute\":\"qmp_capabilities\"}\n"; + send_qmp_cmd(client, qmp_answer, sizeof(qmp_answer)); } void @@ -183,18 +208,154 @@ handle_qmp_event(struct Client *client, struct json_object *obj) if (!json_object_object_get_ex(obj, "event", &event)) { return; } - VERBOSE_PRINT("%s: got QMP event: %s\n", client->vmid, - json_object_get_string(event)); + VERBOSE_PRINT("%s: got QMP event: %s\n", client->qemu.vmid, json_object_get_string(event)); + + if (client->state == STATE_TERMINATING) { + // QEMU sometimes sends a second SHUTDOWN after SIGTERM, ignore + VERBOSE_PRINT("%s: event was after termination, ignoring\n", client->qemu.vmid); + return; + } + // event, check if shutdown and get guest parameter if (!strcmp(json_object_get_string(event), "SHUTDOWN")) { - client->graceful = 1; + client->qemu.graceful = 1; struct json_object *data; struct json_object *guest; if (json_object_object_get_ex(obj, "data", &data) && json_object_object_get_ex(data, "guest", &guest)) { - client->guest = (unsigned short)json_object_get_boolean(guest); + client->qemu.guest = (unsigned short)json_object_get_boolean(guest); } + + // check if a backup is running and kill QEMU process if not + terminate_check(client); + } +} + +void +terminate_check(struct Client *client) +{ + if (client->state != STATE_IDLE) { + // if we're already in a request, queue this one until after + VERBOSE_PRINT("%s: terminate_check queued\n", client->qemu.vmid); + client->qemu.term_check_queued = true; + return; + } + + client->qemu.term_check_queued = false; + + VERBOSE_PRINT("%s: query-status\n", client->qemu.vmid); + client->state = STATE_EXPECT_STATUS_RESP; + static const char qmp_req[] = "{\"execute\":\"query-status\"}\n"; + send_qmp_cmd(client, qmp_req, sizeof(qmp_req)); +} + +void +handle_qmp_return(struct Client *client, struct json_object *data, bool error) +{ + if (error) { + const char *msg = "n/a"; + struct json_object *desc; + if (json_object_object_get_ex(data, "desc", &desc)) { + msg = json_object_get_string(desc); + } + fprintf(stderr, "%s: received error from QMP: %s\n", + client->qemu.vmid, msg); + client->state = STATE_IDLE; + goto out; + } + + struct json_object *status; + json_bool has_status = data && + json_object_object_get_ex(data, "status", &status); + + bool active = false; + if (has_status) { + const char *status_str = json_object_get_string(status); + active = status_str && ( + !strcmp(status_str, "running") + || !strcmp(status_str, "paused") + || !strcmp(status_str, "suspended") + || !strcmp(status_str, "prelaunch") + ); + } + + switch (client->state) { + case STATE_EXPECT_STATUS_RESP: + client->state = STATE_IDLE; + if (active) { + VERBOSE_PRINT("%s: got status: VM is active\n", client->qemu.vmid); + } else if (!client->qemu.backup) { + terminate_client(client); + } else { + // if we're in a backup, don't do anything, vzdump will notify + // us when the backup finishes + VERBOSE_PRINT("%s: not active, but running backup - keep alive\n", + client->qemu.vmid); + } + break; + + // this means we received the empty return from our handshake answer + case STATE_HANDSHAKE: + client->state = STATE_IDLE; + VERBOSE_PRINT("%s: QMP handshake complete\n", client->qemu.vmid); + break; + + // we expect an empty return object after sending quit + case STATE_TERMINATING: + break; + case STATE_IDLE: + VERBOSE_PRINT("%s: spurious return value received\n", + client->qemu.vmid); + break; + } + +out: + if (client->qemu.term_check_queued) { + terminate_check(client); + } +} + +/* + * VZDump specific client functions + */ + +void +handle_vzdump_handshake(struct Client *client, struct json_object *data) +{ + client->state = STATE_IDLE; + + struct json_object *vmid_obj; + json_bool has_vmid = data && json_object_object_get_ex(data, "vmid", &vmid_obj); + + if (!has_vmid) { + VERBOSE_PRINT("pid%d: invalid vzdump handshake: no vmid\n", client->pid); + return; + } + + const char *vmid_str = json_object_get_string(vmid_obj); + + if (!vmid_str) { + VERBOSE_PRINT("pid%d: invalid vzdump handshake: vmid is not a string\n", client->pid); + return; + } + + int res = snprintf(client->vzdump.vmid, sizeof(client->vzdump.vmid), "%s", vmid_str); + if (res < 0 || res >= (int)sizeof(client->vzdump.vmid)) { + VERBOSE_PRINT("pid%d: invalid vzdump handshake: vmid too long or invalid\n", client->pid); + return; + } + + struct Client *vmc = (struct Client*) g_hash_table_lookup(vm_clients, client->vzdump.vmid); + if (vmc) { + vmc->qemu.backup = true; + + // only mark as VZDUMP once we have set everything up, otherwise 'cleanup' + // might try to access an invalid value + client->type = CLIENT_VZDUMP; + VERBOSE_PRINT("%s: vzdump backup started\n", client->vzdump.vmid); + } else { + VERBOSE_PRINT("%s: vzdump requested backup start for unregistered VM\n", client->vzdump.vmid); } } @@ -206,30 +367,30 @@ void add_new_client(int client_fd) { struct Client *client = calloc(sizeof(struct Client), 1); + if (client == NULL) { + fprintf(stderr, "could not add new client - allocation failed!\n"); + fflush(stderr); + return; + } + client->state = STATE_HANDSHAKE; + client->type = CLIENT_NONE; client->fd = client_fd; client->pid = get_pid_from_fd(client_fd); if (client->pid == 0) { fprintf(stderr, "could not get pid from client\n"); goto err; } - unsigned long vmid = get_vmid_from_pid(client->pid); - int res = snprintf(client->vmid, sizeof(client->vmid), "%lu", vmid); - if (vmid == 0 || res < 0 || res >= (int)sizeof(client->vmid)) { - fprintf(stderr, "could not get vmid from pid %d\n", client->pid); - goto err; - } struct epoll_event ev; ev.events = EPOLLIN; ev.data.ptr = client; - res = epoll_ctl(epoll_fd, EPOLL_CTL_ADD, client_fd, &ev); + int res = epoll_ctl(epoll_fd, EPOLL_CTL_ADD, client_fd, &ev); if (res < 0) { perror("epoll_ctl client add"); goto err; } - VERBOSE_PRINT("added new client, pid: %d, vmid: %s\n", client->pid, - client->vmid); + VERBOSE_PRINT("added new client, pid: %d\n", client->pid); return; err: @@ -237,20 +398,16 @@ err: free(client); } -void -cleanup_client(struct Client *client) +static void +cleanup_qemu_client(struct Client *client) { - VERBOSE_PRINT("%s: client exited, status: graceful: %d, guest: %d\n", - client->vmid, client->graceful, client->guest); - log_neg(epoll_ctl(epoll_fd, EPOLL_CTL_DEL, client->fd, NULL), "epoll del"); - (void)close(client->fd); - - unsigned short graceful = client->graceful; - unsigned short guest = client->guest; - char vmid[sizeof(client->vmid)]; - strncpy(vmid, client->vmid, sizeof(vmid)); - free(client); - VERBOSE_PRINT("%s: executing cleanup\n", vmid); + unsigned short graceful = client->qemu.graceful; + unsigned short guest = client->qemu.guest; + char vmid[sizeof(client->qemu.vmid)]; + strncpy(vmid, client->qemu.vmid, sizeof(vmid)); + g_hash_table_remove(vm_clients, &vmid); // frees key, ignore errors + VERBOSE_PRINT("%s: executing cleanup (graceful: %d, guest: %d)\n", + vmid, graceful, guest); int pid = fork(); if (pid < 0) { @@ -275,10 +432,89 @@ cleanup_client(struct Client *client) } } +void +cleanup_client(struct Client *client) +{ + log_neg(epoll_ctl(epoll_fd, EPOLL_CTL_DEL, client->fd, NULL), "epoll del"); + (void)close(client->fd); + + struct Client *vmc; + switch (client->type) { + case CLIENT_QEMU: + cleanup_qemu_client(client); + break; + + case CLIENT_VZDUMP: + vmc = (struct Client*) g_hash_table_lookup(vm_clients, client->vzdump.vmid); + if (vmc) { + VERBOSE_PRINT("%s: backup ended\n", client->vzdump.vmid); + vmc->qemu.backup = false; + terminate_check(vmc); + } + break; + + case CLIENT_NONE: + // do nothing, only close socket + break; + } + + if (client->pidfd > 0) { + (void)close(client->pidfd); + } + VERBOSE_PRINT("removing %s from forced cleanups\n", client->qemu.vmid); + forced_cleanups = g_slist_remove(forced_cleanups, client); + free(client); +} + +void +terminate_client(struct Client *client) +{ + VERBOSE_PRINT("%s: terminating client (pid %d)\n", client->qemu.vmid, client->pid); + + client->state = STATE_TERMINATING; + + // open a pidfd before kill for later cleanup + int pidfd = pidfd_open(client->pid, 0); + if (pidfd < 0) { + switch (errno) { + case ESRCH: + // process already dead for some reason, cleanup done + VERBOSE_PRINT("%s: failed to open pidfd, process already dead (pid %d)\n", + client->qemu.vmid, client->pid); + return; + + // otherwise fall back to just using the PID directly, but don't + // print if we only failed because we're running on an older kernel + case ENOSYS: + break; + default: + perror("failed to open QEMU pidfd for cleanup"); + break; + } + } + + // try to send a 'quit' command first, fallback to SIGTERM of the pid + static const char qmp_quit_command[] = "{\"execute\":\"quit\"}\n"; + VERBOSE_PRINT("%s: sending 'quit' via QMP\n", client->qemu.vmid); + if (!must_write(client->fd, qmp_quit_command, sizeof(qmp_quit_command) - 1)) { + VERBOSE_PRINT("%s: sending 'SIGTERM' to pid %d\n", client->qemu.vmid, client->pid); + int err = kill(client->pid, SIGTERM); + log_neg(err, "kill"); + } + + time_t timeout = time(NULL) + kill_timeout; + + client->pidfd = pidfd; + client->timeout = timeout; + + forced_cleanups = g_slist_prepend(forced_cleanups, (void *)client); + needs_cleanup = 1; +} + void handle_client(struct Client *client) { - VERBOSE_PRINT("%s: entering handle\n", client->vmid); + VERBOSE_PRINT("pid%d: entering handle\n", client->pid); ssize_t len; do { len = read(client->fd, (client->buf+client->buflen), @@ -292,12 +528,12 @@ handle_client(struct Client *client) } return; } else if (len == 0) { - VERBOSE_PRINT("%s: got EOF\n", client->vmid); + VERBOSE_PRINT("pid%d: got EOF\n", client->pid); cleanup_client(client); return; } - VERBOSE_PRINT("%s: read %ld bytes\n", client->vmid, len); + VERBOSE_PRINT("pid%d: read %ld bytes\n", client->pid, len); client->buflen += len; struct json_tokener *tok = json_tokener_new(); @@ -318,20 +554,24 @@ handle_client(struct Client *client) handle_qmp_handshake(client); } else if (json_object_object_get_ex(jobj, "event", &obj)) { handle_qmp_event(client, jobj); + } else if (json_object_object_get_ex(jobj, "return", &obj)) { + handle_qmp_return(client, obj, false); + } else if (json_object_object_get_ex(jobj, "error", &obj)) { + handle_qmp_return(client, obj, true); + } else if (json_object_object_get_ex(jobj, "vzdump", &obj)) { + handle_vzdump_handshake(client, obj); } // else ignore message } break; case json_tokener_continue: if (client->buflen >= sizeof(client->buf)) { - VERBOSE_PRINT("%s, msg too large, discarding buffer\n", - client->vmid); + VERBOSE_PRINT("pid%d: msg too large, discarding buffer\n", client->pid); memset(client->buf, 0, sizeof(client->buf)); client->buflen = 0; } // else we have enough space try again after next read break; default: - VERBOSE_PRINT("%s: parse error: %d, discarding buffer\n", - client->vmid, jerr); + VERBOSE_PRINT("pid%d: parse error: %d, discarding buffer\n", client->pid, jerr); memset(client->buf, 0, client->buflen); client->buflen = 0; break; @@ -341,6 +581,50 @@ handle_client(struct Client *client) json_tokener_free(tok); } +static void +sigkill(void *ptr, void *time_ptr) +{ + struct Client *data = ptr; + int err; + + if (data->timeout != 0 && data->timeout > *(time_t *)time_ptr) { + return; + } + + if (data->pidfd > 0) { + err = pidfd_send_signal(data->pidfd, SIGKILL, NULL, 0); + (void)close(data->pidfd); + data->pidfd = -1; + } else { + err = kill(data->pid, SIGKILL); + } + + if (err < 0) { + if (errno != ESRCH) { + fprintf(stderr, "SIGKILL cleanup of pid '%d' failed - %s\n", + data->pid, strerror(errno)); + } + } else { + fprintf(stderr, "cleanup failed, terminating pid '%d' with SIGKILL\n", + data->pid); + } + + data->timeout = 0; + + // remove ourselves from the list + forced_cleanups = g_slist_remove(forced_cleanups, ptr); +} + +static void +handle_forced_cleanup() +{ + if (g_slist_length(forced_cleanups) > 0) { + VERBOSE_PRINT("clearing forced cleanup backlog\n"); + time_t cur_time = time(NULL); + g_slist_foreach(forced_cleanups, sigkill, &cur_time); + } + needs_cleanup = g_slist_length(forced_cleanups) > 0; +} int main(int argc, char *argv[]) @@ -350,7 +634,7 @@ main(int argc, char *argv[]) char *socket_path = NULL; progname = argv[0]; - while ((opt = getopt(argc, argv, "hfv")) != -1) { + while ((opt = getopt(argc, argv, "hfvt:")) != -1) { switch (opt) { case 'f': daemonize = 0; @@ -358,6 +642,15 @@ main(int argc, char *argv[]) case 'v': verbose = 1; break; + case 't': + errno = 0; + char *endptr = NULL; + kill_timeout = strtoul(optarg, &endptr, 10); + if (errno != 0 || *endptr != '\0' || kill_timeout == 0) { + usage(); + exit(EXIT_FAILURE); + } + break; case 'h': usage(); exit(EXIT_SUCCESS); @@ -402,12 +695,13 @@ main(int argc, char *argv[]) bail_neg(daemon(0, 1), "daemon"); } + vm_clients = g_hash_table_new_full(g_str_hash, g_str_equal, free, NULL); + int nevents; for(;;) { - nevents = epoll_wait(epoll_fd, events, 1, -1); + nevents = epoll_wait(epoll_fd, events, 1, needs_cleanup ? 10*1000 : -1); if (nevents < 0 && errno == EINTR) { - // signal happened, try again continue; } bail_neg(nevents, "epoll_wait"); @@ -415,8 +709,7 @@ main(int argc, char *argv[]) for (int n = 0; n < nevents; n++) { if (events[n].data.fd == sock) { - int conn_sock = accept4(sock, NULL, NULL, - SOCK_NONBLOCK | SOCK_CLOEXEC); + int conn_sock = accept4(sock, NULL, NULL, SOCK_NONBLOCK | SOCK_CLOEXEC); log_neg(conn_sock, "accept"); if (conn_sock > -1) { add_new_client(conn_sock); @@ -425,5 +718,6 @@ main(int argc, char *argv[]) handle_client((struct Client *)events[n].data.ptr); } } + handle_forced_cleanup(); } } diff --git a/qmeventd/qmeventd.h b/qmeventd/qmeventd.h index 0c2ffc1..9afc935 100644 --- a/qmeventd/qmeventd.h +++ b/qmeventd/qmeventd.h @@ -1,27 +1,22 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later /* - - Copyright (C) 2018 Proxmox Server Solutions GmbH - - Copyright: qemumonitor is under GNU GPL, the GNU General Public License. - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; version 2 dated June, 1991. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA - 02111-1307, USA. + Copyright (C) 2018 - 2021 Proxmox Server Solutions GmbH Author: Dominik Csapak + Author: Stefan Reiter */ -#define VERBOSE_PRINT(...) do { if (verbose) { printf(__VA_ARGS__); } } while (0) +#include +#include + +#ifndef __NR_pidfd_open +#define __NR_pidfd_open 434 +#endif +#ifndef __NR_pidfd_send_signal +#define __NR_pidfd_send_signal 424 +#endif + +#define VERBOSE_PRINT(...) do { if (verbose) { printf(__VA_ARGS__); fflush(stdout); } } while (0) static inline void log_neg(int errval, const char *msg) { @@ -38,18 +33,65 @@ static inline void bail_neg(int errval, const char *msg) } } +static inline int +pidfd_open(pid_t pid, unsigned int flags) +{ + return syscall(__NR_pidfd_open, pid, flags); +} + +static inline int +pidfd_send_signal(int pidfd, int sig, siginfo_t *info, unsigned int flags) +{ + return syscall(__NR_pidfd_send_signal, pidfd, sig, info, flags); +} + +typedef enum { + CLIENT_NONE, + CLIENT_QEMU, + CLIENT_VZDUMP +} ClientType; + +typedef enum { + STATE_HANDSHAKE, + STATE_IDLE, + STATE_EXPECT_STATUS_RESP, + STATE_TERMINATING +} ClientState; + struct Client { char buf[4096]; - char vmid[16]; + unsigned int buflen; + int fd; pid_t pid; - unsigned int buflen; - unsigned short graceful; - unsigned short guest; + int pidfd; + time_t timeout; + + ClientType type; + ClientState state; + + // only relevant for type=CLIENT_QEMU + struct { + char vmid[16]; + unsigned short graceful; + unsigned short guest; + bool term_check_queued; + bool backup; + } qemu; + + // only relevant for type=CLIENT_VZDUMP + struct { + // vmid of referenced backup + char vmid[16]; + } vzdump; }; void handle_qmp_handshake(struct Client *client); void handle_qmp_event(struct Client *client, struct json_object *obj); +void handle_qmp_return(struct Client *client, struct json_object *data, bool error); +void handle_vzdump_handshake(struct Client *client, struct json_object *data); void handle_client(struct Client *client); void add_new_client(int client_fd); void cleanup_client(struct Client *client); +void terminate_client(struct Client *client); +void terminate_check(struct Client *client); diff --git a/qmeventd/qmeventd.service b/qmeventd/qmeventd.service index ad3ca00..1e2465b 100644 --- a/qmeventd/qmeventd.service +++ b/qmeventd/qmeventd.service @@ -1,6 +1,8 @@ [Unit] Description=PVE Qemu Event Daemon RequiresMountsFor=/var/run +Before=pve-ha-lrm.service +Before=pve-guests.service [Service] ExecStart=/usr/sbin/qmeventd /var/run/qmeventd.sock diff --git a/test/Makefile b/test/Makefile index d88cbd2..9e6d39e 100644 --- a/test/Makefile +++ b/test/Makefile @@ -1,6 +1,6 @@ all: test -test: test_snapshot test_ovf test_cfg_to_cmd test_pci_addr_conflicts test_qemu_img_convert +test: test_snapshot test_ovf test_cfg_to_cmd test_pci_addr_conflicts test_qemu_img_convert test_migration test_restore_config test_snapshot: run_snapshot_tests.pl ./run_snapshot_tests.pl @@ -17,3 +17,17 @@ test_qemu_img_convert: run_qemu_img_convert_tests.pl test_pci_addr_conflicts: run_pci_addr_checks.pl ./run_pci_addr_checks.pl + +MIGRATION_TEST_TARGETS := $(addprefix test_migration_,$(shell perl -ne 'print "$$1 " if /^\s*name\s*=>\s*["'\'']([^\s"'\'']+)["'\'']\s*,\s*$$/; END { print "\n" }' run_qemu_migrate_tests.pl)) + +test_migration: run_qemu_migrate_tests.pl MigrationTest/*.pm $(MIGRATION_TEST_TARGETS) + +$(MIGRATION_TEST_TARGETS): + ./run_qemu_migrate_tests.pl $(@:test_migration_%=%) + +test_restore_config: run_qemu_restore_config_tests.pl + ./run_qemu_restore_config_tests.pl + +.PHONY: clean +clean: + rm -rf MigrationTest/run diff --git a/test/MigrationTest/QemuMigrateMock.pm b/test/MigrationTest/QemuMigrateMock.pm new file mode 100644 index 0000000..1efabe2 --- /dev/null +++ b/test/MigrationTest/QemuMigrateMock.pm @@ -0,0 +1,347 @@ +package MigrationTest::QemuMigrateMock; + +use strict; +use warnings; + +use JSON; +use Test::MockModule; + +use MigrationTest::Shared; + +use PVE::API2::Qemu; +use PVE::Storage; +use PVE::Tools qw(file_set_contents file_get_contents); + +use PVE::CLIHandler; +use base qw(PVE::CLIHandler); + +my $RUN_DIR_PATH = $ENV{RUN_DIR_PATH} or die "no RUN_DIR_PATH set\n"; +my $QM_LIB_PATH = $ENV{QM_LIB_PATH} or die "no QM_LIB_PATH set\n"; + +my $source_volids = decode_json(file_get_contents("${RUN_DIR_PATH}/source_volids")); +my $source_vdisks = decode_json(file_get_contents("${RUN_DIR_PATH}/source_vdisks")); +my $vm_status = decode_json(file_get_contents("${RUN_DIR_PATH}/vm_status")); +my $expected_calls = decode_json(file_get_contents("${RUN_DIR_PATH}/expected_calls")); +my $fail_config = decode_json(file_get_contents("${RUN_DIR_PATH}/fail_config")); +my $storage_migrate_map = decode_json(file_get_contents("${RUN_DIR_PATH}/storage_migrate_map")); +my $migrate_params = decode_json(file_get_contents("${RUN_DIR_PATH}/migrate_params")); + +my $test_vmid = $migrate_params->{vmid}; +my $test_target = $migrate_params->{target}; +my $test_opts = $migrate_params->{opts}; +my $current_log = ''; + +my $vm_stop_executed = 0; + +# mocked modules + +my $inotify_module = Test::MockModule->new("PVE::INotify"); +$inotify_module->mock( + nodename => sub { + return 'pve0'; + }, +); + +$MigrationTest::Shared::qemu_config_module->mock( + move_config_to_node => sub { + my ($self, $vmid, $target) = @_; + die "moving wrong config: '$vmid'\n" if $vmid ne $test_vmid; + die "moving config to wrong node: '$target'\n" if $target ne $test_target; + delete $expected_calls->{move_config_to_node}; + }, +); + +my $tunnel_module = Test::MockModule->new("PVE::Tunnel"); +$tunnel_module->mock( + finish_tunnel => sub { + delete $expected_calls->{'finish_tunnel'}; + return; + }, + write_tunnel => sub { + my ($tunnel, $timeout, $command) = @_; + + if ($command =~ m/^resume (\d+)$/) { + my $vmid = $1; + die "resuming wrong VM '$vmid'\n" if $vmid ne $test_vmid; + return; + } + die "write_tunnel (mocked) - implement me: $command\n"; + }, +); + +my $qemu_migrate_module = Test::MockModule->new("PVE::QemuMigrate"); +$qemu_migrate_module->mock( + fork_tunnel => sub { + die "fork_tunnel (mocked) - implement me\n"; # currently no call should lead here + }, + read_tunnel => sub { + die "read_tunnel (mocked) - implement me\n"; # currently no call should lead here + }, + start_remote_tunnel => sub { + my ($self, $raddr, $rport, $ruri, $unix_socket_info) = @_; + $expected_calls->{'finish_tunnel'} = 1; + $self->{tunnel} = { + writer => "mocked", + reader => "mocked", + pid => 123456, + version => 1, + }; + }, + log => sub { + my ($self, $level, $message) = @_; + $current_log .= "$level: $message\n"; + }, + mon_cmd => sub { + my ($vmid, $command, %params) = @_; + + if ($command eq 'nbd-server-start') { + return; + } elsif ($command eq 'block-dirty-bitmap-add') { + my $drive = $params{node}; + delete $expected_calls->{"block-dirty-bitmap-add-${drive}"}; + return; + } elsif ($command eq 'block-dirty-bitmap-remove') { + return; + } elsif ($command eq 'query-migrate') { + return { status => 'failed' } if $fail_config->{'query-migrate'}; + return { status => 'completed' }; + } elsif ($command eq 'migrate') { + return; + } elsif ($command eq 'migrate-set-parameters') { + return; + } elsif ($command eq 'migrate_cancel') { + return; + } + die "mon_cmd (mocked) - implement me: $command"; + }, + transfer_replication_state => sub { + delete $expected_calls->{transfer_replication_state}; + }, + switch_replication_job_target => sub { + delete $expected_calls->{switch_replication_job_target}; + }, +); + +$MigrationTest::Shared::qemu_server_module->mock( + kvm_user_version => sub { + return "5.0.0"; + }, + qemu_blockjobs_cancel => sub { + return; + }, + qemu_drive_mirror => sub { + my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_; + + die "drive_mirror with wrong vmid: '$vmid'\n" if $vmid ne $test_vmid; + die "qemu_drive_mirror '$drive' error\n" + if $fail_config->{qemu_drive_mirror} && $fail_config->{qemu_drive_mirror} eq $drive; + + my $nbd_info = decode_json(file_get_contents("${RUN_DIR_PATH}/nbd_info")); + die "target does not expect drive mirror for '$drive'\n" + if !defined($nbd_info->{$drive}); + delete $nbd_info->{$drive}; + file_set_contents("${RUN_DIR_PATH}/nbd_info", to_json($nbd_info)); + }, + qemu_drive_mirror_monitor => sub { + my ($vmid, $vmiddst, $jobs, $completion, $qga) = @_; + + if ($fail_config->{qemu_drive_mirror_monitor} + && $fail_config->{qemu_drive_mirror_monitor} eq $completion + ) { + die "qemu_drive_mirror_monitor '$completion' error\n"; + } + return; + }, + set_migration_caps => sub { + return; + }, + vm_stop => sub { + $vm_stop_executed = 1; + delete $expected_calls->{'vm_stop'}; + }, + del_nets_bridge_fdb => sub { return; }, +); + +my $qemu_server_cpuconfig_module = Test::MockModule->new("PVE::QemuServer::CPUConfig"); +$qemu_server_cpuconfig_module->mock( + get_cpu_from_running_vm => sub { + die "invalid test: if you specify a custom CPU model you need to " . + "specify runningcpu as well\n" if !defined($vm_status->{runningcpu}); + return $vm_status->{runningcpu}; + } +); + +my $qemu_server_helpers_module = Test::MockModule->new("PVE::QemuServer::Helpers"); +$qemu_server_helpers_module->mock( + vm_running_locally => sub { + return $vm_status->{running} && !$vm_stop_executed; + }, +); + +my $qemu_server_machine_module = Test::MockModule->new("PVE::QemuServer::Machine"); +$qemu_server_machine_module->mock( + qemu_machine_pxe => sub { + die "invalid test: no runningmachine specified\n" + if !defined($vm_status->{runningmachine}); + return $vm_status->{runningmachine}; + }, +); + +my $ssh_info_module = Test::MockModule->new("PVE::SSHInfo"); +$ssh_info_module->mock( + get_ssh_info => sub { + my ($node, $network_cidr) = @_; + return { + ip => '1.2.3.4', + name => $node, + network => $network_cidr, + }; + }, +); + +$MigrationTest::Shared::storage_module->mock( + storage_migrate => sub { + my ($cfg, $volid, $target_sshinfo, $target_storeid, $opts, $logfunc) = @_; + + die "storage_migrate '$volid' error\n" + if $fail_config->{storage_migrate} && $fail_config->{storage_migrate} eq $volid; + + my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid); + + die "invalid test: need to add entry for '$volid' to storage_migrate_map\n" + if $storeid ne $target_storeid && !defined($storage_migrate_map->{$volid}); + + my $target_volname = $storage_migrate_map->{$volid} // $opts->{target_volname} // $volname; + my $target_volid = "${target_storeid}:${target_volname}"; + MigrationTest::Shared::add_target_volid($target_volid); + + return $target_volid; + }, + vdisk_list => sub { # expects vmid to be set + my ($cfg, $storeid, $vmid, $vollist) = @_; + + my @storeids = defined($storeid) ? ($storeid) : keys %{$source_vdisks}; + + my $res = {}; + foreach my $storeid (@storeids) { + my $list_for_storeid = $source_vdisks->{$storeid}; + my @list_for_vm = grep { $_->{vmid} eq $vmid } @{$list_for_storeid}; + $res->{$storeid} = \@list_for_vm; + } + return $res; + }, + vdisk_free => sub { + my ($scfg, $volid) = @_; + + PVE::Storage::parse_volume_id($volid); + + die "vdisk_free '$volid' error\n" + if defined($fail_config->{vdisk_free}) && $fail_config->{vdisk_free} eq $volid; + + delete $source_volids->{$volid}; + }, + volume_size_info => sub { + my ($scfg, $volid) = @_; + my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid); + + for my $v ($source_vdisks->{$storeid}->@*) { + return wantarray ? ($v->{size}, $v->{format}, $v->{used}, $v->{parent}) : $v->{size} + if $v->{volid} eq $volid; + } + die "could not find '$volid' in mock 'source_vdisks'\n"; + }, +); + +$MigrationTest::Shared::tools_module->mock( + get_host_address_family => sub { + die "get_host_address_family (mocked) - implement me\n"; # currently no call should lead here + }, + next_migrate_port => sub { + die "next_migrate_port (mocked) - implement me\n"; # currently no call should lead here + }, + run_command => sub { + my ($cmd_tail, %param) = @_; + + my $cmd_msg = to_json($cmd_tail); + + my $cmd = shift @{$cmd_tail}; + + if ($cmd =~ m@^(?:/usr/bin/)?ssh$@) { + while (scalar(@{$cmd_tail})) { + $cmd = shift @{$cmd_tail}; + if ($cmd eq '/bin/true') { + return 0; + } elsif ($cmd eq 'qm') { + $cmd = shift @{$cmd_tail}; + if ($cmd eq 'start') { + delete $expected_calls->{ssh_qm_start}; + + delete $vm_status->{runningmachine}; + delete $vm_status->{runningcpu}; + + my @options = ( @{$cmd_tail} ); + while (scalar(@options)) { + my $opt = shift @options; + if ($opt eq '--machine') { + $vm_status->{runningmachine} = shift @options; + } elsif ($opt eq '--force-cpu') { + $vm_status->{runningcpu} = shift @options; + } + } + + return $MigrationTest::Shared::tools_module->original('run_command')->( + [ + '/usr/bin/perl', + "-I${QM_LIB_PATH}", + "-I${QM_LIB_PATH}/test", + "${QM_LIB_PATH}/test/MigrationTest/QmMock.pm", + 'start', + @{$cmd_tail}, + ], + %param, + ); + + } elsif ($cmd eq 'nbdstop') { + delete $expected_calls->{ssh_nbdstop}; + return 0; + } elsif ($cmd eq 'resume') { + return 0; + } elsif ($cmd eq 'unlock') { + my $vmid = shift @{$cmd_tail};; + die "unlocking wrong vmid: $vmid\n" if $vmid ne $test_vmid; + PVE::QemuConfig->remove_lock($vmid); + return 0; + } elsif ($cmd eq 'stop') { + return 0; + } + die "run_command (mocked) ssh qm command - implement me: ${cmd_msg}"; + } elsif ($cmd eq 'pvesm') { + $cmd = shift @{$cmd_tail}; + if ($cmd eq 'free') { + my $volid = shift @{$cmd_tail}; + PVE::Storage::parse_volume_id($volid); + return 1 + if $fail_config->{ssh_pvesm_free} && $fail_config->{ssh_pvesm_free} eq $volid; + MigrationTest::Shared::remove_target_volid($volid); + return 0; + } + die "run_command (mocked) ssh pvesm command - implement me: ${cmd_msg}"; + } + } + die "run_command (mocked) ssh command - implement me: ${cmd_msg}"; + } + die "run_command (mocked) - implement me: ${cmd_msg}"; + }, +); + +eval { PVE::QemuMigrate->migrate($test_target, undef, $test_vmid, $test_opts) }; +my $error = $@; + +file_set_contents("${RUN_DIR_PATH}/source_volids", to_json($source_volids)); +file_set_contents("${RUN_DIR_PATH}/vm_status", to_json($vm_status)); +file_set_contents("${RUN_DIR_PATH}/expected_calls", to_json($expected_calls)); +file_set_contents("${RUN_DIR_PATH}/log", $current_log); + +die $error if $error; + +1; diff --git a/test/MigrationTest/QmMock.pm b/test/MigrationTest/QmMock.pm new file mode 100644 index 0000000..fb94c58 --- /dev/null +++ b/test/MigrationTest/QmMock.pm @@ -0,0 +1,160 @@ +package MigrationTest::QmMock; + +use strict; +use warnings; + +use JSON; +use Test::MockModule; + +use MigrationTest::Shared; + +use PVE::API2::Qemu; +use PVE::Storage; +use PVE::Tools qw(file_set_contents file_get_contents); + +use PVE::CLIHandler; +use base qw(PVE::CLIHandler); + +my $RUN_DIR_PATH = $ENV{RUN_DIR_PATH} or die "no RUN_DIR_PATH set\n"; + +my $target_volids = decode_json(file_get_contents("${RUN_DIR_PATH}/target_volids")); +my $fail_config = decode_json(file_get_contents("${RUN_DIR_PATH}/fail_config")); +my $migrate_params = decode_json(file_get_contents("${RUN_DIR_PATH}/migrate_params")); +my $nodename = $migrate_params->{target}; + +my $kvm_exectued = 0; +my $forcemachine; + +sub setup_environment { + my $rpcenv = PVE::RPCEnvironment::init('MigrationTest::QmMock', 'cli'); +} + +# mock RPCEnvironment directly + +sub get_user { + return 'root@pam'; +} + +sub fork_worker { + my ($self, $dtype, $id, $user, $function, $background) = @_; + $function->(123456); + return '123456'; +} + +# mocked modules + +my $inotify_module = Test::MockModule->new("PVE::INotify"); +$inotify_module->mock( + nodename => sub { + return $nodename; + }, +); + +$MigrationTest::Shared::qemu_server_module->mock( + nodename => sub { + return $nodename; + }, + config_to_command => sub { + return [ 'mocked_kvm_command' ]; + }, + vm_start_nolock => sub { + my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_; + $forcemachine = $params->{forcemachine} + or die "mocked vm_start_nolock - expected 'forcemachine' parameter\n"; + $MigrationTest::Shared::qemu_server_module->original('vm_start_nolock')->(@_); + }, +); + +my $qemu_server_helpers_module = Test::MockModule->new("PVE::QemuServer::Helpers"); +$qemu_server_helpers_module->mock( + vm_running_locally => sub { + return $kvm_exectued; + }, +); + +our $qemu_server_machine_module = Test::MockModule->new("PVE::QemuServer::Machine"); +$qemu_server_machine_module->mock( + get_current_qemu_machine => sub { + return wantarray ? ($forcemachine, 0) : $forcemachine; + }, +); + +# to make sure we get valid and predictable names +my $disk_counter = 10; + +$MigrationTest::Shared::storage_module->mock( + vdisk_alloc => sub { + my ($cfg, $storeid, $vmid, $fmt, $name, $size) = @_; + + die "vdisk_alloc (mocked) - name is not expected to be set - implement me\n" + if defined($name); + + my $name_without_extension = "vm-${vmid}-disk-${disk_counter}"; + $disk_counter++; + + my $volid; + my $scfg = PVE::Storage::storage_config($cfg, $storeid); + if ($scfg->{path}) { + $volid = "${storeid}:${vmid}/${name_without_extension}.${fmt}"; + } else { + $volid = "${storeid}:${name_without_extension}"; + } + + PVE::Storage::parse_volume_id($volid); + + die "vdisk_alloc '$volid' error\n" if $fail_config->{vdisk_alloc} + && $fail_config->{vdisk_alloc} eq $volid; + + MigrationTest::Shared::add_target_volid($volid); + + return $volid; + }, +); + +$MigrationTest::Shared::qemu_server_module->mock( + mon_cmd => sub { + my ($vmid, $command, %params) = @_; + + if ($command eq 'nbd-server-start') { + return; + } elsif ($command eq 'block-export-add') { + return; + } elsif ($command eq 'query-block') { + return []; + } elsif ($command eq 'qom-set') { + return; + } + die "mon_cmd (mocked) - implement me: $command"; + }, + run_command => sub { + my ($cmd_full, %param) = @_; + + my $cmd_msg = to_json($cmd_full); + + my $cmd = shift @{$cmd_full}; + + if ($cmd eq '/bin/systemctl') { + return; + } elsif ($cmd eq 'mocked_kvm_command') { + $kvm_exectued = 1; + return 0; + } + die "run_command (mocked) - implement me: ${cmd_msg}"; + }, + set_migration_caps => sub { + return; + }, + vm_migrate_alloc_nbd_disks => sub{ + my $nbd = $MigrationTest::Shared::qemu_server_module->original('vm_migrate_alloc_nbd_disks')->(@_); + file_set_contents("${RUN_DIR_PATH}/nbd_info", to_json($nbd)); + return $nbd; + }, +); + +our $cmddef = { + start => [ "PVE::API2::Qemu", 'vm_start', ['vmid'], { node => $nodename } ], +}; + +MigrationTest::QmMock->run_cli_handler(); + +1; diff --git a/test/MigrationTest/Shared.pm b/test/MigrationTest/Shared.pm new file mode 100644 index 0000000..aa7203d --- /dev/null +++ b/test/MigrationTest/Shared.pm @@ -0,0 +1,214 @@ +package MigrationTest::Shared; + +use strict; +use warnings; + +use JSON; +use Test::MockModule; +use Socket qw(AF_INET); + +use PVE::QemuConfig; +use PVE::Tools qw(file_set_contents file_get_contents lock_file_full); + +my $RUN_DIR_PATH = $ENV{RUN_DIR_PATH} or die "no RUN_DIR_PATH set\n"; + +my $storage_config = decode_json(file_get_contents("${RUN_DIR_PATH}/storage_config")); +my $replication_config = decode_json(file_get_contents("${RUN_DIR_PATH}/replication_config")); +my $fail_config = decode_json(file_get_contents("${RUN_DIR_PATH}/fail_config")); +my $migrate_params = decode_json(file_get_contents("${RUN_DIR_PATH}/migrate_params")); +my $test_vmid = $migrate_params->{vmid}; + +# helpers + +sub add_target_volid { + my ($volid) = @_; + + PVE::Storage::parse_volume_id($volid); + + lock_file_full("${RUN_DIR_PATH}/target_volids.lock", undef, 0, sub { + my $target_volids = decode_json(file_get_contents("${RUN_DIR_PATH}/target_volids")); + die "target volid already present " if defined($target_volids->{$volid}); + $target_volids->{$volid} = 1; + file_set_contents("${RUN_DIR_PATH}/target_volids", to_json($target_volids)); + }); + die $@ if $@; +} + +sub remove_target_volid { + my ($volid) = @_; + + PVE::Storage::parse_volume_id($volid); + + lock_file_full("${RUN_DIR_PATH}/target_volids.lock", undef, 0, sub { + my $target_volids = decode_json(file_get_contents("${RUN_DIR_PATH}/target_volids")); + die "target volid does not exist " if !defined($target_volids->{$volid}); + delete $target_volids->{$volid}; + file_set_contents("${RUN_DIR_PATH}/target_volids", to_json($target_volids)); + }); + die $@ if $@; +} + +my $mocked_cfs_read_file = sub { + my ($file) = @_; + + if ($file eq 'datacenter.cfg') { + return {}; + } elsif ($file eq 'replication.cfg') { + return $replication_config; + } + die "cfs_read_file (mocked) - implement me: $file\n"; +}; + +# mocked modules + +our $cgroup_module = Test::MockModule->new("PVE::CGroup"); +$cgroup_module->mock( + cgroup_mode => sub { + return 2; + }, +); + +our $cluster_module = Test::MockModule->new("PVE::Cluster"); +$cluster_module->mock( + cfs_read_file => $mocked_cfs_read_file, + check_cfs_quorum => sub { + return 1; + }, +); + +our $mapping_usb_module = Test::MockModule->new("PVE::Mapping::USB"); +$mapping_usb_module->mock( + config => sub { + return {}; + }, +); + +our $mapping_pci_module = Test::MockModule->new("PVE::Mapping::PCI"); +$mapping_pci_module->mock( + config => sub { + return {}; + }, +); + +our $ha_config_module = Test::MockModule->new("PVE::HA::Config"); +$ha_config_module->mock( + vm_is_ha_managed => sub { + return 0; + }, +); + +our $qemu_config_module = Test::MockModule->new("PVE::QemuConfig"); +$qemu_config_module->mock( + assert_config_exists_on_node => sub { + return; + }, + load_config => sub { + my ($class, $vmid, $node) = @_; + die "trying to load wrong config: '$vmid'\n" if $vmid ne $test_vmid; + return decode_json(file_get_contents("${RUN_DIR_PATH}/vm_config")); + }, + lock_config => sub { # no use locking here because lock is local to node + my ($self, $vmid, $code, @param) = @_; + return $code->(@param); + }, + write_config => sub { + my ($class, $vmid, $conf) = @_; + die "trying to write wrong config: '$vmid'\n" if $vmid ne $test_vmid; + file_set_contents("${RUN_DIR_PATH}/vm_config", to_json($conf)); + }, +); + +our $qemu_server_cloudinit_module = Test::MockModule->new("PVE::QemuServer::Cloudinit"); +$qemu_server_cloudinit_module->mock( + generate_cloudinitconfig => sub { + return; + }, +); + +our $qemu_server_module = Test::MockModule->new("PVE::QemuServer"); +$qemu_server_module->mock( + clear_reboot_request => sub { + return 1; + }, + get_efivars_size => sub { + return 128 * 1024; + }, +); + +our $replication_module = Test::MockModule->new("PVE::Replication"); +$replication_module->mock( + run_replication => sub { + die "run_replication error" if $fail_config->{run_replication}; + + my $vm_config = PVE::QemuConfig->load_config($test_vmid); + return PVE::QemuConfig->get_replicatable_volumes( + $storage_config, + $test_vmid, + $vm_config, + ); + }, +); + +our $replication_config_module = Test::MockModule->new("PVE::ReplicationConfig"); +$replication_config_module->mock( + cfs_read_file => $mocked_cfs_read_file, +); + +our $safe_syslog_module = Test::MockModule->new("PVE::SafeSyslog"); +$safe_syslog_module->mock( + initlog => sub {}, + syslog => sub {}, +); + +our $storage_module = Test::MockModule->new("PVE::Storage"); +$storage_module->mock( + activate_volumes => sub { + return 1; + }, + deactivate_volumes => sub { + return 1; + }, + config => sub { + return $storage_config; + }, + get_bandwidth_limit => sub { + return 123456; + }, + cfs_read_file => $mocked_cfs_read_file, +); + +our $storage_plugin_module = Test::MockModule->new("PVE::Storage::Plugin"); +$storage_plugin_module->mock( + cluster_lock_storage => sub { + my ($class, $storeid, $shared, $timeout, $func, @param) = @_; + + mkdir "${RUN_DIR_PATH}/lock"; + + my $path = "${RUN_DIR_PATH}/lock/pve-storage-${storeid}"; + return PVE::Tools::lock_file($path, $timeout, $func, @param); + }, +); + +our $systemd_module = Test::MockModule->new("PVE::Systemd"); +$systemd_module->mock( + wait_for_unit_removed => sub { + return; + }, + enter_systemd_scope => sub { + return; + }, +); + +my $migrate_port_counter = 60000; + +our $tools_module = Test::MockModule->new("PVE::Tools"); +$tools_module->mock( + get_host_address_family => sub { + return AF_INET; + }, + next_migrate_port => sub { + return $migrate_port_counter++; + }, +); + +1; diff --git a/test/cfg2cmd/bootorder-empty.conf.cmd b/test/cfg2cmd/bootorder-empty.conf.cmd index 1f2b2fb..855c6e2 100644 --- a/test/cfg2cmd/bootorder-empty.conf.cmd +++ b/test/cfg2cmd/bootorder-empty.conf.cmd @@ -1,7 +1,8 @@ /usr/bin/kvm \ -id 8006 \ - -name simple \ - -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server,nowait' \ + -name 'simple,debug-threads=on' \ + -no-shutdown \ + -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ -mon 'chardev=qmp,mode=control' \ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' \ -mon 'chardev=qmp-event,mode=control' \ @@ -11,7 +12,7 @@ -smp '3,sockets=1,cores=3,maxcpus=3' \ -nodefaults \ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ - -vnc unix:/var/run/qemu-server/8006.vnc,password \ + -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ -m 768 \ -object 'iothread,id=iothread-virtio0' \ diff --git a/test/cfg2cmd/bootorder-legacy.conf.cmd b/test/cfg2cmd/bootorder-legacy.conf.cmd index f624ea2..2320abb 100644 --- a/test/cfg2cmd/bootorder-legacy.conf.cmd +++ b/test/cfg2cmd/bootorder-legacy.conf.cmd @@ -1,7 +1,8 @@ /usr/bin/kvm \ -id 8006 \ - -name simple \ - -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server,nowait' \ + -name 'simple,debug-threads=on' \ + -no-shutdown \ + -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ -mon 'chardev=qmp,mode=control' \ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' \ -mon 'chardev=qmp-event,mode=control' \ @@ -11,7 +12,7 @@ -smp '3,sockets=1,cores=3,maxcpus=3' \ -nodefaults \ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ - -vnc unix:/var/run/qemu-server/8006.vnc,password \ + -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ -m 768 \ -object 'iothread,id=iothread-virtio0' \ diff --git a/test/cfg2cmd/bootorder.conf.cmd b/test/cfg2cmd/bootorder.conf.cmd index 86cae07..8ba36dc 100644 --- a/test/cfg2cmd/bootorder.conf.cmd +++ b/test/cfg2cmd/bootorder.conf.cmd @@ -1,7 +1,8 @@ /usr/bin/kvm \ -id 8006 \ - -name simple \ - -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server,nowait' \ + -name 'simple,debug-threads=on' \ + -no-shutdown \ + -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ -mon 'chardev=qmp,mode=control' \ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' \ -mon 'chardev=qmp-event,mode=control' \ @@ -11,7 +12,7 @@ -smp '3,sockets=1,cores=3,maxcpus=3' \ -nodefaults \ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ - -vnc unix:/var/run/qemu-server/8006.vnc,password \ + -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ -m 768 \ -object 'iothread,id=iothread-virtio0' \ diff --git a/test/cfg2cmd/cputype-icelake-client-deprecation.conf b/test/cfg2cmd/cputype-icelake-client-deprecation.conf new file mode 100644 index 0000000..523dd27 --- /dev/null +++ b/test/cfg2cmd/cputype-icelake-client-deprecation.conf @@ -0,0 +1,14 @@ +# TEST: test CPU type depreacation for Icelake-Client (never existed in the wild) +# QEMU_VERSION: 7.1 +bootdisk: scsi0 +cores: 2 +cpu: Icelake-Client +ide2: none,media=cdrom +memory: 768 +name: simple +ostype: l26 +scsi0: local:8006/base-8006-disk-0.qcow2,discard=on,size=104858K +scsihw: virtio-scsi-pci +smbios1: uuid=7b10d7af-b932-4c66-b2c3-3996152ec465 +sockets: 1 +vmgenid: c773c261-d800-4348-9f5d-167fadd53cf8 diff --git a/test/cfg2cmd/cputype-icelake-client-deprecation.conf.cmd b/test/cfg2cmd/cputype-icelake-client-deprecation.conf.cmd new file mode 100644 index 0000000..bf08443 --- /dev/null +++ b/test/cfg2cmd/cputype-icelake-client-deprecation.conf.cmd @@ -0,0 +1,31 @@ +/usr/bin/kvm \ + -id 8006 \ + -name 'simple,debug-threads=on' \ + -no-shutdown \ + -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ + -mon 'chardev=qmp,mode=control' \ + -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' \ + -mon 'chardev=qmp-event,mode=control' \ + -pidfile /var/run/qemu-server/8006.pid \ + -daemonize \ + -smbios 'type=1,uuid=7b10d7af-b932-4c66-b2c3-3996152ec465' \ + -smp '2,sockets=1,cores=2,maxcpus=2' \ + -nodefaults \ + -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ + -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ + -cpu 'Icelake-Server,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,vendor=GenuineIntel' \ + -m 768 \ + -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \ + -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \ + -device 'vmgenid,guid=c773c261-d800-4348-9f5d-167fadd53cf8' \ + -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \ + -device 'usb-tablet,id=tablet,bus=uhci.0,port=1' \ + -device 'VGA,id=vga,bus=pci.0,addr=0x2' \ + -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \ + -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ + -drive 'if=none,id=drive-ide2,media=cdrom,aio=io_uring' \ + -device 'ide-cd,bus=ide.1,unit=0,drive=drive-ide2,id=ide2,bootindex=200' \ + -device 'virtio-scsi-pci,id=scsihw0,bus=pci.0,addr=0x5' \ + -drive 'file=/var/lib/vz/images/8006/base-8006-disk-0.qcow2,if=none,id=drive-scsi0,discard=on,format=qcow2,cache=none,aio=io_uring,detect-zeroes=unmap' \ + -device 'scsi-hd,bus=scsihw0.0,channel=0,scsi-id=0,lun=0,drive=drive-scsi0,id=scsi0,bootindex=100' \ + -machine 'type=pc+pve0' diff --git a/test/cfg2cmd/custom-cpu-model-defaults.conf.cmd b/test/cfg2cmd/custom-cpu-model-defaults.conf.cmd index ca8fcb0..15b31fb 100644 --- a/test/cfg2cmd/custom-cpu-model-defaults.conf.cmd +++ b/test/cfg2cmd/custom-cpu-model-defaults.conf.cmd @@ -1,7 +1,8 @@ /usr/bin/kvm \ -id 8006 \ - -name customcpu-defaults \ - -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server,nowait' \ + -name 'customcpu-defaults,debug-threads=on' \ + -no-shutdown \ + -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ -mon 'chardev=qmp,mode=control' \ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' \ -mon 'chardev=qmp-event,mode=control' \ @@ -11,7 +12,7 @@ -smp '3,sockets=1,cores=3,maxcpus=3' \ -nodefaults \ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ - -vnc unix:/var/run/qemu-server/8006.vnc,password \ + -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ -m 512 \ -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \ @@ -19,6 +20,6 @@ -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \ -device 'usb-tablet,id=tablet,bus=uhci.0,port=1' \ -device 'VGA,id=vga,bus=pci.0,addr=0x2' \ - -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3' \ + -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ -machine 'type=pc+pve0' diff --git a/test/cfg2cmd/custom-cpu-model-host-phys-bits.conf.cmd b/test/cfg2cmd/custom-cpu-model-host-phys-bits.conf.cmd index fb6e8c8..077437d 100644 --- a/test/cfg2cmd/custom-cpu-model-host-phys-bits.conf.cmd +++ b/test/cfg2cmd/custom-cpu-model-host-phys-bits.conf.cmd @@ -1,7 +1,8 @@ /usr/bin/kvm \ -id 8006 \ - -name customcpu \ - -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server,nowait' \ + -name 'customcpu,debug-threads=on' \ + -no-shutdown \ + -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ -mon 'chardev=qmp,mode=control' \ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' \ -mon 'chardev=qmp-event,mode=control' \ @@ -11,8 +12,7 @@ -smp '3,sockets=1,cores=3,maxcpus=3' \ -nodefaults \ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ - -vnc unix:/var/run/qemu-server/8006.vnc,password \ - -no-hpet \ + -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ -cpu 'athlon,+aes,+avx,enforce,hv_ipi,hv_relaxed,hv_reset,hv_runtime,hv_spinlocks=0x1fff,hv_stimer,hv_synic,hv_time,hv_vapic,hv_vendor_id=testvend,hv_vpindex,+kvm_pv_eoi,-kvm_pv_unhalt,vendor=AuthenticAMD,host-phys-bits=true' \ -m 512 \ -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \ @@ -23,5 +23,5 @@ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3' \ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ -rtc 'driftfix=slew,base=localtime' \ - -machine 'type=pc+pve0' \ + -machine 'hpet=off,type=pc-i440fx-5.1+pve0' \ -global 'kvm-pit.lost_tick_policy=discard' diff --git a/test/cfg2cmd/custom-cpu-model.conf.cmd b/test/cfg2cmd/custom-cpu-model.conf.cmd index b30163c..5a9daed 100644 --- a/test/cfg2cmd/custom-cpu-model.conf.cmd +++ b/test/cfg2cmd/custom-cpu-model.conf.cmd @@ -1,7 +1,8 @@ /usr/bin/kvm \ -id 8006 \ - -name customcpu \ - -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server,nowait' \ + -name 'customcpu,debug-threads=on' \ + -no-shutdown \ + -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ -mon 'chardev=qmp,mode=control' \ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' \ -mon 'chardev=qmp-event,mode=control' \ @@ -11,8 +12,7 @@ -smp '3,sockets=1,cores=3,maxcpus=3' \ -nodefaults \ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ - -vnc unix:/var/run/qemu-server/8006.vnc,password \ - -no-hpet \ + -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ -cpu 'athlon,+aes,+avx,enforce,hv_ipi,hv_relaxed,hv_reset,hv_runtime,hv_spinlocks=0x1fff,hv_stimer,hv_synic,hv_time,hv_vapic,hv_vendor_id=testvend,hv_vpindex,+kvm_pv_eoi,-kvm_pv_unhalt,vendor=AuthenticAMD,+virt-ssbd,phys-bits=40' \ -m 512 \ -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \ @@ -23,5 +23,5 @@ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3' \ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ -rtc 'driftfix=slew,base=localtime' \ - -machine 'type=pc+pve0' \ + -machine 'hpet=off,type=pc-i440fx-5.1+pve0' \ -global 'kvm-pit.lost_tick_policy=discard' diff --git a/test/cfg2cmd/efi-raw-old.conf.cmd b/test/cfg2cmd/efi-raw-old.conf.cmd index 666cdb1..dfd381d 100644 --- a/test/cfg2cmd/efi-raw-old.conf.cmd +++ b/test/cfg2cmd/efi-raw-old.conf.cmd @@ -1,19 +1,20 @@ /usr/bin/kvm \ -id 8006 \ - -name vm8006 \ - -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server,nowait' \ + -name 'vm8006,debug-threads=on' \ + -no-shutdown \ + -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ -mon 'chardev=qmp,mode=control' \ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' \ -mon 'chardev=qmp-event,mode=control' \ -pidfile /var/run/qemu-server/8006.pid \ -daemonize \ -smbios 'type=1,uuid=7b10d7af-b932-4c66-b2c3-3996152ec465' \ - -drive 'if=pflash,unit=0,format=raw,readonly,file=/usr/share/pve-edk2-firmware//OVMF_CODE.fd' \ - -drive 'if=pflash,unit=1,format=raw,id=drive-efidisk0,file=/var/lib/vz/images/100/vm-disk-100-0.raw' \ + -drive 'if=pflash,unit=0,format=raw,readonly=on,file=/usr/share/pve-edk2-firmware//OVMF_CODE.fd' \ + -drive 'if=pflash,unit=1,id=drive-efidisk0,format=raw,file=/var/lib/vz/images/100/vm-disk-100-0.raw' \ -smp '1,sockets=1,cores=1,maxcpus=1' \ -nodefaults \ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ - -vnc unix:/var/run/qemu-server/8006.vnc,password \ + -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ -m 512 \ -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \ diff --git a/test/cfg2cmd/efi-raw-template.conf b/test/cfg2cmd/efi-raw-template.conf new file mode 100644 index 0000000..a181522 --- /dev/null +++ b/test/cfg2cmd/efi-raw-template.conf @@ -0,0 +1,5 @@ +# TEST: Test raw efidisk size parameter +smbios1: uuid=7b10d7af-b932-4c66-b2c3-3996152ec465 +bios: ovmf +efidisk0: local:100/base-disk-100-0.raw +template: 1 diff --git a/test/cfg2cmd/efi-raw-template.conf.cmd b/test/cfg2cmd/efi-raw-template.conf.cmd new file mode 100644 index 0000000..b1d4d1f --- /dev/null +++ b/test/cfg2cmd/efi-raw-template.conf.cmd @@ -0,0 +1,28 @@ +/usr/bin/kvm \ + -id 8006 \ + -name 'vm8006,debug-threads=on' \ + -no-shutdown \ + -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ + -mon 'chardev=qmp,mode=control' \ + -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' \ + -mon 'chardev=qmp-event,mode=control' \ + -pidfile /var/run/qemu-server/8006.pid \ + -daemonize \ + -smbios 'type=1,uuid=7b10d7af-b932-4c66-b2c3-3996152ec465' \ + -drive 'if=pflash,unit=0,format=raw,readonly=on,file=/usr/share/pve-edk2-firmware//OVMF_CODE.fd' \ + -drive 'if=pflash,unit=1,id=drive-efidisk0,format=raw,file=/var/lib/vz/images/100/base-disk-100-0.raw,size=131072,readonly=on' \ + -smp '1,sockets=1,cores=1,maxcpus=1' \ + -nodefaults \ + -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ + -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ + -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ + -m 512 \ + -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \ + -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \ + -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \ + -device 'usb-tablet,id=tablet,bus=uhci.0,port=1' \ + -device 'VGA,id=vga,bus=pci.0,addr=0x2' \ + -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \ + -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ + -machine 'type=pc+pve0' \ + -snapshot diff --git a/test/cfg2cmd/efi-raw.conf.cmd b/test/cfg2cmd/efi-raw.conf.cmd index cb0e984..cf9804b 100644 --- a/test/cfg2cmd/efi-raw.conf.cmd +++ b/test/cfg2cmd/efi-raw.conf.cmd @@ -1,19 +1,20 @@ /usr/bin/kvm \ -id 8006 \ - -name vm8006 \ - -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server,nowait' \ + -name 'vm8006,debug-threads=on' \ + -no-shutdown \ + -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ -mon 'chardev=qmp,mode=control' \ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' \ -mon 'chardev=qmp-event,mode=control' \ -pidfile /var/run/qemu-server/8006.pid \ -daemonize \ -smbios 'type=1,uuid=7b10d7af-b932-4c66-b2c3-3996152ec465' \ - -drive 'if=pflash,unit=0,format=raw,readonly,file=/usr/share/pve-edk2-firmware//OVMF_CODE.fd' \ - -drive 'if=pflash,unit=1,format=raw,id=drive-efidisk0,size=131072,file=/var/lib/vz/images/100/vm-disk-100-0.raw' \ + -drive 'if=pflash,unit=0,format=raw,readonly=on,file=/usr/share/pve-edk2-firmware//OVMF_CODE.fd' \ + -drive 'if=pflash,unit=1,id=drive-efidisk0,format=raw,file=/var/lib/vz/images/100/vm-disk-100-0.raw,size=131072' \ -smp '1,sockets=1,cores=1,maxcpus=1' \ -nodefaults \ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ - -vnc unix:/var/run/qemu-server/8006.vnc,password \ + -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ -m 512 \ -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \ @@ -21,6 +22,6 @@ -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \ -device 'usb-tablet,id=tablet,bus=uhci.0,port=1' \ -device 'VGA,id=vga,bus=pci.0,addr=0x2' \ - -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3' \ + -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ -machine 'type=pc+pve0' diff --git a/test/cfg2cmd/efi-secboot-and-tpm-q35.conf b/test/cfg2cmd/efi-secboot-and-tpm-q35.conf new file mode 100644 index 0000000..5d4b5f5 --- /dev/null +++ b/test/cfg2cmd/efi-secboot-and-tpm-q35.conf @@ -0,0 +1,6 @@ +# TEST: Test newer 4MB efidisk with secureboot, smm enforce and a TPM device on Q35 +smbios1: uuid=7b10d7af-b932-4c66-b2c3-3996152ec465 +bios: ovmf +machine: q35 +efidisk0: local:100/vm-disk-100-0.raw,efitype=4m,pre-enrolled-keys=1,size=528K +tpmstate0: local:108/vm-100-disk-1.raw,size=4M,version=v2.0 diff --git a/test/cfg2cmd/efi-secboot-and-tpm-q35.conf.cmd b/test/cfg2cmd/efi-secboot-and-tpm-q35.conf.cmd new file mode 100644 index 0000000..911ead0 --- /dev/null +++ b/test/cfg2cmd/efi-secboot-and-tpm-q35.conf.cmd @@ -0,0 +1,28 @@ +/usr/bin/kvm \ + -id 8006 \ + -name 'vm8006,debug-threads=on' \ + -no-shutdown \ + -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ + -mon 'chardev=qmp,mode=control' \ + -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' \ + -mon 'chardev=qmp-event,mode=control' \ + -pidfile /var/run/qemu-server/8006.pid \ + -daemonize \ + -smbios 'type=1,uuid=7b10d7af-b932-4c66-b2c3-3996152ec465' \ + -drive 'if=pflash,unit=0,format=raw,readonly=on,file=/usr/share/pve-edk2-firmware//OVMF_CODE_4M.secboot.fd' \ + -drive 'if=pflash,unit=1,id=drive-efidisk0,format=raw,file=/var/lib/vz/images/100/vm-disk-100-0.raw,size=540672' \ + -smp '1,sockets=1,cores=1,maxcpus=1' \ + -nodefaults \ + -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ + -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ + -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ + -m 512 \ + -readconfig /usr/share/qemu-server/pve-q35-4.0.cfg \ + -device 'usb-tablet,id=tablet,bus=ehci.0,port=1' \ + -chardev 'socket,id=tpmchar,path=/var/run/qemu-server/8006.swtpm' \ + -tpmdev 'emulator,id=tpmdev,chardev=tpmchar' \ + -device 'tpm-tis,tpmdev=tpmdev' \ + -device 'VGA,id=vga,bus=pcie.0,addr=0x1' \ + -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \ + -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ + -machine 'type=q35+pve0' diff --git a/test/cfg2cmd/efi-secboot-and-tpm.conf b/test/cfg2cmd/efi-secboot-and-tpm.conf new file mode 100644 index 0000000..915424e --- /dev/null +++ b/test/cfg2cmd/efi-secboot-and-tpm.conf @@ -0,0 +1,5 @@ +# TEST: Test newer 4MB efidisk with secureboot and a TPM device +smbios1: uuid=7b10d7af-b932-4c66-b2c3-3996152ec465 +bios: ovmf +efidisk0: local:100/vm-disk-100-0.raw,efitype=4m,pre-enrolled-keys=1,size=528K +tpmstate0: local:108/vm-100-disk-1.raw,size=4M,version=v2.0 diff --git a/test/cfg2cmd/efi-secboot-and-tpm.conf.cmd b/test/cfg2cmd/efi-secboot-and-tpm.conf.cmd new file mode 100644 index 0000000..68a85ea --- /dev/null +++ b/test/cfg2cmd/efi-secboot-and-tpm.conf.cmd @@ -0,0 +1,30 @@ +/usr/bin/kvm \ + -id 8006 \ + -name 'vm8006,debug-threads=on' \ + -no-shutdown \ + -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ + -mon 'chardev=qmp,mode=control' \ + -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' \ + -mon 'chardev=qmp-event,mode=control' \ + -pidfile /var/run/qemu-server/8006.pid \ + -daemonize \ + -smbios 'type=1,uuid=7b10d7af-b932-4c66-b2c3-3996152ec465' \ + -drive 'if=pflash,unit=0,format=raw,readonly=on,file=/usr/share/pve-edk2-firmware//OVMF_CODE_4M.fd' \ + -drive 'if=pflash,unit=1,id=drive-efidisk0,format=raw,file=/var/lib/vz/images/100/vm-disk-100-0.raw,size=540672' \ + -smp '1,sockets=1,cores=1,maxcpus=1' \ + -nodefaults \ + -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ + -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ + -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ + -m 512 \ + -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \ + -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \ + -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \ + -device 'usb-tablet,id=tablet,bus=uhci.0,port=1' \ + -chardev 'socket,id=tpmchar,path=/var/run/qemu-server/8006.swtpm' \ + -tpmdev 'emulator,id=tpmdev,chardev=tpmchar' \ + -device 'tpm-tis,tpmdev=tpmdev' \ + -device 'VGA,id=vga,bus=pci.0,addr=0x2' \ + -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \ + -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ + -machine 'type=pc+pve0' diff --git a/test/cfg2cmd/efidisk-on-rbd.conf b/test/cfg2cmd/efidisk-on-rbd.conf new file mode 100644 index 0000000..1958fe6 --- /dev/null +++ b/test/cfg2cmd/efidisk-on-rbd.conf @@ -0,0 +1,11 @@ +# TEST: Config with efi disk on RBD is very slow without cache - #3329 +bios: ovmf +bootdisk: scsi0 +cores: 1 +efidisk0: rbd-store:vm-100-disk-1,size=128K +memory: 512 +net0: virtio=2E:01:68:F9:9C:87,bridge=vmbr0 +numa: 1 +ostype: l26 +smbios1: uuid=3dd750ce-d910-44d0-9493-525c0be4e688 +vmgenid: 54d1c06c-8f5b-440f-b5b2-6eab1380e13a diff --git a/test/cfg2cmd/efidisk-on-rbd.conf.cmd b/test/cfg2cmd/efidisk-on-rbd.conf.cmd new file mode 100644 index 0000000..f02039a --- /dev/null +++ b/test/cfg2cmd/efidisk-on-rbd.conf.cmd @@ -0,0 +1,32 @@ +/usr/bin/kvm \ + -id 8006 \ + -name 'vm8006,debug-threads=on' \ + -no-shutdown \ + -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ + -mon 'chardev=qmp,mode=control' \ + -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' \ + -mon 'chardev=qmp-event,mode=control' \ + -pidfile /var/run/qemu-server/8006.pid \ + -daemonize \ + -smbios 'type=1,uuid=3dd750ce-d910-44d0-9493-525c0be4e688' \ + -drive 'if=pflash,unit=0,format=raw,readonly=on,file=/usr/share/pve-edk2-firmware//OVMF_CODE.fd' \ + -drive 'if=pflash,unit=1,id=drive-efidisk0,cache=writeback,format=raw,file=rbd:cpool/vm-100-disk-1:mon_host=127.0.0.42;127.0.0.21;[\:\:1]:auth_supported=none:rbd_cache_policy=writeback,size=131072' \ + -smp '1,sockets=1,cores=1,maxcpus=1' \ + -nodefaults \ + -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ + -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ + -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ + -m 512 \ + -object 'memory-backend-ram,id=ram-node0,size=512M' \ + -numa 'node,nodeid=0,cpus=0,memdev=ram-node0' \ + -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \ + -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \ + -device 'vmgenid,guid=54d1c06c-8f5b-440f-b5b2-6eab1380e13a' \ + -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \ + -device 'usb-tablet,id=tablet,bus=uhci.0,port=1' \ + -device 'VGA,id=vga,bus=pci.0,addr=0x2' \ + -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \ + -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ + -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/var/lib/qemu-server/pve-bridge,downscript=/var/lib/qemu-server/pve-bridgedown,vhost=on' \ + -device 'virtio-net-pci,mac=2E:01:68:F9:9C:87,netdev=net0,bus=pci.0,addr=0x12,id=net0,rx_queue_size=1024,tx_queue_size=256,bootindex=300' \ + -machine 'type=pc+pve0' diff --git a/test/cfg2cmd/i440fx-viommu-intel.conf b/test/cfg2cmd/i440fx-viommu-intel.conf new file mode 100644 index 0000000..bc1eb95 --- /dev/null +++ b/test/cfg2cmd/i440fx-viommu-intel.conf @@ -0,0 +1,2 @@ +# EXPECT_ERROR: to use Intel vIOMMU please set the machine type to q35 +machine: pc,viommu=intel diff --git a/test/cfg2cmd/i440fx-viommu-virtio.conf b/test/cfg2cmd/i440fx-viommu-virtio.conf new file mode 100644 index 0000000..fe7b514 --- /dev/null +++ b/test/cfg2cmd/i440fx-viommu-virtio.conf @@ -0,0 +1 @@ +machine: pc,viommu=virtio diff --git a/test/cfg2cmd/i440fx-viommu-virtio.conf.cmd b/test/cfg2cmd/i440fx-viommu-virtio.conf.cmd new file mode 100644 index 0000000..0352354 --- /dev/null +++ b/test/cfg2cmd/i440fx-viommu-virtio.conf.cmd @@ -0,0 +1,25 @@ +/usr/bin/kvm \ + -id 8006 \ + -name 'vm8006,debug-threads=on' \ + -no-shutdown \ + -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ + -mon 'chardev=qmp,mode=control' \ + -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' \ + -mon 'chardev=qmp-event,mode=control' \ + -pidfile /var/run/qemu-server/8006.pid \ + -daemonize \ + -smp '1,sockets=1,cores=1,maxcpus=1' \ + -nodefaults \ + -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ + -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ + -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ + -m 512 \ + -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \ + -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \ + -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \ + -device 'usb-tablet,id=tablet,bus=uhci.0,port=1' \ + -device 'VGA,id=vga,bus=pci.0,addr=0x2' \ + -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \ + -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ + -device virtio-iommu-pci \ + -machine 'type=pc+pve0' diff --git a/test/cfg2cmd/i440fx-win10-hostpci.conf.cmd b/test/cfg2cmd/i440fx-win10-hostpci.conf.cmd index 2fe34a1..455c898 100644 --- a/test/cfg2cmd/i440fx-win10-hostpci.conf.cmd +++ b/test/cfg2cmd/i440fx-win10-hostpci.conf.cmd @@ -1,20 +1,20 @@ /usr/bin/kvm \ -id 8006 \ - -name vm8006 \ - -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server,nowait' \ + -name 'vm8006,debug-threads=on' \ + -no-shutdown \ + -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ -mon 'chardev=qmp,mode=control' \ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' \ -mon 'chardev=qmp-event,mode=control' \ -pidfile /var/run/qemu-server/8006.pid \ -daemonize \ -smbios 'type=1,uuid=3dd750ce-d910-44d0-9493-525c0be4e687' \ - -drive 'if=pflash,unit=0,format=raw,readonly,file=/usr/share/pve-edk2-firmware//OVMF_CODE.fd' \ - -drive 'if=pflash,unit=1,format=qcow2,id=drive-efidisk0,file=/var/lib/vz/images/100/vm-100-disk-1.qcow2' \ + -drive 'if=pflash,unit=0,format=raw,readonly=on,file=/usr/share/pve-edk2-firmware//OVMF_CODE.fd' \ + -drive 'if=pflash,unit=1,id=drive-efidisk0,format=qcow2,file=/var/lib/vz/images/100/vm-100-disk-1.qcow2' \ -smp '2,sockets=2,cores=1,maxcpus=2' \ -nodefaults \ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ - -vnc unix:/var/run/qemu-server/8006.vnc,password \ - -no-hpet \ + -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ -cpu 'kvm64,enforce,hv_ipi,hv_relaxed,hv_reset,hv_runtime,hv_spinlocks=0x1fff,hv_stimer,hv_synic,hv_time,hv_vapic,hv_vpindex,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep' \ -m 512 \ -object 'memory-backend-ram,id=ram-node0,size=256M' \ @@ -33,5 +33,5 @@ -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/var/lib/qemu-server/pve-bridge,downscript=/var/lib/qemu-server/pve-bridgedown,vhost=on' \ -device 'virtio-net-pci,mac=2E:01:68:F9:9C:87,netdev=net0,bus=pci.0,addr=0x12,id=net0,bootindex=300' \ -rtc 'driftfix=slew,base=localtime' \ - -machine 'type=pc+pve0' \ + -machine 'hpet=off,type=pc-i440fx-5.1+pve0' \ -global 'kvm-pit.lost_tick_policy=discard' diff --git a/test/cfg2cmd/ide.conf b/test/cfg2cmd/ide.conf new file mode 100644 index 0000000..0c48aac --- /dev/null +++ b/test/cfg2cmd/ide.conf @@ -0,0 +1,14 @@ +# TEST: Config with default machine type, Linux & four IDE CD-ROMs +bootdisk: scsi0 +cores: 2 +ide0: cifs-store:iso/zero.iso,media=cdrom,size=112M +ide1: cifs-store:iso/one.iso,media=cdrom,size=112M +ide2: cifs-store:iso/two.iso,media=cdrom,size=112M +ide3: cifs-store:iso/three.iso,media=cdrom,size=112M +memory: 512 +net0: virtio=2E:01:68:F9:9C:87,bridge=vmbr0 +ostype: l26 +scsi0: local:100/vm-100-disk-2.qcow2,size=10G +scsihw: virtio-scsi-pci +smbios1: uuid=3dd750ce-d910-44d0-9493-525c0be4e687 +vmgenid: 54d1c06c-8f5b-440f-b5b2-6eab1380e13d diff --git a/test/cfg2cmd/ide.conf.cmd b/test/cfg2cmd/ide.conf.cmd new file mode 100644 index 0000000..7fd4888 --- /dev/null +++ b/test/cfg2cmd/ide.conf.cmd @@ -0,0 +1,39 @@ +/usr/bin/kvm \ + -id 8006 \ + -name 'vm8006,debug-threads=on' \ + -no-shutdown \ + -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ + -mon 'chardev=qmp,mode=control' \ + -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' \ + -mon 'chardev=qmp-event,mode=control' \ + -pidfile /var/run/qemu-server/8006.pid \ + -daemonize \ + -smbios 'type=1,uuid=3dd750ce-d910-44d0-9493-525c0be4e687' \ + -smp '2,sockets=1,cores=2,maxcpus=2' \ + -nodefaults \ + -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ + -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ + -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ + -m 512 \ + -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \ + -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \ + -device 'vmgenid,guid=54d1c06c-8f5b-440f-b5b2-6eab1380e13d' \ + -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \ + -device 'usb-tablet,id=tablet,bus=uhci.0,port=1' \ + -device 'VGA,id=vga,bus=pci.0,addr=0x2' \ + -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \ + -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ + -drive 'file=/mnt/pve/cifs-store/template/iso/zero.iso,if=none,id=drive-ide0,media=cdrom,aio=threads' \ + -device 'ide-cd,bus=ide.0,unit=0,drive=drive-ide0,id=ide0,bootindex=200' \ + -drive 'file=/mnt/pve/cifs-store/template/iso/one.iso,if=none,id=drive-ide1,media=cdrom,aio=threads' \ + -device 'ide-cd,bus=ide.0,unit=1,drive=drive-ide1,id=ide1,bootindex=201' \ + -drive 'file=/mnt/pve/cifs-store/template/iso/two.iso,if=none,id=drive-ide2,media=cdrom,aio=threads' \ + -device 'ide-cd,bus=ide.1,unit=0,drive=drive-ide2,id=ide2,bootindex=202' \ + -drive 'file=/mnt/pve/cifs-store/template/iso/three.iso,if=none,id=drive-ide3,media=cdrom,aio=threads' \ + -device 'ide-cd,bus=ide.1,unit=1,drive=drive-ide3,id=ide3,bootindex=203' \ + -device 'virtio-scsi-pci,id=scsihw0,bus=pci.0,addr=0x5' \ + -drive 'file=/var/lib/vz/images/100/vm-100-disk-2.qcow2,if=none,id=drive-scsi0,format=qcow2,cache=none,aio=io_uring,detect-zeroes=on' \ + -device 'scsi-hd,bus=scsihw0.0,channel=0,scsi-id=0,lun=0,drive=drive-scsi0,id=scsi0,bootindex=100' \ + -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/var/lib/qemu-server/pve-bridge,downscript=/var/lib/qemu-server/pve-bridgedown,vhost=on' \ + -device 'virtio-net-pci,mac=2E:01:68:F9:9C:87,netdev=net0,bus=pci.0,addr=0x12,id=net0,rx_queue_size=1024,tx_queue_size=256,bootindex=300' \ + -machine 'type=pc+pve0' diff --git a/test/cfg2cmd/memory-hotplug-hugepages.conf b/test/cfg2cmd/memory-hotplug-hugepages.conf new file mode 100644 index 0000000..6cba31e --- /dev/null +++ b/test/cfg2cmd/memory-hotplug-hugepages.conf @@ -0,0 +1,12 @@ +# TEST: memory hotplug with 1GB hugepage +# QEMU_VERSION: 3.0 +cores: 2 +memory: 18432 +name: simple +numa: 1 +ostype: l26 +smbios1: uuid=7b10d7af-b932-4c66-b2c3-3996152ec465 +sockets: 2 +vmgenid: c773c261-d800-4348-9f5d-167fadd53cf8 +hotplug: memory +hugepages: 1024 \ No newline at end of file diff --git a/test/cfg2cmd/memory-hotplug-hugepages.conf.cmd b/test/cfg2cmd/memory-hotplug-hugepages.conf.cmd new file mode 100644 index 0000000..6d4d8e8 --- /dev/null +++ b/test/cfg2cmd/memory-hotplug-hugepages.conf.cmd @@ -0,0 +1,62 @@ +/usr/bin/kvm \ + -id 8006 \ + -name 'simple,debug-threads=on' \ + -no-shutdown \ + -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ + -mon 'chardev=qmp,mode=control' \ + -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' \ + -mon 'chardev=qmp-event,mode=control' \ + -pidfile /var/run/qemu-server/8006.pid \ + -daemonize \ + -smbios 'type=1,uuid=7b10d7af-b932-4c66-b2c3-3996152ec465' \ + -smp '4,sockets=2,cores=2,maxcpus=4' \ + -nodefaults \ + -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ + -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ + -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ + -m 'size=2048,slots=255,maxmem=524288M' \ + -object 'memory-backend-file,id=ram-node0,size=1024M,mem-path=/run/hugepages/kvm/1048576kB,share=on,prealloc=yes' \ + -numa 'node,nodeid=0,cpus=0-1,memdev=ram-node0' \ + -object 'memory-backend-file,id=ram-node1,size=1024M,mem-path=/run/hugepages/kvm/1048576kB,share=on,prealloc=yes' \ + -numa 'node,nodeid=1,cpus=2-3,memdev=ram-node1' \ + -object 'memory-backend-file,id=mem-dimm0,size=1024M,mem-path=/run/hugepages/kvm/1048576kB,share=on,prealloc=yes' \ + -device 'pc-dimm,id=dimm0,memdev=mem-dimm0,node=0' \ + -object 'memory-backend-file,id=mem-dimm1,size=1024M,mem-path=/run/hugepages/kvm/1048576kB,share=on,prealloc=yes' \ + -device 'pc-dimm,id=dimm1,memdev=mem-dimm1,node=1' \ + -object 'memory-backend-file,id=mem-dimm2,size=1024M,mem-path=/run/hugepages/kvm/1048576kB,share=on,prealloc=yes' \ + -device 'pc-dimm,id=dimm2,memdev=mem-dimm2,node=0' \ + -object 'memory-backend-file,id=mem-dimm3,size=1024M,mem-path=/run/hugepages/kvm/1048576kB,share=on,prealloc=yes' \ + -device 'pc-dimm,id=dimm3,memdev=mem-dimm3,node=1' \ + -object 'memory-backend-file,id=mem-dimm4,size=1024M,mem-path=/run/hugepages/kvm/1048576kB,share=on,prealloc=yes' \ + -device 'pc-dimm,id=dimm4,memdev=mem-dimm4,node=0' \ + -object 'memory-backend-file,id=mem-dimm5,size=1024M,mem-path=/run/hugepages/kvm/1048576kB,share=on,prealloc=yes' \ + -device 'pc-dimm,id=dimm5,memdev=mem-dimm5,node=1' \ + -object 'memory-backend-file,id=mem-dimm6,size=1024M,mem-path=/run/hugepages/kvm/1048576kB,share=on,prealloc=yes' \ + -device 'pc-dimm,id=dimm6,memdev=mem-dimm6,node=0' \ + -object 'memory-backend-file,id=mem-dimm7,size=1024M,mem-path=/run/hugepages/kvm/1048576kB,share=on,prealloc=yes' \ + -device 'pc-dimm,id=dimm7,memdev=mem-dimm7,node=1' \ + -object 'memory-backend-file,id=mem-dimm8,size=1024M,mem-path=/run/hugepages/kvm/1048576kB,share=on,prealloc=yes' \ + -device 'pc-dimm,id=dimm8,memdev=mem-dimm8,node=0' \ + -object 'memory-backend-file,id=mem-dimm9,size=1024M,mem-path=/run/hugepages/kvm/1048576kB,share=on,prealloc=yes' \ + -device 'pc-dimm,id=dimm9,memdev=mem-dimm9,node=1' \ + -object 'memory-backend-file,id=mem-dimm10,size=1024M,mem-path=/run/hugepages/kvm/1048576kB,share=on,prealloc=yes' \ + -device 'pc-dimm,id=dimm10,memdev=mem-dimm10,node=0' \ + -object 'memory-backend-file,id=mem-dimm11,size=1024M,mem-path=/run/hugepages/kvm/1048576kB,share=on,prealloc=yes' \ + -device 'pc-dimm,id=dimm11,memdev=mem-dimm11,node=1' \ + -object 'memory-backend-file,id=mem-dimm12,size=1024M,mem-path=/run/hugepages/kvm/1048576kB,share=on,prealloc=yes' \ + -device 'pc-dimm,id=dimm12,memdev=mem-dimm12,node=0' \ + -object 'memory-backend-file,id=mem-dimm13,size=1024M,mem-path=/run/hugepages/kvm/1048576kB,share=on,prealloc=yes' \ + -device 'pc-dimm,id=dimm13,memdev=mem-dimm13,node=1' \ + -object 'memory-backend-file,id=mem-dimm14,size=1024M,mem-path=/run/hugepages/kvm/1048576kB,share=on,prealloc=yes' \ + -device 'pc-dimm,id=dimm14,memdev=mem-dimm14,node=0' \ + -object 'memory-backend-file,id=mem-dimm15,size=1024M,mem-path=/run/hugepages/kvm/1048576kB,share=on,prealloc=yes' \ + -device 'pc-dimm,id=dimm15,memdev=mem-dimm15,node=1' \ + -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \ + -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \ + -device 'vmgenid,guid=c773c261-d800-4348-9f5d-167fadd53cf8' \ + -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \ + -device 'usb-tablet,id=tablet,bus=uhci.0,port=1' \ + -device 'VGA,id=vga,bus=pci.0,addr=0x2' \ + -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3' \ + -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ + -machine 'type=pc' diff --git a/test/cfg2cmd/memory-hotplug.conf b/test/cfg2cmd/memory-hotplug.conf new file mode 100644 index 0000000..386e61f --- /dev/null +++ b/test/cfg2cmd/memory-hotplug.conf @@ -0,0 +1,11 @@ +# TEST: basic memory hotplug +# QEMU_VERSION: 3.0 +cores: 2 +memory: 66560 +name: simple +numa: 1 +ostype: l26 +smbios1: uuid=7b10d7af-b932-4c66-b2c3-3996152ec465 +sockets: 2 +vmgenid: c773c261-d800-4348-9f5d-167fadd53cf8 +hotplug: memory diff --git a/test/cfg2cmd/memory-hotplug.conf.cmd b/test/cfg2cmd/memory-hotplug.conf.cmd new file mode 100644 index 0000000..107435d --- /dev/null +++ b/test/cfg2cmd/memory-hotplug.conf.cmd @@ -0,0 +1,174 @@ +/usr/bin/kvm \ + -id 8006 \ + -name 'simple,debug-threads=on' \ + -no-shutdown \ + -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ + -mon 'chardev=qmp,mode=control' \ + -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' \ + -mon 'chardev=qmp-event,mode=control' \ + -pidfile /var/run/qemu-server/8006.pid \ + -daemonize \ + -smbios 'type=1,uuid=7b10d7af-b932-4c66-b2c3-3996152ec465' \ + -smp '4,sockets=2,cores=2,maxcpus=4' \ + -nodefaults \ + -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ + -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ + -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ + -m 'size=1024,slots=255,maxmem=524288M' \ + -object 'memory-backend-ram,id=ram-node0,size=512M' \ + -numa 'node,nodeid=0,cpus=0-1,memdev=ram-node0' \ + -object 'memory-backend-ram,id=ram-node1,size=512M' \ + -numa 'node,nodeid=1,cpus=2-3,memdev=ram-node1' \ + -object 'memory-backend-ram,id=mem-dimm0,size=512M' \ + -device 'pc-dimm,id=dimm0,memdev=mem-dimm0,node=0' \ + -object 'memory-backend-ram,id=mem-dimm1,size=512M' \ + -device 'pc-dimm,id=dimm1,memdev=mem-dimm1,node=1' \ + -object 'memory-backend-ram,id=mem-dimm2,size=512M' \ + -device 'pc-dimm,id=dimm2,memdev=mem-dimm2,node=0' \ + -object 'memory-backend-ram,id=mem-dimm3,size=512M' \ + -device 'pc-dimm,id=dimm3,memdev=mem-dimm3,node=1' \ + -object 'memory-backend-ram,id=mem-dimm4,size=512M' \ + -device 'pc-dimm,id=dimm4,memdev=mem-dimm4,node=0' \ + -object 'memory-backend-ram,id=mem-dimm5,size=512M' \ + -device 'pc-dimm,id=dimm5,memdev=mem-dimm5,node=1' \ + -object 'memory-backend-ram,id=mem-dimm6,size=512M' \ + -device 'pc-dimm,id=dimm6,memdev=mem-dimm6,node=0' \ + -object 'memory-backend-ram,id=mem-dimm7,size=512M' \ + -device 'pc-dimm,id=dimm7,memdev=mem-dimm7,node=1' \ + -object 'memory-backend-ram,id=mem-dimm8,size=512M' \ + -device 'pc-dimm,id=dimm8,memdev=mem-dimm8,node=0' \ + -object 'memory-backend-ram,id=mem-dimm9,size=512M' \ + -device 'pc-dimm,id=dimm9,memdev=mem-dimm9,node=1' \ + -object 'memory-backend-ram,id=mem-dimm10,size=512M' \ + -device 'pc-dimm,id=dimm10,memdev=mem-dimm10,node=0' \ + -object 'memory-backend-ram,id=mem-dimm11,size=512M' \ + -device 'pc-dimm,id=dimm11,memdev=mem-dimm11,node=1' \ + -object 'memory-backend-ram,id=mem-dimm12,size=512M' \ + -device 'pc-dimm,id=dimm12,memdev=mem-dimm12,node=0' \ + -object 'memory-backend-ram,id=mem-dimm13,size=512M' \ + -device 'pc-dimm,id=dimm13,memdev=mem-dimm13,node=1' \ + -object 'memory-backend-ram,id=mem-dimm14,size=512M' \ + -device 'pc-dimm,id=dimm14,memdev=mem-dimm14,node=0' \ + -object 'memory-backend-ram,id=mem-dimm15,size=512M' \ + -device 'pc-dimm,id=dimm15,memdev=mem-dimm15,node=1' \ + -object 'memory-backend-ram,id=mem-dimm16,size=512M' \ + -device 'pc-dimm,id=dimm16,memdev=mem-dimm16,node=0' \ + -object 'memory-backend-ram,id=mem-dimm17,size=512M' \ + -device 'pc-dimm,id=dimm17,memdev=mem-dimm17,node=1' \ + -object 'memory-backend-ram,id=mem-dimm18,size=512M' \ + -device 'pc-dimm,id=dimm18,memdev=mem-dimm18,node=0' \ + -object 'memory-backend-ram,id=mem-dimm19,size=512M' \ + -device 'pc-dimm,id=dimm19,memdev=mem-dimm19,node=1' \ + -object 'memory-backend-ram,id=mem-dimm20,size=512M' \ + -device 'pc-dimm,id=dimm20,memdev=mem-dimm20,node=0' \ + -object 'memory-backend-ram,id=mem-dimm21,size=512M' \ + -device 'pc-dimm,id=dimm21,memdev=mem-dimm21,node=1' \ + -object 'memory-backend-ram,id=mem-dimm22,size=512M' \ + -device 'pc-dimm,id=dimm22,memdev=mem-dimm22,node=0' \ + -object 'memory-backend-ram,id=mem-dimm23,size=512M' \ + -device 'pc-dimm,id=dimm23,memdev=mem-dimm23,node=1' \ + -object 'memory-backend-ram,id=mem-dimm24,size=512M' \ + -device 'pc-dimm,id=dimm24,memdev=mem-dimm24,node=0' \ + -object 'memory-backend-ram,id=mem-dimm25,size=512M' \ + -device 'pc-dimm,id=dimm25,memdev=mem-dimm25,node=1' \ + -object 'memory-backend-ram,id=mem-dimm26,size=512M' \ + -device 'pc-dimm,id=dimm26,memdev=mem-dimm26,node=0' \ + -object 'memory-backend-ram,id=mem-dimm27,size=512M' \ + -device 'pc-dimm,id=dimm27,memdev=mem-dimm27,node=1' \ + -object 'memory-backend-ram,id=mem-dimm28,size=512M' \ + -device 'pc-dimm,id=dimm28,memdev=mem-dimm28,node=0' \ + -object 'memory-backend-ram,id=mem-dimm29,size=512M' \ + -device 'pc-dimm,id=dimm29,memdev=mem-dimm29,node=1' \ + -object 'memory-backend-ram,id=mem-dimm30,size=512M' \ + -device 'pc-dimm,id=dimm30,memdev=mem-dimm30,node=0' \ + -object 'memory-backend-ram,id=mem-dimm31,size=512M' \ + -device 'pc-dimm,id=dimm31,memdev=mem-dimm31,node=1' \ + -object 'memory-backend-ram,id=mem-dimm32,size=1024M' \ + -device 'pc-dimm,id=dimm32,memdev=mem-dimm32,node=0' \ + -object 'memory-backend-ram,id=mem-dimm33,size=1024M' \ + -device 'pc-dimm,id=dimm33,memdev=mem-dimm33,node=1' \ + -object 'memory-backend-ram,id=mem-dimm34,size=1024M' \ + -device 'pc-dimm,id=dimm34,memdev=mem-dimm34,node=0' \ + -object 'memory-backend-ram,id=mem-dimm35,size=1024M' \ + -device 'pc-dimm,id=dimm35,memdev=mem-dimm35,node=1' \ + -object 'memory-backend-ram,id=mem-dimm36,size=1024M' \ + -device 'pc-dimm,id=dimm36,memdev=mem-dimm36,node=0' \ + -object 'memory-backend-ram,id=mem-dimm37,size=1024M' \ + -device 'pc-dimm,id=dimm37,memdev=mem-dimm37,node=1' \ + -object 'memory-backend-ram,id=mem-dimm38,size=1024M' \ + -device 'pc-dimm,id=dimm38,memdev=mem-dimm38,node=0' \ + -object 'memory-backend-ram,id=mem-dimm39,size=1024M' \ + -device 'pc-dimm,id=dimm39,memdev=mem-dimm39,node=1' \ + -object 'memory-backend-ram,id=mem-dimm40,size=1024M' \ + -device 'pc-dimm,id=dimm40,memdev=mem-dimm40,node=0' \ + -object 'memory-backend-ram,id=mem-dimm41,size=1024M' \ + -device 'pc-dimm,id=dimm41,memdev=mem-dimm41,node=1' \ + -object 'memory-backend-ram,id=mem-dimm42,size=1024M' \ + -device 'pc-dimm,id=dimm42,memdev=mem-dimm42,node=0' \ + -object 'memory-backend-ram,id=mem-dimm43,size=1024M' \ + -device 'pc-dimm,id=dimm43,memdev=mem-dimm43,node=1' \ + -object 'memory-backend-ram,id=mem-dimm44,size=1024M' \ + -device 'pc-dimm,id=dimm44,memdev=mem-dimm44,node=0' \ + -object 'memory-backend-ram,id=mem-dimm45,size=1024M' \ + -device 'pc-dimm,id=dimm45,memdev=mem-dimm45,node=1' \ + -object 'memory-backend-ram,id=mem-dimm46,size=1024M' \ + -device 'pc-dimm,id=dimm46,memdev=mem-dimm46,node=0' \ + -object 'memory-backend-ram,id=mem-dimm47,size=1024M' \ + -device 'pc-dimm,id=dimm47,memdev=mem-dimm47,node=1' \ + -object 'memory-backend-ram,id=mem-dimm48,size=1024M' \ + -device 'pc-dimm,id=dimm48,memdev=mem-dimm48,node=0' \ + -object 'memory-backend-ram,id=mem-dimm49,size=1024M' \ + -device 'pc-dimm,id=dimm49,memdev=mem-dimm49,node=1' \ + -object 'memory-backend-ram,id=mem-dimm50,size=1024M' \ + -device 'pc-dimm,id=dimm50,memdev=mem-dimm50,node=0' \ + -object 'memory-backend-ram,id=mem-dimm51,size=1024M' \ + -device 'pc-dimm,id=dimm51,memdev=mem-dimm51,node=1' \ + -object 'memory-backend-ram,id=mem-dimm52,size=1024M' \ + -device 'pc-dimm,id=dimm52,memdev=mem-dimm52,node=0' \ + -object 'memory-backend-ram,id=mem-dimm53,size=1024M' \ + -device 'pc-dimm,id=dimm53,memdev=mem-dimm53,node=1' \ + -object 'memory-backend-ram,id=mem-dimm54,size=1024M' \ + -device 'pc-dimm,id=dimm54,memdev=mem-dimm54,node=0' \ + -object 'memory-backend-ram,id=mem-dimm55,size=1024M' \ + -device 'pc-dimm,id=dimm55,memdev=mem-dimm55,node=1' \ + -object 'memory-backend-ram,id=mem-dimm56,size=1024M' \ + -device 'pc-dimm,id=dimm56,memdev=mem-dimm56,node=0' \ + -object 'memory-backend-ram,id=mem-dimm57,size=1024M' \ + -device 'pc-dimm,id=dimm57,memdev=mem-dimm57,node=1' \ + -object 'memory-backend-ram,id=mem-dimm58,size=1024M' \ + -device 'pc-dimm,id=dimm58,memdev=mem-dimm58,node=0' \ + -object 'memory-backend-ram,id=mem-dimm59,size=1024M' \ + -device 'pc-dimm,id=dimm59,memdev=mem-dimm59,node=1' \ + -object 'memory-backend-ram,id=mem-dimm60,size=1024M' \ + -device 'pc-dimm,id=dimm60,memdev=mem-dimm60,node=0' \ + -object 'memory-backend-ram,id=mem-dimm61,size=1024M' \ + -device 'pc-dimm,id=dimm61,memdev=mem-dimm61,node=1' \ + -object 'memory-backend-ram,id=mem-dimm62,size=1024M' \ + -device 'pc-dimm,id=dimm62,memdev=mem-dimm62,node=0' \ + -object 'memory-backend-ram,id=mem-dimm63,size=1024M' \ + -device 'pc-dimm,id=dimm63,memdev=mem-dimm63,node=1' \ + -object 'memory-backend-ram,id=mem-dimm64,size=2048M' \ + -device 'pc-dimm,id=dimm64,memdev=mem-dimm64,node=0' \ + -object 'memory-backend-ram,id=mem-dimm65,size=2048M' \ + -device 'pc-dimm,id=dimm65,memdev=mem-dimm65,node=1' \ + -object 'memory-backend-ram,id=mem-dimm66,size=2048M' \ + -device 'pc-dimm,id=dimm66,memdev=mem-dimm66,node=0' \ + -object 'memory-backend-ram,id=mem-dimm67,size=2048M' \ + -device 'pc-dimm,id=dimm67,memdev=mem-dimm67,node=1' \ + -object 'memory-backend-ram,id=mem-dimm68,size=2048M' \ + -device 'pc-dimm,id=dimm68,memdev=mem-dimm68,node=0' \ + -object 'memory-backend-ram,id=mem-dimm69,size=2048M' \ + -device 'pc-dimm,id=dimm69,memdev=mem-dimm69,node=1' \ + -object 'memory-backend-ram,id=mem-dimm70,size=2048M' \ + -device 'pc-dimm,id=dimm70,memdev=mem-dimm70,node=0' \ + -object 'memory-backend-ram,id=mem-dimm71,size=2048M' \ + -device 'pc-dimm,id=dimm71,memdev=mem-dimm71,node=1' \ + -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \ + -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \ + -device 'vmgenid,guid=c773c261-d800-4348-9f5d-167fadd53cf8' \ + -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \ + -device 'usb-tablet,id=tablet,bus=uhci.0,port=1' \ + -device 'VGA,id=vga,bus=pci.0,addr=0x2' \ + -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3' \ + -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ + -machine 'type=pc' diff --git a/test/cfg2cmd/memory-hugepages-1g.conf b/test/cfg2cmd/memory-hugepages-1g.conf new file mode 100644 index 0000000..8db2cca --- /dev/null +++ b/test/cfg2cmd/memory-hugepages-1g.conf @@ -0,0 +1,11 @@ +# TEST: memory wih 1gb hugepages +# QEMU_VERSION: 3.0 +cores: 2 +memory: 8192 +name: simple +numa: 1 +ostype: l26 +smbios1: uuid=7b10d7af-b932-4c66-b2c3-3996152ec465 +sockets: 2 +vmgenid: c773c261-d800-4348-9f5d-167fadd53cf8 +hugepages: 1024 \ No newline at end of file diff --git a/test/cfg2cmd/memory-hugepages-1g.conf.cmd b/test/cfg2cmd/memory-hugepages-1g.conf.cmd new file mode 100644 index 0000000..63792d2 --- /dev/null +++ b/test/cfg2cmd/memory-hugepages-1g.conf.cmd @@ -0,0 +1,30 @@ +/usr/bin/kvm \ + -id 8006 \ + -name 'simple,debug-threads=on' \ + -no-shutdown \ + -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ + -mon 'chardev=qmp,mode=control' \ + -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' \ + -mon 'chardev=qmp-event,mode=control' \ + -pidfile /var/run/qemu-server/8006.pid \ + -daemonize \ + -smbios 'type=1,uuid=7b10d7af-b932-4c66-b2c3-3996152ec465' \ + -smp '4,sockets=2,cores=2,maxcpus=4' \ + -nodefaults \ + -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ + -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ + -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ + -m 8192 \ + -object 'memory-backend-file,id=ram-node0,size=4096M,mem-path=/run/hugepages/kvm/1048576kB,share=on,prealloc=yes' \ + -numa 'node,nodeid=0,cpus=0-1,memdev=ram-node0' \ + -object 'memory-backend-file,id=ram-node1,size=4096M,mem-path=/run/hugepages/kvm/1048576kB,share=on,prealloc=yes' \ + -numa 'node,nodeid=1,cpus=2-3,memdev=ram-node1' \ + -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \ + -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \ + -device 'vmgenid,guid=c773c261-d800-4348-9f5d-167fadd53cf8' \ + -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \ + -device 'usb-tablet,id=tablet,bus=uhci.0,port=1' \ + -device 'VGA,id=vga,bus=pci.0,addr=0x2' \ + -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3' \ + -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ + -machine 'type=pc' diff --git a/test/cfg2cmd/memory-hugepages-2m.conf b/test/cfg2cmd/memory-hugepages-2m.conf new file mode 100644 index 0000000..f0d65fb --- /dev/null +++ b/test/cfg2cmd/memory-hugepages-2m.conf @@ -0,0 +1,11 @@ +# TEST: memory wih 2mb hugepages +# QEMU_VERSION: 3.0 +cores: 2 +memory: 8192 +name: simple +numa: 1 +ostype: l26 +smbios1: uuid=7b10d7af-b932-4c66-b2c3-3996152ec465 +sockets: 2 +vmgenid: c773c261-d800-4348-9f5d-167fadd53cf8 +hugepages: 2 \ No newline at end of file diff --git a/test/cfg2cmd/memory-hugepages-2m.conf.cmd b/test/cfg2cmd/memory-hugepages-2m.conf.cmd new file mode 100644 index 0000000..287c0ed --- /dev/null +++ b/test/cfg2cmd/memory-hugepages-2m.conf.cmd @@ -0,0 +1,30 @@ +/usr/bin/kvm \ + -id 8006 \ + -name 'simple,debug-threads=on' \ + -no-shutdown \ + -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ + -mon 'chardev=qmp,mode=control' \ + -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' \ + -mon 'chardev=qmp-event,mode=control' \ + -pidfile /var/run/qemu-server/8006.pid \ + -daemonize \ + -smbios 'type=1,uuid=7b10d7af-b932-4c66-b2c3-3996152ec465' \ + -smp '4,sockets=2,cores=2,maxcpus=4' \ + -nodefaults \ + -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ + -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ + -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ + -m 8192 \ + -object 'memory-backend-file,id=ram-node0,size=4096M,mem-path=/run/hugepages/kvm/2048kB,share=on,prealloc=yes' \ + -numa 'node,nodeid=0,cpus=0-1,memdev=ram-node0' \ + -object 'memory-backend-file,id=ram-node1,size=4096M,mem-path=/run/hugepages/kvm/2048kB,share=on,prealloc=yes' \ + -numa 'node,nodeid=1,cpus=2-3,memdev=ram-node1' \ + -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \ + -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \ + -device 'vmgenid,guid=c773c261-d800-4348-9f5d-167fadd53cf8' \ + -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \ + -device 'usb-tablet,id=tablet,bus=uhci.0,port=1' \ + -device 'VGA,id=vga,bus=pci.0,addr=0x2' \ + -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3' \ + -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ + -machine 'type=pc' diff --git a/test/cfg2cmd/minimal-defaults.conf.cmd b/test/cfg2cmd/minimal-defaults.conf.cmd index 0735f43..8da69fe 100644 --- a/test/cfg2cmd/minimal-defaults.conf.cmd +++ b/test/cfg2cmd/minimal-defaults.conf.cmd @@ -1,7 +1,8 @@ /usr/bin/kvm \ -id 8006 \ - -name vm8006 \ - -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server,nowait' \ + -name 'vm8006,debug-threads=on' \ + -no-shutdown \ + -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ -mon 'chardev=qmp,mode=control' \ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' \ -mon 'chardev=qmp-event,mode=control' \ @@ -11,7 +12,7 @@ -smp '1,sockets=1,cores=1,maxcpus=1' \ -nodefaults \ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ - -vnc unix:/var/run/qemu-server/8006.vnc,password \ + -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ -m 512 \ -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \ @@ -19,6 +20,6 @@ -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \ -device 'usb-tablet,id=tablet,bus=uhci.0,port=1' \ -device 'VGA,id=vga,bus=pci.0,addr=0x2' \ - -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3' \ + -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ -machine 'type=pc+pve0' diff --git a/test/cfg2cmd/netdev-7.0-multiqueues.conf b/test/cfg2cmd/netdev-7.0-multiqueues.conf new file mode 100644 index 0000000..342ad88 --- /dev/null +++ b/test/cfg2cmd/netdev-7.0-multiqueues.conf @@ -0,0 +1,9 @@ +# TEST: Simple test for netdev multi queue on 7.0 machine version +# QEMU_VERSION: 7.0 +bootdisk: scsi0 +cores: 3 +memory: 768 +name: netdev-multiq +net0: virtio=A2:C0:43:77:08:A0,bridge=vmbr0,mtu=900,queues=2 +ostype: l26 +smbios1: uuid=7b10d7af-b932-4c66-b2c3-3996152ec465 diff --git a/test/cfg2cmd/netdev-7.0-multiqueues.conf.cmd b/test/cfg2cmd/netdev-7.0-multiqueues.conf.cmd new file mode 100644 index 0000000..6892de3 --- /dev/null +++ b/test/cfg2cmd/netdev-7.0-multiqueues.conf.cmd @@ -0,0 +1,27 @@ +/usr/bin/kvm \ + -id 8006 \ + -name 'netdev-multiq,debug-threads=on' \ + -no-shutdown \ + -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ + -mon 'chardev=qmp,mode=control' \ + -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' \ + -mon 'chardev=qmp-event,mode=control' \ + -pidfile /var/run/qemu-server/8006.pid \ + -daemonize \ + -smbios 'type=1,uuid=7b10d7af-b932-4c66-b2c3-3996152ec465' \ + -smp '3,sockets=1,cores=3,maxcpus=3' \ + -nodefaults \ + -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ + -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ + -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ + -m 768 \ + -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \ + -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \ + -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \ + -device 'usb-tablet,id=tablet,bus=uhci.0,port=1' \ + -device 'VGA,id=vga,bus=pci.0,addr=0x2' \ + -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \ + -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ + -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/var/lib/qemu-server/pve-bridge,downscript=/var/lib/qemu-server/pve-bridgedown,vhost=on,queues=2' \ + -device 'virtio-net-pci,mac=A2:C0:43:77:08:A0,netdev=net0,bus=pci.0,addr=0x12,id=net0,vectors=6,mq=on,bootindex=300,host_mtu=900' \ + -machine 'type=pc+pve0' diff --git a/test/cfg2cmd/netdev-7.1-multiqueues.conf b/test/cfg2cmd/netdev-7.1-multiqueues.conf new file mode 100644 index 0000000..da5f111 --- /dev/null +++ b/test/cfg2cmd/netdev-7.1-multiqueues.conf @@ -0,0 +1,8 @@ +# TEST: Simple test for netdev related stuff +# QEMU_VERSION: 7.1 +bootdisk: scsi0 +cores: 3 +memory: 768 +name: netdev +net0: virtio=A2:C0:43:77:08:A0,bridge=vmbr0,mtu=900,queues=2 +ostype: l26 diff --git a/test/cfg2cmd/netdev-7.1-multiqueues.conf.cmd b/test/cfg2cmd/netdev-7.1-multiqueues.conf.cmd new file mode 100644 index 0000000..2c6c905 --- /dev/null +++ b/test/cfg2cmd/netdev-7.1-multiqueues.conf.cmd @@ -0,0 +1,26 @@ +/usr/bin/kvm \ + -id 8006 \ + -name 'netdev,debug-threads=on' \ + -no-shutdown \ + -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ + -mon 'chardev=qmp,mode=control' \ + -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' \ + -mon 'chardev=qmp-event,mode=control' \ + -pidfile /var/run/qemu-server/8006.pid \ + -daemonize \ + -smp '3,sockets=1,cores=3,maxcpus=3' \ + -nodefaults \ + -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ + -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ + -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ + -m 768 \ + -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \ + -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \ + -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \ + -device 'usb-tablet,id=tablet,bus=uhci.0,port=1' \ + -device 'VGA,id=vga,bus=pci.0,addr=0x2' \ + -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \ + -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ + -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/var/lib/qemu-server/pve-bridge,downscript=/var/lib/qemu-server/pve-bridgedown,vhost=on,queues=2' \ + -device 'virtio-net-pci,mac=A2:C0:43:77:08:A0,netdev=net0,bus=pci.0,addr=0x12,id=net0,vectors=6,mq=on,packed=on,rx_queue_size=1024,tx_queue_size=256,bootindex=300,host_mtu=900' \ + -machine 'type=pc+pve0' diff --git a/test/cfg2cmd/netdev-7.1.conf b/test/cfg2cmd/netdev-7.1.conf new file mode 100644 index 0000000..82be056 --- /dev/null +++ b/test/cfg2cmd/netdev-7.1.conf @@ -0,0 +1,8 @@ +# TEST: Simple test for netdev related stuff +# QEMU_VERSION: 7.1 +bootdisk: scsi0 +cores: 3 +memory: 768 +name: netdev +net0: virtio=A2:C0:43:77:08:A0,bridge=vmbr0,mtu=900 +ostype: l26 diff --git a/test/cfg2cmd/netdev-7.1.conf.cmd b/test/cfg2cmd/netdev-7.1.conf.cmd new file mode 100644 index 0000000..6ffa971 --- /dev/null +++ b/test/cfg2cmd/netdev-7.1.conf.cmd @@ -0,0 +1,26 @@ +/usr/bin/kvm \ + -id 8006 \ + -name 'netdev,debug-threads=on' \ + -no-shutdown \ + -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ + -mon 'chardev=qmp,mode=control' \ + -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' \ + -mon 'chardev=qmp-event,mode=control' \ + -pidfile /var/run/qemu-server/8006.pid \ + -daemonize \ + -smp '3,sockets=1,cores=3,maxcpus=3' \ + -nodefaults \ + -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ + -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ + -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ + -m 768 \ + -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \ + -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \ + -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \ + -device 'usb-tablet,id=tablet,bus=uhci.0,port=1' \ + -device 'VGA,id=vga,bus=pci.0,addr=0x2' \ + -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \ + -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ + -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/var/lib/qemu-server/pve-bridge,downscript=/var/lib/qemu-server/pve-bridgedown,vhost=on' \ + -device 'virtio-net-pci,mac=A2:C0:43:77:08:A0,netdev=net0,bus=pci.0,addr=0x12,id=net0,rx_queue_size=1024,tx_queue_size=256,bootindex=300,host_mtu=900' \ + -machine 'type=pc+pve0' diff --git a/test/cfg2cmd/netdev.conf.cmd b/test/cfg2cmd/netdev.conf.cmd index 4294fa0..3ae6075 100644 --- a/test/cfg2cmd/netdev.conf.cmd +++ b/test/cfg2cmd/netdev.conf.cmd @@ -1,7 +1,8 @@ /usr/bin/kvm \ -id 8006 \ - -name netdev \ - -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server,nowait' \ + -name 'netdev,debug-threads=on' \ + -no-shutdown \ + -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ -mon 'chardev=qmp,mode=control' \ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' \ -mon 'chardev=qmp-event,mode=control' \ @@ -10,7 +11,7 @@ -smp '3,sockets=1,cores=3,maxcpus=3' \ -nodefaults \ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ - -vnc unix:/var/run/qemu-server/8006.vnc,password \ + -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ -m 768 \ -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \ diff --git a/test/cfg2cmd/ostype-usb13-error.conf b/test/cfg2cmd/ostype-usb13-error.conf new file mode 100644 index 0000000..5c75b62 --- /dev/null +++ b/test/cfg2cmd/ostype-usb13-error.conf @@ -0,0 +1,13 @@ +# TEST: Test error for old ostype type with newer usb config +# QEMU_VERSION: 7.1.0 +# EXPECT_ERROR: using usb13 is only possible with machine type >= 7.1 and ostype l26 or windows > 7 +cores: 2 +memory: 768 +name: q35-usb3-error +net0: virtio=A2:C0:43:77:08:A1,bridge=vmbr0 +ostype: w2k +scsihw: virtio-scsi-pci +smbios1: uuid=7b10d7af-b932-4c66-b2c3-3996152ec465 +vmgenid: c773c261-d800-4348-9f5d-167fadd53cf8 +vga: qxl +usb13: spice diff --git a/test/cfg2cmd/pinned-version-pxe-pve.conf b/test/cfg2cmd/pinned-version-pxe-pve.conf new file mode 100644 index 0000000..36169d7 --- /dev/null +++ b/test/cfg2cmd/pinned-version-pxe-pve.conf @@ -0,0 +1,17 @@ +# TEST: for a basic configuration with a .pxe machine and +pve pinned +bootdisk: scsi0 +cores: 3 +ide2: none,media=cdrom +machine: pc-q35-4.1+pve2.pxe +memory: 1024 +name: pinned +net0: virtio=A2:C0:43:77:08:A1,bridge=vmbr0 +numa: 0 +ostype: l26 +scsi0: local:8006/vm-8006-disk-0.raw,discard=on,size=104858K +scsihw: virtio-scsi-pci +smbios1: uuid=c7fdd046-fefc-11e9-832e-770e1d5636a0 +sockets: 1 +vmgenid: bdd46b98-fefc-11e9-97b4-d72c378e0f96 +# add rng0 to stress +pve2 version requirement +rng0: source=/dev/urandom diff --git a/test/cfg2cmd/pinned-version-pxe-pve.conf.cmd b/test/cfg2cmd/pinned-version-pxe-pve.conf.cmd new file mode 100644 index 0000000..d17d4de --- /dev/null +++ b/test/cfg2cmd/pinned-version-pxe-pve.conf.cmd @@ -0,0 +1,33 @@ +/usr/bin/kvm \ + -id 8006 \ + -name 'pinned,debug-threads=on' \ + -no-shutdown \ + -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ + -mon 'chardev=qmp,mode=control' \ + -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' \ + -mon 'chardev=qmp-event,mode=control' \ + -pidfile /var/run/qemu-server/8006.pid \ + -daemonize \ + -smbios 'type=1,uuid=c7fdd046-fefc-11e9-832e-770e1d5636a0' \ + -smp '3,sockets=1,cores=3,maxcpus=3' \ + -nodefaults \ + -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ + -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ + -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ + -m 1024 \ + -readconfig /usr/share/qemu-server/pve-q35-4.0.cfg \ + -device 'vmgenid,guid=bdd46b98-fefc-11e9-97b4-d72c378e0f96' \ + -device 'usb-tablet,id=tablet,bus=ehci.0,port=1' \ + -device 'VGA,id=vga,bus=pcie.0,addr=0x1' \ + -object 'rng-random,filename=/dev/urandom,id=rng0' \ + -device 'virtio-rng-pci,rng=rng0,max-bytes=1024,period=1000,bus=pci.1,addr=0x1d' \ + -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3' \ + -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ + -drive 'if=none,id=drive-ide2,media=cdrom,aio=io_uring' \ + -device 'ide-cd,bus=ide.1,unit=0,drive=drive-ide2,id=ide2,bootindex=200' \ + -device 'virtio-scsi-pci,id=scsihw0,bus=pci.0,addr=0x5' \ + -drive 'file=/var/lib/vz/images/8006/vm-8006-disk-0.raw,if=none,id=drive-scsi0,discard=on,format=raw,cache=none,aio=io_uring,detect-zeroes=unmap' \ + -device 'scsi-hd,bus=scsihw0.0,channel=0,scsi-id=0,lun=0,drive=drive-scsi0,id=scsi0,bootindex=100' \ + -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/var/lib/qemu-server/pve-bridge,downscript=/var/lib/qemu-server/pve-bridgedown,vhost=on' \ + -device 'virtio-net-pci,mac=A2:C0:43:77:08:A1,netdev=net0,bus=pci.0,addr=0x12,id=net0,bootindex=300,romfile=pxe-virtio.rom' \ + -machine 'type=pc-q35-4.1+pve2' diff --git a/test/cfg2cmd/pinned-version-pxe.conf b/test/cfg2cmd/pinned-version-pxe.conf new file mode 100644 index 0000000..738868f --- /dev/null +++ b/test/cfg2cmd/pinned-version-pxe.conf @@ -0,0 +1,15 @@ +# TEST: for a basic configuration with a .pxe machine +bootdisk: scsi0 +cores: 3 +ide2: none,media=cdrom +machine: pc-q35-5.1.pxe +memory: 1024 +name: pinned +net0: virtio=A2:C0:43:77:08:A1,bridge=vmbr0 +numa: 0 +ostype: l26 +scsi0: local:8006/vm-8006-disk-0.raw,discard=on,size=104858K +scsihw: virtio-scsi-pci +smbios1: uuid=c7fdd046-fefc-11e9-832e-770e1d5636a0 +sockets: 1 +vmgenid: bdd46b98-fefc-11e9-97b4-d72c378e0f96 diff --git a/test/cfg2cmd/pinned-version-pxe.conf.cmd b/test/cfg2cmd/pinned-version-pxe.conf.cmd new file mode 100644 index 0000000..892fc14 --- /dev/null +++ b/test/cfg2cmd/pinned-version-pxe.conf.cmd @@ -0,0 +1,31 @@ +/usr/bin/kvm \ + -id 8006 \ + -name 'pinned,debug-threads=on' \ + -no-shutdown \ + -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ + -mon 'chardev=qmp,mode=control' \ + -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' \ + -mon 'chardev=qmp-event,mode=control' \ + -pidfile /var/run/qemu-server/8006.pid \ + -daemonize \ + -smbios 'type=1,uuid=c7fdd046-fefc-11e9-832e-770e1d5636a0' \ + -smp '3,sockets=1,cores=3,maxcpus=3' \ + -nodefaults \ + -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ + -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ + -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ + -m 1024 \ + -readconfig /usr/share/qemu-server/pve-q35-4.0.cfg \ + -device 'vmgenid,guid=bdd46b98-fefc-11e9-97b4-d72c378e0f96' \ + -device 'usb-tablet,id=tablet,bus=ehci.0,port=1' \ + -device 'VGA,id=vga,bus=pcie.0,addr=0x1' \ + -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3' \ + -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ + -drive 'if=none,id=drive-ide2,media=cdrom,aio=io_uring' \ + -device 'ide-cd,bus=ide.1,unit=0,drive=drive-ide2,id=ide2,bootindex=200' \ + -device 'virtio-scsi-pci,id=scsihw0,bus=pci.0,addr=0x5' \ + -drive 'file=/var/lib/vz/images/8006/vm-8006-disk-0.raw,if=none,id=drive-scsi0,discard=on,format=raw,cache=none,aio=io_uring,detect-zeroes=unmap' \ + -device 'scsi-hd,bus=scsihw0.0,channel=0,scsi-id=0,lun=0,drive=drive-scsi0,id=scsi0,bootindex=100' \ + -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/var/lib/qemu-server/pve-bridge,downscript=/var/lib/qemu-server/pve-bridgedown,vhost=on' \ + -device 'virtio-net-pci,mac=A2:C0:43:77:08:A1,netdev=net0,bus=pci.0,addr=0x12,id=net0,bootindex=300,romfile=pxe-virtio.rom' \ + -machine 'type=pc-q35-5.1+pve0' diff --git a/test/cfg2cmd/pinned-version.conf.cmd b/test/cfg2cmd/pinned-version.conf.cmd index a7d0ae2..13361ed 100644 --- a/test/cfg2cmd/pinned-version.conf.cmd +++ b/test/cfg2cmd/pinned-version.conf.cmd @@ -1,7 +1,8 @@ /usr/bin/kvm \ -id 8006 \ - -name pinned \ - -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server,nowait' \ + -name 'pinned,debug-threads=on' \ + -no-shutdown \ + -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ -mon 'chardev=qmp,mode=control' \ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' \ -mon 'chardev=qmp-event,mode=control' \ @@ -11,7 +12,7 @@ -smp '3,sockets=1,cores=3,maxcpus=3' \ -nodefaults \ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ - -vnc unix:/var/run/qemu-server/8006.vnc,password \ + -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ -m 1024 \ -readconfig /usr/share/qemu-server/pve-q35.cfg \ @@ -20,10 +21,10 @@ -device 'VGA,id=vga,bus=pcie.0,addr=0x1' \ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3' \ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ - -drive 'if=none,id=drive-ide2,media=cdrom,aio=threads' \ + -drive 'if=none,id=drive-ide2,media=cdrom,aio=io_uring' \ -device 'ide-cd,bus=ide.1,unit=0,drive=drive-ide2,id=ide2,bootindex=200' \ -device 'virtio-scsi-pci,id=scsihw0,bus=pci.0,addr=0x5' \ - -drive 'file=/var/lib/vz/images/8006/vm-8006-disk-0.raw,if=none,id=drive-scsi0,discard=on,format=raw,cache=none,aio=native,detect-zeroes=unmap' \ + -drive 'file=/var/lib/vz/images/8006/vm-8006-disk-0.raw,if=none,id=drive-scsi0,discard=on,format=raw,cache=none,aio=io_uring,detect-zeroes=unmap' \ -device 'scsi-hd,bus=scsihw0.0,channel=0,scsi-id=0,lun=0,drive=drive-scsi0,id=scsi0,bootindex=100' \ -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/var/lib/qemu-server/pve-bridge,downscript=/var/lib/qemu-server/pve-bridgedown,vhost=on' \ -device 'virtio-net-pci,mac=A2:C0:43:77:08:A1,netdev=net0,bus=pci.0,addr=0x12,id=net0,bootindex=300' \ diff --git a/test/cfg2cmd/q35-ide.conf b/test/cfg2cmd/q35-ide.conf new file mode 100644 index 0000000..bfef0a1 --- /dev/null +++ b/test/cfg2cmd/q35-ide.conf @@ -0,0 +1,15 @@ +# TEST: Config with q35, Linux & four IDE CD-ROMs +bootdisk: scsi0 +cores: 2 +ide0: cifs-store:iso/zero.iso,media=cdrom,size=112M +ide1: cifs-store:iso/one.iso,media=cdrom,size=112M +ide2: cifs-store:iso/two.iso,media=cdrom,size=112M +ide3: cifs-store:iso/three.iso,media=cdrom,size=112M +machine: q35 +memory: 512 +net0: virtio=2E:01:68:F9:9C:87,bridge=vmbr0 +ostype: l26 +scsi0: local:100/vm-100-disk-2.qcow2,size=10G +scsihw: virtio-scsi-pci +smbios1: uuid=3dd750ce-d910-44d0-9493-525c0be4e687 +vmgenid: 54d1c06c-8f5b-440f-b5b2-6eab1380e13d diff --git a/test/cfg2cmd/q35-ide.conf.cmd b/test/cfg2cmd/q35-ide.conf.cmd new file mode 100644 index 0000000..20ccc98 --- /dev/null +++ b/test/cfg2cmd/q35-ide.conf.cmd @@ -0,0 +1,38 @@ +/usr/bin/kvm \ + -id 8006 \ + -name 'vm8006,debug-threads=on' \ + -no-shutdown \ + -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ + -mon 'chardev=qmp,mode=control' \ + -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' \ + -mon 'chardev=qmp-event,mode=control' \ + -pidfile /var/run/qemu-server/8006.pid \ + -daemonize \ + -smbios 'type=1,uuid=3dd750ce-d910-44d0-9493-525c0be4e687' \ + -global 'ICH9-LPC.acpi-pci-hotplug-with-bridge-support=off' \ + -smp '2,sockets=1,cores=2,maxcpus=2' \ + -nodefaults \ + -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ + -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ + -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ + -m 512 \ + -readconfig /usr/share/qemu-server/pve-q35-4.0.cfg \ + -device 'vmgenid,guid=54d1c06c-8f5b-440f-b5b2-6eab1380e13d' \ + -device 'usb-tablet,id=tablet,bus=ehci.0,port=1' \ + -device 'VGA,id=vga,bus=pcie.0,addr=0x1' \ + -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \ + -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ + -drive 'file=/mnt/pve/cifs-store/template/iso/zero.iso,if=none,id=drive-ide0,media=cdrom,aio=threads' \ + -device 'ide-cd,bus=ide.0,unit=0,drive=drive-ide0,id=ide0,bootindex=200' \ + -drive 'file=/mnt/pve/cifs-store/template/iso/one.iso,if=none,id=drive-ide1,media=cdrom,aio=threads' \ + -device 'ide-cd,bus=ide.2,unit=0,drive=drive-ide1,id=ide1,bootindex=201' \ + -drive 'file=/mnt/pve/cifs-store/template/iso/two.iso,if=none,id=drive-ide2,media=cdrom,aio=threads' \ + -device 'ide-cd,bus=ide.1,unit=0,drive=drive-ide2,id=ide2,bootindex=202' \ + -drive 'file=/mnt/pve/cifs-store/template/iso/three.iso,if=none,id=drive-ide3,media=cdrom,aio=threads' \ + -device 'ide-cd,bus=ide.3,unit=0,drive=drive-ide3,id=ide3,bootindex=203' \ + -device 'virtio-scsi-pci,id=scsihw0,bus=pci.0,addr=0x5' \ + -drive 'file=/var/lib/vz/images/100/vm-100-disk-2.qcow2,if=none,id=drive-scsi0,format=qcow2,cache=none,aio=io_uring,detect-zeroes=on' \ + -device 'scsi-hd,bus=scsihw0.0,channel=0,scsi-id=0,lun=0,drive=drive-scsi0,id=scsi0,bootindex=100' \ + -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/var/lib/qemu-server/pve-bridge,downscript=/var/lib/qemu-server/pve-bridgedown,vhost=on' \ + -device 'virtio-net-pci,mac=2E:01:68:F9:9C:87,netdev=net0,bus=pci.0,addr=0x12,id=net0,rx_queue_size=1024,tx_queue_size=256,bootindex=300' \ + -machine 'type=q35+pve0' diff --git a/test/cfg2cmd/q35-linux-hostpci-mapping.conf b/test/cfg2cmd/q35-linux-hostpci-mapping.conf new file mode 100644 index 0000000..2366fc4 --- /dev/null +++ b/test/cfg2cmd/q35-linux-hostpci-mapping.conf @@ -0,0 +1,17 @@ +# TEST: Config with q35, NUMA, hostpci mapping passthrough, EFI & Linux +bios: ovmf +bootdisk: scsi0 +cores: 1 +efidisk0: local:100/vm-100-disk-1.qcow2,size=128K +hostpci0: mapping=someNic +hostpci1: mapping=someGpu,mdev=some-model +hostpci2: mapping=someNic +machine: q35 +memory: 512 +net0: virtio=2E:01:68:F9:9C:87,bridge=vmbr0 +numa: 1 +ostype: l26 +scsihw: virtio-scsi-pci +smbios1: uuid=3dd750ce-d910-44d0-9493-525c0be4e687 +sockets: 2 +vmgenid: 54d1c06c-8f5b-440f-b5b2-6eab1380e13d diff --git a/test/cfg2cmd/q35-linux-hostpci-mapping.conf.cmd b/test/cfg2cmd/q35-linux-hostpci-mapping.conf.cmd new file mode 100644 index 0000000..bc48c5a --- /dev/null +++ b/test/cfg2cmd/q35-linux-hostpci-mapping.conf.cmd @@ -0,0 +1,36 @@ +/usr/bin/kvm \ + -id 8006 \ + -name 'vm8006,debug-threads=on' \ + -no-shutdown \ + -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ + -mon 'chardev=qmp,mode=control' \ + -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' \ + -mon 'chardev=qmp-event,mode=control' \ + -pidfile /var/run/qemu-server/8006.pid \ + -daemonize \ + -smbios 'type=1,uuid=3dd750ce-d910-44d0-9493-525c0be4e687' \ + -drive 'if=pflash,unit=0,format=raw,readonly=on,file=/usr/share/pve-edk2-firmware//OVMF_CODE.fd' \ + -drive 'if=pflash,unit=1,id=drive-efidisk0,format=qcow2,file=/var/lib/vz/images/100/vm-100-disk-1.qcow2' \ + -global 'ICH9-LPC.acpi-pci-hotplug-with-bridge-support=off' \ + -smp '2,sockets=2,cores=1,maxcpus=2' \ + -nodefaults \ + -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ + -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ + -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ + -m 512 \ + -object 'memory-backend-ram,id=ram-node0,size=256M' \ + -numa 'node,nodeid=0,cpus=0,memdev=ram-node0' \ + -object 'memory-backend-ram,id=ram-node1,size=256M' \ + -numa 'node,nodeid=1,cpus=1,memdev=ram-node1' \ + -readconfig /usr/share/qemu-server/pve-q35-4.0.cfg \ + -device 'vmgenid,guid=54d1c06c-8f5b-440f-b5b2-6eab1380e13d' \ + -device 'usb-tablet,id=tablet,bus=ehci.0,port=1' \ + -device 'vfio-pci,host=0000:07:10.0,id=hostpci0,bus=pci.0,addr=0x10' \ + -device 'vfio-pci,sysfsdev=/sys/bus/mdev/devices/00000001-0000-0000-0000-000000008006,id=hostpci1,bus=pci.0,addr=0x11' \ + -device 'vfio-pci,host=0000:07:10.4,id=hostpci2,bus=pci.0,addr=0x1b' \ + -device 'VGA,id=vga,bus=pcie.0,addr=0x1' \ + -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \ + -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ + -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/var/lib/qemu-server/pve-bridge,downscript=/var/lib/qemu-server/pve-bridgedown,vhost=on' \ + -device 'virtio-net-pci,mac=2E:01:68:F9:9C:87,netdev=net0,bus=pci.0,addr=0x12,id=net0,rx_queue_size=1024,tx_queue_size=256,bootindex=300' \ + -machine 'type=q35+pve0' diff --git a/test/cfg2cmd/q35-linux-hostpci-multifunction.conf.cmd b/test/cfg2cmd/q35-linux-hostpci-multifunction.conf.cmd index a008939..0b1d85a 100644 --- a/test/cfg2cmd/q35-linux-hostpci-multifunction.conf.cmd +++ b/test/cfg2cmd/q35-linux-hostpci-multifunction.conf.cmd @@ -1,19 +1,21 @@ /usr/bin/kvm \ -id 8006 \ - -name vm8006 \ - -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server,nowait' \ + -name 'vm8006,debug-threads=on' \ + -no-shutdown \ + -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ -mon 'chardev=qmp,mode=control' \ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' \ -mon 'chardev=qmp-event,mode=control' \ -pidfile /var/run/qemu-server/8006.pid \ -daemonize \ -smbios 'type=1,uuid=3dd750ce-d910-44d0-9493-525c0be4e687' \ - -drive 'if=pflash,unit=0,format=raw,readonly,file=/usr/share/pve-edk2-firmware//OVMF_CODE.fd' \ - -drive 'if=pflash,unit=1,format=qcow2,id=drive-efidisk0,file=/var/lib/vz/images/100/vm-100-disk-1.qcow2' \ + -drive 'if=pflash,unit=0,format=raw,readonly=on,file=/usr/share/pve-edk2-firmware//OVMF_CODE.fd' \ + -drive 'if=pflash,unit=1,id=drive-efidisk0,format=qcow2,file=/var/lib/vz/images/100/vm-100-disk-1.qcow2' \ + -global 'ICH9-LPC.acpi-pci-hotplug-with-bridge-support=off' \ -smp '2,sockets=2,cores=1,maxcpus=2' \ -nodefaults \ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ - -vnc unix:/var/run/qemu-server/8006.vnc,password \ + -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ -m 512 \ -object 'memory-backend-ram,id=ram-node0,size=256M' \ @@ -27,8 +29,8 @@ -device 'vfio-pci,host=0000:f0:43.1,id=hostpci0.1,bus=pci.0,addr=0x10.1' \ -device 'vfio-pci,host=1234:f0:43.1,id=hostpci1,bus=pci.0,addr=0x11' \ -device 'VGA,id=vga,bus=pcie.0,addr=0x1' \ - -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3' \ + -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/var/lib/qemu-server/pve-bridge,downscript=/var/lib/qemu-server/pve-bridgedown,vhost=on' \ - -device 'virtio-net-pci,mac=2E:01:68:F9:9C:87,netdev=net0,bus=pci.0,addr=0x12,id=net0,bootindex=300' \ + -device 'virtio-net-pci,mac=2E:01:68:F9:9C:87,netdev=net0,bus=pci.0,addr=0x12,id=net0,rx_queue_size=1024,tx_queue_size=256,bootindex=300' \ -machine 'type=q35+pve0' diff --git a/test/cfg2cmd/q35-linux-hostpci-x-pci-overrides.conf b/test/cfg2cmd/q35-linux-hostpci-x-pci-overrides.conf new file mode 100644 index 0000000..b726a3a --- /dev/null +++ b/test/cfg2cmd/q35-linux-hostpci-x-pci-overrides.conf @@ -0,0 +1,16 @@ +# TEST: Overriding PCI vendor/device IDs reported to guest +bios: ovmf +bootdisk: scsi0 +cores: 1 +efidisk0: local:100/vm-100-disk-1.qcow2,size=128K +hostpci0: 00:ff.1,vendor-id=0x1234,device-id=0x5678,sub-vendor-id=0x2233,sub-device-id=0x0000 +hostpci1: d0:13.0,pcie=1,vendor-id=0x1234,device-id=0x5678 +machine: q35 +memory: 512 +net0: virtio=2E:01:68:F9:9C:87,bridge=vmbr0 +numa: 1 +ostype: l26 +scsihw: virtio-scsi-pci +smbios1: uuid=3dd750ce-d910-44d0-9493-525c0be4e687 +sockets: 2 +vmgenid: 54d1c06c-8f5b-440f-b5b2-6eab1380e13d diff --git a/test/cfg2cmd/q35-linux-hostpci-x-pci-overrides.conf.cmd b/test/cfg2cmd/q35-linux-hostpci-x-pci-overrides.conf.cmd new file mode 100644 index 0000000..c7698d1 --- /dev/null +++ b/test/cfg2cmd/q35-linux-hostpci-x-pci-overrides.conf.cmd @@ -0,0 +1,35 @@ +/usr/bin/kvm \ + -id 8006 \ + -name 'vm8006,debug-threads=on' \ + -no-shutdown \ + -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ + -mon 'chardev=qmp,mode=control' \ + -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' \ + -mon 'chardev=qmp-event,mode=control' \ + -pidfile /var/run/qemu-server/8006.pid \ + -daemonize \ + -smbios 'type=1,uuid=3dd750ce-d910-44d0-9493-525c0be4e687' \ + -drive 'if=pflash,unit=0,format=raw,readonly=on,file=/usr/share/pve-edk2-firmware//OVMF_CODE.fd' \ + -drive 'if=pflash,unit=1,id=drive-efidisk0,format=qcow2,file=/var/lib/vz/images/100/vm-100-disk-1.qcow2' \ + -global 'ICH9-LPC.acpi-pci-hotplug-with-bridge-support=off' \ + -smp '2,sockets=2,cores=1,maxcpus=2' \ + -nodefaults \ + -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ + -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ + -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ + -m 512 \ + -object 'memory-backend-ram,id=ram-node0,size=256M' \ + -numa 'node,nodeid=0,cpus=0,memdev=ram-node0' \ + -object 'memory-backend-ram,id=ram-node1,size=256M' \ + -numa 'node,nodeid=1,cpus=1,memdev=ram-node1' \ + -readconfig /usr/share/qemu-server/pve-q35-4.0.cfg \ + -device 'vmgenid,guid=54d1c06c-8f5b-440f-b5b2-6eab1380e13d' \ + -device 'usb-tablet,id=tablet,bus=ehci.0,port=1' \ + -device 'vfio-pci,host=0000:00:ff.1,id=hostpci0,bus=pci.0,addr=0x10,x-pci-vendor-id=0x1234,x-pci-device-id=0x5678,x-pci-sub-vendor-id=0x2233,x-pci-sub-device-id=0x0000' \ + -device 'vfio-pci,host=0000:d0:13.0,id=hostpci1,bus=ich9-pcie-port-2,addr=0x0,x-pci-vendor-id=0x1234,x-pci-device-id=0x5678' \ + -device 'VGA,id=vga,bus=pcie.0,addr=0x1' \ + -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \ + -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ + -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/var/lib/qemu-server/pve-bridge,downscript=/var/lib/qemu-server/pve-bridgedown,vhost=on' \ + -device 'virtio-net-pci,mac=2E:01:68:F9:9C:87,netdev=net0,bus=pci.0,addr=0x12,id=net0,rx_queue_size=1024,tx_queue_size=256,bootindex=300' \ + -machine 'type=q35+pve0' diff --git a/test/cfg2cmd/q35-linux-hostpci.conf b/test/cfg2cmd/q35-linux-hostpci.conf index 749f983..7290120 100644 --- a/test/cfg2cmd/q35-linux-hostpci.conf +++ b/test/cfg2cmd/q35-linux-hostpci.conf @@ -8,7 +8,7 @@ hostpci1: d0:13.0,pcie=1 hostpci2: 00:f4.0 hostpci3: d0:15.1,pcie=1 hostpci4: d0:17.0,pcie=1,rombar=0 -hostpci7: d0:15.1,pcie=1 +hostpci7: d0:15.2,pcie=1 machine: q35 memory: 512 net0: virtio=2E:01:68:F9:9C:87,bridge=vmbr0 diff --git a/test/cfg2cmd/q35-linux-hostpci.conf.cmd b/test/cfg2cmd/q35-linux-hostpci.conf.cmd index 7e829ae..5289ec6 100644 --- a/test/cfg2cmd/q35-linux-hostpci.conf.cmd +++ b/test/cfg2cmd/q35-linux-hostpci.conf.cmd @@ -1,19 +1,21 @@ /usr/bin/kvm \ -id 8006 \ - -name vm8006 \ - -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server,nowait' \ + -name 'vm8006,debug-threads=on' \ + -no-shutdown \ + -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ -mon 'chardev=qmp,mode=control' \ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' \ -mon 'chardev=qmp-event,mode=control' \ -pidfile /var/run/qemu-server/8006.pid \ -daemonize \ -smbios 'type=1,uuid=3dd750ce-d910-44d0-9493-525c0be4e687' \ - -drive 'if=pflash,unit=0,format=raw,readonly,file=/usr/share/pve-edk2-firmware//OVMF_CODE.fd' \ - -drive 'if=pflash,unit=1,format=qcow2,id=drive-efidisk0,file=/var/lib/vz/images/100/vm-100-disk-1.qcow2' \ + -drive 'if=pflash,unit=0,format=raw,readonly=on,file=/usr/share/pve-edk2-firmware//OVMF_CODE.fd' \ + -drive 'if=pflash,unit=1,id=drive-efidisk0,format=qcow2,file=/var/lib/vz/images/100/vm-100-disk-1.qcow2' \ + -global 'ICH9-LPC.acpi-pci-hotplug-with-bridge-support=off' \ -smp '2,sockets=2,cores=1,maxcpus=2' \ -nodefaults \ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ - -vnc unix:/var/run/qemu-server/8006.vnc,password \ + -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ -m 512 \ -object 'memory-backend-ram,id=ram-node0,size=256M' \ @@ -30,10 +32,10 @@ -device 'pcie-root-port,id=ich9-pcie-port-5,addr=10.0,x-speed=16,x-width=32,multifunction=on,bus=pcie.0,port=5,chassis=5' \ -device 'vfio-pci,host=0000:d0:17.0,id=hostpci4,bus=ich9-pcie-port-5,addr=0x0,rombar=0' \ -device 'pcie-root-port,id=ich9-pcie-port-8,addr=10.3,x-speed=16,x-width=32,multifunction=on,bus=pcie.0,port=8,chassis=8' \ - -device 'vfio-pci,host=0000:d0:15.1,id=hostpci7,bus=ich9-pcie-port-8,addr=0x0' \ + -device 'vfio-pci,host=0000:d0:15.2,id=hostpci7,bus=ich9-pcie-port-8,addr=0x0' \ -device 'VGA,id=vga,bus=pcie.0,addr=0x1' \ - -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3' \ + -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/var/lib/qemu-server/pve-bridge,downscript=/var/lib/qemu-server/pve-bridgedown,vhost=on' \ - -device 'virtio-net-pci,mac=2E:01:68:F9:9C:87,netdev=net0,bus=pci.0,addr=0x12,id=net0,bootindex=300' \ + -device 'virtio-net-pci,mac=2E:01:68:F9:9C:87,netdev=net0,bus=pci.0,addr=0x12,id=net0,rx_queue_size=1024,tx_queue_size=256,bootindex=300' \ -machine 'type=q35+pve0' diff --git a/test/cfg2cmd/q35-simple-6.0.conf b/test/cfg2cmd/q35-simple-6.0.conf new file mode 100644 index 0000000..70426b3 --- /dev/null +++ b/test/cfg2cmd/q35-simple-6.0.conf @@ -0,0 +1,13 @@ +# TEST: Config with q35, Linux & nothing much else but on 6.0 +# QEMU_VERSION: 6.0.0 +bios: ovmf +bootdisk: scsi0 +cores: 2 +efidisk0: local:100/vm-100-disk-1.qcow2,size=128K +machine: q35 +memory: 512 +net0: virtio=2E:01:68:F9:9C:87,bridge=vmbr0 +ostype: l26 +scsihw: virtio-scsi-pci +smbios1: uuid=3dd750ce-d910-44d0-9493-525c0be4e687 +vmgenid: 54d1c06c-8f5b-440f-b5b2-6eab1380e13d diff --git a/test/cfg2cmd/q35-simple-6.0.conf.cmd b/test/cfg2cmd/q35-simple-6.0.conf.cmd new file mode 100644 index 0000000..789c240 --- /dev/null +++ b/test/cfg2cmd/q35-simple-6.0.conf.cmd @@ -0,0 +1,28 @@ +/usr/bin/kvm \ + -id 8006 \ + -name 'vm8006,debug-threads=on' \ + -no-shutdown \ + -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ + -mon 'chardev=qmp,mode=control' \ + -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' \ + -mon 'chardev=qmp-event,mode=control' \ + -pidfile /var/run/qemu-server/8006.pid \ + -daemonize \ + -smbios 'type=1,uuid=3dd750ce-d910-44d0-9493-525c0be4e687' \ + -drive 'if=pflash,unit=0,format=raw,readonly=on,file=/usr/share/pve-edk2-firmware//OVMF_CODE.fd' \ + -drive 'if=pflash,unit=1,id=drive-efidisk0,format=qcow2,file=/var/lib/vz/images/100/vm-100-disk-1.qcow2' \ + -smp '2,sockets=1,cores=2,maxcpus=2' \ + -nodefaults \ + -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ + -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ + -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ + -m 512 \ + -readconfig /usr/share/qemu-server/pve-q35-4.0.cfg \ + -device 'vmgenid,guid=54d1c06c-8f5b-440f-b5b2-6eab1380e13d' \ + -device 'usb-tablet,id=tablet,bus=ehci.0,port=1' \ + -device 'VGA,id=vga,bus=pcie.0,addr=0x1' \ + -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3' \ + -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ + -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/var/lib/qemu-server/pve-bridge,downscript=/var/lib/qemu-server/pve-bridgedown,vhost=on' \ + -device 'virtio-net-pci,mac=2E:01:68:F9:9C:87,netdev=net0,bus=pci.0,addr=0x12,id=net0,bootindex=300' \ + -machine 'type=q35+pve0' diff --git a/test/cfg2cmd/q35-simple-7.0.conf b/test/cfg2cmd/q35-simple-7.0.conf new file mode 100644 index 0000000..4618e23 --- /dev/null +++ b/test/cfg2cmd/q35-simple-7.0.conf @@ -0,0 +1,14 @@ +# TEST: Config with q35, Linux & nothing much else but on 7.0 +# QEMU_VERSION: 7.0.0 +bios: ovmf +bootdisk: scsi0 +cores: 2 +efidisk0: local:100/vm-100-disk-1.qcow2,size=128K +machine: q35 +meta: creation-qemu=6.1 +memory: 512 +net0: virtio=2E:01:68:F9:9C:87,bridge=vmbr0 +ostype: l26 +scsihw: virtio-scsi-pci +smbios1: uuid=3dd750ce-d910-44d0-9493-525c0be4e687 +vmgenid: 54d1c06c-8f5b-440f-b5b2-6eab1380e13d diff --git a/test/cfg2cmd/q35-simple-7.0.conf.cmd b/test/cfg2cmd/q35-simple-7.0.conf.cmd new file mode 100644 index 0000000..9344f5a --- /dev/null +++ b/test/cfg2cmd/q35-simple-7.0.conf.cmd @@ -0,0 +1,28 @@ +/usr/bin/kvm \ + -id 8006 \ + -name 'vm8006,debug-threads=on' \ + -no-shutdown \ + -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ + -mon 'chardev=qmp,mode=control' \ + -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' \ + -mon 'chardev=qmp-event,mode=control' \ + -pidfile /var/run/qemu-server/8006.pid \ + -daemonize \ + -smbios 'type=1,uuid=3dd750ce-d910-44d0-9493-525c0be4e687' \ + -drive 'if=pflash,unit=0,format=raw,readonly=on,file=/usr/share/pve-edk2-firmware//OVMF_CODE.fd' \ + -drive 'if=pflash,unit=1,id=drive-efidisk0,format=qcow2,file=/var/lib/vz/images/100/vm-100-disk-1.qcow2' \ + -smp '2,sockets=1,cores=2,maxcpus=2' \ + -nodefaults \ + -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ + -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ + -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ + -m 512 \ + -readconfig /usr/share/qemu-server/pve-q35-4.0.cfg \ + -device 'vmgenid,guid=54d1c06c-8f5b-440f-b5b2-6eab1380e13d' \ + -device 'usb-tablet,id=tablet,bus=ehci.0,port=1' \ + -device 'VGA,id=vga,bus=pcie.0,addr=0x1' \ + -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \ + -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ + -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/var/lib/qemu-server/pve-bridge,downscript=/var/lib/qemu-server/pve-bridgedown,vhost=on' \ + -device 'virtio-net-pci,mac=2E:01:68:F9:9C:87,netdev=net0,bus=pci.0,addr=0x12,id=net0,bootindex=300' \ + -machine 'type=q35+pve0' diff --git a/test/cfg2cmd/q35-simple-pinned-6.1.conf b/test/cfg2cmd/q35-simple-pinned-6.1.conf new file mode 100644 index 0000000..9ecfe00 --- /dev/null +++ b/test/cfg2cmd/q35-simple-pinned-6.1.conf @@ -0,0 +1,13 @@ +# TEST: Config with q35, Linux & nothing much else +# +bios: ovmf +bootdisk: scsi0 +cores: 2 +efidisk0: local:100/vm-100-disk-1.qcow2,size=128K +machine: pc-q35-6.1 +memory: 512 +net0: virtio=2E:01:68:F9:9C:87,bridge=vmbr0 +ostype: l26 +scsihw: virtio-scsi-pci +smbios1: uuid=3dd750ce-d910-44d0-9493-525c0be4e687 +vmgenid: 54d1c06c-8f5b-440f-b5b2-6eab1380e13d diff --git a/test/cfg2cmd/q35-simple-pinned-6.1.conf.cmd b/test/cfg2cmd/q35-simple-pinned-6.1.conf.cmd new file mode 100644 index 0000000..b8c59df --- /dev/null +++ b/test/cfg2cmd/q35-simple-pinned-6.1.conf.cmd @@ -0,0 +1,28 @@ +/usr/bin/kvm \ + -id 8006 \ + -name 'vm8006,debug-threads=on' \ + -no-shutdown \ + -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ + -mon 'chardev=qmp,mode=control' \ + -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' \ + -mon 'chardev=qmp-event,mode=control' \ + -pidfile /var/run/qemu-server/8006.pid \ + -daemonize \ + -smbios 'type=1,uuid=3dd750ce-d910-44d0-9493-525c0be4e687' \ + -drive 'if=pflash,unit=0,format=raw,readonly=on,file=/usr/share/pve-edk2-firmware//OVMF_CODE.fd' \ + -drive 'if=pflash,unit=1,id=drive-efidisk0,format=qcow2,file=/var/lib/vz/images/100/vm-100-disk-1.qcow2' \ + -smp '2,sockets=1,cores=2,maxcpus=2' \ + -nodefaults \ + -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ + -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ + -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ + -m 512 \ + -readconfig /usr/share/qemu-server/pve-q35-4.0.cfg \ + -device 'vmgenid,guid=54d1c06c-8f5b-440f-b5b2-6eab1380e13d' \ + -device 'usb-tablet,id=tablet,bus=ehci.0,port=1' \ + -device 'VGA,id=vga,bus=pcie.0,addr=0x1' \ + -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3' \ + -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ + -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/var/lib/qemu-server/pve-bridge,downscript=/var/lib/qemu-server/pve-bridgedown,vhost=on' \ + -device 'virtio-net-pci,mac=2E:01:68:F9:9C:87,netdev=net0,bus=pci.0,addr=0x12,id=net0,bootindex=300' \ + -machine 'type=pc-q35-6.1+pve0' diff --git a/test/cfg2cmd/q35-simple.conf b/test/cfg2cmd/q35-simple.conf new file mode 100644 index 0000000..21f7812 --- /dev/null +++ b/test/cfg2cmd/q35-simple.conf @@ -0,0 +1,13 @@ +# TEST: Config with q35, Linux & nothing much else +# +bios: ovmf +bootdisk: scsi0 +cores: 2 +efidisk0: local:100/vm-100-disk-1.qcow2,size=128K +machine: q35 +memory: 512 +net0: virtio=2E:01:68:F9:9C:87,bridge=vmbr0 +ostype: l26 +scsihw: virtio-scsi-pci +smbios1: uuid=3dd750ce-d910-44d0-9493-525c0be4e687 +vmgenid: 54d1c06c-8f5b-440f-b5b2-6eab1380e13d diff --git a/test/cfg2cmd/q35-simple.conf.cmd b/test/cfg2cmd/q35-simple.conf.cmd new file mode 100644 index 0000000..98b22f4 --- /dev/null +++ b/test/cfg2cmd/q35-simple.conf.cmd @@ -0,0 +1,29 @@ +/usr/bin/kvm \ + -id 8006 \ + -name 'vm8006,debug-threads=on' \ + -no-shutdown \ + -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ + -mon 'chardev=qmp,mode=control' \ + -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' \ + -mon 'chardev=qmp-event,mode=control' \ + -pidfile /var/run/qemu-server/8006.pid \ + -daemonize \ + -smbios 'type=1,uuid=3dd750ce-d910-44d0-9493-525c0be4e687' \ + -drive 'if=pflash,unit=0,format=raw,readonly=on,file=/usr/share/pve-edk2-firmware//OVMF_CODE.fd' \ + -drive 'if=pflash,unit=1,id=drive-efidisk0,format=qcow2,file=/var/lib/vz/images/100/vm-100-disk-1.qcow2' \ + -global 'ICH9-LPC.acpi-pci-hotplug-with-bridge-support=off' \ + -smp '2,sockets=1,cores=2,maxcpus=2' \ + -nodefaults \ + -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ + -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ + -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ + -m 512 \ + -readconfig /usr/share/qemu-server/pve-q35-4.0.cfg \ + -device 'vmgenid,guid=54d1c06c-8f5b-440f-b5b2-6eab1380e13d' \ + -device 'usb-tablet,id=tablet,bus=ehci.0,port=1' \ + -device 'VGA,id=vga,bus=pcie.0,addr=0x1' \ + -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \ + -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ + -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/var/lib/qemu-server/pve-bridge,downscript=/var/lib/qemu-server/pve-bridgedown,vhost=on' \ + -device 'virtio-net-pci,mac=2E:01:68:F9:9C:87,netdev=net0,bus=pci.0,addr=0x12,id=net0,rx_queue_size=1024,tx_queue_size=256,bootindex=300' \ + -machine 'type=q35+pve0' diff --git a/test/cfg2cmd/q35-usb13-error.conf b/test/cfg2cmd/q35-usb13-error.conf new file mode 100644 index 0000000..a9c25eb --- /dev/null +++ b/test/cfg2cmd/q35-usb13-error.conf @@ -0,0 +1,14 @@ +# TEST: Test usb error for q35 and older machine type +# QEMU_VERSION: 4.0.0 +# EXPECT_ERROR: using usb13 is only possible with machine type >= 7.1 and ostype l26 or windows > 7 +cores: 2 +memory: 768 +name: q35-usb3-error +net0: virtio=A2:C0:43:77:08:A1,bridge=vmbr0 +ostype: l26 +machine: q35 +scsihw: virtio-scsi-pci +smbios1: uuid=7b10d7af-b932-4c66-b2c3-3996152ec465 +vmgenid: c773c261-d800-4348-9f5d-167fadd53cf8 +vga: qxl +usb13: spice diff --git a/test/cfg2cmd/q35-usb2.conf b/test/cfg2cmd/q35-usb2.conf new file mode 100644 index 0000000..4d9b28a --- /dev/null +++ b/test/cfg2cmd/q35-usb2.conf @@ -0,0 +1,13 @@ +# TEST: Test Q35 USB2 passthrough combination +# QEMU_VERSION: 4.0.0 +cores: 2 +memory: 768 +name: q35-usb2 +net0: virtio=A2:C0:43:77:08:A1,bridge=vmbr0 +ostype: l26 +machine: q35 +scsihw: virtio-scsi-pci +smbios1: uuid=7b10d7af-b932-4c66-b2c3-3996152ec465 +vmgenid: c773c261-d800-4348-9f5d-167fadd53cf8 +vga: qxl +usb1: spice diff --git a/test/cfg2cmd/q35-usb2.conf.cmd b/test/cfg2cmd/q35-usb2.conf.cmd new file mode 100644 index 0000000..825f27f --- /dev/null +++ b/test/cfg2cmd/q35-usb2.conf.cmd @@ -0,0 +1,31 @@ +/usr/bin/kvm \ + -id 8006 \ + -name 'q35-usb2,debug-threads=on' \ + -no-shutdown \ + -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ + -mon 'chardev=qmp,mode=control' \ + -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' \ + -mon 'chardev=qmp-event,mode=control' \ + -pidfile /var/run/qemu-server/8006.pid \ + -daemonize \ + -smbios 'type=1,uuid=7b10d7af-b932-4c66-b2c3-3996152ec465' \ + -smp '2,sockets=1,cores=2,maxcpus=2' \ + -nodefaults \ + -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ + -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ + -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ + -m 768 \ + -readconfig /usr/share/qemu-server/pve-q35-4.0.cfg \ + -device 'vmgenid,guid=c773c261-d800-4348-9f5d-167fadd53cf8' \ + -chardev 'spicevmc,id=usbredirchardev1,name=usbredir' \ + -device 'usb-redir,chardev=usbredirchardev1,id=usbredirdev1,bus=ehci.0' \ + -device 'qxl-vga,id=vga,bus=pcie.0,addr=0x1' \ + -device 'virtio-serial,id=spice,bus=pci.0,addr=0x9' \ + -chardev 'spicevmc,id=vdagent,name=vdagent' \ + -device 'virtserialport,chardev=vdagent,name=com.redhat.spice.0' \ + -spice 'tls-port=61000,addr=127.0.0.1,tls-ciphers=HIGH,seamless-migration=on' \ + -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3' \ + -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ + -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/var/lib/qemu-server/pve-bridge,downscript=/var/lib/qemu-server/pve-bridgedown,vhost=on' \ + -device 'virtio-net-pci,mac=A2:C0:43:77:08:A1,netdev=net0,bus=pci.0,addr=0x12,id=net0,bootindex=300' \ + -machine 'type=q35' diff --git a/test/cfg2cmd/q35-usb3.conf b/test/cfg2cmd/q35-usb3.conf new file mode 100644 index 0000000..1046e0e --- /dev/null +++ b/test/cfg2cmd/q35-usb3.conf @@ -0,0 +1,13 @@ +# TEST: Test Q35 USB3 passthrough combination +# QEMU_VERSION: 4.0.0 +cores: 2 +memory: 768 +name: q35-usb3 +net0: virtio=A2:C0:43:77:08:A1,bridge=vmbr0 +ostype: l26 +machine: q35 +scsihw: virtio-scsi-pci +smbios1: uuid=7b10d7af-b932-4c66-b2c3-3996152ec465 +vmgenid: c773c261-d800-4348-9f5d-167fadd53cf8 +vga: qxl +usb1: spice,usb3=1 diff --git a/test/cfg2cmd/q35-usb3.conf.cmd b/test/cfg2cmd/q35-usb3.conf.cmd new file mode 100644 index 0000000..956481f --- /dev/null +++ b/test/cfg2cmd/q35-usb3.conf.cmd @@ -0,0 +1,32 @@ +/usr/bin/kvm \ + -id 8006 \ + -name 'q35-usb3,debug-threads=on' \ + -no-shutdown \ + -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ + -mon 'chardev=qmp,mode=control' \ + -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' \ + -mon 'chardev=qmp-event,mode=control' \ + -pidfile /var/run/qemu-server/8006.pid \ + -daemonize \ + -smbios 'type=1,uuid=7b10d7af-b932-4c66-b2c3-3996152ec465' \ + -smp '2,sockets=1,cores=2,maxcpus=2' \ + -nodefaults \ + -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ + -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ + -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ + -m 768 \ + -readconfig /usr/share/qemu-server/pve-q35-4.0.cfg \ + -device 'vmgenid,guid=c773c261-d800-4348-9f5d-167fadd53cf8' \ + -device 'nec-usb-xhci,id=xhci,bus=pci.1,addr=0x1b' \ + -chardev 'spicevmc,id=usbredirchardev1,name=usbredir' \ + -device 'usb-redir,chardev=usbredirchardev1,id=usbredirdev1,bus=xhci.0' \ + -device 'qxl-vga,id=vga,bus=pcie.0,addr=0x1' \ + -device 'virtio-serial,id=spice,bus=pci.0,addr=0x9' \ + -chardev 'spicevmc,id=vdagent,name=vdagent' \ + -device 'virtserialport,chardev=vdagent,name=com.redhat.spice.0' \ + -spice 'tls-port=61000,addr=127.0.0.1,tls-ciphers=HIGH,seamless-migration=on' \ + -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3' \ + -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ + -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/var/lib/qemu-server/pve-bridge,downscript=/var/lib/qemu-server/pve-bridgedown,vhost=on' \ + -device 'virtio-net-pci,mac=A2:C0:43:77:08:A1,netdev=net0,bus=pci.0,addr=0x12,id=net0,bootindex=300' \ + -machine 'type=q35' diff --git a/test/cfg2cmd/q35-viommu-intel.conf b/test/cfg2cmd/q35-viommu-intel.conf new file mode 100644 index 0000000..e500ab0 --- /dev/null +++ b/test/cfg2cmd/q35-viommu-intel.conf @@ -0,0 +1 @@ +machine: q35,viommu=intel diff --git a/test/cfg2cmd/q35-viommu-intel.conf.cmd b/test/cfg2cmd/q35-viommu-intel.conf.cmd new file mode 100644 index 0000000..24e873d --- /dev/null +++ b/test/cfg2cmd/q35-viommu-intel.conf.cmd @@ -0,0 +1,23 @@ +/usr/bin/kvm \ + -id 8006 \ + -name 'vm8006,debug-threads=on' \ + -no-shutdown \ + -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ + -mon 'chardev=qmp,mode=control' \ + -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' \ + -mon 'chardev=qmp-event,mode=control' \ + -pidfile /var/run/qemu-server/8006.pid \ + -daemonize \ + -smp '1,sockets=1,cores=1,maxcpus=1' \ + -nodefaults \ + -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ + -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ + -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ + -m 512 \ + -device 'intel-iommu,intremap=on,caching-mode=on' \ + -readconfig /usr/share/qemu-server/pve-q35-4.0.cfg \ + -device 'usb-tablet,id=tablet,bus=ehci.0,port=1' \ + -device 'VGA,id=vga,bus=pcie.0,addr=0x1' \ + -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \ + -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ + -machine 'type=q35+pve0,kernel-irqchip=split' diff --git a/test/cfg2cmd/q35-viommu-virtio.conf b/test/cfg2cmd/q35-viommu-virtio.conf new file mode 100644 index 0000000..d31b339 --- /dev/null +++ b/test/cfg2cmd/q35-viommu-virtio.conf @@ -0,0 +1 @@ +machine: type=q35,viommu=virtio diff --git a/test/cfg2cmd/q35-viommu-virtio.conf.cmd b/test/cfg2cmd/q35-viommu-virtio.conf.cmd new file mode 100644 index 0000000..294c353 --- /dev/null +++ b/test/cfg2cmd/q35-viommu-virtio.conf.cmd @@ -0,0 +1,23 @@ +/usr/bin/kvm \ + -id 8006 \ + -name 'vm8006,debug-threads=on' \ + -no-shutdown \ + -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ + -mon 'chardev=qmp,mode=control' \ + -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' \ + -mon 'chardev=qmp-event,mode=control' \ + -pidfile /var/run/qemu-server/8006.pid \ + -daemonize \ + -smp '1,sockets=1,cores=1,maxcpus=1' \ + -nodefaults \ + -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ + -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ + -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ + -m 512 \ + -readconfig /usr/share/qemu-server/pve-q35-4.0.cfg \ + -device 'usb-tablet,id=tablet,bus=ehci.0,port=1' \ + -device 'VGA,id=vga,bus=pcie.0,addr=0x1' \ + -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \ + -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ + -device virtio-iommu-pci \ + -machine 'type=q35+pve0' diff --git a/test/cfg2cmd/q35-win10-hostpci.conf.cmd b/test/cfg2cmd/q35-win10-hostpci.conf.cmd index 133c086..cf03989 100644 --- a/test/cfg2cmd/q35-win10-hostpci.conf.cmd +++ b/test/cfg2cmd/q35-win10-hostpci.conf.cmd @@ -1,20 +1,20 @@ /usr/bin/kvm \ -id 8006 \ - -name vm8006 \ - -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server,nowait' \ + -name 'vm8006,debug-threads=on' \ + -no-shutdown \ + -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ -mon 'chardev=qmp,mode=control' \ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' \ -mon 'chardev=qmp-event,mode=control' \ -pidfile /var/run/qemu-server/8006.pid \ -daemonize \ -smbios 'type=1,uuid=3dd750ce-d910-44d0-9493-525c0be4e687' \ - -drive 'if=pflash,unit=0,format=raw,readonly,file=/usr/share/pve-edk2-firmware//OVMF_CODE.fd' \ - -drive 'if=pflash,unit=1,format=qcow2,id=drive-efidisk0,file=/var/lib/vz/images/100/vm-100-disk-1.qcow2' \ + -drive 'if=pflash,unit=0,format=raw,readonly=on,file=/usr/share/pve-edk2-firmware//OVMF_CODE.fd' \ + -drive 'if=pflash,unit=1,id=drive-efidisk0,format=qcow2,file=/var/lib/vz/images/100/vm-100-disk-1.qcow2' \ -smp '2,sockets=2,cores=1,maxcpus=2' \ -nodefaults \ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ - -vnc unix:/var/run/qemu-server/8006.vnc,password \ - -no-hpet \ + -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ -cpu 'kvm64,enforce,hv_ipi,hv_relaxed,hv_reset,hv_runtime,hv_spinlocks=0x1fff,hv_stimer,hv_synic,hv_time,hv_vapic,hv_vpindex,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep' \ -m 512 \ -object 'memory-backend-ram,id=ram-node0,size=256M' \ @@ -34,5 +34,5 @@ -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/var/lib/qemu-server/pve-bridge,downscript=/var/lib/qemu-server/pve-bridgedown,vhost=on' \ -device 'virtio-net-pci,mac=2E:01:68:F9:9C:87,netdev=net0,bus=pci.0,addr=0x12,id=net0,bootindex=300' \ -rtc 'driftfix=slew,base=localtime' \ - -machine 'type=q35+pve0' \ + -machine 'hpet=off,type=pc-q35-5.1+pve0' \ -global 'kvm-pit.lost_tick_policy=discard' diff --git a/test/cfg2cmd/qemu-xhci-7.1.conf b/test/cfg2cmd/qemu-xhci-7.1.conf new file mode 100644 index 0000000..e7cac65 --- /dev/null +++ b/test/cfg2cmd/qemu-xhci-7.1.conf @@ -0,0 +1,14 @@ +# TEST: Test for new xhci controller with new machine version +# QEMU_VERSION: 7.1.0 +cores: 2 +memory: 768 +name: spiceusb3 +net0: virtio=A2:C0:43:77:08:A1,bridge=vmbr0 +ostype: l26 +scsihw: virtio-scsi-pci +smbios1: uuid=7b10d7af-b932-4c66-b2c3-3996152ec465 +vmgenid: c773c261-d800-4348-9f5d-167fadd53cf8 +vga: qxl +usb1: spice +usb5: spice +usb13: host=1-14 diff --git a/test/cfg2cmd/qemu-xhci-7.1.conf.cmd b/test/cfg2cmd/qemu-xhci-7.1.conf.cmd new file mode 100644 index 0000000..2492e57 --- /dev/null +++ b/test/cfg2cmd/qemu-xhci-7.1.conf.cmd @@ -0,0 +1,37 @@ +/usr/bin/kvm \ + -id 8006 \ + -name 'spiceusb3,debug-threads=on' \ + -no-shutdown \ + -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ + -mon 'chardev=qmp,mode=control' \ + -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' \ + -mon 'chardev=qmp-event,mode=control' \ + -pidfile /var/run/qemu-server/8006.pid \ + -daemonize \ + -smbios 'type=1,uuid=7b10d7af-b932-4c66-b2c3-3996152ec465' \ + -smp '2,sockets=1,cores=2,maxcpus=2' \ + -nodefaults \ + -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ + -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ + -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ + -m 768 \ + -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \ + -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \ + -device 'vmgenid,guid=c773c261-d800-4348-9f5d-167fadd53cf8' \ + -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \ + -device 'qemu-xhci,p2=15,p3=15,id=xhci,bus=pci.1,addr=0x1b' \ + -chardev 'spicevmc,id=usbredirchardev1,name=usbredir' \ + -device 'usb-redir,chardev=usbredirchardev1,id=usbredirdev1,bus=xhci.0,port=2' \ + -chardev 'spicevmc,id=usbredirchardev5,name=usbredir' \ + -device 'usb-redir,chardev=usbredirchardev5,id=usbredirdev5,bus=xhci.0,port=6' \ + -device 'usb-host,bus=xhci.0,port=14,hostbus=1,hostport=14,id=usb13' \ + -device 'qxl-vga,id=vga,max_outputs=4,bus=pci.0,addr=0x2' \ + -device 'virtio-serial,id=spice,bus=pci.0,addr=0x9' \ + -chardev 'spicevmc,id=vdagent,name=vdagent' \ + -device 'virtserialport,chardev=vdagent,name=com.redhat.spice.0' \ + -spice 'tls-port=61000,addr=127.0.0.1,tls-ciphers=HIGH,seamless-migration=on' \ + -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \ + -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ + -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/var/lib/qemu-server/pve-bridge,downscript=/var/lib/qemu-server/pve-bridgedown,vhost=on' \ + -device 'virtio-net-pci,mac=A2:C0:43:77:08:A1,netdev=net0,bus=pci.0,addr=0x12,id=net0,rx_queue_size=1024,tx_queue_size=256,bootindex=300' \ + -machine 'type=pc+pve0' diff --git a/test/cfg2cmd/qemu-xhci-q35-7.1.conf b/test/cfg2cmd/qemu-xhci-q35-7.1.conf new file mode 100644 index 0000000..40a5901 --- /dev/null +++ b/test/cfg2cmd/qemu-xhci-q35-7.1.conf @@ -0,0 +1,12 @@ +# TEST: Test Q35 USB passthrough combination with qemu-xhci +# QEMU_VERSION: 7.1.0 +cores: 2 +memory: 768 +name: q35-qemu-xhci +net0: virtio=A2:C0:43:77:08:A1,bridge=vmbr0 +ostype: l26 +machine: pc-q35-7.1 +scsihw: virtio-scsi-pci +smbios1: uuid=7b10d7af-b932-4c66-b2c3-3996152ec465 +vmgenid: c773c261-d800-4348-9f5d-167fadd53cf8 +usb1: spice diff --git a/test/cfg2cmd/qemu-xhci-q35-7.1.conf.cmd b/test/cfg2cmd/qemu-xhci-q35-7.1.conf.cmd new file mode 100644 index 0000000..87d0f4b --- /dev/null +++ b/test/cfg2cmd/qemu-xhci-q35-7.1.conf.cmd @@ -0,0 +1,29 @@ +/usr/bin/kvm \ + -id 8006 \ + -name 'q35-qemu-xhci,debug-threads=on' \ + -no-shutdown \ + -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ + -mon 'chardev=qmp,mode=control' \ + -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' \ + -mon 'chardev=qmp-event,mode=control' \ + -pidfile /var/run/qemu-server/8006.pid \ + -daemonize \ + -smbios 'type=1,uuid=7b10d7af-b932-4c66-b2c3-3996152ec465' \ + -smp '2,sockets=1,cores=2,maxcpus=2' \ + -nodefaults \ + -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ + -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ + -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ + -m 768 \ + -readconfig /usr/share/qemu-server/pve-q35-4.0.cfg \ + -device 'vmgenid,guid=c773c261-d800-4348-9f5d-167fadd53cf8' \ + -device 'qemu-xhci,p2=15,p3=15,id=xhci,bus=pci.1,addr=0x1b' \ + -device 'usb-tablet,id=tablet,bus=ehci.0,port=1' \ + -chardev 'spicevmc,id=usbredirchardev1,name=usbredir' \ + -device 'usb-redir,chardev=usbredirchardev1,id=usbredirdev1,bus=xhci.0,port=2' \ + -device 'VGA,id=vga,bus=pcie.0,addr=0x1' \ + -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \ + -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ + -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/var/lib/qemu-server/pve-bridge,downscript=/var/lib/qemu-server/pve-bridgedown,vhost=on' \ + -device 'virtio-net-pci,mac=A2:C0:43:77:08:A1,netdev=net0,bus=pci.0,addr=0x12,id=net0,rx_queue_size=1024,tx_queue_size=256,bootindex=300' \ + -machine 'type=pc-q35-7.1+pve0' diff --git a/test/cfg2cmd/seabios_serial.conf b/test/cfg2cmd/seabios_serial.conf new file mode 100644 index 0000000..7ebfa50 --- /dev/null +++ b/test/cfg2cmd/seabios_serial.conf @@ -0,0 +1,16 @@ +# TEST: Test for smm-related regression with SeaBIOS and serial display +bootdisk: scsi0 +cores: 3 +ide2: none,media=cdrom +memory: 768 +name: seabiosserial +net0: virtio=A2:C0:43:77:08:A0,bridge=vmbr0 +numa: 0 +ostype: l26 +scsi0: local:8006/vm-8006-disk-0.qcow2,discard=on,size=104858K +scsihw: virtio-scsi-pci +serial0: socket +smbios1: uuid=7b10d7af-b932-4c66-b2c3-3996152ec465 +sockets: 1 +vga: serial0 +vmgenid: c773c261-d800-4348-9f5d-167fadd53cf8 diff --git a/test/cfg2cmd/seabios_serial.conf.cmd b/test/cfg2cmd/seabios_serial.conf.cmd new file mode 100644 index 0000000..1c4e102 --- /dev/null +++ b/test/cfg2cmd/seabios_serial.conf.cmd @@ -0,0 +1,33 @@ +/usr/bin/kvm \ + -id 8006 \ + -name 'seabiosserial,debug-threads=on' \ + -no-shutdown \ + -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ + -mon 'chardev=qmp,mode=control' \ + -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' \ + -mon 'chardev=qmp-event,mode=control' \ + -pidfile /var/run/qemu-server/8006.pid \ + -daemonize \ + -smbios 'type=1,uuid=7b10d7af-b932-4c66-b2c3-3996152ec465' \ + -smp '3,sockets=1,cores=3,maxcpus=3' \ + -nodefaults \ + -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ + -nographic \ + -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ + -m 768 \ + -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \ + -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \ + -device 'vmgenid,guid=c773c261-d800-4348-9f5d-167fadd53cf8' \ + -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \ + -chardev 'socket,id=serial0,path=/var/run/qemu-server/8006.serial0,server=on,wait=off' \ + -device 'isa-serial,chardev=serial0' \ + -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \ + -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ + -drive 'if=none,id=drive-ide2,media=cdrom,aio=io_uring' \ + -device 'ide-cd,bus=ide.1,unit=0,drive=drive-ide2,id=ide2,bootindex=200' \ + -device 'virtio-scsi-pci,id=scsihw0,bus=pci.0,addr=0x5' \ + -drive 'file=/var/lib/vz/images/8006/vm-8006-disk-0.qcow2,if=none,id=drive-scsi0,discard=on,format=qcow2,cache=none,aio=io_uring,detect-zeroes=unmap' \ + -device 'scsi-hd,bus=scsihw0.0,channel=0,scsi-id=0,lun=0,drive=drive-scsi0,id=scsi0,bootindex=100' \ + -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/var/lib/qemu-server/pve-bridge,downscript=/var/lib/qemu-server/pve-bridgedown,vhost=on' \ + -device 'virtio-net-pci,mac=A2:C0:43:77:08:A0,netdev=net0,bus=pci.0,addr=0x12,id=net0,rx_queue_size=1024,tx_queue_size=256,bootindex=300' \ + -machine 'smm=off,type=pc+pve0' diff --git a/test/cfg2cmd/simple-balloon-free-page-reporting.conf b/test/cfg2cmd/simple-balloon-free-page-reporting.conf new file mode 100644 index 0000000..e7cd1e4 --- /dev/null +++ b/test/cfg2cmd/simple-balloon-free-page-reporting.conf @@ -0,0 +1,15 @@ +# TEST: Simple test for balloon free page reporting enabled by default on 6.2 +# QEMU_VERSION: 6.2 +bootdisk: scsi0 +cores: 3 +ide2: none,media=cdrom +memory: 768 +name: simple +net0: virtio=A2:C0:43:77:08:A0,bridge=vmbr0 +numa: 0 +ostype: l26 +scsi0: local:8006/vm-8006-disk-0.qcow2,discard=on,size=104858K +scsihw: virtio-scsi-pci +smbios1: uuid=7b10d7af-b932-4c66-b2c3-3996152ec465 +sockets: 1 +vmgenid: c773c261-d800-4348-1010-1010add53cf8 diff --git a/test/cfg2cmd/simple-balloon-free-page-reporting.conf.cmd b/test/cfg2cmd/simple-balloon-free-page-reporting.conf.cmd new file mode 100644 index 0000000..a094704 --- /dev/null +++ b/test/cfg2cmd/simple-balloon-free-page-reporting.conf.cmd @@ -0,0 +1,33 @@ +/usr/bin/kvm \ + -id 8006 \ + -name 'simple,debug-threads=on' \ + -no-shutdown \ + -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ + -mon 'chardev=qmp,mode=control' \ + -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' \ + -mon 'chardev=qmp-event,mode=control' \ + -pidfile /var/run/qemu-server/8006.pid \ + -daemonize \ + -smbios 'type=1,uuid=7b10d7af-b932-4c66-b2c3-3996152ec465' \ + -smp '3,sockets=1,cores=3,maxcpus=3' \ + -nodefaults \ + -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ + -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ + -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ + -m 768 \ + -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \ + -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \ + -device 'vmgenid,guid=c773c261-d800-4348-1010-1010add53cf8' \ + -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \ + -device 'usb-tablet,id=tablet,bus=uhci.0,port=1' \ + -device 'VGA,id=vga,bus=pci.0,addr=0x2' \ + -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \ + -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ + -drive 'if=none,id=drive-ide2,media=cdrom,aio=io_uring' \ + -device 'ide-cd,bus=ide.1,unit=0,drive=drive-ide2,id=ide2,bootindex=200' \ + -device 'virtio-scsi-pci,id=scsihw0,bus=pci.0,addr=0x5' \ + -drive 'file=/var/lib/vz/images/8006/vm-8006-disk-0.qcow2,if=none,id=drive-scsi0,discard=on,format=qcow2,cache=none,aio=io_uring,detect-zeroes=unmap' \ + -device 'scsi-hd,bus=scsihw0.0,channel=0,scsi-id=0,lun=0,drive=drive-scsi0,id=scsi0,bootindex=100' \ + -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/var/lib/qemu-server/pve-bridge,downscript=/var/lib/qemu-server/pve-bridgedown,vhost=on' \ + -device 'virtio-net-pci,mac=A2:C0:43:77:08:A0,netdev=net0,bus=pci.0,addr=0x12,id=net0,bootindex=300' \ + -machine 'type=pc+pve0' diff --git a/test/cfg2cmd/simple-btrfs.conf b/test/cfg2cmd/simple-btrfs.conf new file mode 100644 index 0000000..06503d0 --- /dev/null +++ b/test/cfg2cmd/simple-btrfs.conf @@ -0,0 +1,15 @@ +# TEST: Simple test for a BTRFS backed VM, which shouldn't use cache=none like other storages +# QEMU_VERSION: 6.0 +bootdisk: scsi0 +cores: 3 +ide2: none,media=cdrom +memory: 768 +name: simple +net0: virtio=A2:C0:43:77:08:A0,bridge=vmbr0 +numa: 0 +ostype: l26 +scsi0: btrfs-store:8006/vm-8006-disk-0.raw,discard=on,size=104858K +scsihw: virtio-scsi-pci +smbios1: uuid=7b10d7af-b932-4c66-b2c3-3996152ec465 +sockets: 1 +vmgenid: c773c261-d800-4348-1010-1010add53cf8 diff --git a/test/cfg2cmd/simple-btrfs.conf.cmd b/test/cfg2cmd/simple-btrfs.conf.cmd new file mode 100644 index 0000000..148e688 --- /dev/null +++ b/test/cfg2cmd/simple-btrfs.conf.cmd @@ -0,0 +1,33 @@ +/usr/bin/kvm \ + -id 8006 \ + -name 'simple,debug-threads=on' \ + -no-shutdown \ + -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ + -mon 'chardev=qmp,mode=control' \ + -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' \ + -mon 'chardev=qmp-event,mode=control' \ + -pidfile /var/run/qemu-server/8006.pid \ + -daemonize \ + -smbios 'type=1,uuid=7b10d7af-b932-4c66-b2c3-3996152ec465' \ + -smp '3,sockets=1,cores=3,maxcpus=3' \ + -nodefaults \ + -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ + -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ + -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ + -m 768 \ + -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \ + -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \ + -device 'vmgenid,guid=c773c261-d800-4348-1010-1010add53cf8' \ + -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \ + -device 'usb-tablet,id=tablet,bus=uhci.0,port=1' \ + -device 'VGA,id=vga,bus=pci.0,addr=0x2' \ + -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3' \ + -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ + -drive 'if=none,id=drive-ide2,media=cdrom,aio=io_uring' \ + -device 'ide-cd,bus=ide.1,unit=0,drive=drive-ide2,id=ide2,bootindex=200' \ + -device 'virtio-scsi-pci,id=scsihw0,bus=pci.0,addr=0x5' \ + -drive 'file=/butter/bread/images/8006/vm-8006-disk-0/disk.raw,if=none,id=drive-scsi0,discard=on,format=raw,aio=io_uring,detect-zeroes=unmap' \ + -device 'scsi-hd,bus=scsihw0.0,channel=0,scsi-id=0,lun=0,drive=drive-scsi0,id=scsi0,bootindex=100' \ + -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/var/lib/qemu-server/pve-bridge,downscript=/var/lib/qemu-server/pve-bridgedown,vhost=on' \ + -device 'virtio-net-pci,mac=A2:C0:43:77:08:A0,netdev=net0,bus=pci.0,addr=0x12,id=net0,bootindex=300' \ + -machine 'type=pc+pve0' diff --git a/test/cfg2cmd/simple-virtio-blk.conf.cmd b/test/cfg2cmd/simple-virtio-blk.conf.cmd index 6933edf..4e063f3 100644 --- a/test/cfg2cmd/simple-virtio-blk.conf.cmd +++ b/test/cfg2cmd/simple-virtio-blk.conf.cmd @@ -1,7 +1,8 @@ /usr/bin/kvm \ -id 8006 \ - -name simple \ - -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server,nowait' \ + -name 'simple,debug-threads=on' \ + -no-shutdown \ + -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ -mon 'chardev=qmp,mode=control' \ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' \ -mon 'chardev=qmp-event,mode=control' \ @@ -11,7 +12,7 @@ -smp '3,sockets=1,cores=3,maxcpus=3' \ -nodefaults \ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ - -vnc unix:/var/run/qemu-server/8006.vnc,password \ + -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ -m 768 \ -object 'iothread,id=iothread-virtio0' \ diff --git a/test/cfg2cmd/simple1-template.conf b/test/cfg2cmd/simple1-template.conf new file mode 100644 index 0000000..c26de40 --- /dev/null +++ b/test/cfg2cmd/simple1-template.conf @@ -0,0 +1,17 @@ +# TEST: Simple test for a basic template configuration +# QEMU_VERSION: 3.0 +bootdisk: scsi0 +cores: 3 +ide2: none,media=cdrom +memory: 768 +name: simple +net0: virtio=A2:C0:43:77:08:A0,bridge=vmbr0 +numa: 0 +ostype: l26 +sata0: local:8006/base-8006-disk-0.qcow2,discard=on,size=104858K +scsi0: local:8006/base-8006-disk-1.qcow2,discard=on,size=104858K +scsihw: virtio-scsi-pci +smbios1: uuid=7b10d7af-b932-4c66-b2c3-3996152ec465 +sockets: 1 +template: 1 +vmgenid: c773c261-d800-4348-9f5d-167fadd53cf8 diff --git a/test/cfg2cmd/simple1-template.conf.cmd b/test/cfg2cmd/simple1-template.conf.cmd new file mode 100644 index 0000000..a24151f --- /dev/null +++ b/test/cfg2cmd/simple1-template.conf.cmd @@ -0,0 +1,37 @@ +/usr/bin/kvm \ + -id 8006 \ + -name 'simple,debug-threads=on' \ + -no-shutdown \ + -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ + -mon 'chardev=qmp,mode=control' \ + -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' \ + -mon 'chardev=qmp-event,mode=control' \ + -pidfile /var/run/qemu-server/8006.pid \ + -daemonize \ + -smbios 'type=1,uuid=7b10d7af-b932-4c66-b2c3-3996152ec465' \ + -smp '3,sockets=1,cores=3,maxcpus=3' \ + -nodefaults \ + -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ + -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ + -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ + -m 768 \ + -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \ + -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \ + -device 'vmgenid,guid=c773c261-d800-4348-9f5d-167fadd53cf8' \ + -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \ + -device 'usb-tablet,id=tablet,bus=uhci.0,port=1' \ + -device 'VGA,id=vga,bus=pci.0,addr=0x2' \ + -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3' \ + -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ + -drive 'if=none,id=drive-ide2,media=cdrom,aio=threads' \ + -device 'ide-cd,bus=ide.1,unit=0,drive=drive-ide2,id=ide2,bootindex=200' \ + -device 'virtio-scsi-pci,id=scsihw0,bus=pci.0,addr=0x5' \ + -drive 'file=/var/lib/vz/images/8006/base-8006-disk-1.qcow2,if=none,id=drive-scsi0,discard=on,format=qcow2,cache=none,aio=native,detect-zeroes=unmap,readonly=on' \ + -device 'scsi-hd,bus=scsihw0.0,channel=0,scsi-id=0,lun=0,drive=drive-scsi0,id=scsi0,bootindex=100' \ + -device 'ahci,id=ahci0,multifunction=on,bus=pci.0,addr=0x7' \ + -drive 'file=/var/lib/vz/images/8006/base-8006-disk-0.qcow2,if=none,id=drive-sata0,discard=on,format=qcow2,cache=none,aio=native,detect-zeroes=unmap' \ + -device 'ide-hd,bus=ahci0.0,drive=drive-sata0,id=sata0' \ + -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/var/lib/qemu-server/pve-bridge,downscript=/var/lib/qemu-server/pve-bridgedown,vhost=on' \ + -device 'virtio-net-pci,mac=A2:C0:43:77:08:A0,netdev=net0,bus=pci.0,addr=0x12,id=net0,bootindex=300' \ + -machine 'type=pc' \ + -snapshot diff --git a/test/cfg2cmd/simple1.conf.cmd b/test/cfg2cmd/simple1.conf.cmd index 3485064..2b1b185 100644 --- a/test/cfg2cmd/simple1.conf.cmd +++ b/test/cfg2cmd/simple1.conf.cmd @@ -1,7 +1,8 @@ /usr/bin/kvm \ -id 8006 \ - -name simple \ - -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server,nowait' \ + -name 'simple,debug-threads=on' \ + -no-shutdown \ + -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ -mon 'chardev=qmp,mode=control' \ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' \ -mon 'chardev=qmp-event,mode=control' \ @@ -11,7 +12,7 @@ -smp '3,sockets=1,cores=3,maxcpus=3' \ -nodefaults \ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ - -vnc unix:/var/run/qemu-server/8006.vnc,password \ + -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ -m 768 \ -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \ diff --git a/test/cfg2cmd/spice-enhancments.conf.cmd b/test/cfg2cmd/spice-enhancments.conf.cmd index 3951c06..81acdcc 100644 --- a/test/cfg2cmd/spice-enhancments.conf.cmd +++ b/test/cfg2cmd/spice-enhancments.conf.cmd @@ -1,7 +1,8 @@ /usr/bin/kvm \ -id 8006 \ - -name vm8006 \ - -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server,nowait' \ + -name 'vm8006,debug-threads=on' \ + -no-shutdown \ + -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ -mon 'chardev=qmp,mode=control' \ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' \ -mon 'chardev=qmp-event,mode=control' \ @@ -11,7 +12,7 @@ -smp '1,sockets=1,cores=1,maxcpus=1' \ -nodefaults \ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ - -vnc unix:/var/run/qemu-server/8006.vnc,password \ + -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ -m 512 \ -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \ diff --git a/test/cfg2cmd/spice-linux-4.1.conf.cmd b/test/cfg2cmd/spice-linux-4.1.conf.cmd index 2748cc9..e4a60a7 100644 --- a/test/cfg2cmd/spice-linux-4.1.conf.cmd +++ b/test/cfg2cmd/spice-linux-4.1.conf.cmd @@ -1,7 +1,8 @@ /usr/bin/kvm \ -id 8006 \ - -name spicelinux \ - -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server,nowait' \ + -name 'spicelinux,debug-threads=on' \ + -no-shutdown \ + -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ -mon 'chardev=qmp,mode=control' \ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' \ -mon 'chardev=qmp-event,mode=control' \ @@ -11,7 +12,7 @@ -smp '2,sockets=1,cores=2,maxcpus=2' \ -nodefaults \ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ - -vnc unix:/var/run/qemu-server/8006.vnc,password \ + -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ -m 768 \ -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \ diff --git a/test/cfg2cmd/spice-usb3.conf.cmd b/test/cfg2cmd/spice-usb3.conf.cmd index c515644..ab35b29 100644 --- a/test/cfg2cmd/spice-usb3.conf.cmd +++ b/test/cfg2cmd/spice-usb3.conf.cmd @@ -1,7 +1,8 @@ /usr/bin/kvm \ -id 8006 \ - -name spiceusb3 \ - -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server,nowait' \ + -name 'spiceusb3,debug-threads=on' \ + -no-shutdown \ + -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ -mon 'chardev=qmp,mode=control' \ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' \ -mon 'chardev=qmp-event,mode=control' \ @@ -11,7 +12,7 @@ -smp '2,sockets=1,cores=2,maxcpus=2' \ -nodefaults \ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ - -vnc unix:/var/run/qemu-server/8006.vnc,password \ + -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ -m 768 \ -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \ diff --git a/test/cfg2cmd/spice-win.conf.cmd b/test/cfg2cmd/spice-win.conf.cmd index 22dfa9d..f12c035 100644 --- a/test/cfg2cmd/spice-win.conf.cmd +++ b/test/cfg2cmd/spice-win.conf.cmd @@ -1,7 +1,8 @@ /usr/bin/kvm \ -id 8006 \ - -name spiceusb3 \ - -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server,nowait' \ + -name 'spiceusb3,debug-threads=on' \ + -no-shutdown \ + -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ -mon 'chardev=qmp,mode=control' \ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' \ -mon 'chardev=qmp-event,mode=control' \ @@ -11,8 +12,7 @@ -smp '2,sockets=1,cores=2,maxcpus=2' \ -nodefaults \ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ - -vnc unix:/var/run/qemu-server/8006.vnc,password \ - -no-hpet \ + -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ -cpu 'kvm64,enforce,hv_ipi,hv_relaxed,hv_reset,hv_runtime,hv_spinlocks=0x1fff,hv_stimer,hv_synic,hv_time,hv_vapic,hv_vpindex,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep' \ -m 768 \ -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \ @@ -33,5 +33,5 @@ -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/var/lib/qemu-server/pve-bridge,downscript=/var/lib/qemu-server/pve-bridgedown,vhost=on' \ -device 'virtio-net-pci,mac=A2:C0:43:77:08:A1,netdev=net0,bus=pci.0,addr=0x12,id=net0,bootindex=300' \ -rtc 'driftfix=slew,base=localtime' \ - -machine 'type=pc' \ + -machine 'hpet=off,type=pc-i440fx-4.0' \ -global 'kvm-pit.lost_tick_policy=discard' diff --git a/test/cfg2cmd/usb13-error.conf b/test/cfg2cmd/usb13-error.conf new file mode 100644 index 0000000..48dedc3 --- /dev/null +++ b/test/cfg2cmd/usb13-error.conf @@ -0,0 +1,13 @@ +# TEST: Test error for old machine type with newer usb config +# QEMU_VERSION: 4.0.0 +# EXPECT_ERROR: using usb13 is only possible with machine type >= 7.1 and ostype l26 or windows > 7 +cores: 2 +memory: 768 +name: q35-usb3-error +net0: virtio=A2:C0:43:77:08:A1,bridge=vmbr0 +ostype: l26 +scsihw: virtio-scsi-pci +smbios1: uuid=7b10d7af-b932-4c66-b2c3-3996152ec465 +vmgenid: c773c261-d800-4348-9f5d-167fadd53cf8 +vga: qxl +usb13: spice diff --git a/test/cfg2cmd/vnc-clipboard-spice.conf b/test/cfg2cmd/vnc-clipboard-spice.conf new file mode 100644 index 0000000..f4c917d --- /dev/null +++ b/test/cfg2cmd/vnc-clipboard-spice.conf @@ -0,0 +1,2 @@ +# TEST: Test for a VNC clipboard with a SPICE QXL display +vga: qxl,clipboard=vnc diff --git a/test/cfg2cmd/vnc-clipboard-spice.conf.cmd b/test/cfg2cmd/vnc-clipboard-spice.conf.cmd new file mode 100644 index 0000000..f24cc7f --- /dev/null +++ b/test/cfg2cmd/vnc-clipboard-spice.conf.cmd @@ -0,0 +1,27 @@ +/usr/bin/kvm \ + -id 8006 \ + -name 'vm8006,debug-threads=on' \ + -no-shutdown \ + -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ + -mon 'chardev=qmp,mode=control' \ + -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' \ + -mon 'chardev=qmp-event,mode=control' \ + -pidfile /var/run/qemu-server/8006.pid \ + -daemonize \ + -smp '1,sockets=1,cores=1,maxcpus=1' \ + -nodefaults \ + -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ + -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ + -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ + -m 512 \ + -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \ + -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \ + -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \ + -device 'qxl-vga,id=vga,max_outputs=4,bus=pci.0,addr=0x2' \ + -device 'virtio-serial,id=spice,bus=pci.0,addr=0x9' \ + -chardev 'qemu-vdagent,id=vdagent,name=vdagent,clipboard=on' \ + -device 'virtserialport,chardev=vdagent,name=com.redhat.spice.0' \ + -spice 'tls-port=61000,addr=127.0.0.1,tls-ciphers=HIGH,seamless-migration=on' \ + -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \ + -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ + -machine 'type=pc+pve0' diff --git a/test/cfg2cmd/vnc-clipboard-std.conf b/test/cfg2cmd/vnc-clipboard-std.conf new file mode 100644 index 0000000..1474766 --- /dev/null +++ b/test/cfg2cmd/vnc-clipboard-std.conf @@ -0,0 +1,2 @@ +# TEST: Test for a VNC clipboard with a std display +vga: std,clipboard=vnc diff --git a/test/cfg2cmd/vnc-clipboard-std.conf.cmd b/test/cfg2cmd/vnc-clipboard-std.conf.cmd new file mode 100644 index 0000000..c0c6cd2 --- /dev/null +++ b/test/cfg2cmd/vnc-clipboard-std.conf.cmd @@ -0,0 +1,27 @@ +/usr/bin/kvm \ + -id 8006 \ + -name 'vm8006,debug-threads=on' \ + -no-shutdown \ + -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \ + -mon 'chardev=qmp,mode=control' \ + -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' \ + -mon 'chardev=qmp-event,mode=control' \ + -pidfile /var/run/qemu-server/8006.pid \ + -daemonize \ + -smp '1,sockets=1,cores=1,maxcpus=1' \ + -nodefaults \ + -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \ + -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \ + -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \ + -m 512 \ + -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \ + -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \ + -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \ + -device 'usb-tablet,id=tablet,bus=uhci.0,port=1' \ + -device 'VGA,id=vga,bus=pci.0,addr=0x2' \ + -device 'virtio-serial,id=spice,bus=pci.0,addr=0x9' \ + -chardev 'qemu-vdagent,id=vdagent,name=vdagent,clipboard=on' \ + -device 'virtserialport,chardev=vdagent,name=com.redhat.spice.0' \ + -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \ + -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \ + -machine 'type=pc+pve0' diff --git a/test/restore-config-expected/139.conf b/test/restore-config-expected/139.conf new file mode 100644 index 0000000..94425f7 --- /dev/null +++ b/test/restore-config-expected/139.conf @@ -0,0 +1,16 @@ +# regular VM with an EFI disk +bios: ovmf +boot: order=scsi0;ide2;net0 +cores: 1 +efidisk0: target:139/vm-139-disk-0.qcow2,size=128K +ide2: local:iso/debian-10.6.0-amd64-netinst.iso,media=cdrom +memory: 2048 +name: eficloneclone +net0: virtio=7A:6C:A5:8B:11:93,bridge=vmbr0,firewall=1 +numa: 0 +ostype: l26 +scsi0: target:139/vm-139-disk-1.raw,size=4G +scsihw: virtio-scsi-pci +smbios1: uuid=21a7e7bc-3cd2-4232-a009-a41f4ee992ae +sockets: 1 +vmgenid: 0 diff --git a/test/restore-config-expected/142.conf b/test/restore-config-expected/142.conf new file mode 100644 index 0000000..ac2d2ad --- /dev/null +++ b/test/restore-config-expected/142.conf @@ -0,0 +1,15 @@ +# plain VM +bootdisk: scsi0 +cores: 1 +ide2: none,media=cdrom +memory: 512 +name: apache +net0: virtio=92:38:11:FD:ED:87,bridge=vmbr0,firewall=1 +numa: 0 +ostype: l26 +scsi0: target:142/vm-142-disk-0.qcow2,size=4G +scsihw: virtio-scsi-pci +smbios1: uuid=ddf91b3f-a597-42be-9a7e-fb6421dcd5cd +sockets: 1 +vmgenid: 0 +tags: foo bar diff --git a/test/restore-config-expected/1422.conf b/test/restore-config-expected/1422.conf new file mode 100644 index 0000000..2d77a44 --- /dev/null +++ b/test/restore-config-expected/1422.conf @@ -0,0 +1,14 @@ +# some properties should be filtered +bootdisk: scsi0 +cores: 1 +ide2: none,media=cdrom +memory: 512 +name: apache +net0: virtio=92:38:11:FD:ED:87,bridge=vmbr0,firewall=1 +numa: 0 +ostype: l26 +scsi0: target:1422/vm-1422-disk-0.qcow2,size=4G +scsihw: virtio-scsi-pci +smbios1: uuid=ddf91b3f-a597-42be-9a7e-fb6421dcd5cd +sockets: 1 +vmgenid: 0 diff --git a/test/restore-config-expected/179.conf b/test/restore-config-expected/179.conf new file mode 100644 index 0000000..4444efb --- /dev/null +++ b/test/restore-config-expected/179.conf @@ -0,0 +1,17 @@ +# many disks +boot: order=scsi0;ide2;net0 +cores: 1 +ide2: none,media=cdrom +memory: 2048 +net0: virtio=26:15:5B:73:3F:7C,bridge=vmbr0,firewall=1 +numa: 0 +ostype: l26 +scsi0: target:179/vm-179-disk-0.qcow2,cache=none,discard=on,size=32G,ssd=1 +scsi1: target:179/vm-179-disk-1.qcow2,cache=writethrough,size=32G +scsi2: target:179/vm-179-disk-2.qcow2,mbps_rd=7,mbps_wr=7,replicate=0,size=32G +scsi3: target:179/vm-179-disk-3.vmdk,size=32G +#scsi4: myfs:179/vm-179-disk-1.qcow2,backup=0,size=32G +scsihw: virtio-scsi-pci +smbios1: uuid=1819ead7-a55d-4544-8d38-29ca94869a9c +sockets: 1 +vmgenid: 0 diff --git a/test/restore-config-input/139.conf b/test/restore-config-input/139.conf new file mode 100644 index 0000000..5acb4d4 --- /dev/null +++ b/test/restore-config-input/139.conf @@ -0,0 +1,18 @@ +# regular VM with an EFI disk +bios: ovmf +boot: order=scsi0;ide2;net0 +cores: 1 +efidisk0: mydir:139/vm-139-disk-0.qcow2,size=128K +ide2: local:iso/debian-10.6.0-amd64-netinst.iso,media=cdrom +memory: 2048 +name: eficloneclone +net0: virtio=7A:6C:A5:8B:11:93,bridge=vmbr0,firewall=1 +numa: 0 +ostype: l26 +scsi0: rbdkvm:vm-139-disk-1,size=4G +scsihw: virtio-scsi-pci +smbios1: uuid=21a7e7bc-3cd2-4232-a009-a41f4ee992ae +sockets: 1 +vmgenid: 0 +#qmdump#map:efidisk0:drive-efidisk0:mydir:qcow2: +#qmdump#map:scsi0:drive-scsi0:rbdkvm:: diff --git a/test/restore-config-input/142.conf b/test/restore-config-input/142.conf new file mode 100644 index 0000000..f3633aa --- /dev/null +++ b/test/restore-config-input/142.conf @@ -0,0 +1,16 @@ +# plain VM +bootdisk: scsi0 +cores: 1 +ide2: none,media=cdrom +memory: 512 +name: apache +net0: virtio=92:38:11:FD:ED:87,bridge=vmbr0,firewall=1 +numa: 0 +ostype: l26 +scsi0: mydir:142/vm-142-disk-0.qcow2,size=4G +scsihw: virtio-scsi-pci +smbios1: uuid=ddf91b3f-a597-42be-9a7e-fb6421dcd5cd +sockets: 1 +vmgenid: 0 +tags: foo bar +#qmdump#map:scsi0:drive-scsi0:mydir:qcow2: diff --git a/test/restore-config-input/1422.conf b/test/restore-config-input/1422.conf new file mode 100644 index 0000000..d315502 --- /dev/null +++ b/test/restore-config-input/1422.conf @@ -0,0 +1,18 @@ +# some properties should be filtered +bootdisk: scsi0 +cores: 1 +ide2: none,media=cdrom +memory: 512 +name: apache +net0: virtio=92:38:11:FD:ED:87,bridge=vmbr0,firewall=1 +numa: 0 +ostype: l26 +scsi0: mydir:1422/vm-1422-disk-0.qcow2,size=4G +unused7: mydir:1422/vm-1422-disk-8.qcow2 +parent: snap +lock: backup +scsihw: virtio-scsi-pci +smbios1: uuid=ddf91b3f-a597-42be-9a7e-fb6421dcd5cd +sockets: 1 +vmgenid: 0 +#qmdump#map:scsi0:drive-scsi0:mydir:qcow2: diff --git a/test/restore-config-input/179.conf b/test/restore-config-input/179.conf new file mode 100644 index 0000000..e1ee01a --- /dev/null +++ b/test/restore-config-input/179.conf @@ -0,0 +1,21 @@ +# many disks +boot: order=scsi0;ide2;net0 +cores: 1 +ide2: none,media=cdrom +memory: 2048 +net0: virtio=26:15:5B:73:3F:7C,bridge=vmbr0,firewall=1 +numa: 0 +ostype: l26 +scsi0: myfs:179/vm-179-disk-4.qcow2,cache=none,discard=on,size=32G,ssd=1 +scsi1: myfs:179/vm-179-disk-0.qcow2,cache=writethrough,size=32G +scsi2: myfs:179/vm-179-disk-2.qcow2,mbps_rd=7,mbps_wr=7,replicate=0,size=32G +scsi3: myfs:179/vm-179-disk-3.vmdk,size=32G +scsi4: myfs:179/vm-179-disk-1.qcow2,backup=0,size=32G +scsihw: virtio-scsi-pci +smbios1: uuid=1819ead7-a55d-4544-8d38-29ca94869a9c +sockets: 1 +vmgenid: 0 +#qmdump#map:scsi0:drive-scsi0:myfs:qcow2: +#qmdump#map:scsi1:drive-scsi1:myfs:qcow2: +#qmdump#map:scsi2:drive-scsi2:myfs:qcow2: +#qmdump#map:scsi3:drive-scsi3:myfs:vmdk: diff --git a/test/run_config2command_tests.pl b/test/run_config2command_tests.pl index 8011265..7212acc 100755 --- a/test/run_config2command_tests.pl +++ b/test/run_config2command_tests.pl @@ -30,6 +30,13 @@ my $base_env = { type => 'dir', shared => 0, }, + 'btrfs-store' => { + content => { + images => 1, + }, + path => '/butter/bread', + type => 'btrfs', + }, 'cifs-store' => { shared => 1, path => '/mnt/pve/cifs-store', @@ -38,11 +45,13 @@ my $base_env = { type => 'cifs', share => 'CIFShare', content => { - images => 1 + images => 1, + iso => 1, }, }, 'rbd-store' => { monhost => '127.0.0.42,127.0.0.21,::1', + fsid => 'fc4181a6-56eb-4f68-b452-8ba1f381ca2a', content => { images => 1 }, @@ -73,13 +82,43 @@ my $pci_devs = [ "0000:0f:f2.0", "0000:d0:13.0", "0000:d0:15.1", + "0000:d0:15.2", "0000:d0:17.0", "0000:f0:42.0", "0000:f0:43.0", "0000:f0:43.1", "1234:f0:43.1", + "0000:01:00.4", + "0000:01:00.5", + "0000:01:00.6", + "0000:07:10.0", + "0000:07:10.1", + "0000:07:10.4", ]; +my $pci_map_config = { + ids => { + someGpu => { + type => 'pci', + map => [ + 'node=localhost,path=0000:01:00.4,id=10de:2231,iommugroup=1', + 'node=localhost,path=0000:01:00.5,id=10de:2231,iommugroup=1', + 'node=localhost,path=0000:01:00.6,id=10de:2231,iommugroup=1', + ], + }, + someNic => { + type => 'pci', + map => [ + 'node=localhost,path=0000:07:10.0,id=8086:1520,iommugroup=2', + 'node=localhost,path=0000:07:10.1,id=8086:1520,iommugroup=2', + 'node=localhost,path=0000:07:10.4,id=8086:1520,iommugroup=2', + ], + }, + }, +}; + +my $usb_map_config = {}, + my $current_test; # = { # description => 'Test description', # if available # qemu_version => '2.12', @@ -170,6 +209,21 @@ $qemu_server_config->mock( }, ); +my $qemu_server_memory; +$qemu_server_memory = Test::MockModule->new('PVE::QemuServer::Memory'); +$qemu_server_memory->mock( + hugepages_chunk_size_supported => sub { + return 1; + }, + host_numanode_exists => sub { + my ($id) = @_; + return 1; + }, + get_host_phys_address_bits => sub { + return 46; + } +); + my $pve_common_tools; $pve_common_tools = Test::MockModule->new('PVE::Tools'); $pve_common_tools->mock( @@ -251,6 +305,28 @@ $pve_common_sysfstools->mock( } sort @$pci_devs ]; }, + pci_device_info => sub { + my ($path, $noerr) = @_; + + if ($path =~ m/^0000:01:00/) { + return { + mdev => 1, + iommugroup => 1, + mdev => 1, + vendor => "0x10de", + device => "0x2231", + }; + } elsif ($path =~ m/^0000:07:10/) { + return { + iommugroup => 2, + mdev => 0, + vendor => "0x8086", + device => "0x1520", + }; + } else { + return {}; + } + }, ); my $qemu_monitor_module; @@ -279,6 +355,37 @@ $qemu_monitor_module->mock( ); $qemu_monitor_module->mock('qmp_cmd', \&qmp_cmd); +my $mapping_usb_module = Test::MockModule->new("PVE::Mapping::USB"); +$mapping_usb_module->mock( + config => sub { + return $usb_map_config; + }, +); + +my $mapping_pci_module = Test::MockModule->new("PVE::Mapping::PCI"); +$mapping_pci_module->mock( + config => sub { + return $pci_map_config; + }, +); + +my $pci_module = Test::MockModule->new("PVE::QemuServer::PCI"); +$pci_module->mock( + reserve_pci_usage => sub { + my ($ids, $vmid, $timeout, $pid, $dryrun) = @_; + + $ids = [$ids] if !ref($ids); + + for my $id (@$ids) { + if ($id eq "0000:07:10.1") { + die "reserved"; + } + } + + return undef; + }, +); + sub diff($$) { my ($a, $b) = @_; return if $a eq $b; diff --git a/test/run_qemu_img_convert_tests.pl b/test/run_qemu_img_convert_tests.pl index 7e25bd4..20ff387 100755 --- a/test/run_qemu_img_convert_tests.pl +++ b/test/run_qemu_img_convert_tests.pl @@ -23,6 +23,7 @@ my $storage_config = { }, "rbd-store" => { monhost => "127.0.0.42,127.0.0.21,::1", + fsid => 'fc4181a6-56eb-4f68-b452-8ba1f381ca2a', content => { images => 1 }, @@ -54,7 +55,7 @@ my $storage_config = { my $tests = [ { name => 'qcow2raw', - parameters => [ "local:$vmid/vm-$vmid-disk-0.qcow2", "local:$vmid/vm-$vmid-disk-0.raw", 1024*10, undef, 0 ], + parameters => [ "local:$vmid/vm-$vmid-disk-0.qcow2", "local:$vmid/vm-$vmid-disk-0.raw", 1024*10, undef, 0, undef ], expected => [ "/usr/bin/qemu-img", "convert", "-p", "-n", "-f", "qcow2", "-O", "raw", "/var/lib/vz/images/$vmid/vm-$vmid-disk-0.qcow2", "/var/lib/vz/images/$vmid/vm-$vmid-disk-0.raw" @@ -62,7 +63,7 @@ my $tests = [ }, { name => "raw2qcow2", - parameters => [ "local:$vmid/vm-$vmid-disk-0.raw", "local:$vmid/vm-$vmid-disk-0.qcow2", 1024*10, undef, 0 ], + parameters => [ "local:$vmid/vm-$vmid-disk-0.raw", "local:$vmid/vm-$vmid-disk-0.qcow2", 1024*10, undef, 0, undef ], expected => [ "/usr/bin/qemu-img", "convert", "-p", "-n", "-f", "raw", "-O", "qcow2", "/var/lib/vz/images/$vmid/vm-$vmid-disk-0.raw", "/var/lib/vz/images/$vmid/vm-$vmid-disk-0.qcow2" @@ -70,7 +71,7 @@ my $tests = [ }, { name => "local2rbd", - parameters => [ "local:$vmid/vm-$vmid-disk-0.raw", "rbd-store:vm-$vmid-disk-0", 1024*10, undef, 0 ], + parameters => [ "local:$vmid/vm-$vmid-disk-0.raw", "rbd-store:vm-$vmid-disk-0", 1024*10, undef, 0, undef ], expected => [ "/usr/bin/qemu-img", "convert", "-p", "-n", "-f", "raw", "-O", "raw", "/var/lib/vz/images/$vmid/vm-$vmid-disk-0.raw", "rbd:cpool/vm-$vmid-disk-0:mon_host=127.0.0.42;127.0.0.21;[\\:\\:1]:auth_supported=none" @@ -78,7 +79,7 @@ my $tests = [ }, { name => "rbd2local", - parameters => [ "rbd-store:vm-$vmid-disk-0", "local:$vmid/vm-$vmid-disk-0.raw", 1024*10, undef, 0 ], + parameters => [ "rbd-store:vm-$vmid-disk-0", "local:$vmid/vm-$vmid-disk-0.raw", 1024*10, undef, 0, undef ], expected => [ "/usr/bin/qemu-img", "convert", "-p", "-n", "-f", "raw", "-O", "raw", "rbd:cpool/vm-$vmid-disk-0:mon_host=127.0.0.42;127.0.0.21;[\\:\\:1]:auth_supported=none", "/var/lib/vz/images/$vmid/vm-$vmid-disk-0.raw" @@ -86,7 +87,7 @@ my $tests = [ }, { name => "local2zos", - parameters => [ "local:$vmid/vm-$vmid-disk-0.raw", "zfs-over-iscsi:vm-$vmid-disk-0", 1024*10, undef, 0 ], + parameters => [ "local:$vmid/vm-$vmid-disk-0.raw", "zfs-over-iscsi:vm-$vmid-disk-0", 1024*10, undef, 0, undef ], expected => [ "/usr/bin/qemu-img", "convert", "-p", "-n", "-f", "raw", "--target-image-opts", "/var/lib/vz/images/$vmid/vm-$vmid-disk-0.raw", @@ -95,7 +96,7 @@ my $tests = [ }, { name => "zos2local", - parameters => [ "zfs-over-iscsi:vm-$vmid-disk-0", "local:$vmid/vm-$vmid-disk-0.raw", 1024*10, undef, 0 ], + parameters => [ "zfs-over-iscsi:vm-$vmid-disk-0", "local:$vmid/vm-$vmid-disk-0.raw", 1024*10, undef, 0, undef ], expected => [ "/usr/bin/qemu-img", "convert", "-p", "-n", "--image-opts", "-O", "raw", "file.driver=iscsi,file.transport=tcp,file.initiator-name=foobar,file.portal=127.0.0.1,file.target=iqn.2019-10.org.test:foobar,file.lun=1,driver=raw", @@ -104,7 +105,7 @@ my $tests = [ }, { name => "zos2rbd", - parameters => [ "zfs-over-iscsi:vm-$vmid-disk-0", "rbd-store:vm-$vmid-disk-0", 1024*10, undef, 0 ], + parameters => [ "zfs-over-iscsi:vm-$vmid-disk-0", "rbd-store:vm-$vmid-disk-0", 1024*10, undef, 0, undef ], expected => [ "/usr/bin/qemu-img", "convert", "-p", "-n", "--image-opts", "-O", "raw", "file.driver=iscsi,file.transport=tcp,file.initiator-name=foobar,file.portal=127.0.0.1,file.target=iqn.2019-10.org.test:foobar,file.lun=1,driver=raw", @@ -113,7 +114,7 @@ my $tests = [ }, { name => "rbd2zos", - parameters => [ "rbd-store:vm-$vmid-disk-0", "zfs-over-iscsi:vm-$vmid-disk-0", 1024*10, undef, 0 ], + parameters => [ "rbd-store:vm-$vmid-disk-0", "zfs-over-iscsi:vm-$vmid-disk-0", 1024*10, undef, 0, undef ], expected => [ "/usr/bin/qemu-img", "convert", "-p", "-n", "-f", "raw", "--target-image-opts", "rbd:cpool/vm-$vmid-disk-0:mon_host=127.0.0.42;127.0.0.21;[\\:\\:1]:auth_supported=none", @@ -122,7 +123,7 @@ my $tests = [ }, { name => "local2lvmthin", - parameters => [ "local:$vmid/vm-$vmid-disk-0.raw", "local-lvm:vm-$vmid-disk-0", 1024*10, undef, 0 ], + parameters => [ "local:$vmid/vm-$vmid-disk-0.raw", "local-lvm:vm-$vmid-disk-0", 1024*10, undef, 0, undef ], expected => [ "/usr/bin/qemu-img", "convert", "-p", "-n", "-f", "raw", "-O", "raw", "/var/lib/vz/images/$vmid/vm-$vmid-disk-0.raw", @@ -131,7 +132,7 @@ my $tests = [ }, { name => "lvmthin2local", - parameters => [ "local-lvm:vm-$vmid-disk-0", "local:$vmid/vm-$vmid-disk-0.raw", 1024*10, undef, 0 ], + parameters => [ "local-lvm:vm-$vmid-disk-0", "local:$vmid/vm-$vmid-disk-0.raw", 1024*10, undef, 0, undef ], expected => [ "/usr/bin/qemu-img", "convert", "-p", "-n", "-f", "raw", "-O", "raw", "/dev/pve/vm-$vmid-disk-0", @@ -140,7 +141,7 @@ my $tests = [ }, { name => "zeroinit", - parameters => [ "local-lvm:vm-$vmid-disk-0", "local:$vmid/vm-$vmid-disk-0.raw", 1024*10, undef, 1 ], + parameters => [ "local-lvm:vm-$vmid-disk-0", "local:$vmid/vm-$vmid-disk-0.raw", 1024*10, undef, 1, undef ], expected => [ "/usr/bin/qemu-img", "convert", "-p", "-n", "-f", "raw", "-O", "raw", "/dev/pve/vm-$vmid-disk-0", @@ -149,12 +150,12 @@ my $tests = [ }, { name => "notexistingstorage", - parameters => [ "local-lvm:vm-$vmid-disk-0", "not-existing:$vmid/vm-$vmid-disk-0.raw", 1024*10, undef, 1 ], + parameters => [ "local-lvm:vm-$vmid-disk-0", "not-existing:$vmid/vm-$vmid-disk-0.raw", 1024*10, undef, 1, undef ], expected => "storage 'not-existing' does not exist\n", }, { name => "vmdkfile", - parameters => [ "./test.vmdk", "local:$vmid/vm-$vmid-disk-0.raw", 1024*10, undef, 0 ], + parameters => [ "./test.vmdk", "local:$vmid/vm-$vmid-disk-0.raw", 1024*10, undef, 0, undef ], expected => [ "/usr/bin/qemu-img", "convert", "-p", "-n", "-f", "vmdk", "-O", "raw", "./test.vmdk", @@ -163,12 +164,12 @@ my $tests = [ }, { name => "notexistingfile", - parameters => [ "/foo/bar", "local:$vmid/vm-$vmid-disk-0.raw", 1024*10, undef, 0 ], + parameters => [ "/foo/bar", "local:$vmid/vm-$vmid-disk-0.raw", 1024*10, undef, 0, undef ], expected => "source '/foo/bar' is not a valid volid nor path for qemu-img convert\n", }, { name => "efidisk", - parameters => [ "/usr/share/kvm/OVMF_VARS-pure-efi.fd", "local:$vmid/vm-$vmid-disk-0.raw", 1024*10, undef, 0 ], + parameters => [ "/usr/share/kvm/OVMF_VARS-pure-efi.fd", "local:$vmid/vm-$vmid-disk-0.raw", 1024*10, undef, 0, undef ], expected => [ "/usr/bin/qemu-img", "convert", "-p", "-n", "-O", "raw", "/usr/share/kvm/OVMF_VARS-pure-efi.fd", @@ -177,13 +178,22 @@ my $tests = [ }, { name => "efi2zos", - parameters => [ "/usr/share/kvm/OVMF_VARS-pure-efi.fd", "zfs-over-iscsi:vm-$vmid-disk-0", 1024*10, undef, 0 ], + parameters => [ "/usr/share/kvm/OVMF_VARS-pure-efi.fd", "zfs-over-iscsi:vm-$vmid-disk-0", 1024*10, undef, 0, undef ], expected => [ "/usr/bin/qemu-img", "convert", "-p", "-n", "--target-image-opts", "/usr/share/kvm/OVMF_VARS-pure-efi.fd", "file.driver=iscsi,file.transport=tcp,file.initiator-name=foobar,file.portal=127.0.0.1,file.target=iqn.2019-10.org.test:foobar,file.lun=1,driver=raw", ] - } + }, + { + name => "bwlimit", + parameters => [ "local-lvm:vm-$vmid-disk-0", "local:$vmid/vm-$vmid-disk-0.raw", 1024*10, undef, 0, 1024 ], + expected => [ + "/usr/bin/qemu-img", "convert", "-p", "-n", "-r", "1024K", "-f", "raw", "-O", "raw", + "/dev/pve/vm-$vmid-disk-0", + "/var/lib/vz/images/$vmid/vm-$vmid-disk-0.raw", + ] + }, ]; my $command; diff --git a/test/run_qemu_migrate_tests.pl b/test/run_qemu_migrate_tests.pl new file mode 100755 index 0000000..4373a38 --- /dev/null +++ b/test/run_qemu_migrate_tests.pl @@ -0,0 +1,1773 @@ +#!/usr/bin/perl + +use strict; +use warnings; + +use JSON; +use Test::More; +use Test::MockModule; + +use PVE::JSONSchema; +use PVE::Tools qw(file_set_contents file_get_contents run_command); + +my $QM_LIB_PATH = '..'; +my $MIGRATE_LIB_PATH = '..'; +my $RUN_DIR_PATH = './MigrationTest/run/'; + +# test configuration shared by all tests + +my $replication_config = { + 'ids' => { + '105-0' => { + 'guest' => '105', + 'id' => '105-0', + 'jobnum' => '0', + 'source' => 'pve0', + 'target' => 'pve2', + 'type' => 'local' + }, + }, + 'order' => { + '105-0' => 1, + } +}; + +my $storage_config = { + ids => { + local => { + content => { + images => 1, + }, + path => "/var/lib/vz", + type => "dir", + shared => 0, + }, + "local-lvm" => { + content => { + images => 1, + }, + nodes => { + pve0 => 1, + pve1 => 1, + }, + type => "lvmthin", + thinpool => "data", + vgname => "pve", + }, + "local-zfs" => { + content => { + images => 1, + rootdir => 1, + }, + pool => "rpool/data", + sparse => 1, + type => "zfspool", + }, + "rbd-store" => { + monhost => "127.0.0.42,127.0.0.21,::1", + fsid => 'fc4181a6-56eb-4f68-b452-8ba1f381ca2a', + content => { + images => 1, + }, + type => "rbd", + pool => "cpool", + username => "admin", + shared => 1, + }, + "local-dir" => { + content => { + images => 1, + }, + path => "/some/dir/", + type => "dir", + }, + "other-dir" => { + content => { + images => 1, + }, + path => "/some/other/dir/", + type => "dir", + }, + "zfs-alias-1" => { + content => { + images => 1, + rootdir => 1, + }, + pool => "aliaspool", + sparse => 1, + type => "zfspool", + }, + "zfs-alias-2" => { + content => { + images => 1, + rootdir => 1, + }, + pool => "aliaspool", + sparse => 1, + type => "zfspool", + }, + }, +}; + +my $vm_configs = { + 105 => { + 'bootdisk' => 'scsi0', + 'cores' => 1, + 'ide0' => 'local-zfs:vm-105-disk-1,size=103M', + 'ide2' => 'none,media=cdrom', + 'memory' => 512, + 'name' => 'Copy-of-VM-newapache', + 'net0' => 'virtio=4A:A3:E4:4C:CF:F0,bridge=vmbr0,firewall=1', + 'numa' => 0, + 'ostype' => 'l26', + 'parent' => 'ohsnap', + 'pending' => {}, + 'scsi0' => 'local-zfs:vm-105-disk-0,size=4G', + 'scsihw' => 'virtio-scsi-pci', + 'smbios1' => 'uuid=1ddfe18b-77e0-47f6-a4bd-f1761bf6d763', + 'snapshots' => { + 'ohsnap' => { + 'bootdisk' => 'scsi0', + 'cores' => 1, + 'ide2' => 'none,media=cdrom', + 'memory' => 512, + 'name' => 'Copy-of-VM-newapache', + 'net0' => 'virtio=4A:A3:E4:4C:CF:F0,bridge=vmbr0,firewall=1', + 'numa' => 0, + 'ostype' => 'l26', + 'scsi0' => 'local-zfs:vm-105-disk-0,size=4G', + 'scsihw' => 'virtio-scsi-pci', + 'smbios1' => 'uuid=1ddfe18b-77e0-47f6-a4bd-f1761bf6d763', + 'snaptime' => 1580976924, + 'sockets' => 1, + 'startup' => 'order=2', + 'vmgenid' => '4eb1d535-9381-4ddc-a8aa-af50c4d9177b' + }, + }, + 'sockets' => 1, + 'startup' => 'order=2', + 'vmgenid' => '4eb1d535-9381-4ddc-a8aa-af50c4d9177b', + }, + 111 => { + 'bootdisk' => 'scsi0', + 'cores' => 1, + 'ide0' => 'local-lvm:vm-111-disk-0,size=4096M', + 'ide2' => 'none,media=cdrom', + 'memory' => 512, + 'name' => 'pending-test', + 'net0' => 'virtio=4A:A3:E4:4C:CF:F0,bridge=vmbr0,firewall=1', + 'numa' => 0, + 'ostype' => 'l26', + 'pending' => { + 'scsi0' => 'local-zfs:vm-111-disk-0,size=103M', + }, + 'scsihw' => 'virtio-scsi-pci', + 'snapshots' => {}, + 'smbios1' => 'uuid=5ad71d4d-8f73-4377-853e-2d22c10c96a5', + 'sockets' => 1, + 'vmgenid' => '2c00c030-0b5b-4988-a371-6ab259893f22', + }, + 123 => { + 'bootdisk' => 'scsi0', + 'cores' => 1, + 'scsi0' => 'zfs-alias-1:vm-123-disk-0,size=4096M', + 'scsi1' => 'zfs-alias-2:vm-123-disk-0,size=4096M', + 'ide2' => 'none,media=cdrom', + 'memory' => 512, + 'name' => 'alias-test', + 'net0' => 'virtio=4A:A3:E4:4C:CF:F0,bridge=vmbr0,firewall=1', + 'numa' => 0, + 'ostype' => 'l26', + 'pending' => {}, + 'scsihw' => 'virtio-scsi-pci', + 'snapshots' => {}, + 'smbios1' => 'uuid=5ad71d4d-8f73-4377-853e-2d22c10c96a5', + 'sockets' => 1, + 'vmgenid' => '2c00c030-0b5b-4988-a371-6ab259893f22', + }, + 149 => { + 'agent' => '0', + 'bootdisk' => 'scsi0', + 'cores' => 1, + 'hotplug' => 'disk,network,usb,memory,cpu', + 'ide2' => 'none,media=cdrom', + 'memory' => 4096, + 'name' => 'asdf', + 'net0' => 'virtio=52:5D:7E:62:85:97,bridge=vmbr1', + 'numa' => 1, + 'ostype' => 'l26', + 'scsi0' => 'local-lvm:vm-149-disk-0,format=raw,size=4G', + 'scsi1' => 'local-dir:149/vm-149-disk-0.qcow2,format=qcow2,size=1G', + 'scsihw' => 'virtio-scsi-pci', + 'snapshots' => {}, + 'smbios1' => 'uuid=e980bd43-a405-42e2-b5f4-31efe6517460', + 'sockets' => 1, + 'startup' => 'order=2', + 'vmgenid' => '36c6c50c-6ef5-4adc-9b6f-6ba9c8071db0', + }, + 341 => { + 'arch' => 'aarch64', + 'bootdisk' => 'scsi0', + 'cores' => 1, + 'efidisk0' => 'local-lvm:vm-341-disk-0', + 'ide2' => 'none,media=cdrom', + 'ipconfig0' => 'ip=103.214.69.10/25,gw=103.214.69.1', + 'memory' => 4096, + 'name' => 'VM1033', + 'net0' => 'virtio=4E:F1:82:6D:D7:4B,bridge=vmbr0,firewall=1,rate=10', + 'numa' => 0, + 'ostype' => 'l26', + 'scsi0' => 'rbd-store:vm-341-disk-0,size=1G', + 'scsihw' => 'virtio-scsi-pci', + 'snapshots' => {}, + 'smbios1' => 'uuid=e01e4c73-46f1-47c8-af79-288fdf6b7462', + 'sockets' => 2, + 'vmgenid' => 'af47c000-eb0c-48e8-8991-ca4593cd6916', + }, + 1033 => { + 'bootdisk' => 'scsi0', + 'cores' => 1, + 'ide0' => 'rbd-store:vm-1033-cloudinit,media=cdrom,size=4M', + 'ide2' => 'none,media=cdrom', + 'ipconfig0' => 'ip=103.214.69.10/25,gw=103.214.69.1', + 'memory' => 4096, + 'name' => 'VM1033', + 'net0' => 'virtio=4E:F1:82:6D:D7:4B,bridge=vmbr0,firewall=1,rate=10', + 'numa' => 0, + 'ostype' => 'l26', + 'scsi0' => 'rbd-store:vm-1033-disk-1,size=1G', + 'scsihw' => 'virtio-scsi-pci', + 'snapshots' => {}, + 'smbios1' => 'uuid=e01e4c73-46f1-47c8-af79-288fdf6b7462', + 'sockets' => 2, + 'vmgenid' => 'af47c000-eb0c-48e8-8991-ca4593cd6916', + }, + 4567 => { + 'bootdisk' => 'scsi0', + 'cores' => 1, + 'ide2' => 'none,media=cdrom', + 'memory' => 512, + 'name' => 'snapme', + 'net0' => 'virtio=A6:D1:F1:EB:7B:C2,bridge=vmbr0,firewall=1', + 'numa' => 0, + 'ostype' => 'l26', + 'parent' => 'snap1', + 'pending' => {}, + 'scsi0' => 'local-dir:4567/vm-4567-disk-0.qcow2,size=4G', + 'scsihw' => 'virtio-scsi-pci', + 'smbios1' => 'uuid=2925fdec-a066-4228-b46b-eef8662f5e74', + 'snapshots' => { + 'snap1' => { + 'bootdisk' => 'scsi0', + 'cores' => 1, + 'ide2' => 'none,media=cdrom', + 'memory' => 512, + 'name' => 'snapme', + 'net0' => 'virtio=A6:D1:F1:EB:7B:C2,bridge=vmbr0,firewall=1', + 'numa' => 0, + 'ostype' => 'l26', + 'runningcpu' => 'kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep', + 'runningmachine' => 'pc-i440fx-5.0+pve0', + 'scsi0' => 'local-dir:4567/vm-4567-disk-0.qcow2,size=4G', + 'scsihw' => 'virtio-scsi-pci', + 'smbios1' => 'uuid=2925fdec-a066-4228-b46b-eef8662f5e74', + 'snaptime' => 1595928799, + 'sockets' => 1, + 'startup' => 'order=2', + 'vmgenid' => '932b227a-8a39-4ede-955a-dbd4bc4385ed', + 'vmstate' => 'local-dir:4567/vm-4567-state-snap1.raw', + }, + 'snap2' => { + 'bootdisk' => 'scsi0', + 'cores' => 1, + 'ide2' => 'none,media=cdrom', + 'memory' => 512, + 'name' => 'snapme', + 'net0' => 'virtio=A6:D1:F1:EB:7B:C2,bridge=vmbr0,firewall=1', + 'numa' => 0, + 'ostype' => 'l26', + 'parent' => 'snap1', + 'runningcpu' => 'kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep', + 'runningmachine' => 'pc-i440fx-5.0+pve0', + 'scsi0' => 'local-dir:4567/vm-4567-disk-0.qcow2,size=4G', + 'scsi1' => 'local-zfs:vm-4567-disk-0,size=1G', + 'scsihw' => 'virtio-scsi-pci', + 'smbios1' => 'uuid=2925fdec-a066-4228-b46b-eef8662f5e74', + 'snaptime' => 1595928871, + 'sockets' => 1, + 'startup' => 'order=2', + 'vmgenid' => '932b227a-8a39-4ede-955a-dbd4bc4385ed', + 'vmstate' => 'local-dir:4567/vm-4567-state-snap2.raw', + }, + }, + 'sockets' => 1, + 'startup' => 'order=2', + 'unused0' => 'local-zfs:vm-4567-disk-0', + 'vmgenid' => 'e698e60c-9278-4dd9-941f-416075383f2a', + }, +}; + +my $source_vdisks = { + 'local-dir' => [ + { + 'ctime' => 1589439681, + 'format' => 'qcow2', + 'parent' => undef, + 'size' => 1073741824, + 'used' => 335872, + 'vmid' => '149', + 'volid' => 'local-dir:149/vm-149-disk-0.qcow2', + }, + { + 'ctime' => 1595928898, + 'format' => 'qcow2', + 'parent' => undef, + 'size' => 4294967296, + 'used' => 1811664896, + 'vmid' => '4567', + 'volid' => 'local-dir:4567/vm-4567-disk-0.qcow2', + }, + { + 'ctime' => 1595928800, + 'format' => 'raw', + 'parent' => undef, + 'size' => 274666496, + 'used' => 274669568, + 'vmid' => '4567', + 'volid' => 'local-dir:4567/vm-4567-state-snap1.raw', + }, + { + 'ctime' => 1595928872, + 'format' => 'raw', + 'parent' => undef, + 'size' => 273258496, + 'used' => 273260544, + 'vmid' => '4567', + 'volid' => 'local-dir:4567/vm-4567-state-snap2.raw', + }, + ], + 'local-lvm' => [ + { + 'ctime' => '1589277334', + 'format' => 'raw', + 'size' => 4294967296, + 'vmid' => '149', + 'volid' => 'local-lvm:vm-149-disk-0', + }, + { + 'ctime' => '1589277334', + 'format' => 'raw', + 'size' => 4194304, + 'vmid' => '341', + 'volid' => 'local-lvm:vm-341-disk-0', + }, + { + 'ctime' => '1589277334', + 'format' => 'raw', + 'size' => 4294967296, + 'vmid' => '111', + 'volid' => 'local-lvm:vm-111-disk-0', + }, + ], + 'local-zfs' => [ + { + 'ctime' => '1589277334', + 'format' => 'raw', + 'size' => 4294967296, + 'vmid' => '105', + 'volid' => 'local-zfs:vm-105-disk-0', + }, + { + 'ctime' => '1589277334', + 'format' => 'raw', + 'size' => 108003328, + 'vmid' => '105', + 'volid' => 'local-zfs:vm-105-disk-1', + }, + { + 'ctime' => '1589277334', + 'format' => 'raw', + 'size' => 108003328, + 'vmid' => '111', + 'volid' => 'local-zfs:vm-111-disk-0', + }, + { + 'format' => 'raw', + 'name' => 'vm-4567-disk-0', + 'parent' => undef, + 'size' => 1073741824, + 'vmid' => '4567', + 'volid' => 'local-zfs:vm-4567-disk-0', + }, + ], + 'rbd-store' => [ + { + 'ctime' => '1589277334', + 'format' => 'raw', + 'size' => 1073741824, + 'vmid' => '1033', + 'volid' => 'rbd-store:vm-1033-disk-1', + }, + { + 'ctime' => '1589277334', + 'format' => 'raw', + 'size' => 1073741824, + 'vmid' => '1033', + 'volid' => 'rbd-store:vm-1033-cloudinit', + }, + ], + 'zfs-alias-1' => [ + { + 'ctime' => '1589277334', + 'format' => 'raw', + 'size' => 4294967296, + 'vmid' => '123', + 'volid' => 'zfs-alias-1:vm-123-disk-0', + }, + ], + 'zfs-alias-2' => [ + { + 'ctime' => '1589277334', + 'format' => 'raw', + 'size' => 4294967296, + 'vmid' => '123', + 'volid' => 'zfs-alias-2:vm-123-disk-0', + }, + ], +}; + +my $default_expected_calls_online = { + move_config_to_node => 1, + ssh_qm_start => 1, + vm_stop => 1, +}; + +my $default_expected_calls_offline = { + move_config_to_node => 1, +}; + +my $replicated_expected_calls_online = { + %{$default_expected_calls_online}, + transfer_replication_state => 1, + switch_replication_job_target => 1, +}; + +my $replicated_expected_calls_offline = { + %{$default_expected_calls_offline}, + transfer_replication_state => 1, + switch_replication_job_target => 1, +}; + +# helpers + +sub get_patched_config { + my ($vmid, $patch) = @_; + + my $new_config = { %{$vm_configs->{$vmid}} }; + patch_config($new_config, $patch) if defined($patch); + + return $new_config; +} + +sub patch_config { + my ($config, $patch) = @_; + + foreach my $key (keys %{$patch}) { + if ($key eq 'snapshots' && defined($patch->{$key})) { + my $new_snapshot_configs = {}; + foreach my $snap (keys %{$patch->{snapshots}}) { + my $new_snapshot_config = { %{$config->{snapshots}->{$snap}} }; + patch_config($new_snapshot_config, $patch->{snapshots}->{$snap}); + $new_snapshot_configs->{$snap} = $new_snapshot_config; + } + $config->{snapshots} = $new_snapshot_configs; + } elsif (defined($patch->{$key})) { + $config->{$key} = $patch->{$key}; + } else { # use undef value for deletion + delete $config->{$key}; + } + } +} + +sub local_volids_for_vm { + my ($vmid) = @_; + + my $res = {}; + foreach my $storeid (keys %{$source_vdisks}) { + next if $storage_config->{ids}->{$storeid}->{shared}; + $res = { + %{$res}, + map { $_->{vmid} eq $vmid ? ($_->{volid} => 1) : () } @{$source_vdisks->{$storeid}} + }; + } + return $res; +} + +my $tests = [ +# each test consists of the following: +# name - unique name for the test which also serves as a dir name. +# NOTE: gets passed to make, so don't use whitespace or slash +# and adapt buildsys (regex) on code structure changes +# target - hostname of target node +# vmid - ID of the VM to migrate +# opts - options for the migrate() call +# target_volids - hash of volids on the target at the beginning +# vm_status - hash with running, runningmachine and optionally runningcpu +# expected_calls - hash whose keys are calls which are required +# to be made if the migration gets far enough +# expect_die - expect the migration call to fail, and an error message +# matching the specified text in the log +# expected - hash consisting of: +# source_volids - hash of volids expected on the source +# target_volids - hash of volids expected on the target +# vm_config - vm configuration hash +# vm_status - hash with running, runningmachine and optionally runningcpu + { + # NOTE get_efivars_size is mocked and returns 128K + name => '341_running_efidisk_targetstorage_dir', + target => 'pve1', + vmid => 341, + vm_status => { + running => 1, + runningmachine => 'pc-q35-5.0+pve0', + }, + opts => { + online => 1, + 'with-local-disks' => 1, + targetstorage => 'local-dir', + }, + expected_calls => $default_expected_calls_online, + expected => { + source_volids => {}, + target_volids => { + 'local-dir:341/vm-341-disk-10.raw' => 1, + }, + vm_config => get_patched_config(341, { + efidisk0 => 'local-dir:341/vm-341-disk-10.raw,format=raw,size=128K', + }), + vm_status => { + running => 1, + runningmachine => 'pc-q35-5.0+pve0', + }, + }, + }, + { + # NOTE get_efivars_size is mocked and returns 128K + name => '341_running_efidisk', + target => 'pve1', + vmid => 341, + vm_status => { + running => 1, + runningmachine => 'pc-q35-5.0+pve0', + }, + opts => { + online => 1, + 'with-local-disks' => 1, + }, + expected_calls => $default_expected_calls_online, + expected => { + source_volids => {}, + target_volids => { + 'local-lvm:vm-341-disk-10' => 1, + }, + vm_config => get_patched_config(341, { + efidisk0 => 'local-lvm:vm-341-disk-10,format=raw,size=128K', + }), + vm_status => { + running => 1, + runningmachine => 'pc-q35-5.0+pve0', + }, + }, + }, + { + name => '149_running_vdisk_alloc_and_pvesm_free_fail', + target => 'pve1', + vmid => 149, + vm_status => { + running => 1, + runningmachine => 'pc-q35-5.0+pve0', + }, + opts => { + online => 1, + 'with-local-disks' => 1, + }, + fail_config => { + vdisk_alloc => 'local-dir:149/vm-149-disk-11.qcow2', + pvesm_free => 'local-lvm:vm-149-disk-10', + }, + expected_calls => {}, + expect_die => "remote command failed with exit code", + expected => { + source_volids => local_volids_for_vm(149), + target_volids => { + 'local-lvm:vm-149-disk-10' => 1, + }, + vm_config => $vm_configs->{149}, + vm_status => { + running => 1, + runningmachine => 'pc-q35-5.0+pve0', + }, + }, + }, + { + name => '149_running_vdisk_alloc_fail', + target => 'pve1', + vmid => 149, + vm_status => { + running => 1, + runningmachine => 'pc-q35-5.0+pve0', + }, + opts => { + online => 1, + 'with-local-disks' => 1, + }, + fail_config => { + vdisk_alloc => 'local-lvm:vm-149-disk-10', + }, + expected_calls => {}, + expect_die => "remote command failed with exit code", + expected => { + source_volids => local_volids_for_vm(149), + target_volids => {}, + vm_config => $vm_configs->{149}, + vm_status => { + running => 1, + runningmachine => 'pc-q35-5.0+pve0', + }, + }, + }, + { + name => '149_vdisk_free_fail', + target => 'pve1', + vmid => 149, + vm_status => { + running => 0, + }, + opts => { + 'with-local-disks' => 1, + }, + fail_config => { + 'vdisk_free' => 'local-lvm:vm-149-disk-0', + }, + expected_calls => $default_expected_calls_offline, + expect_die => "vdisk_free 'local-lvm:vm-149-disk-0' error", + expected => { + source_volids => { + 'local-lvm:vm-149-disk-0' => 1, + }, + target_volids => local_volids_for_vm(149), + vm_config => $vm_configs->{149}, + vm_status => { + running => 0, + }, + }, + }, + { + name => '105_replicated_run_replication_fail', + target => 'pve2', + vmid => 105, + vm_status => { + running => 0, + }, + target_volids => local_volids_for_vm(105), + fail_config => { + run_replication => 1, + }, + expected_calls => {}, + expect_die => 'run_replication error', + expected => { + source_volids => local_volids_for_vm(105), + target_volids => local_volids_for_vm(105), + vm_config => $vm_configs->{105}, + vm_status => { + running => 0, + }, + }, + }, + { + name => '1033_running_query_migrate_fail', + target => 'pve2', + vmid => 1033, + vm_status => { + running => 1, + runningmachine => 'pc-q35-5.0+pve0', + }, + opts => { + online => 1, + }, + fail_config => { + 'query-migrate' => 1, + }, + expected_calls => {}, + expect_die => 'online migrate failure - aborting', + expected => { + source_volids => {}, + target_volids => {}, + vm_config => $vm_configs->{1033}, + vm_status => { + running => 1, + runningmachine => 'pc-q35-5.0+pve0', + }, + }, + }, + { + name => '4567_targetstorage_dirotherdir', + target => 'pve1', + vmid => 4567, + vm_status => { + running => 0, + }, + opts => { + targetstorage => 'local-dir:other-dir,local-zfs:local-zfs', + }, + storage_migrate_map => { + 'local-dir:4567/vm-4567-disk-0.qcow2' => '4567/vm-4567-disk-0.qcow2', + 'local-dir:4567/vm-4567-state-snap1.raw' => '4567/vm-4567-state-snap1.raw', + 'local-dir:4567/vm-4567-state-snap2.raw' => '4567/vm-4567-state-snap2.raw', + }, + expected_calls => $default_expected_calls_offline, + expected => { + source_volids => {}, + target_volids => { + 'other-dir:4567/vm-4567-disk-0.qcow2' => 1, + 'other-dir:4567/vm-4567-state-snap1.raw' => 1, + 'other-dir:4567/vm-4567-state-snap2.raw' => 1, + 'local-zfs:vm-4567-disk-0' => 1, + }, + vm_config => get_patched_config(4567, { + 'scsi0' => 'other-dir:4567/vm-4567-disk-0.qcow2,size=4G', + snapshots => { + snap1 => { + 'scsi0' => 'other-dir:4567/vm-4567-disk-0.qcow2,size=4G', + 'vmstate' => 'other-dir:4567/vm-4567-state-snap1.raw', + }, + snap2 => { + 'scsi0' => 'other-dir:4567/vm-4567-disk-0.qcow2,size=4G', + 'scsi1' => 'local-zfs:vm-4567-disk-0,size=1G', + 'vmstate' => 'other-dir:4567/vm-4567-state-snap2.raw', + }, + }, + }), + vm_status => { + running => 0, + }, + }, + }, + { + name => '4567_running', + target => 'pve1', + vmid => 4567, + vm_status => { + running => 1, + runningmachine => 'pc-i440fx-5.0+pve0', + }, + opts => { + online => 1, + 'with-local-disks' => 1, + }, + expected_calls => {}, + expect_die => 'online storage migration not possible if non-replicated snapshot exists', + expected => { + source_volids => local_volids_for_vm(4567), + target_volids => {}, + vm_config => $vm_configs->{4567}, + vm_status => { + running => 1, + runningmachine => 'pc-i440fx-5.0+pve0', + }, + }, + }, + { + name => '4567_offline', + target => 'pve1', + vmid => 4567, + vm_status => { + running => 0, + }, + expected_calls => $default_expected_calls_offline, + expected => { + source_volids => {}, + target_volids => local_volids_for_vm(4567), + vm_config => $vm_configs->{4567}, + vm_status => { + running => 0, + }, + }, + }, + { + name => '149_running_orphaned_disk_targetstorage_zfs', + target => 'pve1', + vmid => 149, + vm_status => { + running => 1, + runningmachine => 'pc-q35-5.0+pve0', + }, + opts => { + online => 1, + 'with-local-disks' => 1, + targetstorage => 'local-zfs', + }, + config_patch => { + scsi1 => undef, + }, + storage_migrate_map => { + 'local-dir:149/vm-149-disk-0.qcow2' => 'vm-149-disk-0', + }, + expected_calls => $default_expected_calls_online, + expected => { + source_volids => { + 'local-dir:149/vm-149-disk-0.qcow2' => 1, + }, + target_volids => { + 'local-zfs:vm-149-disk-10' => 1, + }, + vm_config => get_patched_config(149, { + scsi0 => 'local-zfs:vm-149-disk-10,format=raw,size=4G', + scsi1 => undef, + }), + vm_status => { + running => 1, + runningmachine => 'pc-q35-5.0+pve0', + }, + }, + }, + { + name => '149_running_orphaned_disk', + target => 'pve1', + vmid => 149, + vm_status => { + running => 1, + runningmachine => 'pc-q35-5.0+pve0', + }, + opts => { + online => 1, + 'with-local-disks' => 1, + }, + config_patch => { + scsi1 => undef, + }, + storage_migrate_map => { + 'local-dir:149/vm-149-disk-0.qcow2' => '149/vm-149-disk-0.qcow2', + }, + expected_calls => $default_expected_calls_online, + expected => { + source_volids => { + 'local-dir:149/vm-149-disk-0.qcow2' => 1, + }, + target_volids => { + 'local-lvm:vm-149-disk-10' => 1, + }, + vm_config => get_patched_config(149, { + scsi0 => 'local-lvm:vm-149-disk-10,format=raw,size=4G', + scsi1 => undef, + }), + vm_status => { + running => 1, + runningmachine => 'pc-q35-5.0+pve0', + }, + }, + }, + { + # FIXME: This test is not (yet) a realistic situation, because + # storage_migrate currently never changes the format (AFAICT) + # But if such migrations become possible, we need to either update + # the 'format' property or simply remove it for drives migrated + # with storage_migrate (the property is optional, so it shouldn't be a problem) + name => '149_targetstorage_map_lvmzfs_defaultlvm', + target => 'pve1', + vmid => 149, + vm_status => { + running => 0, + }, + opts => { + targetstorage => 'local-lvm:local-zfs,local-lvm', + }, + storage_migrate_map => { + 'local-lvm:vm-149-disk-0' => 'vm-149-disk-0', + 'local-dir:149/vm-149-disk-0.qcow2' => 'vm-149-disk-0', + }, + expected_calls => $default_expected_calls_offline, + expected => { + source_volids => {}, + target_volids => { + 'local-zfs:vm-149-disk-0' => 1, + 'local-lvm:vm-149-disk-0' => 1, + }, + vm_config => get_patched_config(149, { + scsi0 => 'local-zfs:vm-149-disk-0,format=raw,size=4G', + scsi1 => 'local-lvm:vm-149-disk-0,format=qcow2,size=1G', + }), + vm_status => { + running => 0, + }, + }, + }, + { + # FIXME same as for the previous test + name => '149_targetstorage_map_dirzfs_lvmdir', + target => 'pve1', + vmid => 149, + vm_status => { + running => 0, + }, + opts => { + online => 1, + 'with-local-disks' => 1, + targetstorage => 'local-dir:local-zfs,local-lvm:local-dir', + }, + storage_migrate_map => { + 'local-lvm:vm-149-disk-0' => '149/vm-149-disk-0.raw', + 'local-dir:149/vm-149-disk-0.qcow2' => 'vm-149-disk-0', + }, + expected_calls => $default_expected_calls_offline, + expected => { + source_volids => {}, + target_volids => { + 'local-dir:149/vm-149-disk-0.raw' => 1, + 'local-zfs:vm-149-disk-0' => 1, + }, + vm_config => get_patched_config(149, { + scsi0 => 'local-dir:149/vm-149-disk-0.raw,format=raw,size=4G', + scsi1 => 'local-zfs:vm-149-disk-0,format=qcow2,size=1G', + }), + vm_status => { + running => 0, + }, + }, + }, + { + name => '149_running_targetstorage_map_lvmzfs_defaultlvm', + target => 'pve1', + vmid => 149, + vm_status => { + running => 1, + runningmachine => 'pc-q35-5.0+pve0', + }, + opts => { + online => 1, + 'with-local-disks' => 1, + targetstorage => 'local-lvm:local-zfs,local-lvm', + }, + expected_calls => $default_expected_calls_online, + expected => { + source_volids => {}, + target_volids => { + 'local-zfs:vm-149-disk-10' => 1, + 'local-lvm:vm-149-disk-11' => 1, + }, + vm_config => get_patched_config(149, { + scsi0 => 'local-zfs:vm-149-disk-10,format=raw,size=4G', + scsi1 => 'local-lvm:vm-149-disk-11,format=raw,size=1G', + }), + vm_status => { + running => 1, + runningmachine => 'pc-q35-5.0+pve0', + }, + }, + }, + { + name => '149_running_targetstorage_map_lvmzfs_dirdir', + target => 'pve1', + vmid => 149, + vm_status => { + running => 1, + runningmachine => 'pc-q35-5.0+pve0', + }, + opts => { + online => 1, + 'with-local-disks' => 1, + targetstorage => 'local-lvm:local-zfs,local-dir:local-dir', + }, + expected_calls => $default_expected_calls_online, + expected => { + source_volids => {}, + target_volids => { + 'local-zfs:vm-149-disk-10' => 1, + 'local-dir:149/vm-149-disk-11.qcow2' => 1, + }, + vm_config => get_patched_config(149, { + scsi0 => 'local-zfs:vm-149-disk-10,format=raw,size=4G', + scsi1 => 'local-dir:149/vm-149-disk-11.qcow2,format=qcow2,size=1G', + }), + vm_status => { + running => 1, + runningmachine => 'pc-q35-5.0+pve0', + }, + }, + }, + { + name => '149_running_targetstorage_zfs', + target => 'pve1', + vmid => 149, + vm_status => { + running => 1, + runningmachine => 'pc-q35-5.0+pve0', + }, + opts => { + online => 1, + 'with-local-disks' => 1, + targetstorage => 'local-zfs', + }, + expected_calls => $default_expected_calls_online, + expected => { + source_volids => {}, + target_volids => { + 'local-zfs:vm-149-disk-10' => 1, + 'local-zfs:vm-149-disk-11' => 1, + }, + vm_config => get_patched_config(149, { + scsi0 => 'local-zfs:vm-149-disk-10,format=raw,size=4G', + scsi1 => 'local-zfs:vm-149-disk-11,format=raw,size=1G', + }), + vm_status => { + running => 1, + runningmachine => 'pc-q35-5.0+pve0', + }, + }, + }, + { + name => '149_running_wrong_size', + target => 'pve1', + vmid => 149, + vm_status => { + running => 1, + runningmachine => 'pc-q35-5.0+pve0', + }, + opts => { + online => 1, + 'with-local-disks' => 1, + }, + config_patch => { + scsi0 => 'local-lvm:vm-149-disk-0,size=123T', + }, + expected_calls => $default_expected_calls_online, + expected => { + source_volids => {}, + target_volids => { + 'local-lvm:vm-149-disk-10' => 1, + 'local-dir:149/vm-149-disk-11.qcow2' => 1, + }, + vm_config => get_patched_config(149, { + scsi0 => 'local-lvm:vm-149-disk-10,format=raw,size=4G', + scsi1 => 'local-dir:149/vm-149-disk-11.qcow2,format=qcow2,size=1G', + }), + vm_status => { + running => 1, + runningmachine => 'pc-q35-5.0+pve0', + }, + }, + }, + { + name => '149_running_missing_size', + target => 'pve1', + vmid => 149, + vm_status => { + running => 1, + runningmachine => 'pc-q35-5.0+pve0', + }, + opts => { + online => 1, + 'with-local-disks' => 1, + }, + config_patch => { + scsi0 => 'local-lvm:vm-149-disk-0', + }, + expected_calls => $default_expected_calls_online, + expected => { + source_volids => {}, + target_volids => { + 'local-lvm:vm-149-disk-10' => 1, + 'local-dir:149/vm-149-disk-11.qcow2' => 1, + }, + vm_config => get_patched_config(149, { + scsi0 => 'local-lvm:vm-149-disk-10,format=raw,size=4G', + scsi1 => 'local-dir:149/vm-149-disk-11.qcow2,format=qcow2,size=1G', + }), + vm_status => { + running => 1, + runningmachine => 'pc-q35-5.0+pve0', + }, + }, + }, + { + name => '105_local_device_shared', + target => 'pve1', + vmid => 105, + vm_status => { + running => 0, + }, + config_patch => { + ide2 => '/dev/sde,shared=1', + }, + expected_calls => $default_expected_calls_offline, + expected => { + source_volids => {}, + target_volids => local_volids_for_vm(105), + vm_config => get_patched_config(105, { + ide2 => '/dev/sde,shared=1', + }), + vm_status => { + running => 0, + }, + }, + }, + { + name => '105_local_device_in_snapshot', + target => 'pve1', + vmid => 105, + vm_status => { + running => 0, + }, + config_patch => { + snapshots => { + ohsnap => { + ide2 => '/dev/sde', + }, + }, + }, + expected_calls => {}, + expect_die => "can't migrate local disk '/dev/sde': local file/device", + expected => { + source_volids => local_volids_for_vm(105), + target_volids => {}, + vm_config => get_patched_config(105, { + snapshots => { + ohsnap => { + ide2 => '/dev/sde', + }, + }, + }), + vm_status => { + running => 0, + }, + }, + }, + { + name => '105_local_device', + target => 'pve1', + vmid => 105, + vm_status => { + running => 0, + }, + config_patch => { + ide2 => '/dev/sde', + }, + expected_calls => {}, + expect_die => "can't migrate local disk '/dev/sde': local file/device", + expected => { + source_volids => local_volids_for_vm(105), + target_volids => {}, + vm_config => get_patched_config(105, { + ide2 => '/dev/sde', + }), + vm_status => { + running => 0, + }, + }, + }, + { + name => '105_cdrom_in_snapshot', + target => 'pve1', + vmid => 105, + vm_status => { + running => 0, + }, + config_patch => { + snapshots => { + ohsnap => { + ide2 => 'cdrom,media=cdrom', + }, + }, + }, + expected_calls => {}, + expect_die => "can't migrate local cdrom drive (referenced in snapshot - ohsnap", + expected => { + source_volids => local_volids_for_vm(105), + target_volids => {}, + vm_config => get_patched_config(105, { + snapshots => { + ohsnap => { + ide2 => 'cdrom,media=cdrom', + }, + }, + }), + vm_status => { + running => 0, + }, + }, + }, + { + name => '105_cdrom', + target => 'pve1', + vmid => 105, + vm_status => { + running => 0, + }, + config_patch => { + ide2 => 'cdrom,media=cdrom', + }, + expected_calls => {}, + expect_die => "can't migrate local cdrom drive", + expected => { + source_volids => local_volids_for_vm(105), + target_volids => {}, + vm_config => get_patched_config(105, { + ide2 => 'cdrom,media=cdrom', + }), + vm_status => { + running => 0, + }, + }, + }, + { + name => '149_running_missing_option_withlocaldisks', + target => 'pve1', + vmid => 149, + vm_status => { + running => 1, + runningmachine => 'pc-q35-5.0+pve0', + }, + opts => { + online => 1, + }, + expected_calls => {}, + expect_die => "can't live migrate attached local disks without with-local-disks option", + expected => { + source_volids => local_volids_for_vm(149), + target_volids => {}, + vm_config => $vm_configs->{149}, + vm_status => { + running => 1, + runningmachine => 'pc-q35-5.0+pve0', + }, + }, + }, + { + name => '149_running_missing_option_online', + target => 'pve1', + vmid => 149, + vm_status => { + running => 1, + runningmachine => 'pc-q35-5.0+pve0', + }, + opts => { + 'with-local-disks' => 1, + }, + expected_calls => {}, + expect_die => "can't migrate running VM without --online", + expected => { + source_volids => local_volids_for_vm(149), + target_volids => {}, + vm_config => $vm_configs->{149}, + vm_status => { + running => 1, + runningmachine => 'pc-q35-5.0+pve0', + }, + }, + }, + { + name => '1033_running_customcpu', + target => 'pve1', + vmid => 1033, + vm_status => { + running => 1, + runningmachine => 'pc-q35-5.0+pve0', + runningcpu => 'host,+kvm_pv_eoi,+kvm_pv_unhalt', + }, + opts => { + online => 1, + }, + config_patch => { + cpu => 'custom-mycpu', + }, + expected_calls => $default_expected_calls_online, + expected => { + source_volids => {}, + target_volids => {}, + vm_config => get_patched_config(1033, { + cpu => 'custom-mycpu', + }), + vm_status => { + running => 1, + runningmachine => 'pc-q35-5.0+pve0', + runningcpu => 'host,+kvm_pv_eoi,+kvm_pv_unhalt', + }, + }, + }, + { + name => '105_replicated_to_non_replication_target', + target => 'pve1', + vmid => 105, + vm_status => { + running => 0, + }, + target_volids => {}, + expected_calls => $replicated_expected_calls_offline, + expected => { + source_volids => {}, + target_volids => local_volids_for_vm(105), + vm_config => $vm_configs->{105}, + vm_status => { + running => 0, + }, + }, + }, + { + name => '105_running_replicated', + target => 'pve2', + vmid => 105, + vm_status => { + running => 1, + runningmachine => 'pc-i440fx-5.0+pve0', + }, + opts => { + online => 1, + 'with-local-disks' => 1, + }, + target_volids => local_volids_for_vm(105), + expected_calls => { + %{$replicated_expected_calls_online}, + 'block-dirty-bitmap-add-drive-scsi0' => 1, + 'block-dirty-bitmap-add-drive-ide0' => 1, + }, + expected => { + source_volids => local_volids_for_vm(105), + target_volids => local_volids_for_vm(105), + vm_config => $vm_configs->{105}, + vm_status => { + running => 1, + runningmachine => 'pc-i440fx-5.0+pve0', + }, + }, + }, + { + name => '105_replicated', + target => 'pve2', + vmid => 105, + vm_status => { + running => 0, + }, + target_volids => local_volids_for_vm(105), + expected_calls => $replicated_expected_calls_offline, + expected => { + source_volids => local_volids_for_vm(105), + target_volids => local_volids_for_vm(105), + vm_config => $vm_configs->{105}, + vm_status => { + running => 0, + }, + }, + }, + { + name => '105_running_replicated_without_snapshot', + target => 'pve2', + vmid => 105, + vm_status => { + running => 1, + runningmachine => 'pc-i440fx-5.0+pve0', + }, + config_patch => { + snapshots => undef, + }, + opts => { + online => 1, + 'with-local-disks' => 1, + }, + target_volids => local_volids_for_vm(105), + expected_calls => { + %{$replicated_expected_calls_online}, + 'block-dirty-bitmap-add-drive-scsi0' => 1, + 'block-dirty-bitmap-add-drive-ide0' => 1, + }, + expected => { + source_volids => local_volids_for_vm(105), + target_volids => local_volids_for_vm(105), + vm_config => get_patched_config(105, { + snapshots => {}, + }), + vm_status => { + running => 1, + runningmachine => 'pc-i440fx-5.0+pve0', + }, + }, + }, + { + name => '105_replicated_without_snapshot', + target => 'pve2', + vmid => 105, + vm_status => { + running => 0, + }, + config_patch => { + snapshots => undef, + }, + opts => { + online => 1, + }, + target_volids => local_volids_for_vm(105), + expected_calls => $replicated_expected_calls_offline, + expected => { + source_volids => local_volids_for_vm(105), + target_volids => local_volids_for_vm(105), + vm_config => get_patched_config(105, { + snapshots => {}, + }), + vm_status => { + running => 0, + }, + }, + }, + { + name => '1033_running', + target => 'pve2', + vmid => 1033, + vm_status => { + running => 1, + runningmachine => 'pc-q35-5.0+pve0', + }, + opts => { + online => 1, + }, + expected_calls => $default_expected_calls_online, + expected => { + source_volids => {}, + target_volids => {}, + vm_config => $vm_configs->{1033}, + vm_status => { + running => 1, + runningmachine => 'pc-q35-5.0+pve0', + }, + }, + }, + { + name => '149_locked', + target => 'pve2', + vmid => 149, + vm_status => { + running => 0, + }, + config_patch => { + lock => 'locked', + }, + expected_calls => {}, + expect_die => "VM is locked", + expected => { + source_volids => local_volids_for_vm(149), + target_volids => {}, + vm_config => get_patched_config(149, { + lock => 'locked', + }), + vm_status => { + running => 0, + }, + }, + }, + { + name => '149_storage_not_available', + target => 'pve2', + vmid => 149, + vm_status => { + running => 0, + }, + expected_calls => {}, + expect_die => "storage 'local-lvm' is not available on node 'pve2'", + expected => { + source_volids => local_volids_for_vm(149), + target_volids => {}, + vm_config => $vm_configs->{149}, + vm_status => { + running => 0, + }, + }, + }, + { + name => '149_running', + target => 'pve1', + vmid => 149, + vm_status => { + running => 1, + runningmachine => 'pc-q35-5.0+pve0', + }, + opts => { + online => 1, + 'with-local-disks' => 1, + }, + expected_calls => $default_expected_calls_online, + expected => { + source_volids => {}, + target_volids => { + 'local-lvm:vm-149-disk-10' => 1, + 'local-dir:149/vm-149-disk-11.qcow2' => 1, + }, + vm_config => get_patched_config(149, { + scsi0 => 'local-lvm:vm-149-disk-10,format=raw,size=4G', + scsi1 => 'local-dir:149/vm-149-disk-11.qcow2,format=qcow2,size=1G', + }), + vm_status => { + running => 1, + runningmachine => 'pc-q35-5.0+pve0', + }, + }, + }, + { + name => '149_running_drive_mirror_fail', + target => 'pve1', + vmid => 149, + vm_status => { + running => 1, + runningmachine => 'pc-q35-5.0+pve0', + }, + opts => { + online => 1, + 'with-local-disks' => 1, + }, + expected_calls => {}, + expect_die => "qemu_drive_mirror 'scsi1' error", + fail_config => { + 'qemu_drive_mirror' => 'scsi1', + }, + expected => { + source_volids => local_volids_for_vm(149), + target_volids => {}, + vm_config => $vm_configs->{149}, + vm_status => { + running => 1, + runningmachine => 'pc-q35-5.0+pve0', + }, + }, + }, + { + name => '149_running_unused_block_job_cancel_fail', + target => 'pve1', + vmid => 149, + vm_status => { + running => 1, + runningmachine => 'pc-q35-5.0+pve0', + }, + opts => { + online => 1, + 'with-local-disks' => 1, + }, + config_patch => { + scsi1 => undef, + unused0 => 'local-dir:149/vm-149-disk-0.qcow2', + }, + expected_calls => {}, + expect_die => "qemu_drive_mirror_monitor 'cancel' error", + # note that 'cancel' is also used to finish and that's what this test is about + fail_config => { + 'qemu_drive_mirror_monitor' => 'cancel', + }, + expected => { + source_volids => local_volids_for_vm(149), + target_volids => {}, + vm_config => get_patched_config(149, { + scsi1 => undef, + unused0 => 'local-dir:149/vm-149-disk-0.qcow2', + }), + vm_status => { + running => 1, + runningmachine => 'pc-q35-5.0+pve0', + }, + }, + }, + { + name => '149_offline', + target => 'pve1', + vmid => 149, + vm_status => { + running => 0, + }, + opts => { + 'with-local-disks' => 1, + }, + expected_calls => $default_expected_calls_offline, + expected => { + source_volids => {}, + target_volids => local_volids_for_vm(149), + vm_config => $vm_configs->{149}, + vm_status => { + running => 0, + }, + }, + }, + { + name => '149_storage_migrate_fail', + target => 'pve1', + vmid => 149, + vm_status => { + running => 0, + }, + opts => { + 'with-local-disks' => 1, + }, + fail_config => { + 'storage_migrate' => 'local-lvm:vm-149-disk-0', + }, + expected_calls => {}, + expect_die => "storage_migrate 'local-lvm:vm-149-disk-0' error", + expected => { + source_volids => local_volids_for_vm(149), + target_volids => {}, + vm_config => $vm_configs->{149}, + vm_status => { + running => 0, + }, + }, + }, + { + name => '111_running_pending', + target => 'pve1', + vmid => 111, + vm_status => { + running => 1, + runningmachine => 'pc-q35-5.0+pve0', + }, + opts => { + online => 1, + 'with-local-disks' => 1, + }, + expected_calls => $default_expected_calls_online, + expected => { + source_volids => {}, + target_volids => { + 'local-zfs:vm-111-disk-0' => 1, + 'local-lvm:vm-111-disk-10' => 1, + }, + vm_config => get_patched_config(111, { + ide0 => 'local-lvm:vm-111-disk-10,format=raw,size=4G', + pending => { + scsi0 => 'local-zfs:vm-111-disk-0,size=103M', + }, + }), + vm_status => { + running => 1, + runningmachine => 'pc-q35-5.0+pve0', + }, + }, + }, + { + name => '123_alias_fail', + target => 'pve1', + vmid => 123, + vm_status => { + running => 0, + }, + opts => { + 'with-local-disks' => 1, + }, + expected_calls => {}, + expect_die => "detected not supported aliased volumes", + expected => { + source_volids => local_volids_for_vm(123), + target_volids => {}, + vm_config => $vm_configs->{123}, + vm_status => { + running => 0, + }, + }, + }, +]; + +my $single_test_name = shift; + +mkdir $RUN_DIR_PATH; + +foreach my $test (@{$tests}) { + my $name = $test->{name}; + next if defined($single_test_name) && $name ne $single_test_name; + + my $run_dir = "${RUN_DIR_PATH}/${name}"; + + mkdir $run_dir; + file_set_contents("${run_dir}/replication_config", to_json($replication_config)); + file_set_contents("${run_dir}/storage_config", to_json($storage_config)); + file_set_contents("${run_dir}/source_vdisks", to_json($source_vdisks)); + + my $expect_die = $test->{expect_die}; + my $expected = $test->{expected}; + + my $source_volids = local_volids_for_vm($test->{vmid}); + my $target_volids = $test->{target_volids} // {}; + + my $config_patch = $test->{config_patch}; + my $vm_config = get_patched_config($test->{vmid}, $test->{config_patch}); + + my $fail_config = $test->{fail_config} // {}; + my $storage_migrate_map = $test->{storage_migrate_map} // {}; + + if (my $targetstorage = $test->{opts}->{targetstorage}) { + $test->{opts}->{storagemap} = PVE::JSONSchema::parse_idmap($targetstorage, 'pve-storage-id'); + } + + my $migrate_params = { + target => $test->{target}, + vmid => $test->{vmid}, + opts => $test->{opts}, + }; + + file_set_contents("${run_dir}/nbd_info", to_json({})); + file_set_contents("${run_dir}/source_volids", to_json($source_volids)); + file_set_contents("${run_dir}/target_volids", to_json($target_volids)); + file_set_contents("${run_dir}/vm_config", to_json($vm_config)); + file_set_contents("${run_dir}/vm_status", to_json($test->{vm_status})); + file_set_contents("${run_dir}/expected_calls", to_json($test->{expected_calls})); + file_set_contents("${run_dir}/fail_config", to_json($fail_config)); + file_set_contents("${run_dir}/storage_migrate_map", to_json($storage_migrate_map)); + file_set_contents("${run_dir}/migrate_params", to_json($migrate_params)); + + $ENV{QM_LIB_PATH} = $QM_LIB_PATH; + $ENV{RUN_DIR_PATH} = $run_dir; + my $exitcode = run_command([ + '/usr/bin/perl', + "-I${MIGRATE_LIB_PATH}", + "-I${MIGRATE_LIB_PATH}/test", + "${MIGRATE_LIB_PATH}/test/MigrationTest/QemuMigrateMock.pm", + ], noerr => 1, errfunc => sub {print "#$name - $_[0]\n"} ); + + if (defined($expect_die) && $exitcode) { + my $log = file_get_contents("${run_dir}/log"); + my @lines = split /\n/, $log; + + my $matched = 0; + foreach my $line (@lines) { + $matched = 1 if $line =~ m/^err:.*\Q${expect_die}\E/; + $matched = 1 if $line =~ m/^warn:.*\Q${expect_die}\E/; + } + if (!$matched) { + fail($name); + note("expected error message is not present in log"); + } + } elsif (defined($expect_die) && !$exitcode) { + fail($name); + note("mocked migrate call didn't fail, but it was expected to - check log"); + } elsif (!defined($expect_die) && $exitcode) { + fail($name); + note("mocked migrate call failed, but it was not expected - check log"); + } + + my $expected_calls = decode_json(file_get_contents("${run_dir}/expected_calls")); + foreach my $call (keys %{$expected_calls}) { + fail($name); + note("expected call '$call' was not made"); + } + + if (!defined($expect_die)) { + my $nbd_info = decode_json(file_get_contents("${run_dir}/nbd_info")); + foreach my $drive (keys %{$nbd_info}) { + fail($name); + note("drive '$drive' was not mirrored"); + } + } + + my $actual = { + source_volids => decode_json(file_get_contents("${run_dir}/source_volids")), + target_volids => decode_json(file_get_contents("${run_dir}/target_volids")), + vm_config => decode_json(file_get_contents("${run_dir}/vm_config")), + vm_status => decode_json(file_get_contents("${run_dir}/vm_status")), + }; + + is_deeply($actual, $expected, $name); +} + +done_testing(); diff --git a/test/run_qemu_restore_config_tests.pl b/test/run_qemu_restore_config_tests.pl new file mode 100755 index 0000000..1e1e807 --- /dev/null +++ b/test/run_qemu_restore_config_tests.pl @@ -0,0 +1,88 @@ +#!/usr/bin/perl + +use strict; +use warnings; + +use lib qw(..); + +use Test::MockModule; +use Test::More; +use Test::MockModule; + +use File::Basename; + +use PVE::QemuServer; +use PVE::Tools qw(dir_glob_foreach file_get_contents); + +my $INPUT_DIR = './restore-config-input'; +my $EXPECTED_DIR = './restore-config-expected'; + +my $pve_cluster_module = Test::MockModule->new('PVE::Cluster'); +$pve_cluster_module->mock( + cfs_read_file => sub { + return {}; + }, +); + +# NOTE update when you add/remove tests +plan tests => 4; + +my $cfs_mock = Test::MockModule->new("PVE::Cluster"); +$cfs_mock->mock( + cfs_read_file => sub { + my ($file) = @_; + + if ($file eq 'datacenter.cfg') { + return {}; + } else { + die "'cfs_read_file' called - missing mock?\n"; + } + }, +); + +dir_glob_foreach('./restore-config-input', '[0-9]+.conf', sub { + my ($file) = @_; + + my $vmid = basename($file, ('.conf')); + + my $fh = IO::File->new("${INPUT_DIR}/${file}", "r") or + die "unable to read '$file' - $!\n"; + + my $map = {}; + my $disknum = 0; + + # NOTE For now, the map is hardcoded to a file-based 'target' storage. + # In the future, the test could be extended to include parse_backup_hints + # and restore_allocate_devices. Even better if the config-related logic from + # the restore_XYZ_archive functions could become a separate function. + while (defined(my $line = <$fh>)) { + if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) { + my ($drive, undef, $storeid, $fmt) = ($1, $2, $3, $4); + + $fmt ||= 'raw'; + + $map->{$drive} = "target:${vmid}/vm-${vmid}-disk-${disknum}.${fmt}"; + $disknum++; + } + } + + $fh->seek(0, 0) or die "seek failed - $!\n"; + + my $got = ''; + my $cookie = { netcount => 0 }; + + while (defined(my $line = <$fh>)) { + $got .= PVE::QemuServer::restore_update_config_line( + $cookie, + $map, + $line, + 0, + ); + } + + my $expected = file_get_contents("${EXPECTED_DIR}/${file}"); + + is_deeply($got, $expected, $file); +}); + +done_testing(); diff --git a/test/snapshot-expected/commit/qemu-server/101.conf b/test/snapshot-expected/commit/qemu-server/101.conf index 060676e..82c9522 100644 --- a/test/snapshot-expected/commit/qemu-server/101.conf +++ b/test/snapshot-expected/commit/qemu-server/101.conf @@ -18,7 +18,7 @@ bootdisk: ide0 cores: 4 ide0: somestore:somedisk,discard=on,size=32G ide2: none,media=cdrom -machine: somemachine +machine: q35 memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 diff --git a/test/snapshot-expected/commit/qemu-server/102.conf b/test/snapshot-expected/commit/qemu-server/102.conf index 13f33a3..01b8531 100644 --- a/test/snapshot-expected/commit/qemu-server/102.conf +++ b/test/snapshot-expected/commit/qemu-server/102.conf @@ -34,7 +34,7 @@ bootdisk: ide0 cores: 4 ide0: somestore:somedisk,discard=on,size=32G ide2: none,media=cdrom -machine: somemachine +machine: q35 memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 diff --git a/test/snapshot-expected/commit/qemu-server/201.conf b/test/snapshot-expected/commit/qemu-server/201.conf index 63a6d77..f8e99dd 100644 --- a/test/snapshot-expected/commit/qemu-server/201.conf +++ b/test/snapshot-expected/commit/qemu-server/201.conf @@ -34,7 +34,7 @@ bootdisk: ide0 cores: 4 ide0: somestore:somedisk,discard=on,size=32G ide2: none,media=cdrom -machine: somemachine +machine: q35 memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 diff --git a/test/snapshot-expected/commit/qemu-server/202.conf b/test/snapshot-expected/commit/qemu-server/202.conf index 4c6b84d..a221ba0 100644 --- a/test/snapshot-expected/commit/qemu-server/202.conf +++ b/test/snapshot-expected/commit/qemu-server/202.conf @@ -35,7 +35,7 @@ bootdisk: ide0 cores: 4 ide0: somestore:somedisk,discard=on,size=32G ide2: none,media=cdrom -machine: somemachine +machine: q35 memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 diff --git a/test/snapshot-expected/commit/qemu-server/203.conf b/test/snapshot-expected/commit/qemu-server/203.conf index 5acf20d..e10b68e 100644 --- a/test/snapshot-expected/commit/qemu-server/203.conf +++ b/test/snapshot-expected/commit/qemu-server/203.conf @@ -35,7 +35,7 @@ bootdisk: ide0 cores: 4 ide0: somestore:somedisk,discard=on,size=32G ide2: none,media=cdrom -machine: somemachine +machine: q35 memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 diff --git a/test/snapshot-expected/create/qemu-server/102.conf b/test/snapshot-expected/create/qemu-server/102.conf index 9b57004..d507956 100644 --- a/test/snapshot-expected/create/qemu-server/102.conf +++ b/test/snapshot-expected/create/qemu-server/102.conf @@ -25,7 +25,7 @@ name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 -runningmachine: somemachine +runningmachine: q35 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd snaptime: 1234567890 sockets: 1 diff --git a/test/snapshot-expected/create/qemu-server/104.conf b/test/snapshot-expected/create/qemu-server/104.conf index 54f1c21..385625f 100644 --- a/test/snapshot-expected/create/qemu-server/104.conf +++ b/test/snapshot-expected/create/qemu-server/104.conf @@ -20,7 +20,7 @@ bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom -machine: somemachine +machine: q35 memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 @@ -45,7 +45,7 @@ net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 parent: test -runningmachine: somemachine +runningmachine: q35 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd snaptime: 1234567890 sockets: 1 diff --git a/test/snapshot-expected/create/qemu-server/106.conf b/test/snapshot-expected/create/qemu-server/106.conf index 9b57004..d507956 100644 --- a/test/snapshot-expected/create/qemu-server/106.conf +++ b/test/snapshot-expected/create/qemu-server/106.conf @@ -25,7 +25,7 @@ name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 -runningmachine: somemachine +runningmachine: q35 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd snaptime: 1234567890 sockets: 1 diff --git a/test/snapshot-expected/create/qemu-server/301.conf b/test/snapshot-expected/create/qemu-server/301.conf index 9c49b1d..8cba2dc 100644 --- a/test/snapshot-expected/create/qemu-server/301.conf +++ b/test/snapshot-expected/create/qemu-server/301.conf @@ -25,7 +25,7 @@ name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 -runningmachine: somemachine +runningmachine: q35 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd snaptime: 1234567890 sockets: 1 diff --git a/test/snapshot-expected/create/qemu-server/302.conf b/test/snapshot-expected/create/qemu-server/302.conf index 9c49b1d..8cba2dc 100644 --- a/test/snapshot-expected/create/qemu-server/302.conf +++ b/test/snapshot-expected/create/qemu-server/302.conf @@ -25,7 +25,7 @@ name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 -runningmachine: somemachine +runningmachine: q35 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd snaptime: 1234567890 sockets: 1 diff --git a/test/snapshot-expected/create/qemu-server/303.conf b/test/snapshot-expected/create/qemu-server/303.conf new file mode 100644 index 0000000..2731bd1 --- /dev/null +++ b/test/snapshot-expected/create/qemu-server/303.conf @@ -0,0 +1,13 @@ +bootdisk: ide0 +cores: 4 +ide0: local:snapshotable-disk-1,discard=on,size=32G +ide2: none,media=cdrom +machine: q35 +memory: 8192 +name: win +net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 +numa: 0 +ostype: win7 +smbios1: uuid=01234567-890a-bcde-f012-34567890abcd +sockets: 1 +vga: qxl diff --git a/test/snapshot-expected/delete/qemu-server/203.conf b/test/snapshot-expected/delete/qemu-server/203.conf index c406640..ed93cf7 100644 --- a/test/snapshot-expected/delete/qemu-server/203.conf +++ b/test/snapshot-expected/delete/qemu-server/203.conf @@ -21,7 +21,7 @@ bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom -machine: somemachine +machine: q35 memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 diff --git a/test/snapshot-expected/delete/qemu-server/204.conf b/test/snapshot-expected/delete/qemu-server/204.conf new file mode 100644 index 0000000..fe63204 --- /dev/null +++ b/test/snapshot-expected/delete/qemu-server/204.conf @@ -0,0 +1,33 @@ +agent: 1 +bootdisk: ide0 +cores: 4 +ide0: local:snapshotable-disk-1,discard=on,size=32G +ide2: none,media=cdrom +memory: 8192 +name: win +net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 +numa: 0 +ostype: win7 +parent: test +smbios1: uuid=01234567-890a-bcde-f012-34567890abcd +sockets: 1 +vga: qxl + +[test] +#test comment +agent: 1 +bootdisk: ide0 +cores: 4 +ide0: local:snapshotable-disk-1,discard=on,size=32G +ide2: none,media=cdrom +machine: q35 +memory: 8192 +name: win +net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 +numa: 0 +ostype: win7 +smbios1: uuid=01234567-890a-bcde-f012-34567890abcd +snaptime: 1234567890 +sockets: 1 +vga: qxl +vmstate: somestorage:state-volume diff --git a/test/snapshot-expected/prepare/qemu-server/102.conf b/test/snapshot-expected/prepare/qemu-server/102.conf index 92db74a..8f998ca 100644 --- a/test/snapshot-expected/prepare/qemu-server/102.conf +++ b/test/snapshot-expected/prepare/qemu-server/102.conf @@ -23,7 +23,7 @@ name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 -runningmachine: somemachine +runningmachine: q35 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd snapstate: prepare snaptime: 1234567890 diff --git a/test/snapshot-expected/prepare/qemu-server/104.conf b/test/snapshot-expected/prepare/qemu-server/104.conf index 02e2d3c..2f2ec96 100644 --- a/test/snapshot-expected/prepare/qemu-server/104.conf +++ b/test/snapshot-expected/prepare/qemu-server/104.conf @@ -41,7 +41,7 @@ net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 parent: test -runningmachine: somemachine +runningmachine: q35 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd snapstate: prepare snaptime: 1234567890 diff --git a/test/snapshot-expected/rollback/qemu-server/101.conf b/test/snapshot-expected/rollback/qemu-server/101.conf index 91de880..17257e3 100644 --- a/test/snapshot-expected/rollback/qemu-server/101.conf +++ b/test/snapshot-expected/rollback/qemu-server/101.conf @@ -21,7 +21,7 @@ bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom -machine: somemachine +machine: q35 memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 diff --git a/test/snapshot-expected/rollback/qemu-server/106.conf b/test/snapshot-expected/rollback/qemu-server/106.conf index aa5fa9e..729bc93 100644 --- a/test/snapshot-expected/rollback/qemu-server/106.conf +++ b/test/snapshot-expected/rollback/qemu-server/106.conf @@ -3,7 +3,7 @@ bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom -machine: somemachine +machine: q35 memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 @@ -21,7 +21,7 @@ bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom -machine: somemachine +machine: q35 memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 diff --git a/test/snapshot-expected/rollback/qemu-server/201.conf b/test/snapshot-expected/rollback/qemu-server/201.conf index c521154..fe63204 100644 --- a/test/snapshot-expected/rollback/qemu-server/201.conf +++ b/test/snapshot-expected/rollback/qemu-server/201.conf @@ -20,7 +20,7 @@ bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom -machine: somemachine +machine: q35 memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 diff --git a/test/snapshot-expected/rollback/qemu-server/202.conf b/test/snapshot-expected/rollback/qemu-server/202.conf index 691f5a2..d09b5d0 100644 --- a/test/snapshot-expected/rollback/qemu-server/202.conf +++ b/test/snapshot-expected/rollback/qemu-server/202.conf @@ -20,7 +20,7 @@ bootdisk: ide0 cores: 4 ide0: local:unsnapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom -machine: somemachine +machine: q35 memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 diff --git a/test/snapshot-expected/rollback/qemu-server/203.conf b/test/snapshot-expected/rollback/qemu-server/203.conf index 6e53b27..8abf841 100644 --- a/test/snapshot-expected/rollback/qemu-server/203.conf +++ b/test/snapshot-expected/rollback/qemu-server/203.conf @@ -20,7 +20,7 @@ bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom -machine: somemachine +machine: q35 memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 diff --git a/test/snapshot-expected/rollback/qemu-server/204.conf b/test/snapshot-expected/rollback/qemu-server/204.conf index c406640..ed93cf7 100644 --- a/test/snapshot-expected/rollback/qemu-server/204.conf +++ b/test/snapshot-expected/rollback/qemu-server/204.conf @@ -21,7 +21,7 @@ bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom -machine: somemachine +machine: q35 memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 diff --git a/test/snapshot-expected/rollback/qemu-server/205.conf b/test/snapshot-expected/rollback/qemu-server/205.conf index c521154..fe63204 100644 --- a/test/snapshot-expected/rollback/qemu-server/205.conf +++ b/test/snapshot-expected/rollback/qemu-server/205.conf @@ -20,7 +20,7 @@ bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom -machine: somemachine +machine: q35 memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 diff --git a/test/snapshot-expected/rollback/qemu-server/301.conf b/test/snapshot-expected/rollback/qemu-server/301.conf index c521154..fe63204 100644 --- a/test/snapshot-expected/rollback/qemu-server/301.conf +++ b/test/snapshot-expected/rollback/qemu-server/301.conf @@ -20,7 +20,7 @@ bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom -machine: somemachine +machine: q35 memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 diff --git a/test/snapshot-expected/rollback/qemu-server/302.conf b/test/snapshot-expected/rollback/qemu-server/302.conf index 828e8b0..5110016 100644 --- a/test/snapshot-expected/rollback/qemu-server/302.conf +++ b/test/snapshot-expected/rollback/qemu-server/302.conf @@ -27,7 +27,7 @@ name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 -runningmachine: somemachine +runningmachine: q35 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd snaptime: 1234567890 sockets: 1 diff --git a/test/snapshot-expected/rollback/qemu-server/303.conf b/test/snapshot-expected/rollback/qemu-server/303.conf new file mode 100644 index 0000000..473a9a0 --- /dev/null +++ b/test/snapshot-expected/rollback/qemu-server/303.conf @@ -0,0 +1,34 @@ +agent: 1 +bootdisk: ide0 +cores: 4 +ide0: local:snapshotable-disk-1,discard=on,size=32G +ide2: none,media=cdrom +memory: 8192 +name: win +net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 +numa: 0 +ostype: win7 +parent: test +smbios1: uuid=01234567-890a-bcde-f012-34567890abcd +sockets: 1 +vga: qxl + +[test] +#test comment +agent: 1 +bootdisk: ide0 +cores: 4 +ide0: local:snapshotable-disk-1,discard=on,size=32G +ide2: none,media=cdrom +machine: q35 +memory: 8192 +name: win +net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 +numa: 0 +ostype: win7 +runningmachine: q35 +smbios1: uuid=01234567-890a-bcde-f012-34567890abcd +snaptime: 1234567890 +sockets: 1 +vga: qxl +vmstate: somestorage:state-volume diff --git a/test/snapshot-input/commit/qemu-server/101.conf b/test/snapshot-input/commit/qemu-server/101.conf index 4ab1787..92c1f6a 100644 --- a/test/snapshot-input/commit/qemu-server/101.conf +++ b/test/snapshot-input/commit/qemu-server/101.conf @@ -18,7 +18,7 @@ bootdisk: ide0 cores: 4 ide0: somestore:somedisk,discard=on,size=32G ide2: none,media=cdrom -machine: somemachine +machine: q35 memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 diff --git a/test/snapshot-input/commit/qemu-server/102.conf b/test/snapshot-input/commit/qemu-server/102.conf index b62f2c6..99bca5e 100644 --- a/test/snapshot-input/commit/qemu-server/102.conf +++ b/test/snapshot-input/commit/qemu-server/102.conf @@ -35,7 +35,7 @@ bootdisk: ide0 cores: 4 ide0: somestore:somedisk,discard=on,size=32G ide2: none,media=cdrom -machine: somemachine +machine: q35 memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 diff --git a/test/snapshot-input/commit/qemu-server/201.conf b/test/snapshot-input/commit/qemu-server/201.conf index 63a6d77..f8e99dd 100644 --- a/test/snapshot-input/commit/qemu-server/201.conf +++ b/test/snapshot-input/commit/qemu-server/201.conf @@ -34,7 +34,7 @@ bootdisk: ide0 cores: 4 ide0: somestore:somedisk,discard=on,size=32G ide2: none,media=cdrom -machine: somemachine +machine: q35 memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 diff --git a/test/snapshot-input/commit/qemu-server/202.conf b/test/snapshot-input/commit/qemu-server/202.conf index 4c6b84d..a221ba0 100644 --- a/test/snapshot-input/commit/qemu-server/202.conf +++ b/test/snapshot-input/commit/qemu-server/202.conf @@ -35,7 +35,7 @@ bootdisk: ide0 cores: 4 ide0: somestore:somedisk,discard=on,size=32G ide2: none,media=cdrom -machine: somemachine +machine: q35 memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 diff --git a/test/snapshot-input/commit/qemu-server/203.conf b/test/snapshot-input/commit/qemu-server/203.conf index 5acf20d..e10b68e 100644 --- a/test/snapshot-input/commit/qemu-server/203.conf +++ b/test/snapshot-input/commit/qemu-server/203.conf @@ -35,7 +35,7 @@ bootdisk: ide0 cores: 4 ide0: somestore:somedisk,discard=on,size=32G ide2: none,media=cdrom -machine: somemachine +machine: q35 memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 diff --git a/test/snapshot-input/create/qemu-server/104.conf b/test/snapshot-input/create/qemu-server/104.conf index c521154..fe63204 100644 --- a/test/snapshot-input/create/qemu-server/104.conf +++ b/test/snapshot-input/create/qemu-server/104.conf @@ -20,7 +20,7 @@ bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom -machine: somemachine +machine: q35 memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 diff --git a/test/snapshot-input/create/qemu-server/303.conf b/test/snapshot-input/create/qemu-server/303.conf new file mode 100644 index 0000000..2731bd1 --- /dev/null +++ b/test/snapshot-input/create/qemu-server/303.conf @@ -0,0 +1,13 @@ +bootdisk: ide0 +cores: 4 +ide0: local:snapshotable-disk-1,discard=on,size=32G +ide2: none,media=cdrom +machine: q35 +memory: 8192 +name: win +net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 +numa: 0 +ostype: win7 +smbios1: uuid=01234567-890a-bcde-f012-34567890abcd +sockets: 1 +vga: qxl diff --git a/test/snapshot-input/delete/qemu-server/101.conf b/test/snapshot-input/delete/qemu-server/101.conf index c521154..fe63204 100644 --- a/test/snapshot-input/delete/qemu-server/101.conf +++ b/test/snapshot-input/delete/qemu-server/101.conf @@ -20,7 +20,7 @@ bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom -machine: somemachine +machine: q35 memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 diff --git a/test/snapshot-input/delete/qemu-server/203.conf b/test/snapshot-input/delete/qemu-server/203.conf index c406640..ed93cf7 100644 --- a/test/snapshot-input/delete/qemu-server/203.conf +++ b/test/snapshot-input/delete/qemu-server/203.conf @@ -21,7 +21,7 @@ bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom -machine: somemachine +machine: q35 memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 diff --git a/test/snapshot-input/delete/qemu-server/204.conf b/test/snapshot-input/delete/qemu-server/204.conf new file mode 100644 index 0000000..fe63204 --- /dev/null +++ b/test/snapshot-input/delete/qemu-server/204.conf @@ -0,0 +1,33 @@ +agent: 1 +bootdisk: ide0 +cores: 4 +ide0: local:snapshotable-disk-1,discard=on,size=32G +ide2: none,media=cdrom +memory: 8192 +name: win +net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 +numa: 0 +ostype: win7 +parent: test +smbios1: uuid=01234567-890a-bcde-f012-34567890abcd +sockets: 1 +vga: qxl + +[test] +#test comment +agent: 1 +bootdisk: ide0 +cores: 4 +ide0: local:snapshotable-disk-1,discard=on,size=32G +ide2: none,media=cdrom +machine: q35 +memory: 8192 +name: win +net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 +numa: 0 +ostype: win7 +smbios1: uuid=01234567-890a-bcde-f012-34567890abcd +snaptime: 1234567890 +sockets: 1 +vga: qxl +vmstate: somestorage:state-volume diff --git a/test/snapshot-input/rollback/qemu-server/101.conf b/test/snapshot-input/rollback/qemu-server/101.conf index 4fea865..0fa6a61 100644 --- a/test/snapshot-input/rollback/qemu-server/101.conf +++ b/test/snapshot-input/rollback/qemu-server/101.conf @@ -20,7 +20,7 @@ bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom -machine: somemachine +machine: q35 memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 diff --git a/test/snapshot-input/rollback/qemu-server/102.conf b/test/snapshot-input/rollback/qemu-server/102.conf index 8f0db83..3fcffe1 100644 --- a/test/snapshot-input/rollback/qemu-server/102.conf +++ b/test/snapshot-input/rollback/qemu-server/102.conf @@ -4,7 +4,7 @@ bootdisk: ide2 cores: 2 ide0: local:snapshotable-disk-1,size=32G ide2: none,media=cdrom -machine: someothermachine +machine: pc memory: 4096 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 diff --git a/test/snapshot-input/rollback/qemu-server/103.conf b/test/snapshot-input/rollback/qemu-server/103.conf index 8f0db83..3fcffe1 100644 --- a/test/snapshot-input/rollback/qemu-server/103.conf +++ b/test/snapshot-input/rollback/qemu-server/103.conf @@ -4,7 +4,7 @@ bootdisk: ide2 cores: 2 ide0: local:snapshotable-disk-1,size=32G ide2: none,media=cdrom -machine: someothermachine +machine: pc memory: 4096 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 diff --git a/test/snapshot-input/rollback/qemu-server/104.conf b/test/snapshot-input/rollback/qemu-server/104.conf index ff50151..2f12761 100644 --- a/test/snapshot-input/rollback/qemu-server/104.conf +++ b/test/snapshot-input/rollback/qemu-server/104.conf @@ -4,7 +4,7 @@ bootdisk: ide2 cores: 2 ide0: local:snapshotable-disk-1,size=32G ide2: none,media=cdrom -machine: someothermachine +machine: pc memory: 4096 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 diff --git a/test/snapshot-input/rollback/qemu-server/106.conf b/test/snapshot-input/rollback/qemu-server/106.conf index 64bf9bf..56d7199 100644 --- a/test/snapshot-input/rollback/qemu-server/106.conf +++ b/test/snapshot-input/rollback/qemu-server/106.conf @@ -3,7 +3,7 @@ bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom -machine: someothermachine +machine: pc memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 @@ -21,7 +21,7 @@ bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom -machine: somemachine +machine: q35 memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 diff --git a/test/snapshot-input/rollback/qemu-server/201.conf b/test/snapshot-input/rollback/qemu-server/201.conf index c521154..fe63204 100644 --- a/test/snapshot-input/rollback/qemu-server/201.conf +++ b/test/snapshot-input/rollback/qemu-server/201.conf @@ -20,7 +20,7 @@ bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom -machine: somemachine +machine: q35 memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 diff --git a/test/snapshot-input/rollback/qemu-server/202.conf b/test/snapshot-input/rollback/qemu-server/202.conf index 691f5a2..d09b5d0 100644 --- a/test/snapshot-input/rollback/qemu-server/202.conf +++ b/test/snapshot-input/rollback/qemu-server/202.conf @@ -20,7 +20,7 @@ bootdisk: ide0 cores: 4 ide0: local:unsnapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom -machine: somemachine +machine: q35 memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 diff --git a/test/snapshot-input/rollback/qemu-server/203.conf b/test/snapshot-input/rollback/qemu-server/203.conf index 6e53b27..8abf841 100644 --- a/test/snapshot-input/rollback/qemu-server/203.conf +++ b/test/snapshot-input/rollback/qemu-server/203.conf @@ -20,7 +20,7 @@ bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom -machine: somemachine +machine: q35 memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 diff --git a/test/snapshot-input/rollback/qemu-server/204.conf b/test/snapshot-input/rollback/qemu-server/204.conf index c406640..ed93cf7 100644 --- a/test/snapshot-input/rollback/qemu-server/204.conf +++ b/test/snapshot-input/rollback/qemu-server/204.conf @@ -21,7 +21,7 @@ bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom -machine: somemachine +machine: q35 memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 diff --git a/test/snapshot-input/rollback/qemu-server/205.conf b/test/snapshot-input/rollback/qemu-server/205.conf index c521154..fe63204 100644 --- a/test/snapshot-input/rollback/qemu-server/205.conf +++ b/test/snapshot-input/rollback/qemu-server/205.conf @@ -20,7 +20,7 @@ bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom -machine: somemachine +machine: q35 memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 diff --git a/test/snapshot-input/rollback/qemu-server/301.conf b/test/snapshot-input/rollback/qemu-server/301.conf index c521154..fe63204 100644 --- a/test/snapshot-input/rollback/qemu-server/301.conf +++ b/test/snapshot-input/rollback/qemu-server/301.conf @@ -20,7 +20,7 @@ bootdisk: ide0 cores: 4 ide0: local:snapshotable-disk-1,discard=on,size=32G ide2: none,media=cdrom -machine: somemachine +machine: q35 memory: 8192 name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 diff --git a/test/snapshot-input/rollback/qemu-server/302.conf b/test/snapshot-input/rollback/qemu-server/302.conf index 518c954..473a9a0 100644 --- a/test/snapshot-input/rollback/qemu-server/302.conf +++ b/test/snapshot-input/rollback/qemu-server/302.conf @@ -26,7 +26,7 @@ name: win net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 numa: 0 ostype: win7 -runningmachine: somemachine +runningmachine: q35 smbios1: uuid=01234567-890a-bcde-f012-34567890abcd snaptime: 1234567890 sockets: 1 diff --git a/test/snapshot-input/rollback/qemu-server/303.conf b/test/snapshot-input/rollback/qemu-server/303.conf new file mode 100644 index 0000000..473a9a0 --- /dev/null +++ b/test/snapshot-input/rollback/qemu-server/303.conf @@ -0,0 +1,34 @@ +agent: 1 +bootdisk: ide0 +cores: 4 +ide0: local:snapshotable-disk-1,discard=on,size=32G +ide2: none,media=cdrom +memory: 8192 +name: win +net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 +numa: 0 +ostype: win7 +parent: test +smbios1: uuid=01234567-890a-bcde-f012-34567890abcd +sockets: 1 +vga: qxl + +[test] +#test comment +agent: 1 +bootdisk: ide0 +cores: 4 +ide0: local:snapshotable-disk-1,discard=on,size=32G +ide2: none,media=cdrom +machine: q35 +memory: 8192 +name: win +net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1 +numa: 0 +ostype: win7 +runningmachine: q35 +smbios1: uuid=01234567-890a-bcde-f012-34567890abcd +snaptime: 1234567890 +sockets: 1 +vga: qxl +vmstate: somestorage:state-volume diff --git a/test/snapshot-test.pm b/test/snapshot-test.pm index 8f9f60a..f130a5a 100644 --- a/test/snapshot-test.pm +++ b/test/snapshot-test.pm @@ -15,6 +15,7 @@ use PVE::ReplicationConfig; use Test::MockModule; use Test::More; +my $activate_storage_possible = 1; my $nodename; my $snapshot_possible; my $vol_snapshot_possible = {}; @@ -105,6 +106,15 @@ sub mocked_volume_rollback_is_possible { die "volume_rollback_is_possible failed\n"; } +sub mocked_activate_storage { + my ($storecfg, $storeid) = @_; + die "Storage config not mocked! aborting\n" + if defined($storecfg); + die "storage activation failed\n" + if !$activate_storage_possible; + return; +} + sub mocked_activate_volumes { my ($storecfg, $volumes) = @_; die "Storage config not mocked! aborting\n" @@ -295,7 +305,7 @@ sub __snapshot_save_vmstate { my $snap = $conf->{snapshots}->{$snapname}; $snap->{vmstate} = "somestorage:state-volume"; - $snap->{runningmachine} = "somemachine" + $snap->{runningmachine} = "q35" } sub assert_config_exists_on_node { @@ -318,9 +328,6 @@ sub qmp_cmd { my ($vmid, $cmd) = @_; my $exec = $cmd->{execute}; - if ($exec eq "delete-drive-snapshot") { - return; - } if ($exec eq "guest-ping") { die "guest-ping disabled\n" if !$vm_mon->{guest_ping}; @@ -342,7 +349,11 @@ sub qmp_cmd { return; } if ($exec eq "query-savevm") { - return { "status" => "completed" }; + return { + "status" => "completed", + "bytes" => 1024*1024*1024, + "total-time" => 5000, + }; } die "unexpected vm_qmp_command!\n"; } @@ -376,6 +387,8 @@ sub vm_stop { return; } +sub set_migration_caps {} # ignored + # END redefine PVE::QemuServer methods PVE::Tools::run_command("rm -rf snapshot-working"); @@ -404,6 +417,7 @@ $repl_config_module->mock('check_for_existing_jobs' => sub { return }); my $storage_module = Test::MockModule->new('PVE::Storage'); $storage_module->mock('config', sub { return; }); $storage_module->mock('path', sub { return "/some/store/statefile/path"; }); +$storage_module->mock('activate_storage', \&mocked_activate_storage); $storage_module->mock('activate_volumes', \&mocked_activate_volumes); $storage_module->mock('deactivate_volumes', \&mocked_deactivate_volumes); $storage_module->mock('vdisk_free', \&mocked_vdisk_free); @@ -542,13 +556,20 @@ printf("Expected error for snapshot_create when volume snapshot is not possible testcase_create("202", "test", 0, "test comment", "volume snapshot disabled\n\n", { "local:snapshotable-disk-1" => "test" }, { "local:snapshotable-disk-1" => "test" }); $vm_mon->{savevm_start} = 0; -printf("Expected error for snapshot_create when Qemu mon command 'savevm-start' fails\n"); +printf("Expected error for snapshot_create when QEMU mon command 'savevm-start' fails\n"); testcase_create("203", "test", 0, "test comment", "savevm-start disabled\n\n"); $vm_mon->{savevm_start} = 1; printf("Successful snapshot_create with no existing snapshots but set machine type\n"); testcase_create("301", "test", 1, "test comment", "", { "local:snapshotable-disk-1" => "test" }); +$activate_storage_possible = 0; + +printf("Expected error for snapshot_create when storage activation is not possible\n"); +testcase_create("303", "test", 1, "test comment", "storage activation failed\n\n"); + +$activate_storage_possible = 1; + $nodename = "delete"; printf("\n"); printf("Running delete tests\n"); @@ -581,6 +602,13 @@ testcase_delete("202", "test", 0, "volume snapshot delete disabled\n", { "local: printf("Expected error for snapshot_delete with locked config\n"); testcase_delete("203", "test", 0, "VM is locked (backup)\n"); +$activate_storage_possible = 0; + +printf("Expected error for snapshot_delete when storage activation is not possible\n"); +testcase_delete("204", "test", 0, "storage activation failed\n"); + +$activate_storage_possible = 1; + $nodename = "rollback"; printf("\n"); printf("Running rollback tests\n"); @@ -637,6 +665,13 @@ testcase_rollback("301", "test", "", { "local:snapshotable-disk-1" => "test" }); printf("Successful snapshot_rollback with saved vmstate and machine config and runningmachine \n"); testcase_rollback("302", "test", "", { "local:snapshotable-disk-1" => "test" }); +$activate_storage_possible = 0; + +printf("Expected error for snapshot_rollback when storage activation is not possible\n"); +testcase_rollback("303", "test", "storage activation failed\n"); + +$activate_storage_possible = 1; + done_testing(); 1; diff --git a/vm-network-scripts/pve-bridge b/vm-network-scripts/pve-bridge index d37ce33..299be1f 100755 --- a/vm-network-scripts/pve-bridge +++ b/vm-network-scripts/pve-bridge @@ -6,10 +6,12 @@ use warnings; use PVE::QemuServer; use PVE::Tools qw(run_command); use PVE::Network; +use PVE::Firewall; my $have_sdn; eval { require PVE::Network::SDN::Zones; + require PVE::Network::SDN::Vnets; $have_sdn = 1; }; @@ -36,19 +38,23 @@ my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom); my $netconf = $conf->{$netid}; $netconf = $conf->{pending}->{$netid} if !$migratedfrom && defined($conf->{pending}->{$netid}); - + die "unable to get network config '$netid'\n" if !defined($netconf); my $net = PVE::QemuServer::parse_net($netconf); die "unable to parse network config '$netid'\n" if !$net; +# The nftable-based implementation from the newer proxmox-firewall does not requires FW bridges +my $create_firewall_bridges = $net->{firewall} && !PVE::Firewall::is_nftables(); + if ($have_sdn) { + PVE::Network::SDN::Vnets::add_dhcp_mapping($net->{bridge}, $net->{macaddr}, $vmid, $conf->{name}); PVE::Network::SDN::Zones::tap_create($iface, $net->{bridge}); - PVE::Network::SDN::Zones::tap_plug($iface, $net->{bridge}, $net->{tag}, $net->{firewall}, $net->{trunks}, $net->{rate}); + PVE::Network::SDN::Zones::tap_plug($iface, $net->{bridge}, $net->{tag}, $create_firewall_bridges, $net->{trunks}, $net->{rate}); } else { PVE::Network::tap_create($iface, $net->{bridge}); - PVE::Network::tap_plug($iface, $net->{bridge}, $net->{tag}, $net->{firewall}, $net->{trunks}, $net->{rate}); + PVE::Network::tap_plug($iface, $net->{bridge}, $net->{tag}, $create_firewall_bridges, $net->{trunks}, $net->{rate}); } exit 0;