-*.deb
-*.buildinfo
-*.changes
*.1.gz
*.1.pod
*.5.gz
*.5.pod
-sparsecp
-vmtar
+*.buildinfo
+*.changes
+*.deb
+/.vscode/
+/qemu-server-[0-9]*/
build
qm.bash-completion
+sparsecp
+vmtar
-include /usr/share/dpkg/pkg-info.mk
-include /usr/share/dpkg/architecture.mk
+include /usr/share/dpkg/default.mk
PACKAGE=qemu-server
-BUILDDIR ?= ${PACKAGE}-${DEB_VERSION_UPSTREAM}
+BUILDDIR ?= $(PACKAGE)-$(DEB_VERSION_UPSTREAM)
DESTDIR=
PREFIX=/usr
-SBINDIR=${PREFIX}/sbin
-LIBDIR=${PREFIX}/lib/${PACKAGE}
-MANDIR=${PREFIX}/share/man
-DOCDIR=${PREFIX}/share/doc
-MAN1DIR=${MANDIR}/man1/
-MAN5DIR=${MANDIR}/man5/
-BASHCOMPLDIR=${PREFIX}/share/bash-completion/completions/
-ZSHCOMPLDIR=${PREFIX}/share/zsh/vendor-completions/
-export PERLDIR=${PREFIX}/share/perl5
-PERLINCDIR=${PERLDIR}/asm-x86_64
+SBINDIR=$(PREFIX)/sbin
+LIBDIR=$(PREFIX)/lib/$(PACKAGE)
+MANDIR=$(PREFIX)/share/man
+DOCDIR=$(PREFIX)/share/doc
+MAN1DIR=$(MANDIR)/man1/
+MAN5DIR=$(MANDIR)/man5/
+BASHCOMPLDIR=$(PREFIX)/share/bash-completion/completions/
+ZSHCOMPLDIR=$(PREFIX)/share/zsh/vendor-completions/
+export PERLDIR=$(PREFIX)/share/perl5
+PERLINCDIR=$(PERLDIR)/asm-x86_64
GITVERSION:=$(shell git rev-parse HEAD)
-DEB=${PACKAGE}_${DEB_VERSION_UPSTREAM_REVISION}_${DEB_BUILD_ARCH}.deb
-DBG_DEB=${PACKAGE}-dbgsym_${DEB_VERSION_UPSTREAM_REVISION}_${DEB_BUILD_ARCH}.deb
-DSC=${PACKAGE}_${DEB_VERSION_UPSTREAM_REVISION}.dsc
+DEB=$(PACKAGE)_$(DEB_VERSION_UPSTREAM_REVISION)_$(DEB_BUILD_ARCH).deb
+DBG_DEB=$(PACKAGE)-dbgsym_$(DEB_VERSION_UPSTREAM_REVISION)_$(DEB_BUILD_ARCH).deb
+DSC=$(PACKAGE)_$(DEB_VERSION_UPSTREAM_REVISION).dsc
-DEBS=${DEB} ${DBG_DEB}
+DEBS=$(DEB) $(DBG_DEB)
-# this requires package pve-doc-generator
-export NOVIEW=1
-include /usr/share/pve-doc-generator/pve-doc-generator.mk
+-include /usr/share/pve-doc-generator/pve-doc-generator.mk
all:
.PHONY: dinstall
dinstall: deb
- dpkg -i ${DEB}
+ dpkg -i $(DEB)
qm.bash-completion:
PVE_GENERATING_DOCS=1 perl -I. -T -e "use PVE::CLI::qm; PVE::CLI::qm->generate_bash_completions();" >$@.tmp
qm.zsh-completion qmrestore.zsh-completion cpu-models.conf.5
.PHONY: install
-install: ${PKGSOURCES}
- install -d ${DESTDIR}/${SBINDIR}
- install -d ${DESTDIR}${LIBDIR}
- install -d ${DESTDIR}/${MAN1DIR}
- install -d ${DESTDIR}/${MAN5DIR}
- install -d ${DESTDIR}/usr/share/${PACKAGE}
- install -m 0644 -D qm.bash-completion ${DESTDIR}/${BASHCOMPLDIR}/qm
- install -m 0644 -D qmrestore.bash-completion ${DESTDIR}/${BASHCOMPLDIR}/qmrestore
- install -m 0644 -D qm.zsh-completion ${DESTDIR}/${ZSHCOMPLDIR}/_qm
- install -m 0644 -D qmrestore.zsh-completion ${DESTDIR}/${ZSHCOMPLDIR}/_qmrestore
- install -m 0644 -D bootsplash.jpg ${DESTDIR}/usr/share/${PACKAGE}
+install: $(PKGSOURCES)
+ install -d $(DESTDIR)/$(SBINDIR)
+ install -d $(DESTDIR)$(LIBDIR)
+ install -d $(DESTDIR)/$(MAN1DIR)
+ install -d $(DESTDIR)/$(MAN5DIR)
+ install -d $(DESTDIR)/usr/share/$(PACKAGE)
+ install -m 0644 -D qm.bash-completion $(DESTDIR)/$(BASHCOMPLDIR)/qm
+ install -m 0644 -D qmrestore.bash-completion $(DESTDIR)/$(BASHCOMPLDIR)/qmrestore
+ install -m 0644 -D qm.zsh-completion $(DESTDIR)/$(ZSHCOMPLDIR)/_qm
+ install -m 0644 -D qmrestore.zsh-completion $(DESTDIR)/$(ZSHCOMPLDIR)/_qmrestore
+ install -m 0644 -D bootsplash.jpg $(DESTDIR)/usr/share/$(PACKAGE)
$(MAKE) -C PVE install
$(MAKE) -C qmeventd install
$(MAKE) -C qemu-configs install
$(MAKE) -C vm-network-scripts install
- install -m 0755 qm ${DESTDIR}${SBINDIR}
- install -m 0755 qmrestore ${DESTDIR}${SBINDIR}
- install -D -m 0644 modules-load.conf ${DESTDIR}/etc/modules-load.d/qemu-server.conf
- install -m 0755 qmextract ${DESTDIR}${LIBDIR}
- install -m 0644 qm.1 ${DESTDIR}/${MAN1DIR}
- install -m 0644 qmrestore.1 ${DESTDIR}/${MAN1DIR}
- install -m 0644 cpu-models.conf.5 ${DESTDIR}/${MAN5DIR}
- install -m 0644 qm.conf.5 ${DESTDIR}/${MAN5DIR}
- cd ${DESTDIR}/${MAN5DIR}; ln -s -f qm.conf.5.gz vm.conf.5.gz
-
-${BUILDDIR}:
- rm -rf $(BUILDDIR)
- rsync -a * $(BUILDDIR)
- echo "git clone git://git.proxmox.com/git/qemu-server.git\\ngit checkout $(GITVERSION)" > $(BUILDDIR)/debian/SOURCE
+ install -m 0755 qm $(DESTDIR)$(SBINDIR)
+ install -m 0755 qmrestore $(DESTDIR)$(SBINDIR)
+ install -D -m 0644 modules-load.conf $(DESTDIR)/etc/modules-load.d/qemu-server.conf
+ install -m 0755 qmextract $(DESTDIR)$(LIBDIR)
+ install -m 0644 qm.1 $(DESTDIR)/$(MAN1DIR)
+ install -m 0644 qmrestore.1 $(DESTDIR)/$(MAN1DIR)
+ install -m 0644 cpu-models.conf.5 $(DESTDIR)/$(MAN5DIR)
+ install -m 0644 qm.conf.5 $(DESTDIR)/$(MAN5DIR)
+ cd $(DESTDIR)/$(MAN5DIR); ln -s -f qm.conf.5.gz vm.conf.5.gz
+
+$(BUILDDIR):
+ rm -rf $(BUILDDIR) $(BUILDDIR).tmp
+ rsync -a * $(BUILDDIR).tmp
+ echo "git clone git://git.proxmox.com/git/qemu-server.git\\ngit checkout $(GITVERSION)" > $(BUILDDIR).tmp/debian/SOURCE
+ mv $(BUILDDIR).tmp $(BUILDDIR)
.PHONY: deb
-deb: ${DEBS}
-${DBG_DEB}: ${DEB}
-${DEB}: $(BUILDDIR)
+deb: $(DEBS)
+$(DBG_DEB): $(DEB)
+$(DEB): $(BUILDDIR)
cd $(BUILDDIR); dpkg-buildpackage -b -us -uc
- lintian ${DEBS}
+ lintian $(DEBS)
.PHONY: dsc
-dsc: ${DSC}
-${DSC}: ${BUILDDIR}
- cd ${BUILDDIR}; dpkg-buildpackage -S -us -uc -d
- lintian ${DSC}
+dsc: $(DSC)
+$(DSC): $(BUILDDIR)
+ cd $(BUILDDIR); dpkg-buildpackage -S -us -uc -d
+ lintian $(DSC)
+
+sbuild: $(DSC)
+ sbuild $(DSC)
.PHONY: test
test:
$(MAKE) -C test
.PHONY: upload
-upload: ${DEB}
- tar cf - ${DEBS} | ssh -X repoman@repo.proxmox.com upload --product pve --dist buster
+upload: UPLOAD_DIST ?= $(DEB_DISTRIBUTION)
+upload: $(DEB)
+ tar cf - $(DEBS) | ssh -X repoman@repo.proxmox.com upload --product pve --dist $(UPLOAD_DIST)
.PHONY: clean
clean:
- rm -rf $(PACKAGE)-*/ *.deb *.buildinfo *.changes *.dsc $(PACKAGE)_*.tar.gz
- $(MAKE) cleanup-docgen
- find . -name '*~' -exec rm {} ';'
+ $(MAKE) -C test $@
+ rm -rf $(PACKAGE)-*/ *.deb *.build *.buildinfo *.changes *.dsc $(PACKAGE)_*.tar.?z
+ rm -f *.xml.tmp *.1 *.5 *.8 *{synopsis,opts}.adoc docinfo.xml
.PHONY: distclean
use warnings;
use Cwd 'abs_path';
use Net::SSLeay;
-use POSIX;
use IO::Socket::IP;
+use IO::Socket::UNIX;
+use IPC::Open3;
+use JSON;
use URI::Escape;
use Crypt::OpenSSL::Random;
+use Socket qw(SOCK_STREAM);
+use PVE::APIClient::LWP;
+use PVE::CGroup;
use PVE::Cluster qw (cfs_read_file cfs_write_file);;
use PVE::RRD;
use PVE::SafeSyslog;
use PVE::JSONSchema qw(get_standard_option);
use PVE::RESTHandler;
use PVE::ReplicationConfig;
-use PVE::GuestHelpers;
+use PVE::GuestHelpers qw(assert_tag_permissions);
use PVE::QemuConfig;
use PVE::QemuServer;
-use PVE::QemuServer::Drive;
+use PVE::QemuServer::Cloudinit;
use PVE::QemuServer::CPUConfig;
+use PVE::QemuServer::Drive;
+use PVE::QemuServer::ImportDisk;
use PVE::QemuServer::Monitor qw(mon_cmd);
+use PVE::QemuServer::Machine;
+use PVE::QemuServer::Memory qw(get_current_memory);
+use PVE::QemuServer::PCI;
+use PVE::QemuServer::USB;
use PVE::QemuMigrate;
use PVE::RPCEnvironment;
use PVE::AccessControl;
use PVE::VZDump::Plugin;
use PVE::DataCenterConfig;
use PVE::SSHInfo;
+use PVE::Replication;
+use PVE::StorageTunnel;
+use PVE::RESTEnvironment qw(log_warn);
BEGIN {
if (!$ENV{PVE_GENERATING_DOCS}) {
}
}
-use Data::Dumper; # fixme: remove
-
use base qw(PVE::RESTHandler);
my $opt_force_description = "Force physical removal. Without this, we simple remove the disk from the config file and create an additional configuration entry called 'unused[n]', which contains the volume ID. Unlink of unused[n] always cause physical removal.";
}
};
-my $NEW_DISK_RE = qr!^(([^/:\s]+):)?(\d+(\.\d+)?)$!;
+# Used in import-enabled API endpoints. Parses drives using the extended '_with_alloc' schema.
+my $foreach_volume_with_alloc = sub {
+ my ($param, $func) = @_;
+
+ for my $opt (sort keys $param->%*) {
+ next if !PVE::QemuServer::is_valid_drivename($opt);
+
+ my $drive = PVE::QemuServer::Drive::parse_drive($opt, $param->{$opt}, 1);
+ next if !$drive;
+
+ $func->($opt, $drive);
+ }
+};
+
+my $check_drive_param = sub {
+ my ($param, $storecfg, $extra_checks) = @_;
+
+ for my $opt (sort keys $param->%*) {
+ next if !PVE::QemuServer::is_valid_drivename($opt);
+
+ my $drive = PVE::QemuServer::parse_drive($opt, $param->{$opt}, 1);
+ raise_param_exc({ $opt => "unable to parse drive options" }) if !$drive;
+
+ if ($drive->{'import-from'}) {
+ if ($drive->{file} !~ $PVE::QemuServer::Drive::NEW_DISK_RE || $3 != 0) {
+ raise_param_exc({
+ $opt => "'import-from' requires special syntax - ".
+ "use <storage ID>:0,import-from=<source>",
+ });
+ }
+
+ if ($opt eq 'efidisk0') {
+ for my $required (qw(efitype pre-enrolled-keys)) {
+ if (!defined($drive->{$required})) {
+ raise_param_exc({
+ $opt => "need to specify '$required' when using 'import-from'",
+ });
+ }
+ }
+ } elsif ($opt eq 'tpmstate0') {
+ raise_param_exc({ $opt => "need to specify 'version' when using 'import-from'" })
+ if !defined($drive->{version});
+ }
+ }
+
+ PVE::QemuServer::cleanup_drive_path($opt, $storecfg, $drive);
+
+ $extra_checks->($drive) if $extra_checks;
+
+ $param->{$opt} = PVE::QemuServer::print_drive($drive, 1);
+ }
+};
+
my $check_storage_access = sub {
my ($rpcenv, $authuser, $storecfg, $vmid, $settings, $default_storage) = @_;
- PVE::QemuConfig->foreach_volume($settings, sub {
+ $foreach_volume_with_alloc->($settings, sub {
my ($ds, $drive) = @_;
my $isCDROM = PVE::QemuServer::drive_is_cdrom($drive);
# nothing to check
} elsif ($isCDROM && ($volid eq 'cdrom')) {
$rpcenv->check($authuser, "/", ['Sys.Console']);
- } elsif (!$isCDROM && ($volid =~ $NEW_DISK_RE)) {
+ } elsif (!$isCDROM && ($volid =~ $PVE::QemuServer::Drive::NEW_DISK_RE)) {
my ($storeid, $size) = ($2 || $default_storage, $3);
die "no storage ID specified (and no default storage)\n" if !$storeid;
$rpcenv->check($authuser, "/storage/$storeid", ['Datastore.AllocateSpace']);
if !$scfg->{content}->{images};
} else {
PVE::Storage::check_volume_access($rpcenv, $authuser, $storecfg, $vmid, $volid);
+ if ($storeid) {
+ my ($vtype) = PVE::Storage::parse_volname($storecfg, $volid);
+ raise_param_exc({ $ds => "content type needs to be 'images' or 'iso'" })
+ if $vtype ne 'images' && $vtype ne 'iso';
+ }
+ }
+
+ if (my $src_image = $drive->{'import-from'}) {
+ my $src_vmid;
+ if (PVE::Storage::parse_volume_id($src_image, 1)) { # PVE-managed volume
+ (my $vtype, undef, $src_vmid) = PVE::Storage::parse_volname($storecfg, $src_image);
+ raise_param_exc({ $ds => "$src_image has wrong type '$vtype' - not an image" })
+ if $vtype ne 'images';
+ }
+
+ if ($src_vmid) { # might be actively used by VM and will be copied via clone_disk()
+ $rpcenv->check($authuser, "/vms/${src_vmid}", ['VM.Clone']);
+ } else {
+ PVE::Storage::check_volume_access($rpcenv, $authuser, $storecfg, $vmid, $src_image);
+ }
}
});
return $sharedvm;
};
+my $check_storage_access_migrate = sub {
+ my ($rpcenv, $authuser, $storecfg, $storage, $node) = @_;
+
+ PVE::Storage::storage_check_enabled($storecfg, $storage, $node);
+
+ $rpcenv->check($authuser, "/storage/$storage", ['Datastore.AllocateSpace']);
+
+ my $scfg = PVE::Storage::storage_config($storecfg, $storage);
+ die "storage '$storage' does not support vm images\n"
+ if !$scfg->{content}->{images};
+};
+
+my $import_from_volid = sub {
+ my ($storecfg, $src_volid, $dest_info, $vollist) = @_;
+
+ die "could not get size of $src_volid\n"
+ if !PVE::Storage::volume_size_info($storecfg, $src_volid, 10);
+
+ die "cannot import from cloudinit disk\n"
+ if PVE::QemuServer::Drive::drive_is_cloudinit({ file => $src_volid });
+
+ my $src_vmid = (PVE::Storage::parse_volname($storecfg, $src_volid))[2];
+
+ my $src_vm_state = sub {
+ my $exists = $src_vmid && PVE::Cluster::get_vmlist()->{ids}->{$src_vmid} ? 1 : 0;
+
+ my $runs = 0;
+ if ($exists) {
+ eval { PVE::QemuConfig::assert_config_exists_on_node($src_vmid); };
+ die "owner VM $src_vmid not on local node\n" if $@;
+ $runs = PVE::QemuServer::Helpers::vm_running_locally($src_vmid) || 0;
+ }
+
+ return ($exists, $runs);
+ };
+
+ my ($src_vm_exists, $running) = $src_vm_state->();
+
+ die "cannot import from '$src_volid' - full clone feature is not supported\n"
+ if !PVE::Storage::volume_has_feature($storecfg, 'copy', $src_volid, undef, $running);
+
+ my $clonefn = sub {
+ my ($src_vm_exists_now, $running_now) = $src_vm_state->();
+
+ die "owner VM $src_vmid changed state unexpectedly\n"
+ if $src_vm_exists_now != $src_vm_exists || $running_now != $running;
+
+ my $src_conf = $src_vm_exists_now ? PVE::QemuConfig->load_config($src_vmid) : {};
+
+ my $src_drive = { file => $src_volid };
+ my $src_drivename;
+ PVE::QemuConfig->foreach_volume($src_conf, sub {
+ my ($ds, $drive) = @_;
+
+ return if $src_drivename;
+
+ if ($drive->{file} eq $src_volid) {
+ $src_drive = $drive;
+ $src_drivename = $ds;
+ }
+ });
+
+ my $source_info = {
+ vmid => $src_vmid,
+ running => $running_now,
+ drivename => $src_drivename,
+ drive => $src_drive,
+ snapname => undef,
+ };
+
+ my ($src_storeid) = PVE::Storage::parse_volume_id($src_volid);
+
+ return PVE::QemuServer::clone_disk(
+ $storecfg,
+ $source_info,
+ $dest_info,
+ 1,
+ $vollist,
+ undef,
+ undef,
+ $src_conf->{agent},
+ PVE::Storage::get_bandwidth_limit('clone', [$src_storeid, $dest_info->{storage}]),
+ );
+ };
+
+ my $cloned;
+ if ($running) {
+ $cloned = PVE::QemuConfig->lock_config_full($src_vmid, 30, $clonefn);
+ } elsif ($src_vmid) {
+ $cloned = PVE::QemuConfig->lock_config_shared($src_vmid, 30, $clonefn);
+ } else {
+ $cloned = $clonefn->();
+ }
+
+ return $cloned->@{qw(file size)};
+};
+
# Note: $pool is only needed when creating a VM, because pool permissions
# are automatically inherited if VM already exists inside a pool.
-my $create_disks = sub {
- my ($rpcenv, $authuser, $conf, $arch, $storecfg, $vmid, $pool, $settings, $default_storage) = @_;
+my sub create_disks : prototype($$$$$$$$$$) {
+ my (
+ $rpcenv,
+ $authuser,
+ $conf,
+ $arch,
+ $storecfg,
+ $vmid,
+ $pool,
+ $settings,
+ $default_storage,
+ $is_live_import,
+ ) = @_;
my $vollist = [];
my $res = {};
+ my $live_import_mapping = {};
+
my $code = sub {
my ($ds, $disk) = @_;
} elsif (defined($volname) && $volname eq 'cloudinit') {
$storeid = $storeid // $default_storage;
die "no storage ID specified (and no default storage)\n" if !$storeid;
+
+ if (
+ my $ci_key = PVE::QemuConfig->has_cloudinit($conf, $ds)
+ || PVE::QemuConfig->has_cloudinit($conf->{pending} || {}, $ds)
+ || PVE::QemuConfig->has_cloudinit($res, $ds)
+ ) {
+ die "$ds - cloud-init drive is already attached at '$ci_key'\n";
+ }
+
my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
my $name = "vm-$vmid-cloudinit";
push @$vollist, $volid;
delete $disk->{format}; # no longer needed
$res->{$ds} = PVE::QemuServer::print_drive($disk);
- } elsif ($volid =~ $NEW_DISK_RE) {
+ print "$ds: successfully created disk '$res->{$ds}'\n";
+ } elsif ($volid =~ $PVE::QemuServer::Drive::NEW_DISK_RE) {
my ($storeid, $size) = ($2 || $default_storage, $3);
die "no storage ID specified (and no default storage)\n" if !$storeid;
- my $defformat = PVE::Storage::storage_default_format($storecfg, $storeid);
- my $fmt = $disk->{format} || $defformat;
$size = PVE::Tools::convert_size($size, 'gb' => 'kb'); # vdisk_alloc uses kb
- my $volid;
- if ($ds eq 'efidisk0') {
- ($volid, $size) = PVE::QemuServer::create_efidisk($storecfg, $storeid, $vmid, $fmt, $arch);
- } else {
- $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $size);
- }
- push @$vollist, $volid;
- $disk->{file} = $volid;
- $disk->{size} = PVE::Tools::convert_size($size, 'kb' => 'b');
- delete $disk->{format}; # no longer needed
- $res->{$ds} = PVE::QemuServer::print_drive($disk);
- } else {
-
- PVE::Storage::check_volume_access($rpcenv, $authuser, $storecfg, $vmid, $volid);
-
- my $volid_is_new = 1;
+ my $live_import = $is_live_import && $ds ne 'efidisk0';
+ my $needs_creation = 1;
+
+ if (my $source = delete $disk->{'import-from'}) {
+ my $dst_volid;
+
+ $needs_creation = $live_import;
+
+ if (PVE::Storage::parse_volume_id($source, 1)) { # PVE-managed volume
+ if ($live_import && $ds ne 'efidisk0') {
+ my $path = PVE::Storage::path($storecfg, $source)
+ or die "failed to get a path for '$source'\n";
+ $source = $path;
+ ($size, my $source_format) = PVE::Storage::file_size_info($source);
+ die "could not get file size of $source\n" if !$size;
+ $live_import_mapping->{$ds} = {
+ path => $source,
+ format => $source_format,
+ };
+ } else {
+ my $dest_info = {
+ vmid => $vmid,
+ drivename => $ds,
+ storage => $storeid,
+ format => $disk->{format},
+ };
+
+ $dest_info->{efisize} = PVE::QemuServer::get_efivars_size($conf, $disk)
+ if $ds eq 'efidisk0';
+
+ ($dst_volid, $size) = eval {
+ $import_from_volid->($storecfg, $source, $dest_info, $vollist);
+ };
+ die "cannot import from '$source' - $@" if $@;
+ }
+ } else {
+ $source = PVE::Storage::abs_filesystem_path($storecfg, $source, 1);
+ ($size, my $source_format) = PVE::Storage::file_size_info($source);
+ die "could not get file size of $source\n" if !$size;
+
+ if ($live_import && $ds ne 'efidisk0') {
+ $live_import_mapping->{$ds} = {
+ path => $source,
+ format => $source_format,
+ };
+ } else {
+ (undef, $dst_volid) = PVE::QemuServer::ImportDisk::do_import(
+ $source,
+ $vmid,
+ $storeid,
+ {
+ drive_name => $ds,
+ format => $disk->{format},
+ 'skip-config-update' => 1,
+ },
+ );
+ push @$vollist, $dst_volid;
+ }
+ }
- if ($conf->{$ds}) {
- my $olddrive = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
- $volid_is_new = undef if $olddrive->{file} && $olddrive->{file} eq $volid;
+ if ($needs_creation) {
+ $size = PVE::Tools::convert_size($size, 'b' => 'kb'); # vdisk_alloc uses kb
+ } else {
+ $disk->{file} = $dst_volid;
+ $disk->{size} = $size;
+ delete $disk->{format}; # no longer needed
+ $res->{$ds} = PVE::QemuServer::print_drive($disk);
+ }
}
- if ($volid_is_new) {
-
- PVE::Storage::activate_volumes($storecfg, [ $volid ]) if $storeid;
+ if ($needs_creation) {
+ my $defformat = PVE::Storage::storage_default_format($storecfg, $storeid);
+ my $fmt = $disk->{format} || $defformat;
+
+ my $volid;
+ if ($ds eq 'efidisk0') {
+ my $smm = PVE::QemuServer::Machine::machine_type_is_q35($conf);
+ ($volid, $size) = PVE::QemuServer::create_efidisk(
+ $storecfg, $storeid, $vmid, $fmt, $arch, $disk, $smm);
+ } elsif ($ds eq 'tpmstate0') {
+ # swtpm can only use raw volumes, and uses a fixed size
+ $size = PVE::Tools::convert_size(PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE, 'b' => 'kb');
+ $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, "raw", undef, $size);
+ } else {
+ $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $size);
+ }
+ push @$vollist, $volid;
+ $disk->{file} = $volid;
+ $disk->{size} = PVE::Tools::convert_size($size, 'kb' => 'b');
+ delete $disk->{format}; # no longer needed
+ $res->{$ds} = PVE::QemuServer::print_drive($disk);
+ }
- my $size = PVE::Storage::volume_size_info($storecfg, $volid);
+ print "$ds: successfully created disk '$res->{$ds}'\n";
+ } else {
+ PVE::Storage::check_volume_access($rpcenv, $authuser, $storecfg, $vmid, $volid);
+ if ($storeid) {
+ my ($vtype) = PVE::Storage::parse_volname($storecfg, $volid);
+ die "cannot use volume $volid - content type needs to be 'images' or 'iso'"
+ if $vtype ne 'images' && $vtype ne 'iso';
+
+ if (PVE::QemuServer::Drive::drive_is_cloudinit($disk)) {
+ if (
+ my $ci_key = PVE::QemuConfig->has_cloudinit($conf, $ds)
+ || PVE::QemuConfig->has_cloudinit($conf->{pending} || {}, $ds)
+ || PVE::QemuConfig->has_cloudinit($res, $ds)
+ ) {
+ die "$ds - cloud-init drive is already attached at '$ci_key'\n";
+ }
+ }
+ }
- die "volume $volid does not exist\n" if !$size;
+ PVE::Storage::activate_volumes($storecfg, [ $volid ]) if $storeid;
- $disk->{size} = $size;
- }
+ my $size = PVE::Storage::volume_size_info($storecfg, $volid);
+ die "volume $volid does not exist\n" if !$size;
+ $disk->{size} = $size;
$res->{$ds} = PVE::QemuServer::print_drive($disk);
}
};
- eval { PVE::QemuConfig->foreach_volume($settings, $code); };
+ eval { $foreach_volume_with_alloc->($settings, $code); };
# free allocated images on error
if (my $err = $@) {
die $err;
}
- # modify vm config if everything went well
- foreach my $ds (keys %$res) {
- $conf->{$ds} = $res->{$ds};
- }
+ # don't return empty import mappings
+ $live_import_mapping = undef if !%$live_import_mapping;
- return $vollist;
+ return ($vollist, $res, $live_import_mapping);
};
my $check_cpu_model_access = sub {
'startup' => 1,
'tdf' => 1,
'template' => 1,
- 'tags' => 1,
};
my $vmpoweroptions = {
cipassword => 1,
citype => 1,
ciuser => 1,
+ ciupgrade => 1,
nameserver => 1,
searchdomain => 1,
sshkeys => 1,
};
+my $check_vm_create_serial_perm = sub {
+ my ($rpcenv, $authuser, $vmid, $pool, $param) = @_;
+
+ return 1 if $authuser eq 'root@pam';
+
+ foreach my $opt (keys %{$param}) {
+ next if $opt !~ m/^serial\d+$/;
+
+ if ($param->{$opt} eq 'socket') {
+ $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.HWType']);
+ } else {
+ die "only root can set '$opt' config for real devices\n";
+ }
+ }
+
+ return 1;
+};
+
+my sub check_usb_perm {
+ my ($rpcenv, $authuser, $vmid, $pool, $opt, $value) = @_;
+
+ return 1 if $authuser eq 'root@pam';
+
+ $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.HWType']);
+
+ my $device = PVE::JSONSchema::parse_property_string('pve-qm-usb', $value);
+ if ($device->{host} && $device->{host} !~ m/^spice$/i) {
+ die "only root can set '$opt' config for real devices\n";
+ } elsif ($device->{mapping}) {
+ $rpcenv->check_full($authuser, "/mapping/usb/$device->{mapping}", ['Mapping.Use']);
+ } else {
+ die "either 'host' or 'mapping' must be set.\n";
+ }
+
+ return 1;
+}
+
+my sub check_vm_create_usb_perm {
+ my ($rpcenv, $authuser, $vmid, $pool, $param) = @_;
+
+ return 1 if $authuser eq 'root@pam';
+
+ foreach my $opt (keys %{$param}) {
+ next if $opt !~ m/^usb\d+$/;
+ check_usb_perm($rpcenv, $authuser, $vmid, $pool, $opt, $param->{$opt});
+ }
+
+ return 1;
+};
+
+my sub check_hostpci_perm {
+ my ($rpcenv, $authuser, $vmid, $pool, $opt, $value) = @_;
+
+ return 1 if $authuser eq 'root@pam';
+
+ my $device = PVE::JSONSchema::parse_property_string('pve-qm-hostpci', $value);
+ if ($device->{host}) {
+ die "only root can set '$opt' config for non-mapped devices\n";
+ } elsif ($device->{mapping}) {
+ $rpcenv->check_full($authuser, "/mapping/pci/$device->{mapping}", ['Mapping.Use']);
+ $rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.HWType']);
+ } else {
+ die "either 'host' or 'mapping' must be set.\n";
+ }
+
+ return 1;
+}
+
+my sub check_vm_create_hostpci_perm {
+ my ($rpcenv, $authuser, $vmid, $pool, $param) = @_;
+
+ return 1 if $authuser eq 'root@pam';
+
+ foreach my $opt (keys %{$param}) {
+ next if $opt !~ m/^hostpci\d+$/;
+ check_hostpci_perm($rpcenv, $authuser, $vmid, $pool, $opt, $param->{$opt});
+ }
+
+ return 1;
+};
+
my $check_vm_modify_config_perm = sub {
my ($rpcenv, $authuser, $vmid, $pool, $key_list) = @_;
# else, as there the permission can be value dependend
next if PVE::QemuServer::is_valid_drivename($opt);
next if $opt eq 'cdrom';
- next if $opt =~ m/^(?:unused|serial|usb)\d+$/;
+ next if $opt =~ m/^(?:unused|serial|usb|hostpci)\d+$/;
+ next if $opt eq 'tags';
if ($cpuoptions->{$opt} || $opt =~ m/^numa\d+$/) {
$rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.PowerMgmt']);
} elsif ($diskoptions->{$opt}) {
$rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Disk']);
- } elsif ($opt =~ m/^(?:net|ipconfig)\d+$/) {
+ } elsif ($opt =~ m/^net\d+$/) {
$rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Network']);
- } elsif ($cloudinitoptions->{$opt}) {
+ } elsif ($cloudinitoptions->{$opt} || $opt =~ m/^ipconfig\d+$/) {
$rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Cloudinit', 'VM.Config.Network'], 1);
} elsif ($opt eq 'vmstate') {
# the user needs Disk and PowerMgmt privileges to change the vmstate
# also needs privileges on the storage, that will be checked later
$rpcenv->check_vm_perm($authuser, $vmid, $pool, ['VM.Config.Disk', 'VM.PowerMgmt' ]);
} else {
- # catches hostpci\d+, args, lock, etc.
+ # catches args, lock, etc.
# new options will be checked here
die "only root can set '$opt' config\n";
}
return 1;
};
+sub assert_scsi_feature_compatibility {
+ my ($opt, $conf, $storecfg, $drive_attributes) = @_;
+
+ my $drive = PVE::QemuServer::Drive::parse_drive($opt, $drive_attributes, 1);
+
+ my $machine_type = PVE::QemuServer::get_vm_machine($conf, undef, $conf->{arch});
+ my $machine_version = PVE::QemuServer::Machine::extract_version(
+ $machine_type, PVE::QemuServer::kvm_user_version());
+ my $drivetype = PVE::QemuServer::Drive::get_scsi_device_type(
+ $drive, $storecfg, $machine_version);
+
+ if ($drivetype ne 'hd' && $drivetype ne 'cd') {
+ if ($drive->{product}) {
+ raise_param_exc({
+ $opt => "Passing of product information is only supported for 'scsi-hd' and "
+ ."'scsi-cd' devices (e.g. not pass-through).",
+ });
+ }
+ if ($drive->{vendor}) {
+ raise_param_exc({
+ $opt => "Passing of vendor information is only supported for 'scsi-hd' and "
+ ."'scsi-cd' devices (e.g. not pass-through).",
+ });
+ }
+ }
+}
+
__PACKAGE__->register_method({
name => 'vmlist',
path => '',
proxyto => 'node',
protected => 1, # qemu pid files are only readable by root
parameters => {
- additionalProperties => 0,
+ additionalProperties => 0,
properties => {
node => get_standard_option('pve-node'),
full => {
my ($archive_storeid, $archive_volname) = PVE::Storage::parse_volume_id($archive, 1);
+ my $res = {};
+
if (defined($archive_storeid)) {
my $scfg = PVE::Storage::storage_config($storecfg, $archive_storeid);
+ $res->{volid} = $archive;
if ($scfg->{type} eq 'pbs') {
- return {
- type => 'pbs',
- volid => $archive,
- };
+ $res->{type} = 'pbs';
+ return $res;
}
}
my $path = PVE::Storage::abs_filesystem_path($storecfg, $archive);
- return {
- type => 'file',
- path => $path,
- };
+ $res->{type} = 'file';
+ $res->{path} = $path;
+ return $res;
};
-
__PACKAGE__->register_method({
name => 'create_vm',
path => '',
permissions => {
description => "You need 'VM.Allocate' permissions on /vms/{vmid} or on the VM pool /pool/{pool}. " .
"For restore (option 'archive'), it is enough if the user has 'VM.Backup' permission and the VM already exists. " .
- "If you create disks you need 'Datastore.AllocateSpace' on any used storage.",
+ "If you create disks you need 'Datastore.AllocateSpace' on any used storage." .
+ "If you use a bridge/vlan, you need 'SDN.Use' on any used bridge/vlan.",
user => 'all', # check inside
},
protected => 1,
proxyto => 'node',
parameters => {
- additionalProperties => 0,
+ additionalProperties => 0,
properties => PVE::QemuServer::json_config_properties(
{
node => get_standard_option('pve-node'),
description => "Assign a unique random ethernet address.",
requires => 'archive',
},
+ 'live-restore' => {
+ optional => 1,
+ type => 'boolean',
+ description => "Start the VM immediately while importing or restoring in the background.",
+ },
pool => {
optional => 1,
type => 'string', format => 'pve-poolid',
default => 0,
description => "Start VM after it was created successfully.",
},
- }),
+ },
+ 1, # with_disk_alloc
+ ),
},
returns => {
type => 'string',
my $start_after_create = extract_param($param, 'start');
my $storage = extract_param($param, 'storage');
my $unique = extract_param($param, 'unique');
+ my $live_restore = extract_param($param, 'live-restore');
if (defined(my $ssh_keys = $param->{sshkeys})) {
$ssh_keys = URI::Escape::uri_unescape($ssh_keys);
PVE::Tools::validate_ssh_public_keys($ssh_keys);
}
+ $param->{cpuunits} = PVE::CGroup::clamp_cpu_shares($param->{cpuunits})
+ if defined($param->{cpuunits}); # clamp value depending on cgroup version
+
PVE::Cluster::check_cfs_quorum();
my $filename = PVE::QemuConfig->config_file($vmid);
# OK
} elsif ($archive && $force && (-f $filename) &&
$rpcenv->check($authuser, "/vms/$vmid", ['VM.Backup'], 1)) {
- # OK: user has VM.Backup permissions, and want to restore an existing VM
+ # OK: user has VM.Backup permissions and wants to restore an existing VM
} else {
raise_perm_exc();
}
- if (!$archive) {
+ if ($archive) {
+ for my $opt (sort keys $param->%*) {
+ if (PVE::QemuServer::Drive::is_valid_drivename($opt)) {
+ raise_param_exc({ $opt => "option conflicts with option 'archive'" });
+ }
+ }
+
+ if ($archive eq '-') {
+ die "pipe requires cli environment\n" if $rpcenv->{type} ne 'cli';
+ $archive = { type => 'pipe' };
+ } else {
+ PVE::Storage::check_volume_access(
+ $rpcenv,
+ $authuser,
+ $storecfg,
+ $vmid,
+ $archive,
+ 'backup',
+ );
+
+ $archive = $parse_restore_archive->($storecfg, $archive);
+ }
+ }
+
+ if (scalar(keys $param->%*) > 0) {
&$resolve_cdrom_alias($param);
&$check_storage_access($rpcenv, $authuser, $storecfg, $vmid, $param, $storage);
&$check_vm_modify_config_perm($rpcenv, $authuser, $vmid, $pool, [ keys %$param]);
- &$check_cpu_model_access($rpcenv, $authuser, $param);
+ &$check_vm_create_serial_perm($rpcenv, $authuser, $vmid, $pool, $param);
+ check_vm_create_usb_perm($rpcenv, $authuser, $vmid, $pool, $param);
+ check_vm_create_hostpci_perm($rpcenv, $authuser, $vmid, $pool, $param);
- foreach my $opt (keys %$param) {
- if (PVE::QemuServer::is_valid_drivename($opt)) {
- my $drive = PVE::QemuServer::parse_drive($opt, $param->{$opt});
- raise_param_exc({ $opt => "unable to parse drive options" }) if !$drive;
+ PVE::QemuServer::check_bridge_access($rpcenv, $authuser, $param);
+ &$check_cpu_model_access($rpcenv, $authuser, $param);
- PVE::QemuServer::cleanup_drive_path($opt, $storecfg, $drive);
- $param->{$opt} = PVE::QemuServer::print_drive($drive);
- }
- }
+ $check_drive_param->($param, $storecfg);
PVE::QemuServer::add_random_macs($param);
- } else {
- my $keystr = join(' ', keys %$param);
- raise_param_exc({ archive => "option conflicts with other options ($keystr)"}) if $keystr;
-
- if ($archive eq '-') {
- die "pipe requires cli environment\n"
- if $rpcenv->{type} ne 'cli';
- $archive = { type => 'pipe' };
- } else {
- PVE::Storage::check_volume_access($rpcenv, $authuser, $storecfg, $vmid, $archive);
-
- $archive = $parse_restore_archive->($storecfg, $archive);
- }
}
my $emsg = $is_restore ? "unable to restore VM $vmid -" : "unable to create VM $vmid -";
eval { PVE::QemuConfig->create_and_lock_config($vmid, $force) };
die "$emsg $@" if $@;
+ my $restored_data = 0;
my $restorefn = sub {
my $conf = PVE::QemuConfig->load_config($vmid);
pool => $pool,
unique => $unique,
bwlimit => $bwlimit,
+ live => $live_restore,
+ override_conf => $param,
};
+ if (my $volid = $archive->{volid}) {
+ # best effort, real check is after restoring!
+ my $merged = eval {
+ my $old_conf = PVE::Storage::extract_vzdump_config($storecfg, $volid);
+ PVE::QemuServer::restore_merge_config("backup/qemu-server/$vmid.conf", $old_conf, $param);
+ };
+ if ($@) {
+ warn "Could not extract backed up config: $@\n";
+ warn "Skipping early checks!\n";
+ } else {
+ PVE::QemuServer::check_restore_permissions($rpcenv, $authuser, $merged);
+ }
+ }
if ($archive->{type} eq 'file' || $archive->{type} eq 'pipe') {
+ die "live-restore is only compatible with backup images from a Proxmox Backup Server\n"
+ if $live_restore;
PVE::QemuServer::restore_file_archive($archive->{path} // '-', $vmid, $authuser, $restore_options);
} elsif ($archive->{type} eq 'pbs') {
PVE::QemuServer::restore_proxmox_backup_archive($archive->{volid}, $vmid, $authuser, $restore_options);
} else {
die "unknown backup archive type\n";
}
+ $restored_data = 1;
+
my $restored_conf = PVE::QemuConfig->load_config($vmid);
# Convert restored VM to template if backup was VM template
if (PVE::QemuConfig->is_template($restored_conf)) {
warn $@ if $@;
}
- PVE::AccessControl::add_vm_to_pool($vmid, $pool) if $pool;
+ PVE::QemuServer::create_ifaces_ipams_ips($restored_conf, $vmid) if $unique;
};
# ensure no old replication state are exists
PVE::QemuConfig->lock_config_full($vmid, 1, $realcmd);
- if ($start_after_create) {
+ if ($start_after_create && !$live_restore) {
print "Execute autostart\n";
eval { PVE::API2::Qemu->vm_start({ vmid => $vmid, node => $node }) };
warn $@ if $@;
};
my $createfn = sub {
+ my $live_import_mapping = {};
+
# ensure no old replication state are exists
PVE::ReplicationState::delete_guest_states($vmid);
my $conf = $param;
my $arch = PVE::QemuServer::get_vm_arch($conf);
+ for my $opt (sort keys $param->%*) {
+ next if $opt !~ m/^scsi\d+$/;
+ assert_scsi_feature_compatibility($opt, $conf, $storecfg, $param->{$opt});
+ }
+
+ $conf->{meta} = PVE::QemuServer::new_meta_info_string();
+
my $vollist = [];
eval {
- $vollist = &$create_disks($rpcenv, $authuser, $conf, $arch, $storecfg, $vmid, $pool, $param, $storage);
+ ($vollist, my $created_opts, $live_import_mapping) = create_disks(
+ $rpcenv,
+ $authuser,
+ $conf,
+ $arch,
+ $storecfg,
+ $vmid,
+ $pool,
+ $param,
+ $storage,
+ $live_restore,
+ );
+ $conf->{$_} = $created_opts->{$_} for keys $created_opts->%*;
if (!$conf->{boot}) {
my $devs = PVE::QemuServer::get_default_bootdevices($conf);
$conf->{boot} = PVE::QemuServer::print_bootorder($devs);
}
+ my $vga = PVE::QemuServer::parse_vga($conf->{vga});
+ PVE::QemuServer::assert_clipboard_config($vga);
+
# auto generate uuid if user did not specify smbios1 option
if (!$conf->{smbios1}) {
$conf->{smbios1} = PVE::QemuServer::generate_smbios1_uuid();
$conf->{vmgenid} = PVE::QemuServer::generate_uuid();
}
- PVE::QemuConfig->write_config($vmid, $conf);
+ my $machine_conf = PVE::QemuServer::Machine::parse_machine($conf->{machine});
+ my $machine = $machine_conf->{type};
+ if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
+ # always pin Windows' machine version on create, they get to easily confused
+ if (PVE::QemuServer::Helpers::windows_version($conf->{ostype})) {
+ $machine_conf->{type} = PVE::QemuServer::windows_get_pinned_machine_version($machine);
+ $conf->{machine} = PVE::QemuServer::Machine::print_machine($machine_conf);
+ }
+ }
+ PVE::QemuServer::Machine::assert_valid_machine_property($conf, $machine_conf);
+ $conf->{lock} = 'import' if $live_import_mapping;
+
+ PVE::QemuConfig->write_config($vmid, $conf);
};
my $err = $@;
}
PVE::AccessControl::add_vm_to_pool($vmid, $pool) if $pool;
+
+ PVE::QemuServer::create_ifaces_ipams_ips($conf, $vmid);
};
PVE::QemuConfig->lock_config_full($vmid, 1, $realcmd);
- if ($start_after_create) {
+ if ($start_after_create && !$live_restore) {
print "Execute autostart\n";
eval { PVE::API2::Qemu->vm_start({vmid => $vmid, node => $node}) };
warn $@ if $@;
+ return;
+ } else {
+ return $live_import_mapping;
}
};
if (my $err = $@) {
eval { PVE::QemuConfig->remove_lock($vmid, 'create') };
warn $@ if $@;
+ if ($restored_data) {
+ warn "error after data was restored, VM disks should be OK but config may "
+ ."require adaptions. VM $vmid state is NOT cleaned up.\n";
+ } else {
+ warn "error before or during data restore, some or all disks were not "
+ ."completely restored. VM $vmid state is NOT cleaned up.\n";
+ }
die $err;
}
};
} else {
$worker_name = 'qmcreate';
$code = sub {
- eval { $createfn->() };
+ # If a live import was requested the create function returns
+ # the mapping for the startup.
+ my $live_import_mapping = eval { $createfn->() };
if (my $err = $@) {
eval {
my $conffile = PVE::QemuConfig->config_file($vmid);
warn $@ if $@;
die $err;
}
+
+ if ($live_import_mapping) {
+ my $import_options = {
+ bwlimit => $bwlimit,
+ live => 1,
+ };
+
+ my $conf = PVE::QemuConfig->load_config($vmid);
+ PVE::QemuServer::live_import_from_files(
+ $live_import_mapping,
+ $vmid,
+ $conf,
+ $import_options,
+ );
+ }
};
}
user => 'all',
},
parameters => {
- additionalProperties => 0,
+ additionalProperties => 0,
properties => {
node => get_standard_option('pve-node'),
vmid => get_standard_option('pve-vmid'),
my $res = [
{ subdir => 'config' },
+ { subdir => 'cloudinit' },
{ subdir => 'pending' },
{ subdir => 'status' },
{ subdir => 'unlink' },
{ subdir => 'spiceproxy' },
{ subdir => 'sendkey' },
{ subdir => 'firewall' },
- ];
+ { subdir => 'mtunnel' },
+ { subdir => 'remote_migrate' },
+ ];
return $res;
}});
},
description => "Read VM RRD statistics (returns PNG)",
parameters => {
- additionalProperties => 0,
+ additionalProperties => 0,
properties => {
node => get_standard_option('pve-node'),
vmid => get_standard_option('pve-vmid'),
},
description => "Read VM RRD statistics",
parameters => {
- additionalProperties => 0,
+ additionalProperties => 0,
properties => {
node => get_standard_option('pve-node'),
vmid => get_standard_option('pve-vmid'),
check => ['perm', '/vms/{vmid}', [ 'VM.Audit' ]],
},
parameters => {
- additionalProperties => 0,
+ additionalProperties => 0,
properties => {
node => get_standard_option('pve-node'),
vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }),
return PVE::GuestHelpers::config_with_pending_array($conf, $pending_delete_hash);
}});
-# POST/PUT {vmid}/config implementation
-#
-# The original API used PUT (idempotent) an we assumed that all operations
-# are fast. But it turned out that almost any configuration change can
-# involve hot-plug actions, or disk alloc/free. Such actions can take long
-# time to complete and have side effects (not idempotent).
-#
-# The new implementation uses POST and forks a worker process. We added
-# a new option 'background_delay'. If specified we wait up to
-# 'background_delay' second for the worker task to complete. It returns null
-# if the task is finished within that time, else we return the UPID.
-
-my $update_vm_api = sub {
- my ($param, $sync) = @_;
-
- my $rpcenv = PVE::RPCEnvironment::get();
-
- my $authuser = $rpcenv->get_user();
-
- my $node = extract_param($param, 'node');
-
- my $vmid = extract_param($param, 'vmid');
-
- my $digest = extract_param($param, 'digest');
-
- my $background_delay = extract_param($param, 'background_delay');
-
- if (defined(my $cipassword = $param->{cipassword})) {
- # Same logic as in cloud-init (but with the regex fixed...)
- $param->{cipassword} = PVE::Tools::encrypt_pw($cipassword)
- if $cipassword !~ /^\$(?:[156]|2[ay])(\$.+){2}/;
- }
-
- my @paramarr = (); # used for log message
- foreach my $key (sort keys %$param) {
- my $value = $key eq 'cipassword' ? '<hidden>' : $param->{$key};
- push @paramarr, "-$key", $value;
- }
-
- my $skiplock = extract_param($param, 'skiplock');
+__PACKAGE__->register_method({
+ name => 'cloudinit_pending',
+ path => '{vmid}/cloudinit',
+ method => 'GET',
+ proxyto => 'node',
+ description => "Get the cloudinit configuration with both current and pending values.",
+ permissions => {
+ check => ['perm', '/vms/{vmid}', [ 'VM.Audit' ]],
+ },
+ parameters => {
+ additionalProperties => 0,
+ properties => {
+ node => get_standard_option('pve-node'),
+ vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }),
+ },
+ },
+ returns => {
+ type => "array",
+ items => {
+ type => "object",
+ properties => {
+ key => {
+ description => "Configuration option name.",
+ type => 'string',
+ },
+ value => {
+ description => "Value as it was used to generate the current cloudinit image.",
+ type => 'string',
+ optional => 1,
+ },
+ pending => {
+ description => "The new pending value.",
+ type => 'string',
+ optional => 1,
+ },
+ delete => {
+ description => "Indicates a pending delete request if present and not 0. ",
+ type => 'integer',
+ minimum => 0,
+ maximum => 1,
+ optional => 1,
+ },
+ },
+ },
+ },
+ code => sub {
+ my ($param) = @_;
+
+ my $vmid = $param->{vmid};
+ my $conf = PVE::QemuConfig->load_config($vmid);
+
+ my $ci = $conf->{cloudinit};
+
+ $conf->{cipassword} = '**********' if exists $conf->{cipassword};
+ $ci->{cipassword} = '**********' if exists $ci->{cipassword};
+
+ my $res = [];
+
+ # All the values that got added
+ my $added = delete($ci->{added}) // '';
+ for my $key (PVE::Tools::split_list($added)) {
+ push @$res, { key => $key, pending => $conf->{$key} };
+ }
+
+ # All already existing values (+ their new value, if it exists)
+ for my $opt (keys %$cloudinitoptions) {
+ next if !$conf->{$opt};
+ next if $added =~ m/$opt/;
+ my $item = {
+ key => $opt,
+ };
+
+ if (my $pending = $ci->{$opt}) {
+ $item->{value} = $pending;
+ $item->{pending} = $conf->{$opt};
+ } else {
+ $item->{value} = $conf->{$opt},
+ }
+
+ push @$res, $item;
+ }
+
+ # Now, we'll find the deleted ones
+ for my $opt (keys %$ci) {
+ next if $conf->{$opt};
+ push @$res, { key => $opt, delete => 1 };
+ }
+
+ return $res;
+ }});
+
+__PACKAGE__->register_method({
+ name => 'cloudinit_update',
+ path => '{vmid}/cloudinit',
+ method => 'PUT',
+ protected => 1,
+ proxyto => 'node',
+ description => "Regenerate and change cloudinit config drive.",
+ permissions => {
+ check => ['perm', '/vms/{vmid}', ['VM.Config.Cloudinit']],
+ },
+ parameters => {
+ additionalProperties => 0,
+ properties => {
+ node => get_standard_option('pve-node'),
+ vmid => get_standard_option('pve-vmid'),
+ },
+ },
+ returns => { type => 'null' },
+ code => sub {
+ my ($param) = @_;
+
+ my $rpcenv = PVE::RPCEnvironment::get();
+ my $authuser = $rpcenv->get_user();
+
+ my $vmid = extract_param($param, 'vmid');
+
+ PVE::QemuConfig->lock_config($vmid, sub {
+ my $conf = PVE::QemuConfig->load_config($vmid);
+ PVE::QemuConfig->check_lock($conf);
+
+ my $storecfg = PVE::Storage::config();
+ PVE::QemuServer::vmconfig_update_cloudinit_drive($storecfg, $conf, $vmid);
+ });
+ return;
+ }});
+
+# POST/PUT {vmid}/config implementation
+#
+# The original API used PUT (idempotent) an we assumed that all operations
+# are fast. But it turned out that almost any configuration change can
+# involve hot-plug actions, or disk alloc/free. Such actions can take long
+# time to complete and have side effects (not idempotent).
+#
+# The new implementation uses POST and forks a worker process. We added
+# a new option 'background_delay'. If specified we wait up to
+# 'background_delay' second for the worker task to complete. It returns null
+# if the task is finished within that time, else we return the UPID.
+
+my $update_vm_api = sub {
+ my ($param, $sync) = @_;
+
+ my $rpcenv = PVE::RPCEnvironment::get();
+
+ my $authuser = $rpcenv->get_user();
+
+ my $node = extract_param($param, 'node');
+
+ my $vmid = extract_param($param, 'vmid');
+
+ my $digest = extract_param($param, 'digest');
+
+ my $background_delay = extract_param($param, 'background_delay');
+
+ my $skip_cloud_init = extract_param($param, 'skip_cloud_init');
+
+ if (defined(my $cipassword = $param->{cipassword})) {
+ # Same logic as in cloud-init (but with the regex fixed...)
+ $param->{cipassword} = PVE::Tools::encrypt_pw($cipassword)
+ if $cipassword !~ /^\$(?:[156]|2[ay])(\$.+){2}/;
+ }
+
+ my @paramarr = (); # used for log message
+ foreach my $key (sort keys %$param) {
+ my $value = $key eq 'cipassword' ? '<hidden>' : $param->{$key};
+ push @paramarr, "-$key", $value;
+ }
+
+ my $skiplock = extract_param($param, 'skiplock');
raise_param_exc({ skiplock => "Only root may use this option." })
if $skiplock && $authuser ne 'root@pam';
PVE::Tools::validate_ssh_public_keys($ssh_keys);
}
+ $param->{cpuunits} = PVE::CGroup::clamp_cpu_shares($param->{cpuunits})
+ if defined($param->{cpuunits}); # clamp value depending on cgroup version
+
die "no options specified\n" if !$delete_str && !$revert_str && !scalar(keys %$param);
my $storecfg = PVE::Storage::config();
- my $defaults = PVE::QemuServer::load_defaults();
-
&$resolve_cdrom_alias($param);
# now try to verify all parameters
return if defined($volname) && $volname eq 'cloudinit';
my $format;
- if ($volid =~ $NEW_DISK_RE) {
+ if ($volid =~ $PVE::QemuServer::Drive::NEW_DISK_RE) {
$storeid = $2;
$format = $drive->{format} || PVE::Storage::storage_default_format($storecfg, $storeid);
} else {
die "cannot add non-replicatable volume to a replicated VM\n";
};
+ $check_drive_param->($param, $storecfg, $check_replication);
+
foreach my $opt (keys %$param) {
- if (PVE::QemuServer::is_valid_drivename($opt)) {
- # cleanup drive path
- my $drive = PVE::QemuServer::parse_drive($opt, $param->{$opt});
- raise_param_exc({ $opt => "unable to parse drive options" }) if !$drive;
- PVE::QemuServer::cleanup_drive_path($opt, $storecfg, $drive);
- $check_replication->($drive);
- $param->{$opt} = PVE::QemuServer::print_drive($drive);
- } elsif ($opt =~ m/^net(\d+)$/) {
+ if ($opt =~ m/^net(\d+)$/) {
# add macaddr
my $net = PVE::QemuServer::parse_net($param->{$opt});
$param->{$opt} = PVE::QemuServer::print_net($net);
&$check_storage_access($rpcenv, $authuser, $storecfg, $vmid, $param);
+ PVE::QemuServer::check_bridge_access($rpcenv, $authuser, $param);
+
my $updatefn = sub {
my $conf = PVE::QemuConfig->load_config($vmid);
}
if ($param->{memory} || defined($param->{balloon})) {
- my $maxmem = $param->{memory} || $conf->{pending}->{memory} || $conf->{memory} || $defaults->{memory};
+
+ my $memory = $param->{memory} || $conf->{pending}->{memory} || $conf->{memory};
+ my $maxmem = get_current_memory($memory);
my $balloon = defined($param->{balloon}) ? $param->{balloon} : $conf->{pending}->{balloon} || $conf->{balloon};
die "balloon value too large (must be smaller than assigned memory)\n"
}
my $bootorder_deleted = grep {$_ eq 'bootorder'} @delete;
+ my $check_drive_perms = sub {
+ my ($opt, $val) = @_;
+ my $drive = PVE::QemuServer::parse_drive($opt, $val, 1);
+ if (PVE::QemuServer::drive_is_cloudinit($drive)) {
+ $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Cloudinit', 'VM.Config.CDROM']);
+ } elsif (PVE::QemuServer::drive_is_cdrom($drive, 1)) { # CDROM
+ $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.CDROM']);
+ } else {
+ $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
+
+ }
+ };
+
foreach my $opt (@delete) {
$modified->{$opt} = 1;
$conf = PVE::QemuConfig->load_config($vmid); # update/reload
}
} elsif (PVE::QemuServer::is_valid_drivename($opt)) {
PVE::QemuConfig->check_protection($conf, "can't remove drive '$opt'");
- $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
+ $check_drive_perms->($opt, $val);
PVE::QemuServer::vmconfig_register_unused_drive($storecfg, $vmid, $conf, PVE::QemuServer::parse_drive($opt, $val))
if $is_pending_val;
PVE::QemuConfig->add_to_pending_delete($conf, $opt, $force);
PVE::QemuConfig->add_to_pending_delete($conf, $opt, $force);
PVE::QemuConfig->write_config($vmid, $conf);
} elsif ($opt =~ m/^usb\d+$/) {
- if ($val =~ m/spice/) {
- $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.HWType']);
- } elsif ($authuser ne 'root@pam') {
- die "only root can delete '$opt' config for real devices\n";
+ check_usb_perm($rpcenv, $authuser, $vmid, undef, $opt, $val);
+ PVE::QemuConfig->add_to_pending_delete($conf, $opt, $force);
+ PVE::QemuConfig->write_config($vmid, $conf);
+ } elsif ($opt =~ m/^hostpci\d+$/) {
+ check_hostpci_perm($rpcenv, $authuser, $vmid, undef, $opt, $val);
+ PVE::QemuConfig->add_to_pending_delete($conf, $opt, $force);
+ PVE::QemuConfig->write_config($vmid, $conf);
+ } elsif ($opt eq 'tags') {
+ assert_tag_permissions($vmid, $val, '', $rpcenv, $authuser);
+ delete $conf->{$opt};
+ PVE::QemuConfig->write_config($vmid, $conf);
+ } elsif ($opt =~ m/^net\d+$/) {
+ if ($conf->{$opt}) {
+ PVE::QemuServer::check_bridge_access(
+ $rpcenv,
+ $authuser,
+ { $opt => $conf->{$opt} },
+ );
}
PVE::QemuConfig->add_to_pending_delete($conf, $opt, $force);
PVE::QemuConfig->write_config($vmid, $conf);
my $arch = PVE::QemuServer::get_vm_arch($conf);
if (PVE::QemuServer::is_valid_drivename($opt)) {
- my $drive = PVE::QemuServer::parse_drive($opt, $param->{$opt});
- # FIXME: cloudinit: CDROM or Disk?
- if (PVE::QemuServer::drive_is_cdrom($drive)) { # CDROM
- $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.CDROM']);
- } else {
- $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
+ # old drive
+ if ($conf->{$opt}) {
+ $check_drive_perms->($opt, $conf->{$opt});
}
+
+ # new drive
+ $check_drive_perms->($opt, $param->{$opt});
PVE::QemuServer::vmconfig_register_unused_drive($storecfg, $vmid, $conf, PVE::QemuServer::parse_drive($opt, $conf->{pending}->{$opt}))
if defined($conf->{pending}->{$opt});
- &$create_disks($rpcenv, $authuser, $conf->{pending}, $arch, $storecfg, $vmid, undef, {$opt => $param->{$opt}});
+ assert_scsi_feature_compatibility($opt, $conf, $storecfg, $param->{$opt})
+ if $opt =~ m/^scsi\d+$/;
+
+ my (undef, $created_opts) = create_disks(
+ $rpcenv,
+ $authuser,
+ $conf,
+ $arch,
+ $storecfg,
+ $vmid,
+ undef,
+ {$opt => $param->{$opt}},
+ undef,
+ undef,
+ );
+ $conf->{pending}->{$_} = $created_opts->{$_} for keys $created_opts->%*;
+
+ # default legacy boot order implies all cdroms anyway
+ if (@bootorder) {
+ # append new CD drives to bootorder to mark them bootable
+ my $drive = PVE::QemuServer::parse_drive($opt, $param->{$opt}, 1);
+ if (PVE::QemuServer::drive_is_cdrom($drive, 1) && !grep(/^$opt$/, @bootorder)) {
+ push @bootorder, $opt;
+ $conf->{pending}->{boot} = PVE::QemuServer::print_bootorder(\@bootorder);
+ $modified->{boot} = 1;
+ }
+ }
} elsif ($opt =~ m/^serial\d+/) {
if ((!defined($conf->{$opt}) || $conf->{$opt} eq 'socket') && $param->{$opt} eq 'socket') {
$rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.HWType']);
die "only root can modify '$opt' config for real devices\n";
}
$conf->{pending}->{$opt} = $param->{$opt};
+ } elsif ($opt eq 'vga') {
+ my $vga = PVE::QemuServer::parse_vga($param->{$opt});
+ PVE::QemuServer::assert_clipboard_config($vga);
+ $conf->{pending}->{$opt} = $param->{$opt};
} elsif ($opt =~ m/^usb\d+/) {
- if ((!defined($conf->{$opt}) || $conf->{$opt} =~ m/spice/) && $param->{$opt} =~ m/spice/) {
- $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.HWType']);
- } elsif ($authuser ne 'root@pam') {
- die "only root can modify '$opt' config for real devices\n";
+ if (my $olddevice = $conf->{$opt}) {
+ check_usb_perm($rpcenv, $authuser, $vmid, undef, $opt, $conf->{$opt});
+ }
+ check_usb_perm($rpcenv, $authuser, $vmid, undef, $opt, $param->{$opt});
+ $conf->{pending}->{$opt} = $param->{$opt};
+ } elsif ($opt =~ m/^hostpci\d+$/) {
+ if (my $oldvalue = $conf->{$opt}) {
+ check_hostpci_perm($rpcenv, $authuser, $vmid, undef, $opt, $oldvalue);
}
+ check_hostpci_perm($rpcenv, $authuser, $vmid, undef, $opt, $param->{$opt});
+ $conf->{pending}->{$opt} = $param->{$opt};
+ } elsif ($opt eq 'tags') {
+ assert_tag_permissions($vmid, $conf->{$opt}, $param->{$opt}, $rpcenv, $authuser);
+ $conf->{pending}->{$opt} = PVE::GuestHelpers::get_unique_tags($param->{$opt});
+ } elsif ($opt =~ m/^net\d+$/) {
+ if ($conf->{$opt}) {
+ PVE::QemuServer::check_bridge_access(
+ $rpcenv,
+ $authuser,
+ { $opt => $conf->{$opt} },
+ );
+ }
+ $conf->{pending}->{$opt} = $param->{$opt};
+ } elsif ($opt eq 'machine') {
+ my $machine_conf = PVE::QemuServer::Machine::parse_machine($param->{$opt});
+ PVE::QemuServer::Machine::assert_valid_machine_property($conf, $machine_conf);
$conf->{pending}->{$opt} = $param->{$opt};
} else {
$conf->{pending}->{$opt} = $param->{$opt};
if ($new_bootcfg->{order}) {
my @devs = PVE::Tools::split_list($new_bootcfg->{order});
for my $dev (@devs) {
- my $exists = $conf->{$dev} || $conf->{pending}->{$dev};
+ my $exists = $conf->{$dev} || $conf->{pending}->{$dev} || $param->{$dev};
my $deleted = grep {$_ eq $dev} @delete;
die "invalid bootorder: device '$dev' does not exist'\n"
if !$exists || $deleted;
if ($running) {
PVE::QemuServer::vmconfig_hotplug_pending($vmid, $conf, $storecfg, $modified, $errors);
} else {
- PVE::QemuServer::vmconfig_apply_pending($vmid, $conf, $storecfg, $running, $errors);
+ # cloud_init must be skipped if we are in an incoming, remote live migration
+ PVE::QemuServer::vmconfig_apply_pending($vmid, $conf, $storecfg, $errors, $skip_cloud_init);
}
raise_param_exc($errors) if scalar(keys %$errors);
if (!$running) {
my $status = PVE::Tools::upid_read_status($upid);
- return if $status eq 'OK';
- die $status;
+ return if !PVE::Tools::upid_status_is_error($status);
+ die "failed to update VM $vmid: $status\n";
}
}
check => ['perm', '/vms/{vmid}', $vm_config_perm_list, any => 1],
},
parameters => {
- additionalProperties => 0,
+ additionalProperties => 0,
properties => PVE::QemuServer::json_config_properties(
{
node => get_standard_option('pve-node'),
maximum => 30,
optional => 1,
},
- }),
+ },
+ 1, # with_disk_alloc
+ ),
},
returns => {
type => 'string',
check => ['perm', '/vms/{vmid}', $vm_config_perm_list, any => 1],
},
parameters => {
- additionalProperties => 0,
+ additionalProperties => 0,
properties => PVE::QemuServer::json_config_properties(
{
node => get_standard_option('pve-node'),
maxLength => 40,
optional => 1,
},
- }),
+ },
+ 1, # with_disk_alloc
+ ),
},
returns => { type => 'null' },
code => sub {
method => 'DELETE',
protected => 1,
proxyto => 'node',
- description => "Destroy the vm (also delete all used/owned volumes).",
+ description => "Destroy the VM and all used/owned volumes. Removes any VM specific permissions"
+ ." and firewall rules",
permissions => {
check => [ 'perm', '/vms/{vmid}', ['VM.Allocate']],
},
parameters => {
- additionalProperties => 0,
+ additionalProperties => 0,
properties => {
node => get_standard_option('pve-node'),
vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid_stopped }),
skiplock => get_standard_option('skiplock'),
purge => {
type => 'boolean',
- description => "Remove vmid from backup cron jobs.",
+ description => "Remove VMID from configurations, like backup & replication jobs and HA.",
+ optional => 1,
+ },
+ 'destroy-unreferenced-disks' => {
+ type => 'boolean',
+ description => "If set, destroy additionally all disks not referenced in the config"
+ ." but with a matching VMID from all enabled storages.",
optional => 1,
+ default => 0,
},
},
},
# repeat, config might have changed
my $ha_managed = $early_checks->();
- PVE::QemuServer::destroy_vm($storecfg, $vmid, $skiplock, { lock => 'destroyed' });
+ my $purge_unreferenced = $param->{'destroy-unreferenced-disks'};
+
+ PVE::QemuServer::destroy_vm(
+ $storecfg,
+ $vmid,
+ $skiplock, { lock => 'destroyed' },
+ $purge_unreferenced,
+ );
PVE::AccessControl::remove_vm_access($vmid);
PVE::Firewall::remove_vmfw_conf($vmid);
check => [ 'perm', '/vms/{vmid}', ['VM.Config.Disk']],
},
parameters => {
- additionalProperties => 0,
+ additionalProperties => 0,
properties => {
node => get_standard_option('pve-node'),
vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }),
},
description => "Creates a TCP VNC proxy connections.",
parameters => {
- additionalProperties => 0,
+ additionalProperties => 0,
properties => {
node => get_standard_option('pve-node'),
vmid => get_standard_option('pve-vmid'),
websocket => {
optional => 1,
type => 'boolean',
- description => "starts websockify instead of vncproxy",
+ description => "Prepare for websocket upgrade (only required when using "
+ ."serial terminal, otherwise upgrade is always possible).",
},
'generate-password' => {
optional => 1,
},
},
returns => {
- additionalProperties => 0,
+ additionalProperties => 0,
properties => {
user => { type => 'string' },
ticket => { type => 'string' },
} else {
- $ENV{LC_PVE_TICKET} = $password if $websocket; # set ticket with "qm vncproxy"
+ $ENV{LC_PVE_TICKET} = $password; # set ticket with "qm vncproxy"
$cmd = [@$remcmd, "/usr/sbin/qm", 'vncproxy', $vmid];
},
description => "Opens a weksocket for VNC traffic.",
parameters => {
- additionalProperties => 0,
+ additionalProperties => 0,
properties => {
node => get_standard_option('pve-node'),
vmid => get_standard_option('pve-vmid'),
},
description => "Returns a SPICE configuration to connect to the VM.",
parameters => {
- additionalProperties => 0,
+ additionalProperties => 0,
properties => {
node => get_standard_option('pve-node'),
vmid => get_standard_option('pve-vmid'),
user => 'all',
},
parameters => {
- additionalProperties => 0,
+ additionalProperties => 0,
properties => {
node => get_standard_option('pve-node'),
vmid => get_standard_option('pve-vmid'),
check => ['perm', '/vms/{vmid}', [ 'VM.Audit' ]],
},
parameters => {
- additionalProperties => 0,
+ additionalProperties => 0,
properties => {
node => get_standard_option('pve-node'),
vmid => get_standard_option('pve-vmid'),
type => 'object',
},
spice => {
- description => "Qemu VGA configuration supports spice.",
+ description => "QEMU VGA configuration supports spice.",
type => 'boolean',
optional => 1,
},
agent => {
- description => "Qemu GuestAgent enabled in config.",
+ description => "QEMU Guest Agent is enabled in config.",
type => 'boolean',
optional => 1,
},
+ clipboard => {
+ description => 'Enable a specific clipboard. If not set, depending on'
+ .' the display type the SPICE one will be added.',
+ type => 'string',
+ enum => ['vnc'],
+ optional => 1,
+ },
},
},
code => sub {
$status->{ha} = PVE::HA::Config::get_service_status("vm:$param->{vmid}");
- $status->{spice} = 1 if PVE::QemuServer::vga_conf_has_spice($conf->{vga});
- $status->{agent} = 1 if (PVE::QemuServer::parse_guest_agent($conf)->{enabled});
+ if ($conf->{vga}) {
+ my $vga = PVE::QemuServer::parse_vga($conf->{vga});
+ my $spice = defined($vga->{type}) && $vga->{type} =~ /^virtio/;
+ $spice ||= PVE::QemuServer::vga_conf_has_spice($conf->{vga});
+ $status->{spice} = 1 if $spice;
+ $status->{clipboard} = $vga->{clipboard};
+ }
+ $status->{agent} = 1 if PVE::QemuServer::get_qga_key($conf, 'enabled');
return $status;
}});
check => ['perm', '/vms/{vmid}', [ 'VM.PowerMgmt' ]],
},
parameters => {
- additionalProperties => 0,
+ additionalProperties => 0,
properties => {
node => get_standard_option('pve-node'),
vmid => get_standard_option('pve-vmid',
my $node = extract_param($param, 'node');
my $vmid = extract_param($param, 'vmid');
my $timeout = extract_param($param, 'timeout');
-
my $machine = extract_param($param, 'machine');
- my $force_cpu = extract_param($param, 'force-cpu');
my $get_root_param = sub {
my $value = extract_param($param, $_[0]);
my $migration_type = $get_root_param->('migration_type');
my $migration_network = $get_root_param->('migration_network');
my $targetstorage = $get_root_param->('targetstorage');
+ my $force_cpu = $get_root_param->('force-cpu');
my $storagemap;
my $spice_ticket;
my $nbd_protocol_version = 0;
my $replicated_volumes = {};
+ my $offline_volumes = {};
if ($stateuri && ($stateuri eq 'tcp' || $stateuri eq 'unix') && $migratedfrom && ($rpcenv->{type} eq 'cli')) {
while (defined(my $line = <STDIN>)) {
chomp $line;
$nbd_protocol_version = $1;
} elsif ($line =~ m/^replicated_volume: (.*)$/) {
$replicated_volumes->{$1} = 1;
- } else {
+ } elsif ($line =~ m/^tpmstate0: (.*)$/) { # Deprecated, use offline_volume instead
+ $offline_volumes->{tpmstate0} = $1;
+ } elsif ($line =~ m/^offline_volume: ([^:]+): (.*)$/) {
+ $offline_volumes->{$1} = $2;
+ } elsif (!$spice_ticket) {
# fallback for old source node
$spice_ticket = $line;
+ } else {
+ warn "unknown 'start' parameter on STDIN: '$line'\n";
}
}
}
storagemap => $storagemap,
nbd_proto_version => $nbd_protocol_version,
replicated_volumes => $replicated_volumes,
+ offline_volumes => $offline_volumes,
};
my $params = {
method => 'POST',
protected => 1,
proxyto => 'node',
- description => "Stop virtual machine. The qemu process will exit immediately. This" .
- "is akin to pulling the power plug of a running computer and may damage the VM data",
+ description => "Stop virtual machine. The qemu process will exit immediately. This"
+ ." is akin to pulling the power plug of a running computer and may damage the VM data.",
permissions => {
check => ['perm', '/vms/{vmid}', [ 'VM.PowerMgmt' ]],
},
parameters => {
- additionalProperties => 0,
+ additionalProperties => 0,
properties => {
node => get_standard_option('pve-node'),
vmid => get_standard_option('pve-vmid',
type => 'boolean',
optional => 1,
default => 0,
- }
+ },
+ 'overrule-shutdown' => {
+ description => "Try to abort active 'qmshutdown' tasks before stopping.",
+ optional => 1,
+ type => 'boolean',
+ default => 0,
+ },
},
},
returns => {
raise_param_exc({ migratedfrom => "Only root may use this option." })
if $migratedfrom && $authuser ne 'root@pam';
+ my $overrule_shutdown = extract_param($param, 'overrule-shutdown');
my $storecfg = PVE::Storage::config();
if (PVE::HA::Config::vm_is_ha_managed($vmid) && ($rpcenv->{type} ne 'ha') && !defined($migratedfrom)) {
+ raise_param_exc({ 'overrule-shutdown' => "Not applicable for HA resources." })
+ if $overrule_shutdown;
my $hacmd = sub {
my $upid = shift;
syslog('info', "stop VM $vmid: $upid\n");
+ if ($overrule_shutdown) {
+ my $overruled_tasks = PVE::GuestHelpers::abort_guest_tasks(
+ $rpcenv, 'qmshutdown', $vmid);
+ my $overruled_tasks_list = join(", ", $overruled_tasks->@*);
+ print "overruled qmshutdown tasks: $overruled_tasks_list\n"
+ if @$overruled_tasks;
+ };
+
PVE::QemuServer::vm_stop($storecfg, $vmid, $skiplock, 0,
$param->{timeout}, 0, 1, $keepActive, $migratedfrom);
return;
check => ['perm', '/vms/{vmid}', [ 'VM.PowerMgmt' ]],
},
parameters => {
- additionalProperties => 0,
+ additionalProperties => 0,
properties => {
node => get_standard_option('pve-node'),
vmid => get_standard_option('pve-vmid',
return $rpcenv->fork_worker('qmreset', $vmid, $authuser, $realcmd);
}});
-my sub vm_is_paused {
- my ($vmid) = @_;
- my $qmpstatus = eval {
- PVE::QemuConfig::assert_config_exists_on_node($vmid);
- mon_cmd($vmid, "query-status");
- };
- warn "$@\n" if $@;
- return $qmpstatus && $qmpstatus->{status} eq "paused";
-}
-
__PACKAGE__->register_method({
name => 'vm_shutdown',
path => '{vmid}/status/shutdown',
method => 'POST',
protected => 1,
proxyto => 'node',
- description => "Shutdown virtual machine. This is similar to pressing the power button on a physical machine." .
- "This will send an ACPI event for the guest OS, which should then proceed to a clean shutdown.",
+ description => "Shutdown virtual machine. This is similar to pressing the power button on a"
+ ." physical machine. This will send an ACPI event for the guest OS, which should then"
+ ." proceed to a clean shutdown.",
permissions => {
check => ['perm', '/vms/{vmid}', [ 'VM.PowerMgmt' ]],
},
parameters => {
- additionalProperties => 0,
+ additionalProperties => 0,
properties => {
node => get_standard_option('pve-node'),
vmid => get_standard_option('pve-vmid',
my $shutdown = 1;
- # if vm is paused, do not shutdown (but stop if forceStop = 1)
- # otherwise, we will infer a shutdown command, but run into the timeout,
- # then when the vm is resumed, it will instantly shutdown
- #
- # checking the qmp status here to get feedback to the gui/cli/api
- # and the status query should not take too long
- if (vm_is_paused($vmid)) {
+ # sending a graceful shutdown command to paused VMs runs into timeouts, and even worse, when
+ # the VM gets resumed later, it still gets the request delivered and powers off
+ if (PVE::QemuServer::vm_is_paused($vmid, 1)) {
if ($param->{forceStop}) {
warn "VM is paused - stop instead of shutdown\n";
$shutdown = 0;
my $node = extract_param($param, 'node');
my $vmid = extract_param($param, 'vmid');
- die "VM is paused - cannot shutdown\n" if vm_is_paused($vmid);
+ die "VM is paused - cannot shutdown\n" if PVE::QemuServer::vm_is_paused($vmid, 1);
die "VM $vmid not running\n" if !PVE::QemuServer::check_running($vmid);
check => ['perm', '/vms/{vmid}', [ 'VM.PowerMgmt' ]],
},
parameters => {
- additionalProperties => 0,
+ additionalProperties => 0,
properties => {
node => get_standard_option('pve-node'),
vmid => get_standard_option('pve-vmid',
# early check for storage permission, for better user feedback
if ($todisk) {
$rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
+ my $conf = PVE::QemuConfig->load_config($vmid);
+
+ # cannot save the state of a non-virtualized PCIe device, so resume cannot really work
+ for my $key (keys %$conf) {
+ next if $key !~ /^hostpci\d+/;
+ die "cannot suspend VM to disk due to passed-through PCI device(s), which lack the"
+ ." possibility to save/restore their internal state\n";
+ }
if (!$statestorage) {
# get statestorage from config if none is given
- my $conf = PVE::QemuConfig->load_config($vmid);
my $storecfg = PVE::Storage::config();
$statestorage = PVE::QemuServer::find_vmstate_storage($conf, $storecfg);
}
check => ['perm', '/vms/{vmid}', [ 'VM.PowerMgmt' ]],
},
parameters => {
- additionalProperties => 0,
+ additionalProperties => 0,
properties => {
node => get_standard_option('pve-node'),
vmid => get_standard_option('pve-vmid',
raise_param_exc({ skiplock => "Only root may use this option." })
if $skiplock && $authuser ne 'root@pam';
+ # nocheck is used as part of migration when config file might be still
+ # be on source node
my $nocheck = extract_param($param, 'nocheck');
raise_param_exc({ nocheck => "Only root may use this option." })
if $nocheck && $authuser ne 'root@pam';
check => ['perm', '/vms/{vmid}', [ 'VM.Console' ]],
},
parameters => {
- additionalProperties => 0,
+ additionalProperties => 0,
properties => {
node => get_standard_option('pve-node'),
vmid => get_standard_option('pve-vmid',
check => ['perm', '/vms/{vmid}', [ 'VM.Audit' ]],
},
parameters => {
- additionalProperties => 0,
+ additionalProperties => 0,
properties => {
node => get_standard_option('pve-node'),
vmid => get_standard_option('pve-vmid'),
permissions => {
description => "You need 'VM.Clone' permissions on /vms/{vmid}, and 'VM.Allocate' permissions " .
"on /vms/{newid} (or on the VM pool /pool/{pool}). You also need " .
- "'Datastore.AllocateSpace' on any used storage.",
+ "'Datastore.AllocateSpace' on any used storage and 'SDN.Use' on any used bridge/vnet",
check =>
[ 'and',
['perm', '/vms/{vmid}', [ 'VM.Clone' ]],
]
},
parameters => {
- additionalProperties => 0,
+ additionalProperties => 0,
properties => {
node => get_standard_option('pve-node'),
vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }),
my $vmid = extract_param($param, 'vmid');
my $newid = extract_param($param, 'newid');
my $pool = extract_param($param, 'pool');
- $rpcenv->check_pool_exist($pool) if defined($pool);
my $snapname = extract_param($param, 'snapname');
my $storage = extract_param($param, 'storage');
undef $target;
}
- PVE::Cluster::check_node_exists($target) if $target;
-
- my $storecfg = PVE::Storage::config();
+ my $running = PVE::QemuServer::check_running($vmid) || 0;
- if ($storage) {
- # check if storage is enabled on local node
- PVE::Storage::storage_check_enabled($storecfg, $storage);
- if ($target) {
- # check if storage is available on target node
- PVE::Storage::storage_check_node($storecfg, $storage, $target);
- # clone only works if target storage is shared
- my $scfg = PVE::Storage::storage_config($storecfg, $storage);
- die "can't clone to non-shared storage '$storage'\n" if !$scfg->{shared};
- }
- }
+ my $load_and_check = sub {
+ $rpcenv->check_pool_exist($pool) if defined($pool);
+ PVE::Cluster::check_node_exists($target) if $target;
- PVE::Cluster::check_cfs_quorum();
+ my $storecfg = PVE::Storage::config();
- my $running = PVE::QemuServer::check_running($vmid) || 0;
+ if ($storage) {
+ # check if storage is enabled on local node
+ PVE::Storage::storage_check_enabled($storecfg, $storage);
+ if ($target) {
+ # check if storage is available on target node
+ PVE::Storage::storage_check_enabled($storecfg, $storage, $target);
+ # clone only works if target storage is shared
+ my $scfg = PVE::Storage::storage_config($storecfg, $storage);
+ die "can't clone to non-shared storage '$storage'\n"
+ if !$scfg->{shared};
+ }
+ }
- my $clonefn = sub {
- # do all tests after lock but before forking worker - if possible
+ PVE::Cluster::check_cfs_quorum();
my $conf = PVE::QemuConfig->load_config($vmid);
PVE::QemuConfig->check_lock($conf);
die "snapshot '$snapname' does not exist\n"
if $snapname && !defined( $conf->{snapshots}->{$snapname});
- my $full = extract_param($param, 'full') // !PVE::QemuConfig->is_template($conf);
+ my $full = $param->{full} // !PVE::QemuConfig->is_template($conf);
die "parameter 'storage' not allowed for linked clones\n"
if defined($storage) && !$full;
my $oldconf = $snapname ? $conf->{snapshots}->{$snapname} : $conf;
my $sharedvm = &$check_storage_access_clone($rpcenv, $authuser, $storecfg, $oldconf, $storage);
+ PVE::QemuServer::check_mapping_access($rpcenv, $authuser, $oldconf);
+
+ PVE::QemuServer::check_bridge_access($rpcenv, $authuser, $oldconf);
die "can't clone VM to node '$target' (VM uses local storage)\n"
if $target && !$sharedvm;
# no need to copy unused images, because VMID(owner) changes anyways
next if $opt =~ m/^unused\d+$/;
+ die "cannot clone TPM state while VM is running\n"
+ if $full && $running && !$snapname && $opt eq 'tpmstate0';
+
# always change MAC! address
if ($opt =~ m/^net(\d+)$/) {
my $net = PVE::QemuServer::parse_net($value);
}
}
- # auto generate a new uuid
+ return ($conffile, $newconf, $oldconf, $vollist, $drives, $fullclone);
+ };
+
+ my $clonefn = sub {
+ my ($conffile, $newconf, $oldconf, $vollist, $drives, $fullclone) = $load_and_check->();
+ my $storecfg = PVE::Storage::config();
+
+ # auto generate a new uuid
my $smbios1 = PVE::QemuServer::parse_smbios1($newconf->{smbios1} || '');
$smbios1->{uuid} = PVE::QemuServer::generate_uuid();
$newconf->{smbios1} = PVE::QemuServer::print_smbios1($smbios1);
# FIXME use PVE::QemuConfig->create_and_lock_config and adapt code
PVE::Tools::file_set_contents($conffile, "# qmclone temporary file\nlock: clone\n");
- my $realcmd = sub {
- my $upid = shift;
+ PVE::Firewall::clone_vmfw_conf($vmid, $newid);
- my $newvollist = [];
- my $jobs = {};
+ my $newvollist = [];
+ my $jobs = {};
- eval {
- local $SIG{INT} =
- local $SIG{TERM} =
- local $SIG{QUIT} =
- local $SIG{HUP} = sub { die "interrupted by signal\n"; };
+ eval {
+ local $SIG{INT} =
+ local $SIG{TERM} =
+ local $SIG{QUIT} =
+ local $SIG{HUP} = sub { die "interrupted by signal\n"; };
- PVE::Storage::activate_volumes($storecfg, $vollist, $snapname);
+ PVE::Storage::activate_volumes($storecfg, $vollist, $snapname);
- my $bwlimit = extract_param($param, 'bwlimit');
+ my $bwlimit = extract_param($param, 'bwlimit');
- my $total_jobs = scalar(keys %{$drives});
- my $i = 1;
+ my $total_jobs = scalar(keys %{$drives});
+ my $i = 1;
- foreach my $opt (keys %$drives) {
- my $drive = $drives->{$opt};
- my $skipcomplete = ($total_jobs != $i); # finish after last drive
- my $completion = $skipcomplete ? 'skip' : 'complete';
+ foreach my $opt (sort keys %$drives) {
+ my $drive = $drives->{$opt};
+ my $skipcomplete = ($total_jobs != $i); # finish after last drive
+ my $completion = $skipcomplete ? 'skip' : 'complete';
- my $src_sid = PVE::Storage::parse_volume_id($drive->{file});
- my $storage_list = [ $src_sid ];
- push @$storage_list, $storage if defined($storage);
- my $clonelimit = PVE::Storage::get_bandwidth_limit('clone', $storage_list, $bwlimit);
+ my $src_sid = PVE::Storage::parse_volume_id($drive->{file});
+ my $storage_list = [ $src_sid ];
+ push @$storage_list, $storage if defined($storage);
+ my $clonelimit = PVE::Storage::get_bandwidth_limit('clone', $storage_list, $bwlimit);
- my $newdrive = PVE::QemuServer::clone_disk($storecfg, $vmid, $running, $opt, $drive, $snapname,
- $newid, $storage, $format, $fullclone->{$opt}, $newvollist,
- $jobs, $completion, $oldconf->{agent}, $clonelimit, $oldconf);
+ my $source_info = {
+ vmid => $vmid,
+ running => $running,
+ drivename => $opt,
+ drive => $drive,
+ snapname => $snapname,
+ };
- $newconf->{$opt} = PVE::QemuServer::print_drive($newdrive);
+ my $dest_info = {
+ vmid => $newid,
+ drivename => $opt,
+ storage => $storage,
+ format => $format,
+ };
- PVE::QemuConfig->write_config($newid, $newconf);
- $i++;
- }
+ $dest_info->{efisize} = PVE::QemuServer::get_efivars_size($oldconf)
+ if $opt eq 'efidisk0';
- delete $newconf->{lock};
+ my $newdrive = PVE::QemuServer::clone_disk(
+ $storecfg,
+ $source_info,
+ $dest_info,
+ $fullclone->{$opt},
+ $newvollist,
+ $jobs,
+ $completion,
+ $oldconf->{agent},
+ $clonelimit,
+ );
- # do not write pending changes
- if (my @changes = keys %{$newconf->{pending}}) {
- my $pending = join(',', @changes);
- warn "found pending changes for '$pending', discarding for clone\n";
- delete $newconf->{pending};
- }
+ $newconf->{$opt} = PVE::QemuServer::print_drive($newdrive);
PVE::QemuConfig->write_config($newid, $newconf);
+ $i++;
+ }
- if ($target) {
- # always deactivate volumes - avoid lvm LVs to be active on several nodes
- PVE::Storage::deactivate_volumes($storecfg, $vollist, $snapname) if !$running;
- PVE::Storage::deactivate_volumes($storecfg, $newvollist);
+ delete $newconf->{lock};
- my $newconffile = PVE::QemuConfig->config_file($newid, $target);
- die "Failed to move config to node '$target' - rename failed: $!\n"
- if !rename($conffile, $newconffile);
- }
+ # do not write pending changes
+ if (my @changes = keys %{$newconf->{pending}}) {
+ my $pending = join(',', @changes);
+ warn "found pending changes for '$pending', discarding for clone\n";
+ delete $newconf->{pending};
+ }
- PVE::AccessControl::add_vm_to_pool($newid, $pool) if $pool;
- };
- if (my $err = $@) {
- eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
- sleep 1; # some storage like rbd need to wait before release volume - really?
+ PVE::QemuConfig->write_config($newid, $newconf);
- foreach my $volid (@$newvollist) {
- eval { PVE::Storage::vdisk_free($storecfg, $volid); };
- warn $@ if $@;
- }
+ PVE::QemuServer::create_ifaces_ipams_ips($newconf, $newid);
- PVE::Firewall::remove_vmfw_conf($newid);
+ if ($target) {
+ if (!$running) {
+ # always deactivate volumes – avoids that LVM LVs are active on several nodes
+ eval { PVE::Storage::deactivate_volumes($storecfg, $vollist, $snapname) };
+ # but only warn when that fails (e.g., parallel clones keeping them active)
+ log_warn($@) if $@;
+ }
- unlink $conffile; # avoid races -> last thing before die
+ PVE::Storage::deactivate_volumes($storecfg, $newvollist);
- die "clone failed: $err";
+ my $newconffile = PVE::QemuConfig->config_file($newid, $target);
+ die "Failed to move config to node '$target' - rename failed: $!\n"
+ if !rename($conffile, $newconffile);
}
- return;
+ PVE::AccessControl::add_vm_to_pool($newid, $pool) if $pool;
};
+ if (my $err = $@) {
+ eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
+ sleep 1; # some storage like rbd need to wait before release volume - really?
- PVE::Firewall::clone_vmfw_conf($vmid, $newid);
+ foreach my $volid (@$newvollist) {
+ eval { PVE::Storage::vdisk_free($storecfg, $volid); };
+ warn $@ if $@;
+ }
+
+ PVE::Firewall::remove_vmfw_conf($newid);
+
+ unlink $conffile; # avoid races -> last thing before die
- return $rpcenv->fork_worker('qmclone', $vmid, $authuser, $realcmd);
+ die "clone failed: $err";
+ }
+
+ return;
};
# Aquire exclusive lock lock for $newid
return PVE::QemuConfig->lock_config_full($newid, 1, $clonefn);
};
- # exclusive lock if VM is running - else shared lock is enough;
- if ($running) {
- return PVE::QemuConfig->lock_config_full($vmid, 1, $lock_target_vm);
- } else {
- return PVE::QemuConfig->lock_config_shared($vmid, 1, $lock_target_vm);
- }
+ my $lock_source_vm = sub {
+ # exclusive lock if VM is running - else shared lock is enough;
+ if ($running) {
+ return PVE::QemuConfig->lock_config_full($vmid, 1, $lock_target_vm);
+ } else {
+ return PVE::QemuConfig->lock_config_shared($vmid, 1, $lock_target_vm);
+ }
+ };
+
+ $load_and_check->(); # early checks before forking/locking
+
+ return $rpcenv->fork_worker('qmclone', $vmid, $authuser, $lock_source_vm);
}});
__PACKAGE__->register_method({
method => 'POST',
protected => 1,
proxyto => 'node',
- description => "Move volume to different storage.",
+ description => "Move volume to different storage or to a different VM.",
permissions => {
- description => "You need 'VM.Config.Disk' permissions on /vms/{vmid}, and 'Datastore.AllocateSpace' permissions on the storage.",
- check => [ 'and',
- ['perm', '/vms/{vmid}', [ 'VM.Config.Disk' ]],
- ['perm', '/storage/{storage}', [ 'Datastore.AllocateSpace' ]],
- ],
+ description => "You need 'VM.Config.Disk' permissions on /vms/{vmid}, " .
+ "and 'Datastore.AllocateSpace' permissions on the storage. To move ".
+ "a disk to another VM, you need the permissions on the target VM as well.",
+ check => ['perm', '/vms/{vmid}', [ 'VM.Config.Disk' ]],
},
parameters => {
- additionalProperties => 0,
+ additionalProperties => 0,
properties => {
node => get_standard_option('pve-node'),
vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }),
+ 'target-vmid' => get_standard_option('pve-vmid', {
+ completion => \&PVE::QemuServer::complete_vmid,
+ optional => 1,
+ }),
disk => {
type => 'string',
description => "The disk you want to move.",
- enum => [PVE::QemuServer::Drive::valid_drive_names()],
+ enum => [PVE::QemuServer::Drive::valid_drive_names_with_unused()],
},
storage => get_standard_option('pve-storage-id', {
description => "Target storage.",
completion => \&PVE::QemuServer::complete_storage,
+ optional => 1,
}),
- 'format' => {
- type => 'string',
- description => "Target Format.",
- enum => [ 'raw', 'qcow2', 'vmdk' ],
- optional => 1,
- },
+ 'format' => {
+ type => 'string',
+ description => "Target Format.",
+ enum => [ 'raw', 'qcow2', 'vmdk' ],
+ optional => 1,
+ },
delete => {
type => 'boolean',
- description => "Delete the original disk after successful copy. By default the original disk is kept as unused disk.",
+ description => "Delete the original disk after successful copy. By default the"
+ ." original disk is kept as unused disk.",
optional => 1,
default => 0,
},
digest => {
type => 'string',
- description => 'Prevent changes if current configuration file has different SHA1 digest. This can be used to prevent concurrent modifications.',
+ description => 'Prevent changes if current configuration file has different SHA1"
+ ." digest. This can be used to prevent concurrent modifications.',
maxLength => 40,
optional => 1,
},
minimum => '0',
default => 'move limit from datacenter or storage config',
},
+ 'target-disk' => {
+ type => 'string',
+ description => "The config key the disk will be moved to on the target VM"
+ ." (for example, ide0 or scsi1). Default is the source disk key.",
+ enum => [PVE::QemuServer::Drive::valid_drive_names_with_unused()],
+ optional => 1,
+ },
+ 'target-digest' => {
+ type => 'string',
+ description => 'Prevent changes if the current config file of the target VM has a"
+ ." different SHA1 digest. This can be used to detect concurrent modifications.',
+ maxLength => 40,
+ optional => 1,
+ },
},
},
returns => {
my $node = extract_param($param, 'node');
my $vmid = extract_param($param, 'vmid');
+ my $target_vmid = extract_param($param, 'target-vmid');
my $digest = extract_param($param, 'digest');
+ my $target_digest = extract_param($param, 'target-digest');
my $disk = extract_param($param, 'disk');
+ my $target_disk = extract_param($param, 'target-disk') // $disk;
my $storeid = extract_param($param, 'storage');
my $format = extract_param($param, 'format');
my $storecfg = PVE::Storage::config();
- my $updatefn = sub {
+ my $load_and_check_move = sub {
my $conf = PVE::QemuConfig->load_config($vmid);
PVE::QemuConfig->check_lock($conf);
- die "VM config checksum missmatch (file change by other user?)\n"
- if $digest && $digest ne $conf->{digest};
+ PVE::Tools::assert_if_modified($digest, $conf->{digest});
die "disk '$disk' does not exist\n" if !$conf->{$disk};
$oldfmt = $1;
}
- die "you can't move to the same storage with same format\n" if $oldstoreid eq $storeid &&
- (!$format || !$oldfmt || $oldfmt eq $format);
+ die "you can't move to the same storage with same format\n"
+ if $oldstoreid eq $storeid && (!$format || !$oldfmt || $oldfmt eq $format);
# this only checks snapshots because $disk is passed!
- my $snapshotted = PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $disk, $old_volid);
+ my $snapshotted = PVE::QemuServer::Drive::is_volume_in_use(
+ $storecfg,
+ $conf,
+ $disk,
+ $old_volid
+ );
die "you can't move a disk with snapshots and delete the source\n"
if $snapshotted && $param->{delete};
- PVE::Cluster::log_msg('info', $authuser, "move disk VM $vmid: move --disk $disk --storage $storeid");
+ return ($conf, $drive, $oldstoreid, $snapshotted);
+ };
+
+ my $move_updatefn = sub {
+ my ($conf, $drive, $oldstoreid, $snapshotted) = $load_and_check_move->();
+ my $old_volid = $drive->{file};
+
+ PVE::Cluster::log_msg(
+ 'info',
+ $authuser,
+ "move disk VM $vmid: move --disk $disk --storage $storeid"
+ );
my $running = PVE::QemuServer::check_running($vmid);
PVE::Storage::activate_volumes($storecfg, [ $drive->{file} ]);
- my $realcmd = sub {
- my $newvollist = [];
+ my $newvollist = [];
+
+ eval {
+ local $SIG{INT} =
+ local $SIG{TERM} =
+ local $SIG{QUIT} =
+ local $SIG{HUP} = sub { die "interrupted by signal\n"; };
+
+ warn "moving disk with snapshots, snapshots will not be moved!\n"
+ if $snapshotted;
+
+ my $bwlimit = extract_param($param, 'bwlimit');
+ my $movelimit = PVE::Storage::get_bandwidth_limit(
+ 'move',
+ [$oldstoreid, $storeid],
+ $bwlimit
+ );
+
+ my $source_info = {
+ vmid => $vmid,
+ running => $running,
+ drivename => $disk,
+ drive => $drive,
+ snapname => undef,
+ };
+
+ my $dest_info = {
+ vmid => $vmid,
+ drivename => $disk,
+ storage => $storeid,
+ format => $format,
+ };
+
+ $dest_info->{efisize} = PVE::QemuServer::get_efivars_size($conf)
+ if $disk eq 'efidisk0';
+
+ my $newdrive = PVE::QemuServer::clone_disk(
+ $storecfg,
+ $source_info,
+ $dest_info,
+ 1,
+ $newvollist,
+ undef,
+ undef,
+ undef,
+ $movelimit,
+ );
+ $conf->{$disk} = PVE::QemuServer::print_drive($newdrive);
+
+ PVE::QemuConfig->add_unused_volume($conf, $old_volid) if !$param->{delete};
+
+ # convert moved disk to base if part of template
+ PVE::QemuServer::template_create($vmid, $conf, $disk)
+ if PVE::QemuConfig->is_template($conf);
+
+ PVE::QemuConfig->write_config($vmid, $conf);
+
+ my $do_trim = PVE::QemuServer::get_qga_key($conf, 'fstrim_cloned_disks');
+ if ($running && $do_trim && PVE::QemuServer::qga_check_running($vmid)) {
+ eval { mon_cmd($vmid, "guest-fstrim") };
+ }
eval {
- local $SIG{INT} =
- local $SIG{TERM} =
- local $SIG{QUIT} =
- local $SIG{HUP} = sub { die "interrupted by signal\n"; };
+ # try to deactivate volumes - avoid lvm LVs to be active on several nodes
+ PVE::Storage::deactivate_volumes($storecfg, [ $newdrive->{file} ])
+ if !$running;
+ };
+ warn $@ if $@;
+ };
+ if (my $err = $@) {
+ foreach my $volid (@$newvollist) {
+ eval { PVE::Storage::vdisk_free($storecfg, $volid) };
+ warn $@ if $@;
+ }
+ die "storage migration failed: $err";
+ }
- warn "moving disk with snapshots, snapshots will not be moved!\n"
- if $snapshotted;
+ if ($param->{delete}) {
+ eval {
+ PVE::Storage::deactivate_volumes($storecfg, [$old_volid]);
+ PVE::Storage::vdisk_free($storecfg, $old_volid);
+ };
+ warn $@ if $@;
+ }
+ };
- my $bwlimit = extract_param($param, 'bwlimit');
- my $movelimit = PVE::Storage::get_bandwidth_limit('move', [$oldstoreid, $storeid], $bwlimit);
+ my $load_and_check_reassign_configs = sub {
+ my $vmlist = PVE::Cluster::get_vmlist()->{ids};
- my $newdrive = PVE::QemuServer::clone_disk($storecfg, $vmid, $running, $disk, $drive, undef,
- $vmid, $storeid, $format, 1, $newvollist, undef, undef, undef, $movelimit, $conf);
+ die "could not find VM ${vmid}\n" if !exists($vmlist->{$vmid});
+ die "could not find target VM ${target_vmid}\n" if !exists($vmlist->{$target_vmid});
- $conf->{$disk} = PVE::QemuServer::print_drive($newdrive);
+ my $source_node = $vmlist->{$vmid}->{node};
+ my $target_node = $vmlist->{$target_vmid}->{node};
- PVE::QemuConfig->add_unused_volume($conf, $old_volid) if !$param->{delete};
+ die "Both VMs need to be on the same node ($source_node != $target_node)\n"
+ if $source_node ne $target_node;
- # convert moved disk to base if part of template
- PVE::QemuServer::template_create($vmid, $conf, $disk)
- if PVE::QemuConfig->is_template($conf);
+ my $source_conf = PVE::QemuConfig->load_config($vmid);
+ PVE::QemuConfig->check_lock($source_conf);
+ my $target_conf = PVE::QemuConfig->load_config($target_vmid);
+ PVE::QemuConfig->check_lock($target_conf);
- PVE::QemuConfig->write_config($vmid, $conf);
+ die "Can't move disks from or to template VMs\n"
+ if ($source_conf->{template} || $target_conf->{template});
- my $do_trim = PVE::QemuServer::parse_guest_agent($conf)->{fstrim_cloned_disks};
- if ($running && $do_trim && PVE::QemuServer::qga_check_running($vmid)) {
- eval { mon_cmd($vmid, "guest-fstrim") };
- }
+ if ($digest) {
+ eval { PVE::Tools::assert_if_modified($digest, $source_conf->{digest}) };
+ die "VM ${vmid}: $@" if $@;
+ }
- eval {
- # try to deactivate volumes - avoid lvm LVs to be active on several nodes
- PVE::Storage::deactivate_volumes($storecfg, [ $newdrive->{file} ])
- if !$running;
- };
- warn $@ if $@;
- };
- if (my $err = $@) {
- foreach my $volid (@$newvollist) {
- eval { PVE::Storage::vdisk_free($storecfg, $volid) };
- warn $@ if $@;
+ if ($target_digest) {
+ eval { PVE::Tools::assert_if_modified($target_digest, $target_conf->{digest}) };
+ die "VM ${target_vmid}: $@" if $@;
+ }
+
+ die "Disk '${disk}' for VM '$vmid' does not exist\n" if !defined($source_conf->{$disk});
+
+ die "Target disk key '${target_disk}' is already in use for VM '$target_vmid'\n"
+ if $target_conf->{$target_disk};
+
+ my $drive = PVE::QemuServer::parse_drive(
+ $disk,
+ $source_conf->{$disk},
+ );
+ die "failed to parse source disk - $@\n" if !$drive;
+
+ my $source_volid = $drive->{file};
+
+ die "disk '${disk}' has no associated volume\n" if !$source_volid;
+ die "CD drive contents can't be moved to another VM\n"
+ if PVE::QemuServer::drive_is_cdrom($drive, 1);
+
+ my $storeid = PVE::Storage::parse_volume_id($source_volid, 1);
+ die "Volume '$source_volid' not managed by PVE\n" if !defined($storeid);
+
+ die "Can't move disk used by a snapshot to another VM\n"
+ if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $source_conf, $disk, $source_volid);
+ die "Storage does not support moving of this disk to another VM\n"
+ if (!PVE::Storage::volume_has_feature($storecfg, 'rename', $source_volid));
+ die "Cannot move disk to another VM while the source VM is running - detach first\n"
+ if PVE::QemuServer::check_running($vmid) && $disk !~ m/^unused\d+$/;
+
+ # now re-parse using target disk slot format
+ if ($target_disk =~ /^unused\d+$/) {
+ $drive = PVE::QemuServer::parse_drive(
+ $target_disk,
+ $source_volid,
+ );
+ } else {
+ $drive = PVE::QemuServer::parse_drive(
+ $target_disk,
+ $source_conf->{$disk},
+ );
+ }
+ die "failed to parse source disk for target disk format - $@\n" if !$drive;
+
+ my $repl_conf = PVE::ReplicationConfig->new();
+ if ($repl_conf->check_for_existing_jobs($target_vmid, 1)) {
+ my $format = (PVE::Storage::parse_volname($storecfg, $source_volid))[6];
+ die "Cannot move disk to a replicated VM. Storage does not support replication!\n"
+ if !PVE::Storage::storage_can_replicate($storecfg, $storeid, $format);
+ }
+
+ return ($source_conf, $target_conf, $drive);
+ };
+
+ my $logfunc = sub {
+ my ($msg) = @_;
+ print STDERR "$msg\n";
+ };
+
+ my $disk_reassignfn = sub {
+ return PVE::QemuConfig->lock_config($vmid, sub {
+ return PVE::QemuConfig->lock_config($target_vmid, sub {
+ my ($source_conf, $target_conf, $drive) = &$load_and_check_reassign_configs();
+
+ my $source_volid = $drive->{file};
+
+ print "moving disk '$disk' from VM '$vmid' to '$target_vmid'\n";
+ my ($storeid, $source_volname) = PVE::Storage::parse_volume_id($source_volid);
+
+ my $fmt = (PVE::Storage::parse_volname($storecfg, $source_volid))[6];
+
+ my $new_volid = PVE::Storage::rename_volume(
+ $storecfg,
+ $source_volid,
+ $target_vmid,
+ );
+
+ $drive->{file} = $new_volid;
+
+ my $boot_order = PVE::QemuServer::device_bootorder($source_conf);
+ if (defined(delete $boot_order->{$disk})) {
+ print "removing disk '$disk' from boot order config\n";
+ my $boot_devs = [ sort { $boot_order->{$a} <=> $boot_order->{$b} } keys %$boot_order ];
+ $source_conf->{boot} = PVE::QemuServer::print_bootorder($boot_devs);
}
- die "storage migration failed: $err";
- }
- if ($param->{delete}) {
- eval {
- PVE::Storage::deactivate_volumes($storecfg, [$old_volid]);
- PVE::Storage::vdisk_free($storecfg, $old_volid);
- };
- warn $@ if $@;
- }
- };
+ delete $source_conf->{$disk};
+ print "removing disk '${disk}' from VM '${vmid}' config\n";
+ PVE::QemuConfig->write_config($vmid, $source_conf);
- return $rpcenv->fork_worker('qmmove', $vmid, $authuser, $realcmd);
+ my $drive_string = PVE::QemuServer::print_drive($drive);
+
+ if ($target_disk =~ /^unused\d+$/) {
+ $target_conf->{$target_disk} = $drive_string;
+ PVE::QemuConfig->write_config($target_vmid, $target_conf);
+ } else {
+ &$update_vm_api(
+ {
+ node => $node,
+ vmid => $target_vmid,
+ digest => $target_digest,
+ $target_disk => $drive_string,
+ },
+ 1,
+ );
+ }
+
+ # remove possible replication snapshots
+ if (PVE::Storage::volume_has_feature(
+ $storecfg,
+ 'replicate',
+ $source_volid),
+ ) {
+ eval {
+ PVE::Replication::prepare(
+ $storecfg,
+ [$new_volid],
+ undef,
+ 1,
+ undef,
+ $logfunc,
+ )
+ };
+ if (my $err = $@) {
+ print "Failed to remove replication snapshots on moved disk " .
+ "'$target_disk'. Manual cleanup could be necessary.\n";
+ }
+ }
+ });
+ });
};
- return PVE::QemuConfig->lock_config($vmid, $updatefn);
+ if ($target_vmid && $storeid) {
+ my $msg = "either set 'storage' or 'target-vmid', but not both";
+ raise_param_exc({ 'target-vmid' => $msg, 'storage' => $msg });
+ } elsif ($target_vmid) {
+ $rpcenv->check_vm_perm($authuser, $target_vmid, undef, ['VM.Config.Disk'])
+ if $authuser ne 'root@pam';
+
+ raise_param_exc({ 'target-vmid' => "must be different than source VMID to reassign disk" })
+ if $vmid eq $target_vmid;
+
+ my (undef, undef, $drive) = &$load_and_check_reassign_configs();
+ my $storage = PVE::Storage::parse_volume_id($drive->{file});
+ $rpcenv->check($authuser, "/storage/$storage", ['Datastore.AllocateSpace']);
+
+ return $rpcenv->fork_worker(
+ 'qmmove',
+ "${vmid}-${disk}>${target_vmid}-${target_disk}",
+ $authuser,
+ $disk_reassignfn
+ );
+ } elsif ($storeid) {
+ $rpcenv->check($authuser, "/storage/$storeid", ['Datastore.AllocateSpace']);
+
+ $load_and_check_move->(); # early checks before forking/locking
+
+ my $realcmd = sub {
+ PVE::QemuConfig->lock_config($vmid, $move_updatefn);
+ };
+
+ return $rpcenv->fork_worker('qmmove', $vmid, $authuser, $realcmd);
+ } else {
+ my $msg = "both 'storage' and 'target-vmid' missing, either needs to be set";
+ raise_param_exc({ 'target-vmid' => $msg, 'storage' => $msg });
+ }
}});
my $check_vm_disks_local = sub {
local_resources => {
type => 'array',
description => "List local resources e.g. pci, usb"
- }
+ },
+ 'mapped-resources' => {
+ type => 'array',
+ description => "List of mapped resources e.g. pci, usb"
+ },
},
},
code => sub {
$res->{running} = PVE::QemuServer::check_running($vmid) ? 1:0;
- # if vm is not running, return target nodes where local storage is available
+ my ($local_resources, $mapped_resources, $missing_mappings_by_node) =
+ PVE::QemuServer::check_local_resources($vmconf, 1);
+ delete $missing_mappings_by_node->{$localnode};
+
+ my $vga = PVE::QemuServer::parse_vga($vmconf->{vga});
+ if ($res->{running} && $vga->{'clipboard'} && $vga->{'clipboard'} eq 'vnc') {
+ push $local_resources->@*, "clipboard=vnc";
+ }
+
+ # if vm is not running, return target nodes where local storage/mapped devices are available
# for offline migration
if (!$res->{running}) {
$res->{allowed_nodes} = [];
delete $checked_nodes->{$localnode};
foreach my $node (keys %$checked_nodes) {
- if (!defined $checked_nodes->{$node}->{unavailable_storages}) {
+ my $missing_mappings = $missing_mappings_by_node->{$node};
+ if (scalar($missing_mappings->@*)) {
+ $checked_nodes->{$node}->{'unavailable-resources'} = $missing_mappings;
+ next;
+ }
+
+ if (!defined($checked_nodes->{$node}->{unavailable_storages})) {
push @{$res->{allowed_nodes}}, $node;
}
$res->{not_allowed_nodes} = $checked_nodes;
}
-
my $local_disks = &$check_vm_disks_local($storecfg, $vmconf, $vmid);
$res->{local_disks} = [ values %$local_disks ];;
- my $local_resources = PVE::QemuServer::check_local_resources($vmconf, 1);
-
$res->{local_resources} = $local_resources;
+ $res->{'mapped-resources'} = $mapped_resources;
return $res;
check => ['perm', '/vms/{vmid}', [ 'VM.Migrate' ]],
},
parameters => {
- additionalProperties => 0,
+ additionalProperties => 0,
properties => {
node => get_standard_option('pve-node'),
vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }),
my $repl_conf = PVE::ReplicationConfig->new();
my $is_replicated = $repl_conf->check_for_existing_jobs($vmid, 1);
my $is_replicated_to_target = defined($repl_conf->find_local_replication_job($vmid, $target));
- if ($is_replicated && !$is_replicated_to_target) {
- if ($param->{force}) {
- warn "WARNING: Node '$target' is not a replication target. Existing replication " .
- "jobs will fail after migration!\n";
- } else {
- die "Cannot live-migrate replicated VM to node '$target' - not a replication target." .
- " Use 'force' to override.\n";
- }
+ if (!$param->{force} && $is_replicated && !$is_replicated_to_target) {
+ die "Cannot live-migrate replicated VM to node '$target' - not a replication " .
+ "target. Use 'force' to override.\n";
}
} else {
warn "VM isn't running. Doing offline migration instead.\n" if $param->{online};
}
my $storecfg = PVE::Storage::config();
-
if (my $targetstorage = $param->{targetstorage}) {
- my $check_storage = sub {
- my ($target_sid) = @_;
- PVE::Storage::storage_check_node($storecfg, $target_sid, $target);
- $rpcenv->check($authuser, "/storage/$target_sid", ['Datastore.AllocateSpace']);
- my $scfg = PVE::Storage::storage_config($storecfg, $target_sid);
- raise_param_exc({ targetstorage => "storage '$target_sid' does not support vm images"})
- if !$scfg->{content}->{images};
- };
-
my $storagemap = eval { PVE::JSONSchema::parse_idmap($targetstorage, 'pve-storage-id') };
raise_param_exc({ targetstorage => "failed to parse storage map: $@" })
if $@;
$rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk'])
if !defined($storagemap->{identity});
- foreach my $source (values %{$storagemap->{entries}}) {
- $check_storage->($source);
+ foreach my $target_sid (values %{$storagemap->{entries}}) {
+ $check_storage_access_migrate->($rpcenv, $authuser, $storecfg, $target_sid, $target);
}
- $check_storage->($storagemap->{default})
+ $check_storage_access_migrate->($rpcenv, $authuser, $storecfg, $storagemap->{default}, $target)
if $storagemap->{default};
PVE::QemuServer::check_storage_availability($storecfg, $conf, $target)
}});
+__PACKAGE__->register_method({
+ name => 'remote_migrate_vm',
+ path => '{vmid}/remote_migrate',
+ method => 'POST',
+ protected => 1,
+ proxyto => 'node',
+ description => "Migrate virtual machine to a remote cluster. Creates a new migration task. EXPERIMENTAL feature!",
+ permissions => {
+ check => ['perm', '/vms/{vmid}', [ 'VM.Migrate' ]],
+ },
+ parameters => {
+ additionalProperties => 0,
+ properties => {
+ node => get_standard_option('pve-node'),
+ vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }),
+ 'target-vmid' => get_standard_option('pve-vmid', { optional => 1 }),
+ 'target-endpoint' => get_standard_option('proxmox-remote', {
+ description => "Remote target endpoint",
+ }),
+ online => {
+ type => 'boolean',
+ description => "Use online/live migration if VM is running. Ignored if VM is stopped.",
+ optional => 1,
+ },
+ delete => {
+ type => 'boolean',
+ description => "Delete the original VM and related data after successful migration. By default the original VM is kept on the source cluster in a stopped state.",
+ optional => 1,
+ default => 0,
+ },
+ 'target-storage' => get_standard_option('pve-targetstorage', {
+ completion => \&PVE::QemuServer::complete_migration_storage,
+ optional => 0,
+ }),
+ 'target-bridge' => {
+ type => 'string',
+ description => "Mapping from source to target bridges. Providing only a single bridge ID maps all source bridges to that bridge. Providing the special value '1' will map each source bridge to itself.",
+ format => 'bridge-pair-list',
+ },
+ bwlimit => {
+ description => "Override I/O bandwidth limit (in KiB/s).",
+ optional => 1,
+ type => 'integer',
+ minimum => '0',
+ default => 'migrate limit from datacenter or storage config',
+ },
+ },
+ },
+ returns => {
+ type => 'string',
+ description => "the task ID.",
+ },
+ code => sub {
+ my ($param) = @_;
+
+ my $rpcenv = PVE::RPCEnvironment::get();
+ my $authuser = $rpcenv->get_user();
+
+ my $source_vmid = extract_param($param, 'vmid');
+ my $target_endpoint = extract_param($param, 'target-endpoint');
+ my $target_vmid = extract_param($param, 'target-vmid') // $source_vmid;
+
+ my $delete = extract_param($param, 'delete') // 0;
+
+ PVE::Cluster::check_cfs_quorum();
+
+ # test if VM exists
+ my $conf = PVE::QemuConfig->load_config($source_vmid);
+
+ PVE::QemuConfig->check_lock($conf);
+
+ raise_param_exc({ vmid => "cannot migrate HA-managed VM to remote cluster" })
+ if PVE::HA::Config::vm_is_ha_managed($source_vmid);
+
+ my $remote = PVE::JSONSchema::parse_property_string('proxmox-remote', $target_endpoint);
+
+ # TODO: move this as helper somewhere appropriate?
+ my $conn_args = {
+ protocol => 'https',
+ host => $remote->{host},
+ port => $remote->{port} // 8006,
+ apitoken => $remote->{apitoken},
+ };
+
+ my $fp;
+ if ($fp = $remote->{fingerprint}) {
+ $conn_args->{cached_fingerprints} = { uc($fp) => 1 };
+ }
+
+ print "Establishing API connection with remote at '$remote->{host}'\n";
+
+ my $api_client = PVE::APIClient::LWP->new(%$conn_args);
+
+ if (!defined($fp)) {
+ my $cert_info = $api_client->get("/nodes/localhost/certificates/info");
+ foreach my $cert (@$cert_info) {
+ my $filename = $cert->{filename};
+ next if $filename ne 'pveproxy-ssl.pem' && $filename ne 'pve-ssl.pem';
+ $fp = $cert->{fingerprint} if !$fp || $filename eq 'pveproxy-ssl.pem';
+ }
+ $conn_args->{cached_fingerprints} = { uc($fp) => 1 }
+ if defined($fp);
+ }
+
+ my $repl_conf = PVE::ReplicationConfig->new();
+ my $is_replicated = $repl_conf->check_for_existing_jobs($source_vmid, 1);
+ die "cannot remote-migrate replicated VM\n" if $is_replicated;
+
+ if (PVE::QemuServer::check_running($source_vmid)) {
+ die "can't migrate running VM without --online\n" if !$param->{online};
+
+ } else {
+ warn "VM isn't running. Doing offline migration instead.\n" if $param->{online};
+ $param->{online} = 0;
+ }
+
+ my $storecfg = PVE::Storage::config();
+ my $target_storage = extract_param($param, 'target-storage');
+ my $storagemap = eval { PVE::JSONSchema::parse_idmap($target_storage, 'pve-storage-id') };
+ raise_param_exc({ 'target-storage' => "failed to parse storage map: $@" })
+ if $@;
+
+ my $target_bridge = extract_param($param, 'target-bridge');
+ my $bridgemap = eval { PVE::JSONSchema::parse_idmap($target_bridge, 'pve-bridge-id') };
+ raise_param_exc({ 'target-bridge' => "failed to parse bridge map: $@" })
+ if $@;
+
+ die "remote migration requires explicit storage mapping!\n"
+ if $storagemap->{identity};
+
+ $param->{storagemap} = $storagemap;
+ $param->{bridgemap} = $bridgemap;
+ $param->{remote} = {
+ conn => $conn_args, # re-use fingerprint for tunnel
+ client => $api_client,
+ vmid => $target_vmid,
+ };
+ $param->{migration_type} = 'websocket';
+ $param->{'with-local-disks'} = 1;
+ $param->{delete} = $delete if $delete;
+
+ my $cluster_status = $api_client->get("/cluster/status");
+ my $target_node;
+ foreach my $entry (@$cluster_status) {
+ next if $entry->{type} ne 'node';
+ if ($entry->{local}) {
+ $target_node = $entry->{name};
+ last;
+ }
+ }
+
+ die "couldn't determine endpoint's node name\n"
+ if !defined($target_node);
+
+ my $realcmd = sub {
+ PVE::QemuMigrate->migrate($target_node, $remote->{host}, $source_vmid, $param);
+ };
+
+ my $worker = sub {
+ return PVE::GuestHelpers::guest_migration_lock($source_vmid, 10, $realcmd);
+ };
+
+ return $rpcenv->fork_worker('qmigrate', $source_vmid, $authuser, $worker);
+ }});
+
__PACKAGE__->register_method({
name => 'monitor',
path => '{vmid}/monitor',
method => 'POST',
protected => 1,
proxyto => 'node',
- description => "Execute Qemu monitor commands.",
+ description => "Execute QEMU monitor commands.",
permissions => {
description => "Sys.Modify is required for (sub)commands which are not read-only ('info *' and 'help')",
check => ['perm', '/vms/{vmid}', [ 'VM.Monitor' ]],
},
parameters => {
- additionalProperties => 0,
+ additionalProperties => 0,
properties => {
node => get_standard_option('pve-node'),
vmid => get_standard_option('pve-vmid'),
check => ['perm', '/vms/{vmid}', [ 'VM.Config.Disk' ]],
},
parameters => {
- additionalProperties => 0,
- properties => {
+ additionalProperties => 0,
+ properties => {
node => get_standard_option('pve-node'),
vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }),
skiplock => get_standard_option('skiplock'),
},
},
},
- returns => { type => 'null'},
+ returns => {
+ type => 'string',
+ description => "the task ID.",
+ },
code => sub {
my ($param) = @_;
my (undef, undef, undef, undef, undef, undef, $format) =
PVE::Storage::parse_volname($storecfg, $drive->{file});
- die "can't resize volume: $disk if snapshot exists\n"
- if %{$conf->{snapshots}} && $format eq 'qcow2';
-
my $volid = $drive->{file};
die "disk '$disk' has no associated volume\n" if !$volid;
PVE::QemuConfig->write_config($vmid, $conf);
};
- PVE::QemuConfig->lock_config($vmid, $updatefn);
- return;
+ my $worker = sub {
+ PVE::QemuConfig->lock_config($vmid, $updatefn);
+ };
+
+ return $rpcenv->fork_worker('resize', $vmid, $authuser, $worker);
}});
__PACKAGE__->register_method({
proxyto => 'node',
protected => 1, # qemu pid files are only readable by root
parameters => {
- additionalProperties => 0,
+ additionalProperties => 0,
properties => {
vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }),
node => get_standard_option('pve-node'),
user => 'all',
},
parameters => {
- additionalProperties => 0,
+ additionalProperties => 0,
properties => {
vmid => get_standard_option('pve-vmid'),
node => get_standard_option('pve-node'),
node => get_standard_option('pve-node'),
vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }),
snapname => get_standard_option('pve-snapshot-name'),
+ start => {
+ type => 'boolean',
+ description => "Whether the VM should get started after rolling back successfully."
+ . " (Note: VMs will be automatically started if the snapshot includes RAM.)",
+ optional => 1,
+ default => 0,
+ },
},
},
returns => {
my $realcmd = sub {
PVE::Cluster::log_msg('info', $authuser, "rollback snapshot VM $vmid: $snapname");
PVE::QemuConfig->snapshot_rollback($vmid, $snapname);
+
+ if ($param->{start} && !PVE::QemuServer::Helpers::vm_running_locally($vmid)) {
+ PVE::API2::Qemu->vm_start({ vmid => $vmid, node => $node });
+ }
};
my $worker = sub {
my $snapname = extract_param($param, 'snapname');
- my $realcmd = sub {
+ my $lock_obtained;
+ my $do_delete = sub {
+ $lock_obtained = 1;
PVE::Cluster::log_msg('info', $authuser, "delete snapshot VM $vmid: $snapname");
PVE::QemuConfig->snapshot_delete($vmid, $snapname, $param->{force});
};
+ my $realcmd = sub {
+ if ($param->{force}) {
+ $do_delete->();
+ } else {
+ eval { PVE::GuestHelpers::guest_migration_lock($vmid, 10, $do_delete); };
+ if (my $err = $@) {
+ die $err if $lock_obtained;
+ die "Failed to obtain guest migration lock - replication running?\n";
+ }
+ }
+ };
+
return $rpcenv->fork_worker('qmdelsnapshot', $vmid, $authuser, $realcmd);
}});
},
},
- returns => { type => 'null'},
+ returns => {
+ type => 'string',
+ description => "the task ID.",
+ },
code => sub {
my ($param) = @_;
my $disk = extract_param($param, 'disk');
- my $updatefn = sub {
-
+ my $load_and_check = sub {
my $conf = PVE::QemuConfig->load_config($vmid);
PVE::QemuConfig->check_lock($conf);
die "you can't convert a VM to template if VM is running\n"
if PVE::QemuServer::check_running($vmid);
- my $realcmd = sub {
- PVE::QemuServer::template_create($vmid, $conf, $disk);
- };
+ return $conf;
+ };
- $conf->{template} = 1;
- PVE::QemuConfig->write_config($vmid, $conf);
+ $load_and_check->();
+
+ my $realcmd = sub {
+ PVE::QemuConfig->lock_config($vmid, sub {
+ my $conf = $load_and_check->();
- return $rpcenv->fork_worker('qmtemplate', $vmid, $authuser, $realcmd);
+ $conf->{template} = 1;
+ PVE::QemuConfig->write_config($vmid, $conf);
+
+ PVE::QemuServer::template_create($vmid, $conf, $disk);
+ });
};
- PVE::QemuConfig->lock_config($vmid, $updatefn);
- return;
+ return $rpcenv->fork_worker('qmtemplate', $vmid, $authuser, $realcmd);
}});
__PACKAGE__->register_method({
return PVE::QemuServer::Cloudinit::dump_cloudinit_config($conf, $param->{vmid}, $param->{type});
}});
+__PACKAGE__->register_method({
+ name => 'mtunnel',
+ path => '{vmid}/mtunnel',
+ method => 'POST',
+ protected => 1,
+ description => 'Migration tunnel endpoint - only for internal use by VM migration.',
+ permissions => {
+ check =>
+ [ 'and',
+ ['perm', '/vms/{vmid}', [ 'VM.Allocate' ]],
+ ['perm', '/', [ 'Sys.Incoming' ]],
+ ],
+ description => "You need 'VM.Allocate' permissions on '/vms/{vmid}' and Sys.Incoming" .
+ " on '/'. Further permission checks happen during the actual migration.",
+ },
+ parameters => {
+ additionalProperties => 0,
+ properties => {
+ node => get_standard_option('pve-node'),
+ vmid => get_standard_option('pve-vmid'),
+ storages => {
+ type => 'string',
+ format => 'pve-storage-id-list',
+ optional => 1,
+ description => 'List of storages to check permission and availability. Will be checked again for all actually used storages during migration.',
+ },
+ bridges => {
+ type => 'string',
+ format => 'pve-bridge-id-list',
+ optional => 1,
+ description => 'List of network bridges to check availability. Will be checked again for actually used bridges during migration.',
+ },
+ },
+ },
+ returns => {
+ additionalProperties => 0,
+ properties => {
+ upid => { type => 'string' },
+ ticket => { type => 'string' },
+ socket => { type => 'string' },
+ },
+ },
+ code => sub {
+ my ($param) = @_;
+
+ my $rpcenv = PVE::RPCEnvironment::get();
+ my $authuser = $rpcenv->get_user();
+
+ my $node = extract_param($param, 'node');
+ my $vmid = extract_param($param, 'vmid');
+
+ my $storages = extract_param($param, 'storages');
+ my $bridges = extract_param($param, 'bridges');
+
+ my $nodename = PVE::INotify::nodename();
+
+ raise_param_exc({ node => "node needs to be 'localhost' or local hostname '$nodename'" })
+ if $node ne 'localhost' && $node ne $nodename;
+
+ $node = $nodename;
+
+ my $storecfg = PVE::Storage::config();
+ foreach my $storeid (PVE::Tools::split_list($storages)) {
+ $check_storage_access_migrate->($rpcenv, $authuser, $storecfg, $storeid, $node);
+ }
+
+ foreach my $bridge (PVE::Tools::split_list($bridges)) {
+ PVE::Network::read_bridge_mtu($bridge);
+ }
+
+ PVE::Cluster::check_cfs_quorum();
+
+ my $lock = 'create';
+ eval { PVE::QemuConfig->create_and_lock_config($vmid, 0, $lock); };
+
+ raise_param_exc({ vmid => "unable to create empty VM config - $@"})
+ if $@;
+
+ my $realcmd = sub {
+ my $state = {
+ storecfg => PVE::Storage::config(),
+ lock => $lock,
+ vmid => $vmid,
+ };
+
+ my $run_locked = sub {
+ my ($code, $params) = @_;
+ return PVE::QemuConfig->lock_config($state->{vmid}, sub {
+ my $conf = PVE::QemuConfig->load_config($state->{vmid});
+
+ $state->{conf} = $conf;
+
+ die "Encountered wrong lock - aborting mtunnel command handling.\n"
+ if $state->{lock} && !PVE::QemuConfig->has_lock($conf, $state->{lock});
+
+ return $code->($params);
+ });
+ };
+
+ my $cmd_desc = {
+ config => {
+ conf => {
+ type => 'string',
+ description => 'Full VM config, adapted for target cluster/node',
+ },
+ 'firewall-config' => {
+ type => 'string',
+ description => 'VM firewall config',
+ optional => 1,
+ },
+ },
+ disk => {
+ format => PVE::JSONSchema::get_standard_option('pve-qm-image-format'),
+ storage => {
+ type => 'string',
+ format => 'pve-storage-id',
+ },
+ drive => {
+ type => 'object',
+ description => 'parsed drive information without volid and format',
+ },
+ },
+ start => {
+ start_params => {
+ type => 'object',
+ description => 'params passed to vm_start_nolock',
+ },
+ migrate_opts => {
+ type => 'object',
+ description => 'migrate_opts passed to vm_start_nolock',
+ },
+ },
+ ticket => {
+ path => {
+ type => 'string',
+ description => 'socket path for which the ticket should be valid. must be known to current mtunnel instance.',
+ },
+ },
+ quit => {
+ cleanup => {
+ type => 'boolean',
+ description => 'remove VM config and disks, aborting migration',
+ default => 0,
+ },
+ },
+ 'disk-import' => $PVE::StorageTunnel::cmd_schema->{'disk-import'},
+ 'query-disk-import' => $PVE::StorageTunnel::cmd_schema->{'query-disk-import'},
+ bwlimit => $PVE::StorageTunnel::cmd_schema->{bwlimit},
+ };
+
+ my $cmd_handlers = {
+ 'version' => sub {
+ # compared against other end's version
+ # bump/reset for breaking changes
+ # bump/bump for opt-in changes
+ return {
+ api => $PVE::QemuMigrate::WS_TUNNEL_VERSION,
+ age => 0,
+ };
+ },
+ 'config' => sub {
+ my ($params) = @_;
+
+ # parse and write out VM FW config if given
+ if (my $fw_conf = $params->{'firewall-config'}) {
+ my ($path, $fh) = PVE::Tools::tempfile_contents($fw_conf, 700);
+
+ my $empty_conf = {
+ rules => [],
+ options => {},
+ aliases => {},
+ ipset => {} ,
+ ipset_comments => {},
+ };
+ my $cluster_fw_conf = PVE::Firewall::load_clusterfw_conf();
+
+ # TODO: add flag for strict parsing?
+ # TODO: add import sub that does all this given raw content?
+ my $vmfw_conf = PVE::Firewall::generic_fw_config_parser($path, $cluster_fw_conf, $empty_conf, 'vm');
+ $vmfw_conf->{vmid} = $state->{vmid};
+ PVE::Firewall::save_vmfw_conf($state->{vmid}, $vmfw_conf);
+
+ $state->{cleanup}->{fw} = 1;
+ }
+
+ my $conf_fn = "incoming/qemu-server/$state->{vmid}.conf";
+ my $new_conf = PVE::QemuServer::parse_vm_config($conf_fn, $params->{conf}, 1);
+ delete $new_conf->{lock};
+ delete $new_conf->{digest};
+
+ # TODO handle properly?
+ delete $new_conf->{snapshots};
+ delete $new_conf->{parent};
+ delete $new_conf->{pending};
+
+ # not handled by update_vm_api
+ my $vmgenid = delete $new_conf->{vmgenid};
+ my $meta = delete $new_conf->{meta};
+ my $cloudinit = delete $new_conf->{cloudinit}; # this is informational only
+ $new_conf->{skip_cloud_init} = 1; # re-use image from source side
+
+ $new_conf->{vmid} = $state->{vmid};
+ $new_conf->{node} = $node;
+
+ PVE::QemuConfig->remove_lock($state->{vmid}, 'create');
+
+ eval {
+ $update_vm_api->($new_conf, 1);
+ };
+ if (my $err = $@) {
+ # revert to locked previous config
+ my $conf = PVE::QemuConfig->load_config($state->{vmid});
+ $conf->{lock} = 'create';
+ PVE::QemuConfig->write_config($state->{vmid}, $conf);
+
+ die $err;
+ }
+
+ my $conf = PVE::QemuConfig->load_config($state->{vmid});
+ $conf->{lock} = 'migrate';
+ $conf->{vmgenid} = $vmgenid if defined($vmgenid);
+ $conf->{meta} = $meta if defined($meta);
+ $conf->{cloudinit} = $cloudinit if defined($cloudinit);
+ PVE::QemuConfig->write_config($state->{vmid}, $conf);
+
+ $state->{lock} = 'migrate';
+
+ return;
+ },
+ 'bwlimit' => sub {
+ my ($params) = @_;
+ return PVE::StorageTunnel::handle_bwlimit($params);
+ },
+ 'disk' => sub {
+ my ($params) = @_;
+
+ my $format = $params->{format};
+ my $storeid = $params->{storage};
+ my $drive = $params->{drive};
+
+ $check_storage_access_migrate->($rpcenv, $authuser, $state->{storecfg}, $storeid, $node);
+
+ my $storagemap = {
+ default => $storeid,
+ };
+
+ my $source_volumes = {
+ 'disk' => [
+ undef,
+ $storeid,
+ $drive,
+ 0,
+ $format,
+ ],
+ };
+
+ my $res = PVE::QemuServer::vm_migrate_alloc_nbd_disks($state->{storecfg}, $state->{vmid}, $source_volumes, $storagemap);
+ if (defined($res->{disk})) {
+ $state->{cleanup}->{volumes}->{$res->{disk}->{volid}} = 1;
+ return $res->{disk};
+ } else {
+ die "failed to allocate NBD disk..\n";
+ }
+ },
+ 'disk-import' => sub {
+ my ($params) = @_;
+
+ $check_storage_access_migrate->(
+ $rpcenv,
+ $authuser,
+ $state->{storecfg},
+ $params->{storage},
+ $node
+ );
+
+ $params->{unix} = "/run/qemu-server/$state->{vmid}.storage";
+
+ return PVE::StorageTunnel::handle_disk_import($state, $params);
+ },
+ 'query-disk-import' => sub {
+ my ($params) = @_;
+
+ return PVE::StorageTunnel::handle_query_disk_import($state, $params);
+ },
+ 'start' => sub {
+ my ($params) = @_;
+
+ my $info = PVE::QemuServer::vm_start_nolock(
+ $state->{storecfg},
+ $state->{vmid},
+ $state->{conf},
+ $params->{start_params},
+ $params->{migrate_opts},
+ );
+
+
+ if ($info->{migrate}->{proto} ne 'unix') {
+ PVE::QemuServer::vm_stop(undef, $state->{vmid}, 1, 1);
+ die "migration over non-UNIX sockets not possible\n";
+ }
+
+ my $socket = $info->{migrate}->{addr};
+ chown $state->{socket_uid}, -1, $socket;
+ $state->{sockets}->{$socket} = 1;
+
+ my $unix_sockets = $info->{migrate}->{unix_sockets};
+ foreach my $socket (@$unix_sockets) {
+ chown $state->{socket_uid}, -1, $socket;
+ $state->{sockets}->{$socket} = 1;
+ }
+ return $info;
+ },
+ 'fstrim' => sub {
+ if (PVE::QemuServer::qga_check_running($state->{vmid})) {
+ eval { mon_cmd($state->{vmid}, "guest-fstrim") };
+ warn "fstrim failed: $@\n" if $@;
+ }
+ return;
+ },
+ 'stop' => sub {
+ PVE::QemuServer::vm_stop(undef, $state->{vmid}, 1, 1);
+ return;
+ },
+ 'nbdstop' => sub {
+ PVE::QemuServer::nbd_stop($state->{vmid});
+ return;
+ },
+ 'resume' => sub {
+ if (PVE::QemuServer::Helpers::vm_running_locally($state->{vmid})) {
+ PVE::QemuServer::vm_resume($state->{vmid}, 1, 1);
+ } else {
+ die "VM $state->{vmid} not running\n";
+ }
+ return;
+ },
+ 'unlock' => sub {
+ PVE::QemuConfig->remove_lock($state->{vmid}, $state->{lock});
+ delete $state->{lock};
+ return;
+ },
+ 'ticket' => sub {
+ my ($params) = @_;
+
+ my $path = $params->{path};
+
+ die "Not allowed to generate ticket for unknown socket '$path'\n"
+ if !defined($state->{sockets}->{$path});
+
+ return { ticket => PVE::AccessControl::assemble_tunnel_ticket($authuser, "/socket/$path") };
+ },
+ 'quit' => sub {
+ my ($params) = @_;
+
+ if ($params->{cleanup}) {
+ if ($state->{cleanup}->{fw}) {
+ PVE::Firewall::remove_vmfw_conf($state->{vmid});
+ }
+
+ for my $volid (keys $state->{cleanup}->{volumes}->%*) {
+ print "freeing volume '$volid' as part of cleanup\n";
+ eval { PVE::Storage::vdisk_free($state->{storecfg}, $volid) };
+ warn $@ if $@;
+ }
+
+ PVE::QemuServer::destroy_vm($state->{storecfg}, $state->{vmid}, 1);
+ }
+
+ print "switching to exit-mode, waiting for client to disconnect\n";
+ $state->{exit} = 1;
+ return;
+ },
+ };
+
+ $run_locked->(sub {
+ my $socket_addr = "/run/qemu-server/$state->{vmid}.mtunnel";
+ unlink $socket_addr;
+
+ $state->{socket} = IO::Socket::UNIX->new(
+ Type => SOCK_STREAM(),
+ Local => $socket_addr,
+ Listen => 1,
+ );
+
+ $state->{socket_uid} = getpwnam('www-data')
+ or die "Failed to resolve user 'www-data' to numeric UID\n";
+ chown $state->{socket_uid}, -1, $socket_addr;
+ });
+
+ print "mtunnel started\n";
+
+ my $conn = eval { PVE::Tools::run_with_timeout(300, sub { $state->{socket}->accept() }) };
+ if ($@) {
+ warn "Failed to accept tunnel connection - $@\n";
+
+ warn "Removing tunnel socket..\n";
+ unlink $state->{socket};
+
+ warn "Removing temporary VM config..\n";
+ $run_locked->(sub {
+ PVE::QemuServer::destroy_vm($state->{storecfg}, $state->{vmid}, 1);
+ });
+
+ die "Exiting mtunnel\n";
+ }
+
+ $state->{conn} = $conn;
+
+ my $reply_err = sub {
+ my ($msg) = @_;
+
+ my $reply = JSON::encode_json({
+ success => JSON::false,
+ msg => $msg,
+ });
+ $conn->print("$reply\n");
+ $conn->flush();
+ };
+
+ my $reply_ok = sub {
+ my ($res) = @_;
+
+ $res->{success} = JSON::true;
+ my $reply = JSON::encode_json($res);
+ $conn->print("$reply\n");
+ $conn->flush();
+ };
+
+ while (my $line = <$conn>) {
+ chomp $line;
+
+ # untaint, we validate below if needed
+ ($line) = $line =~ /^(.*)$/;
+ my $parsed = eval { JSON::decode_json($line) };
+ if ($@) {
+ $reply_err->("failed to parse command - $@");
+ next;
+ }
+
+ my $cmd = delete $parsed->{cmd};
+ if (!defined($cmd)) {
+ $reply_err->("'cmd' missing");
+ } elsif ($state->{exit}) {
+ $reply_err->("tunnel is in exit-mode, processing '$cmd' cmd not possible");
+ next;
+ } elsif (my $handler = $cmd_handlers->{$cmd}) {
+ print "received command '$cmd'\n";
+ eval {
+ if ($cmd_desc->{$cmd}) {
+ PVE::JSONSchema::validate($parsed, $cmd_desc->{$cmd});
+ } else {
+ $parsed = {};
+ }
+ my $res = $run_locked->($handler, $parsed);
+ $reply_ok->($res);
+ };
+ $reply_err->("failed to handle '$cmd' command - $@")
+ if $@;
+ } else {
+ $reply_err->("unknown command '$cmd' given");
+ }
+ }
+
+ if ($state->{exit}) {
+ print "mtunnel exited\n";
+ } else {
+ die "mtunnel exited unexpectedly\n";
+ }
+ };
+
+ my $socket_addr = "/run/qemu-server/$vmid.mtunnel";
+ my $ticket = PVE::AccessControl::assemble_tunnel_ticket($authuser, "/socket/$socket_addr");
+ my $upid = $rpcenv->fork_worker('qmtunnel', $vmid, $authuser, $realcmd);
+
+ return {
+ ticket => $ticket,
+ upid => $upid,
+ socket => $socket_addr,
+ };
+ }});
+
+__PACKAGE__->register_method({
+ name => 'mtunnelwebsocket',
+ path => '{vmid}/mtunnelwebsocket',
+ method => 'GET',
+ permissions => {
+ description => "You need to pass a ticket valid for the selected socket. Tickets can be created via the mtunnel API call, which will check permissions accordingly.",
+ user => 'all', # check inside
+ },
+ description => 'Migration tunnel endpoint for websocket upgrade - only for internal use by VM migration.',
+ parameters => {
+ additionalProperties => 0,
+ properties => {
+ node => get_standard_option('pve-node'),
+ vmid => get_standard_option('pve-vmid'),
+ socket => {
+ type => "string",
+ description => "unix socket to forward to",
+ },
+ ticket => {
+ type => "string",
+ description => "ticket return by initial 'mtunnel' API call, or retrieved via 'ticket' tunnel command",
+ },
+ },
+ },
+ returns => {
+ type => "object",
+ properties => {
+ port => { type => 'string', optional => 1 },
+ socket => { type => 'string', optional => 1 },
+ },
+ },
+ code => sub {
+ my ($param) = @_;
+
+ my $rpcenv = PVE::RPCEnvironment::get();
+ my $authuser = $rpcenv->get_user();
+
+ my $nodename = PVE::INotify::nodename();
+ my $node = extract_param($param, 'node');
+
+ raise_param_exc({ node => "node needs to be 'localhost' or local hostname '$nodename'" })
+ if $node ne 'localhost' && $node ne $nodename;
+
+ my $vmid = $param->{vmid};
+ # check VM exists
+ PVE::QemuConfig->load_config($vmid);
+
+ my $socket = $param->{socket};
+ PVE::AccessControl::verify_tunnel_ticket($param->{ticket}, $authuser, "/socket/$socket");
+
+ return { socket => $socket };
+ }});
+
1;
path => '',
proxyto => 'node',
method => 'GET',
- description => "Qemu Agent command index.",
+ description => "QEMU Guest Agent command index.",
permissions => {
user => 'all',
},
properties => {},
},
links => [ { rel => 'child', href => '{name}' } ],
- description => "Returns the list of Qemu Agent commands",
+ description => "Returns the list of QEMU Guest Agent commands",
},
code => sub {
my ($param) = @_;
properties => {
node => get_standard_option('pve-node'),
vmid => get_standard_option('pve-vmid', {
- completion => \&PVE::QemuServer::complete_vmid_running }),
+ completion => \&PVE::QemuServer::complete_vmid_running,
+ }),
command => {
type => 'string',
description => "The QGA command.",
},
};
- my $description = "Execute Qemu Guest Agent commands.";
+ my $description = "Execute QEMU Guest Agent commands.";
my $name = 'agent';
if ($command ne '') {
vmid => get_standard_option('pve-vmid', {
completion => \&PVE::QemuServer::complete_vmid_running }),
command => {
- type => 'string',
- format => 'string-alist',
- description => 'The command as a list of program + arguments',
- optional => 1,
+ type => 'array',
+ description => 'The command as a list of program + arguments.',
+ items => {
+ format => 'string',
+ description => 'A single part of the program + arguments.',
+ }
},
'input-data' => {
type => 'string',
my ($param) = @_;
my $vmid = $param->{vmid};
- my $cmd = undef;
- if ($param->{command}) {
- $cmd = [PVE::Tools::split_list($param->{command})];
- }
+ my $cmd = $param->{command};
my $res = PVE::QemuServer::Agent::qemu_exec($vmid, $param->{'input-data'}, $cmd);
return $res;
},
content => {
type => 'string',
- maxLength => 60*1024, # 60k, smaller than our 64k POST limit
+ maxLength => 60 * 1024, # 60k, smaller than our 64k POST limit
description => "The content to write into the file."
- }
+ },
+ encode => {
+ type => 'boolean',
+ description => "If set, the content will be encoded as base64 (required by QEMU)."
+ ."Otherwise the content needs to be encoded beforehand - defaults to true.",
+ optional => 1,
+ default => 1,
+ },
},
},
returns => { type => 'null' },
my ($param) = @_;
my $vmid = $param->{vmid};
- my $buf = encode_base64($param->{content});
+
+ my $buf = ($param->{encode} // 1) ? encode_base64($param->{content}) : $param->{content};
my $qgafh = agent_cmd($vmid, "file-open", { path => $param->{file}, mode => 'wb' }, "can't open file");
my $write = agent_cmd($vmid, "file-write", { handle => $qgafh, 'buf-b64' => $buf }, "can't write to file");
--- /dev/null
+package PVE::API2::Qemu::Machine;
+
+use strict;
+use warnings;
+
+use JSON;
+
+use PVE::JSONSchema qw(get_standard_option);
+use PVE::RESTHandler;
+use PVE::Tools qw(file_get_contents);
+
+use base qw(PVE::RESTHandler);
+
+__PACKAGE__->register_method({
+ name => 'types',
+ path => '',
+ method => 'GET',
+ proxyto => 'node',
+ description => "Get available QEMU/KVM machine types.",
+ permissions => {
+ user => 'all',
+ },
+ parameters => {
+ additionalProperties => 0,
+ properties => {
+ node => get_standard_option('pve-node'),
+ },
+ },
+ returns => {
+ type => 'array',
+ items => {
+ type => 'object',
+ additionalProperties => 1,
+ properties => {
+ id => {
+ type => 'string',
+ description => "Full name of machine type and version.",
+ },
+ type => {
+ type => 'string',
+ enum => ['q35', 'i440fx'],
+ description => "The machine type.",
+ },
+ version => {
+ type => 'string',
+ description => "The machine version.",
+ },
+ },
+ },
+ },
+ code => sub {
+ my $machines = eval {
+ my $raw = file_get_contents('/usr/share/kvm/machine-versions-x86_64.json');
+ return from_json($raw, { utf8 => 1 });
+ };
+ die "could not load supported machine versions - $@\n" if $@;
+ return $machines;
+ }
+});
+
+1;
-SOURCES=Agent.pm CPU.pm
+SOURCES=Agent.pm CPU.pm Machine.pm OVF.pm
.PHONY: install
install:
--- /dev/null
+package PVE::API2::Qemu::OVF;
+
+use strict;
+use warnings;
+
+use PVE::JSONSchema qw(get_standard_option);
+use PVE::QemuServer::OVF;
+use PVE::RESTHandler;
+
+use base qw(PVE::RESTHandler);
+
+__PACKAGE__->register_method ({
+ name => 'readovf',
+ path => '',
+ method => 'GET',
+ proxyto => 'node',
+ description => "Read an .ovf manifest.",
+ protected => 1,
+ parameters => {
+ additionalProperties => 0,
+ properties => {
+ node => get_standard_option('pve-node'),
+ manifest => {
+ description => "Path to .ovf manifest.",
+ type => 'string',
+ },
+ },
+ },
+ returns => {
+ type => 'object',
+ additionalProperties => 1,
+ properties => PVE::QemuServer::json_ovf_properties(),
+ description => "VM config according to .ovf manifest.",
+ },
+ code => sub {
+ my ($param) = @_;
+
+ my $manifest = $param->{manifest};
+ die "check for file $manifest failed - $!\n" if !-f $manifest;
+
+ my $parsed = PVE::QemuServer::OVF::parse_ovf($manifest);
+ my $result;
+ $result->{cores} = $parsed->{qm}->{cores};
+ $result->{name} = $parsed->{qm}->{name};
+ $result->{memory} = $parsed->{qm}->{memory};
+ my $disks = $parsed->{disks};
+ for my $disk (@$disks) {
+ $result->{$disk->{disk_address}} = $disk->{backing_file};
+ }
+ return $result;
+ }});
+
+1;
use Term::ReadLine;
use URI::Escape;
+use PVE::APIClient::LWP;
use PVE::Cluster;
use PVE::Exception qw(raise_param_exc);
use PVE::GuestHelpers;
use PVE::Network;
use PVE::RPCEnvironment;
use PVE::SafeSyslog;
-use PVE::Tools qw(extract_param);
+use PVE::Tools qw(extract_param file_get_contents);
use PVE::API2::Qemu::Agent;
use PVE::API2::Qemu;
my $upid_exit = sub {
my $upid = shift;
my $status = PVE::Tools::upid_read_status($upid);
- exit($status eq 'OK' ? 0 : -1);
+ exit(PVE::Tools::upid_status_is_error($status) ? -1 : 0);
};
my $nodename = PVE::INotify::nodename();
+my %node = (node => $nodename);
sub setup_environment {
PVE::RPCEnvironment->setup_default_cli_env();
if (defined($key)) {
print "$prefix$key:\n";
}
- foreach my $itemkey (keys %$hash) {
+ for my $itemkey (sort keys %$hash) {
print_recursive_hash("\t$prefix", $hash->{$itemkey}, $itemkey);
}
} elsif (ref($hash) eq 'ARRAY') {
if (defined($key)) {
print "$prefix$key:\n";
}
- foreach my $item (@$hash) {
+ for my $item (@$hash) {
print_recursive_hash("\t$prefix", $item);
}
- } elsif (!ref($hash) && defined($hash)) {
+ } elsif ((!ref($hash) && defined($hash)) || ref($hash) eq 'JSON::PP::Boolean') {
if (defined($key)) {
print "$prefix$key: $hash\n";
} else {
return;
}});
+
+__PACKAGE__->register_method({
+ name => 'remote_migrate_vm',
+ path => 'remote_migrate_vm',
+ method => 'POST',
+ description => "Migrate virtual machine to a remote cluster. Creates a new migration task. EXPERIMENTAL feature!",
+ permissions => {
+ check => ['perm', '/vms/{vmid}', [ 'VM.Migrate' ]],
+ },
+ parameters => {
+ additionalProperties => 0,
+ properties => {
+ node => get_standard_option('pve-node'),
+ vmid => get_standard_option('pve-vmid', { completion => \&PVE::QemuServer::complete_vmid }),
+ 'target-vmid' => get_standard_option('pve-vmid', { optional => 1 }),
+ 'target-endpoint' => get_standard_option('proxmox-remote', {
+ description => "Remote target endpoint",
+ }),
+ online => {
+ type => 'boolean',
+ description => "Use online/live migration if VM is running. Ignored if VM is stopped.",
+ optional => 1,
+ },
+ delete => {
+ type => 'boolean',
+ description => "Delete the original VM and related data after successful migration. By default the original VM is kept on the source cluster in a stopped state.",
+ optional => 1,
+ default => 0,
+ },
+ 'target-storage' => get_standard_option('pve-targetstorage', {
+ completion => \&PVE::QemuServer::complete_migration_storage,
+ optional => 0,
+ }),
+ 'target-bridge' => {
+ type => 'string',
+ description => "Mapping from source to target bridges. Providing only a single bridge ID maps all source bridges to that bridge. Providing the special value '1' will map each source bridge to itself.",
+ format => 'bridge-pair-list',
+ },
+ bwlimit => {
+ description => "Override I/O bandwidth limit (in KiB/s).",
+ optional => 1,
+ type => 'integer',
+ minimum => '0',
+ default => 'migrate limit from datacenter or storage config',
+ },
+ },
+ },
+ returns => {
+ type => 'string',
+ description => "the task ID.",
+ },
+ code => sub {
+ my ($param) = @_;
+
+ my $rpcenv = PVE::RPCEnvironment::get();
+ my $authuser = $rpcenv->get_user();
+
+ my $source_vmid = $param->{vmid};
+ my $target_endpoint = $param->{'target-endpoint'};
+ my $target_vmid = $param->{'target-vmid'} // $source_vmid;
+
+ my $remote = PVE::JSONSchema::parse_property_string('proxmox-remote', $target_endpoint);
+
+ # TODO: move this as helper somewhere appropriate?
+ my $conn_args = {
+ protocol => 'https',
+ host => $remote->{host},
+ port => $remote->{port} // 8006,
+ apitoken => $remote->{apitoken},
+ };
+
+ $conn_args->{cached_fingerprints} = { uc($remote->{fingerprint}) => 1 }
+ if defined($remote->{fingerprint});
+
+ my $api_client = PVE::APIClient::LWP->new(%$conn_args);
+ my $resources = $api_client->get("/cluster/resources", { type => 'vm' });
+ if (grep { defined($_->{vmid}) && $_->{vmid} eq $target_vmid } @$resources) {
+ raise_param_exc({ target_vmid => "Guest with ID '$target_vmid' already exists on remote cluster" });
+ }
+
+ my $storages = $api_client->get("/nodes/localhost/storage", { enabled => 1 });
+
+ my $storecfg = PVE::Storage::config();
+ my $target_storage = $param->{'target-storage'};
+ my $storagemap = eval { PVE::JSONSchema::parse_idmap($target_storage, 'pve-storage-id') };
+ raise_param_exc({ 'target-storage' => "failed to parse storage map: $@" })
+ if $@;
+
+ my $check_remote_storage = sub {
+ my ($storage) = @_;
+ my $found = [ grep { $_->{storage} eq $storage } @$storages ];
+ die "remote: storage '$storage' does not exist (or missing permission)!\n"
+ if !@$found;
+
+ $found = @$found[0];
+
+ my $content_types = [ PVE::Tools::split_list($found->{content}) ];
+ die "remote: storage '$storage' cannot store images\n"
+ if !grep { $_ eq 'images' } @$content_types;
+ };
+
+ foreach my $target_sid (values %{$storagemap->{entries}}) {
+ $check_remote_storage->($target_sid);
+ }
+
+ $check_remote_storage->($storagemap->{default})
+ if $storagemap->{default};
+
+ return PVE::API2::Qemu->remote_migrate_vm($param);
+ }});
+
__PACKAGE__->register_method ({
name => 'status',
path => 'status',
my $vnc_socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
if (my $ticket = $ENV{LC_PVE_TICKET}) { # NOTE: ssh on debian only pass LC_* variables
- mon_cmd($vmid, "change", device => 'vnc', target => "unix:$vnc_socket,password");
mon_cmd($vmid, "set_password", protocol => 'vnc', password => $ticket);
mon_cmd($vmid, "expire_password", protocol => 'vnc', time => "+30");
} else {
- # FIXME: remove or allow to add tls-creds object, as x509 vnc param is removed with qemu 4??
- mon_cmd($vmid, "change", device => 'vnc', target => "unix:$vnc_socket,password");
+ die "LC_PVE_TICKET not set, VNC proxy without password is forbidden\n";
}
run_vnc_proxy($vnc_socket);
last;
} elsif ($line =~ /^resume (\d+)$/) {
my $vmid = $1;
+ # check_running and vm_resume with nocheck, since local node
+ # might not have processed config move/rename yet
if (PVE::QemuServer::check_running($vmid, 1)) {
eval { PVE::QemuServer::vm_resume($vmid, 1, 1); };
if ($@) {
name => 'monitor',
path => 'monitor',
method => 'POST',
- description => "Enter Qemu Monitor interface.",
+ description => "Enter QEMU Monitor interface.",
parameters => {
additionalProperties => 0,
properties => {
my $conf = PVE::QemuConfig->load_config ($vmid); # check if VM exists
- print "Entering Qemu Monitor for VM $vmid - type 'help' for help\n";
+ print "Entering QEMU Monitor for VM $vmid - type 'help' for help\n";
my $term = Term::ReadLine->new('qm');
$conf->{memory} = $parsed->{qm}->{memory} if defined($parsed->{qm}->{memory});
$conf->{cores} = $parsed->{qm}->{cores} if defined($parsed->{qm}->{cores});
+ my $imported_disks = [];
eval {
# order matters, as do_import() will load_config() internally
$conf->{vmgenid} = PVE::QemuServer::generate_uuid();
foreach my $disk (@{ $parsed->{disks} }) {
my ($file, $drive) = ($disk->{backing_file}, $disk->{disk_address});
- PVE::QemuServer::ImportDisk::do_import($file, $vmid, $storeid, {
+ my ($name, $volid) = PVE::QemuServer::ImportDisk::do_import($file, $vmid, $storeid, {
drive_name => $drive,
format => $format,
skiplock => 1,
});
+ # for cleanup on (later) error
+ push @$imported_disks, $volid;
}
# reload after disks entries have been created
PVE::QemuConfig->write_config($vmid, $conf);
};
- my $err = $@;
- if ($err) {
+ if (my $err = $@) {
my $skiplock = 1;
- # eval for additional safety in error path
+ warn "error during import, cleaning up created resources...\n";
+ for my $volid (@$imported_disks) {
+ eval { PVE::Storage::vdisk_free($storecfg, $volid) };
+ warn "cleanup of $volid failed: $@\n" if $@;
+ }
eval { PVE::QemuServer::destroy_vm($storecfg, $vmid, $skiplock) };
warn "Could not destroy VM $vmid: $@" if "$@";
die "import failed - $err";
my $storecfg = PVE::Storage::config();
warn "Starting cleanup for $vmid\n";
- PVE::QemuConfig->lock_config($vmid, sub {
+ # mdev cleanup can take a while, so wait up to 60 seconds
+ PVE::QemuConfig->lock_config_full($vmid, 60, sub {
my $conf = PVE::QemuConfig->load_config ($vmid);
my $pid = PVE::QemuServer::check_running ($vmid);
die "vm still running\n" if $pid;
# we have to cleanup the tap devices after a crash
foreach my $opt (keys %$conf) {
- next if $opt !~ m/^net(\d)+$/;
+ next if $opt !~ m/^net(\d+)$/;
my $interface = $1;
PVE::Network::tap_unplug("tap${vmid}i${interface}");
}
warn "Restarting VM $vmid\n";
PVE::API2::Qemu->vm_start({
vmid => $vmid,
- node => $nodename,
+ %node,
});
}
return;
}});
+__PACKAGE__->register_method({
+ name => 'vm_import',
+ path => 'vm-import',
+ description => "Import a foreign virtual guest from a supported import source, such as an ESXi storage.",
+ parameters => {
+ additionalProperties => 0,
+ properties => PVE::QemuServer::json_config_properties({
+ vmid => get_standard_option('pve-vmid', { completion => \&PVE::Cluster::complete_next_vmid }),
+ 'source' => {
+ type => 'string',
+ description => 'The import source volume id.',
+ },
+ storage => get_standard_option('pve-storage-id', {
+ description => "Default storage.",
+ completion => \&PVE::QemuServer::complete_storage,
+ }),
+ 'live-import' => {
+ type => 'boolean',
+ optional => 1,
+ default => 0,
+ description => "Immediately start the VM and copy the data in the background.",
+ },
+ 'dryrun' => {
+ type => 'boolean',
+ optional => 1,
+ default => 0,
+ description => "Show the create command and exit without doing anything.",
+ },
+ delete => {
+ type => 'string', format => 'pve-configid-list',
+ description => "A list of settings you want to delete.",
+ optional => 1,
+ },
+ format => {
+ type => 'string',
+ description => 'Target format',
+ enum => [ 'raw', 'qcow2', 'vmdk' ],
+ optional => 1,
+ },
+ }),
+ },
+ returns => { type => 'null' },
+ code => sub {
+ my ($param) = @_;
+
+ my ($vmid, $source, $storage, $format, $live_import, $dryrun, $delete) =
+ delete $param->@{qw(vmid source storage format live-import dryrun delete)};
+
+ if (defined($format)) {
+ $format = ",format=$format";
+ } else {
+ $format = '';
+ }
+
+ my $storecfg = PVE::Storage::config();
+ my $metadata = PVE::Storage::get_import_metadata($storecfg, $source);
+
+ my $create_args = $metadata->{'create-args'};
+ if (my $netdevs = $metadata->{net}) {
+ for my $net (keys $netdevs->%*) {
+ my $value = $netdevs->{$net};
+ $create_args->{$net} = join(',', map { $_ . '=' . $value->{$_} } sort keys %$value);
+ }
+ }
+ if (my $disks = $metadata->{disks}) {
+ if (delete $disks->{efidisk0}) {
+ $create_args->{efidisk0} = "$storage:1$format,efitype=4m";
+ }
+ for my $disk (keys $disks->%*) {
+ my $value = $disks->{$disk}->{volid};
+ $create_args->{$disk} = "$storage:0${format},import-from=$value";
+ }
+ }
+
+ $create_args->{'live-restore'} = 1 if $live_import;
+
+ $create_args->{$_} = $param->{$_} for keys $param->%*;
+ delete $create_args->{$_} for PVE::Tools::split_list($delete);
+
+ if ($dryrun) {
+ print("# dry-run – the resulting create command for the import would be:\n");
+ print("qm create $vmid \\\n ");
+ print(join(" \\\n ", map { "--$_ $create_args->{$_}" } sort keys $create_args->%*));
+ print("\n");
+ return;
+ }
+
+ PVE::API2::Qemu->create_vm({
+ %node,
+ vmid => $vmid,
+ %$create_args,
+ });
+ return;
+ }
+});
+
my $print_agent_result = sub {
my ($data) = @_;
return;
}
- print to_json($result, { pretty => 1, canonical => 1});
+ print to_json($result, { pretty => 1, canonical => 1, utf8 => 1});
};
sub param_mapping {
my ($name) = @_;
my $ssh_key_map = ['sshkeys', sub {
- return URI::Escape::uri_escape(PVE::Tools::file_get_contents($_[0]));
+ return URI::Escape::uri_escape(file_get_contents($_[0]));
}];
my $cipassword_map = PVE::CLIHandler::get_standard_mapping('pve-password', { name => 'cipassword' });
my $password_map = PVE::CLIHandler::get_standard_mapping('pve-password');
}
our $cmddef = {
- list => [ "PVE::API2::Qemu", 'vmlist', [],
- { node => $nodename }, sub {
- my $vmlist = shift;
-
- exit 0 if (!scalar(@$vmlist));
-
- printf "%10s %-20s %-10s %-10s %12s %-10s\n",
- qw(VMID NAME STATUS MEM(MB) BOOTDISK(GB) PID);
-
- foreach my $rec (sort { $a->{vmid} <=> $b->{vmid} } @$vmlist) {
- printf "%10s %-20s %-10s %-10s %12.2f %-10s\n", $rec->{vmid}, $rec->{name},
- $rec->{qmpstatus} || $rec->{status},
- ($rec->{maxmem} || 0)/(1024*1024),
- ($rec->{maxdisk} || 0)/(1024*1024*1024),
- $rec->{pid}||0;
- }
-
-
- } ],
-
- create => [ "PVE::API2::Qemu", 'create_vm', ['vmid'], { node => $nodename }, $upid_exit ],
-
- destroy => [ "PVE::API2::Qemu", 'destroy_vm', ['vmid'], { node => $nodename }, $upid_exit ],
-
- clone => [ "PVE::API2::Qemu", 'clone_vm', ['vmid', 'newid'], { node => $nodename }, $upid_exit ],
-
- migrate => [ "PVE::API2::Qemu", 'migrate_vm', ['vmid', 'target'], { node => $nodename }, $upid_exit ],
-
- set => [ "PVE::API2::Qemu", 'update_vm', ['vmid'], { node => $nodename } ],
+ list=> [ "PVE::API2::Qemu", 'vmlist', [], { %node }, sub {
+ my $vmlist = shift;
+ exit 0 if (!scalar(@$vmlist));
+
+ printf "%10s %-20s %-10s %-10s %12s %-10s\n",
+ qw(VMID NAME STATUS MEM(MB) BOOTDISK(GB) PID);
+
+ foreach my $rec (sort { $a->{vmid} <=> $b->{vmid} } @$vmlist) {
+ printf "%10s %-20s %-10s %-10s %12.2f %-10s\n", $rec->{vmid}, $rec->{name},
+ $rec->{qmpstatus} || $rec->{status},
+ ($rec->{maxmem} || 0)/(1024*1024),
+ ($rec->{maxdisk} || 0)/(1024*1024*1024),
+ $rec->{pid} || 0;
+ }
+ }],
- resize => [ "PVE::API2::Qemu", 'resize_vm', ['vmid', 'disk', 'size'], { node => $nodename } ],
+ create => [ "PVE::API2::Qemu", 'create_vm', ['vmid'], { %node }, $upid_exit ],
+ destroy => [ "PVE::API2::Qemu", 'destroy_vm', ['vmid'], { %node }, $upid_exit ],
+ clone => [ "PVE::API2::Qemu", 'clone_vm', ['vmid', 'newid'], { %node }, $upid_exit ],
- move_disk => [ "PVE::API2::Qemu", 'move_vm_disk', ['vmid', 'disk', 'storage'], { node => $nodename }, $upid_exit ],
+ migrate => [ "PVE::API2::Qemu", 'migrate_vm', ['vmid', 'target'], { %node }, $upid_exit ],
+ 'remote-migrate' => [ __PACKAGE__, 'remote_migrate_vm', ['vmid', 'target-vmid', 'target-endpoint'], { %node }, $upid_exit ],
- unlink => [ "PVE::API2::Qemu", 'unlink', ['vmid'], { node => $nodename } ],
+ set => [ "PVE::API2::Qemu", 'update_vm', ['vmid'], { %node } ],
- config => [ "PVE::API2::Qemu", 'vm_config', ['vmid'],
- { node => $nodename }, sub {
- my $config = shift;
- foreach my $k (sort (keys %$config)) {
- next if $k eq 'digest';
- my $v = $config->{$k};
- if ($k eq 'description') {
- $v = PVE::Tools::encode_text($v);
- }
- print "$k: $v\n";
- }
- }],
+ config => [ "PVE::API2::Qemu", 'vm_config', ['vmid'], { %node }, sub {
+ my $config = shift;
+ foreach my $k (sort (keys %$config)) {
+ next if $k eq 'digest';
+ my $v = $config->{$k};
+ if ($k eq 'description') {
+ $v = PVE::Tools::encode_text($v);
+ }
+ print "$k: $v\n";
+ }
+ }],
- pending => [ "PVE::API2::Qemu", 'vm_pending', ['vmid'], { node => $nodename }, \&PVE::GuestHelpers::format_pending ],
+ pending => [ "PVE::API2::Qemu", 'vm_pending', ['vmid'], { %node }, \&PVE::GuestHelpers::format_pending ],
showcmd => [ __PACKAGE__, 'showcmd', ['vmid']],
status => [ __PACKAGE__, 'status', ['vmid']],
- snapshot => [ "PVE::API2::Qemu", 'snapshot', ['vmid', 'snapname'], { node => $nodename } , $upid_exit ],
-
- delsnapshot => [ "PVE::API2::Qemu", 'delsnapshot', ['vmid', 'snapname'], { node => $nodename } , $upid_exit ],
-
- listsnapshot => [ "PVE::API2::Qemu", 'snapshot_list', ['vmid'], { node => $nodename }, \&PVE::GuestHelpers::print_snapshot_tree],
-
- rollback => [ "PVE::API2::Qemu", 'rollback', ['vmid', 'snapname'], { node => $nodename } , $upid_exit ],
-
- template => [ "PVE::API2::Qemu", 'template', ['vmid'], { node => $nodename }],
+ # FIXME: for 8.0 move to command group snapshot { create, list, destroy, rollback }
+ snapshot => [ "PVE::API2::Qemu", 'snapshot', ['vmid', 'snapname'], { %node } , $upid_exit ],
+ delsnapshot => [ "PVE::API2::Qemu", 'delsnapshot', ['vmid', 'snapname'], { %node } , $upid_exit ],
+ listsnapshot => [ "PVE::API2::Qemu", 'snapshot_list', ['vmid'], { %node }, \&PVE::GuestHelpers::print_snapshot_tree],
+ rollback => [ "PVE::API2::Qemu", 'rollback', ['vmid', 'snapname'], { %node } , $upid_exit ],
- start => [ "PVE::API2::Qemu", 'vm_start', ['vmid'], { node => $nodename } , $upid_exit ],
+ template => [ "PVE::API2::Qemu", 'template', ['vmid'], { %node }],
- stop => [ "PVE::API2::Qemu", 'vm_stop', ['vmid'], { node => $nodename }, $upid_exit ],
+ # FIXME: should be in a power command group?
+ start => [ "PVE::API2::Qemu", 'vm_start', ['vmid'], { %node } , $upid_exit ],
+ stop => [ "PVE::API2::Qemu", 'vm_stop', ['vmid'], { %node }, $upid_exit ],
+ reset => [ "PVE::API2::Qemu", 'vm_reset', ['vmid'], { %node }, $upid_exit ],
+ shutdown => [ "PVE::API2::Qemu", 'vm_shutdown', ['vmid'], { %node }, $upid_exit ],
+ reboot => [ "PVE::API2::Qemu", 'vm_reboot', ['vmid'], { %node }, $upid_exit ],
+ suspend => [ "PVE::API2::Qemu", 'vm_suspend', ['vmid'], { %node }, $upid_exit ],
+ resume => [ "PVE::API2::Qemu", 'vm_resume', ['vmid'], { %node }, $upid_exit ],
- reset => [ "PVE::API2::Qemu", 'vm_reset', ['vmid'], { node => $nodename }, $upid_exit ],
-
- shutdown => [ "PVE::API2::Qemu", 'vm_shutdown', ['vmid'], { node => $nodename }, $upid_exit ],
-
- reboot => [ "PVE::API2::Qemu", 'vm_reboot', ['vmid'], { node => $nodename }, $upid_exit ],
-
- suspend => [ "PVE::API2::Qemu", 'vm_suspend', ['vmid'], { node => $nodename }, $upid_exit ],
-
- resume => [ "PVE::API2::Qemu", 'vm_resume', ['vmid'], { node => $nodename }, $upid_exit ],
-
- sendkey => [ "PVE::API2::Qemu", 'vm_sendkey', ['vmid', 'key'], { node => $nodename } ],
+ sendkey => [ "PVE::API2::Qemu", 'vm_sendkey', ['vmid', 'key'], { %node } ],
vncproxy => [ __PACKAGE__, 'vncproxy', ['vmid']],
unlock => [ __PACKAGE__, 'unlock', ['vmid']],
- rescan => [ __PACKAGE__, 'rescan', []],
+ # TODO: evluate dropping below aliases for 8.0, if no usage is left
+ importdisk => { alias => 'disk import' },
+ 'move-disk' => { alias => 'disk move' },
+ move_disk => { alias => 'disk move' },
+ rescan => { alias => 'disk rescan' },
+ resize => { alias => 'disk resize' },
+ unlink => { alias => 'disk unlink' },
+
+ disk => {
+ import => [ __PACKAGE__, 'importdisk', ['vmid', 'source', 'storage']],
+ 'move' => [ "PVE::API2::Qemu", 'move_vm_disk', ['vmid', 'disk', 'storage'], { %node }, $upid_exit ],
+ rescan => [ __PACKAGE__, 'rescan', []],
+ resize => [ "PVE::API2::Qemu", 'resize_vm', ['vmid', 'disk', 'size'], { %node } ],
+ unlink => [ "PVE::API2::Qemu", 'unlink', ['vmid'], { %node } ],
+ },
monitor => [ __PACKAGE__, 'monitor', ['vmid']],
- agent => { alias => 'guest cmd' },
+ agent => { alias => 'guest cmd' }, # FIXME: remove with PVE 8.0
guest => {
- cmd => [ "PVE::API2::Qemu::Agent", 'agent', ['vmid', 'command'], { node => $nodename }, $print_agent_result ],
- passwd => [ "PVE::API2::Qemu::Agent", 'set-user-password', [ 'vmid', 'username' ], { node => $nodename }],
- exec => [ __PACKAGE__, 'exec', [ 'vmid', 'extra-args' ], { node => $nodename }, $print_agent_result],
- 'exec-status' => [ "PVE::API2::Qemu::Agent", 'exec-status', [ 'vmid', 'pid' ], { node => $nodename }, $print_agent_result],
+ cmd => [ "PVE::API2::Qemu::Agent", 'agent', ['vmid', 'command'], { %node }, $print_agent_result ],
+ passwd => [ "PVE::API2::Qemu::Agent", 'set-user-password', [ 'vmid', 'username' ], { %node }],
+ exec => [ __PACKAGE__, 'exec', [ 'vmid', 'extra-args' ], { %node }, $print_agent_result],
+ 'exec-status' => [ "PVE::API2::Qemu::Agent", 'exec-status', [ 'vmid', 'pid' ], { %node }, $print_agent_result],
},
mtunnel => [ __PACKAGE__, 'mtunnel', []],
terminal => [ __PACKAGE__, 'terminal', ['vmid']],
- importdisk => [ __PACKAGE__, 'importdisk', ['vmid', 'source', 'storage']],
-
importovf => [ __PACKAGE__, 'importovf', ['vmid', 'manifest', 'storage']],
- cleanup => [ __PACKAGE__, 'cleanup', ['vmid', 'clean-shutdown', 'guest-requested'], { node => $nodename }],
+ cleanup => [ __PACKAGE__, 'cleanup', ['vmid', 'clean-shutdown', 'guest-requested'], { %node }],
cloudinit => {
- dump => [ "PVE::API2::Qemu", 'cloudinit_generated_config_dump', ['vmid', 'type'], { node => $nodename }, sub {
- my $data = shift;
- print "$data\n";
- }],
+ dump => [ "PVE::API2::Qemu", 'cloudinit_generated_config_dump', ['vmid', 'type'], { %node }, sub { print "$_[0]\n"; }],
+ pending => [ "PVE::API2::Qemu", 'cloudinit_pending', ['vmid'], { %node }, \&PVE::GuestHelpers::format_pending ],
+ update => [ "PVE::API2::Qemu", 'cloudinit_update', ['vmid'], { node => $nodename }],
},
+ import => [ __PACKAGE__, 'vm_import', ['vmid', 'source']],
};
1;
description => "Add the VM to the specified pool.",
},
bwlimit => {
- description => "Override i/o bandwidth limit (in KiB/s).",
+ description => "Override I/O bandwidth limit (in KiB/s).",
optional => 1,
type => 'number',
minimum => '0',
- }
+ },
+ 'live-restore' => {
+ optional => 1,
+ type => 'boolean',
+ description => "Start the VM immediately from the backup and restore in background. PBS only.",
+ },
},
},
returns => {
sub {
my $upid = shift;
my $status = PVE::Tools::upid_read_status($upid);
- exit($status eq 'OK' ? 0 : -1);
+ exit(PVE::Tools::upid_status_is_error($status) ? -1 : 0);
}];
1;
use warnings;
use IO::Multiplex;
+use IO::Socket::UNIX;
use JSON;
use POSIX qw(EINTR EAGAIN);
use Scalar::Util qw(weaken);
use PVE::IPCC;
use PVE::QemuServer::Helpers;
-# Qemu Monitor Protocol (QMP) client.
+# QEMU Monitor Protocol (QMP) client.
#
# This implementation uses IO::Multiplex (libio-multiplex-perl) and
# allows you to issue qmp and qga commands to different VMs in parallel.
# locked state with high probability, so use an generous timeout
$timeout = 60*60; # 1 hour
} elsif ($cmd->{execute} eq 'guest-fsfreeze-thaw') {
- # thaw has no possible long blocking actions, either it returns
- # instantly or never (dead locked)
- $timeout = 10;
- } elsif ($cmd->{execute} eq 'savevm-start' ||
- $cmd->{execute} eq 'savevm-end' ||
- $cmd->{execute} eq 'query-backup' ||
- $cmd->{execute} eq 'query-block-jobs' ||
- $cmd->{execute} eq 'block-job-cancel' ||
- $cmd->{execute} eq 'block-job-complete' ||
- $cmd->{execute} eq 'backup-cancel' ||
- $cmd->{execute} eq 'query-savevm' ||
- $cmd->{execute} eq 'delete-drive-snapshot' ||
- $cmd->{execute} eq 'guest-shutdown' ||
- $cmd->{execute} eq 'blockdev-snapshot-internal-sync' ||
- $cmd->{execute} eq 'blockdev-snapshot-delete-internal-sync' ||
- $cmd->{execute} eq 'snapshot-drive' ) {
- $timeout = 10*60; # 10 mins ?
+ # While it should return instantly or never (dead locked) for Linux guests,
+ # the variance for Windows guests can be big. And there might be hook scripts
+ # that are executed upon thaw, so use 3 minutes to be on the safe side.
+ $timeout = 3 * 60;
+ } elsif (
+ $cmd->{execute} eq 'backup-cancel' ||
+ $cmd->{execute} eq 'blockdev-snapshot-delete-internal-sync' ||
+ $cmd->{execute} eq 'blockdev-snapshot-internal-sync' ||
+ $cmd->{execute} eq 'block-job-cancel' ||
+ $cmd->{execute} eq 'block-job-complete' ||
+ $cmd->{execute} eq 'drive-mirror' ||
+ $cmd->{execute} eq 'guest-fstrim' ||
+ $cmd->{execute} eq 'guest-shutdown' ||
+ $cmd->{execute} eq 'query-backup' ||
+ $cmd->{execute} eq 'query-block-jobs' ||
+ $cmd->{execute} eq 'query-savevm' ||
+ $cmd->{execute} eq 'savevm-end' ||
+ $cmd->{execute} eq 'savevm-start'
+ ) {
+ $timeout = 10*60; # 10 mins
} else {
- $timeout = 3; # default
+ # NOTE: if you came here as user and want to change this, try using IO-Threads first
+ # which move out quite some processing of the main thread, leaving more time for QMP
+ $timeout = 5; # default
}
}
use PVE::QemuServer::Monitor qw(mon_cmd);
use PVE::QemuServer;
use PVE::QemuServer::Machine;
+use PVE::QemuServer::Memory qw(get_current_memory);
use PVE::Storage;
use PVE::Tools;
+use PVE::Format qw(render_bytes render_duration);
use base qw(PVE::AbstractConfig);
$target = PVE::QemuServer::find_vmstate_storage($conf, $storecfg);
}
- my $defaults = PVE::QemuServer::load_defaults();
- my $mem_size = $conf->{memory} // $defaults->{memory};
+ my $mem_size = get_current_memory($conf->{memory});
my $driver_state_size = 500; # assume 500MB is enough to safe all driver state;
# our savevm-start does live-save of the memory until the space left in the
# volume is just enough for the remaining memory content + internal state
return $statefile;
}
+sub __snapshot_activate_storages {
+ my ($class, $conf, $include_vmstate) = @_;
+
+ my $storecfg = PVE::Storage::config();
+ my $opts = $include_vmstate ? { 'extra_keys' => ['vmstate'] } : {};
+ my $storage_hash = {};
+
+ $class->foreach_volume_full($conf, $opts, sub {
+ my ($key, $drive) = @_;
+
+ return if PVE::QemuServer::drive_is_cdrom($drive);
+
+ my ($storeid) = PVE::Storage::parse_volume_id($drive->{file});
+ $storage_hash->{$storeid} = 1;
+ });
+
+ PVE::Storage::activate_storage_list($storecfg, [ sort keys $storage_hash->%* ]);
+}
+
sub __snapshot_check_running {
my ($class, $vmid) = @_;
return PVE::QemuServer::Helpers::vm_running_locally($vmid);
if ($snap->{vmstate}) {
my $path = PVE::Storage::path($storecfg, $snap->{vmstate});
PVE::Storage::activate_volumes($storecfg, [$snap->{vmstate}]);
+ my $state_storage_id = PVE::Storage::parse_volume_id($snap->{vmstate});
+ PVE::QemuServer::set_migration_caps($vmid, 1);
mon_cmd($vmid, "savevm-start", statefile => $path);
+ print "saving VM state and RAM using storage '$state_storage_id'\n";
+ my $render_state = sub {
+ my ($stat) = @_;
+ my $b = render_bytes($stat->{bytes});
+ my $t = render_duration($stat->{'total-time'} / 1000);
+ return ($b, $t);
+ };
+ my $round = 0;
for(;;) {
+ $round++;
my $stat = mon_cmd($vmid, "query-savevm");
if (!$stat->{status}) {
die "savevm not active\n";
} elsif ($stat->{status} eq 'active') {
+ if ($round < 60 || $round % 10 == 0) {
+ my ($b, $t) = $render_state->($stat);
+ print "$b in $t\n";
+ }
+ print "reducing reporting rate to every 10s\n" if $round == 60;
sleep(1);
next;
} elsif ($stat->{status} eq 'completed') {
+ my ($b, $t) = $render_state->($stat);
+ print "completed saving the VM state in $t, saved $b\n";
last;
+ } elsif ($stat->{status} eq 'failed') {
+ my $err = $stat->{error} || 'unknown error';
+ die "unable to save VM state and RAM - $err\n";
} else {
- die "query-savevm returned status '$stat->{status}'\n";
+ die "query-savevm returned unexpected status '$stat->{status}'\n";
}
}
} else {
my $device = "drive-$ds";
my $storecfg = PVE::Storage::config();
+ print "snapshotting '$device' ($drive->{file})\n";
+
PVE::QemuServer::qemu_volume_snapshot($vmid, $device, $storecfg, $volid, $snapname);
}
return if PVE::QemuServer::drive_is_cdrom($drive);
my $storecfg = PVE::Storage::config();
my $volid = $drive->{file};
- my $device = "drive-$ds";
- PVE::QemuServer::qemu_volume_snapshot_delete($vmid, $device, $storecfg, $volid, $snapname);
+ PVE::QemuServer::qemu_volume_snapshot_delete($vmid, $storecfg, $volid, $snapname);
push @$unused, $volid;
}
} else {
# Note: old code did not store 'machine', so we try to be smart
# and guess the snapshot was generated with kvm 1.4 (pc-i440fx-1.4).
- $data->{forcemachine} = $conf->{machine} || 'pc-i440fx-1.4';
+ my $machine_conf = PVE::QemuServer::Machine::parse_machine($conf->{machine});
+ $data->{forcemachine} = $machine_conf->{type} || 'pc-i440fx-1.4';
# we remove the 'machine' configuration if not explicitly specified
# in the original config.
}
sub __snapshot_rollback_vol_possible {
- my ($class, $drive, $snapname) = @_;
+ my ($class, $drive, $snapname, $blockers) = @_;
return if PVE::QemuServer::drive_is_cdrom($drive);
my $storecfg = PVE::Storage::config();
my $volid = $drive->{file};
- PVE::Storage::volume_rollback_is_possible($storecfg, $volid, $snapname);
+ PVE::Storage::volume_rollback_is_possible($storecfg, $volid, $snapname, $blockers);
}
sub __snapshot_rollback_vol_rollback {
$class->foreach_volume($conf, sub {
my ($vs, $volume) = @_;
- return if PVE::QemuServer::drive_is_cdrom($volume);
+ return if PVE::QemuServer::drive_is_cdrom($volume, 1);
my $found = 0;
my $volid = $volume->{file};
my ($ds, $drive) = @_;
return if $found;
- return if PVE::QemuServer::drive_is_cdrom($drive);
+ return if PVE::QemuServer::drive_is_cdrom($drive, 1);
$found = 1
if ($drive->{file} && $drive->{file} eq $volid);
return $unused;
}
+sub add_unused_volume {
+ my ($class, $config, $volid) = @_;
+
+ if ($volid =~ m/vm-\d+-cloudinit/) {
+ print "found unused cloudinit disk '$volid', removing it\n";
+ my $storecfg = PVE::Storage::config();
+ PVE::Storage::vdisk_free($storecfg, $volid);
+ return undef;
+ } else {
+ return $class->SUPER::add_unused_volume($config, $volid);
+ }
+}
+
+sub load_current_config {
+ my ($class, $vmid, $current) = @_;
+
+ my $conf = $class->SUPER::load_current_config($vmid, $current);
+ delete $conf->{cloudinit};
+ return $conf;
+}
+
+sub get_derived_property {
+ my ($class, $conf, $name) = @_;
+
+ my $defaults = PVE::QemuServer::load_defaults();
+
+ if ($name eq 'max-cpu') {
+ my $cpus =
+ ($conf->{sockets} || $defaults->{sockets}) * ($conf->{cores} || $defaults->{cores});
+ return $conf->{vcpus} || $cpus;
+ } elsif ($name eq 'max-memory') { # current usage maximum, not maximum hotpluggable
+ return get_current_memory($conf->{memory}) * 1024 * 1024;
+ } else {
+ die "unknown derived property - $name\n";
+ }
+}
+
# END implemented abstract methods from PVE::AbstractConfig
+sub has_cloudinit {
+ my ($class, $conf, $skip) = @_;
+
+ my $found;
+
+ $class->foreach_volume($conf, sub {
+ my ($key, $volume) = @_;
+
+ return if ($skip && $skip eq $key) || $found;
+ $found = $key if PVE::QemuServer::Drive::drive_is_cloudinit($volume);
+ });
+
+ return $found;
+}
+
1;
use IO::File;
use IPC::Open2;
-use POSIX qw( WNOHANG );
use Time::HiRes qw( usleep );
use PVE::Cluster;
+use PVE::Format qw(render_bytes);
+use PVE::GuestHelpers qw(safe_boolean_ne safe_string_ne);
use PVE::INotify;
use PVE::RPCEnvironment;
use PVE::Replication;
use PVE::ReplicationConfig;
use PVE::ReplicationState;
use PVE::Storage;
+use PVE::StorageTunnel;
use PVE::Tools;
+use PVE::Tunnel;
use PVE::QemuConfig;
use PVE::QemuServer::CPUConfig;
use PVE::QemuServer::Helpers qw(min_version);
use PVE::QemuServer::Machine;
use PVE::QemuServer::Monitor qw(mon_cmd);
+use PVE::QemuServer::Memory qw(get_current_memory);
use PVE::QemuServer;
use PVE::AbstractMigrate;
use base qw(PVE::AbstractMigrate);
-sub fork_command_pipe {
- my ($self, $cmd) = @_;
+# compared against remote end's minimum version
+our $WS_TUNNEL_VERSION = 2;
- my $reader = IO::File->new();
- my $writer = IO::File->new();
-
- my $orig_pid = $$;
-
- my $cpid;
-
- eval { $cpid = open2($reader, $writer, @$cmd); };
-
- my $err = $@;
-
- # catch exec errors
- if ($orig_pid != $$) {
- $self->log('err', "can't fork command pipe\n");
- POSIX::_exit(1);
- kill('KILL', $$);
- }
+sub fork_tunnel {
+ my ($self, $ssh_forward_info) = @_;
- die $err if $err;
+ my $cmd = ['/usr/sbin/qm', 'mtunnel'];
+ my $log = sub {
+ my ($level, $msg) = @_;
+ $self->log($level, $msg);
+ };
- return { writer => $writer, reader => $reader, pid => $cpid };
+ return PVE::Tunnel::fork_ssh_tunnel($self->{rem_ssh}, $cmd, $ssh_forward_info, $log);
}
-sub finish_command_pipe {
- my ($self, $cmdpipe, $timeout) = @_;
-
- my $cpid = $cmdpipe->{pid};
- return if !defined($cpid);
+sub fork_websocket_tunnel {
+ my ($self, $storages, $bridges) = @_;
- my $writer = $cmdpipe->{writer};
- my $reader = $cmdpipe->{reader};
+ my $remote = $self->{opts}->{remote};
+ my $conn = $remote->{conn};
- $writer->close();
- $reader->close();
-
- my $collect_child_process = sub {
- my $res = waitpid($cpid, WNOHANG);
- if (defined($res) && ($res == $cpid)) {
- delete $cmdpipe->{cpid};
- return 1;
- } else {
- return 0;
- }
- };
-
- if ($timeout) {
- for (my $i = 0; $i < $timeout; $i++) {
- return if &$collect_child_process();
- sleep(1);
- }
- }
-
- $self->log('info', "ssh tunnel still running - terminating now with SIGTERM\n");
- kill(15, $cpid);
-
- # wait again
- for (my $i = 0; $i < 10; $i++) {
- return if &$collect_child_process();
- sleep(1);
- }
-
- $self->log('info', "ssh tunnel still running - terminating now with SIGKILL\n");
- kill 9, $cpid;
- sleep 1;
-
- $self->log('err', "ssh tunnel child process (PID $cpid) couldn't be collected\n")
- if !&$collect_child_process();
-}
+ my $log = sub {
+ my ($level, $msg) = @_;
+ $self->log($level, $msg);
+ };
-sub read_tunnel {
- my ($self, $tunnel, $timeout) = @_;
+ my $websocket_url = "https://$conn->{host}:$conn->{port}/api2/json/nodes/$self->{node}/qemu/$remote->{vmid}/mtunnelwebsocket";
+ my $url = "/nodes/$self->{node}/qemu/$remote->{vmid}/mtunnel";
- $timeout = 60 if !defined($timeout);
+ my $tunnel_params = {
+ url => $websocket_url,
+ };
- my $reader = $tunnel->{reader};
+ my $storage_list = join(',', keys %$storages);
+ my $bridge_list = join(',', keys %$bridges);
- my $output;
- eval {
- PVE::Tools::run_with_timeout($timeout, sub { $output = <$reader>; });
+ my $req_params = {
+ storages => $storage_list,
+ bridges => $bridge_list,
};
- die "reading from tunnel failed: $@\n" if $@;
- chomp $output;
-
- return $output;
+ return PVE::Tunnel::fork_websocket_tunnel($conn, $url, $req_params, $tunnel_params, $log);
}
-sub write_tunnel {
- my ($self, $tunnel, $timeout, $command) = @_;
+# tunnel_info:
+# proto: unix (secure) or tcp (insecure/legacy compat)
+# addr: IP or UNIX socket path
+# port: optional TCP port
+# unix_sockets: additional UNIX socket paths to forward
+sub start_remote_tunnel {
+ my ($self, $tunnel_info) = @_;
- $timeout = 60 if !defined($timeout);
+ my $nodename = PVE::INotify::nodename();
+ my $migration_type = $self->{opts}->{migration_type};
- my $writer = $tunnel->{writer};
+ if ($migration_type eq 'secure') {
- eval {
- PVE::Tools::run_with_timeout($timeout, sub {
- print $writer "$command\n";
- $writer->flush();
- });
- };
- die "writing to tunnel failed: $@\n" if $@;
+ if ($tunnel_info->{proto} eq 'unix') {
+ my $ssh_forward_info = [];
- if ($tunnel->{version} && $tunnel->{version} >= 1) {
- my $res = eval { $self->read_tunnel($tunnel, 10); };
- die "no reply to command '$command': $@\n" if $@;
+ my $unix_sockets = [ keys %{$tunnel_info->{unix_sockets}} ];
+ push @$unix_sockets, $tunnel_info->{addr};
+ for my $sock (@$unix_sockets) {
+ push @$ssh_forward_info, "$sock:$sock";
+ unlink $sock;
+ }
- if ($res eq 'OK') {
- return;
- } else {
- die "tunnel replied '$res' to command '$command'\n";
- }
- }
-}
+ $self->{tunnel} = $self->fork_tunnel($ssh_forward_info);
-sub fork_tunnel {
- my ($self, $ssh_forward_info) = @_;
+ my $unix_socket_try = 0; # wait for the socket to become ready
+ while ($unix_socket_try <= 100) {
+ $unix_socket_try++;
+ my $available = 0;
+ foreach my $sock (@$unix_sockets) {
+ if (-S $sock) {
+ $available++;
+ }
+ }
- my @localtunnelinfo = ();
- foreach my $addr (@$ssh_forward_info) {
- push @localtunnelinfo, '-L', $addr;
- }
+ if ($available == @$unix_sockets) {
+ last;
+ }
- my $cmd = [@{$self->{rem_ssh}}, '-o ExitOnForwardFailure=yes', @localtunnelinfo, '/usr/sbin/qm', 'mtunnel' ];
+ usleep(50000);
+ }
+ if ($unix_socket_try > 100) {
+ $self->{errors} = 1;
+ PVE::Tunnel::finish_tunnel($self->{tunnel});
+ die "Timeout, migration socket $tunnel_info->{addr} did not get ready";
+ }
+ $self->{tunnel}->{unix_sockets} = $unix_sockets if (@$unix_sockets);
- my $tunnel = $self->fork_command_pipe($cmd);
+ } elsif ($tunnel_info->{proto} eq 'tcp') {
+ my $ssh_forward_info = [];
+ if ($tunnel_info->{addr} eq "localhost") {
+ # for backwards compatibility with older qemu-server versions
+ my $pfamily = PVE::Tools::get_host_address_family($nodename);
+ my $lport = PVE::Tools::next_migrate_port($pfamily);
+ push @$ssh_forward_info, "$lport:localhost:$tunnel_info->{port}";
+ }
- eval {
- my $helo = $self->read_tunnel($tunnel, 60);
- die "no reply\n" if !$helo;
- die "no quorum on target node\n" if $helo =~ m/^no quorum$/;
- die "got strange reply from mtunnel ('$helo')\n"
- if $helo !~ m/^tunnel online$/;
- };
- my $err = $@;
+ $self->{tunnel} = $self->fork_tunnel($ssh_forward_info);
- eval {
- my $ver = $self->read_tunnel($tunnel, 10);
- if ($ver =~ /^ver (\d+)$/) {
- $tunnel->{version} = $1;
- $self->log('info', "ssh tunnel $ver\n");
} else {
- $err = "received invalid tunnel version string '$ver'\n" if !$err;
+ die "unsupported protocol in migration URI: $tunnel_info->{proto}\n";
}
- };
-
- if ($err) {
- $self->finish_command_pipe($tunnel);
- die "can't open migration tunnel - $err";
- }
- return $tunnel;
-}
-
-sub finish_tunnel {
- my ($self, $tunnel) = @_;
-
- eval { $self->write_tunnel($tunnel, 30, 'quit'); };
- my $err = $@;
-
- $self->finish_command_pipe($tunnel, 30);
-
- if (my $unix_sockets = $tunnel->{unix_sockets}) {
- # ssh does not clean up on local host
- my $cmd = ['rm', '-f', @$unix_sockets];
- PVE::Tools::run_command($cmd);
-
- # .. and just to be sure check on remote side
- unshift @{$cmd}, @{$self->{rem_ssh}};
- PVE::Tools::run_command($cmd);
+ } else {
+ #fork tunnel for insecure migration, to send faster commands like resume
+ $self->{tunnel} = $self->fork_tunnel();
}
-
- die $err if $err;
}
sub lock_vm {
return PVE::QemuConfig->lock_config($vmid, $code, @param);
}
+sub target_storage_check_available {
+ my ($self, $storecfg, $targetsid, $volid) = @_;
+
+ if (!$self->{opts}->{remote}) {
+ # check if storage is available on target node
+ my $target_scfg = PVE::Storage::storage_check_enabled(
+ $storecfg,
+ $targetsid,
+ $self->{node},
+ );
+ my ($vtype) = PVE::Storage::parse_volname($storecfg, $volid);
+ die "$volid: content type '$vtype' is not available on storage '$targetsid'\n"
+ if !$target_scfg->{content}->{$vtype};
+ }
+}
+
sub prepare {
my ($self, $vmid) = @_;
my $online = $self->{opts}->{online};
- $self->{storecfg} = PVE::Storage::config();
+ my $storecfg = $self->{storecfg} = PVE::Storage::config();
# test if VM exists
my $conf = $self->{vmconf} = PVE::QemuConfig->load_config($vmid);
+ my $version = PVE::QemuServer::Helpers::get_node_pvecfg_version($self->{node});
+ my $cloudinit_config = $conf->{cloudinit};
+
+ if (
+ PVE::QemuConfig->has_cloudinit($conf) && defined($cloudinit_config)
+ && scalar(keys %$cloudinit_config) > 0
+ && !PVE::QemuServer::Helpers::pvecfg_min_version($version, 7, 2, 13)
+ ) {
+ die "target node is too old (manager <= 7.2-13) and doesn't support new cloudinit section\n";
+ }
+
+ my $repl_conf = PVE::ReplicationConfig->new();
+ $self->{replication_jobcfg} = $repl_conf->find_local_replication_job($vmid, $self->{node});
+ $self->{is_replicated} = $repl_conf->check_for_existing_jobs($vmid, 1);
+
+ if ($self->{replication_jobcfg} && defined($self->{replication_jobcfg}->{remove_job})) {
+ die "refusing to migrate replicated VM whose replication job is marked for removal\n";
+ }
+
PVE::QemuConfig->check_lock($conf);
my $running = 0;
die "can't migrate running VM without --online\n" if !$online;
$running = $pid;
+ if ($self->{is_replicated} && !$self->{replication_jobcfg}) {
+ if ($self->{opts}->{force}) {
+ $self->log('warn', "WARNING: Node '$self->{node}' is not a replication target. Existing " .
+ "replication jobs will fail after migration!\n");
+ } else {
+ die "Cannot live-migrate replicated VM to node '$self->{node}' - not a replication " .
+ "target. Use 'force' to override.\n";
+ }
+ }
+
$self->{forcemachine} = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
# To support custom CPU types, we keep QEMU's "-cpu" parameter intact.
$self->{forcecpu} = PVE::QemuServer::CPUConfig::get_cpu_from_running_vm($pid);
}
}
+
+ # Do not treat a suspended VM as paused, as it might wake up
+ # during migration and remain paused after migration finishes.
+ $self->{vm_was_paused} = 1 if PVE::QemuServer::vm_is_paused($vmid, 0);
}
- my $loc_res = PVE::QemuServer::check_local_resources($conf, 1);
- if (scalar @$loc_res) {
+ my ($loc_res, $mapped_res, $missing_mappings_by_node) = PVE::QemuServer::check_local_resources($conf, 1);
+ my $blocking_resources = [];
+ for my $res ($loc_res->@*) {
+ if (!grep($res, $mapped_res->@*)) {
+ push $blocking_resources->@*, $res;
+ }
+ }
+ if (scalar($blocking_resources->@*)) {
if ($self->{running} || !$self->{opts}->{force}) {
- die "can't migrate VM which uses local devices: " . join(", ", @$loc_res) . "\n";
+ die "can't migrate VM which uses local devices: " . join(", ", $blocking_resources->@*) . "\n";
} else {
$self->log('info', "migrating VM which uses local devices");
}
}
+ if (scalar($mapped_res->@*)) {
+ my $missing_mappings = $missing_mappings_by_node->{$self->{node}};
+ if ($running) {
+ die "can't migrate running VM which uses mapped devices: " . join(", ", $mapped_res->@*) . "\n";
+ } elsif (scalar($missing_mappings->@*)) {
+ die "can't migrate to '$self->{node}': missing mapped devices " . join(", ", $missing_mappings->@*) . "\n";
+ } else {
+ $self->log('info', "migrating VM which uses mapped local devices");
+ }
+ }
+
+ my $vga = PVE::QemuServer::parse_vga($conf->{vga});
+ if ($running && $vga->{'clipboard'} && $vga->{'clipboard'} eq 'vnc') {
+ die "VMs with 'clipboard' set to 'vnc' are not live migratable!\n";
+ }
+
my $vollist = PVE::QemuServer::get_vm_volumes($conf);
- my $need_activate = [];
+ my $storages = {};
foreach my $volid (@$vollist) {
my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
- # check if storage is available on both nodes
- my $targetsid = PVE::QemuServer::map_storage($self->{opts}->{storagemap}, $sid);
+ # check if storage is available on source node
+ my $scfg = PVE::Storage::storage_check_enabled($storecfg, $sid);
+
+ my $targetsid = $sid;
+ # NOTE: local ignores shared mappings, remote maps them
+ if (!$scfg->{shared} || $self->{opts}->{remote}) {
+ $targetsid = PVE::JSONSchema::map_id($self->{opts}->{storagemap}, $sid);
+ }
+
+ $storages->{$targetsid} = 1;
- my $scfg = PVE::Storage::storage_check_node($self->{storecfg}, $sid);
- PVE::Storage::storage_check_node($self->{storecfg}, $targetsid, $self->{node});
+ $self->target_storage_check_available($storecfg, $targetsid, $volid);
if ($scfg->{shared}) {
# PVE::Storage::activate_storage checks this for non-shared storages
my $plugin = PVE::Storage::Plugin->lookup($scfg->{type});
warn "Used shared storage '$sid' is not online on source node!\n"
if !$plugin->check_connection($sid, $scfg);
- } else {
- # only activate if not shared
- next if ($volid =~ m/vm-\d+-cloudinit/);
- push @$need_activate, $volid;
}
}
- # activate volumes
- PVE::Storage::activate_volumes($self->{storecfg}, $need_activate);
-
- # test ssh connection
- my $cmd = [ @{$self->{rem_ssh}}, '/bin/true' ];
- eval { $self->cmd_quiet($cmd); };
- die "Can't connect to destination address using public key\n" if $@;
+ if ($self->{opts}->{remote}) {
+ # test & establish websocket connection
+ my $bridges = map_bridges($conf, $self->{opts}->{bridgemap}, 1);
+ my $tunnel = $self->fork_websocket_tunnel($storages, $bridges);
+ my $min_version = $tunnel->{version} - $tunnel->{age};
+ $self->log('info', "local WS tunnel version: $WS_TUNNEL_VERSION");
+ $self->log('info', "remote WS tunnel version: $tunnel->{version}");
+ $self->log('info', "minimum required WS tunnel version: $min_version");
+ die "Remote tunnel endpoint not compatible, upgrade required\n"
+ if $WS_TUNNEL_VERSION < $min_version;
+ die "Remote tunnel endpoint too old, upgrade required\n"
+ if $WS_TUNNEL_VERSION > $tunnel->{version};
+
+ print "websocket tunnel started\n";
+ $self->{tunnel} = $tunnel;
+ } else {
+ # test ssh connection
+ my $cmd = [ @{$self->{rem_ssh}}, '/bin/true' ];
+ eval { $self->cmd_quiet($cmd); };
+ die "Can't connect to destination address using public key\n" if $@;
+ }
return $running;
}
-sub sync_disks {
+sub scan_local_volumes {
my ($self, $vmid) = @_;
my $conf = $self->{vmconf};
# local volumes which have been copied
# and their old_id => new_id pairs
- $self->{volumes} = [];
$self->{volume_map} = {};
+ $self->{local_volumes} = {};
my $storecfg = $self->{storecfg};
eval {
-
# found local volumes and their origin
- my $local_volumes = {};
+ my $local_volumes = $self->{local_volumes};
my $local_volumes_errors = {};
my $other_errors = [];
my $abort = 0;
+ my $path_to_volid = {};
my $log_error = sub {
my ($msg, $volid) = @_;
$abort = 1;
};
- my @sids = PVE::Storage::storage_ids($storecfg);
- foreach my $storeid (@sids) {
- my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
- next if $scfg->{shared};
- next if !PVE::Storage::storage_check_enabled($storecfg, $storeid, undef, 1);
-
- # get list from PVE::Storage (for unused volumes)
- my $dl = PVE::Storage::vdisk_list($storecfg, $storeid, $vmid);
-
- next if @{$dl->{$storeid}} == 0;
-
- my $targetsid = PVE::QemuServer::map_storage($self->{opts}->{storagemap}, $storeid);
- # check if storage is available on target node
- PVE::Storage::storage_check_node($storecfg, $targetsid, $self->{node});
-
- # grandfather in existing mismatches
- if ($targetsid ne $storeid) {
- my $target_scfg = PVE::Storage::storage_config($storecfg, $targetsid);
- die "content type 'images' is not available on storage '$targetsid'\n"
- if !$target_scfg->{content}->{images};
- }
-
- PVE::Storage::foreach_volid($dl, sub {
- my ($volid, $sid, $volinfo) = @_;
-
- $local_volumes->{$volid}->{ref} = 'storage';
-
- # If with_snapshots is not set for storage migrate, it tries to use
- # a raw+size stream, but on-the-fly conversion from qcow2 to raw+size
- # back to qcow2 is currently not possible.
- $local_volumes->{$volid}->{snapshots} = ($volinfo->{format} =~ /^(?:qcow2|vmdk)$/);
- $local_volumes->{$volid}->{format} = $volinfo->{format};
- });
- }
-
- my $rep_cfg = PVE::ReplicationConfig->new();
- my $replication_jobcfg = $rep_cfg->find_local_replication_job($vmid, $self->{node});
- my $replicatable_volumes = !$replication_jobcfg ? {}
+ my $replicatable_volumes = !$self->{replication_jobcfg} ? {}
: PVE::QemuConfig->get_replicatable_volumes($storecfg, $vmid, $conf, 0, 1);
+ foreach my $volid (keys %{$replicatable_volumes}) {
+ $local_volumes->{$volid}->{replicated} = 1;
+ }
my $test_volid = sub {
my ($volid, $attr) = @_;
if ($attr->{cdrom}) {
if ($volid eq 'cdrom') {
my $msg = "can't migrate local cdrom drive";
- if (defined($snaprefs) && !$attr->{referenced_in_config}) {
+ if (defined($snaprefs) && !$attr->{is_attached}) {
my $snapnames = join(', ', sort keys %$snaprefs);
$msg .= " (referenced in snapshot - $snapnames)";
}
my ($sid, $volname) = PVE::Storage::parse_volume_id($volid);
- my $targetsid = PVE::QemuServer::map_storage($self->{opts}->{storagemap}, $sid);
# check if storage is available on both nodes
- my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
- PVE::Storage::storage_check_node($storecfg, $targetsid, $self->{node});
+ my $scfg = PVE::Storage::storage_check_enabled($storecfg, $sid);
+
+ my $targetsid = $sid;
+ # NOTE: local ignores shared mappings, remote maps them
+ if (!$scfg->{shared} || $self->{opts}->{remote}) {
+ $targetsid = PVE::JSONSchema::map_id($self->{opts}->{storagemap}, $sid);
+ }
+
+ $self->target_storage_check_available($storecfg, $targetsid, $volid);
+ return if $scfg->{shared} && !$self->{opts}->{remote};
+
+ $local_volumes->{$volid}->{ref} = 'pending' if $attr->{referenced_in_pending};
+ $local_volumes->{$volid}->{ref} = 'snapshot' if $attr->{referenced_in_snapshot};
+ $local_volumes->{$volid}->{ref} = 'unused' if $attr->{is_unused};
+ $local_volumes->{$volid}->{ref} = 'attached' if $attr->{is_attached};
+ $local_volumes->{$volid}->{ref} = 'generated' if $attr->{is_tpmstate};
- return if $scfg->{shared};
+ $local_volumes->{$volid}->{bwlimit} = $self->get_bwlimit($sid, $targetsid);
+ $local_volumes->{$volid}->{targetsid} = $targetsid;
- $local_volumes->{$volid}->{ref} = $attr->{referenced_in_config} ? 'config' : 'snapshot';
- $local_volumes->{$volid}->{ref} = 'storage' if $attr->{is_unused};
+ $local_volumes->{$volid}->@{qw(size format)} = PVE::Storage::volume_size_info($storecfg, $volid);
$local_volumes->{$volid}->{is_vmstate} = $attr->{is_vmstate} ? 1 : 0;
+ $local_volumes->{$volid}->{drivename} = $attr->{drivename}
+ if $attr->{drivename};
+
+ # If with_snapshots is not set for storage migrate, it tries to use
+ # a raw+size stream, but on-the-fly conversion from qcow2 to raw+size
+ # back to qcow2 is currently not possible.
+ $local_volumes->{$volid}->{snapshots} = ($local_volumes->{$volid}->{format} =~ /^(?:qcow2|vmdk)$/);
+
if ($attr->{cdrom}) {
if ($volid =~ /vm-\d+-cloudinit/) {
$local_volumes->{$volid}->{ref} = 'generated';
die "owned by other VM (owner = VM $owner)\n"
if !$owner || ($owner != $vmid);
+ $path_to_volid->{$path}->{$volid} = 1;
+
return if $attr->{is_vmstate};
if (defined($snaprefs)) {
# we cannot migrate shapshots on local storage
# exceptions: 'zfspool' or 'qcow2' files (on directory storage)
- die "online storage migration not possible if snapshot exists\n" if $self->{running};
- if (!($scfg->{type} eq 'zfspool' || $local_volumes->{$volid}->{format} eq 'qcow2')) {
+ die "online storage migration not possible if non-replicated snapshot exists\n"
+ if $self->{running} && !$local_volumes->{$volid}->{replicated};
+
+ die "remote migration with snapshots not supported yet\n" if $self->{opts}->{remote};
+
+ if (!($scfg->{type} eq 'zfspool'
+ || ($scfg->{type} eq 'btrfs' && $local_volumes->{$volid}->{format} eq 'raw')
+ || $local_volumes->{$volid}->{format} eq 'qcow2'
+ )) {
die "non-migratable snapshot exists\n";
}
}
}
});
+ for my $path (keys %$path_to_volid) {
+ my @volids = keys $path_to_volid->{$path}->%*;
+ die "detected not supported aliased volumes: '" . join("', '", @volids) . "'\n"
+ if (scalar(@volids) > 1);
+ }
+
foreach my $vol (sort keys %$local_volumes) {
my $type = $replicatable_volumes->{$vol} ? 'local, replicated' : 'local';
my $ref = $local_volumes->{$vol}->{ref};
- if ($ref eq 'storage') {
- $self->log('info', "found $type disk '$vol' (via storage)\n");
- } elsif ($ref eq 'config') {
+ if ($ref eq 'attached') {
&$log_error("can't live migrate attached local disks without with-local-disks option\n", $vol)
if $self->{running} && !$self->{opts}->{"with-local-disks"};
- $self->log('info', "found $type disk '$vol' (in current VM config)\n");
+ $self->log('info', "found $type disk '$vol' (attached)\n");
+ } elsif ($ref eq 'unused') {
+ $self->log('info', "found $type disk '$vol' (unused)\n");
} elsif ($ref eq 'snapshot') {
$self->log('info', "found $type disk '$vol' (referenced by snapshot(s))\n");
+ } elsif ($ref eq 'pending') {
+ $self->log('info', "found $type disk '$vol' (pending change)\n");
} elsif ($ref eq 'generated') {
$self->log('info', "found generated disk '$vol' (in current VM config)\n");
} else {
my ($sid, $volname) = PVE::Storage::parse_volume_id($volid);
my $scfg = PVE::Storage::storage_config($storecfg, $sid);
- my $migratable = $scfg->{type} =~ /^(?:dir|zfspool|lvmthin|lvm)$/;
+ my $migratable = $scfg->{type} =~ /^(?:dir|btrfs|zfspool|lvmthin|lvm)$/;
+
+ # TODO: what is this even here for?
+ $migratable = 1 if $self->{opts}->{remote};
die "can't migrate '$volid' - storage type '$scfg->{type}' not supported\n"
if !$migratable;
}
}
- if ($replication_jobcfg) {
- if ($self->{running}) {
+ foreach my $volid (sort keys %$local_volumes) {
+ my $ref = $local_volumes->{$volid}->{ref};
+ if ($self->{running} && $ref eq 'attached') {
+ $local_volumes->{$volid}->{migration_mode} = 'online';
+ } elsif ($self->{running} && $ref eq 'generated') {
+ # offline migrate the cloud-init ISO and don't regenerate on VM start
+ #
+ # tpmstate will also be offline migrated first, and in case of
+ # live migration then updated by QEMU/swtpm if necessary
+ $local_volumes->{$volid}->{migration_mode} = 'offline';
+ } else {
+ $local_volumes->{$volid}->{migration_mode} = 'offline';
+ }
+ }
+ };
+ die "Problem found while scanning volumes - $@" if $@;
+}
+
+sub handle_replication {
+ my ($self, $vmid) = @_;
- my $version = PVE::QemuServer::kvm_user_version();
- if (!min_version($version, 4, 2)) {
- die "can't live migrate VM with replicated volumes, pve-qemu to old (< 4.2)!\n"
- }
+ my $conf = $self->{vmconf};
+ my $local_volumes = $self->{local_volumes};
- my $live_replicatable_volumes = {};
- PVE::QemuConfig->foreach_volume($conf, sub {
- my ($ds, $drive) = @_;
+ return if !$self->{replication_jobcfg};
- my $volid = $drive->{file};
- $live_replicatable_volumes->{$ds} = $volid
- if defined($replicatable_volumes->{$volid});
- });
- foreach my $drive (keys %$live_replicatable_volumes) {
- my $volid = $live_replicatable_volumes->{$drive};
+ die "can't migrate VM with replicated volumes to remote cluster/node\n"
+ if $self->{opts}->{remote};
- my $bitmap = "repl_$drive";
+ if ($self->{running}) {
- # start tracking before replication to get full delta + a few duplicates
- $self->log('info', "$drive: start tracking writes using block-dirty-bitmap '$bitmap'");
- mon_cmd($vmid, 'block-dirty-bitmap-add', node => "drive-$drive", name => $bitmap);
+ my $version = PVE::QemuServer::kvm_user_version();
+ if (!min_version($version, 4, 2)) {
+ die "can't live migrate VM with replicated volumes, pve-qemu to old (< 4.2)!\n"
+ }
- # other info comes from target node in phase 2
- $self->{target_drive}->{$drive}->{bitmap} = $bitmap;
- }
- }
- $self->log('info', "replicating disk images");
+ my @live_replicatable_volumes = $self->filter_local_volumes('online', 1);
+ foreach my $volid (@live_replicatable_volumes) {
+ my $drive = $local_volumes->{$volid}->{drivename};
+ die "internal error - no drive for '$volid'\n" if !defined($drive);
+
+ my $bitmap = "repl_$drive";
- my $start_time = time();
- my $logfunc = sub { $self->log('info', shift) };
- $self->{replicated_volumes} = PVE::Replication::run_replication(
- 'PVE::QemuConfig', $replication_jobcfg, $start_time, $start_time, $logfunc);
+ # start tracking before replication to get full delta + a few duplicates
+ $self->log('info', "$drive: start tracking writes using block-dirty-bitmap '$bitmap'");
+ mon_cmd($vmid, 'block-dirty-bitmap-add', node => "drive-$drive", name => $bitmap);
+
+ # other info comes from target node in phase 2
+ $self->{target_drive}->{$drive}->{bitmap} = $bitmap;
}
+ }
+ $self->log('info', "replicating disk images");
+
+ my $start_time = time();
+ my $logfunc = sub { $self->log('info', shift) };
+ my $actual_replicated_volumes = PVE::Replication::run_replication(
+ 'PVE::QemuConfig', $self->{replication_jobcfg}, $start_time, $start_time, $logfunc);
+
+ # extra safety check
+ my @replicated_volumes = $self->filter_local_volumes(undef, 1);
+ foreach my $volid (@replicated_volumes) {
+ die "expected volume '$volid' to get replicated, but it wasn't\n"
+ if !$actual_replicated_volumes->{$volid};
+ }
+}
- # sizes in config have to be accurate for remote node to correctly
- # allocate disks, rescan to be sure
- my $volid_hash = PVE::QemuServer::scan_volids($storecfg, $vmid);
- PVE::QemuConfig->foreach_volume($conf, sub {
- my ($key, $drive) = @_;
- return if $key eq 'efidisk0'; # skip efidisk, will be handled later
-
- my $volid = $drive->{file};
- return if !defined($local_volumes->{$volid}); # only update sizes for local volumes
- return if !defined($volid_hash->{$volid});
-
- my ($updated, $msg) = PVE::QemuServer::Drive::update_disksize($drive, $volid_hash->{$volid}->{size});
- if (defined($updated)) {
- $conf->{$key} = PVE::QemuServer::print_drive($updated);
- $self->log('info', "drive '$key': $msg");
- }
- });
+sub config_update_local_disksizes {
+ my ($self) = @_;
- # we want to set the efidisk size in the config to the size of the
- # real OVMF_VARS.fd image, else we can create a too big image, which does not work
- if (defined($conf->{efidisk0})) {
- PVE::QemuServer::update_efidisk_size($conf);
+ my $conf = $self->{vmconf};
+ my $local_volumes = $self->{local_volumes};
+
+ PVE::QemuConfig->foreach_volume($conf, sub {
+ my ($key, $drive) = @_;
+ # skip special disks, will be handled later
+ return if $key eq 'efidisk0';
+ return if $key eq 'tpmstate0';
+
+ my $volid = $drive->{file};
+ return if !defined($local_volumes->{$volid}); # only update sizes for local volumes
+
+ my ($updated, $msg) = PVE::QemuServer::Drive::update_disksize($drive, $local_volumes->{$volid}->{size});
+ if (defined($updated)) {
+ $conf->{$key} = PVE::QemuServer::print_drive($updated);
+ $self->log('info', "drive '$key': $msg");
}
+ });
- $self->log('info', "copying local disk images") if scalar(%$local_volumes);
+ # we want to set the efidisk size in the config to the size of the
+ # real OVMF_VARS.fd image, else we can create a too big image, which does not work
+ if (defined($conf->{efidisk0})) {
+ PVE::QemuServer::update_efidisk_size($conf);
+ }
- foreach my $volid (keys %$local_volumes) {
- my ($sid, $volname) = PVE::Storage::parse_volume_id($volid);
- my $targetsid = PVE::QemuServer::map_storage($self->{opts}->{storagemap}, $sid);
- my $ref = $local_volumes->{$volid}->{ref};
- if ($self->{running} && $ref eq 'config') {
- push @{$self->{online_local_volumes}}, $volid;
- } elsif ($ref eq 'generated') {
- die "can't live migrate VM with local cloudinit disk. use a shared storage instead\n" if $self->{running};
- # skip all generated volumes but queue them for deletion in phase3_cleanup
- push @{$self->{volumes}}, $volid;
- next;
- } else {
- next if $self->{replicated_volumes}->{$volid};
- push @{$self->{volumes}}, $volid;
- my $opts = $self->{opts};
- # use 'migrate' limit for transfer to other node
- my $bwlimit = PVE::Storage::get_bandwidth_limit('migration', [$targetsid, $sid], $opts->{bwlimit});
- # JSONSchema and get_bandwidth_limit use kbps - storage_migrate bps
- $bwlimit = $bwlimit * 1024 if defined($bwlimit);
-
- my $storage_migrate_opts = {
- 'ratelimit_bps' => $bwlimit,
- 'insecure' => $opts->{migration_type} eq 'insecure',
- 'with_snapshots' => $local_volumes->{$volid}->{snapshots},
- 'allow_rename' => !$local_volumes->{$volid}->{is_vmstate},
- };
+ # TPM state might have an irregular filesize, to avoid problems on transfer
+ # we always assume the static size of 4M to allocate on the target
+ if (defined($conf->{tpmstate0})) {
+ PVE::QemuServer::update_tpmstate_size($conf);
+ }
+}
- my $logfunc = sub { $self->log('info', $_[0]); };
- my $new_volid = eval {
- PVE::Storage::storage_migrate($storecfg, $volid, $self->{ssh_info},
- $targetsid, $storage_migrate_opts, $logfunc);
- };
- if (my $err = $@) {
- die "storage migration for '$volid' to storage '$targetsid' failed - $err\n";
- }
+sub filter_local_volumes {
+ my ($self, $migration_mode, $replicated) = @_;
+
+ my $volumes = $self->{local_volumes};
+ my @filtered_volids;
+
+ foreach my $volid (sort keys %{$volumes}) {
+ next if defined($migration_mode) && safe_string_ne($volumes->{$volid}->{migration_mode}, $migration_mode);
+ next if defined($replicated) && safe_boolean_ne($volumes->{$volid}->{replicated}, $replicated);
+ push @filtered_volids, $volid;
+ }
+
+ return @filtered_volids;
+}
- $self->{volume_map}->{$volid} = $new_volid;
- $self->log('info', "volume '$volid' is '$new_volid' on the target\n");
+sub sync_offline_local_volumes {
+ my ($self) = @_;
+
+ my $local_volumes = $self->{local_volumes};
+ my @volids = $self->filter_local_volumes('offline', 0);
+
+ my $storecfg = $self->{storecfg};
+ my $opts = $self->{opts};
+
+ $self->log('info', "copying local disk images") if scalar(@volids);
+
+ foreach my $volid (@volids) {
+ my $new_volid;
+
+ my $opts = $self->{opts};
+ if ($opts->{remote}) {
+ my $log = sub {
+ my ($level, $msg) = @_;
+ $self->log($level, $msg);
+ };
+
+ $new_volid = PVE::StorageTunnel::storage_migrate(
+ $self->{tunnel},
+ $storecfg,
+ $volid,
+ $self->{vmid},
+ $opts->{remote}->{vmid},
+ $local_volumes->{$volid},
+ $log,
+ );
+ } else {
+ my $targetsid = $local_volumes->{$volid}->{targetsid};
+
+ my $bwlimit = $local_volumes->{$volid}->{bwlimit};
+ $bwlimit = $bwlimit * 1024 if defined($bwlimit); # storage_migrate uses bps
+
+ my $storage_migrate_opts = {
+ 'ratelimit_bps' => $bwlimit,
+ 'insecure' => $opts->{migration_type} eq 'insecure',
+ 'with_snapshots' => $local_volumes->{$volid}->{snapshots},
+ 'allow_rename' => !$local_volumes->{$volid}->{is_vmstate},
+ };
+
+ my $logfunc = sub { $self->log('info', $_[0]); };
+ $new_volid = eval {
+ PVE::Storage::storage_migrate(
+ $storecfg,
+ $volid,
+ $self->{ssh_info},
+ $targetsid,
+ $storage_migrate_opts,
+ $logfunc,
+ );
+ };
+ if (my $err = $@) {
+ die "storage migration for '$volid' to storage '$targetsid' failed - $err\n";
}
}
- };
- die "Failed to sync data - $@" if $@;
+
+ $self->{volume_map}->{$volid} = $new_volid;
+ $self->log('info', "volume '$volid' is '$new_volid' on the target\n");
+
+ eval { PVE::Storage::deactivate_volumes($storecfg, [$volid]); };
+ if (my $err = $@) {
+ $self->log('warn', $err);
+ }
+ }
}
sub cleanup_remotedisks {
my ($self) = @_;
- foreach my $target_drive (keys %{$self->{target_drive}}) {
- my $drivestr = $self->{target_drive}->{$target_drive}->{drivestr};
- next if !defined($drivestr);
+ if ($self->{opts}->{remote}) {
+ PVE::Tunnel::finish_tunnel($self->{tunnel}, 1);
+ delete $self->{tunnel};
+ return;
+ }
- my $drive = PVE::QemuServer::parse_drive($target_drive, $drivestr);
+ my $local_volumes = $self->{local_volumes};
+ foreach my $volid (values %{$self->{volume_map}}) {
# don't clean up replicated disks!
- next if defined($self->{replicated_volumes}->{$drive->{file}});
+ next if $local_volumes->{$volid}->{replicated};
- my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
+ my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
my $cmd = [@{$self->{rem_ssh}}, 'pvesm', 'free', "$storeid:$volname"];
$conf->{lock} = 'migrate';
PVE::QemuConfig->write_config($vmid, $conf);
- sync_disks($self, $vmid);
+ $self->scan_local_volumes($vmid);
- # sync_disks fixes disk sizes to match their actual size, write changes so
- # target allocates correct volumes
+ # fix disk sizes to match their actual size and write changes,
+ # so that the target allocates the correct volumes
+ $self->config_update_local_disksizes();
PVE::QemuConfig->write_config($vmid, $conf);
+
+ $self->handle_replication($vmid);
+
+ $self->sync_offline_local_volumes();
+ $self->phase1_remote($vmid) if $self->{opts}->{remote};
};
+sub map_bridges {
+ my ($conf, $map, $scan_only) = @_;
+
+ my $bridges = {};
+
+ foreach my $opt (keys %$conf) {
+ next if $opt !~ m/^net\d+$/;
+
+ next if !$conf->{$opt};
+ my $d = PVE::QemuServer::parse_net($conf->{$opt});
+ next if !$d || !$d->{bridge};
+
+ my $target_bridge = PVE::JSONSchema::map_id($map, $d->{bridge});
+ $bridges->{$target_bridge}->{$opt} = $d->{bridge};
+
+ next if $scan_only;
+
+ $d->{bridge} = $target_bridge;
+ $conf->{$opt} = PVE::QemuServer::print_net($d);
+ }
+
+ return $bridges;
+}
+
+sub phase1_remote {
+ my ($self, $vmid) = @_;
+
+ my $remote_conf = PVE::QemuConfig->load_config($vmid);
+ PVE::QemuConfig->update_volume_ids($remote_conf, $self->{volume_map});
+
+ my $bridges = map_bridges($remote_conf, $self->{opts}->{bridgemap});
+ for my $target (keys $bridges->%*) {
+ for my $nic (keys $bridges->{$target}->%*) {
+ $self->log('info', "mapped: $nic from $bridges->{$target}->{$nic} to $target");
+ }
+ }
+
+ my @online_local_volumes = $self->filter_local_volumes('online');
+
+ my $storage_map = $self->{opts}->{storagemap};
+ $self->{nbd} = {};
+ PVE::QemuConfig->foreach_volume($remote_conf, sub {
+ my ($ds, $drive) = @_;
+
+ # TODO eject CDROM?
+ return if PVE::QemuServer::drive_is_cdrom($drive);
+
+ my $volid = $drive->{file};
+ return if !$volid;
+
+ return if !grep { $_ eq $volid} @online_local_volumes;
+
+ my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
+ my $scfg = PVE::Storage::storage_config($self->{storecfg}, $storeid);
+ my $source_format = PVE::QemuServer::qemu_img_format($scfg, $volname);
+
+ # set by target cluster
+ my $oldvolid = delete $drive->{file};
+ delete $drive->{format};
+
+ my $targetsid = PVE::JSONSchema::map_id($storage_map, $storeid);
+
+ my $params = {
+ format => $source_format,
+ storage => $targetsid,
+ drive => $drive,
+ };
+
+ $self->log('info', "Allocating volume for drive '$ds' on remote storage '$targetsid'..");
+ my $res = PVE::Tunnel::write_tunnel($self->{tunnel}, 600, 'disk', $params);
+
+ $self->log('info', "volume '$oldvolid' is '$res->{volid}' on the target\n");
+ $remote_conf->{$ds} = $res->{drivestr};
+ $self->{nbd}->{$ds} = $res;
+ });
+
+ my $conf_str = PVE::QemuServer::write_vm_config("remote", $remote_conf);
+
+ # TODO expose in PVE::Firewall?
+ my $vm_fw_conf_path = "/etc/pve/firewall/$vmid.fw";
+ my $fw_conf_str;
+ $fw_conf_str = PVE::Tools::file_get_contents($vm_fw_conf_path)
+ if -e $vm_fw_conf_path;
+ my $params = {
+ conf => $conf_str,
+ 'firewall-config' => $fw_conf_str,
+ };
+
+ PVE::Tunnel::write_tunnel($self->{tunnel}, 10, 'config', $params);
+}
+
sub phase1_cleanup {
my ($self, $vmid, $err) = @_;
$self->log('err', $err);
}
- if ($self->{volumes}) {
- foreach my $volid (@{$self->{volumes}}) {
- $self->log('err', "found stale volume copy '$volid' on node '$self->{node}'");
- # fixme: try to remove ?
- }
+ eval { $self->cleanup_remotedisks() };
+ if (my $err = $@) {
+ $self->log('err', $err);
}
eval { $self->cleanup_bitmaps() };
if (my $err =$@) {
$self->log('err', $err);
}
-
}
-sub phase2 {
- my ($self, $vmid) = @_;
+sub phase2_start_local_cluster {
+ my ($self, $vmid, $params) = @_;
my $conf = $self->{vmconf};
+ my $local_volumes = $self->{local_volumes};
+ my @online_local_volumes = $self->filter_local_volumes('online');
+
+ my $start = $params->{start_params};
+ my $migrate = $params->{migrate_opts};
$self->log('info', "starting VM $vmid on remote node '$self->{node}'");
- my $raddr;
- my $rport;
- my $ruri; # the whole migration dst. URI (protocol:address[:port])
- my $nodename = PVE::INotify::nodename();
+ my $tunnel_info = {};
## start on remote node
my $cmd = [@{$self->{rem_ssh}}];
- my $spice_ticket;
- if (PVE::QemuServer::vga_conf_has_spice($conf->{vga})) {
- my $res = mon_cmd($vmid, 'query-spice');
- $spice_ticket = $res->{ticket};
- }
+ push @$cmd, 'qm', 'start', $vmid;
- push @$cmd , 'qm', 'start', $vmid, '--skiplock', '--migratedfrom', $nodename;
+ if ($start->{skiplock}) {
+ push @$cmd, '--skiplock';
+ }
- my $migration_type = $self->{opts}->{migration_type};
+ push @$cmd, '--migratedfrom', $migrate->{migratedfrom};
- push @$cmd, '--migration_type', $migration_type;
+ push @$cmd, '--migration_type', $migrate->{type};
- push @$cmd, '--migration_network', $self->{opts}->{migration_network}
- if $self->{opts}->{migration_network};
+ push @$cmd, '--migration_network', $migrate->{network}
+ if $migrate->{network};
- if ($migration_type eq 'insecure') {
- push @$cmd, '--stateuri', 'tcp';
- } else {
- push @$cmd, '--stateuri', 'unix';
- }
+ push @$cmd, '--stateuri', $start->{statefile};
- if ($self->{forcemachine}) {
- push @$cmd, '--machine', $self->{forcemachine};
+ if ($start->{forcemachine}) {
+ push @$cmd, '--machine', $start->{forcemachine};
}
- if ($self->{forcecpu}) {
- push @$cmd, '--force-cpu', $self->{forcecpu};
+ if ($start->{forcecpu}) {
+ push @$cmd, '--force-cpu', $start->{forcecpu};
}
- if ($self->{online_local_volumes}) {
+ if ($self->{storage_migration}) {
push @$cmd, '--targetstorage', ($self->{opts}->{targetstorage} // '1');
}
my $spice_port;
- my $unix_socket_info = {};
- # version > 0 for unix socket support
- my $nbd_protocol_version = 1;
- # TODO change to 'spice_ticket: <ticket>\n' in 7.0
- my $input = $spice_ticket ? "$spice_ticket\n" : "\n";
- $input .= "nbd_protocol_version: $nbd_protocol_version\n";
-
- my $number_of_online_replicated_volumes = 0;
-
- # prevent auto-vivification
- if ($self->{online_local_volumes}) {
- foreach my $volid (@{$self->{online_local_volumes}}) {
- next if !$self->{replicated_volumes}->{$volid};
- $number_of_online_replicated_volumes++;
- $input .= "replicated_volume: $volid\n";
+ my $input = "nbd_protocol_version: $migrate->{nbd_proto_version}\n";
+
+ my @offline_local_volumes = $self->filter_local_volumes('offline');
+ for my $volid (@offline_local_volumes) {
+ my $drivename = $local_volumes->{$volid}->{drivename};
+ next if !$drivename || !$conf->{$drivename};
+
+ my $new_volid = $self->{volume_map}->{$volid};
+ next if !$new_volid || $volid eq $new_volid;
+
+ # FIXME PVE 8.x only use offline_volume variant once all targets can handle it
+ if ($drivename eq 'tpmstate0') {
+ $input .= "$drivename: $new_volid\n"
+ } else {
+ $input .= "offline_volume: $drivename: $new_volid\n"
}
}
+ $input .= "spice_ticket: $migrate->{spice_ticket}\n" if $migrate->{spice_ticket};
+
+ my @online_replicated_volumes = $self->filter_local_volumes('online', 1);
+ foreach my $volid (@online_replicated_volumes) {
+ $input .= "replicated_volume: $volid\n";
+ }
+
+ my $handle_storage_migration_listens = sub {
+ my ($drive_key, $drivestr, $nbd_uri) = @_;
+
+ $self->{stopnbd} = 1;
+ $self->{target_drive}->{$drive_key}->{drivestr} = $drivestr;
+ $self->{target_drive}->{$drive_key}->{nbd_uri} = $nbd_uri;
+
+ my $source_drive = PVE::QemuServer::parse_drive($drive_key, $conf->{$drive_key});
+ my $target_drive = PVE::QemuServer::parse_drive($drive_key, $drivestr);
+ my $source_volid = $source_drive->{file};
+ my $target_volid = $target_drive->{file};
+
+ $self->{volume_map}->{$source_volid} = $target_volid;
+ $self->log('info', "volume '$source_volid' is '$target_volid' on the target\n");
+ };
+
my $target_replicated_volumes = {};
# Note: We try to keep $spice_ticket secret (do not pass via command line parameter)
my $exitcode = PVE::Tools::run_command($cmd, input => $input, outfunc => sub {
my $line = shift;
- if ($line =~ m/^migration listens on tcp:(localhost|[\d\.]+|\[[\d\.:a-fA-F]+\]):(\d+)$/) {
- $raddr = $1;
- $rport = int($2);
- $ruri = "tcp:$raddr:$rport";
+ if ($line =~ m/^migration listens on (tcp):(localhost|[\d\.]+|\[[\d\.:a-fA-F]+\]):(\d+)$/) {
+ $tunnel_info->{addr} = $2;
+ $tunnel_info->{port} = int($3);
+ $tunnel_info->{proto} = $1;
}
- elsif ($line =~ m!^migration listens on unix:(/run/qemu-server/(\d+)\.migrate)$!) {
- $raddr = $1;
- die "Destination UNIX sockets VMID does not match source VMID" if $vmid ne $2;
- $ruri = "unix:$raddr";
+ elsif ($line =~ m!^migration listens on (unix):(/run/qemu-server/(\d+)\.migrate)$!) {
+ $tunnel_info->{addr} = $2;
+ die "Destination UNIX sockets VMID does not match source VMID" if $vmid ne $3;
+ $tunnel_info->{proto} = $1;
}
elsif ($line =~ m/^migration listens on port (\d+)$/) {
- $raddr = "localhost";
- $rport = int($1);
- $ruri = "tcp:$raddr:$rport";
+ $tunnel_info->{addr} = "localhost";
+ $tunnel_info->{port} = int($1);
+ $tunnel_info->{proto} = "tcp";
}
elsif ($line =~ m/^spice listens on port (\d+)$/) {
$spice_port = int($1);
my $targetdrive = $3;
$targetdrive =~ s/drive-//g;
- $self->{stopnbd} = 1;
- $self->{target_drive}->{$targetdrive}->{drivestr} = $drivestr;
- $self->{target_drive}->{$targetdrive}->{nbd_uri} = $nbd_uri;
+ $handle_storage_migration_listens->($targetdrive, $drivestr, $nbd_uri);
} elsif ($line =~ m!^storage migration listens on nbd:unix:(/run/qemu-server/(\d+)_nbd\.migrate):exportname=(\S+) volume:(\S+)$!) {
my $drivestr = $4;
die "Destination UNIX socket's VMID does not match source VMID" if $vmid ne $2;
my $targetdrive = $3;
$targetdrive =~ s/drive-//g;
- $self->{stopnbd} = 1;
- $self->{target_drive}->{$targetdrive}->{drivestr} = $drivestr;
- $self->{target_drive}->{$targetdrive}->{nbd_uri} = $nbd_uri;
- $unix_socket_info->{$nbd_unix_addr} = 1;
+ $handle_storage_migration_listens->($targetdrive, $drivestr, $nbd_uri);
+ $tunnel_info->{unix_sockets}->{$nbd_unix_addr} = 1;
} elsif ($line =~ m/^re-using replicated volume: (\S+) - (.*)$/) {
my $drive = $1;
my $volid = $2;
die "remote command failed with exit code $exitcode\n" if $exitcode;
- die "unable to detect remote migration address\n" if !$raddr;
+ die "unable to detect remote migration address\n" if !$tunnel_info->{addr} || !$tunnel_info->{proto};
- if (scalar(keys %$target_replicated_volumes) != $number_of_online_replicated_volumes) {
+ if (scalar(keys %$target_replicated_volumes) != scalar(@online_replicated_volumes)) {
die "number of replicated disks on source and target node do not match - target node too old?\n"
}
- $self->log('info', "start remote tunnel");
+ return ($tunnel_info, $spice_port);
+}
- if ($migration_type eq 'secure') {
+sub phase2_start_remote_cluster {
+ my ($self, $vmid, $params) = @_;
- if ($ruri =~ /^unix:/) {
- my $ssh_forward_info = ["$raddr:$raddr"];
- $unix_socket_info->{$raddr} = 1;
+ die "insecure migration to remote cluster not implemented\n"
+ if $params->{migrate_opts}->{type} ne 'websocket';
- my $unix_sockets = [ keys %$unix_socket_info ];
- for my $sock (@$unix_sockets) {
- push @$ssh_forward_info, "$sock:$sock";
- unlink $sock;
- }
+ my $remote_vmid = $self->{opts}->{remote}->{vmid};
- $self->{tunnel} = $self->fork_tunnel($ssh_forward_info);
+ # like regular start but with some overhead accounted for
+ my $memory = get_current_memory($self->{vmconf}->{memory});
+ my $timeout = PVE::QemuServer::Helpers::config_aware_timeout($self->{vmconf}, $memory) + 10;
- my $unix_socket_try = 0; # wait for the socket to become ready
- while ($unix_socket_try <= 100) {
- $unix_socket_try++;
- my $available = 0;
- foreach my $sock (@$unix_sockets) {
- if (-S $sock) {
- $available++;
- }
- }
+ my $res = PVE::Tunnel::write_tunnel($self->{tunnel}, $timeout, "start", $params);
- if ($available == @$unix_sockets) {
- last;
- }
+ foreach my $drive (keys %{$res->{drives}}) {
+ $self->{stopnbd} = 1;
+ $self->{target_drive}->{$drive}->{drivestr} = $res->{drives}->{$drive}->{drivestr};
+ my $nbd_uri = $res->{drives}->{$drive}->{nbd_uri};
+ die "unexpected NBD uri for '$drive': $nbd_uri\n"
+ if $nbd_uri !~ s!/run/qemu-server/$remote_vmid\_!/run/qemu-server/$vmid\_!;
- usleep(50000);
- }
- if ($unix_socket_try > 100) {
- $self->{errors} = 1;
- $self->finish_tunnel($self->{tunnel});
- die "Timeout, migration socket $ruri did not get ready";
- }
- $self->{tunnel}->{unix_sockets} = $unix_sockets if (@$unix_sockets);
+ $self->{target_drive}->{$drive}->{nbd_uri} = $nbd_uri;
+ }
- } elsif ($ruri =~ /^tcp:/) {
- my $ssh_forward_info = [];
- if ($raddr eq "localhost") {
- # for backwards compatibility with older qemu-server versions
- my $pfamily = PVE::Tools::get_host_address_family($nodename);
- my $lport = PVE::Tools::next_migrate_port($pfamily);
- push @$ssh_forward_info, "$lport:localhost:$rport";
- }
+ return ($res->{migrate}, $res->{spice_port});
+}
- $self->{tunnel} = $self->fork_tunnel($ssh_forward_info);
+sub phase2 {
+ my ($self, $vmid) = @_;
- } else {
- die "unsupported protocol in migration URI: $ruri\n";
+ my $conf = $self->{vmconf};
+ my $local_volumes = $self->{local_volumes};
+
+ # version > 0 for unix socket support
+ my $nbd_protocol_version = 1;
+
+ my $spice_ticket;
+ if (PVE::QemuServer::vga_conf_has_spice($conf->{vga})) {
+ my $res = mon_cmd($vmid, 'query-spice');
+ $spice_ticket = $res->{ticket};
+ }
+
+ my $migration_type = $self->{opts}->{migration_type};
+ my $state_uri = $migration_type eq 'insecure' ? 'tcp' : 'unix';
+
+ my $params = {
+ start_params => {
+ statefile => $state_uri,
+ forcemachine => $self->{forcemachine},
+ forcecpu => $self->{forcecpu},
+ skiplock => 1,
+ },
+ migrate_opts => {
+ spice_ticket => $spice_ticket,
+ type => $migration_type,
+ network => $self->{opts}->{migration_network},
+ storagemap => $self->{opts}->{storagemap},
+ migratedfrom => PVE::INotify::nodename(),
+ nbd_proto_version => $nbd_protocol_version,
+ nbd => $self->{nbd},
+ },
+ };
+
+ my ($tunnel_info, $spice_port);
+
+ my @online_local_volumes = $self->filter_local_volumes('online');
+ $self->{storage_migration} = 1 if scalar(@online_local_volumes);
+
+ if (my $remote = $self->{opts}->{remote}) {
+ my $remote_vmid = $remote->{vmid};
+ $params->{migrate_opts}->{remote_node} = $self->{node};
+ ($tunnel_info, $spice_port) = $self->phase2_start_remote_cluster($vmid, $params);
+ die "only UNIX sockets are supported for remote migration\n"
+ if $tunnel_info->{proto} ne 'unix';
+
+ my $remote_socket = $tunnel_info->{addr};
+ my $local_socket = $remote_socket;
+ $local_socket =~ s/$remote_vmid/$vmid/g;
+ $tunnel_info->{addr} = $local_socket;
+
+ $self->log('info', "Setting up tunnel for '$local_socket'");
+ PVE::Tunnel::forward_unix_socket($self->{tunnel}, $local_socket, $remote_socket);
+
+ foreach my $remote_socket (@{$tunnel_info->{unix_sockets}}) {
+ my $local_socket = $remote_socket;
+ $local_socket =~ s/$remote_vmid/$vmid/g;
+ next if $self->{tunnel}->{forwarded}->{$local_socket};
+ $self->log('info', "Setting up tunnel for '$local_socket'");
+ PVE::Tunnel::forward_unix_socket($self->{tunnel}, $local_socket, $remote_socket);
}
} else {
- #fork tunnel for insecure migration, to send faster commands like resume
- $self->{tunnel} = $self->fork_tunnel();
+ ($tunnel_info, $spice_port) = $self->phase2_start_local_cluster($vmid, $params);
+
+ $self->log('info', "start remote tunnel");
+ $self->start_remote_tunnel($tunnel_info);
}
- my $start = time();
- my $opt_bwlimit = $self->{opts}->{bwlimit};
+ my $migrate_uri = "$tunnel_info->{proto}:$tunnel_info->{addr}";
+ $migrate_uri .= ":$tunnel_info->{port}"
+ if defined($tunnel_info->{port});
- if (defined($self->{online_local_volumes})) {
- $self->{storage_migration} = 1;
+ if ($self->{storage_migration}) {
$self->{storage_migration_jobs} = {};
$self->log('info', "starting storage migration");
die "The number of local disks does not match between the source and the destination.\n"
- if (scalar(keys %{$self->{target_drive}}) != scalar @{$self->{online_local_volumes}});
+ if (scalar(keys %{$self->{target_drive}}) != scalar(@online_local_volumes));
foreach my $drive (keys %{$self->{target_drive}}){
my $target = $self->{target_drive}->{$drive};
my $nbd_uri = $target->{nbd_uri};
my $source_drive = PVE::QemuServer::parse_drive($drive, $conf->{$drive});
- my $target_drive = PVE::QemuServer::parse_drive($drive, $target->{drivestr});
-
my $source_volid = $source_drive->{file};
- my $target_volid = $target_drive->{file};
- my $source_sid = PVE::Storage::Plugin::parse_volume_id($source_volid);
- my $target_sid = PVE::Storage::Plugin::parse_volume_id($target_volid);
-
- my $bwlimit = PVE::Storage::get_bandwidth_limit('migration', [$source_sid, $target_sid], $opt_bwlimit);
+ my $bwlimit = $self->{local_volumes}->{$source_volid}->{bwlimit};
my $bitmap = $target->{bitmap};
$self->log('info', "$drive: start migration to $nbd_uri");
PVE::QemuServer::qemu_drive_mirror($vmid, $drive, $nbd_uri, $vmid, undef, $self->{storage_migration_jobs}, 'skip', undef, $bwlimit, $bitmap);
-
- $self->{volume_map}->{$source_volid} = $target_volid;
- $self->log('info', "volume '$source_volid' is '$target_volid' on the target\n");
}
}
- $self->log('info', "starting online/live migration on $ruri");
+ $self->log('info', "starting online/live migration on $migrate_uri");
$self->{livemigration} = 1;
# load_defaults
my $defaults = PVE::QemuServer::load_defaults();
- $self->log('info', "set migration_caps");
- eval {
- PVE::QemuServer::set_migration_caps($vmid);
- };
+ $self->log('info', "set migration capabilities");
+ eval { PVE::QemuServer::set_migration_caps($vmid) };
warn $@ if $@;
my $qemu_migrate_params = {};
# migrate speed can be set via bwlimit (datacenter.cfg and API) and via the
# migrate_speed parameter in qm.conf - take the lower of the two.
- my $bwlimit = PVE::Storage::get_bandwidth_limit('migration', undef, $opt_bwlimit) // 0;
- my $migrate_speed = $conf->{migrate_speed} // $bwlimit;
- # migrate_speed is in MB/s, bwlimit in KB/s
- $migrate_speed *= 1024;
+ my $bwlimit = $self->get_bwlimit();
- $migrate_speed = ($bwlimit < $migrate_speed) ? $bwlimit : $migrate_speed;
+ my $migrate_speed = $conf->{migrate_speed} // 0;
+ $migrate_speed *= 1024; # migrate_speed is in MB/s, bwlimit in KB/s
- # always set migrate speed (overwrite kvm default of 32m) we set a very high
- # default of 8192m which is basically unlimited
- $migrate_speed ||= ($defaults->{migrate_speed} || 8192) * 1024;
+ if ($bwlimit && $migrate_speed) {
+ $migrate_speed = ($bwlimit < $migrate_speed) ? $bwlimit : $migrate_speed;
+ } else {
+ $migrate_speed ||= $bwlimit;
+ }
+ $migrate_speed ||= ($defaults->{migrate_speed} || 0) * 1024;
- # qmp takes migrate_speed in B/s.
- $migrate_speed *= 1024;
- $self->log('info', "migration speed limit: $migrate_speed B/s");
+ if ($migrate_speed) {
+ $migrate_speed *= 1024; # qmp takes migrate_speed in B/s.
+ $self->log('info', "migration speed limit: ". render_bytes($migrate_speed, 1) ."/s");
+ } else {
+ # always set migrate speed as QEMU default to 128 MiBps == 1 Gbps, use 16 GiBps == 128 Gbps
+ $migrate_speed = (16 << 30);
+ }
$qemu_migrate_params->{'max-bandwidth'} = int($migrate_speed);
my $migrate_downtime = $defaults->{migrate_downtime};
$qemu_migrate_params->{'downtime-limit'} = int($migrate_downtime);
# set cachesize to 10% of the total memory
- my $memory = $conf->{memory} || $defaults->{memory};
+ my $memory = get_current_memory($conf->{memory});
my $cachesize = int($memory * 1048576 / 10);
$cachesize = round_powerof2($cachesize);
- $self->log('info', "migration cachesize: $cachesize B");
+ $self->log('info', "migration cachesize: " . render_bytes($cachesize, 1));
$qemu_migrate_params->{'xbzrle-cache-size'} = int($cachesize);
$self->log('info', "set migration parameters");
};
$self->log('info', "migrate-set-parameters error: $@") if $@;
- if (PVE::QemuServer::vga_conf_has_spice($conf->{vga})) {
+ if (PVE::QemuServer::vga_conf_has_spice($conf->{vga}) && !$self->{opts}->{remote}) {
my $rpcenv = PVE::RPCEnvironment::get();
my $authuser = $rpcenv->get_user();
}
- $self->log('info', "start migrate command to $ruri");
+ my $start = time();
+
+ $self->log('info', "start migrate command to $migrate_uri");
eval {
- mon_cmd($vmid, "migrate", uri => $ruri);
+ mon_cmd($vmid, "migrate", uri => $migrate_uri);
};
my $merr = $@;
- $self->log('info', "migrate uri => $ruri failed: $merr") if $merr;
+ $self->log('info', "migrate uri => $migrate_uri failed: $merr") if $merr;
- my $lstat = 0;
+ my $last_mem_transferred = 0;
my $usleep = 1000000;
my $i = 0;
my $err_count = 0;
my $downtimecounter = 0;
while (1) {
$i++;
- my $avglstat = $lstat ? $lstat / $i : 0;
+ my $avglstat = $last_mem_transferred ? $last_mem_transferred / $i : 0;
usleep($usleep);
- my $stat;
- eval {
- $stat = mon_cmd($vmid, "query-migrate");
- };
+
+ my $stat = eval { mon_cmd($vmid, "query-migrate") };
if (my $err = $@) {
$err_count++;
warn "query migrate failed: $err\n";
$self->log('info', "query migrate failed: $err");
if ($err_count <= 5) {
- usleep(1000000);
+ usleep(1_000_000);
next;
}
die "too many query migrate failures - aborting\n";
}
- if (defined($stat->{status}) && $stat->{status} =~ m/^(setup)$/im) {
- sleep(1);
- next;
- }
-
- if (defined($stat->{status}) && $stat->{status} =~ m/^(active|completed|failed|cancelled)$/im) {
- $merr = undef;
- $err_count = 0;
- if ($stat->{status} eq 'completed') {
- my $delay = time() - $start;
- if ($delay > 0) {
- my $mbps = sprintf "%.2f", $memory / $delay;
- my $downtime = $stat->{downtime} || 0;
- $self->log('info', "migration speed: $mbps MB/s - downtime $downtime ms");
- }
- }
+ my $status = $stat->{status};
+ if (defined($status) && $status =~ m/^(setup)$/im) {
+ sleep(1);
+ next;
+ }
- if ($stat->{status} eq 'failed' || $stat->{status} eq 'cancelled') {
- $self->log('info', "migration status error: $stat->{status}");
- die "aborting\n"
+ if (!defined($status) || $status !~ m/^(active|completed|failed|cancelled)$/im) {
+ die $merr if $merr;
+ die "unable to parse migration status '$status' - aborting\n";
+ }
+ $merr = undef;
+ $err_count = 0;
+
+ my $memstat = $stat->{ram};
+
+ if ($status eq 'completed') {
+ my $delay = time() - $start;
+ if ($delay > 0) {
+ my $total = $memstat->{total} || 0;
+ my $avg_speed = render_bytes($total / $delay, 1);
+ my $downtime = $stat->{downtime} || 0;
+ $self->log('info', "average migration speed: $avg_speed/s - downtime $downtime ms");
}
+ }
+
+ if ($status eq 'failed' || $status eq 'cancelled') {
+ my $message = $stat->{'error-desc'} ? "$status - $stat->{'error-desc'}" : $status;
+ $self->log('info', "migration status error: $message");
+ die "aborting\n"
+ }
+
+ if ($status ne 'active') {
+ $self->log('info', "migration status: $status");
+ last;
+ }
+
+ if ($memstat->{transferred} ne $last_mem_transferred) {
+ my $trans = $memstat->{transferred} || 0;
+ my $rem = $memstat->{remaining} || 0;
+ my $total = $memstat->{total} || 0;
+ my $speed = ($memstat->{'pages-per-second'} // 0) * ($memstat->{'page-size'} // 0);
+ my $dirty_rate = ($memstat->{'dirty-pages-rate'} // 0) * ($memstat->{'page-size'} // 0);
+
+ # reduce sleep if remainig memory is lower than the average transfer speed
+ $usleep = 100_000 if $avglstat && $rem < $avglstat;
+
+ # also reduce loggin if we poll more frequent
+ my $should_log = $usleep > 100_000 ? 1 : ($i % 10) == 0;
+
+ my $total_h = render_bytes($total, 1);
+ my $transferred_h = render_bytes($trans, 1);
+ my $speed_h = render_bytes($speed, 1);
- if ($stat->{status} ne 'active') {
- $self->log('info', "migration status: $stat->{status}");
- last;
+ my $progress = "transferred $transferred_h of $total_h VM-state, ${speed_h}/s";
+
+ if ($dirty_rate > $speed) {
+ my $dirty_rate_h = render_bytes($dirty_rate, 1);
+ $progress .= ", VM dirties lots of memory: $dirty_rate_h/s";
}
- if ($stat->{ram}->{transferred} ne $lstat) {
- my $trans = $stat->{ram}->{transferred} || 0;
- my $rem = $stat->{ram}->{remaining} || 0;
- my $total = $stat->{ram}->{total} || 0;
- my $xbzrlecachesize = $stat->{"xbzrle-cache"}->{"cache-size"} || 0;
- my $xbzrlebytes = $stat->{"xbzrle-cache"}->{"bytes"} || 0;
- my $xbzrlepages = $stat->{"xbzrle-cache"}->{"pages"} || 0;
- my $xbzrlecachemiss = $stat->{"xbzrle-cache"}->{"cache-miss"} || 0;
- my $xbzrleoverflow = $stat->{"xbzrle-cache"}->{"overflow"} || 0;
- # reduce sleep if remainig memory is lower than the average transfer speed
- $usleep = 100000 if $avglstat && $rem < $avglstat;
-
- $self->log('info', "migration status: $stat->{status} (transferred ${trans}, " .
- "remaining ${rem}), total ${total})");
-
- if (${xbzrlecachesize}) {
- $self->log('info', "migration xbzrle cachesize: ${xbzrlecachesize} transferred ${xbzrlebytes} pages ${xbzrlepages} cachemiss ${xbzrlecachemiss} overflow ${xbzrleoverflow}");
- }
+ $self->log('info', "migration $status, $progress") if $should_log;
- if (($lastrem && $rem > $lastrem ) || ($rem == 0)) {
- $downtimecounter++;
- }
- $lastrem = $rem;
-
- if ($downtimecounter > 5) {
- $downtimecounter = 0;
- $migrate_downtime *= 2;
- $self->log('info', "auto-increased downtime to continue migration: $migrate_downtime ms");
- eval {
- # migrate-set-parameters does not touch values not
- # specified, so this only changes downtime-limit
- mon_cmd($vmid, "migrate-set-parameters", 'downtime-limit' => int($migrate_downtime));
- };
- $self->log('info', "migrate-set-parameters error: $@") if $@;
- }
+ my $xbzrle = $stat->{"xbzrle-cache"} || {};
+ my ($xbzrlebytes, $xbzrlepages) = $xbzrle->@{'bytes', 'pages'};
+ if ($xbzrlebytes || $xbzrlepages) {
+ my $bytes_h = render_bytes($xbzrlebytes, 1);
+
+ my $msg = "send updates to $xbzrlepages pages in $bytes_h encoded memory";
+ $msg .= sprintf(", cache-miss %.2f%%", $xbzrle->{'cache-miss-rate'} * 100)
+ if $xbzrle->{'cache-miss-rate'};
+
+ $msg .= ", overflow $xbzrle->{overflow}" if $xbzrle->{overflow};
+
+ $self->log('info', "xbzrle: $msg") if $should_log;
}
+ if (($lastrem && $rem > $lastrem) || ($rem == 0)) {
+ $downtimecounter++;
+ }
+ $lastrem = $rem;
+
+ if ($downtimecounter > 5) {
+ $downtimecounter = 0;
+ $migrate_downtime *= 2;
+ $self->log('info', "auto-increased downtime to continue migration: $migrate_downtime ms");
+ eval {
+ # migrate-set-parameters does not touch values not
+ # specified, so this only changes downtime-limit
+ mon_cmd($vmid, "migrate-set-parameters", 'downtime-limit' => int($migrate_downtime));
+ };
+ $self->log('info', "migrate-set-parameters error: $@") if $@;
+ }
+ }
- $lstat = $stat->{ram}->{transferred};
+ $last_mem_transferred = $memstat->{transferred};
+ }
- } else {
- die $merr if $merr;
- die "unable to parse migration status '$stat->{status}' - aborting\n";
+ if ($self->{storage_migration}) {
+ # finish block-job with block-job-cancel, to disconnect source VM from NBD
+ # to avoid it trying to re-establish it. We are in blockjob ready state,
+ # thus, this command changes to it to blockjob complete (see qapi docs)
+ eval { PVE::QemuServer::qemu_drive_mirror_monitor($vmid, undef, $self->{storage_migration_jobs}, 'cancel'); };
+ if (my $err = $@) {
+ die "Failed to complete storage migration: $err\n";
}
}
}
};
$self->log('info', "migrate_cancel error: $@") if $@;
+ my $vm_status = eval {
+ mon_cmd($vmid, 'query-status')->{status} or die "no 'status' in result\n";
+ };
+ $self->log('err', "query-status error: $@") if $@;
+
+ # Can end up in POSTMIGRATE state if failure occurred after convergence. Try going back to
+ # original state. Unfortunately, direct transition from POSTMIGRATE to PAUSED is not possible.
+ if ($vm_status && $vm_status eq 'postmigrate') {
+ if (!$self->{vm_was_paused}) {
+ eval { mon_cmd($vmid, 'cont'); };
+ $self->log('err', "resuming VM failed: $@") if $@;
+ } else {
+ $self->log('err', "VM was paused, but ended in postmigrate state");
+ }
+ }
+
my $conf = $self->{vmconf};
delete $conf->{lock};
eval { PVE::QemuConfig->write_config($vmid, $conf) };
my $nodename = PVE::INotify::nodename();
- my $cmd = [@{$self->{rem_ssh}}, 'qm', 'stop', $vmid, '--skiplock', '--migratedfrom', $nodename];
- eval{ PVE::Tools::run_command($cmd, outfunc => sub {}, errfunc => sub {}) };
- if (my $err = $@) {
- $self->log('err', $err);
- $self->{errors} = 1;
+ if ($self->{tunnel} && $self->{tunnel}->{version} >= 2) {
+ PVE::Tunnel::write_tunnel($self->{tunnel}, 10, 'stop');
+ } else {
+ my $cmd = [@{$self->{rem_ssh}}, 'qm', 'stop', $vmid, '--skiplock', '--migratedfrom', $nodename];
+ eval{ PVE::Tools::run_command($cmd, outfunc => sub {}, errfunc => sub {}) };
+ if (my $err = $@) {
+ $self->log('err', $err);
+ $self->{errors} = 1;
+ }
}
# cleanup after stopping, otherwise disks might be in-use by target VM!
if ($self->{tunnel}) {
- eval { finish_tunnel($self, $self->{tunnel}); };
+ eval { PVE::Tunnel::finish_tunnel($self->{tunnel}); };
if (my $err = $@) {
$self->log('err', $err);
$self->{errors} = 1;
sub phase3 {
my ($self, $vmid) = @_;
- my $volids = $self->{volumes};
- return if $self->{phase2errors};
-
- # destroy local copies
- foreach my $volid (@$volids) {
- eval { PVE::Storage::vdisk_free($self->{storecfg}, $volid); };
- if (my $err = $@) {
- $self->log('err', "removing local copy of '$volid' failed - $err");
- $self->{errors} = 1;
- last if $err =~ /^interrupted by signal$/;
- }
- }
+ return;
}
sub phase3_cleanup {
my $tunnel = $self->{tunnel};
- if ($self->{storage_migration}) {
- # finish block-job with block-job-cancel, to disconnect source VM from NBD
- # to avoid it trying to re-establish it. We are in blockjob ready state,
- # thus, this command changes to it to blockjob complete (see qapi docs)
- eval { PVE::QemuServer::qemu_drive_mirror_monitor($vmid, undef, $self->{storage_migration_jobs}, 'cancel'); };
+ my $sourcevollist = PVE::QemuServer::get_vm_volumes($conf);
- if (my $err = $@) {
- eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $self->{storage_migration_jobs}) };
- eval { PVE::QemuMigrate::cleanup_remotedisks($self) };
- die "Failed to complete storage migration: $err\n";
- }
- }
-
- if ($self->{volume_map}) {
+ if ($self->{volume_map} && !$self->{opts}->{remote}) {
my $target_drives = $self->{target_drive};
# FIXME: for NBD storage migration we now only update the volid, and
}
# transfer replication state before move config
- $self->transfer_replication_state() if $self->{replicated_volumes};
-
- # move config to remote node
- my $conffile = PVE::QemuConfig->config_file($vmid);
- my $newconffile = PVE::QemuConfig->config_file($vmid, $self->{node});
-
- die "Failed to move config to node '$self->{node}' - rename failed: $!\n"
- if !rename($conffile, $newconffile);
-
- $self->switch_replication_job_target() if $self->{replicated_volumes};
+ if (!$self->{opts}->{remote}) {
+ $self->transfer_replication_state() if $self->{is_replicated};
+ PVE::QemuConfig->move_config_to_node($vmid, $self->{node});
+ $self->switch_replication_job_target() if $self->{is_replicated};
+ }
if ($self->{livemigration}) {
if ($self->{stopnbd}) {
$self->log('info', "stopping NBD storage migration server on target.");
# stop nbd server on remote vm - requirement for resume since 2.9
- my $cmd = [@{$self->{rem_ssh}}, 'qm', 'nbdstop', $vmid];
+ if ($tunnel && $tunnel->{version} && $tunnel->{version} >= 2) {
+ eval {
+ PVE::Tunnel::write_tunnel($tunnel, 30, 'nbdstop');
+ };
+ if (my $err = $@) {
+ $self->log('err', $err);
+ $self->{errors} = 1;
+ }
+ } else {
+ my $cmd = [@{$self->{rem_ssh}}, 'qm', 'nbdstop', $vmid];
- eval{ PVE::Tools::run_command($cmd, outfunc => sub {}, errfunc => sub {}) };
- if (my $err = $@) {
- $self->log('err', $err);
- $self->{errors} = 1;
+ eval{ PVE::Tools::run_command($cmd, outfunc => sub {}, errfunc => sub {}) };
+ if (my $err = $@) {
+ $self->log('err', $err);
+ $self->{errors} = 1;
+ }
}
}
- # config moved and nbd server stopped - now we can resume vm on target
- if ($tunnel && $tunnel->{version} && $tunnel->{version} >= 1) {
- eval {
- $self->write_tunnel($tunnel, 30, "resume $vmid");
- };
- if (my $err = $@) {
- $self->log('err', $err);
- $self->{errors} = 1;
- }
- } else {
- my $cmd = [@{$self->{rem_ssh}}, 'qm', 'resume', $vmid, '--skiplock', '--nocheck'];
- my $logf = sub {
- my $line = shift;
- $self->log('err', $line);
- };
- eval { PVE::Tools::run_command($cmd, outfunc => sub {}, errfunc => $logf); };
- if (my $err = $@) {
- $self->log('err', $err);
- $self->{errors} = 1;
+ # deletes local FDB entries if learning is disabled, they'll be re-added on target on resume
+ PVE::QemuServer::del_nets_bridge_fdb($conf, $vmid);
+
+ if (!$self->{vm_was_paused}) {
+ # config moved and nbd server stopped - now we can resume vm on target
+ if ($tunnel && $tunnel->{version} && $tunnel->{version} >= 1) {
+ my $cmd = $tunnel->{version} == 1 ? "resume $vmid" : "resume";
+ eval {
+ PVE::Tunnel::write_tunnel($tunnel, 30, $cmd);
+ };
+ if (my $err = $@) {
+ $self->log('err', $err);
+ $self->{errors} = 1;
+ }
+ } else {
+ # nocheck in case target node hasn't processed the config move/rename yet
+ my $cmd = [@{$self->{rem_ssh}}, 'qm', 'resume', $vmid, '--skiplock', '--nocheck'];
+ my $logf = sub {
+ my $line = shift;
+ $self->log('err', $line);
+ };
+ eval { PVE::Tools::run_command($cmd, outfunc => sub {}, errfunc => $logf); };
+ if (my $err = $@) {
+ $self->log('err', $err);
+ $self->{errors} = 1;
+ }
}
}
- if ($self->{storage_migration} && PVE::QemuServer::parse_guest_agent($conf)->{fstrim_cloned_disks} && $self->{running}) {
- my $cmd = [@{$self->{rem_ssh}}, 'qm', 'guest', 'cmd', $vmid, 'fstrim'];
- eval{ PVE::Tools::run_command($cmd, outfunc => sub {}, errfunc => sub {}) };
+ if (
+ $self->{storage_migration}
+ && PVE::QemuServer::parse_guest_agent($conf)->{fstrim_cloned_disks}
+ && $self->{running}
+ ) {
+ if (!$self->{vm_was_paused}) {
+ $self->log('info', "issuing guest fstrim");
+ if ($self->{opts}->{remote}) {
+ PVE::Tunnel::write_tunnel($self->{tunnel}, 600, 'fstrim');
+ } else {
+ my $cmd = [@{$self->{rem_ssh}}, 'qm', 'guest', 'cmd', $vmid, 'fstrim'];
+ eval{ PVE::Tools::run_command($cmd, outfunc => sub {}, errfunc => sub {}) };
+ if (my $err = $@) {
+ $self->log('err', "fstrim failed - $err");
+ $self->{errors} = 1;
+ }
+ }
+ } else {
+ $self->log('info', "skipping guest fstrim, because VM is paused");
+ }
}
}
# close tunnel on successful migration, on error phase2_cleanup closed it
- if ($tunnel) {
- eval { finish_tunnel($self, $tunnel); };
+ if ($tunnel && $tunnel->{version} == 1) {
+ eval { PVE::Tunnel::finish_tunnel($tunnel); };
if (my $err = $@) {
$self->log('err', $err);
$self->{errors} = 1;
}
+ $tunnel = undef;
+ delete $self->{tunnel};
}
eval {
}
};
- # always stop local VM
+ # always stop local VM with nocheck, since config is moved already
eval { PVE::QemuServer::vm_stop($self->{storecfg}, $vmid, 1, 1); };
if (my $err = $@) {
$self->log('err', "stopping vm failed - $err");
# always deactivate volumes - avoid lvm LVs to be active on several nodes
eval {
- my $vollist = PVE::QemuServer::get_vm_volumes($conf);
- PVE::Storage::deactivate_volumes($self->{storecfg}, $vollist);
+ PVE::Storage::deactivate_volumes($self->{storecfg}, $sourcevollist);
};
if (my $err = $@) {
$self->log('err', $err);
$self->{errors} = 1;
}
- if($self->{storage_migration}) {
- # destroy local copies
- my $volids = $self->{online_local_volumes};
+ my @not_replicated_volumes = $self->filter_local_volumes(undef, 0);
- foreach my $volid (@$volids) {
- # keep replicated volumes!
- next if $self->{replicated_volumes}->{$volid};
+ # destroy local copies
+ foreach my $volid (@not_replicated_volumes) {
+ # remote is cleaned up below
+ next if $self->{opts}->{remote};
- eval { PVE::Storage::vdisk_free($self->{storecfg}, $volid); };
- if (my $err = $@) {
- $self->log('err', "removing local copy of '$volid' failed - $err");
- $self->{errors} = 1;
- last if $err =~ /^interrupted by signal$/;
- }
+ eval { PVE::Storage::vdisk_free($self->{storecfg}, $volid); };
+ if (my $err = $@) {
+ $self->log('err', "removing local copy of '$volid' failed - $err");
+ $self->{errors} = 1;
+ last if $err =~ /^interrupted by signal$/;
}
-
}
# clear migrate lock
- my $cmd = [ @{$self->{rem_ssh}}, 'qm', 'unlock', $vmid ];
- $self->cmd_logerr($cmd, errmsg => "failed to clear migrate lock");
+ if ($tunnel && $tunnel->{version} >= 2) {
+ PVE::Tunnel::write_tunnel($tunnel, 10, "unlock");
+
+ PVE::Tunnel::finish_tunnel($tunnel);
+ } else {
+ my $cmd = [ @{$self->{rem_ssh}}, 'qm', 'unlock', $vmid ];
+ $self->cmd_logerr($cmd, errmsg => "failed to clear migrate lock");
+ }
+
+ if ($self->{opts}->{remote} && $self->{opts}->{delete}) {
+ eval { PVE::QemuServer::destroy_vm($self->{storecfg}, $vmid, 1, undef, 0) };
+ warn "Failed to remove source VM - $@\n" if $@;
+ }
}
sub final_cleanup {
use IO::Socket::UNIX;
use IPC::Open3;
use JSON;
+use List::Util qw(first);
use MIME::Base64;
use POSIX;
use Storable qw(dclone);
-use Time::HiRes qw(gettimeofday);
+use Time::HiRes qw(gettimeofday usleep);
use URI::Escape;
use UUID;
use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file);
+use PVE::CGroup;
+use PVE::CpuSet;
use PVE::DataCenterConfig;
use PVE::Exception qw(raise raise_param_exc);
+use PVE::Format qw(render_duration render_bytes);
use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
+use PVE::HA::Config;
+use PVE::Mapping::PCI;
+use PVE::Mapping::USB;
use PVE::INotify;
use PVE::JSONSchema qw(get_standard_option parse_property_string);
use PVE::ProcFSTools;
+use PVE::PBSClient;
+use PVE::RESTEnvironment qw(log_warn);
use PVE::RPCEnvironment;
use PVE::Storage;
use PVE::SysFSTools;
use PVE::QMPClient;
use PVE::QemuConfig;
-use PVE::QemuServer::Helpers qw(min_version config_aware_timeout);
+use PVE::QemuServer::Helpers qw(config_aware_timeout min_version windows_version);
use PVE::QemuServer::Cloudinit;
-use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
-use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom parse_drive print_drive);
+use PVE::QemuServer::CGroup;
+use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options get_cpu_bitness is_native_arch);
+use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom drive_is_read_only parse_drive print_drive);
use PVE::QemuServer::Machine;
-use PVE::QemuServer::Memory;
+use PVE::QemuServer::Memory qw(get_current_memory);
use PVE::QemuServer::Monitor qw(mon_cmd);
use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
-use PVE::QemuServer::USB qw(parse_usb_device);
+use PVE::QemuServer::QMPHelpers qw(qemu_deviceadd qemu_devicedel qemu_objectadd qemu_objectdel);
+use PVE::QemuServer::USB;
my $have_sdn;
eval {
require PVE::Network::SDN::Zones;
+ require PVE::Network::SDN::Vnets;
$have_sdn = 1;
};
my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
my $OVMF = {
- x86_64 => [
- "$EDK2_FW_BASE/OVMF_CODE.fd",
- "$EDK2_FW_BASE/OVMF_VARS.fd"
- ],
- aarch64 => [
- "$EDK2_FW_BASE/AAVMF_CODE.fd",
- "$EDK2_FW_BASE/AAVMF_VARS.fd"
- ],
+ x86_64 => {
+ '4m-no-smm' => [
+ "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
+ "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
+ ],
+ '4m-no-smm-ms' => [
+ "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
+ "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
+ ],
+ '4m' => [
+ "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
+ "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
+ ],
+ '4m-ms' => [
+ "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
+ "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
+ ],
+ # FIXME: These are legacy 2MB-sized images that modern OVMF doesn't supports to build
+ # anymore. how can we deperacate this sanely without breaking existing instances, or using
+ # older backups and snapshot?
+ default => [
+ "$EDK2_FW_BASE/OVMF_CODE.fd",
+ "$EDK2_FW_BASE/OVMF_VARS.fd",
+ ],
+ },
+ aarch64 => {
+ default => [
+ "$EDK2_FW_BASE/AAVMF_CODE.fd",
+ "$EDK2_FW_BASE/AAVMF_VARS.fd",
+ ],
+ },
};
my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
-# Note about locking: we use flock on the config file protect
-# against concurent actions.
-# Aditionaly, we have a 'lock' setting in the config file. This
-# can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
-# allowed when such lock is set. But you can ignore this kind of
-# lock with the --skiplock flag.
+# Note about locking: we use flock on the config file protect against concurent actions.
+# Aditionaly, we have a 'lock' setting in the config file. This can be set to 'migrate',
+# 'backup', 'snapshot' or 'rollback'. Most actions are not allowed when such lock is set.
+# But you can ignore this kind of lock with the --skiplock flag.
-cfs_register_file('/qemu-server/',
- \&parse_vm_config,
- \&write_vm_config);
+cfs_register_file(
+ '/qemu-server/',
+ \&parse_vm_config,
+ \&write_vm_config
+);
PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
description => "Some command save/restore state from this location.",
optional => 1,
});
-PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
- description => "Specifies the Qemu machine type.",
- type => 'string',
- pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
- maxLength => 40,
- optional => 1,
-});
-
-
-sub map_storage {
- my ($map, $source) = @_;
-
- return $source if !defined($map);
-
- return $map->{entries}->{$source}
- if $map->{entries} && defined($map->{entries}->{$source});
-
- return $map->{default} if $map->{default};
-
- # identity (fallback)
- return $source;
-}
-
-PVE::JSONSchema::register_standard_option('pve-targetstorage', {
- description => "Mapping from source to target storages. Providing only a single storage ID maps all source storages to that storage. Providing the special value '1' will map each source storage to itself.",
- type => 'string',
- format => 'storagepair-list',
- optional => 1,
-});
-
-#no warnings 'redefine';
-
-sub cgroups_write {
- my ($controller, $vmid, $option, $value) = @_;
-
- my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
- PVE::ProcFSTools::write_proc_entry($path, $value);
-
-}
-
+# FIXME: remove in favor of just using the INotify one, it's cached there exactly the same way
my $nodename_cache;
sub nodename {
$nodename_cache //= PVE::INotify::nodename();
my $agent_fmt = {
enabled => {
- description => "Enable/disable Qemu GuestAgent.",
+ description => "Enable/disable communication with a QEMU Guest Agent (QGA) running in the VM.",
type => 'boolean',
default => 0,
default_key => 1,
},
fstrim_cloned_disks => {
- description => "Run fstrim after cloning/moving a disk.",
+ description => "Run fstrim after moving a disk or migrating the VM.",
+ type => 'boolean',
+ optional => 1,
+ default => 0,
+ },
+ 'freeze-fs-on-backup' => {
+ description => "Freeze/thaw guest filesystems on backup for consistency.",
type => 'boolean',
optional => 1,
- default => 0
+ default => 1,
},
type => {
description => "Select the agent type",
default => 'std',
optional => 1,
default_key => 1,
- enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
+ enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio virtio-gl vmware)],
},
memory => {
description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
minimum => 4,
maximum => 512,
},
+ clipboard => {
+ description => 'Enable a specific clipboard. If not set, depending on the display type the'
+ .' SPICE one will be added. Migration with VNC clipboard is not yet supported!',
+ type => 'string',
+ enum => ['vnc'],
+ optional => 1,
+ },
};
my $ivshmem_fmt = {
},
driver => {
type => 'string',
- enum => ['spice'],
+ enum => ['spice', 'none'],
default => 'spice',
optional => 1,
description => "Driver backend for the audio device."
type => 'string',
enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
default_key => 1,
- description => "The file on the host to gather entropy from. In most"
- . " cases /dev/urandom should be preferred over /dev/random"
- . " to avoid entropy-starvation issues on the host. Using"
- . " urandom does *not* decrease security in any meaningful"
- . " way, as it's still seeded from real entropy, and the"
- . " bytes provided will most likely be mixed with real"
- . " entropy on the guest as well. /dev/hwrng can be used"
- . " to pass through a hardware RNG from the host.",
+ description => "The file on the host to gather entropy from. In most cases '/dev/urandom'"
+ ." should be preferred over '/dev/random' to avoid entropy-starvation issues on the"
+ ." host. Using urandom does *not* decrease security in any meaningful way, as it's"
+ ." still seeded from real entropy, and the bytes provided will most likely be mixed"
+ ." with real entropy on the guest as well. '/dev/hwrng' can be used to pass through"
+ ." a hardware RNG from the host.",
},
max_bytes => {
type => 'integer',
- description => "Maximum bytes of entropy injected into the guest every"
- . " 'period' milliseconds. Prefer a lower value when using"
- . " /dev/random as source. Use 0 to disable limiting"
- . " (potentially dangerous!).",
+ description => "Maximum bytes of entropy allowed to get injected into the guest every"
+ ." 'period' milliseconds. Prefer a lower value when using '/dev/random' as source. Use"
+ ." `0` to disable limiting (potentially dangerous!).",
optional => 1,
- # default is 1 KiB/s, provides enough entropy to the guest to avoid
- # boot-starvation issues (e.g. systemd etc...) while allowing no chance
- # of overwhelming the host, provided we're reading from /dev/urandom
+ # default is 1 KiB/s, provides enough entropy to the guest to avoid boot-starvation issues
+ # (e.g. systemd etc...) while allowing no chance of overwhelming the host, provided we're
+ # reading from /dev/urandom
default => 1024,
},
period => {
type => 'integer',
- description => "Every 'period' milliseconds the entropy-injection quota"
- . " is reset, allowing the guest to retrieve another"
- . " 'max_bytes' of entropy.",
+ description => "Every 'period' milliseconds the entropy-injection quota is reset, allowing"
+ ." the guest to retrieve another 'max_bytes' of entropy.",
optional => 1,
default => 1000,
},
};
+my $meta_info_fmt = {
+ 'ctime' => {
+ type => 'integer',
+ description => "The guest creation timestamp as UNIX epoch time",
+ minimum => 0,
+ optional => 1,
+ },
+ 'creation-qemu' => {
+ type => 'string',
+ description => "The QEMU (machine) version from the time this VM was created.",
+ pattern => '\d+(\.\d+)+',
+ optional => 1,
+ },
+};
+
my $confdesc = {
onboot => {
optional => 1,
default => 0,
},
hotplug => {
- optional => 1,
- type => 'string', format => 'pve-hotplug-features',
- description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
+ optional => 1,
+ type => 'string', format => 'pve-hotplug-features',
+ description => "Selectively enable hotplug features. This is a comma separated list of"
+ ." hotplug features: 'network', 'disk', 'cpu', 'memory', 'usb' and 'cloudinit'. Use '0' to disable"
+ ." hotplug completely. Using '1' as value is an alias for the default `network,disk,usb`."
+ ." USB hotplugging is possible for guests with machine version >= 7.1 and ostype l26 or"
+ ." windows > 7.",
default => 'network,disk,usb',
},
reboot => {
optional => 1,
type => 'number',
description => "Limit of CPU usage.",
- verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
+ verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has"
+ ." total of '2' CPU time. Value '0' indicates no CPU limit.",
minimum => 0,
maximum => 128,
- default => 0,
+ default => 0,
},
cpuunits => {
optional => 1,
type => 'integer',
- description => "CPU weight for a VM.",
- verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
- minimum => 2,
+ description => "CPU weight for a VM, will be clamped to [1, 10000] in cgroup v2.",
+ verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler."
+ ." The larger the number is, the more CPU time this VM gets. Number is relative to"
+ ." weights of all the other running VMs.",
+ minimum => 1,
maximum => 262144,
- default => 1024,
+ default => 'cgroup v1: 1024, cgroup v2: 100',
},
memory => {
optional => 1,
- type => 'integer',
- description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
- minimum => 16,
- default => 512,
+ type => 'string',
+ description => "Memory properties.",
+ format => $PVE::QemuServer::Memory::memory_fmt
},
balloon => {
- optional => 1,
- type => 'integer',
- description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
+ optional => 1,
+ type => 'integer',
+ description => "Amount of target RAM for the VM in MiB. Using zero disables the ballon driver.",
minimum => 0,
},
shares => {
- optional => 1,
- type => 'integer',
- description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
+ optional => 1,
+ type => 'integer',
+ description => "Amount of memory shares for auto-ballooning. The larger the number is, the"
+ ." more memory this VM gets. Number is relative to weights of all other running VMs."
+ ." Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
minimum => 0,
maximum => 50000,
default => 1000,
keyboard => {
optional => 1,
type => 'string',
- description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
- "It should not be necessary to set it.",
+ description => "Keyboard layout for VNC server. This option is generally not required and"
+ ." is often better handled from within the guest OS.",
enum => PVE::Tools::kvmkeymaplist(),
default => undef,
},
description => {
optional => 1,
type => 'string',
- description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
+ description => "Description for the VM. Shown in the web-interface VM's summary."
+ ." This is saved as comment inside the configuration file.",
+ maxLength => 1024 * 8,
},
ostype => {
optional => 1,
type => 'string',
- enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
+ # NOTE: When extending, also consider extending `%guest_types` in `Import/ESXi.pm`.
+ enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 win11 l24 l26 solaris)],
description => "Specify guest operating system.",
verbose_description => <<EODESC,
Specify guest operating system. This is used to enable special
wvista;; Microsoft Windows Vista
win7;; Microsoft Windows 7
win8;; Microsoft Windows 8/2012/2012r2
-win10;; Microsoft Windows 10/2016
+win10;; Microsoft Windows 10/2016/2019
+win11;; Microsoft Windows 11/2022/2025
l24;; Linux 2.4 Kernel
-l26;; Linux 2.6 - 5.X Kernel
+l26;; Linux 2.6 - 6.X Kernel
solaris;; Solaris/OpenSolaris/OpenIndiania kernel
EODESC
},
boot => {
optional => 1,
type => 'string', format => 'pve-qm-boot',
- description => "Specify guest boot order. Use with 'order=', usage with"
- . " no key or 'legacy=' is deprecated.",
+ description => "Specify guest boot order. Use the 'order=' sub-property as usage with no"
+ ." key or 'legacy=' is deprecated.",
},
bootdisk => {
optional => 1,
},
agent => {
optional => 1,
- description => "Enable/disable Qemu GuestAgent and its properties.",
+ description => "Enable/disable communication with the QEMU Guest Agent and its properties.",
type => 'string',
format => $agent_fmt,
},
localtime => {
optional => 1,
type => 'boolean',
- description => "Set the real time clock to local time. This is enabled by default if ostype"
- ." indicates a Microsoft OS.",
+ description => "Set the real time clock (RTC) to local time. This is enabled by default if"
+ ." the `ostype` indicates a Microsoft Windows OS.",
},
freeze => {
optional => 1,
verbose_description => <<EODESCR,
Arbitrary arguments passed to kvm, for example:
-args: -no-reboot -no-hpet
+args: -no-reboot -smbios 'type=0,vendor=FOO'
NOTE: this option is for experts only.
EODESCR
description => "Configure a VirtIO-based Random Number Generator.",
optional => 1,
},
+ meta => {
+ type => 'string',
+ format => $meta_info_fmt,
+ description => "Some (read-only) meta-information about this guest.",
+ optional => 1,
+ },
+ affinity => {
+ type => 'string', format => 'pve-cpuset',
+ description => "List of host cores used to execute guest processes, for example: 0,5,8-11",
+ optional => 1,
+ },
};
my $cicustom_fmt = {
network => {
type => 'string',
optional => 1,
- description => 'Specify a custom file containing all network data passed to the VM via'
- .' cloud-init.',
+ description => 'To pass a custom file containing all network data to the VM via cloud-init.',
format => 'pve-volume-id',
format_description => 'volume',
},
user => {
type => 'string',
optional => 1,
- description => 'Specify a custom file containing all user data passed to the VM via'
- .' cloud-init.',
+ description => 'To pass a custom file containing all user data to the VM via cloud-init.',
+ format => 'pve-volume-id',
+ format_description => 'volume',
+ },
+ vendor => {
+ type => 'string',
+ optional => 1,
+ description => 'To pass a custom file containing all vendor data to the VM via cloud-init.',
format => 'pve-volume-id',
format_description => 'volume',
},
};
PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
+# any new option might need to be added to $cloudinitoptions in PVE::API2::Qemu
my $confdesc_cloudinit = {
citype => {
optional => 1,
description => 'Specifies the cloud-init configuration format. The default depends on the'
.' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
.' and `configdrive2` for windows.',
- enum => ['configdrive2', 'nocloud'],
+ enum => ['configdrive2', 'nocloud', 'opennebula'],
},
ciuser => {
optional => 1,
.' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
.' support hashed passwords.',
},
+ ciupgrade => {
+ optional => 1,
+ type => 'boolean',
+ description => 'cloud-init: do an automatic package upgrade after the first boot.',
+ default => 1,
+ },
cicustom => {
optional => 1,
type => 'string',
searchdomain => {
optional => 1,
type => 'string',
- description => "cloud-init: Sets DNS search domains for a container. Create will'
+ description => 'cloud-init: Sets DNS search domains for a container. Create will'
.' automatically use the setting from the host if neither searchdomain nor nameserver'
- .' are set.",
+ .' are set.',
},
nameserver => {
optional => 1,
type => 'string', format => 'address-list',
- description => "cloud-init: Sets DNS server IP address for a container. Create will'
+ description => 'cloud-init: Sets DNS server IP address for a container. Create will'
.' automatically use the setting from the host if neither searchdomain nor nameserver'
- .' are set.",
+ .' are set.',
},
sshkeys => {
optional => 1,
PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
}
-my $MAX_USB_DEVICES = 5;
my $MAX_NETS = 32;
my $MAX_SERIAL_PORTS = 4;
my $MAX_PARALLEL_PORTS = 3;
-my $MAX_NUMA = 8;
-my $numa_fmt = {
- cpus => {
- type => "string",
- pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
- description => "CPUs accessing this NUMA node.",
- format_description => "id[-id];...",
- },
- memory => {
- type => "number",
- description => "Amount of memory this NUMA node provides.",
- optional => 1,
- },
- hostnodes => {
- type => "string",
- pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
- description => "Host NUMA nodes to use.",
- format_description => "id[-id];...",
- optional => 1,
- },
- policy => {
- type => 'string',
- enum => [qw(preferred bind interleave)],
- description => "NUMA allocation policy.",
- optional => 1,
- },
-};
-PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
-my $numadesc = {
- optional => 1,
- type => 'string', format => $numa_fmt,
- description => "NUMA topology.",
-};
-PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
-
-for (my $i = 0; $i < $MAX_NUMA; $i++) {
- $confdesc->{"numa$i"} = $numadesc;
-}
-
-my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000', 'pcnet', 'virtio',
- 'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
- 'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
+for (my $i = 0; $i < $PVE::QemuServer::Memory::MAX_NUMA; $i++) {
+ $confdesc->{"numa$i"} = $PVE::QemuServer::Memory::numadesc;
+}
+
+my $nic_model_list = [
+ 'e1000',
+ 'e1000-82540em',
+ 'e1000-82544gc',
+ 'e1000-82545em',
+ 'e1000e',
+ 'i82551',
+ 'i82557b',
+ 'i82559er',
+ 'ne2k_isa',
+ 'ne2k_pci',
+ 'pcnet',
+ 'rtl8139',
+ 'virtio',
+ 'vmxnet3',
+];
my $nic_model_list_txt = join(' ', sort @$nic_model_list);
my $net_fmt_bridge_descr = <<__EOD__;
default_key => 1,
},
(map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
- bridge => {
- type => 'string',
+ bridge => get_standard_option('pve-bridge-id', {
description => $net_fmt_bridge_descr,
- format_description => 'bridge',
- pattern => '[-_.\w\d]+',
optional => 1,
- },
+ }),
queues => {
type => 'integer',
- minimum => 0, maximum => 16,
+ minimum => 0, maximum => 64,
description => 'Number of packet queues to be used on the device.',
optional => 1,
},
The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
gateway should be provided.
-For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
+For IPv6 the special string 'auto' can be used to use stateless autoconfiguration. This requires
+cloud-init 19.4 or newer.
If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
dhcp on IPv4.
$confdesc->{$key} = $confdesc_cloudinit->{$key};
}
-PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
-sub verify_volume_id_or_qm_path {
- my ($volid, $noerr) = @_;
+PVE::JSONSchema::register_format('pve-cpuset', \&pve_verify_cpuset);
+sub pve_verify_cpuset {
+ my ($set_text, $noerr) = @_;
- if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
- return $volid;
- }
+ my ($count, $members) = eval { PVE::CpuSet::parse_cpuset($set_text) };
- # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
- $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
if ($@) {
return if $noerr;
- die $@;
+ die "unable to parse cpuset option\n";
}
- return $volid;
+
+ return PVE::CpuSet->new($members)->short_string();
}
-my $usb_fmt = {
- host => {
- default_key => 1,
- type => 'string', format => 'pve-qm-usb-device',
- format_description => 'HOSTUSBDEVICE|spice',
- description => <<EODESCR,
-The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
+PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
+sub verify_volume_id_or_qm_path {
+ my ($volid, $noerr) = @_;
- 'bus-port(.port)*' (decimal numbers) or
- 'vendor_id:product_id' (hexadeciaml numbers) or
- 'spice'
+ return $volid if $volid eq 'none' || $volid eq 'cdrom';
-You can use the 'lsusb -t' command to list existing usb devices.
+ return verify_volume_id_or_absolute_path($volid, $noerr);
+}
-NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
-machines - use with special care.
+PVE::JSONSchema::register_format('pve-volume-id-or-absolute-path', \&verify_volume_id_or_absolute_path);
+sub verify_volume_id_or_absolute_path {
+ my ($volid, $noerr) = @_;
-The value 'spice' can be used to add a usb redirection devices for spice.
-EODESCR
- },
- usb3 => {
- optional => 1,
- type => 'boolean',
- description => "Specifies whether if given host option is a USB3 device or port.",
- default => 0,
- },
-};
+ return $volid if $volid =~ m|^/|;
-my $usbdesc = {
- optional => 1,
- type => 'string', format => $usb_fmt,
- description => "Configure an USB device (n is 0 to 4).",
-};
-PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
+ $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
+ if ($@) {
+ return if $noerr;
+ die $@;
+ }
+ return $volid;
+}
my $serialdesc = {
optional => 1,
$confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
}
-for (my $i = 0; $i < $MAX_USB_DEVICES; $i++) {
- $confdesc->{"usb$i"} = $usbdesc;
+for (my $i = 0; $i < $PVE::QemuServer::USB::MAX_USB_DEVICES; $i++) {
+ $confdesc->{"usb$i"} = $PVE::QemuServer::USB::usbdesc;
}
my $boot_fmt = {
sub verify_bootdev {
my ($dev, $noerr) = @_;
- return $dev if PVE::QemuServer::Drive::is_valid_drivename($dev) && $dev !~ m/^efidisk/;
+ my $special = $dev =~ m/^efidisk/ || $dev =~ m/^tpmstate/;
+ return $dev if PVE::QemuServer::Drive::is_valid_drivename($dev) && !$special;
my $check = sub {
my ($base) = @_;
my sub extract_version {
my ($machine_type, $version) = @_;
$version = kvm_user_version() if !defined($version);
- PVE::QemuServer::Machine::extract_version($machine_type, $version)
+ return PVE::QemuServer::Machine::extract_version($machine_type, $version)
}
sub kernel_has_vhost_net {
my $cdrom_path;
sub get_cdrom_path {
- return $cdrom_path if $cdrom_path;
+ return $cdrom_path if defined($cdrom_path);
- return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
- return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
- return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
+ $cdrom_path = first { -l $_ } map { "/dev/cdrom$_" } ('', '1', '2');
+
+ if (!defined($cdrom_path)) {
+ log_warn("no physical CD-ROM available, ignoring");
+ $cdrom_path = '';
+ }
+
+ return $cdrom_path;
}
sub get_iso_path {
$data = $confdesc->{hotplug}->{default} if $data eq '1';
foreach my $feature (PVE::Tools::split_list($data)) {
- if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
+ if ($feature =~ m/^(network|disk|cpu|memory|usb|cloudinit)$/) {
$res->{$1} = 1;
} else {
die "invalid hotplug feature '$feature'\n";
die "unable to parse hotplug option\n";
}
-sub scsi_inquiry {
- my($fh, $noerr) = @_;
-
- my $SG_IO = 0x2285;
- my $SG_GET_VERSION_NUM = 0x2282;
-
- my $versionbuf = "\x00" x 8;
- my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
- if (!$ret) {
- die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
- return;
- }
- my $version = unpack("I", $versionbuf);
- if ($version < 30000) {
- die "scsi generic interface too old\n" if !$noerr;
- return;
- }
-
- my $buf = "\x00" x 36;
- my $sensebuf = "\x00" x 8;
- my $cmd = pack("C x3 C x1", 0x12, 36);
-
- # see /usr/include/scsi/sg.h
- my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
-
- my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
- length($sensebuf), 0, length($buf), $buf,
- $cmd, $sensebuf, 6000);
+sub assert_clipboard_config {
+ my ($vga) = @_;
- $ret = ioctl($fh, $SG_IO, $packet);
- if (!$ret) {
- die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
- return;
- }
+ my $clipboard_regex = qr/^(std|cirrus|vmware|virtio|qxl)/;
- my @res = unpack($sg_io_hdr_t, $packet);
- if ($res[17] || $res[18]) {
- die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
- return;
+ if (
+ $vga->{'clipboard'}
+ && $vga->{'clipboard'} eq 'vnc'
+ && $vga->{type}
+ && $vga->{type} !~ $clipboard_regex
+ ) {
+ die "vga type $vga->{type} is not compatible with VNC clipboard\n";
}
-
- my $res = {};
- (my $byte0, my $byte1, $res->{vendor},
- $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
-
- $res->{removable} = $byte1 & 128 ? 1 : 0;
- $res->{type} = $byte0 & 31;
-
- return $res;
-}
-
-sub path_is_scsi {
- my ($path) = @_;
-
- my $fh = IO::File->new("+<$path") || return;
- my $res = scsi_inquiry($fh, 1);
- close($fh);
-
- return $res;
}
sub print_tabletdevice_full {
# we use uhci for old VMs because tablet driver was buggy in older qemu
my $usbbus;
- if (PVE::QemuServer::Machine::machine_type_is_q35($conf) || $arch eq 'aarch64') {
+ if ($q35 || $arch eq 'aarch64') {
$usbbus = 'ehci';
} else {
$usbbus = 'uhci';
}
sub print_keyboarddevice_full {
- my ($conf, $arch, $machine) = @_;
+ my ($conf, $arch) = @_;
return if $arch ne 'aarch64';
return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
}
+my sub get_drive_id {
+ my ($drive) = @_;
+ return "$drive->{interface}$drive->{index}";
+}
+
sub print_drivedevice_full {
my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
my $device = '';
my $maxdev = 0;
- my $drive_id = "$drive->{interface}$drive->{index}";
+ my $drive_id = get_drive_id($drive);
if ($drive->{interface} eq 'virtio') {
my $pciaddr = print_pci_addr("$drive_id", $bridges, $arch, $machine_type);
$device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
my $unit = $drive->{index} % $maxdev;
- my $devicetype = 'hd';
- my $path = '';
- if (drive_is_cdrom($drive)) {
- $devicetype = 'cd';
- } else {
- if ($drive->{file} =~ m|^/|) {
- $path = $drive->{file};
- if (my $info = path_is_scsi($path)) {
- if ($info->{type} == 0 && $drive->{scsiblock}) {
- $devicetype = 'block';
- } elsif ($info->{type} == 1) { # tape
- $devicetype = 'generic';
- }
- }
- } else {
- $path = PVE::Storage::path($storecfg, $drive->{file});
- }
- # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
- my $version = extract_version($machine_type, kvm_user_version());
- if ($path =~ m/^iscsi\:\/\// &&
- !min_version($version, 4, 1)) {
- $devicetype = 'generic';
- }
- }
+ my $machine_version = extract_version($machine_type, kvm_user_version());
+ my $device_type = PVE::QemuServer::Drive::get_scsi_device_type(
+ $drive, $storecfg, $machine_version);
- if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
- $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
+ if (!$conf->{scsihw} || $conf->{scsihw} =~ m/^lsi/ || $conf->{scsihw} eq 'pvscsi') {
+ $device = "scsi-$device_type,bus=$controller_prefix$controller.0,scsi-id=$unit";
} else {
- $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
+ $device = "scsi-$device_type,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
.",lun=$drive->{index}";
}
$device .= ",drive=drive-$drive_id,id=$drive_id";
- if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
+ if ($drive->{ssd} && ($device_type eq 'block' || $device_type eq 'hd')) {
$device .= ",rotation_rate=1";
}
$device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
+ # only scsi-hd and scsi-cd support passing vendor and product information
+ if ($device_type eq 'hd' || $device_type eq 'cd') {
+ if (my $vendor = $drive->{vendor}) {
+ $device .= ",vendor=$vendor";
+ }
+ if (my $product = $drive->{product}) {
+ $device .= ",product=$product";
+ }
+ }
+
} elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
my $controller = int($drive->{index} / $maxdev);
my $unit = $drive->{index} % $maxdev;
- my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
- $device = "ide-$devicetype";
+ # machine type q35 only supports unit=0 for IDE rather than 2 units. This wasn't handled
+ # correctly before, so e.g. index=2 was mapped to controller=1,unit=0 rather than
+ # controller=2,unit=0. Note that odd indices never worked, as they would be mapped to
+ # unit=1, so to keep backwards compat for migration, it suffices to keep even ones as they
+ # were before. Move odd ones up by 2 where they don't clash.
+ if (PVE::QemuServer::Machine::machine_type_is_q35($conf) && $drive->{interface} eq 'ide') {
+ $controller += 2 * ($unit % 2);
+ $unit = 0;
+ }
+
+ my $device_type = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
+
+ $device = "ide-$device_type";
if ($drive->{interface} eq 'ide') {
$device .= ",bus=ide.$controller,unit=$unit";
} else {
}
$device .= ",drive=drive-$drive_id,id=$drive_id";
- if ($devicetype eq 'hd') {
+ if ($device_type eq 'hd') {
if (my $model = $drive->{model}) {
$model = URI::Escape::uri_unescape($model);
$device .= ",model=$model";
return $initiator;
}
+my sub storage_allows_io_uring_default {
+ my ($scfg, $cache_direct) = @_;
+
+ # io_uring with cache mode writeback or writethrough on krbd will hang...
+ return if $scfg && $scfg->{type} eq 'rbd' && $scfg->{krbd} && !$cache_direct;
+
+ # io_uring with cache mode writeback or writethrough on LVM will hang, without cache only
+ # sometimes, just plain disable...
+ return if $scfg && $scfg->{type} eq 'lvm';
+
+ # io_uring causes problems when used with CIFS since kernel 5.15
+ # Some discussion: https://www.spinics.net/lists/linux-cifs/msg26734.html
+ return if $scfg && $scfg->{type} eq 'cifs';
+
+ return 1;
+}
+
+my sub drive_uses_cache_direct {
+ my ($drive, $scfg) = @_;
+
+ my $cache_direct = 0;
+
+ if (my $cache = $drive->{cache}) {
+ $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
+ } elsif (!drive_is_cdrom($drive) && !($scfg && $scfg->{type} eq 'btrfs' && !$scfg->{nocow})) {
+ $cache_direct = 1;
+ }
+
+ return $cache_direct;
+}
+
sub print_drive_commandline_full {
- my ($storecfg, $vmid, $drive) = @_;
+ my ($storecfg, $vmid, $drive, $live_restore_name, $io_uring) = @_;
my $path;
my $volid = $drive->{file};
- my $format;
+ my $format = $drive->{format};
+ my $drive_id = get_drive_id($drive);
+
+ my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
+ my $scfg = $storeid ? PVE::Storage::storage_config($storecfg, $storeid) : undef;
if (drive_is_cdrom($drive)) {
$path = get_iso_path($storecfg, $vmid, $volid);
+ die "$drive_id: cannot back cdrom drive with a live restore image\n" if $live_restore_name;
} else {
- my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
if ($storeid) {
$path = PVE::Storage::path($storecfg, $volid);
- my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
- $format = qemu_img_format($scfg, $volname);
+ $format //= qemu_img_format($scfg, $volname);
} else {
$path = $volid;
- $format = "raw";
+ $format //= "raw";
}
}
+ my $is_rbd = $path =~ m/^rbd:/;
+
my $opts = '';
- my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
+ my @qemu_drive_options = qw(heads secs cyls trans media cache rerror werror aio discard);
foreach my $o (@qemu_drive_options) {
$opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
}
$opts .= ",snapshot=$v";
}
+ if (defined($drive->{ro})) { # ro maps to QEMUs `readonly`, which accepts `on` or `off` only
+ $opts .= ",readonly=" . ($drive->{ro} ? 'on' : 'off');
+ }
+
foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
my ($dir, $qmpname) = @$type;
if (my $v = $drive->{"mbps$dir"}) {
}
}
- $opts .= ",format=$format" if $format && !$drive->{format};
+ if ($live_restore_name) {
+ $format = "rbd" if $is_rbd;
+ die "$drive_id: Proxmox Backup Server backed drive cannot auto-detect the format\n"
+ if !$format;
+ $opts .= ",format=alloc-track,file.driver=$format";
+ } elsif ($format) {
+ $opts .= ",format=$format";
+ }
- my $cache_direct = 0;
+ my $cache_direct = drive_uses_cache_direct($drive, $scfg);
- if (my $cache = $drive->{cache}) {
- $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
- } elsif (!drive_is_cdrom($drive)) {
- $opts .= ",cache=none";
- $cache_direct = 1;
- }
+ $opts .= ",cache=none" if !$drive->{cache} && $cache_direct;
- # aio native works only with O_DIRECT
if (!$drive->{aio}) {
- if($cache_direct) {
- $opts .= ",aio=native";
+ if ($io_uring && storage_allows_io_uring_default($scfg, $cache_direct)) {
+ # io_uring supports all cache modes
+ $opts .= ",aio=io_uring";
} else {
- $opts .= ",aio=threads";
+ # aio native works only with O_DIRECT
+ if($cache_direct) {
+ $opts .= ",aio=native";
+ } else {
+ $opts .= ",aio=threads";
+ }
}
}
# This used to be our default with discard not being specified:
$detectzeroes = 'on';
}
- $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
+
+ # note: 'detect-zeroes' works per blockdev and we want it to persist
+ # after the alloc-track is removed, so put it on 'file' directly
+ my $dz_param = $live_restore_name ? "file.detect-zeroes" : "detect-zeroes";
+ $opts .= ",$dz_param=$detectzeroes" if $detectzeroes;
+ }
+
+ if ($live_restore_name) {
+ $opts .= ",backing=$live_restore_name";
+ $opts .= ",auto-remove=on";
}
- my $pathinfo = $path ? "file=$path," : '';
+ # my $file_param = $live_restore_name ? "file.file.filename" : "file";
+ my $file_param = "file";
+ if ($live_restore_name) {
+ # non-rbd drivers require the underlying file to be a seperate block
+ # node, so add a second .file indirection
+ $file_param .= ".file" if !$is_rbd;
+ $file_param .= ".filename";
+ }
+ my $pathinfo = $path ? "$file_param=$path," : '';
return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
}
+sub print_pbs_blockdev {
+ my ($pbs_conf, $pbs_name) = @_;
+ my $blockdev = "driver=pbs,node-name=$pbs_name,read-only=on";
+ $blockdev .= ",repository=$pbs_conf->{repository}";
+ $blockdev .= ",namespace=$pbs_conf->{namespace}" if $pbs_conf->{namespace};
+ $blockdev .= ",snapshot=$pbs_conf->{snapshot}";
+ $blockdev .= ",archive=$pbs_conf->{archive}";
+ $blockdev .= ",keyfile=$pbs_conf->{keyfile}" if $pbs_conf->{keyfile};
+ return $blockdev;
+}
+
sub print_netdevice_full {
- my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
+ my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type, $machine_version) = @_;
my $device = $net->{model};
if ($net->{model} eq 'virtio') {
# and out of each queue plus one config interrupt and control vector queue
my $vectors = $net->{queues} * 2 + 2;
$tmpstr .= ",vectors=$vectors,mq=on";
+ if (min_version($machine_version, 7, 1)) {
+ $tmpstr .= ",packed=on";
+ }
+ }
+
+ if (min_version($machine_version, 7, 1) && $net->{model} eq 'virtio'){
+ $tmpstr .= ",rx_queue_size=1024,tx_queue_size=256";
}
+
$tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
if (my $mtu = $net->{mtu}) {
$romfile = 'pxe-virtio.rom';
} elsif ($device eq 'e1000') {
$romfile = 'pxe-e1000.rom';
+ } elsif ($device eq 'e1000e') {
+ $romfile = 'pxe-e1000e.rom';
} elsif ($device eq 'ne2k') {
$romfile = 'pxe-ne2k_pci.rom';
} elsif ($device eq 'pcnet') {
if length($ifname) >= 16;
my $vhostparam = '';
- if (is_native($arch)) {
+ if (is_native_arch($arch)) {
$vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
}
'std' => 'VGA',
'vmware' => 'vmware-svga',
'virtio' => 'virtio-vga',
+ 'virtio-gl' => 'virtio-vga-gl',
};
sub print_vga_device {
}
}
- die "no devicetype for $vga->{type}\n" if !$type;
+ die "no device-type for $vga->{type}\n" if !$type;
my $memory = "";
if ($vgamem_mb) {
- if ($vga->{type} eq 'virtio') {
+ if ($vga->{type} =~ /^virtio/) {
my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
$memory = ",max_hostmem=$bytes";
} elsif ($qxlnum) {
my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
my $vgaid = "vga" . ($id // '');
my $pciaddr;
-
if ($q35 && $vgaid eq 'vga') {
# the first display uses pcie.0 bus on q35 machines
$pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
$pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
}
- return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
-}
+ if ($vga->{type} eq 'virtio-gl') {
+ my $base = '/usr/lib/x86_64-linux-gnu/lib';
+ die "missing libraries for '$vga->{type}' detected! Please install 'libgl1' and 'libegl1'\n"
+ if !-e "${base}EGL.so.1" || !-e "${base}GL.so.1";
-sub parse_number_sets {
- my ($set) = @_;
- my $res = [];
- foreach my $part (split(/;/, $set)) {
- if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
- die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
- push @$res, [ $1, $2 ];
- } else {
- die "invalid range: $part\n";
- }
+ die "no DRM render node detected (/dev/dri/renderD*), no GPU? - needed for '$vga->{type}' display\n"
+ if !PVE::Tools::dir_glob_regex('/dev/dri/', "renderD.*");
}
- return $res;
-}
-sub parse_numa {
- my ($data) = @_;
-
- my $res = parse_property_string($numa_fmt, $data);
- $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
- $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
- return $res;
+ return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
}
# netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
sub parse_net {
- my ($data) = @_;
+ my ($data, $disable_mac_autogen) = @_;
my $res = eval { parse_property_string($net_fmt, $data) };
if ($@) {
warn $@;
return;
}
- if (!defined($res->{macaddr})) {
+ if (!defined($res->{macaddr}) && !$disable_mac_autogen) {
my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
$res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
}
if (drive_is_cloudinit($drive)) {
eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
warn $@ if $@;
+ delete $conf->{cloudinit};
} elsif (!drive_is_cdrom($drive)) {
my $volid = $drive->{file};
if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
}
sub parse_guest_agent {
- my ($value) = @_;
+ my ($conf) = @_;
- return {} if !defined($value->{agent});
+ return {} if !defined($conf->{agent});
- my $res = eval { parse_property_string($agent_fmt, $value->{agent}) };
+ my $res = eval { parse_property_string($agent_fmt, $conf->{agent}) };
warn $@ if $@;
# if the agent is disabled ignore the other potentially set properties
return $res;
}
+sub get_qga_key {
+ my ($conf, $key) = @_;
+ return undef if !defined($conf->{agent});
+
+ my $agent = parse_guest_agent($conf);
+ return $agent->{$key};
+}
+
sub parse_vga {
my ($value) = @_;
return $res;
}
-PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
-sub verify_usb_device {
- my ($value, $noerr) = @_;
+sub parse_meta_info {
+ my ($value) = @_;
- return $value if parse_usb_device($value);
+ return if !$value;
- return if $noerr;
+ my $res = eval { parse_property_string($meta_info_fmt, $value) };
+ warn $@ if $@;
+ return $res;
+}
+
+sub new_meta_info_string {
+ my () = @_; # for now do not allow to override any value
+
+ return PVE::JSONSchema::print_property_string(
+ {
+ 'creation-qemu' => kvm_user_version(),
+ ctime => "". int(time()),
+ },
+ $meta_info_fmt
+ );
+}
+
+sub qemu_created_version_fixups {
+ my ($conf, $forcemachine, $kvmver) = @_;
- die "unable to parse usb device\n";
+ my $meta = parse_meta_info($conf->{meta}) // {};
+ my $forced_vers = PVE::QemuServer::Machine::extract_version($forcemachine);
+
+ # check if we need to apply some handling for VMs that always use the latest machine version but
+ # had a machine version transition happen that affected HW such that, e.g., an OS config change
+ # would be required (we do not want to pin machine version for non-windows OS type)
+ my $machine_conf = PVE::QemuServer::Machine::parse_machine($conf->{machine});
+ if (
+ (!defined($machine_conf->{type}) || $machine_conf->{type} =~ m/^(?:pc|q35|virt)$/) # non-versioned machine
+ && (!defined($meta->{'creation-qemu'}) || !min_version($meta->{'creation-qemu'}, 6, 1)) # created before 6.1
+ && (!$forced_vers || min_version($forced_vers, 6, 1)) # handle snapshot-rollback/migrations
+ && min_version($kvmver, 6, 1) # only need to apply the change since 6.1
+ ) {
+ my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
+ if ($q35 && $conf->{ostype} && $conf->{ostype} eq 'l26') {
+ # this changed to default-on in Q 6.1 for q35 machines, it will mess with PCI slot view
+ # and thus with the predictable interface naming of systemd
+ return ['-global', 'ICH9-LPC.acpi-pci-hotplug-with-bridge-support=off'];
+ }
+ }
+ return;
}
# add JSON properties for create and set function
sub json_config_properties {
- my $prop = shift;
+ my ($prop, $with_disk_alloc) = @_;
+
+ my $skip_json_config_opts = {
+ parent => 1,
+ snaptime => 1,
+ vmstate => 1,
+ runningmachine => 1,
+ runningcpu => 1,
+ meta => 1,
+ };
foreach my $opt (keys %$confdesc) {
- next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' ||
- $opt eq 'runningmachine' || $opt eq 'runningcpu';
- $prop->{$opt} = $confdesc->{$opt};
+ next if $skip_json_config_opts->{$opt};
+
+ if ($with_disk_alloc && is_valid_drivename($opt)) {
+ $prop->{$opt} = $PVE::QemuServer::Drive::drivedesc_hash_with_alloc->{$opt};
+ } else {
+ $prop->{$opt} = $confdesc->{$opt};
+ }
}
return $prop;
}
-# return copy of $confdesc_cloudinit to generate documentation
-sub cloudinit_config_properties {
-
- return dclone($confdesc_cloudinit);
-}
-
-sub check_type {
- my ($key, $value) = @_;
-
- die "unknown setting '$key'\n" if !$confdesc->{$key};
+# Properties that we can read from an OVF file
+sub json_ovf_properties {
+ my $prop = {};
- my $type = $confdesc->{$key}->{type};
+ for my $device (PVE::QemuServer::Drive::valid_drive_names()) {
+ $prop->{$device} = {
+ type => 'string',
+ format => 'pve-volume-id-or-absolute-path',
+ description => "Disk image that gets imported to $device",
+ optional => 1,
+ };
+ }
- if (!defined($value)) {
+ $prop->{cores} = {
+ type => 'integer',
+ description => "The number of CPU cores.",
+ optional => 1,
+ };
+ $prop->{memory} = {
+ type => 'integer',
+ description => "Amount of RAM for the VM in MB.",
+ optional => 1,
+ };
+ $prop->{name} = {
+ type => 'string',
+ description => "Name of the VM.",
+ optional => 1,
+ };
+
+ return $prop;
+}
+
+# return copy of $confdesc_cloudinit to generate documentation
+sub cloudinit_config_properties {
+
+ return dclone($confdesc_cloudinit);
+}
+
+sub cloudinit_pending_properties {
+ my $p = {
+ map { $_ => 1 } keys $confdesc_cloudinit->%*,
+ name => 1,
+ };
+ $p->{"net$_"} = 1 for 0..($MAX_NETS-1);
+ return $p;
+}
+
+sub check_type {
+ my ($key, $value) = @_;
+
+ die "unknown setting '$key'\n" if !$confdesc->{$key};
+
+ my $type = $confdesc->{$key}->{type};
+
+ if (!defined($value)) {
die "got undefined value\n";
}
}
sub destroy_vm {
- my ($storecfg, $vmid, $skiplock, $replacement_conf) = @_;
+ my ($storecfg, $vmid, $skiplock, $replacement_conf, $purge_unreferenced) = @_;
my $conf = PVE::QemuConfig->load_config($vmid);
- PVE::QemuConfig->check_lock($conf) if !$skiplock;
+ if (!$skiplock && !PVE::QemuConfig->has_lock($conf, 'suspended')) {
+ PVE::QemuConfig->check_lock($conf);
+ }
if ($conf->{template}) {
# check if any base image is still used by a linked clone
- PVE::QemuConfig->foreach_volume($conf, sub {
+ PVE::QemuConfig->foreach_volume_full($conf, { include_unused => 1 }, sub {
my ($ds, $drive) = @_;
return if drive_is_cdrom($drive);
});
}
- # only remove disks owned by this VM
- PVE::QemuConfig->foreach_volume($conf, sub {
+ my $volids = {};
+ my $remove_owned_drive = sub {
my ($ds, $drive) = @_;
return if drive_is_cdrom($drive, 1);
my $volid = $drive->{file};
return if !$volid || $volid =~ m|^/|;
+ return if $volids->{$volid};
my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
return if !$path || !$owner || ($owner != $vmid);
+ $volids->{$volid} = 1;
eval { PVE::Storage::vdisk_free($storecfg, $volid) };
warn "Could not remove disk '$volid', check manually: $@" if $@;
- });
+ };
- # also remove unused disk
- my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
- PVE::Storage::foreach_volid($vmdisks, sub {
- my ($volid, $sid, $volname, $d) = @_;
- eval { PVE::Storage::vdisk_free($storecfg, $volid) };
- warn $@ if $@;
- });
+ # only remove disks owned by this VM (referenced in the config)
+ my $include_opts = {
+ include_unused => 1,
+ extra_keys => ['vmstate'],
+ };
+ PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $remove_owned_drive);
+
+ for my $snap (values %{$conf->{snapshots}}) {
+ next if !defined($snap->{vmstate});
+ my $drive = PVE::QemuConfig->parse_volume('vmstate', $snap->{vmstate}, 1);
+ next if !defined($drive);
+ $remove_owned_drive->('vmstate', $drive);
+ }
+
+ PVE::QemuConfig->foreach_volume_full($conf->{pending}, $include_opts, $remove_owned_drive);
+
+ if ($purge_unreferenced) { # also remove unreferenced disk
+ my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid, undef, 'images');
+ PVE::Storage::foreach_volid($vmdisks, sub {
+ my ($volid, $sid, $volname, $d) = @_;
+ eval { PVE::Storage::vdisk_free($storecfg, $volid) };
+ warn $@ if $@;
+ });
+ }
+
+ eval { delete_ifaces_ipams_ips($conf, $vmid)};
+ warn $@ if $@;
if (defined $replacement_conf) {
PVE::QemuConfig->write_config($vmid, $replacement_conf);
}
sub parse_vm_config {
- my ($filename, $raw) = @_;
+ my ($filename, $raw, $strict) = @_;
return if !defined($raw);
digest => Digest::SHA::sha1_hex($raw),
snapshots => {},
pending => {},
+ cloudinit => {},
+ };
+
+ my $handle_error = sub {
+ my ($msg) = @_;
+
+ if ($strict) {
+ die $msg;
+ } else {
+ warn $msg;
+ }
};
$filename =~ m|/qemu-server/(\d+)\.conf$|
my $conf = $res;
my $descr;
+ my $finish_description = sub {
+ if (defined($descr)) {
+ $descr =~ s/\s+$//;
+ $conf->{description} = $descr;
+ }
+ $descr = undef;
+ };
my $section = '';
my @lines = split(/\n/, $raw);
if ($line =~ m/^\[PENDING\]\s*$/i) {
$section = 'pending';
- if (defined($descr)) {
- $descr =~ s/\s+$//;
- $conf->{description} = $descr;
- }
- $descr = undef;
+ $finish_description->();
+ $conf = $res->{$section} = {};
+ next;
+ } elsif ($line =~ m/^\[special:cloudinit\]\s*$/i) {
+ $section = 'cloudinit';
+ $finish_description->();
$conf = $res->{$section} = {};
next;
} elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
$section = $1;
- if (defined($descr)) {
- $descr =~ s/\s+$//;
- $conf->{description} = $descr;
- }
- $descr = undef;
+ $finish_description->();
$conf = $res->{snapshots}->{$section} = {};
next;
}
- if ($line =~ m/^\#(.*)\s*$/) {
+ if ($line =~ m/^\#(.*)$/) {
$descr = '' if !defined($descr);
$descr .= PVE::Tools::decode_text($1) . "\n";
next;
if ($section eq 'pending') {
$conf->{delete} = $value; # we parse this later
} else {
- warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
+ $handle_error->("vm $vmid - property 'delete' is only allowed in [PENDING]\n");
}
} elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
my $key = $1;
my $value = $2;
+ if ($section eq 'cloudinit') {
+ # ignore validation only used for informative purpose
+ $conf->{$key} = $value;
+ next;
+ }
eval { $value = check_type($key, $value); };
if ($@) {
- warn "vm $vmid - unable to parse value of '$key' - $@";
+ $handle_error->("vm $vmid - unable to parse value of '$key' - $@");
} else {
$key = 'ide2' if $key eq 'cdrom';
my $fmt = $confdesc->{$key}->{format};
$v->{file} = $volid;
$value = print_drive($v);
} else {
- warn "vm $vmid - unable to parse value of '$key'\n";
+ $handle_error->("vm $vmid - unable to parse value of '$key'\n");
next;
}
}
$conf->{$key} = $value;
}
+ } else {
+ $handle_error->("vm $vmid - unable to parse config: $line\n");
}
}
- if (defined($descr)) {
- $descr =~ s/\s+$//;
- $conf->{description} = $descr;
- }
+ $finish_description->();
delete $res->{snapstate}; # just to be sure
return $res;
foreach my $key (keys %$cref) {
next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
- $key eq 'snapstate' || $key eq 'pending';
+ $key eq 'snapstate' || $key eq 'pending' || $key eq 'cloudinit';
my $value = $cref->{$key};
if ($key eq 'delete') {
die "propertry 'delete' is only allowed in [PENDING]\n"
}
foreach my $key (sort keys %$conf) {
- next if $key =~ /^(digest|description|pending|snapshots)$/;
+ next if $key =~ /^(digest|description|pending|cloudinit|snapshots)$/;
$raw .= "$key: $conf->{$key}\n";
}
return $raw;
$raw .= &$generate_raw_config($conf->{pending}, 1);
}
+ if (scalar(keys %{$conf->{cloudinit}}) && PVE::QemuConfig->has_cloudinit($conf)){
+ $raw .= "\n[special:cloudinit]\n";
+ $raw .= &$generate_raw_config($conf->{cloudinit});
+ }
+
foreach my $snapname (sort keys %{$conf->{snapshots}}) {
$raw .= "\n[$snapname]\n";
$raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
my ($conf, $noerr) = @_;
my @loc_res = ();
+ my $mapped_res = [];
+
+ my $nodelist = PVE::Cluster::get_nodelist();
+ my $pci_map = PVE::Mapping::PCI::config();
+ my $usb_map = PVE::Mapping::USB::config();
+
+ my $missing_mappings_by_node = { map { $_ => [] } @$nodelist };
+
+ my $add_missing_mapping = sub {
+ my ($type, $key, $id) = @_;
+ for my $node (@$nodelist) {
+ my $entry;
+ if ($type eq 'pci') {
+ $entry = PVE::Mapping::PCI::get_node_mapping($pci_map, $id, $node);
+ } elsif ($type eq 'usb') {
+ $entry = PVE::Mapping::USB::get_node_mapping($usb_map, $id, $node);
+ }
+ if (!scalar($entry->@*)) {
+ push @{$missing_mappings_by_node->{$node}}, $key;
+ }
+ }
+ };
push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
push @loc_res, "ivshmem" if $conf->{ivshmem};
foreach my $k (keys %$conf) {
- next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
+ if ($k =~ m/^usb/) {
+ my $entry = parse_property_string('pve-qm-usb', $conf->{$k});
+ next if $entry->{host} && $entry->{host} =~ m/^spice$/i;
+ if ($entry->{mapping}) {
+ $add_missing_mapping->('usb', $k, $entry->{mapping});
+ push @$mapped_res, $k;
+ }
+ }
+ if ($k =~ m/^hostpci/) {
+ my $entry = parse_property_string('pve-qm-hostpci', $conf->{$k});
+ if ($entry->{mapping}) {
+ $add_missing_mapping->('pci', $k, $entry->{mapping});
+ push @$mapped_res, $k;
+ }
+ }
# sockets are safe: they will recreated be on the target side post-migrate
next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
die "VM uses local resources\n" if scalar @loc_res && !$noerr;
- return \@loc_res;
+ return wantarray ? (\@loc_res, $mapped_res, $missing_mappings_by_node) : \@loc_res;
}
# check if used storages are available on all nodes (use by migrate)
return if !$sid;
# check if storage is available on both nodes
- my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
- PVE::Storage::storage_check_node($storecfg, $sid, $node);
+ my $scfg = PVE::Storage::storage_check_enabled($storecfg, $sid);
+ PVE::Storage::storage_check_enabled($storecfg, $sid, $node);
+
+ my ($vtype) = PVE::Storage::parse_volname($storecfg, $volid);
+
+ die "$volid: content type '$vtype' is not available on storage '$sid'\n"
+ if !$scfg->{content}->{$vtype};
});
}
sub check_running {
my ($vmid, $nocheck, $node) = @_;
+ # $nocheck is set when called during a migration, in which case the config
+ # file might still or already reside on the *other* node
+ # - because rename has already happened, and current node is source
+ # - because rename hasn't happened yet, and current node is target
+ # - because rename has happened, current node is target, but hasn't yet
+ # processed it yet
PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
return PVE::QemuServer::Helpers::vm_running_locally($vmid);
}
our $vmstatus_return_properties = {
vmid => get_standard_option('pve-vmid'),
status => {
- description => "Qemu process status.",
+ description => "QEMU process status.",
type => 'string',
enum => ['stopped', 'running'],
},
optional => 1,
},
qmpstatus => {
- description => "Qemu QMP agent status.",
+ description => "VM run state from the 'query-status' QMP monitor command.",
type => 'string',
optional => 1,
},
type => 'string',
optional => 1,
},
+ 'running-machine' => {
+ description => "The currently running machine type (if running).",
+ type => 'string',
+ optional => 1,
+ },
+ 'running-qemu' => {
+ description => "The currently running QEMU version (if running).",
+ type => 'string',
+ optional => 1,
+ },
};
my $last_proc_pid_stat;
my $conf = PVE::QemuConfig->load_config($vmid);
- my $d = { vmid => $vmid };
- $d->{pid} = $list->{$vmid}->{pid};
+ my $d = { vmid => int($vmid) };
+ $d->{pid} = int($list->{$vmid}->{pid}) if $list->{$vmid}->{pid};
# fixme: better status?
$d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
$d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
$d->{name} = $conf->{name} || "VM $vmid";
- $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
- : $defaults->{memory}*(1024*1024);
+ $d->{maxmem} = get_current_memory($conf->{memory})*(1024*1024);
if ($conf->{balloon}) {
$d->{balloon_min} = $conf->{balloon}*(1024*1024);
$d->{diskread} = 0;
$d->{diskwrite} = 0;
- $d->{template} = PVE::QemuConfig->is_template($conf);
+ $d->{template} = 1 if PVE::QemuConfig->is_template($conf);
$d->{serial} = 1 if conf_has_serial($conf);
$d->{lock} = $conf->{lock} if $conf->{lock};
$d->{netin} += $netdev->{$dev}->{transmit};
if ($full) {
- $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
- $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
+ $d->{nics}->{$dev}->{netout} = int($netdev->{$dev}->{receive});
+ $d->{nics}->{$dev}->{netin} = int($netdev->{$dev}->{transmit});
}
}
$res->{$vmid}->{diskwrite} = $totalwrbytes;
};
+ my $machinecb = sub {
+ my ($vmid, $resp) = @_;
+ my $data = $resp->{'return'} || [];
+
+ $res->{$vmid}->{'running-machine'} =
+ PVE::QemuServer::Machine::current_from_query_machines($data);
+ };
+
+ my $versioncb = sub {
+ my ($vmid, $resp) = @_;
+ my $data = $resp->{'return'} // {};
+ my $version = 'unknown';
+
+ if (my $v = $data->{qemu}) {
+ $version = $v->{major} . "." . $v->{minor} . "." . $v->{micro};
+ }
+
+ $res->{$vmid}->{'running-qemu'} = $version;
+ };
+
my $statuscb = sub {
my ($vmid, $resp) = @_;
$qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
+ $qmpclient->queue_cmd($vmid, $machinecb, 'query-machines');
+ $qmpclient->queue_cmd($vmid, $versioncb, 'query-version');
# this fails if ballon driver is not loaded, so this must be
# the last commnand (following command are aborted if this fails).
$qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
$qmpclient->queue_execute(undef, 2);
+ foreach my $vmid (keys %$list) {
+ next if $opt_vmid && ($vmid ne $opt_vmid);
+ next if !$res->{$vmid}->{pid}; #not running
+
+ # we can't use the $qmpclient since it might have already aborted on
+ # 'query-balloon', but this might also fail for older versions...
+ my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
+ $res->{$vmid}->{'proxmox-support'} = $qemu_support // {};
+ }
+
foreach my $vmid (keys %$list) {
next if $opt_vmid && ($vmid ne $opt_vmid);
$res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
return $devs;
}
+sub get_tpm_paths {
+ my ($vmid) = @_;
+ return {
+ socket => "/var/run/qemu-server/$vmid.swtpm",
+ pid => "/var/run/qemu-server/$vmid.swtpm.pid",
+ };
+}
+
+sub add_tpm_device {
+ my ($vmid, $devices, $conf) = @_;
+
+ return if !$conf->{tpmstate0};
+
+ my $paths = get_tpm_paths($vmid);
+
+ push @$devices, "-chardev", "socket,id=tpmchar,path=$paths->{socket}";
+ push @$devices, "-tpmdev", "emulator,id=tpmdev,chardev=tpmchar";
+ push @$devices, "-device", "tpm-tis,tpmdev=tpmdev";
+}
+
+sub start_swtpm {
+ my ($storecfg, $vmid, $tpmdrive, $migration) = @_;
+
+ return if !$tpmdrive;
+
+ my $state;
+ my $tpm = parse_drive("tpmstate0", $tpmdrive);
+ my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
+ if ($storeid) {
+ $state = PVE::Storage::map_volume($storecfg, $tpm->{file});
+ } else {
+ $state = $tpm->{file};
+ }
+
+ my $paths = get_tpm_paths($vmid);
+
+ # during migration, we will get state from remote
+ #
+ if (!$migration) {
+ # run swtpm_setup to create a new TPM state if it doesn't exist yet
+ my $setup_cmd = [
+ "swtpm_setup",
+ "--tpmstate",
+ "file://$state",
+ "--createek",
+ "--create-ek-cert",
+ "--create-platform-cert",
+ "--lock-nvram",
+ "--config",
+ "/etc/swtpm_setup.conf", # do not use XDG configs
+ "--runas",
+ "0", # force creation as root, error if not possible
+ "--not-overwrite", # ignore existing state, do not modify
+ ];
+
+ push @$setup_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
+ # TPM 2.0 supports ECC crypto, use if possible
+ push @$setup_cmd, "--ecc" if $tpm->{version} eq 'v2.0';
+
+ run_command($setup_cmd, outfunc => sub {
+ print "swtpm_setup: $1\n";
+ });
+ }
+
+ # Used to distinguish different invocations in the log.
+ my $log_prefix = "[id=" . int(time()) . "] ";
+
+ my $emulator_cmd = [
+ "swtpm",
+ "socket",
+ "--tpmstate",
+ "backend-uri=file://$state,mode=0600",
+ "--ctrl",
+ "type=unixio,path=$paths->{socket},mode=0600",
+ "--pid",
+ "file=$paths->{pid}",
+ "--terminate", # terminate on QEMU disconnect
+ "--daemon",
+ "--log",
+ "file=/run/qemu-server/$vmid-swtpm.log,level=1,prefix=$log_prefix",
+ ];
+ push @$emulator_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
+ run_command($emulator_cmd, outfunc => sub { print $1; });
+
+ my $tries = 100; # swtpm may take a bit to start before daemonizing, wait up to 5s for pid
+ while (! -e $paths->{pid}) {
+ die "failed to start swtpm: pid file '$paths->{pid}' wasn't created.\n" if --$tries == 0;
+ usleep(50_000);
+ }
+
+ # return untainted PID of swtpm daemon so it can be killed on error
+ file_read_firstline($paths->{pid}) =~ m/(\d+)/;
+ return $1;
+}
+
sub vga_conf_has_spice {
my ($vga) = @_;
return $1 || 1;
}
-sub is_native($) {
- my ($arch) = @_;
- return get_host_arch() eq $arch;
-}
-
sub get_vm_arch {
my ($conf) = @_;
return $conf->{arch} // get_host_arch();
aarch64 => 'virt',
};
+sub get_installed_machine_version {
+ my ($kvmversion) = @_;
+ $kvmversion = kvm_user_version() if !defined($kvmversion);
+ $kvmversion =~ m/^(\d+\.\d+)/;
+ return $1;
+}
+
+sub windows_get_pinned_machine_version {
+ my ($machine, $base_version, $kvmversion) = @_;
+
+ my $pin_version = $base_version;
+ if (!defined($base_version) ||
+ !PVE::QemuServer::Machine::can_run_pve_machine_version($base_version, $kvmversion)
+ ) {
+ $pin_version = get_installed_machine_version($kvmversion);
+ }
+ if (!$machine || $machine eq 'pc') {
+ $machine = "pc-i440fx-$pin_version";
+ } elsif ($machine eq 'q35') {
+ $machine = "pc-q35-$pin_version";
+ } elsif ($machine eq 'virt') {
+ $machine = "virt-$pin_version";
+ } else {
+ warn "unknown machine type '$machine', not touching that!\n";
+ }
+
+ return $machine;
+}
+
sub get_vm_machine {
my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
- my $machine = $forcemachine || $conf->{machine};
+ my $machine_conf = PVE::QemuServer::Machine::parse_machine($conf->{machine});
+ my $machine = $forcemachine || $machine_conf->{type};
if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
+ $kvmversion //= kvm_user_version();
+ # we must pin Windows VMs without a specific version to 5.1, as 5.2 fixed a bug in ACPI
+ # layout which confuses windows quite a bit and may result in various regressions..
+ # see: https://lists.gnu.org/archive/html/qemu-devel/2021-02/msg08484.html
+ if (windows_version($conf->{ostype})) {
+ $machine = windows_get_pinned_machine_version($machine, '5.1', $kvmversion);
+ }
$arch //= 'x86_64';
$machine ||= $default_machines->{$arch};
if ($add_pve_version) {
- $kvmversion //= kvm_user_version();
my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
$machine .= "+pve$pvever";
}
}
- if ($add_pve_version && $machine !~ m/\+pve\d+$/) {
+ if ($add_pve_version && $machine !~ m/\+pve\d+?(?:\.pxe)?$/) {
+ my $is_pxe = $machine =~ m/^(.*?)\.pxe$/;
+ $machine = $1 if $is_pxe;
+
# for version-pinned machines that do not include a pve-version (e.g.
# pc-q35-4.1), we assume 0 to keep them stable in case we bump
$machine .= '+pve0';
+
+ $machine .= '.pxe' if $is_pxe;
}
return $machine;
}
-sub get_ovmf_files($) {
- my ($arch) = @_;
+sub get_ovmf_files($$$) {
+ my ($arch, $efidisk, $smm) = @_;
- my $ovmf = $OVMF->{$arch}
+ my $types = $OVMF->{$arch}
or die "no OVMF images known for architecture '$arch'\n";
- return @$ovmf;
+ my $type = 'default';
+ if ($arch eq 'x86_64') {
+ if (defined($efidisk->{efitype}) && $efidisk->{efitype} eq '4m') {
+ $type = $smm ? "4m" : "4m-no-smm";
+ $type .= '-ms' if $efidisk->{'pre-enrolled-keys'};
+ } else {
+ # TODO: log_warn about use of legacy images for x86_64 with Promxox VE 9
+ }
+ }
+
+ my ($ovmf_code, $ovmf_vars) = $types->{$type}->@*;
+ die "EFI base image '$ovmf_code' not found\n" if ! -f $ovmf_code;
+ die "EFI vars image '$ovmf_vars' not found\n" if ! -f $ovmf_vars;
+
+ return ($ovmf_code, $ovmf_vars);
}
my $Arch2Qemu = {
};
sub get_command_for_arch($) {
my ($arch) = @_;
- return '/usr/bin/kvm' if is_native($arch);
+ return '/usr/bin/kvm' if is_native_arch($arch);
my $cmd = $Arch2Qemu->{$arch}
or die "don't know how to emulate architecture '$arch'\n";
$qemu_cmd,
'-machine', $default_machine,
'-display', 'none',
- '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server,nowait",
+ '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server=on,wait=off",
'-mon', 'chardev=qmp,mode=control',
'-pidfile', $pidfile,
'-S', '-daemonize'
};
my $err = $@;
- # force stop with 10 sec timeout and 'nocheck'
- # always stop, even if QMP failed
+ # force stop with 10 sec timeout and 'nocheck', always stop, even if QMP failed
vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
die $err if $err;
return \@flags;
}
+# Since commit 277d33454f77ec1d1e0bc04e37621e4dd2424b67 in pve-qemu, smm is not off by default
+# anymore. But smm=off seems to be required when using SeaBIOS and serial display.
+my sub should_disable_smm {
+ my ($conf, $vga, $machine) = @_;
+
+ return if $machine =~ m/^virt/; # there is no smm flag that could be disabled
+
+ return (!defined($conf->{bios}) || $conf->{bios} eq 'seabios') &&
+ $vga->{type} && $vga->{type} =~ m/^(serial\d+|none)$/;
+}
+
+my sub print_ovmf_drive_commandlines {
+ my ($conf, $storecfg, $vmid, $arch, $q35, $version_guard) = @_;
+
+ my $d = $conf->{efidisk0} ? parse_drive('efidisk0', $conf->{efidisk0}) : undef;
+
+ my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch, $d, $q35);
+
+ my $var_drive_str = "if=pflash,unit=1,id=drive-efidisk0";
+ if ($d) {
+ my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
+ my ($path, $format) = $d->@{'file', 'format'};
+ if ($storeid) {
+ $path = PVE::Storage::path($storecfg, $d->{file});
+ if (!defined($format)) {
+ my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
+ $format = qemu_img_format($scfg, $volname);
+ }
+ } elsif (!defined($format)) {
+ die "efidisk format must be specified\n";
+ }
+ # SPI flash does lots of read-modify-write OPs, without writeback this gets really slow #3329
+ if ($path =~ m/^rbd:/) {
+ $var_drive_str .= ',cache=writeback';
+ $path .= ':rbd_cache_policy=writeback'; # avoid write-around, we *need* to cache writes too
+ }
+ $var_drive_str .= ",format=$format,file=$path";
+
+ $var_drive_str .= ",size=" . (-s $ovmf_vars) if $format eq 'raw' && $version_guard->(4, 1, 2);
+ $var_drive_str .= ',readonly=on' if drive_is_read_only($conf, $d);
+ } else {
+ log_warn("no efidisk configured! Using temporary efivars disk.");
+ my $path = "/tmp/$vmid-ovmf.fd";
+ PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
+ $var_drive_str .= ",format=raw,file=$path";
+ $var_drive_str .= ",size=" . (-s $ovmf_vars) if $version_guard->(4, 1, 2);
+ }
+
+ return ("if=pflash,unit=0,format=raw,readonly=on,file=$ovmf_code", $var_drive_str);
+}
+
sub config_to_command {
- my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu) = @_;
+ my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu,
+ $live_restore_backing) = @_;
- my $cmd = [];
- my $globalFlags = [];
- my $machineFlags = [];
- my $rtcFlags = [];
+ my ($globalFlags, $machineFlags, $rtcFlags) = ([], [], []);
my $devices = [];
- my $pciaddr = '';
my $bridges = {};
my $ostype = $conf->{ostype};
my $winversion = windows_version($ostype);
my $kvm = $conf->{kvm};
my $nodename = nodename();
+ my $machine_conf = PVE::QemuServer::Machine::parse_machine($conf->{machine});
+
my $arch = get_vm_arch($conf);
my $kvm_binary = get_command_for_arch($arch);
my $kvmver = kvm_user_version($kvm_binary);
my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
my $machine_version = extract_version($machine_type, $kvmver);
- $kvm //= 1 if is_native($arch);
+ $kvm //= 1 if is_native_arch($arch);
$machine_version =~ m/(\d+)\.(\d+)/;
my ($machine_major, $machine_minor) = ($1, $2);
my $use_old_bios_files = undef;
($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
- my $cpuunits = defined($conf->{cpuunits}) ?
- $conf->{cpuunits} : $defaults->{cpuunits};
+ my $cmd = [];
+ if ($conf->{affinity}) {
+ push @$cmd, '/usr/bin/taskset', '--cpu-list', '--all-tasks', $conf->{affinity};
+ }
push @$cmd, $kvm_binary;
my $vmname = $conf->{name} || "vm$vmid";
- push @$cmd, '-name', $vmname;
+ push @$cmd, '-name', "$vmname,debug-threads=on";
+
+ push @$cmd, '-no-shutdown';
my $use_virtio = 0;
my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
- push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
+ push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server=on,wait=off";
push @$cmd, '-mon', "chardev=qmp,mode=control";
if (min_version($machine_version, 2, 12)) {
}
if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
- my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
- die "uefi base image '$ovmf_code' not found\n" if ! -f $ovmf_code;
-
- my ($path, $format);
- if (my $efidisk = $conf->{efidisk0}) {
- my $d = parse_drive('efidisk0', $efidisk);
- my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
- $format = $d->{format};
- if ($storeid) {
- $path = PVE::Storage::path($storecfg, $d->{file});
- if (!defined($format)) {
- my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
- $format = qemu_img_format($scfg, $volname);
- }
- } else {
- $path = $d->{file};
- die "efidisk format must be specified\n"
- if !defined($format);
- }
- } else {
- warn "no efidisk configured! Using temporary efivars disk.\n";
- $path = "/tmp/$vmid-ovmf.fd";
- PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
- $format = 'raw';
- }
+ die "OVMF (UEFI) BIOS is not supported on 32-bit CPU types\n"
+ if !$forcecpu && get_cpu_bitness($conf->{cpu}, $arch) == 32;
- my $size_str = "";
-
- if ($format eq 'raw' && $version_guard->(4, 1, 2)) {
- $size_str = ",size=" . (-s $ovmf_vars);
- }
-
- push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
- push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0$size_str,file=$path";
+ my ($code_drive_str, $var_drive_str) =
+ print_ovmf_drive_commandlines($conf, $storecfg, $vmid, $arch, $q35, $version_guard);
+ push $cmd->@*, '-drive', $code_drive_str;
+ push $cmd->@*, '-drive', $var_drive_str;
}
- # load q35 config
- if ($q35) {
+ if ($q35) { # tell QEMU to load q35 config early
# we use different pcie-port hardware for qemu >= 4.0 for passthrough
if (min_version($machine_version, 4, 0)) {
push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
}
}
+ if (defined(my $fixups = qemu_created_version_fixups($conf, $forcemachine, $kvmver))) {
+ push @$cmd, $fixups->@*;
+ }
+
if ($conf->{vmgenid}) {
push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
}
# add usb controllers
my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers(
- $conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
+ $conf, $bridges, $arch, $machine_type, $machine_version);
push @$devices, @usbcontrollers if @usbcontrollers;
my $vga = parse_vga($conf->{vga});
}
# enable absolute mouse coordinates (needed by vnc)
- my $tablet;
- if (defined($conf->{tablet})) {
- $tablet = $conf->{tablet};
- } else {
+ my $tablet = $conf->{tablet};
+ if (!defined($tablet)) {
$tablet = $defaults->{tablet};
$tablet = 0 if $qxlnum; # disable for spice because it is not needed
$tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
my $bootorder = device_bootorder($conf);
# host pci device passthrough
- my ($kvm_off, $gpu_passthrough, $legacy_igd) = PVE::QemuServer::PCI::print_hostpci_devices(
- $vmid, $conf, $devices, $vga, $winversion, $q35, $bridges, $arch, $machine_type, $bootorder);
+ my ($kvm_off, $gpu_passthrough, $legacy_igd, $pci_devices) = PVE::QemuServer::PCI::print_hostpci_devices(
+ $vmid, $conf, $devices, $vga, $winversion, $bridges, $arch, $machine_type, $bootorder);
# usb devices
my $usb_dev_features = {};
$usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
my @usbdevices = PVE::QemuServer::USB::get_usb_devices(
- $conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features, $bootorder);
+ $conf, $usb_dev_features, $bootorder, $machine_version);
push @$devices, @usbdevices if @usbdevices;
# serial devices
for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
- if (my $path = $conf->{"serial$i"}) {
- if ($path eq 'socket') {
- my $socket = "/var/run/qemu-server/${vmid}.serial$i";
- push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
- # On aarch64, serial0 is the UART device. Qemu only allows
- # connecting UART devices via the '-serial' command line, as
- # the device has a fixed slot on the hardware...
- if ($arch eq 'aarch64' && $i == 0) {
- push @$devices, '-serial', "chardev:serial$i";
- } else {
- push @$devices, '-device', "isa-serial,chardev=serial$i";
- }
+ my $path = $conf->{"serial$i"} or next;
+ if ($path eq 'socket') {
+ my $socket = "/var/run/qemu-server/${vmid}.serial$i";
+ push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server=on,wait=off";
+ # On aarch64, serial0 is the UART device. QEMU only allows
+ # connecting UART devices via the '-serial' command line, as
+ # the device has a fixed slot on the hardware...
+ if ($arch eq 'aarch64' && $i == 0) {
+ push @$devices, '-serial', "chardev:serial$i";
} else {
- die "no such serial device\n" if ! -c $path;
- push @$devices, '-chardev', "tty,id=serial$i,path=$path";
push @$devices, '-device', "isa-serial,chardev=serial$i";
}
+ } else {
+ die "no such serial device\n" if ! -c $path;
+ push @$devices, '-chardev', "serial,id=serial$i,path=$path";
+ push @$devices, '-device', "isa-serial,chardev=serial$i";
}
}
for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++) {
if (my $path = $conf->{"parallel$i"}) {
die "no such parallel device\n" if ! -c $path;
- my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
+ my $devtype = $path =~ m!^/dev/usb/lp! ? 'serial' : 'parallel';
push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
push @$devices, '-device', "isa-parallel,chardev=parallel$i";
}
push @$devices, @$audio_devs;
}
+ # Add a TPM only if the VM is not a template,
+ # to support backing up template VMs even if the TPM disk is write-protected.
+ add_tpm_device($vmid, $devices, $conf) if (!PVE::QemuConfig->is_template($conf));
+
my $sockets = 1;
$sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
$sockets = $conf->{sockets} if $conf->{sockets};
my $allowed_vcpus = $cpuinfo->{cpus};
- die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
- if ($allowed_vcpus < $maxcpus);
-
- if($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
+ die "MAX $allowed_vcpus vcpus allowed per VM on this node\n" if ($allowed_vcpus < $maxcpus);
+ if ($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
for (my $i = 2; $i <= $vcpus; $i++) {
- my $cpustr = print_cpu_device($conf,$i);
+ my $cpustr = print_cpu_device($conf, $arch, $i);
push @$cmd, '-device', $cpustr;
}
push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
- push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
+ push $machineFlags->@*, 'acpi=off' if defined($conf->{acpi}) && $conf->{acpi} == 0;
push @$cmd, '-no-reboot' if defined($conf->{reboot}) && $conf->{reboot} == 0;
if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
push @$devices, '-device', print_vga_device(
$conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
+
+ push @$cmd, '-display', 'egl-headless,gl=core' if $vga->{type} eq 'virtio-gl'; # VIRGL
+
my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
- push @$cmd, '-vnc', "unix:$socket,password";
+ push @$cmd, '-vnc', "unix:$socket,password=on";
} else {
push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
push @$cmd, '-nographic';
if ($winversion >= 6) {
push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
- push @$cmd, '-no-hpet';
+ push @$machineFlags, 'hpet=off';
}
push @$rtcFlags, 'driftfix=slew' if $tdf;
push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
}
- PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
+ PVE::QemuServer::Memory::config(
+ $conf, $vmid, $sockets, $cores, $hotplug_features->{memory}, $cmd);
push @$cmd, '-S' if $conf->{freeze};
if ($guest_agent->{enabled}) {
my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
- push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
+ push @$devices, '-chardev', "socket,path=$qgasocket,server=on,wait=off,id=qga0";
if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
my $spice_port;
- if ($qxlnum) {
+ assert_clipboard_config($vga);
+ my $is_spice = $qxlnum || $vga->{type} =~ /^virtio/;
+
+ if ($is_spice || ($vga->{'clipboard'} && $vga->{'clipboard'} eq 'vnc')) {
if ($qxlnum > 1) {
if ($winversion){
for (my $i = 1; $i < $qxlnum; $i++){
my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
- my $pfamily = PVE::Tools::get_host_address_family($nodename);
- my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
- die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
-
push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
- push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
+ if ($vga->{'clipboard'} && $vga->{'clipboard'} eq 'vnc') {
+ push @$devices, '-chardev', 'qemu-vdagent,id=vdagent,name=vdagent,clipboard=on';
+ } else {
+ push @$devices, '-chardev', 'spicevmc,id=vdagent,name=vdagent';
+ }
push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
- my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
- $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
+ if ($is_spice) {
+ my $pfamily = PVE::Tools::get_host_address_family($nodename);
+ my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
+ die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
- my $spice_enhancement_str = $conf->{spice_enhancements} // '';
- my $spice_enhancement = parse_property_string($spice_enhancements_fmt, $spice_enhancement_str);
- if ($spice_enhancement->{foldersharing}) {
- push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
- push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
- }
+ my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
+ $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
- my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
- $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
- if $spice_enhancement->{videostreaming};
+ my $spice_enhancement_str = $conf->{spice_enhancements} // '';
+ my $spice_enhancement = parse_property_string($spice_enhancements_fmt, $spice_enhancement_str);
+ if ($spice_enhancement->{foldersharing}) {
+ push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
+ push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
+ }
- push @$devices, '-spice', "$spice_opts";
+ my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
+ $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
+ if $spice_enhancement->{videostreaming};
+ push @$devices, '-spice', "$spice_opts";
+ }
}
# enable balloon by default, unless explicitly disabled
if (!defined($conf->{balloon}) || $conf->{balloon}) {
- $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
- push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
+ my $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
+ my $ballooncmd = "virtio-balloon-pci,id=balloon0$pciaddr";
+ $ballooncmd .= ",free-page-reporting=on" if min_version($machine_version, 6, 2);
+ push @$devices, '-device', $ballooncmd;
}
if ($conf->{watchdog}) {
my $wdopts = parse_watchdog($conf->{watchdog});
- $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
+ my $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
my $watchdog = $wdopts->{model} || 'i6300esb';
push @$devices, '-device', "$watchdog$pciaddr";
push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
my ($ds, $drive) = @_;
if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
+ check_volume_storage_type($storecfg, $drive->{file});
push @$vollist, $drive->{file};
}
# ignore efidisk here, already added in bios/fw handling code above
return if $drive->{interface} eq 'efidisk';
+ # similar for TPM
+ return if $drive->{interface} eq 'tpmstate';
$use_virtio = 1 if $ds =~ m/^virtio/;
die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
- $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
+ my $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
my $iothread = '';
$iothread .= ",iothread=iothread-$controller_prefix$controller";
push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
} elsif ($drive->{iothread}) {
- warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
+ log_warn(
+ "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n"
+ );
}
my $queues = '';
if ($drive->{interface} eq 'sata') {
my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
- $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
+ my $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
if !$ahcicontroller->{$controller};
$ahcicontroller->{$controller}=1;
}
- my $drive_cmd = print_drive_commandline_full($storecfg, $vmid, $drive);
- $drive_cmd .= ',readonly' if PVE::QemuConfig->is_template($conf);
+ my $live_restore = $live_restore_backing->{$ds};
+ my $live_blockdev_name = undef;
+ if ($live_restore) {
+ $live_blockdev_name = $live_restore->{name};
+ push @$devices, '-blockdev', $live_restore->{blockdev};
+ }
+
+ my $drive_cmd = print_drive_commandline_full(
+ $storecfg, $vmid, $drive, $live_blockdev_name, min_version($kvmver, 6, 0));
+
+ # extra protection for templates, but SATA and IDE don't support it..
+ $drive_cmd .= ',readonly=on' if drive_is_read_only($conf, $drive);
push @$devices, '-drive',$drive_cmd;
push @$devices, '-device', print_drivedevice_full(
next if !$conf->{$netname};
my $d = parse_net($conf->{$netname});
next if !$d;
+ # save the MAC addr here (could be auto-gen. in some odd setups) for FDB registering later?
$use_virtio = 1 if $d->{model} eq 'virtio';
push @$devices, '-netdev', $netdevfull;
my $netdevicefull = print_netdevice_full(
- $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type);
+ $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type, $machine_version);
push @$devices, '-device', $netdevicefull;
}
# pci.4 is nested in pci.1
$bridges->{1} = 1 if $bridges->{4};
- if (!$q35) {
- # add pci bridges
- if (min_version($machine_version, 2, 3)) {
+ if (!$q35) { # add pci bridges
+ if (min_version($machine_version, 2, 3)) {
$bridges->{1} = 1;
$bridges->{2} = 1;
}
-
$bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
-
}
for my $k (sort {$b cmp $a} keys %$bridges) {
if ($k == 2 && $legacy_igd) {
$k_name = "$k-igd";
}
- $pciaddr = print_pci_addr("pci.$k_name", undef, $arch, $machine_type);
-
+ my $pciaddr = print_pci_addr("pci.$k_name", undef, $arch, $machine_type);
my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
- if ($q35) {
- # add after -readconfig pve-q35.cfg
+
+ if ($q35) { # add after -readconfig pve-q35.cfg
splice @$devices, 2, 0, '-device', $devstr;
} else {
unshift @$devices, '-device', $devstr if $k > 0;
push @$machineFlags, 'accel=tcg';
}
+ push @$machineFlags, 'smm=off' if should_disable_smm($conf, $vga, $machine_type);
+
my $machine_type_min = $machine_type;
if ($add_pve_version) {
$machine_type_min =~ s/\+pve\d+$//;
}
push @$machineFlags, "type=${machine_type_min}";
+ PVE::QemuServer::Machine::assert_valid_machine_property($conf, $machine_conf);
+
+ if (my $viommu = $machine_conf->{viommu}) {
+ if ($viommu eq 'intel') {
+ unshift @$devices, '-device', 'intel-iommu,intremap=on,caching-mode=on';
+ push @$machineFlags, 'kernel-irqchip=split';
+ } elsif ($viommu eq 'virtio') {
+ push @$devices, '-device', 'virtio-iommu-pci';
+ }
+ }
+
push @$cmd, @$devices;
push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
print "activating and using '$vmstate' as vmstate\n";
}
+ if (PVE::QemuConfig->is_template($conf)) {
+ # needed to workaround base volumes being read-only
+ push @$cmd, '-snapshot';
+ }
+
# add custom args
if ($conf->{args}) {
my $aa = PVE::Tools::split_args($conf->{args});
push @$cmd, @$aa;
}
- return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
+ return wantarray ? ($cmd, $vollist, $spice_port, $pci_devices) : $cmd;
}
sub check_rng_source {
my $to_check = [];
for my $d (@$devices_to_check) {
$devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
- next if !$d->{'pci_bridge'};
+ next if !$d->{'pci_bridge'} || !$d->{'pci_bridge'}->{devices};
$devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
push @$to_check, @{$d->{'pci_bridge'}->{devices}};
# qom-list path=/machine/peripheral
my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
foreach my $per (@$resperipheral) {
- if ($per->{name} =~ m/^usb\d+$/) {
+ if ($per->{name} =~ m/^usb(?:redirdev)?\d+$/) {
$devices->{$per->{name}} = 1;
}
}
qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
if ($deviceid eq 'tablet') {
-
qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
-
} elsif ($deviceid eq 'keyboard') {
-
qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
-
+ } elsif ($deviceid =~ m/^usbredirdev(\d+)$/) {
+ my $id = $1;
+ qemu_spice_usbredir_chardev_add($vmid, "usbredirchardev$id");
+ qemu_deviceadd($vmid, PVE::QemuServer::USB::print_spice_usbdevice($id, "xhci", $id + 1));
} elsif ($deviceid =~ m/^usb(\d+)$/) {
-
- die "usb hotplug currently not reliable\n";
- # since we can't reliably hot unplug all added usb devices and usb
- # passthrough breaks live migration we disable usb hotplugging for now
- #qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
-
+ qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device, {}, $1 + 1));
} elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
-
qemu_iothread_add($vmid, $deviceid, $device);
- qemu_driveadd($storecfg, $vmid, $device);
- my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
+ qemu_driveadd($storecfg, $vmid, $device);
+ my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
- qemu_deviceadd($vmid, $devicefull);
+ qemu_deviceadd($vmid, $devicefull);
eval { qemu_deviceaddverify($vmid, $deviceid); };
if (my $err = $@) {
eval { qemu_drivedel($vmid, $deviceid); };
warn $@ if $@;
die $err;
}
-
} elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
-
-
- my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
- my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
+ my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
+ my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
- my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
+ my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
qemu_iothread_add($vmid, $deviceid, $device);
$devicefull .= ",num_queues=$device->{queues}";
}
- qemu_deviceadd($vmid, $devicefull);
- qemu_deviceaddverify($vmid, $deviceid);
-
+ qemu_deviceadd($vmid, $devicefull);
+ qemu_deviceaddverify($vmid, $deviceid);
} elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
-
qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
qemu_driveadd($storecfg, $vmid, $device);
- my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
+ my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
eval { qemu_deviceadd($vmid, $devicefull); };
if (my $err = $@) {
eval { qemu_drivedel($vmid, $deviceid); };
warn $@ if $@;
die $err;
}
-
} elsif ($deviceid =~ m/^(net)(\d+)$/) {
-
return if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
+ my $machine_version = PVE::QemuServer::Machine::extract_version($machine_type);
my $use_old_bios_files = undef;
($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
my $netdevicefull = print_netdevice_full(
- $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
+ $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type, $machine_version);
qemu_deviceadd($vmid, $netdevicefull);
eval {
qemu_deviceaddverify($vmid, $deviceid);
warn $@ if $@;
die $err;
}
-
} elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
-
my $bridgeid = $2;
my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
qemu_deviceadd($vmid, $devicefull);
qemu_deviceaddverify($vmid, $deviceid);
-
} else {
die "can't hotplug device '$deviceid'\n";
}
my $bootdisks = PVE::QemuServer::Drive::get_bootdisks($conf);
die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
- if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
-
+ if ($deviceid eq 'tablet' || $deviceid eq 'keyboard' || $deviceid eq 'xhci') {
qemu_devicedel($vmid, $deviceid);
-
+ } elsif ($deviceid =~ m/^usbredirdev\d+$/) {
+ qemu_devicedel($vmid, $deviceid);
+ qemu_devicedelverify($vmid, $deviceid);
} elsif ($deviceid =~ m/^usb\d+$/) {
-
- die "usb hotplug currently not reliable\n";
- # when unplugging usb devices this way, there may be remaining usb
- # controllers/hubs so we disable it for now
- #qemu_devicedel($vmid, $deviceid);
- #qemu_devicedelverify($vmid, $deviceid);
-
+ qemu_devicedel($vmid, $deviceid);
+ qemu_devicedelverify($vmid, $deviceid);
} elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
+ my $device = parse_drive($deviceid, $conf->{$deviceid});
- qemu_devicedel($vmid, $deviceid);
- qemu_devicedelverify($vmid, $deviceid);
- qemu_drivedel($vmid, $deviceid);
- qemu_iothread_del($conf, $vmid, $deviceid);
-
+ qemu_devicedel($vmid, $deviceid);
+ qemu_devicedelverify($vmid, $deviceid);
+ qemu_drivedel($vmid, $deviceid);
+ qemu_iothread_del($vmid, $deviceid, $device);
} elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
-
qemu_devicedel($vmid, $deviceid);
qemu_devicedelverify($vmid, $deviceid);
- qemu_iothread_del($conf, $vmid, $deviceid);
-
} elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
+ my $device = parse_drive($deviceid, $conf->{$deviceid});
- qemu_devicedel($vmid, $deviceid);
- qemu_drivedel($vmid, $deviceid);
+ qemu_devicedel($vmid, $deviceid);
+ qemu_devicedelverify($vmid, $deviceid);
+ qemu_drivedel($vmid, $deviceid);
qemu_deletescsihw($conf, $vmid, $deviceid);
+ qemu_iothread_del($vmid, "virtioscsi$device->{index}", $device)
+ if $conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single');
} elsif ($deviceid =~ m/^(net)(\d+)$/) {
-
- qemu_devicedel($vmid, $deviceid);
- qemu_devicedelverify($vmid, $deviceid);
- qemu_netdevdel($vmid, $deviceid);
-
+ qemu_devicedel($vmid, $deviceid);
+ qemu_devicedelverify($vmid, $deviceid);
+ qemu_netdevdel($vmid, $deviceid);
} else {
die "can't unplug device '$deviceid'\n";
}
return 1;
}
-sub qemu_deviceadd {
- my ($vmid, $devicefull) = @_;
-
- $devicefull = "driver=".$devicefull;
- my %options = split(/[=,]/, $devicefull);
-
- mon_cmd($vmid, "device_add" , %options);
-}
-
-sub qemu_devicedel {
- my ($vmid, $deviceid) = @_;
+sub qemu_spice_usbredir_chardev_add {
+ my ($vmid, $id) = @_;
- my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
+ mon_cmd($vmid, "chardev-add" , (
+ id => $id,
+ backend => {
+ type => 'spicevmc',
+ data => {
+ type => "usbredir",
+ },
+ },
+ ));
}
sub qemu_iothread_add {
- my($vmid, $deviceid, $device) = @_;
+ my ($vmid, $deviceid, $device) = @_;
if ($device->{iothread}) {
my $iothreads = vm_iothreads_list($vmid);
}
sub qemu_iothread_del {
- my($conf, $vmid, $deviceid) = @_;
+ my ($vmid, $deviceid, $device) = @_;
- my $confid = $deviceid;
- if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
- $confid = 'scsi' . $1;
- }
- my $device = parse_drive($confid, $conf->{$confid});
if ($device->{iothread}) {
my $iothreads = vm_iothreads_list($vmid);
qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
}
}
-sub qemu_objectadd {
- my($vmid, $objectid, $qomtype) = @_;
-
- mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
-
- return 1;
-}
-
-sub qemu_objectdel {
- my($vmid, $objectid) = @_;
-
- mon_cmd($vmid, "object-del", id => $objectid);
-
- return 1;
-}
-
sub qemu_driveadd {
my ($storecfg, $vmid, $device) = @_;
- my $drive = print_drive_commandline_full($storecfg, $vmid, $device);
+ my $kvmver = get_running_qemu_version($vmid);
+ my $io_uring = min_version($kvmver, 6, 0);
+ my $drive = print_drive_commandline_full($storecfg, $vmid, $device, undef, $io_uring);
$drive =~ s/\\/\\\\/g;
my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
}
sub qemu_drivedel {
- my($vmid, $deviceid) = @_;
+ my ($vmid, $deviceid) = @_;
my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
$ret =~ s/^\s+//;
my $scsihwid="$controller_prefix$controller";
my $devices_list = vm_devices_list($vmid);
- if(!defined($devices_list->{$scsihwid})) {
+ if (!defined($devices_list->{$scsihwid})) {
vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
}
foreach my $opt (keys %{$devices_list}) {
if (is_valid_drivename($opt)) {
my $drive = parse_drive($opt, $conf->{$opt});
- if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
+ if ($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
return 1;
}
}
vm_deviceunplug($vmid, $conf, $deviceid);
# check if xhci controller is necessary and available
- if ($device->{usb3}) {
-
- my $devicelist = vm_devices_list($vmid);
+ my $devicelist = vm_devices_list($vmid);
- if (!$devicelist->{xhci}) {
- my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
- qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
- }
+ if (!$devicelist->{xhci}) {
+ my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
+ qemu_deviceadd($vmid, PVE::QemuServer::USB::print_qemu_xhci_controller($pciaddr));
}
- my $d = parse_usb_device($device->{host});
- $d->{usb3} = $device->{usb3};
# add the new one
- vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
+ vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type);
}
sub qemu_cpu_hotplug {
if scalar(@{$currentrunningvcpus}) != $currentvcpus;
if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
+ my $arch = get_vm_arch($conf);
for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
- my $cpustr = print_cpu_device($conf, $i);
+ my $cpustr = print_cpu_device($conf, $arch, $i);
qemu_deviceadd($vmid, $cpustr);
my $retry = 0;
my $running = check_running($vmid);
- $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
+ PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
return if !$running;
my $padding = (1024 - $size % 1024) % 1024;
$size = $size + $padding;
- mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
-
+ mon_cmd(
+ $vmid,
+ "block_resize",
+ device => $deviceid,
+ size => int($size),
+ timeout => 60,
+ );
}
sub qemu_volume_snapshot {
my $running = check_running($vmid);
- if ($running && do_snapshots_with_qemu($storecfg, $volid)){
+ if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
} else {
PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
}
sub qemu_volume_snapshot_delete {
- my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
+ my ($vmid, $storecfg, $volid, $snap) = @_;
my $running = check_running($vmid);
+ my $attached_deviceid;
- if($running) {
-
- $running = undef;
+ if ($running) {
my $conf = PVE::QemuConfig->load_config($vmid);
PVE::QemuConfig->foreach_volume($conf, sub {
my ($ds, $drive) = @_;
- $running = 1 if $drive->{file} eq $volid;
+ $attached_deviceid = "drive-$ds" if $drive->{file} eq $volid;
});
}
- if ($running && do_snapshots_with_qemu($storecfg, $volid)){
- mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
+ if ($attached_deviceid && do_snapshots_with_qemu($storecfg, $volid, $attached_deviceid)) {
+ mon_cmd(
+ $vmid,
+ 'blockdev-snapshot-delete-internal-sync',
+ device => $attached_deviceid,
+ name => $snap,
+ );
} else {
- PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
+ PVE::Storage::volume_snapshot_delete(
+ $storecfg, $volid, $snap, $attached_deviceid ? 1 : undef);
}
}
sub set_migration_caps {
- my ($vmid) = @_;
+ my ($vmid, $savevm) = @_;
my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
+ my $bitmap_prop = $savevm ? 'pbs-dirty-bitmap-savevm' : 'pbs-dirty-bitmap-migration';
+ my $dirty_bitmaps = $qemu_support->{$bitmap_prop} ? 1 : 0;
+
my $cap_ref = [];
my $enabled_cap = {
"x-rdma-pin-all" => 0,
"zero-blocks" => 0,
"compress" => 0,
- "dirty-bitmaps" => $qemu_support->{'pbs-dirty-bitmap-migration'} ? 1 : 0,
+ "dirty-bitmaps" => $dirty_bitmaps,
};
my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
my $volhash = {};
my $test_volid = sub {
- my ($key, $drive, $snapname) = @_;
+ my ($key, $drive, $snapname, $pending) = @_;
my $volid = $drive->{file};
return if !$volid;
$volhash->{$volid}->{shared} //= 0;
$volhash->{$volid}->{shared} = 1 if $drive->{shared};
- $volhash->{$volid}->{referenced_in_config} //= 0;
- $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
+ $volhash->{$volid}->{is_unused} //= 0;
+ $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
+
+ $volhash->{$volid}->{is_attached} //= 0;
+ $volhash->{$volid}->{is_attached} = 1
+ if !$volhash->{$volid}->{is_unused} && !defined($snapname) && !$pending;
$volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
if defined($snapname);
+ $volhash->{$volid}->{referenced_in_pending} = 1 if $pending;
+
my $size = $drive->{size};
$volhash->{$volid}->{size} //= $size if $size;
$volhash->{$volid}->{is_vmstate} //= 0;
$volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate';
- $volhash->{$volid}->{is_unused} //= 0;
- $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
+ $volhash->{$volid}->{is_tpmstate} //= 0;
+ $volhash->{$volid}->{is_tpmstate} = 1 if $key eq 'tpmstate0';
+
+ $volhash->{$volid}->{drivename} = $key if is_valid_drivename($key);
};
my $include_opts = {
};
PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
+
+ PVE::QemuConfig->foreach_volume_full($conf->{pending}, $include_opts, $test_volid, undef, 1)
+ if defined($conf->{pending}) && $conf->{pending}->%*;
+
foreach my $snapname (keys %{$conf->{snapshots}}) {
my $snap = $conf->{snapshots}->{$snapname};
PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
}
my $fast_plug_option = {
+ 'description' => 1,
+ 'hookscript' => 1,
'lock' => 1,
+ 'migrate_downtime' => 1,
+ 'migrate_speed' => 1,
'name' => 1,
'onboot' => 1,
+ 'protection' => 1,
'shares' => 1,
'startup' => 1,
- 'description' => 1,
- 'protection' => 1,
- 'vmstatestorage' => 1,
- 'hookscript' => 1,
'tags' => 1,
+ 'vmstatestorage' => 1,
+};
+
+for my $opt (keys %$confdesc_cloudinit) {
+ $fast_plug_option->{$opt} = 1;
};
# hotplug changes in [PENDING]
$errors->{$opt} = "hotplug problem - $msg";
};
+ my $cloudinit_pending_properties = PVE::QemuServer::cloudinit_pending_properties();
+
+ my $cloudinit_record_changed = sub {
+ my ($conf, $opt, $old, $new) = @_;
+ return if !$cloudinit_pending_properties->{$opt};
+
+ my $ci = ($conf->{cloudinit} //= {});
+
+ my $recorded = $ci->{$opt};
+ my %added = map { $_ => 1 } PVE::Tools::split_list(delete($ci->{added}) // '');
+
+ if (defined($new)) {
+ if (defined($old)) {
+ # an existing value is being modified
+ if (defined($recorded)) {
+ # the value was already not in sync
+ if ($new eq $recorded) {
+ # a value is being reverted to the cloud-init state:
+ delete $ci->{$opt};
+ delete $added{$opt};
+ } else {
+ # the value was changed multiple times, do nothing
+ }
+ } elsif ($added{$opt}) {
+ # the value had been marked as added and is being changed, do nothing
+ } else {
+ # the value is new, record it:
+ $ci->{$opt} = $old;
+ }
+ } else {
+ # a new value is being added
+ if (defined($recorded)) {
+ # it was already not in sync
+ if ($new eq $recorded) {
+ # a value is being reverted to the cloud-init state:
+ delete $ci->{$opt};
+ delete $added{$opt};
+ } else {
+ # the value had temporarily been removed, do nothing
+ }
+ } elsif ($added{$opt}) {
+ # the value had been marked as added already, do nothing
+ } else {
+ # the value is new, add it
+ $added{$opt} = 1;
+ }
+ }
+ } elsif (!defined($old)) {
+ # a non-existent value is being removed? ignore...
+ } else {
+ # a value is being deleted
+ if (defined($recorded)) {
+ # a value was already recorded, just keep it
+ } elsif ($added{$opt}) {
+ # the value was marked as added, remove it
+ delete $added{$opt};
+ } else {
+ # a previously unrecorded value is being removed, record the old value:
+ $ci->{$opt} = $old;
+ }
+ }
+
+ my $added = join(',', sort keys %added);
+ $ci->{added} = $added if length($added);
+ };
+
my $changes = 0;
foreach my $opt (keys %{$conf->{pending}}) { # add/change
if ($fast_plug_option->{$opt}) {
- $conf->{$opt} = $conf->{pending}->{$opt};
- delete $conf->{pending}->{$opt};
+ my $new = delete $conf->{pending}->{$opt};
+ $cloudinit_record_changed->($conf, $opt, $conf->{$opt}, $new);
+ $conf->{$opt} = $new;
$changes = 1;
}
}
PVE::QemuConfig->write_config($vmid, $conf);
}
+ my $ostype = $conf->{ostype};
+ my $version = extract_version($machine_type, get_running_qemu_version($vmid));
my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
+ my $usb_hotplug = $hotplug_features->{usb}
+ && min_version($version, 7, 1)
+ && defined($ostype) && ($ostype eq 'l26' || windows_version($ostype) > 7);
+ my $cgroup = PVE::QemuServer::CGroup->new($vmid);
my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
+
foreach my $opt (sort keys %$pending_delete_hash) {
next if $selection && !$selection->{$opt};
my $force = $pending_delete_hash->{$opt}->{force};
eval {
if ($opt eq 'hotplug') {
- die "skip\n" if ($conf->{hotplug} =~ /memory/);
+ die "skip\n" if ($conf->{hotplug} =~ /(cpu|memory)/);
} elsif ($opt eq 'tablet') {
die "skip\n" if !$hotplug_features->{usb};
if ($defaults->{tablet}) {
vm_deviceunplug($vmid, $conf, 'tablet');
vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
}
- } elsif ($opt =~ m/^usb\d+/) {
- die "skip\n";
- # since we cannot reliably hot unplug usb devices we are disabling it
- #die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
- #vm_deviceunplug($vmid, $conf, $opt);
+ } elsif ($opt =~ m/^usb(\d+)$/) {
+ my $index = $1;
+ die "skip\n" if !$usb_hotplug;
+ vm_deviceunplug($vmid, $conf, "usbredirdev$index"); # if it's a spice port
+ vm_deviceunplug($vmid, $conf, $opt);
} elsif ($opt eq 'vcpus') {
die "skip\n" if !$hotplug_features->{cpu};
qemu_cpu_hotplug($vmid, $conf, undef);
# enable balloon device is not hotpluggable
die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
# here we reset the ballooning value to memory
- my $balloon = $conf->{memory} || $defaults->{memory};
+ my $balloon = get_current_memory($conf->{memory});
mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
} elsif ($fast_plug_option->{$opt}) {
# do nothing
} elsif ($opt =~ m/^net(\d+)$/) {
die "skip\n" if !$hotplug_features->{network};
vm_deviceunplug($vmid, $conf, $opt);
+ if($have_sdn) {
+ my $net = PVE::QemuServer::parse_net($conf->{$opt});
+ PVE::Network::SDN::Vnets::del_ips_from_mac($net->{bridge}, $net->{macaddr}, $conf->{name});
+ }
} elsif (is_valid_drivename($opt)) {
die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
vm_deviceunplug($vmid, $conf, $opt);
vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
} elsif ($opt =~ m/^memory$/) {
die "skip\n" if !$hotplug_features->{memory};
- PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
+ PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf);
} elsif ($opt eq 'cpuunits') {
- cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
+ $cgroup->change_cpu_shares(undef);
} elsif ($opt eq 'cpulimit') {
- cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
+ $cgroup->change_cpu_quota(undef, undef); # reset, cgroup module can better decide values
} else {
die "skip\n";
}
if (my $err = $@) {
&$add_error($opt, $err) if $err ne "skip\n";
} else {
- delete $conf->{$opt};
+ my $old = delete $conf->{$opt};
+ $cloudinit_record_changed->($conf, $opt, $old, undef);
PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
}
}
- my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
- $apply_pending_cloudinit = sub {
- return if $apply_pending_cloudinit_done; # once is enough
- $apply_pending_cloudinit_done = 1; # once is enough
-
- my ($key, $value) = @_;
-
- my @cloudinit_opts = keys %$confdesc_cloudinit;
- foreach my $opt (keys %{$conf->{pending}}) {
- next if !grep { $_ eq $opt } @cloudinit_opts;
- $conf->{$opt} = delete $conf->{pending}->{$opt};
- }
-
- my $new_conf = { %$conf };
- $new_conf->{$key} = $value;
- PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
- };
-
+ my $cloudinit_opt;
foreach my $opt (keys %{$conf->{pending}}) {
next if $selection && !$selection->{$opt};
my $value = $conf->{pending}->{$opt};
eval {
if ($opt eq 'hotplug') {
die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
+ die "skip\n" if ($value =~ /cpu/) || ($value !~ /cpu/ && $conf->{hotplug} =~ /cpu/);
} elsif ($opt eq 'tablet') {
die "skip\n" if !$hotplug_features->{usb};
if ($value == 1) {
vm_deviceunplug($vmid, $conf, 'tablet');
vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
}
- } elsif ($opt =~ m/^usb\d+$/) {
- die "skip\n";
- # since we cannot reliably hot unplug usb devices we disable it for now
- #die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
- #my $d = eval { parse_property_string($usbdesc->{format}, $value) };
- #die "skip\n" if !$d;
- #qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
+ } elsif ($opt =~ m/^usb(\d+)$/) {
+ my $index = $1;
+ die "skip\n" if !$usb_hotplug;
+ my $d = eval { parse_property_string('pve-qm-usb', $value) };
+ my $id = $opt;
+ if ($d->{host} =~ m/^spice$/i) {
+ $id = "usbredirdev$index";
+ }
+ qemu_usb_hotplug($storecfg, $conf, $vmid, $id, $d, $arch, $machine_type);
} elsif ($opt eq 'vcpus') {
die "skip\n" if !$hotplug_features->{cpu};
qemu_cpu_hotplug($vmid, $conf, $value);
# allow manual ballooning if shares is set to zero
if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
- my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
+ my $memory = get_current_memory($conf->{memory});
+ my $balloon = $conf->{pending}->{balloon} || $memory;
mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
}
} elsif ($opt =~ m/^net(\d+)$/) {
vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
$vmid, $opt, $value, $arch, $machine_type);
} elsif (is_valid_drivename($opt)) {
- die "skip\n" if $opt eq 'efidisk0';
+ die "skip\n" if $opt eq 'efidisk0' || $opt eq 'tpmstate0';
# some changes can be done without hotplug
my $drive = parse_drive($opt, $value);
if (drive_is_cloudinit($drive)) {
- &$apply_pending_cloudinit($opt, $value);
+ $cloudinit_opt = [$opt, $drive];
+ # apply all the other changes first, then generate the cloudinit disk
+ die "skip\n";
}
vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
$vmid, $opt, $value, $arch, $machine_type);
} elsif ($opt =~ m/^memory$/) { #dimms
die "skip\n" if !$hotplug_features->{memory};
- $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
+ $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $value);
} elsif ($opt eq 'cpuunits') {
- cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
+ my $new_cpuunits = PVE::CGroup::clamp_cpu_shares($conf->{pending}->{$opt}); #clamp
+ $cgroup->change_cpu_shares($new_cpuunits);
} elsif ($opt eq 'cpulimit') {
my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
- cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
+ $cgroup->change_cpu_quota($cpulimit, 100000);
+ } elsif ($opt eq 'agent') {
+ vmconfig_update_agent($conf, $opt, $value);
} else {
die "skip\n"; # skip non-hot-pluggable options
}
if (my $err = $@) {
&$add_error($opt, $err) if $err ne "skip\n";
} else {
+ $cloudinit_record_changed->($conf, $opt, $conf->{$opt}, $value);
$conf->{$opt} = $value;
delete $conf->{pending}->{$opt};
}
}
+ if (defined($cloudinit_opt)) {
+ my ($opt, $drive) = @$cloudinit_opt;
+ my $value = $conf->{pending}->{$opt};
+ eval {
+ my $temp = {%$conf, $opt => $value};
+ PVE::QemuServer::Cloudinit::apply_cloudinit_config($temp, $vmid);
+ vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
+ $vmid, $opt, $value, $arch, $machine_type);
+ };
+ if (my $err = $@) {
+ &$add_error($opt, $err) if $err ne "skip\n";
+ } else {
+ $conf->{$opt} = $value;
+ delete $conf->{pending}->{$opt};
+ }
+ }
+
+ # unplug xhci controller if no usb device is left
+ if ($usb_hotplug) {
+ my $has_usb = 0;
+ for (my $i = 0; $i < $PVE::QemuServer::USB::MAX_USB_DEVICES; $i++) {
+ next if !defined($conf->{"usb$i"});
+ $has_usb = 1;
+ last;
+ }
+ if (!$has_usb) {
+ vm_deviceunplug($vmid, $conf, 'xhci');
+ }
+ }
+
PVE::QemuConfig->write_config($vmid, $conf);
+
+ if ($hotplug_features->{cloudinit} && PVE::QemuServer::Cloudinit::has_changes($conf)) {
+ PVE::QemuServer::vmconfig_update_cloudinit_drive($storecfg, $conf, $vmid);
+ }
}
sub try_deallocate_drive {
sub vmconfig_apply_pending {
- my ($vmid, $conf, $storecfg, $errors) = @_;
+ my ($vmid, $conf, $storecfg, $errors, $skip_cloud_init) = @_;
+
+ return if !scalar(keys %{$conf->{pending}});
my $add_apply_error = sub {
my ($opt, $msg) = @_;
die "internal error";
} elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
+ } elsif (defined($conf->{$opt}) && $opt =~ m/^net\d+$/) {
+ if($have_sdn) {
+ my $net = PVE::QemuServer::parse_net($conf->{$opt});
+ eval { PVE::Network::SDN::Vnets::del_ips_from_mac($net->{bridge}, $net->{macaddr}, $conf->{name}) };
+ warn if $@;
+ }
}
};
if (my $err = $@) {
PVE::QemuConfig->cleanup_pending($conf);
+ my $generate_cloudinit = $skip_cloud_init ? 0 : undef;
+
foreach my $opt (keys %{$conf->{pending}}) { # add/change
next if $opt eq 'delete'; # just to be sure
eval {
if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
+ } elsif (defined($conf->{pending}->{$opt}) && $opt =~ m/^net\d+$/) {
+ return if !$have_sdn; # return from eval if SDN is not available
+
+ my $new_net = PVE::QemuServer::parse_net($conf->{pending}->{$opt});
+ if ($conf->{$opt}) {
+ my $old_net = PVE::QemuServer::parse_net($conf->{$opt});
+
+ if (defined($old_net->{bridge}) && defined($old_net->{macaddr}) && (
+ safe_string_ne($old_net->{bridge}, $new_net->{bridge}) ||
+ safe_string_ne($old_net->{macaddr}, $new_net->{macaddr})
+ )) {
+ PVE::Network::SDN::Vnets::del_ips_from_mac($old_net->{bridge}, $old_net->{macaddr}, $conf->{name});
+ }
+ }
+ #fixme: reuse ip if mac change && same bridge
+ PVE::Network::SDN::Vnets::add_next_free_cidr($new_net->{bridge}, $conf->{name}, $new_net->{macaddr}, $vmid, undef, 1);
}
};
if (my $err = $@) {
$add_apply_error->($opt, $err);
} else {
+
+ if (is_valid_drivename($opt)) {
+ my $drive = parse_drive($opt, $conf->{pending}->{$opt});
+ $generate_cloudinit //= 1 if drive_is_cloudinit($drive);
+ }
+
$conf->{$opt} = delete $conf->{pending}->{$opt};
}
}
# write all changes at once to avoid unnecessary i/o
PVE::QemuConfig->write_config($vmid, $conf);
+ if ($generate_cloudinit) {
+ if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) {
+ # After successful generation and if there were changes to be applied, update the
+ # config to drop the {cloudinit} entry.
+ PVE::QemuConfig->write_config($vmid, $conf);
+ }
+ }
}
sub vmconfig_update_net {
if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
- !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
+ safe_num_ne($oldnet->{mtu}, $newnet->{mtu}) ||
+ !($newnet->{bridge} && $oldnet->{bridge})
+ ) { # bridge/nat mode change
# for non online change, we try to hot-unplug
die "skip\n" if !$hotplug;
vm_deviceunplug($vmid, $conf, $opt);
+
+ if ($have_sdn) {
+ PVE::Network::SDN::Vnets::del_ips_from_mac($oldnet->{bridge}, $oldnet->{macaddr}, $conf->{name});
+ }
+
} else {
die "internal error" if $opt !~ m/net(\d+)/;
if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
- safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
+ safe_num_ne($oldnet->{firewall}, $newnet->{firewall})
+ ) {
PVE::Network::tap_unplug($iface);
+ #set link_down in guest if bridge or vlan change to notify guest (dhcp renew for example)
+ if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
+ safe_num_ne($oldnet->{tag}, $newnet->{tag})
+ ) {
+ qemu_set_link_status($vmid, $opt, 0);
+ }
+
+ if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge})) {
+ if ($have_sdn) {
+ PVE::Network::SDN::Vnets::del_ips_from_mac($oldnet->{bridge}, $oldnet->{macaddr}, $conf->{name});
+ PVE::Network::SDN::Vnets::add_next_free_cidr($newnet->{bridge}, $conf->{name}, $newnet->{macaddr}, $vmid, undef, 1);
+ }
+ }
+
if ($have_sdn) {
PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
} else {
PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
}
+
+ #set link_up in guest if bridge or vlan change to notify guest (dhcp renew for example)
+ if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
+ safe_num_ne($oldnet->{tag}, $newnet->{tag})
+ ) {
+ qemu_set_link_status($vmid, $opt, 1);
+ }
+
} elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
# Rate can be applied on its own but any change above needs to
# include the rate in tap_plug since OVS resets everything.
}
if ($hotplug) {
+ if ($have_sdn) {
+ PVE::Network::SDN::Vnets::add_next_free_cidr($newnet->{bridge}, $conf->{name}, $newnet->{macaddr}, $vmid, undef, 1);
+ PVE::Network::SDN::Vnets::add_dhcp_mapping($newnet->{bridge}, $newnet->{macaddr}, $vmid, $conf->{name});
+ }
vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
} else {
die "skip\n";
}
}
+sub vmconfig_update_agent {
+ my ($conf, $opt, $value) = @_;
+
+ die "skip\n" if !$conf->{$opt};
+
+ my $hotplug_options = { fstrim_cloned_disks => 1 };
+
+ my $old_agent = parse_guest_agent($conf);
+ my $agent = parse_guest_agent({$opt => $value});
+
+ for my $option (keys %$agent) { # added/changed options
+ next if defined($hotplug_options->{$option});
+ die "skip\n" if safe_string_ne($agent->{$option}, $old_agent->{$option});
+ }
+
+ for my $option (keys %$old_agent) { # removed options
+ next if defined($hotplug_options->{$option});
+ die "skip\n" if safe_string_ne($old_agent->{$option}, $agent->{$option});
+ }
+
+ return; # either no actual change (e.g., format string reordered) or just hotpluggable changes
+}
+
sub vmconfig_update_disk {
my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
# update existing disk
# skip non hotpluggable value
- if (safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
+ if (safe_string_ne($drive->{aio}, $old_drive->{aio}) ||
+ safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
+ safe_string_ne($drive->{product}, $old_drive->{product}) ||
safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
- safe_string_ne($drive->{ssd}, $old_drive->{ssd})) {
+ safe_string_ne($drive->{ssd}, $old_drive->{ssd}) ||
+ safe_string_ne($drive->{vendor}, $old_drive->{vendor}) ||
+ safe_string_ne($drive->{ro}, $old_drive->{ro})) {
die "skip\n";
}
vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
}
+sub vmconfig_update_cloudinit_drive {
+ my ($storecfg, $conf, $vmid) = @_;
+
+ my $cloudinit_ds = undef;
+ my $cloudinit_drive = undef;
+
+ PVE::QemuConfig->foreach_volume($conf, sub {
+ my ($ds, $drive) = @_;
+ if (PVE::QemuServer::drive_is_cloudinit($drive)) {
+ $cloudinit_ds = $ds;
+ $cloudinit_drive = $drive;
+ }
+ });
+
+ return if !$cloudinit_drive;
+
+ if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) {
+ PVE::QemuConfig->write_config($vmid, $conf);
+ }
+
+ my $running = PVE::QemuServer::check_running($vmid);
+
+ if ($running) {
+ my $path = PVE::Storage::path($storecfg, $cloudinit_drive->{file});
+ if ($path) {
+ mon_cmd($vmid, "eject", force => JSON::true, id => "$cloudinit_ds");
+ mon_cmd($vmid, "blockdev-change-medium", id => "$cloudinit_ds", filename => "$path");
+ }
+ }
+}
+
# called in locked context by incoming migration
sub vm_migrate_get_nbd_disks {
my ($storecfg, $conf, $replicated_volumes) = @_;
my ($ds, $drive) = @_;
return if drive_is_cdrom($drive);
+ return if $ds eq 'tpmstate0';
my $volid = $drive->{file};
my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
return if $scfg->{shared};
+ my $format = qemu_img_format($scfg, $volname);
+
# replicated disks re-use existing state via bitmap
my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
- $local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing];
+ $local_volumes->{$ds} = [$volid, $storeid, $drive, $use_existing, $format];
});
return $local_volumes;
}
sub vm_migrate_alloc_nbd_disks {
my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
- my $format = undef;
-
my $nbd = {};
foreach my $opt (sort keys %$source_volumes) {
- my ($volid, $storeid, $volname, $drive, $use_existing) = @{$source_volumes->{$opt}};
+ my ($volid, $storeid, $drive, $use_existing, $format) = @{$source_volumes->{$opt}};
if ($use_existing) {
$nbd->{$opt}->{drivestr} = print_drive($drive);
next;
}
- # If a remote storage is specified and the format of the original
- # volume is not available there, fall back to the default format.
- # Otherwise use the same format as the original.
- if (!$storagemap->{identity}) {
- $storeid = map_storage($storagemap, $storeid);
- my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
- my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
- my $fileFormat = qemu_img_format($scfg, $volname);
- $format = (grep {$fileFormat eq $_} @{$validFormats}) ? $fileFormat : $defFormat;
- } else {
- my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
- $format = qemu_img_format($scfg, $volname);
- }
+ $storeid = PVE::JSONSchema::map_id($storagemap, $storeid);
+
+ # order of precedence, filtered by whether storage supports it:
+ # 1. explicit requested format
+ # 2. default format of storage
+ my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
+ $format = $defFormat if !$format || !grep { $format eq $_ } $validFormats->@*;
my $size = $drive->{size} / 1024;
my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, $size);
if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf);
my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended');
+ my $has_backup_lock = PVE::QemuConfig->has_lock($conf, 'backup');
+
+ my $running = check_running($vmid, undef, $migrate_opts->{migratedfrom});
+
+ if ($has_backup_lock && $running) {
+ # a backup is currently running, attempt to start the guest in the
+ # existing QEMU instance
+ return vm_resume($vmid);
+ }
PVE::QemuConfig->check_lock($conf)
if !($params->{skiplock} || $has_suspended_lock);
$params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
- die "VM $vmid already running\n" if check_running($vmid, undef, $migrate_opts->{migratedfrom});
+ die "VM $vmid already running\n" if $running;
if (my $storagemap = $migrate_opts->{storagemap}) {
my $replicated = $migrate_opts->{replicated_volumes};
# statefile => 'tcp', 'unix' for migration or path/volid for RAM state
# skiplock => 0/1, skip checking for config lock
# skiptemplate => 0/1, skip checking whether VM is template
-# forcemachine => to force Qemu machine (rollback/migration)
+# forcemachine => to force QEMU machine (rollback/migration)
# forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
# timeout => in seconds
# paused => start VM in paused state (backup)
# resume => resume from hibernation
+# live-restore-backing => {
+# sata0 => {
+# name => blockdev-name,
+# blockdev => "arg to the -blockdev command instantiating device named 'name'",
+# },
+# virtio2 => ...
+# }
# migrate_opts:
# nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
# migratedfrom => source node
# network => CIDR of migration network
# type => secure/insecure - tunnel over encrypted connection or plain-text
# nbd_proto_version => int, 0 for TCP, 1 for UNIX
-# replicated_volumes = which volids should be re-used with bitmaps for nbd migration
+# replicated_volumes => which volids should be re-used with bitmaps for nbd migration
+# offline_volumes => new volids of offline migrated disks like tpmstate and cloudinit, not yet
+# contained in config
sub vm_start_nolock {
my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
$conf = PVE::QemuConfig->load_config($vmid); # update/reload
}
- PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
+ # don't regenerate the ISO if the VM is started as part of a live migration
+ # this way we can reuse the old ISO with the correct config
+ if (!$migratedfrom) {
+ if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) {
+ # FIXME: apply_cloudinit_config updates $conf in this case, and it would only drop
+ # $conf->{cloudinit}, so we could just not do this?
+ # But we do it above, so for now let's be consistent.
+ $conf = PVE::QemuConfig->load_config($vmid); # update/reload
+ }
+ }
+
+ # override offline migrated volumes, conf is out of date still
+ if (my $offline_volumes = $migrate_opts->{offline_volumes}) {
+ for my $key (sort keys $offline_volumes->%*) {
+ my $parsed = parse_drive($key, $conf->{$key});
+ $parsed->{file} = $offline_volumes->{$key};
+ $conf->{$key} = print_drive($parsed);
+ }
+ }
my $defaults = load_defaults();
# set environment variable useful inside network script
- $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
+ # for remote migration the config is available on the target node!
+ if (!$migrate_opts->{remote_node}) {
+ $ENV{PVE_MIGRATED_FROM} = $migratedfrom;
+ }
PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
print "Resuming suspended VM\n";
}
- my ($cmd, $vollist, $spice_port) =
- config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu);
+ my ($cmd, $vollist, $spice_port, $pci_devices) = config_to_command($storecfg, $vmid,
+ $conf, $defaults, $forcemachine, $forcecpu, $params->{'live-restore-backing'});
my $migration_ip;
my $get_migration_ip = sub {
return $migration_ip;
};
- my $migrate_uri;
if ($statefile) {
if ($statefile eq 'tcp') {
- my $localip = "localhost";
+ my $migrate = $res->{migrate} = { proto => 'tcp' };
+ $migrate->{addr} = "localhost";
my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
my $nodename = nodename();
}
if ($migration_type eq 'insecure') {
- $localip = $get_migration_ip->($nodename);
- $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
+ $migrate->{addr} = $get_migration_ip->($nodename);
+ $migrate->{addr} = "[$migrate->{addr}]" if Net::IP::ip_is_ipv6($migrate->{addr});
}
- my $pfamily = PVE::Tools::get_host_address_family($nodename);
- my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
- $migrate_uri = "tcp:${localip}:${migrate_port}";
- push @$cmd, '-incoming', $migrate_uri;
+ # see #4501: port reservation should be done close to usage - tell QEMU where to listen
+ # via QMP later
+ push @$cmd, '-incoming', 'defer';
push @$cmd, '-S';
} elsif ($statefile eq 'unix') {
# should be default for secure migrations as a ssh TCP forward
# tunnel is not deterministic reliable ready and fails regurarly
# to set up in time, so use UNIX socket forwards
- my $socket_addr = "/run/qemu-server/$vmid.migrate";
- unlink $socket_addr;
-
- $migrate_uri = "unix:$socket_addr";
+ my $migrate = $res->{migrate} = { proto => 'unix' };
+ $migrate->{addr} = "/run/qemu-server/$vmid.migrate";
+ unlink $migrate->{addr};
- push @$cmd, '-incoming', $migrate_uri;
+ $migrate->{uri} = "unix:$migrate->{addr}";
+ push @$cmd, '-incoming', $migrate->{uri};
push @$cmd, '-S';
} elsif (-e $statefile) {
push @$cmd, '-S';
}
- # host pci devices
- for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++) {
- my $d = parse_hostpci($conf->{"hostpci$i"});
- next if !$d;
- my $pcidevices = $d->{pciid};
- foreach my $pcidevice (@$pcidevices) {
- my $pciid = $pcidevice->{id};
+ my $memory = get_current_memory($conf->{memory});
+ my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $memory, $resume);
- my $info = PVE::SysFSTools::pci_device_info("$pciid");
- die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
- die "no pci device info for device '$pciid'\n" if !$info;
+ my $pci_reserve_list = [];
+ for my $device (values $pci_devices->%*) {
+ next if $device->{mdev}; # we don't reserve for mdev devices
+ push $pci_reserve_list->@*, map { $_->{id} } $device->{ids}->@*;
+ }
- if ($d->{mdev}) {
- my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
- PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
- } else {
- die "can't unbind/bind PCI group to VFIO '$pciid'\n"
- if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
- die "can't reset PCI device '$pciid'\n"
- if $info->{has_fl_reset} && !PVE::SysFSTools::pci_dev_reset($info);
+ # reserve all PCI IDs before actually doing anything with them
+ PVE::QemuServer::PCI::reserve_pci_usage($pci_reserve_list, $vmid, $start_timeout);
+
+ eval {
+ my $uuid;
+ for my $id (sort keys %$pci_devices) {
+ my $d = $pci_devices->{$id};
+ my ($index) = ($id =~ m/^hostpci(\d+)$/);
+
+ my $chosen_mdev;
+ for my $dev ($d->{ids}->@*) {
+ my $info = eval { PVE::QemuServer::PCI::prepare_pci_device($vmid, $dev->{id}, $index, $d->{mdev}) };
+ if ($d->{mdev}) {
+ warn $@ if $@;
+ $chosen_mdev = $info;
+ last if $chosen_mdev; # if successful, we're done
+ } else {
+ die $@ if $@;
+ }
}
- }
+
+ next if !$d->{mdev};
+ die "could not create mediated device\n" if !defined($chosen_mdev);
+
+ # nvidia grid needs the uuid of the mdev as qemu parameter
+ if (!defined($uuid) && $chosen_mdev->{vendor} =~ m/^(0x)?10de$/) {
+ if (defined($conf->{smbios1})) {
+ my $smbios_conf = parse_smbios1($conf->{smbios1});
+ $uuid = $smbios_conf->{uuid} if defined($smbios_conf->{uuid});
+ }
+ $uuid = PVE::QemuServer::PCI::generate_mdev_uuid($vmid, $index) if !defined($uuid);
+ }
+ }
+ push @$cmd, '-uuid', $uuid if defined($uuid);
+ };
+ if (my $err = $@) {
+ eval { cleanup_pci_devices($vmid, $conf) };
+ warn $@ if $@;
+ die $err;
}
PVE::Storage::activate_volumes($storecfg, $vollist);
- eval {
- run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
- outfunc => sub {}, errfunc => sub {});
- };
+
+ my %silence_std_outs = (outfunc => sub {}, errfunc => sub {});
+ eval { run_command(['/bin/systemctl', 'reset-failed', "$vmid.scope"], %silence_std_outs) };
+ eval { run_command(['/bin/systemctl', 'stop', "$vmid.scope"], %silence_std_outs) };
# Issues with the above 'stop' not being fully completed are extremely rare, a very low
# timeout should be more than enough here...
- PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
+ PVE::Systemd::wait_for_unit_removed("$vmid.scope", 20);
- my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
- : $defaults->{cpuunits};
+ my $cpuunits = PVE::CGroup::clamp_cpu_shares($conf->{cpuunits});
- my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $resume);
my %run_params = (
timeout => $statefile ? undef : $start_timeout,
umask => 0077,
$run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
}
- my %properties = (
+ my %systemd_properties = (
Slice => 'qemu.slice',
- KillMode => 'none',
- CPUShares => $cpuunits
+ KillMode => 'process',
+ SendSIGKILL => 0,
+ TimeoutStopUSec => ULONG_MAX, # infinity
);
+ if (PVE::CGroup::cgroup_mode() == 2) {
+ $systemd_properties{CPUWeight} = $cpuunits;
+ } else {
+ $systemd_properties{CPUShares} = $cpuunits;
+ }
+
if (my $cpulimit = $conf->{cpulimit}) {
- $properties{CPUQuota} = int($cpulimit * 100);
+ $systemd_properties{CPUQuota} = int($cpulimit * 100);
}
- $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
+ $systemd_properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
my $run_qemu = sub {
PVE::Tools::run_fork sub {
- PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
+ PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %systemd_properties);
+
+ my $tpmpid;
+ if ((my $tpm = $conf->{tpmstate0}) && !PVE::QemuConfig->is_template($conf)) {
+ # start the TPM emulator so QEMU can connect on start
+ $tpmpid = start_swtpm($storecfg, $vmid, $tpm, $migratedfrom);
+ }
my $exitcode = run_command($cmd, %run_params);
- die "QEMU exited with code $exitcode\n" if $exitcode;
+ if ($exitcode) {
+ if ($tpmpid) {
+ warn "stopping swtpm instance (pid $tpmpid) due to QEMU startup error\n";
+ kill 'TERM', $tpmpid;
+ }
+ die "QEMU exited with code $exitcode\n";
+ }
};
};
if ($conf->{hugepages}) {
my $code = sub {
- my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
+ my $hotplug_features =
+ parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
+ my $hugepages_topology =
+ PVE::QemuServer::Memory::hugepages_topology($conf, $hotplug_features->{memory});
+
my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
PVE::QemuServer::Memory::hugepages_mount();
if (my $err = $@) {
# deactivate volumes if start fails
eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
+ warn $@ if $@;
+ eval { cleanup_pci_devices($vmid, $conf) };
+ warn $@ if $@;
+
die "start failed: $err";
}
- print "migration listens on $migrate_uri\n" if $migrate_uri;
- $res->{migrate_uri} = $migrate_uri;
+ # re-reserve all PCI IDs now that we can know the actual VM PID
+ my $pid = PVE::QemuServer::Helpers::vm_running_locally($vmid);
+ eval { PVE::QemuServer::PCI::reserve_pci_usage($pci_reserve_list, $vmid, undef, $pid) };
+ warn $@ if $@;
- if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix') {
+ if (defined(my $migrate = $res->{migrate})) {
+ if ($migrate->{proto} eq 'tcp') {
+ my $nodename = nodename();
+ my $pfamily = PVE::Tools::get_host_address_family($nodename);
+ $migrate->{port} = PVE::Tools::next_migrate_port($pfamily);
+ $migrate->{uri} = "tcp:$migrate->{addr}:$migrate->{port}";
+ mon_cmd($vmid, "migrate-incoming", uri => $migrate->{uri});
+ }
+ print "migration listens on $migrate->{uri}\n";
+ } elsif ($statefile) {
eval { mon_cmd($vmid, "cont"); };
warn $@ if $@;
}
my $migrate_storage_uri;
# nbd_protocol_version > 0 for unix socket support
- if ($nbd_protocol_version > 0 && $migration_type eq 'secure') {
+ if ($nbd_protocol_version > 0 && ($migration_type eq 'secure' || $migration_type eq 'websocket')) {
my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
$migrate_storage_uri = "nbd:unix:$socket_path";
+ $res->{migrate}->{unix_sockets} = [$socket_path];
} else {
my $nodename = nodename();
my $localip = $get_migration_ip->($nodename);
$migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
}
- $res->{migrate_storage_uri} = $migrate_storage_uri;
+ my $block_info = mon_cmd($vmid, "query-block");
+ $block_info = { map { $_->{device} => $_ } $block_info->@* };
foreach my $opt (sort keys %$nbd) {
my $drivestr = $nbd->{$opt}->{drivestr};
my $volid = $nbd->{$opt}->{volid};
- mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
+
+ my $block_node = $block_info->{"drive-$opt"}->{inserted}->{'node-name'};
+
+ mon_cmd(
+ $vmid,
+ "block-export-add",
+ id => "drive-$opt",
+ 'node-name' => $block_node,
+ writable => JSON::true,
+ type => "nbd",
+ name => "drive-$opt", # NBD export name
+ );
+
my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
print "storage migration listens on $nbd_uri volume:$drivestr\n";
print "re-using replicated volume: $opt - $volid\n"
my $nicconf = parse_net($conf->{$opt});
qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
}
+ add_nets_bridge_fdb($conf, $vmid);
}
- mon_cmd($vmid, 'qom-set',
+ if (!defined($conf->{balloon}) || $conf->{balloon}) {
+ eval {
+ mon_cmd(
+ $vmid,
+ 'qom-set',
path => "machine/peripheral/balloon0",
property => "guest-stats-polling-interval",
- value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
+ value => 2
+ );
+ };
+ log_warn("could not set polling interval for ballooning - $@") if $@;
+ }
if ($resume) {
print "Resumed VM, removing state\n";
PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
+ my ($current_machine, $is_deprecated) =
+ PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
+ if ($is_deprecated) {
+ log_warn(
+ "current machine version '$current_machine' is deprecated - see the documentation and ".
+ "change to a newer one",
+ );
+ }
+
return $res;
}
my ($storecfg, $vmid, $snapname) = @_;
my $conf = PVE::QemuConfig->load_config($vmid);
- my $forcemachine;
- my $forcecpu;
+ my ($forcemachine, $forcecpu);
if ($snapname) {
my $snapshot = $conf->{snapshots}->{$snapname};
die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
my $defaults = load_defaults();
- my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults,
- $forcemachine, $forcecpu);
+ my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu);
return PVE::Tools::cmd2string($cmd);
}
return $vollist;
}
+sub cleanup_pci_devices {
+ my ($vmid, $conf) = @_;
+
+ foreach my $key (keys %$conf) {
+ next if $key !~ m/^hostpci(\d+)$/;
+ my $hostpciindex = $1;
+ my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
+ my $d = parse_hostpci($conf->{$key});
+ if ($d->{mdev}) {
+ # NOTE: avoid PVE::SysFSTools::pci_cleanup_mdev_device as it requires PCI ID and we
+ # don't want to break ABI just for this two liner
+ my $dev_sysfs_dir = "/sys/bus/mdev/devices/$uuid";
+
+ # some nvidia vgpu driver versions want to clean the mdevs up themselves, and error
+ # out when we do it first. so wait for up to 10 seconds and then try it manually
+ if ($d->{ids}->[0]->[0]->{vendor} =~ m/^(0x)?10de$/ && -e $dev_sysfs_dir) {
+ my $count = 0;
+ while (-e $dev_sysfs_dir && $count < 10) {
+ sleep 1;
+ $count++;
+ }
+ print "waited $count seconds for mediated device driver finishing clean up\n";
+ }
+
+ if (-e $dev_sysfs_dir) {
+ print "actively clean up mediated device with UUID $uuid\n";
+ PVE::SysFSTools::file_write("$dev_sysfs_dir/remove", "1");
+ }
+ }
+ }
+ PVE::QemuServer::PCI::remove_pci_reservation($vmid);
+}
+
sub vm_stop_cleanup {
my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
if (!$keepActive) {
my $vollist = get_vm_volumes($conf);
PVE::Storage::deactivate_volumes($storecfg, $vollist);
+
+ if (my $tpmdrive = $conf->{tpmstate0}) {
+ my $tpm = parse_drive("tpmstate0", $tpmdrive);
+ my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
+ if ($storeid) {
+ PVE::Storage::unmap_volume($storecfg, $tpm->{file});
+ }
+ }
}
foreach my $ext (qw(mon qmp pid vnc qga)) {
unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
}
- foreach my $key (keys %$conf) {
- next if $key !~ m/^hostpci(\d+)$/;
- my $hostpciindex = $1;
- my $d = parse_hostpci($conf->{$key});
- my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
-
- foreach my $pci (@{$d->{pciid}}) {
- my $pciid = $pci->{id};
- PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
- }
- }
+ cleanup_pci_devices($vmid, $conf);
vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
};
eval {
if ($shutdown) {
- if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
+ if (defined($conf) && get_qga_key($conf, 'enabled')) {
mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
} else {
mon_cmd($vmid, "system_powerdown");
PVE::Storage::activate_volumes($storecfg, [$vmstate]);
eval {
+ set_migration_caps($vmid, 1);
mon_cmd($vmid, "savevm-start", statefile => $path);
for(;;) {
my $state = mon_cmd($vmid, "query-savevm");
}
}
+# $nocheck is set when called as part of a migration - in this context the
+# location of the config file (source or target node) is not deterministic,
+# since migration cannot wait for pmxcfs to process the rename
sub vm_resume {
my ($vmid, $skiplock, $nocheck) = @_;
PVE::QemuConfig->lock_config($vmid, sub {
my $res = mon_cmd($vmid, 'query-status');
my $resume_cmd = 'cont';
+ my $reset = 0;
+ my $conf;
+ if ($nocheck) {
+ $conf = eval { PVE::QemuConfig->load_config($vmid) }; # try on target node
+ if ($@) {
+ my $vmlist = PVE::Cluster::get_vmlist();
+ if (exists($vmlist->{ids}->{$vmid})) {
+ my $node = $vmlist->{ids}->{$vmid}->{node};
+ $conf = eval { PVE::QemuConfig->load_config($vmid, $node) }; # try on source node
+ }
+ if (!$conf) {
+ PVE::Cluster::cfs_update(); # vmlist was wrong, invalidate cache
+ $conf = PVE::QemuConfig->load_config($vmid); # last try on target node again
+ }
+ }
+ } else {
+ $conf = PVE::QemuConfig->load_config($vmid);
+ }
- if ($res->{status} && $res->{status} eq 'suspended') {
- $resume_cmd = 'system_wakeup';
+ if ($res->{status}) {
+ return if $res->{status} eq 'running'; # job done, go home
+ $resume_cmd = 'system_wakeup' if $res->{status} eq 'suspended';
+ $reset = 1 if $res->{status} eq 'shutdown';
}
if (!$nocheck) {
-
- my $conf = PVE::QemuConfig->load_config($vmid);
-
PVE::QemuConfig->check_lock($conf)
if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
}
+ if ($reset) {
+ # required if a VM shuts down during a backup and we get a resume
+ # request before the backup finishes for example
+ mon_cmd($vmid, "system_reset");
+ }
+
+ add_nets_bridge_fdb($conf, $vmid) if $resume_cmd eq 'cont';
+
mon_cmd($vmid, $resume_cmd);
});
}
});
}
+sub check_bridge_access {
+ my ($rpcenv, $authuser, $conf) = @_;
+
+ return 1 if $authuser eq 'root@pam';
+
+ for my $opt (sort keys $conf->%*) {
+ next if $opt !~ m/^net\d+$/;
+ my $net = parse_net($conf->{$opt});
+ my ($bridge, $tag, $trunks) = $net->@{'bridge', 'tag', 'trunks'};
+ PVE::GuestHelpers::check_vnet_access($rpcenv, $authuser, $bridge, $tag, $trunks);
+ }
+ return 1;
+};
+
+sub check_mapping_access {
+ my ($rpcenv, $user, $conf) = @_;
+
+ for my $opt (keys $conf->%*) {
+ if ($opt =~ m/^usb\d+$/) {
+ my $device = PVE::JSONSchema::parse_property_string('pve-qm-usb', $conf->{$opt});
+ if (my $host = $device->{host}) {
+ die "only root can set '$opt' config for real devices\n"
+ if $host !~ m/^spice$/i && $user ne 'root@pam';
+ } elsif ($device->{mapping}) {
+ $rpcenv->check_full($user, "/mapping/usb/$device->{mapping}", ['Mapping.Use']);
+ } else {
+ die "either 'host' or 'mapping' must be set.\n";
+ }
+ } elsif ($opt =~ m/^hostpci\d+$/) {
+ my $device = PVE::JSONSchema::parse_property_string('pve-qm-hostpci', $conf->{$opt});
+ if ($device->{host}) {
+ die "only root can set '$opt' config for non-mapped devices\n" if $user ne 'root@pam';
+ } elsif ($device->{mapping}) {
+ $rpcenv->check_full($user, "/mapping/pci/$device->{mapping}", ['Mapping.Use']);
+ } else {
+ die "either 'host' or 'mapping' must be set.\n";
+ }
+ }
+ }
+};
+
+sub check_restore_permissions {
+ my ($rpcenv, $user, $conf) = @_;
+
+ check_bridge_access($rpcenv, $user, $conf);
+ check_mapping_access($rpcenv, $user, $conf);
+}
# vzdump restore implementaion
sub tar_archive_read_firstfile {
my $restore_cleanup_oldconf = sub {
my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
+ my $kept_disks = {};
+
PVE::QemuConfig->foreach_volume($oldconf, sub {
my ($ds, $drive) = @_;
if (my $err = $@) {
warn $err;
}
+ } else {
+ $kept_disks->{$volid} = 1;
}
});
- # delete vmstate files, after the restore we have no snapshots anymore
- foreach my $snapname (keys %{$oldconf->{snapshots}}) {
+ # after the restore we have no snapshots anymore
+ for my $snapname (keys $oldconf->{snapshots}->%*) {
my $snap = $oldconf->{snapshots}->{$snapname};
if ($snap->{vmstate}) {
eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
warn $err;
}
}
+
+ for my $volid (keys $kept_disks->%*) {
+ eval { PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snapname); };
+ warn $@ if $@;
+ }
}
};
my $parse_backup_hints = sub {
my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
- my $virtdev_hash = {};
+ my $check_storage = sub { # assert if an image can be allocate
+ my ($storeid, $scfg) = @_;
+ die "Content type 'images' is not available on storage '$storeid'\n"
+ if !$scfg->{content}->{images};
+ $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace'])
+ if $user ne 'root@pam';
+ };
+ my $virtdev_hash = {};
while (defined(my $line = <$fh>)) {
if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
$devinfo->{$devname}->{format} = $format;
$devinfo->{$devname}->{storeid} = $storeid;
- # check permission on storage
- my $pool = $options->{pool}; # todo: do we need that?
- if ($user ne 'root@pam') {
- $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
- }
+ my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
+ $check_storage->($storeid, $scfg); # permission and content type check
$virtdev_hash->{$virtdev} = $devinfo->{$devname};
} elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
my $virtdev = $1;
my $drive = parse_drive($virtdev, $2);
+
if (drive_is_cloudinit($drive)) {
my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
$storeid = $options->{storage} if defined ($options->{storage});
my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
+ $check_storage->($storeid, $scfg); # permission and content type check
+
$virtdev_hash->{$virtdev} = {
format => $format,
storeid => $storeid,
my $name;
if ($d->{is_cloudinit}) {
$name = "vm-$vmid-cloudinit";
- $name .= ".$d->{format}" if $d->{format} ne 'raw';
+ my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
+ if ($scfg->{path}) {
+ $name .= ".$d->{format}";
+ }
}
my $volid = PVE::Storage::vdisk_alloc(
return $map;
};
-my $restore_update_config_line = sub {
- my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
+sub restore_update_config_line {
+ my ($cookie, $map, $line, $unique) = @_;
+
+ return '' if $line =~ m/^\#qmdump\#/;
+ return '' if $line =~ m/^\#vzdump\#/;
+ return '' if $line =~ m/^lock:/;
+ return '' if $line =~ m/^unused\d+:/;
+ return '' if $line =~ m/^parent:/;
- return if $line =~ m/^\#qmdump\#/;
- return if $line =~ m/^\#vzdump\#/;
- return if $line =~ m/^lock:/;
- return if $line =~ m/^unused\d+:/;
- return if $line =~ m/^parent:/;
+ my $res = '';
my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
};
my $netstr = print_net($net);
- print $outfd "net$cookie->{netcount}: $netstr\n";
+ $res .= "net$cookie->{netcount}: $netstr\n";
$cookie->{netcount}++;
}
} elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
my $net = parse_net($netstr);
$net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
$netstr = print_net($net);
- print $outfd "$id: $netstr\n";
- } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
+ $res .= "$id: $netstr\n";
+ } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk|tpmstate)\d+):\s*(\S+)\s*$/) {
my $virtdev = $1;
my $value = $3;
my $di = parse_drive($virtdev, $value);
if (defined($di->{backup}) && !$di->{backup}) {
- print $outfd "#$line";
+ $res .= "#$line";
} elsif ($map->{$virtdev}) {
delete $di->{format}; # format can change on restore
$di->{file} = $map->{$virtdev};
$value = print_drive($di);
- print $outfd "$virtdev: $value\n";
+ $res .= "$virtdev: $value\n";
} else {
- print $outfd $line;
+ $res .= $line;
}
} elsif (($line =~ m/^vmgenid: (.*)/)) {
my $vmgenid = $1;
# always generate a new vmgenid if there was a valid one setup
$vmgenid = generate_uuid();
}
- print $outfd "vmgenid: $vmgenid\n";
+ $res .= "vmgenid: $vmgenid\n";
} elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
my ($uuid, $uuid_str);
UUID::generate($uuid);
UUID::unparse($uuid, $uuid_str);
my $smbios1 = parse_smbios1($2);
$smbios1->{uuid} = $uuid_str;
- print $outfd $1.print_smbios1($smbios1)."\n";
+ $res .= $1.print_smbios1($smbios1)."\n";
} else {
- print $outfd $line;
+ $res .= $line;
}
-};
+
+ return $res;
+}
my $restore_deactivate_volumes = sub {
- my ($storecfg, $devinfo) = @_;
+ my ($storecfg, $virtdev_hash) = @_;
my $vollist = [];
- foreach my $devname (keys %$devinfo) {
- my $volid = $devinfo->{$devname}->{volid};
- push @$vollist, $volid if $volid;
+ for my $dev (values $virtdev_hash->%*) {
+ push $vollist->@*, $dev->{volid} if $dev->{volid};
}
- PVE::Storage::deactivate_volumes($storecfg, $vollist);
+ eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
+ print STDERR $@ if $@;
};
my $restore_destroy_volumes = sub {
- my ($storecfg, $devinfo) = @_;
+ my ($storecfg, $virtdev_hash) = @_;
- foreach my $devname (keys %$devinfo) {
- my $volid = $devinfo->{$devname}->{volid};
- next if !$volid;
+ for my $dev (values $virtdev_hash->%*) {
+ my $volid = $dev->{volid} or next;
eval {
- if ($volid =~ m|^/|) {
- unlink $volid || die 'unlink failed\n';
- } else {
- PVE::Storage::vdisk_free($storecfg, $volid);
- }
+ PVE::Storage::vdisk_free($storecfg, $volid);
print STDERR "temporary volume '$volid' sucessfuly removed\n";
};
print STDERR "unable to cleanup '$volid' - $@" if $@;
}
};
+sub restore_merge_config {
+ my ($filename, $backup_conf_raw, $override_conf) = @_;
+
+ my $backup_conf = parse_vm_config($filename, $backup_conf_raw);
+ for my $key (keys $override_conf->%*) {
+ $backup_conf->{$key} = $override_conf->{$key};
+ }
+
+ return $backup_conf;
+}
+
sub scan_volids {
my ($cfg, $vmid) = @_;
- my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
+ my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid, undef, 'images');
my $volid_hash = {};
foreach my $storeid (keys %$info) {
my $cfg = PVE::Storage::config();
- # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
- # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
- foreach my $stor (keys %{$cfg->{ids}}) {
- delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
- }
-
print "rescan volumes...\n";
my $volid_hash = scan_volids($cfg, $vmid);
my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive);
my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
- my $server = $scfg->{server};
- my $datastore = $scfg->{datastore};
- my $username = $scfg->{username} // 'root@pam';
my $fingerprint = $scfg->{fingerprint};
my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($storecfg, $storeid);
- my $repo = "$username\@$server:$datastore";
+ my $repo = PVE::PBSClient::get_repository($scfg);
+ my $namespace = $scfg->{namespace};
- # This is only used for `pbs-restore`!
+ # This is only used for `pbs-restore` and the QEMU PBS driver (live-restore)
my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
local $ENV{PBS_PASSWORD} = $password;
local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
mkpath $tmpdir;
my $conffile = PVE::QemuConfig->config_file($vmid);
- my $tmpfn = "$conffile.$$.tmp";
# disable interrupts (always do cleanups)
local $SIG{INT} =
local $SIG{TERM} =
# Note: $oldconf is undef if VM does not exists
my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
+ my $new_conf_raw = '';
my $rpcenv = PVE::RPCEnvironment::get();
- my $devinfo = {};
+ my $devinfo = {}; # info about drives included in backup
+ my $virtdev_hash = {}; # info about allocated drives
eval {
# enable interrupts
my $index = PVE::Tools::file_get_contents($index_fn);
$index = decode_json($index);
- # print Dumper($index);
foreach my $info (@{$index->{files}}) {
if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) {
my $devname = $1;
my $fh = IO::File->new($cfgfn, "r") ||
die "unable to read qemu-server.conf - $!\n";
- my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
+ $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
# fixme: rate limit?
my $d = $virtdev_hash->{$virtdev};
next if $d->{is_cloudinit}; # no need to restore cloudinit
+ # this fails if storage is unavailable
my $volid = $d->{volid};
-
my $path = PVE::Storage::path($storecfg, $volid);
- # This is the ONLY user of the PBS_ env vars set on top of this function!
+ # for live-restore we only want to preload the efidisk and TPM state
+ next if $options->{live} && $virtdev ne 'efidisk0' && $virtdev ne 'tpmstate0';
+
+ my @ns_arg;
+ if (defined(my $ns = $scfg->{namespace})) {
+ @ns_arg = ('--ns', $ns);
+ }
+
my $pbs_restore_cmd = [
'/usr/bin/pbs-restore',
'--repository', $repo,
+ @ns_arg,
$pbs_backup_name,
"$d->{devname}.img.fidx",
$path,
$fh->seek(0, 0) || die "seek failed - $!\n";
- my $outfd = IO::File->new($tmpfn, "w") || die "unable to write config for VM $vmid\n";
-
my $cookie = { netcount => 0 };
while (defined(my $line = <$fh>)) {
- $restore_update_config_line->($outfd, $cookie, $vmid, $map, $line, $options->{unique});
+ $new_conf_raw .= restore_update_config_line(
+ $cookie,
+ $map,
+ $line,
+ $options->{unique},
+ );
}
$fh->close();
- $outfd->close();
};
my $err = $@;
- $restore_deactivate_volumes->($storecfg, $devinfo);
+ if ($err || !$options->{live}) {
+ $restore_deactivate_volumes->($storecfg, $virtdev_hash);
+ }
rmtree $tmpdir;
if ($err) {
- unlink $tmpfn;
- $restore_destroy_volumes->($storecfg, $devinfo);
+ $restore_destroy_volumes->($storecfg, $virtdev_hash);
die $err;
}
- rename($tmpfn, $conffile) ||
- die "unable to commit configuration file '$conffile'\n";
+ if ($options->{live}) {
+ # keep lock during live-restore
+ $new_conf_raw .= "\nlock: create";
+ }
- PVE::Cluster::cfs_update(); # make sure we read new file
+ my $new_conf = restore_merge_config($conffile, $new_conf_raw, $options->{override_conf});
+ check_restore_permissions($rpcenv, $user, $new_conf);
+ PVE::QemuConfig->write_config($vmid, $new_conf);
eval { rescan($vmid, 1); };
warn $@ if $@;
+
+ PVE::AccessControl::add_vm_to_pool($vmid, $options->{pool}) if $options->{pool};
+
+ if ($options->{live}) {
+ # enable interrupts
+ local $SIG{INT} =
+ local $SIG{TERM} =
+ local $SIG{QUIT} =
+ local $SIG{HUP} =
+ local $SIG{PIPE} = sub { die "got signal ($!) - abort\n"; };
+
+ my $conf = PVE::QemuConfig->load_config($vmid);
+ die "cannot do live-restore for template\n" if PVE::QemuConfig->is_template($conf);
+
+ # these special drives are already restored before start
+ delete $devinfo->{'drive-efidisk0'};
+ delete $devinfo->{'drive-tpmstate0-backup'};
+
+ my $pbs_opts = {
+ repo => $repo,
+ keyfile => $keyfile,
+ snapshot => $pbs_backup_name,
+ namespace => $namespace,
+ };
+ pbs_live_restore($vmid, $conf, $storecfg, $devinfo, $pbs_opts);
+
+ PVE::QemuConfig->remove_lock($vmid, "create");
+ }
+}
+
+sub pbs_live_restore {
+ my ($vmid, $conf, $storecfg, $restored_disks, $opts) = @_;
+
+ print "starting VM for live-restore\n";
+ print "repository: '$opts->{repo}', snapshot: '$opts->{snapshot}'\n";
+
+ my $live_restore_backing = {};
+ for my $ds (keys %$restored_disks) {
+ $ds =~ m/^drive-(.*)$/;
+ my $confname = $1;
+ my $pbs_conf = {};
+ $pbs_conf = {
+ repository => $opts->{repo},
+ snapshot => $opts->{snapshot},
+ archive => "$ds.img.fidx",
+ };
+ $pbs_conf->{keyfile} = $opts->{keyfile} if -e $opts->{keyfile};
+ $pbs_conf->{namespace} = $opts->{namespace} if defined($opts->{namespace});
+
+ my $drive = parse_drive($confname, $conf->{$confname});
+ print "restoring '$ds' to '$drive->{file}'\n";
+
+ my $pbs_name = "drive-${confname}-pbs";
+ $live_restore_backing->{$confname} = {
+ name => $pbs_name,
+ blockdev => print_pbs_blockdev($pbs_conf, $pbs_name),
+ };
+ }
+
+ my $drives_streamed = 0;
+ eval {
+ # make sure HA doesn't interrupt our restore by stopping the VM
+ if (PVE::HA::Config::vm_is_ha_managed($vmid)) {
+ run_command(['ha-manager', 'set', "vm:$vmid", '--state', 'started']);
+ }
+
+ # start VM with backing chain pointing to PBS backup, environment vars for PBS driver
+ # in QEMU (PBS_PASSWORD and PBS_FINGERPRINT) are already set by our caller
+ vm_start_nolock($storecfg, $vmid, $conf, {paused => 1, 'live-restore-backing' => $live_restore_backing}, {});
+
+ my $qmeventd_fd = register_qmeventd_handle($vmid);
+
+ # begin streaming, i.e. data copy from PBS to target disk for every vol,
+ # this will effectively collapse the backing image chain consisting of
+ # [target <- alloc-track -> PBS snapshot] to just [target] (alloc-track
+ # removes itself once all backing images vanish with 'auto-remove=on')
+ my $jobs = {};
+ for my $ds (sort keys %$restored_disks) {
+ my $job_id = "restore-$ds";
+ mon_cmd($vmid, 'block-stream',
+ 'job-id' => $job_id,
+ device => "$ds",
+ );
+ $jobs->{$job_id} = {};
+ }
+
+ mon_cmd($vmid, 'cont');
+ qemu_drive_mirror_monitor($vmid, undef, $jobs, 'auto', 0, 'stream');
+
+ print "restore-drive jobs finished successfully, removing all tracking block devices"
+ ." to disconnect from Proxmox Backup Server\n";
+
+ for my $ds (sort keys %$restored_disks) {
+ mon_cmd($vmid, 'blockdev-del', 'node-name' => "$ds-pbs");
+ }
+
+ close($qmeventd_fd);
+ };
+
+ my $err = $@;
+
+ if ($err) {
+ warn "An error occurred during live-restore: $err\n";
+ _do_vm_stop($storecfg, $vmid, 1, 1, 10, 0, 1);
+ die "live-restore failed\n";
+ }
+}
+
+# Inspired by pbs live-restore, this restores with the disks being available as files.
+# Theoretically this can also be used to quick-start a full-clone vm if the
+# disks are all available as files.
+#
+# The mapping should provide a path by config entry, such as
+# `{ scsi0 => { format => <qcow2|raw|...>, path => "/path/to/file", sata1 => ... } }`
+#
+# This is used when doing a `create` call with the `--live-import` parameter,
+# where the disks get an `import-from=` property. The non-live part is
+# therefore already handled in the `$create_disks()` call happening in the
+# `create` api call
+sub live_import_from_files {
+ my ($mapping, $vmid, $conf, $restore_options) = @_;
+
+ my $live_restore_backing = {};
+ for my $dev (keys %$mapping) {
+ die "disk not support for live-restoring: '$dev'\n"
+ if !is_valid_drivename($dev) || $dev =~ /^(?:efidisk|tpmstate)/;
+
+ die "mapping contains disk '$dev' which does not exist in the config\n"
+ if !exists($conf->{$dev});
+
+ my $info = $mapping->{$dev};
+ my ($format, $path) = $info->@{qw(format path)};
+ die "missing path for '$dev' mapping\n" if !$path;
+ die "missing format for '$dev' mapping\n" if !$format;
+ die "invalid format '$format' for '$dev' mapping\n"
+ if !grep { $format eq $_ } qw(raw qcow2 vmdk);
+
+ $live_restore_backing->{$dev} = {
+ name => "drive-$dev-restore",
+ blockdev => "driver=$format,node-name=drive-$dev-restore"
+ . ",read-only=on"
+ . ",file.driver=file,file.filename=$path"
+ };
+ };
+
+ my $storecfg = PVE::Storage::config();
+ eval {
+
+ # make sure HA doesn't interrupt our restore by stopping the VM
+ if (PVE::HA::Config::vm_is_ha_managed($vmid)) {
+ run_command(['ha-manager', 'set', "vm:$vmid", '--state', 'started']);
+ }
+
+ vm_start_nolock($storecfg, $vmid, $conf, {paused => 1, 'live-restore-backing' => $live_restore_backing}, {});
+
+ # prevent shutdowns from qmeventd when the VM powers off from the inside
+ my $qmeventd_fd = register_qmeventd_handle($vmid);
+
+ # begin streaming, i.e. data copy from PBS to target disk for every vol,
+ # this will effectively collapse the backing image chain consisting of
+ # [target <- alloc-track -> PBS snapshot] to just [target] (alloc-track
+ # removes itself once all backing images vanish with 'auto-remove=on')
+ my $jobs = {};
+ for my $ds (sort keys %$live_restore_backing) {
+ my $job_id = "restore-$ds";
+ mon_cmd($vmid, 'block-stream',
+ 'job-id' => $job_id,
+ device => "drive-$ds",
+ );
+ $jobs->{$job_id} = {};
+ }
+
+ mon_cmd($vmid, 'cont');
+ qemu_drive_mirror_monitor($vmid, undef, $jobs, 'auto', 0, 'stream');
+
+ print "restore-drive jobs finished successfully, removing all tracking block devices\n";
+
+ for my $ds (sort keys %$live_restore_backing) {
+ mon_cmd($vmid, 'blockdev-del', 'node-name' => "drive-$ds-restore");
+ }
+
+ close($qmeventd_fd);
+ };
+
+ my $err = $@;
+
+ if ($err) {
+ warn "An error occurred during live-restore: $err\n";
+ _do_vm_stop($storecfg, $vmid, 1, 1, 10, 0, 1);
+ die "live-restore failed\n";
+ }
+
+ PVE::QemuConfig->remove_lock($vmid, "import");
}
sub restore_vma_archive {
$add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
- my $oldtimeout;
- my $timeout = 5;
-
- my $devinfo = {};
+ my $devinfo = {}; # info about drives included in backup
+ my $virtdev_hash = {}; # info about allocated drives
my $rpcenv = PVE::RPCEnvironment::get();
my $conffile = PVE::QemuConfig->config_file($vmid);
- my $tmpfn = "$conffile.$$.tmp";
# Note: $oldconf is undef if VM does not exist
my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
+ my $new_conf_raw = '';
my %storage_limits;
PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
}
- my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
+ $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
+
+ foreach my $info (values %{$virtdev_hash}) {
+ my $storeid = $info->{storeid};
+ next if defined($storage_limits{$storeid});
- foreach my $key (keys %storage_limits) {
- my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
- next if !$limit;
- print STDERR "rate limit for storage $key: $limit KiB/s\n";
- $storage_limits{$key} = $limit * 1024;
+ my $limit = PVE::Storage::get_bandwidth_limit('restore', [$storeid], $bwlimit) // 0;
+ print STDERR "rate limit for storage $storeid: $limit KiB/s\n" if $limit;
+ $storage_limits{$storeid} = $limit * 1024;
}
foreach my $devname (keys %$devinfo) {
$fh->seek(0, 0) || die "seek failed - $!\n";
- my $outfd = IO::File->new($tmpfn, "w") || die "unable to write config for VM $vmid\n";
-
my $cookie = { netcount => 0 };
while (defined(my $line = <$fh>)) {
- $restore_update_config_line->($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
+ $new_conf_raw .= restore_update_config_line(
+ $cookie,
+ $map,
+ $line,
+ $opts->{unique},
+ );
}
$fh->close();
- $outfd->close();
};
+ my $oldtimeout;
+
eval {
# enable interrupts
local $SIG{INT} =
local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
local $SIG{ALRM} = sub { die "got timeout\n"; };
- $oldtimeout = alarm($timeout);
+ $oldtimeout = alarm(5); # for reading the VMA header - might hang with a corrupted one
my $parser = sub {
my $line = shift;
$devinfo->{$devname} = { size => $size, dev_id => $dev_id };
} elsif ($line =~ m/^CTIME: /) {
# we correctly received the vma config, so we can disable
- # the timeout now for disk allocation (set to 10 minutes, so
- # that we always timeout if something goes wrong)
- alarm(600);
+ # the timeout now for disk allocation
+ alarm($oldtimeout || 0);
+ $oldtimeout = undef;
&$print_devmap();
print $fifofh "done\n";
- my $tmp = $oldtimeout || 0;
- $oldtimeout = undef;
- alarm($tmp);
close($fifofh);
$fifofh = undef;
}
alarm($oldtimeout) if $oldtimeout;
- $restore_deactivate_volumes->($cfg, $devinfo);
+ $restore_deactivate_volumes->($cfg, $virtdev_hash);
close($fifofh) if $fifofh;
unlink $mapfifo;
rmtree $tmpdir;
if ($err) {
- unlink $tmpfn;
- $restore_destroy_volumes->($cfg, $devinfo);
+ $restore_destroy_volumes->($cfg, $virtdev_hash);
die $err;
}
- rename($tmpfn, $conffile) ||
- die "unable to commit configuration file '$conffile'\n";
-
- PVE::Cluster::cfs_update(); # make sure we read new file
+ my $new_conf = restore_merge_config($conffile, $new_conf_raw, $opts->{override_conf});
+ check_restore_permissions($rpcenv, $user, $new_conf);
+ PVE::QemuConfig->write_config($vmid, $new_conf);
eval { rescan($vmid, 1); };
warn $@ if $@;
+
+ PVE::AccessControl::add_vm_to_pool($vmid, $opts->{pool}) if $opts->{pool};
}
sub restore_tar_archive {
my ($archive, $vmid, $user, $opts) = @_;
+ if (scalar(keys $opts->{override_conf}->%*) > 0) {
+ my $keystring = join(' ', keys $opts->{override_conf}->%*);
+ die "cannot pass along options ($keystring) when restoring from tar archive\n";
+ }
+
if ($archive ne '-') {
my $firstfile = tar_archive_read_firstfile($archive);
die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
local $ENV{VZDUMP_USER} = $user;
my $conffile = PVE::QemuConfig->config_file($vmid);
- my $tmpfn = "$conffile.$$.tmp";
+ my $new_conf_raw = '';
# disable interrupts (always do cleanups)
local $SIG{INT} =
my $srcfd = IO::File->new($confsrc, "r") || die "unable to open file '$confsrc'\n";
- my $outfd = IO::File->new($tmpfn, "w") || die "unable to write config for VM $vmid\n";
-
my $cookie = { netcount => 0 };
while (defined (my $line = <$srcfd>)) {
- $restore_update_config_line->($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
+ $new_conf_raw .= restore_update_config_line(
+ $cookie,
+ $map,
+ $line,
+ $opts->{unique},
+ );
}
$srcfd->close();
- $outfd->close();
};
if (my $err = $@) {
- unlink $tmpfn;
tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
die $err;
}
rmtree $tmpdir;
- rename $tmpfn, $conffile ||
- die "unable to commit configuration file '$conffile'\n";
+ PVE::Tools::file_set_contents($conffile, $new_conf_raw);
PVE::Cluster::cfs_update(); # make sure we read new file
rbd => 1,
};
sub do_snapshots_with_qemu {
- my ($storecfg, $volid) = @_;
+ my ($storecfg, $volid, $deviceid) = @_;
+
+ return if $deviceid =~ m/tpmstate0/;
my $storage_name = PVE::Storage::parse_volume_id($volid);
my $scfg = $storecfg->{ids}->{$storage_name};
eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
if ($@) {
- warn "Qemu Guest Agent is not running - $@" if !$nowarn;
+ warn "QEMU Guest Agent is not running - $@" if !$nowarn;
return 0;
}
return 1;
}
sub qemu_img_convert {
- my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
+ my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized, $bwlimit) = @_;
my $storecfg = PVE::Storage::config();
my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
$src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
$src_is_iscsi = ($src_path =~ m|^iscsi://|);
$cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
- } elsif (-f $src_volid) {
+ } elsif (-f $src_volid || -b $src_volid) {
$src_path = $src_volid;
if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
$src_format = $1;
if $snapname && $src_format && $src_format eq "qcow2";
push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
push @$cmd, '-T', $cachemode if defined($cachemode);
+ push @$cmd, '-r', "${bwlimit}K" if defined($bwlimit);
if ($src_is_iscsi) {
push @$cmd, '--image-opts';
if($line =~ m/\((\S+)\/100\%\)/){
my $percent = $1;
my $transferred = int($size * $percent / 100);
- my $remaining = $size - $transferred;
+ my $total_h = render_bytes($size, 1);
+ my $transferred_h = render_bytes($transferred, 1);
- print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
+ print "transferred $transferred_h of $total_h ($percent%)\n";
}
};
sub qemu_img_format {
my ($scfg, $volname) = @_;
- if ($scfg->{path} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
+ # FIXME: this entire function is kind of weird given that `parse_volname`
+ # also already gives us a format?
+ my $is_path_storage = $scfg->{path} || $scfg->{type} eq 'esxi';
+
+ if ($is_path_storage && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
return $1;
} else {
return "raw";
# 'complete': wait until all jobs are ready, block-job-complete them (default)
# 'cancel': wait until all jobs are ready, block-job-cancel them
# 'skip': wait until all jobs are ready, return with block jobs in ready state
+# 'auto': wait until all jobs disappear, only use for jobs which complete automatically
sub qemu_drive_mirror_monitor {
- my ($vmid, $vmiddst, $jobs, $completion, $qga) = @_;
+ my ($vmid, $vmiddst, $jobs, $completion, $qga, $op) = @_;
$completion //= 'complete';
+ $op //= "mirror";
eval {
my $err_complete = 0;
+ my $starttime = time ();
while (1) {
- die "storage migration timed out\n" if $err_complete > 300;
+ die "block job ('$op') timed out\n" if $err_complete > 300;
my $stats = mon_cmd($vmid, "query-block-jobs");
+ my $ctime = time();
- my $running_mirror_jobs = {};
- foreach my $stat (@$stats) {
- next if $stat->{type} ne 'mirror';
- $running_mirror_jobs->{$stat->{device}} = $stat;
+ my $running_jobs = {};
+ for my $stat (@$stats) {
+ next if $stat->{type} ne $op;
+ $running_jobs->{$stat->{device}} = $stat;
}
my $readycounter = 0;
- foreach my $job (keys %$jobs) {
+ for my $job_id (sort keys %$jobs) {
+ my $job = $running_jobs->{$job_id};
- if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
- print "$job : finished\n";
- delete $jobs->{$job};
+ my $vanished = !defined($job);
+ my $complete = defined($jobs->{$job_id}->{complete}) && $vanished;
+ if($complete || ($vanished && $completion eq 'auto')) {
+ print "$job_id: $op-job finished\n";
+ delete $jobs->{$job_id};
next;
}
- die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
+ die "$job_id: '$op' has been cancelled\n" if !defined($job);
- my $busy = $running_mirror_jobs->{$job}->{busy};
- my $ready = $running_mirror_jobs->{$job}->{ready};
- if (my $total = $running_mirror_jobs->{$job}->{len}) {
- my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
+ my $busy = $job->{busy};
+ my $ready = $job->{ready};
+ if (my $total = $job->{len}) {
+ my $transferred = $job->{offset} || 0;
my $remaining = $total - $transferred;
my $percent = sprintf "%.2f", ($transferred * 100 / $total);
- print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
+ my $duration = $ctime - $starttime;
+ my $total_h = render_bytes($total, 1);
+ my $transferred_h = render_bytes($transferred, 1);
+
+ my $status = sprintf(
+ "transferred $transferred_h of $total_h ($percent%%) in %s",
+ render_duration($duration),
+ );
+
+ if ($ready) {
+ if ($busy) {
+ $status .= ", still busy"; # shouldn't even happen? but mirror is weird
+ } else {
+ $status .= ", ready";
+ }
+ }
+ print "$job_id: $status\n" if !$jobs->{$job_id}->{ready};
+ $jobs->{$job_id}->{ready} = $ready;
}
- $readycounter++ if $running_mirror_jobs->{$job}->{ready};
+ $readycounter++ if $job->{ready};
}
last if scalar(keys %$jobs) == 0;
if ($readycounter == scalar(keys %$jobs)) {
- print "all mirroring jobs are ready \n";
- last if $completion eq 'skip'; #do the complete later
+ print "all '$op' jobs are ready\n";
+
+ # do the complete later (or has already been done)
+ last if $completion eq 'skip' || $completion eq 'auto';
if ($vmiddst && $vmiddst != $vmid) {
my $agent_running = $qga && qga_check_running($vmid);
if ($agent_running) {
print "freeze filesystem\n";
eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
+ warn $@ if $@;
} else {
print "suspend vm\n";
eval { PVE::QemuServer::vm_suspend($vmid, 1); };
+ warn $@ if $@;
}
# if we clone a disk for a new target vm, we don't switch the disk
if ($agent_running) {
print "unfreeze filesystem\n";
eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
+ warn $@ if $@;
} else {
print "resume vm\n";
- eval { PVE::QemuServer::vm_resume($vmid, 1, 1); };
+ eval { PVE::QemuServer::vm_resume($vmid, 1, 1); };
+ warn $@ if $@;
}
last;
} else {
- foreach my $job (keys %$jobs) {
+ for my $job_id (sort keys %$jobs) {
# try to switch the disk if source and destination are on the same guest
- print "$job: Completing block job...\n";
+ print "$job_id: Completing block job_id...\n";
my $op;
if ($completion eq 'complete') {
} else {
die "invalid completion value: $completion\n";
}
- eval { mon_cmd($vmid, $op, device => $job) };
+ eval { mon_cmd($vmid, $op, device => $job_id) };
if ($@ =~ m/cannot be completed/) {
- print "$job: Block job cannot be completed, try again.\n";
+ print "$job_id: block job cannot be completed, trying again.\n";
$err_complete++;
}else {
- print "$job: Completed successfully.\n";
- $jobs->{$job}->{complete} = 1;
+ print "$job_id: Completed successfully.\n";
+ $jobs->{$job_id}->{complete} = 1;
}
}
}
if ($err) {
eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
- die "mirroring error: $err";
+ die "block job ($op) error: $err";
}
-
}
sub qemu_blockjobs_cancel {
}
}
+# Check for bug #4525: drive-mirror will open the target drive with the same aio setting as the
+# source, but some storages have problems with io_uring, sometimes even leading to crashes.
+my sub clone_disk_check_io_uring {
+ my ($src_drive, $storecfg, $src_storeid, $dst_storeid, $use_drive_mirror) = @_;
+
+ return if !$use_drive_mirror;
+
+ # Don't complain when not changing storage.
+ # Assume if it works for the source, it'll work for the target too.
+ return if $src_storeid eq $dst_storeid;
+
+ my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
+ my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
+
+ my $cache_direct = drive_uses_cache_direct($src_drive);
+
+ my $src_uses_io_uring;
+ if ($src_drive->{aio}) {
+ $src_uses_io_uring = $src_drive->{aio} eq 'io_uring';
+ } else {
+ $src_uses_io_uring = storage_allows_io_uring_default($src_scfg, $cache_direct);
+ }
+
+ die "target storage is known to cause issues with aio=io_uring (used by current drive)\n"
+ if $src_uses_io_uring && !storage_allows_io_uring_default($dst_scfg, $cache_direct);
+}
+
sub clone_disk {
- my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
- $newvmid, $storage, $format, $full, $newvollist, $jobs, $completion, $qga, $bwlimit, $conf) = @_;
+ my ($storecfg, $source, $dest, $full, $newvollist, $jobs, $completion, $qga, $bwlimit) = @_;
+
+ my ($vmid, $running) = $source->@{qw(vmid running)};
+ my ($src_drivename, $drive, $snapname) = $source->@{qw(drivename drive snapname)};
+
+ my ($newvmid, $dst_drivename, $efisize) = $dest->@{qw(vmid drivename efisize)};
+ my ($storage, $format) = $dest->@{qw(storage format)};
+
+ my $unused = defined($src_drivename) && $src_drivename =~ /^unused/;
+ my $use_drive_mirror = $full && $running && $src_drivename && !$snapname && !$unused;
+
+ if ($src_drivename && $dst_drivename && $src_drivename ne $dst_drivename) {
+ die "cloning from/to EFI disk requires EFI disk\n"
+ if $src_drivename eq 'efidisk0' || $dst_drivename eq 'efidisk0';
+ die "cloning from/to TPM state requires TPM state\n"
+ if $src_drivename eq 'tpmstate0' || $dst_drivename eq 'tpmstate0';
+
+ # This would lead to two device nodes in QEMU pointing to the same backing image!
+ die "cannot change drive name when cloning disk from/to the same VM\n"
+ if $use_drive_mirror && $vmid == $newvmid;
+ }
+
+ die "cannot move TPM state while VM is running\n"
+ if $use_drive_mirror && $src_drivename eq 'tpmstate0';
my $newvolid;
+ print "create " . ($full ? 'full' : 'linked') . " clone of drive ";
+ print "$src_drivename " if $src_drivename;
+ print "($drive->{file})\n";
+
if (!$full) {
- print "create linked clone of drive $drivename ($drive->{file})\n";
$newvolid = PVE::Storage::vdisk_clone($storecfg, $drive->{file}, $newvmid, $snapname);
push @$newvollist, $newvolid;
} else {
-
- my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
- $storeid = $storage if $storage;
+ my ($src_storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
+ my $storeid = $storage || $src_storeid;
my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
- print "create full clone of drive $drivename ($drive->{file})\n";
my $name = undef;
my $size = undef;
if (drive_is_cloudinit($drive)) {
$name = "vm-$newvmid-cloudinit";
- $name .= ".$dst_format" if $dst_format ne 'raw';
+ my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
+ if ($scfg->{path}) {
+ $name .= ".$dst_format";
+ }
$snapname = undef;
$size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
- } elsif ($drivename eq 'efidisk0') {
- $size = get_efivars_size($conf);
+ } elsif ($dst_drivename eq 'efidisk0') {
+ $size = $efisize or die "internal error - need to specify EFI disk size\n";
+ } elsif ($dst_drivename eq 'tpmstate0') {
+ $dst_format = 'raw';
+ $size = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
} else {
- ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
+ clone_disk_check_io_uring($drive, $storecfg, $src_storeid, $storeid, $use_drive_mirror);
+
+ $size = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 10);
}
- $size /= 1024;
- $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, $size);
+ $newvolid = PVE::Storage::vdisk_alloc(
+ $storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024)
+ );
push @$newvollist, $newvolid;
PVE::Storage::activate_volumes($storecfg, [$newvolid]);
if (drive_is_cloudinit($drive)) {
+ # when cloning multiple disks (e.g. during clone_vm) it might be the last disk
+ # if this is the case, we have to complete any block-jobs still there from
+ # previous drive-mirrors
+ if (($completion eq 'complete') && (scalar(keys %$jobs) > 0)) {
+ qemu_drive_mirror_monitor($vmid, $newvmid, $jobs, $completion, $qga);
+ }
goto no_data_clone;
}
my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
- if (!$running || $snapname) {
- # TODO: handle bwlimits
- if ($drivename eq 'efidisk0') {
+ if ($use_drive_mirror) {
+ qemu_drive_mirror($vmid, $src_drivename, $newvolid, $newvmid, $sparseinit, $jobs,
+ $completion, $qga, $bwlimit);
+ } else {
+ if ($dst_drivename eq 'efidisk0') {
# the relevant data on the efidisk may be smaller than the source
# e.g. on RBD/ZFS, so we use dd to copy only the amount
# that is given by the OVMF_VARS.fd
- my $src_path = PVE::Storage::path($storecfg, $drive->{file});
+ my $src_path = PVE::Storage::path($storecfg, $drive->{file}, $snapname);
my $dst_path = PVE::Storage::path($storecfg, $newvolid);
- run_command(['qemu-img', 'dd', '-n', '-O', $dst_format, "bs=1", "count=$size",
- "if=$src_path", "of=$dst_path"]);
- } else {
- qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
- }
- } else {
- my $kvmver = get_running_qemu_version ($vmid);
- if (!min_version($kvmver, 2, 7)) {
- die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
- if $drive->{iothread};
- }
+ my $src_format = (PVE::Storage::parse_volname($storecfg, $drive->{file}))[6];
- qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs,
- $completion, $qga, $bwlimit);
+ # better for Ceph if block size is not too small, see bug #3324
+ my $bs = 1024*1024;
+
+ my $cmd = ['qemu-img', 'dd', '-n', '-O', $dst_format];
+
+ if ($src_format eq 'qcow2' && $snapname) {
+ die "cannot clone qcow2 EFI disk snapshot - requires QEMU >= 6.2\n"
+ if !min_version(kvm_user_version(), 6, 2);
+ push $cmd->@*, '-l', $snapname;
+ }
+ push $cmd->@*, "bs=$bs", "osize=$size", "if=$src_path", "of=$dst_path";
+ run_command($cmd);
+ } else {
+ qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit, $bwlimit);
+ }
}
}
no_data_clone:
- my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
+ my $size = eval { PVE::Storage::volume_size_info($storecfg, $newvolid, 10) };
- my $disk = $drive;
- $disk->{format} = undef;
+ my $disk = dclone($drive);
+ delete $disk->{format};
$disk->{file} = $newvolid;
- $disk->{size} = $size;
+ $disk->{size} = $size if defined($size) && !$unused;
return $disk;
}
}
sub get_efivars_size {
- my ($conf) = @_;
+ my ($conf, $efidisk) = @_;
+
my $arch = get_vm_arch($conf);
- my (undef, $ovmf_vars) = get_ovmf_files($arch);
- die "uefi vars image '$ovmf_vars' not found\n" if ! -f $ovmf_vars;
+ $efidisk //= $conf->{efidisk0} ? parse_drive('efidisk0', $conf->{efidisk0}) : undef;
+ my $smm = PVE::QemuServer::Machine::machine_type_is_q35($conf);
+ my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
return -s $ovmf_vars;
}
return;
}
-sub create_efidisk($$$$$) {
- my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
+sub update_tpmstate_size {
+ my ($conf) = @_;
+
+ my $disk = PVE::QemuServer::parse_drive('tpmstate0', $conf->{tpmstate0});
+ $disk->{size} = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
+ $conf->{tpmstate0} = print_drive($disk);
+}
+
+sub create_efidisk($$$$$$$) {
+ my ($storecfg, $storeid, $vmid, $fmt, $arch, $efidisk, $smm) = @_;
- my (undef, $ovmf_vars) = get_ovmf_files($arch);
- die "EFI vars default image not found\n" if ! -f $ovmf_vars;
+ my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
my $vars_size_b = -s $ovmf_vars;
my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
PVE::Storage::activate_volumes($storecfg, [$volid]);
qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
- my ($size) = PVE::Storage::volume_size_info($storecfg, $volid, 3);
+ my $size = PVE::Storage::volume_size_info($storecfg, $volid, 3);
return ($volid, $size/1024);
}
return ($maxdev, $controller, $controller_prefix);
}
-sub windows_version {
- my ($ostype) = @_;
-
- return 0 if !$ostype;
-
- my $winversion = 0;
-
- if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
- $winversion = 5;
- } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
- $winversion = 6;
- } elsif ($ostype =~ m/^win(\d+)$/) {
- $winversion = $1;
- }
-
- return $winversion;
-}
-
sub resolve_dst_disk_format {
my ($storecfg, $storeid, $src_volname, $format) = @_;
my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
sub nbd_stop {
my ($vmid) = @_;
- mon_cmd($vmid, 'nbd-server-stop');
+ mon_cmd($vmid, 'nbd-server-stop', timeout => 25);
}
sub create_reboot_request {
return $bootorder;
}
+sub register_qmeventd_handle {
+ my ($vmid) = @_;
+
+ my $fh;
+ my $peer = "/var/run/qmeventd.sock";
+ my $count = 0;
+
+ for (;;) {
+ $count++;
+ $fh = IO::Socket::UNIX->new(Peer => $peer, Blocking => 0, Timeout => 1);
+ last if $fh;
+ if ($! != EINTR && $! != EAGAIN) {
+ die "unable to connect to qmeventd socket (vmid: $vmid) - $!\n";
+ }
+ if ($count > 4) {
+ die "unable to connect to qmeventd socket (vmid: $vmid) - timeout "
+ . "after $count retries\n";
+ }
+ usleep(25000);
+ }
+
+ # send handshake to mark VM as backing up
+ print $fh to_json({vzdump => {vmid => "$vmid"}});
+
+ # return handle to be closed later when inhibit is no longer required
+ return $fh;
+}
+
# bash completion helper
sub complete_backup_archives {
return $res;
}
+sub vm_is_paused {
+ my ($vmid, $include_suspended) = @_;
+ my $qmpstatus = eval {
+ PVE::QemuConfig::assert_config_exists_on_node($vmid);
+ mon_cmd($vmid, "query-status");
+ };
+ warn "$@\n" if $@;
+ return $qmpstatus && (
+ $qmpstatus->{status} eq "paused" ||
+ $qmpstatus->{status} eq "prelaunch" ||
+ ($include_suspended && $qmpstatus->{status} eq "suspended")
+ );
+}
+
+sub check_volume_storage_type {
+ my ($storecfg, $vol) = @_;
+
+ my ($storeid, $volname) = PVE::Storage::parse_volume_id($vol);
+ my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
+ my ($vtype) = PVE::Storage::parse_volname($storecfg, $vol);
+
+ die "storage '$storeid' does not support content-type '$vtype'\n"
+ if !$scfg->{content}->{$vtype};
+
+ return 1;
+}
+
+sub add_nets_bridge_fdb {
+ my ($conf, $vmid) = @_;
+
+ for my $opt (keys %$conf) {
+ next if $opt !~ m/^net(\d+)$/;
+ my $iface = "tap${vmid}i$1";
+ # NOTE: expect setups with learning off to *not* use auto-random-generation of MAC on start
+ my $net = parse_net($conf->{$opt}, 1) or next;
+
+ my $mac = $net->{macaddr};
+ if (!$mac) {
+ log_warn("MAC learning disabled, but vNIC '$iface' has no static MAC to add to forwarding DB!")
+ if !file_read_firstline("/sys/class/net/$iface/brport/learning");
+ next;
+ }
+
+ my $bridge = $net->{bridge};
+ if (!$bridge) {
+ log_warn("Interface '$iface' not attached to any bridge.");
+ next;
+ }
+ if ($have_sdn) {
+ PVE::Network::SDN::Zones::add_bridge_fdb($iface, $mac, $bridge);
+ } elsif (-d "/sys/class/net/$bridge/bridge") { # avoid fdb management with OVS for now
+ PVE::Network::add_bridge_fdb($iface, $mac);
+ }
+ }
+}
+
+sub del_nets_bridge_fdb {
+ my ($conf, $vmid) = @_;
+
+ for my $opt (keys %$conf) {
+ next if $opt !~ m/^net(\d+)$/;
+ my $iface = "tap${vmid}i$1";
+
+ my $net = parse_net($conf->{$opt}) or next;
+ my $mac = $net->{macaddr} or next;
+
+ my $bridge = $net->{bridge};
+ if ($have_sdn) {
+ PVE::Network::SDN::Zones::del_bridge_fdb($iface, $mac, $bridge);
+ } elsif (-d "/sys/class/net/$bridge/bridge") { # avoid fdb management with OVS for now
+ PVE::Network::del_bridge_fdb($iface, $mac);
+ }
+ }
+}
+
+sub create_ifaces_ipams_ips {
+ my ($conf, $vmid) = @_;
+
+ return if !$have_sdn;
+
+ foreach my $opt (keys %$conf) {
+ if ($opt =~ m/^net(\d+)$/) {
+ my $value = $conf->{$opt};
+ my $net = PVE::QemuServer::parse_net($value);
+ eval { PVE::Network::SDN::Vnets::add_next_free_cidr($net->{bridge}, $conf->{name}, $net->{macaddr}, $vmid, undef, 1) };
+ warn $@ if $@;
+ }
+ }
+}
+
+sub delete_ifaces_ipams_ips {
+ my ($conf, $vmid) = @_;
+
+ return if !$have_sdn;
+
+ foreach my $opt (keys %$conf) {
+ if ($opt =~ m/^net(\d+)$/) {
+ my $net = PVE::QemuServer::parse_net($conf->{$opt});
+ eval { PVE::Network::SDN::Vnets::del_ips_from_mac($net->{bridge}, $net->{macaddr}, $conf->{name}) };
+ warn $@ if $@;
+ }
+ }
+}
+
1;
--- /dev/null
+package PVE::QemuServer::CGroup;
+
+use strict;
+use warnings;
+use PVE::CGroup;
+use base('PVE::CGroup');
+
+sub get_subdir {
+ my ($self, $controller, $limiting) = @_;
+ my $vmid = $self->{vmid};
+ return "qemu.slice/$vmid.scope/";
+}
+
+1;
use PVE::JSONSchema;
use PVE::Cluster qw(cfs_register_file cfs_read_file);
+use PVE::Tools qw(get_host_arch);
use PVE::QemuServer::Helpers qw(min_version);
use base qw(PVE::SectionConfig Exporter);
our @EXPORT_OK = qw(
print_cpu_device
get_cpu_options
+get_cpu_bitness
+is_native_arch
);
# under certain race-conditions, this module might be loaded before pve-cluster
}
my $default_filename = "virtual-guest/cpu-models.conf";
-cfs_register_file($default_filename,
- sub { PVE::QemuServer::CPUConfig->parse_config(@_); },
- sub { PVE::QemuServer::CPUConfig->write_config(@_); });
+cfs_register_file(
+ $default_filename,
+ sub { PVE::QemuServer::CPUConfig->parse_config(@_); },
+ sub { PVE::QemuServer::CPUConfig->write_config(@_); },
+);
sub load_custom_model_conf {
return cfs_read_file($default_filename);
}
+#builtin models : reported-model is mandatory
+my $builtin_models = {
+ 'x86-64-v2' => {
+ 'reported-model' => 'qemu64',
+ flags => "+popcnt;+pni;+sse4.1;+sse4.2;+ssse3",
+ },
+ 'x86-64-v2-AES' => {
+ 'reported-model' => 'qemu64',
+ flags => "+aes;+popcnt;+pni;+sse4.1;+sse4.2;+ssse3",
+ },
+ 'x86-64-v3' => {
+ 'reported-model' => 'qemu64',
+ flags => "+aes;+popcnt;+pni;+sse4.1;+sse4.2;+ssse3;+avx;+avx2;+bmi1;+bmi2;+f16c;+fma;+abm;+movbe;+xsave",
+ },
+ 'x86-64-v4' => {
+ 'reported-model' => 'qemu64',
+ flags => "+aes;+popcnt;+pni;+sse4.1;+sse4.2;+ssse3;+avx;+avx2;+bmi1;+bmi2;+f16c;+fma;+abm;+movbe;+xsave;+avx512f;+avx512bw;+avx512cd;+avx512dq;+avx512vl",
+ },
+};
+
+my $depreacated_cpu_map = {
+ # there never was such a client CPU, so map it to the server one for backward compat
+ 'Icelake-Client' => 'Icelake-Server',
+ 'Icelake-Client-noTSX' => 'Icelake-Server-noTSX',
+};
+
+my $cputypes_32bit = {
+ '486' => 1,
+ 'pentium' => 1,
+ 'pentium2' => 1,
+ 'pentium3' => 1,
+ 'coreduo' => 1,
+ 'athlon' => 1,
+ 'kvm32' => 1,
+ 'qemu32' => 1,
+};
+
my $cpu_vendor_list = {
# Intel CPUs
486 => 'GenuineIntel',
'Skylake-Client' => 'GenuineIntel',
'Skylake-Client-IBRS' => 'GenuineIntel',
'Skylake-Client-noTSX-IBRS' => 'GenuineIntel',
+ 'Skylake-Client-v4' => 'GenuineIntel',
'Skylake-Server' => 'GenuineIntel',
'Skylake-Server-IBRS' => 'GenuineIntel',
'Skylake-Server-noTSX-IBRS' => 'GenuineIntel',
+ 'Skylake-Server-v4' => 'GenuineIntel',
+ 'Skylake-Server-v5' => 'GenuineIntel',
'Cascadelake-Server' => 'GenuineIntel',
+ 'Cascadelake-Server-v2' => 'GenuineIntel',
'Cascadelake-Server-noTSX' => 'GenuineIntel',
+ 'Cascadelake-Server-v4' => 'GenuineIntel',
+ 'Cascadelake-Server-v5' => 'GenuineIntel',
+ 'Cooperlake' => 'GenuineIntel',
+ 'Cooperlake-v2' => 'GenuineIntel',
KnightsMill => 'GenuineIntel',
- 'Icelake-Client' => 'GenuineIntel',
- 'Icelake-Client-noTSX' => 'GenuineIntel',
+ 'Icelake-Client' => 'GenuineIntel', # depreacated, removed with QEMU 7.1
+ 'Icelake-Client-noTSX' => 'GenuineIntel', # depreacated, removed with QEMU 7.1
'Icelake-Server' => 'GenuineIntel',
'Icelake-Server-noTSX' => 'GenuineIntel',
+ 'Icelake-Server-v3' => 'GenuineIntel',
+ 'Icelake-Server-v4' => 'GenuineIntel',
+ 'Icelake-Server-v5' => 'GenuineIntel',
+ 'Icelake-Server-v6' => 'GenuineIntel',
+ 'SapphireRapids' => 'GenuineIntel',
+ 'SapphireRapids-v2' => 'GenuineIntel',
+ 'GraniteRapids' => 'GenuineIntel',
# AMD CPUs
athlon => 'AuthenticAMD',
Opteron_G5 => 'AuthenticAMD',
EPYC => 'AuthenticAMD',
'EPYC-IBPB' => 'AuthenticAMD',
+ 'EPYC-v3' => 'AuthenticAMD',
+ 'EPYC-v4' => 'AuthenticAMD',
'EPYC-Rome' => 'AuthenticAMD',
+ 'EPYC-Rome-v2' => 'AuthenticAMD',
+ 'EPYC-Rome-v3' => 'AuthenticAMD',
+ 'EPYC-Rome-v4' => 'AuthenticAMD',
+ 'EPYC-Milan' => 'AuthenticAMD',
+ 'EPYC-Milan-v2' => 'AuthenticAMD',
+ 'EPYC-Genoa' => 'AuthenticAMD',
# generic types, use vendor from host node
host => 'default',
},
'reported-model' => {
description => "CPU model and vendor to report to the guest. Must be a QEMU/KVM supported model."
- . " Only valid for custom CPU model definitions, default models will always report themselves to the guest OS.",
+ ." Only valid for custom CPU model definitions, default models will always report themselves to the guest OS.",
type => 'string',
enum => [ sort { lc("$a") cmp lc("$b") } keys %$cpu_vendor_list ],
default => 'kvm64',
optional => 1,
},
flags => {
- description => "List of additional CPU flags separated by ';'."
- . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
- . " Custom CPU models can specify any flag supported by"
- . " QEMU/KVM, VM-specific flags must be from the following"
- . " set for security reasons: @{[join(', ', @supported_cpu_flags)]}.",
+ description => "List of additional CPU flags separated by ';'. Use '+FLAG' to enable,"
+ ." '-FLAG' to disable a flag. Custom CPU models can specify any flag supported by"
+ ." QEMU/KVM, VM-specific flags must be from the following set for security reasons: "
+ . join(', ', @supported_cpu_flags),
format_description => '+FLAG[;-FLAG...]',
type => 'string',
pattern => qr/$cpu_flag_any_re(;$cpu_flag_any_re)*/,
type => 'string',
format => 'pve-phys-bits',
format_description => '8-64|host',
- description => "The physical memory address bits that are reported to"
- . " the guest OS. Should be smaller or equal to the host's."
- . " Set to 'host' to use value from host CPU, but note that"
- . " doing so will break live migration to CPUs with other values.",
+ description => "The physical memory address bits that are reported to the guest OS. Should"
+ ." be smaller or equal to the host's. Set to 'host' to use value from host CPU, but"
+ ." note that doing so will break live migration to CPUs with other values.",
optional => 1,
},
};
PVE::JSONSchema::register_format('pve-cpu-conf', $cpu_fmt, \&validate_cpu_conf);
sub validate_cpu_conf {
my ($cpu) = @_;
-
- # required, but can't be forced in schema since it's encoded in section
- # header for custom models
+ # required, but can't be forced in schema since it's encoded in section header for custom models
die "CPU is missing cputype\n" if !$cpu->{cputype};
-
return $cpu;
}
PVE::JSONSchema::register_format('pve-vm-cpu-conf', $cpu_fmt, \&validate_vm_cpu_conf);
get_custom_model($cputype);
} else {
die "Built-in cputype '$cputype' is not defined (missing 'custom-' prefix?)\n"
- if !defined($cpu_vendor_list->{$cputype});
+ if !defined($cpu_vendor_list->{$cputype}) && !defined($builtin_models->{$cputype});
}
# in a VM-specific config, certain properties are limited/forbidden
die "VM-specific CPU flags must be a subset of: @{[join(', ', @supported_cpu_flags)]}\n"
- if ($cpu->{flags} && $cpu->{flags} !~ m/$cpu_flag_supported_re(;$cpu_flag_supported_re)*/);
+ if ($cpu->{flags} && $cpu->{flags} !~ m/^$cpu_flag_supported_re(;$cpu_flag_supported_re)*$/);
die "Property 'reported-model' not allowed in VM-specific CPU config.\n"
if defined($cpu->{'reported-model'});
};
}
+ for my $model (keys %{$builtin_models}) {
+ my $reported_model = $builtin_models->{$model}->{'reported-model'};
+ my $vendor = $cpu_vendor_list->{$reported_model};
+ push @$models, {
+ name => $model,
+ custom => 0,
+ vendor => $vendor,
+ };
+ }
+
return $models if !$include_custom;
my $conf = load_custom_model_conf();
# Print a QEMU device node for a given VM configuration for hotplugging CPUs
sub print_cpu_device {
- my ($conf, $id) = @_;
+ my ($conf, $arch, $id) = @_;
+
+ # FIXME: hot plugging other architectures like our unofficial aarch64 support?
+ die "Hotplug of non x86_64 CPU not yet supported" if $arch ne 'x86_64';
- my $kvm = $conf->{kvm} // 1;
- my $cpu = $kvm ? "kvm64" : "qemu64";
+ my $kvm = $conf->{kvm} // is_native_arch($arch);
+ my $cpu = get_default_cpu_type('x86_64', $kvm);
if (my $cputype = $conf->{cpu}) {
my $cpuconf = PVE::JSONSchema::parse_property_string('pve-vm-cpu-conf', $cputype)
or die "Cannot parse cpu description: $cputype\n";
$cpu = $cpuconf->{cputype};
- if (is_custom_model($cpu)) {
+ if (my $model = $builtin_models->{$cpu}) {
+ $cpu = $model->{'reported-model'};
+ } elsif (is_custom_model($cputype)) {
my $custom_cpu = get_custom_model($cpu);
- $cpu = $custom_cpu->{'reported-model'} //
- $cpu_fmt->{'reported-model'}->{default};
+ $cpu = $custom_cpu->{'reported-model'} // $cpu_fmt->{'reported-model'}->{default};
+ }
+ if (my $replacement_type = $depreacated_cpu_map->{$cpu}) {
+ $cpu = $replacement_type;
}
}
return $res if !$flaglist;
foreach my $flag (split(";", $flaglist)) {
- if ($flag =~ $re) {
+ if ($flag =~ m/^$re$/) {
$res->{$2} = { op => $1, reason => $reason };
}
}
sub get_cpu_options {
my ($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough) = @_;
- my $cputype = $kvm ? "kvm64" : "qemu64";
- if ($arch eq 'aarch64') {
- $cputype = 'cortex-a57';
- }
+ my $cputype = get_default_cpu_type($arch, $kvm);
my $cpu = {};
my $custom_cpu;
+ my $builtin_cpu;
my $hv_vendor_id;
if (my $cpu_prop_str = $conf->{cpu}) {
$cpu = PVE::JSONSchema::parse_property_string('pve-vm-cpu-conf', $cpu_prop_str)
or die "Cannot parse cpu description: $cpu_prop_str\n";
$cputype = $cpu->{cputype};
-
- if (is_custom_model($cputype)) {
+ if (my $model = $builtin_models->{$cputype}) {
+ $cputype = $model->{'reported-model'};
+ $builtin_cpu->{flags} = $model->{'flags'};
+ } elsif (is_custom_model($cputype)) {
$custom_cpu = get_custom_model($cputype);
- $cputype = $custom_cpu->{'reported-model'} //
- $cpu_fmt->{'reported-model'}->{default};
- $kvm_off = $custom_cpu->{hidden}
- if defined($custom_cpu->{hidden});
+ $cputype = $custom_cpu->{'reported-model'} // $cpu_fmt->{'reported-model'}->{default};
+ $kvm_off = $custom_cpu->{hidden} if defined($custom_cpu->{hidden});
$hv_vendor_id = $custom_cpu->{'hv-vendor-id'};
}
+ if (my $replacement_type = $depreacated_cpu_map->{$cputype}) {
+ $cputype = $replacement_type;
+ }
+
# VM-specific settings override custom CPU config
- $kvm_off = $cpu->{hidden}
- if defined($cpu->{hidden});
- $hv_vendor_id = $cpu->{'hv-vendor-id'}
- if defined($cpu->{'hv-vendor-id'});
+ $kvm_off = $cpu->{hidden} if defined($cpu->{hidden});
+ $hv_vendor_id = $cpu->{'hv-vendor-id'} if defined($cpu->{'hv-vendor-id'});
}
my $pve_flags = get_pve_cpu_flags($conf, $kvm, $cputype, $arch, $machine_version);
)
: undef;
- my $custom_cputype_flags = parse_cpuflag_list($cpu_flag_any_re,
- "set by custom CPU model", $custom_cpu->{flags});
+ my $builtin_cputype_flags = parse_cpuflag_list(
+ $cpu_flag_any_re, "set by builtin CPU model", $builtin_cpu->{flags});
- my $vm_flags = parse_cpuflag_list($cpu_flag_supported_re,
- "manually set for VM", $cpu->{flags});
+ my $custom_cputype_flags = parse_cpuflag_list(
+ $cpu_flag_any_re, "set by custom CPU model", $custom_cpu->{flags});
+
+ my $vm_flags = parse_cpuflag_list(
+ $cpu_flag_supported_re, "manually set for VM", $cpu->{flags});
my $pve_forced_flags = {};
$pve_forced_flags->{'enforce'} = {
my $cpu_str = $cputype;
# will be resolved in parameter order
- $cpu_str .= resolve_cpu_flags($pve_flags, $hv_flags, $custom_cputype_flags,
- $vm_flags, $pve_forced_flags);
+ $cpu_str .= resolve_cpu_flags(
+ $pve_flags, $hv_flags, $builtin_cputype_flags, $custom_cputype_flags, $vm_flags, $pve_forced_flags);
my $phys_bits = '';
foreach my $conf ($custom_cpu, $cpu) {
return $1;
}
+sub get_default_cpu_type {
+ my ($arch, $kvm) = @_;
+
+ my $cputype = $kvm ? 'kvm64' : 'qemu64';
+ $cputype = 'cortex-a57' if $arch eq 'aarch64';
+
+ return $cputype;
+}
+
+sub is_native_arch($) {
+ my ($arch) = @_;
+ return get_host_arch() eq $arch;
+}
+
+sub get_cpu_bitness {
+ my ($cpu_prop_str, $arch) = @_;
+
+ $arch //= get_host_arch();
+
+ my $cputype = get_default_cpu_type($arch, 0);
+
+ if ($cpu_prop_str) {
+ my $cpu = PVE::JSONSchema::parse_property_string('pve-vm-cpu-conf', $cpu_prop_str)
+ or die "Cannot parse cpu description: $cpu_prop_str\n";
+
+ $cputype = $cpu->{cputype};
+
+ if (my $model = $builtin_models->{$cputype}) {
+ $cputype = $model->{'reported-model'};
+ } elsif (is_custom_model($cputype)) {
+ my $custom_cpu = get_custom_model($cputype);
+ $cputype = $custom_cpu->{'reported-model'} // $cpu_fmt->{'reported-model'}->{default};
+ }
+ }
+
+ return $cputypes_32bit->{$cputype} ? 32 : 64 if $arch eq 'x86_64';
+ return 64 if $arch eq 'aarch64';
+
+ die "unsupported architecture '$arch'\n";
+}
+
__PACKAGE__->register();
__PACKAGE__->init();
use File::Path;
use Digest::SHA;
use URI::Escape;
+use MIME::Base64 qw(encode_base64);
+use Storable qw(dclone);
use PVE::Tools qw(run_command file_set_contents);
use PVE::Storage;
use PVE::QemuServer;
+use PVE::QemuServer::Helpers;
use constant CLOUDINIT_DISK_SIZE => 4 * 1024 * 1024; # 4MiB in bytes
# the new predicatble network device naming scheme.
if (defined(my $ostype = $conf->{ostype})) {
return 'configdrive2'
- if PVE::QemuServer::windows_version($ostype);
+ if PVE::QemuServer::Helpers::windows_version($ostype);
}
return 'nocloud';
$hostname =~ s/\..*$//;
} elsif (my $search = $conf->{searchdomain}) {
$fqdn = "$hostname.$search";
+ } else {
+ $fqdn = $hostname;
}
return ($hostname, $fqdn);
}
$content .= "hostname: $hostname\n";
$content .= "manage_etc_hosts: true\n";
- $content .= "fqdn: $fqdn\n" if defined($fqdn);
+ $content .= "fqdn: $fqdn\n";
my $username = $conf->{ciuser};
my $password = $conf->{cipassword};
$content .= " - default\n";
}
- $content .= "package_upgrade: true\n";
+ $content .= "package_upgrade: true\n" if !defined($conf->{ciupgrade}) || $conf->{ciupgrade};
return $content;
}
sub generate_configdrive2 {
my ($conf, $vmid, $drive, $volname, $storeid) = @_;
- my ($user_data, $network_data, $meta_data) = get_custom_cloudinit_files($conf);
+ my ($user_data, $network_data, $meta_data, $vendor_data) = get_custom_cloudinit_files($conf);
$user_data = cloudinit_userdata($conf, $vmid) if !defined($user_data);
$network_data = configdrive2_network($conf) if !defined($network_data);
+ $vendor_data = '' if !defined($vendor_data);
if (!defined($meta_data)) {
$meta_data = configdrive2_gen_metadata($user_data, $network_data);
}
+
+ # we always allocate a 4MiB disk for cloudinit and with the overhead of the ISO
+ # make sure we always stay below it by keeping the sum of all files below 3 MiB
+ my $sum = length($user_data) + length($network_data) + length($meta_data) + length($vendor_data);
+ die "Cloud-Init sum of snippets too big (> 3 MiB)\n" if $sum > (3 * 1024 * 1024);
+
my $files = {
'/openstack/latest/user_data' => $user_data,
'/openstack/content/0000' => $network_data,
- '/openstack/latest/meta_data.json' => $meta_data
+ '/openstack/latest/meta_data.json' => $meta_data,
+ '/openstack/latest/vendor_data.json' => $vendor_data
};
commit_cloudinit_disk($conf, $vmid, $drive, $volname, $storeid, $files, 'config-2');
}
+sub generate_opennebula {
+ my ($conf, $vmid, $drive, $volname, $storeid) = @_;
+
+ my $content = "";
+
+ my $username = $conf->{ciuser} || "root";
+ $content .= "USERNAME=$username\n" if defined($username);
+
+ if (defined(my $password = $conf->{cipassword})) {
+ $content .= "CRYPTED_PASSWORD_BASE64=". encode_base64($password) ."\n";
+ }
+
+ if (defined($conf->{sshkeys})) {
+ my $keys = [ split(/\s*\n\s*/, URI::Escape::uri_unescape($conf->{sshkeys})) ];
+ $content .= "SSH_PUBLIC_KEY=\"". join("\n", $keys->@*) ."\"\n";
+ }
+
+ my ($hostname, $fqdn) = get_hostname_fqdn($conf, $vmid);
+ $content .= "SET_HOSTNAME=$hostname\n";
+
+ my ($searchdomains, $nameservers) = get_dns_conf($conf);
+ $content .= 'DNS="' . join(' ', @$nameservers) ."\"\n" if $nameservers && @$nameservers;
+ $content .= 'SEARCH_DOMAIN="'. join(' ', @$searchdomains) ."\"\n" if $searchdomains && @$searchdomains;
+
+ my $networkenabled = undef;
+ my @ifaces = grep { /^net(\d+)$/ } keys %$conf;
+ foreach my $iface (sort @ifaces) {
+ (my $id = $iface) =~ s/^net//;
+ my $net = PVE::QemuServer::parse_net($conf->{$iface});
+ next if !$conf->{"ipconfig$id"};
+ my $ipconfig = PVE::QemuServer::parse_ipconfig($conf->{"ipconfig$id"});
+ my $ethid = "ETH$id";
+
+ my $mac = lc $net->{hwaddr};
+
+ if ($ipconfig->{ip}) {
+ $networkenabled = 1;
+
+ if ($ipconfig->{ip} eq 'dhcp') {
+ $content .= "${ethid}_DHCP=YES\n";
+ } else {
+ my ($addr, $mask) = split_ip4($ipconfig->{ip});
+ $content .= "${ethid}_IP=$addr\n";
+ $content .= "${ethid}_MASK=$mask\n";
+ $content .= "${ethid}_MAC=$mac\n";
+ $content .= "${ethid}_GATEWAY=$ipconfig->{gw}\n" if $ipconfig->{gw};
+ }
+ $content .= "${ethid}_MTU=$net->{mtu}\n" if $net->{mtu};
+ }
+
+ if ($ipconfig->{ip6}) {
+ $networkenabled = 1;
+ if ($ipconfig->{ip6} eq 'dhcp') {
+ $content .= "${ethid}_DHCP6=YES\n";
+ } elsif ($ipconfig->{ip6} eq 'auto') {
+ $content .= "${ethid}_AUTO6=YES\n";
+ } else {
+ my ($addr, $mask) = split('/', $ipconfig->{ip6});
+ $content .= "${ethid}_IP6=$addr\n";
+ $content .= "${ethid}_MASK6=$mask\n";
+ $content .= "${ethid}_MAC6=$mac\n";
+ $content .= "${ethid}_GATEWAY6=$ipconfig->{gw6}\n" if $ipconfig->{gw6};
+ }
+ $content .= "${ethid}_MTU=$net->{mtu}\n" if $net->{mtu};
+ }
+ }
+
+ $content .= "NETWORK=YES\n" if $networkenabled;
+
+ my $files = { '/context.sh' => $content };
+ commit_cloudinit_disk($conf, $vmid, $drive, $volname, $storeid, $files, 'CONTEXT');
+}
+
sub nocloud_network_v2 {
my ($conf) = @_;
if ($ip eq 'dhcp') {
$content .= "${i}- type: dhcp6\n";
} elsif ($ip eq 'auto') {
- # SLAAC is not supported by cloud-init, this fallback should work with an up-to-date netplan at least
- $content .= "${i}- type: dhcp6\n";
+ # SLAAC is only supported by cloud-init since 19.4
+ $content .= "${i}- type: ipv6_slaac\n";
} else {
- $content .= "${i}- type: static\n"
+ $content .= "${i}- type: static6\n"
. "${i} address: '$ip'\n";
if (defined(my $gw = $ipconfig->{gw6})) {
$content .= "${i} gateway: '$gw'\n";
sub generate_nocloud {
my ($conf, $vmid, $drive, $volname, $storeid) = @_;
- my ($user_data, $network_data, $meta_data) = get_custom_cloudinit_files($conf);
+ my ($user_data, $network_data, $meta_data, $vendor_data) = get_custom_cloudinit_files($conf);
$user_data = cloudinit_userdata($conf, $vmid) if !defined($user_data);
$network_data = nocloud_network($conf) if !defined($network_data);
+ $vendor_data = '' if !defined($vendor_data);
if (!defined($meta_data)) {
$meta_data = nocloud_gen_metadata($user_data, $network_data);
}
+ # we always allocate a 4MiB disk for cloudinit and with the overhead of the ISO
+ # make sure we always stay below it by keeping the sum of all files below 3 MiB
+ my $sum = length($user_data) + length($network_data) + length($meta_data) + length($vendor_data);
+ die "Cloud-Init sum of snippets too big (> 3 MiB)\n" if $sum > (3 * 1024 * 1024);
+
my $files = {
'/user-data' => $user_data,
'/network-config' => $network_data,
- '/meta-data' => $meta_data
+ '/meta-data' => $meta_data,
+ '/vendor-data' => $vendor_data
};
commit_cloudinit_disk($conf, $vmid, $drive, $volname, $storeid, $files, 'cidata');
}
my $network_volid = $files->{network};
my $user_volid = $files->{user};
my $meta_volid = $files->{meta};
+ my $vendor_volid = $files->{vendor};
my $storage_conf = PVE::Storage::config();
$meta_data = read_cloudinit_snippets_file($storage_conf, $meta_volid);
}
- return ($user_data, $network_data, $meta_data);
+ my $vendor_data;
+ if ($vendor_volid) {
+ $vendor_data = read_cloudinit_snippets_file($storage_conf, $vendor_volid);
+ }
+
+ return ($user_data, $network_data, $meta_data, $vendor_data);
}
sub read_cloudinit_snippets_file {
my ($storage_conf, $volid) = @_;
- my ($full_path, undef, $type) = PVE::Storage::path($storage_conf, $volid);
- die "$volid is not in the snippets directory\n" if $type ne 'snippets';
+ my ($vtype, undef) = PVE::Storage::parse_volname($storage_conf, $volid);
+
+ die "$volid is not in the snippets directory\n" if $vtype ne 'snippets';
+
+ my $full_path = PVE::Storage::abs_filesystem_path($storage_conf, $volid, 1);
return PVE::Tools::file_get_contents($full_path, 1 * 1024 * 1024);
}
my $cloudinit_methods = {
configdrive2 => \&generate_configdrive2,
nocloud => \&generate_nocloud,
+ opennebula => \&generate_opennebula,
};
-sub generate_cloudinitconfig {
+sub has_changes {
+ my ($conf) = @_;
+
+ return !!$conf->{cloudinit}->%*;
+}
+
+sub generate_cloudinit_config {
my ($conf, $vmid) = @_;
my $format = get_cloudinit_format($conf);
+ my $has_changes = has_changes($conf);
+
PVE::QemuConfig->foreach_volume($conf, sub {
my ($ds, $drive) = @_;
$generator->($conf, $vmid, $drive, $volname, $storeid);
});
+
+ return $has_changes;
+}
+
+sub apply_cloudinit_config {
+ my ($conf, $vmid) = @_;
+
+ my $has_changes = generate_cloudinit_config($conf, $vmid);
+
+ if ($has_changes) {
+ delete $conf->{cloudinit};
+ PVE::QemuConfig->write_config($vmid, $conf);
+ return 1;
+ }
+
+ return $has_changes;
}
sub dump_cloudinit_config {
use strict;
use warnings;
+use Storable qw(dclone);
+
+use IO::File;
+
use PVE::Storage;
use PVE::JSONSchema qw(get_standard_option);
is_valid_drivename
drive_is_cloudinit
drive_is_cdrom
+drive_is_read_only
+get_scsi_devicetype
parse_drive
print_drive
);
my $MAX_VIRTIO_DISKS = 16;
our $MAX_SATA_DISKS = 6;
our $MAX_UNUSED_DISKS = 256;
+our $NEW_DISK_RE = qr!^(([^/:\s]+):)?(\d+(\.\d+)?)$!;
our $drivedesc_hash;
+# Schema when disk allocation is possible.
+our $drivedesc_hash_with_alloc = {};
my %drivedesc_base = (
volume => { alias => 'file' },
},
aio => {
type => 'string',
- enum => [qw(native threads)],
+ enum => [qw(native threads io_uring)],
description => 'AIO type to use.',
optional => 1,
},
optional => 1,
});
+my %product_fmt = (
+ product => {
+ type => 'string',
+ pattern => '[A-Za-z0-9\-_\s]{,16}', # QEMU (8.1) will quietly only use 16 bytes
+ format_description => 'product',
+ description => "The drive's product name, up to 16 bytes long.",
+ optional => 1,
+ },
+);
+
+my %vendor_fmt = (
+ vendor => {
+ type => 'string',
+ pattern => '[A-Za-z0-9\-_\s]{,8}', # QEMU (8.1) will quietly only use 8 bytes
+ format_description => 'vendor',
+ description => "The drive's vendor name, up to 8 bytes long.",
+ optional => 1,
+ },
+);
+
my %model_fmt = (
model => {
type => 'string',
}
);
+my %readonly_fmt = (
+ ro => {
+ type => 'boolean',
+ description => "Whether the drive is read-only.",
+ optional => 1,
+ },
+);
+
my %scsiblock_fmt = (
scsiblock => {
type => 'boolean',
my $idedesc = {
optional => 1,
type => 'string', format => $ide_fmt,
- description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
+ description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS - 1) . ").",
};
PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
my $scsi_fmt = {
%drivedesc_base,
%iothread_fmt,
+ %product_fmt,
%queues_fmt,
+ %readonly_fmt,
%scsiblock_fmt,
%ssd_fmt,
+ %vendor_fmt,
%wwn_fmt,
};
my $scsidesc = {
my $virtio_fmt = {
%drivedesc_base,
%iothread_fmt,
+ %readonly_fmt,
};
my $virtiodesc = {
optional => 1,
};
PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
-my $alldrive_fmt = {
- %drivedesc_base,
- %iothread_fmt,
- %model_fmt,
- %queues_fmt,
- %scsiblock_fmt,
- %ssd_fmt,
- %wwn_fmt,
-};
+my %efitype_fmt = (
+ efitype => {
+ type => 'string',
+ enum => [qw(2m 4m)],
+ description => "Size and type of the OVMF EFI vars. '4m' is newer and recommended,"
+ . " and required for Secure Boot. For backwards compatibility, '2m' is used"
+ . " if not otherwise specified. Ignored for VMs with arch=aarch64 (ARM).",
+ optional => 1,
+ default => '2m',
+ },
+ 'pre-enrolled-keys' => {
+ type => 'boolean',
+ description => "Use am EFI vars template with distribution-specific and Microsoft Standard"
+ ." keys enrolled, if used with 'efitype=4m'. Note that this will enable Secure Boot by"
+ ." default, though it can still be turned off from within the VM.",
+ optional => 1,
+ default => 0,
+ },
+);
my $efidisk_fmt = {
volume => { alias => 'file' },
description => "Disk size. This is purely informational and has no effect.",
optional => 1,
},
+ %efitype_fmt,
};
my $efidisk_desc = {
optional => 1,
type => 'string', format => $efidisk_fmt,
- description => "Configure a Disk for storing EFI vars",
+ description => "Configure a disk for storing EFI vars.",
};
PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
+my %tpmversion_fmt = (
+ version => {
+ type => 'string',
+ enum => [qw(v1.2 v2.0)],
+ description => "The TPM interface version. v2.0 is newer and should be preferred."
+ ." Note that this cannot be changed later on.",
+ optional => 1,
+ default => 'v2.0',
+ },
+);
+my $tpmstate_fmt = {
+ volume => { alias => 'file' },
+ file => {
+ type => 'string',
+ format => 'pve-volume-id-or-qm-path',
+ default_key => 1,
+ format_description => 'volume',
+ description => "The drive's backing volume.",
+ },
+ size => {
+ type => 'string',
+ format => 'disk-size',
+ format_description => 'DiskSize',
+ description => "Disk size. This is purely informational and has no effect.",
+ optional => 1,
+ },
+ %tpmversion_fmt,
+};
+my $tpmstate_desc = {
+ optional => 1,
+ type => 'string', format => $tpmstate_fmt,
+ description => "Configure a Disk for storing TPM state. The format is fixed to 'raw'.",
+};
+use constant TPMSTATE_DISK_SIZE => 4 * 1024 * 1024;
+
+my $alldrive_fmt = {
+ %drivedesc_base,
+ %iothread_fmt,
+ %model_fmt,
+ %product_fmt,
+ %queues_fmt,
+ %readonly_fmt,
+ %scsiblock_fmt,
+ %ssd_fmt,
+ %vendor_fmt,
+ %wwn_fmt,
+ %tpmversion_fmt,
+ %efitype_fmt,
+};
+
+my %import_from_fmt = (
+ 'import-from' => {
+ type => 'string',
+ format => 'pve-volume-id-or-absolute-path',
+ format_description => 'source volume',
+ description => "Create a new disk, importing from this source (volume ID or absolute ".
+ "path). When an absolute path is specified, it's up to you to ensure that the source ".
+ "is not actively used by another process during the import!",
+ optional => 1,
+ },
+);
+
+my $alldrive_fmt_with_alloc = {
+ %$alldrive_fmt,
+ %import_from_fmt,
+};
+
my $unused_fmt = {
volume => { alias => 'file' },
file => {
description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
};
+my $with_alloc_desc_cache = {
+ unused => $unuseddesc, # Allocation for unused is not supported currently.
+};
+my $desc_with_alloc = sub {
+ my ($type, $desc) = @_;
+
+ return $with_alloc_desc_cache->{$type} if $with_alloc_desc_cache->{$type};
+
+ my $new_desc = dclone($desc);
+
+ $new_desc->{format}->{'import-from'} = $import_from_fmt{'import-from'};
+
+ my $extra_note = '';
+ if ($type eq 'efidisk') {
+ $extra_note = " Note that SIZE_IN_GiB is ignored here and that the default EFI vars are ".
+ "copied to the volume instead.";
+ } elsif ($type eq 'tpmstate') {
+ $extra_note = " Note that SIZE_IN_GiB is ignored here and 4 MiB will be used instead.";
+ }
+
+ $new_desc->{description} .= " Use the special syntax STORAGE_ID:SIZE_IN_GiB to allocate a new ".
+ "volume.${extra_note} Use STORAGE_ID:0 and the 'import-from' parameter to import from an ".
+ "existing volume.";
+
+ $with_alloc_desc_cache->{$type} = $new_desc;
+
+ return $new_desc;
+};
+
for (my $i = 0; $i < $MAX_IDE_DISKS; $i++) {
$drivedesc_hash->{"ide$i"} = $idedesc;
+ $drivedesc_hash_with_alloc->{"ide$i"} = $desc_with_alloc->('ide', $idedesc);
}
for (my $i = 0; $i < $MAX_SATA_DISKS; $i++) {
$drivedesc_hash->{"sata$i"} = $satadesc;
+ $drivedesc_hash_with_alloc->{"sata$i"} = $desc_with_alloc->('sata', $satadesc);
}
for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++) {
$drivedesc_hash->{"scsi$i"} = $scsidesc;
+ $drivedesc_hash_with_alloc->{"scsi$i"} = $desc_with_alloc->('scsi', $scsidesc);
}
for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++) {
$drivedesc_hash->{"virtio$i"} = $virtiodesc;
+ $drivedesc_hash_with_alloc->{"virtio$i"} = $desc_with_alloc->('virtio', $virtiodesc);
}
$drivedesc_hash->{efidisk0} = $efidisk_desc;
+$drivedesc_hash_with_alloc->{efidisk0} = $desc_with_alloc->('efidisk', $efidisk_desc);
+
+$drivedesc_hash->{tpmstate0} = $tpmstate_desc;
+$drivedesc_hash_with_alloc->{tpmstate0} = $desc_with_alloc->('tpmstate', $tpmstate_desc);
for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++) {
$drivedesc_hash->{"unused$i"} = $unuseddesc;
+ $drivedesc_hash_with_alloc->{"unused$i"} = $desc_with_alloc->('unused', $unuseddesc);
+}
+
+sub valid_drive_names_for_boot {
+ return grep { $_ ne 'efidisk0' && $_ ne 'tpmstate0' } valid_drive_names();
}
sub valid_drive_names {
(map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
(map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
(map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
- 'efidisk0');
+ 'efidisk0',
+ 'tpmstate0');
+}
+
+sub valid_drive_names_with_unused {
+ return (valid_drive_names(), map {"unused$_"} (0 .. ($MAX_UNUSED_DISKS - 1)));
}
sub is_valid_drivename {
sub drive_is_cloudinit {
my ($drive) = @_;
- return $drive->{file} =~ m@[:/]vm-\d+-cloudinit(?:\.$QEMU_FORMAT_RE)?$@;
+ return $drive->{file} =~ m@[:/](?:vm-\d+-)?cloudinit(?:\.$QEMU_FORMAT_RE)?$@;
}
sub drive_is_cdrom {
return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
}
+sub drive_is_read_only {
+ my ($conf, $drive) = @_;
+
+ return 0 if !PVE::QemuConfig->is_template($conf);
+
+ # don't support being marked read-only
+ return $drive->{interface} ne 'sata' && $drive->{interface} ne 'ide';
+}
+
# ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
# [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
# [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
# [,iothread=on][,serial=serial][,model=model]
sub parse_drive {
- my ($key, $data) = @_;
+ my ($key, $data, $with_alloc) = @_;
my ($interface, $index);
return;
}
- if (!defined($drivedesc_hash->{$key})) {
+ my $desc_hash = $with_alloc ? $drivedesc_hash_with_alloc : $drivedesc_hash;
+
+ if (!defined($desc_hash->{$key})) {
warn "invalid drive key: $key\n";
return;
}
- my $desc = $drivedesc_hash->{$key}->{format};
+ my $desc = $desc_hash->{$key}->{format};
my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
return if !$res;
$res->{interface} = $interface;
}
sub print_drive {
- my ($drive) = @_;
+ my ($drive, $with_alloc) = @_;
my $skip = [ 'index', 'interface' ];
- return PVE::JSONSchema::print_property_string($drive, $alldrive_fmt, $skip);
+ my $fmt = $with_alloc ? $alldrive_fmt_with_alloc : $alldrive_fmt;
+ return PVE::JSONSchema::print_property_string($drive, $fmt, $skip);
}
sub get_bootdisks {
my $bootdisks = get_bootdisks($conf);
return if !@$bootdisks;
- my $bootdisk = $bootdisks->[0];
- return if !is_valid_drivename($bootdisk);
-
- return if !$conf->{$bootdisk};
-
- my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
- return if !defined($drive);
-
- return if drive_is_cdrom($drive);
-
- my $volid = $drive->{file};
- return if !$volid;
+ for my $bootdisk (@$bootdisks) {
+ next if !is_valid_drivename($bootdisk);
+ next if !$conf->{$bootdisk};
+ my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
+ next if !defined($drive);
+ next if drive_is_cdrom($drive);
+ my $volid = $drive->{file};
+ next if !$volid;
+ return $drive->{size};
+ }
- return $drive->{size};
+ return;
}
sub update_disksize {
my $path = PVE::Storage::path($storecfg, $volid);
my $scan_config = sub {
- my ($cref, $snapname) = @_;
+ my ($cref) = @_;
foreach my $key (keys %$cref) {
my $value = $cref->{$key};
next if !$storeid;
my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
next if !$scfg;
- return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
+ return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file});
}
}
}
undef $skip_drive;
- foreach my $snapname (keys %{$conf->{snapshots}}) {
- return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
+ for my $snap (values %{$conf->{snapshots}}) {
+ return 1 if $scan_config->($snap);
}
return 0;
sub resolve_first_disk {
my ($conf, $cdrom) = @_;
- my @disks = valid_drive_names();
+ my @disks = valid_drive_names_for_boot();
foreach my $ds (@disks) {
next if !$conf->{$ds};
my $disk = parse_drive($ds, $conf->{$ds});
return;
}
+sub scsi_inquiry {
+ my($fh, $noerr) = @_;
+
+ my $SG_IO = 0x2285;
+ my $SG_GET_VERSION_NUM = 0x2282;
+
+ my $versionbuf = "\x00" x 8;
+ my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
+ if (!$ret) {
+ die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
+ return;
+ }
+ my $version = unpack("I", $versionbuf);
+ if ($version < 30000) {
+ die "scsi generic interface too old\n" if !$noerr;
+ return;
+ }
+
+ my $buf = "\x00" x 36;
+ my $sensebuf = "\x00" x 8;
+ my $cmd = pack("C x3 C x1", 0x12, 36);
+
+ # see /usr/include/scsi/sg.h
+ my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
+
+ my $packet = pack(
+ $sg_io_hdr_t, ord('S'), -3, length($cmd), length($sensebuf), 0, length($buf), $buf, $cmd, $sensebuf, 6000
+ );
+
+ $ret = ioctl($fh, $SG_IO, $packet);
+ if (!$ret) {
+ die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
+ return;
+ }
+
+ my @res = unpack($sg_io_hdr_t, $packet);
+ if ($res[17] || $res[18]) {
+ die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
+ return;
+ }
+
+ my $res = {};
+ $res->@{qw(type removable vendor product revision)} = unpack("C C x6 A8 A16 A4", $buf);
+
+ $res->{removable} = $res->{removable} & 128 ? 1 : 0;
+ $res->{type} &= 0x1F;
+
+ return $res;
+}
+
+sub path_is_scsi {
+ my ($path) = @_;
+
+ my $fh = IO::File->new("+<$path") || return;
+ my $res = scsi_inquiry($fh, 1);
+ close($fh);
+
+ return $res;
+}
+
+sub get_scsi_device_type {
+ my ($drive, $storecfg, $machine_version) = @_;
+
+ my $devicetype = 'hd';
+ my $path = '';
+ if (drive_is_cdrom($drive) || drive_is_cloudinit($drive)) {
+ $devicetype = 'cd';
+ } else {
+ if ($drive->{file} =~ m|^/|) {
+ $path = $drive->{file};
+ if (my $info = path_is_scsi($path)) {
+ if ($info->{type} == 0 && $drive->{scsiblock}) {
+ $devicetype = 'block';
+ } elsif ($info->{type} == 1) { # tape
+ $devicetype = 'generic';
+ }
+ }
+ } elsif ($drive->{file} =~ $NEW_DISK_RE){
+ # special syntax cannot be parsed to path
+ return $devicetype;
+ } else {
+ $path = PVE::Storage::path($storecfg, $drive->{file});
+ }
+
+ # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
+ if ($path =~ m/^iscsi\:\/\// &&
+ !PVE::QemuServer::Helpers::min_version($machine_version, 4, 1)) {
+ $devicetype = 'generic';
+ }
+ }
+
+ return $devicetype;
+}
1;
use warnings;
use File::stat;
+use JSON;
use PVE::INotify;
use PVE::ProcFSTools;
our @EXPORT_OK = qw(
min_version
config_aware_timeout
+parse_number_sets
+windows_version
);
my $nodename = PVE::INotify::nodename();
}
sub config_aware_timeout {
- my ($config, $is_suspended) = @_;
- my $memory = $config->{memory};
+ my ($config, $memory, $is_suspended) = @_;
my $timeout = 30;
# Based on user reported startup time for vm with 512GiB @ 4-5 minutes
$timeout = int($memory/1024);
}
+ # When using PCI passthrough, users reported much higher startup times,
+ # growing with the amount of memory configured. Constant factor chosen
+ # based on user reports.
+ if (grep(/^hostpci[0-9]+$/, keys %$config)) {
+ $timeout *= 4;
+ }
+
if ($is_suspended && $timeout < 300) {
$timeout = 300;
}
return $timeout;
}
+sub get_node_pvecfg_version {
+ my ($node) = @_;
+
+ my $nodes_version_info = PVE::Cluster::get_node_kv('version-info', $node);
+ return if !$nodes_version_info->{$node};
+
+ my $version_info = decode_json($nodes_version_info->{$node});
+ return $version_info->{version};
+}
+
+sub pvecfg_min_version {
+ my ($verstr, $major, $minor, $release) = @_;
+
+ return 0 if !$verstr;
+
+ if ($verstr =~ m/^(\d+)\.(\d+)(?:[.-](\d+))?/) {
+ return 1 if version_cmp($1, $major, $2, $minor, $3 // 0, $release) >= 0;
+ return 0;
+ }
+
+ die "internal error: cannot check version of invalid string '$verstr'";
+}
+
+sub parse_number_sets {
+ my ($set) = @_;
+ my $res = [];
+ foreach my $part (split(/;/, $set)) {
+ if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
+ die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
+ push @$res, [ $1, $2 ];
+ } else {
+ die "invalid range: $part\n";
+ }
+ }
+ return $res;
+}
+
+sub windows_version {
+ my ($ostype) = @_;
+
+ return 0 if !$ostype;
+
+ my $winversion = 0;
+
+ if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
+ $winversion = 5;
+ } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
+ $winversion = 6;
+ } elsif ($ostype =~ m/^win(\d+)$/) {
+ $winversion = $1;
+ }
+
+ return $winversion;
+}
+
1;
# and creates by default a drive entry unused[n] pointing to the created volume
# $params->{drive_name} may be used to specify ide0, scsi1, etc ...
# $params->{format} may be used to specify qcow2, raw, etc ...
+# $params->{skiplock} may be used to skip checking for a lock in the VM config
+# $params->{'skip-config-update'} may be used to import the disk without updating the VM config
sub do_import {
my ($src_path, $vmid, $storage_id, $params) = @_;
# get target format, target image's path, and whether it's possible to sparseinit
my $storecfg = PVE::Storage::config();
my $dst_format = PVE::QemuServer::resolve_dst_disk_format($storecfg, $storage_id, undef, $format);
+ warn "format '$format' is not supported by the target storage - using '$dst_format' instead\n"
+ if $format && $format ne $dst_format;
my $dst_volid = PVE::Storage::vdisk_alloc($storecfg, $storage_id, $vmid, $dst_format, undef, $src_size / 1024);
PVE::Storage::activate_volumes($storecfg, [$dst_volid]);
PVE::QemuServer::qemu_img_convert($src_path, $dst_volid, $src_size, undef, $zeroinit);
PVE::Storage::deactivate_volumes($storecfg, [$dst_volid]);
- PVE::QemuConfig->lock_config($vmid, $create_drive);
+ PVE::QemuConfig->lock_config($vmid, $create_drive) if !$params->{'skip-config-update'};
};
if (my $err = $@) {
eval { PVE::Storage::vdisk_free($storecfg, $dst_volid) };
use PVE::QemuServer::Helpers;
use PVE::QemuServer::Monitor;
+use PVE::JSONSchema qw(get_standard_option parse_property_string print_property_string);
# Bump this for VM HW layout changes during a release (where the QEMU machine
# version stays the same)
'4.1' => 2,
};
+my $machine_fmt = {
+ type => {
+ default_key => 1,
+ description => "Specifies the QEMU machine type.",
+ type => 'string',
+ pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
+ maxLength => 40,
+ format_description => 'machine type',
+ optional => 1,
+ },
+ viommu => {
+ type => 'string',
+ description => "Enable and set guest vIOMMU variant (Intel vIOMMU needs q35 to be set as"
+ ." machine type).",
+ enum => ['intel', 'virtio'],
+ optional => 1,
+ },
+};
+
+PVE::JSONSchema::register_format('pve-qemu-machine-fmt', $machine_fmt);
+
+PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
+ description => "Specify the QEMU machine.",
+ type => 'string',
+ optional => 1,
+ format => PVE::JSONSchema::get_format('pve-qemu-machine-fmt'),
+});
+
+sub parse_machine {
+ my ($value) = @_;
+
+ return if !$value;
+
+ my $res = parse_property_string($machine_fmt, $value);
+ return $res;
+}
+
+sub print_machine {
+ my ($machine_conf) = @_;
+ return print_property_string($machine_conf, $machine_fmt);
+}
+
+sub assert_valid_machine_property {
+ my ($conf, $machine_conf) = @_;
+ my $q35 = $machine_conf->{type} && ($machine_conf->{type} =~ m/q35/) ? 1 : 0;
+ if ($machine_conf->{viommu} && $machine_conf->{viommu} eq "intel" && !$q35) {
+ die "to use Intel vIOMMU please set the machine type to q35\n";
+ }
+}
+
sub machine_type_is_q35 {
my ($conf) = @_;
- return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
+ my $machine_conf = parse_machine($conf->{machine});
+ return $machine_conf->{type} && ($machine_conf->{type} =~ m/q35/) ? 1 : 0;
}
-# this only works if VM is running
-sub get_current_qemu_machine {
- my ($vmid) = @_;
+# In list context, also returns whether the current machine is deprecated or not.
+sub current_from_query_machines {
+ my ($machines) = @_;
- my $res = PVE::QemuServer::Monitor::mon_cmd($vmid, 'query-machines');
+ my ($current, $default);
+ for my $machine ($machines->@*) {
+ $default = $machine->{name} if $machine->{'is-default'};
- my ($current, $pve_version, $default);
- foreach my $e (@$res) {
- $default = $e->{name} if $e->{'is-default'};
- $current = $e->{name} if $e->{'is-current'};
- $pve_version = $e->{'pve-version'} if $e->{'pve-version'};
+ if ($machine->{'is-current'}) {
+ $current = $machine->{name};
+ # pve-version only exists for the current machine
+ $current .= "+$machine->{'pve-version'}" if $machine->{'pve-version'};
+ return wantarray ? ($current, $machine->{deprecated} ? 1 : 0) : $current;
+ }
}
- $current .= "+$pve_version" if $current && $pve_version;
+ # fallback to the default machine if current is not supported by qemu - assume never deprecated
+ my $fallback = $default || 'pc';
+ return wantarray ? ($fallback, 0) : $fallback;
+}
+
+# This only works if VM is running.
+# In list context, also returns whether the current machine is deprecated or not.
+sub get_current_qemu_machine {
+ my ($vmid) = @_;
+
+ my $res = PVE::QemuServer::Monitor::mon_cmd($vmid, 'query-machines');
- # fallback to the default machine if current is not supported by qemu
- return $current || $default || 'pc';
+ return current_from_query_machines($res);
}
# returns a string with major.minor+pve<VERSION>, patch version-part is ignored
sub extract_version {
my ($machine_type, $kvmversion) = @_;
- if (defined($machine_type) && $machine_type =~ m/^(?:pc(?:-i440fx|-q35)?|virt)-(\d+)\.(\d+)(?:\.(\d+))?(\+pve\d+)?/) {
+ if (defined($machine_type) && $machine_type =~
+ m/^(?:pc(?:-i440fx|-q35)?|virt)-(\d+)\.(\d+)(?:\.(\d+))?(\+pve\d+)?(?:\.pxe)?/)
+ {
my $versionstr = "$1.$2";
$versionstr .= $4 if $4;
return $versionstr;
sub can_run_pve_machine_version {
my ($machine_version, $kvmversion) = @_;
- $machine_version =~ m/^(\d+)\.(\d+)(?:\+pve(\d+))$/;
+ $machine_version =~ m/^(\d+)\.(\d+)(?:\+pve(\d+))?(?:\.pxe)?$/;
my $major = $1;
my $minor = $2;
my $pvever = $3;
my $machine = get_current_qemu_machine($vmid);
- if ($conf->{machine} && $conf->{machine} =~ m/\.pxe$/) {
+ my $machine_conf = parse_machine($conf->{machine});
+ if ($machine_conf->{type} && $machine_conf->{type} =~ m/\.pxe$/) {
$machine .= '.pxe';
}
Monitor.pm \
Machine.pm \
CPUConfig.pm \
+ CGroup.pm \
Drive.pm \
+ QMPHelpers.pm
.PHONY: install
install: ${SOURCES}
use strict;
use warnings;
+use PVE::JSONSchema qw(parse_property_string);
use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
use PVE::Exception qw(raise raise_param_exc);
-use PVE::QemuServer;
+use PVE::QemuServer::Helpers qw(parse_number_sets);
use PVE::QemuServer::Monitor qw(mon_cmd);
+use PVE::QemuServer::QMPHelpers qw(qemu_devicedel qemu_objectdel);
+
+use base qw(Exporter);
+
+our @EXPORT_OK = qw(
+get_current_memory
+);
+
+our $MAX_NUMA = 8;
+
+my $numa_fmt = {
+ cpus => {
+ type => "string",
+ pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
+ description => "CPUs accessing this NUMA node.",
+ format_description => "id[-id];...",
+ },
+ memory => {
+ type => "number",
+ description => "Amount of memory this NUMA node provides.",
+ optional => 1,
+ },
+ hostnodes => {
+ type => "string",
+ pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
+ description => "Host NUMA nodes to use.",
+ format_description => "id[-id];...",
+ optional => 1,
+ },
+ policy => {
+ type => 'string',
+ enum => [qw(preferred bind interleave)],
+ description => "NUMA allocation policy.",
+ optional => 1,
+ },
+};
+PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
+our $numadesc = {
+ optional => 1,
+ type => 'string', format => $numa_fmt,
+ description => "NUMA topology.",
+};
+PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
+
+sub parse_numa {
+ my ($data) = @_;
+
+ my $res = parse_property_string($numa_fmt, $data);
+ $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
+ $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
+ return $res;
+}
-my $MAX_NUMA = 8;
-my $MAX_MEM = 4194304;
my $STATICMEM = 1024;
+our $memory_fmt = {
+ current => {
+ description => "Current amount of online RAM for the VM in MiB. This is the maximum available memory when"
+ ." you use the balloon device.",
+ type => 'integer',
+ default_key => 1,
+ minimum => 16,
+ default => 512,
+ },
+};
+
+sub print_memory {
+ my $memory = shift;
+
+ return PVE::JSONSchema::print_property_string($memory, $memory_fmt);
+}
+
+sub parse_memory {
+ my ($value) = @_;
+
+ return { current => $memory_fmt->{current}->{default} } if !defined($value);
+
+ my $res = PVE::JSONSchema::parse_property_string($memory_fmt, $value);
+
+ return $res;
+}
+
+my $_host_bits;
+sub get_host_phys_address_bits {
+ return $_host_bits if defined($_host_bits);
+
+ my $fh = IO::File->new ('/proc/cpuinfo', "r") or return;
+ while (defined(my $line = <$fh>)) {
+ # hopefully we never need to care about mixed (big.LITTLE) archs
+ if ($line =~ m/^address sizes\s*:\s*(\d+)\s*bits physical/i) {
+ $_host_bits = int($1);
+ $fh->close();
+ return $_host_bits;
+ }
+ }
+ $fh->close();
+ return; # undef, cannot really do anything..
+}
+
+my sub get_max_mem {
+ my ($conf) = @_;
+
+ my $cpu = {};
+ if (my $cpu_prop_str = $conf->{cpu}) {
+ $cpu = PVE::JSONSchema::parse_property_string('pve-vm-cpu-conf', $cpu_prop_str)
+ or die "Cannot parse cpu description: $cpu_prop_str\n";
+ }
+ my $bits;
+ if (my $phys_bits = $cpu->{'phys-bits'}) {
+ if ($phys_bits eq 'host') {
+ $bits = get_host_phys_address_bits();
+ } elsif ($phys_bits =~ /^(\d+)$/) {
+ $bits = int($phys_bits);
+ }
+ }
+
+ if (!defined($bits)) {
+ my $host_bits = get_host_phys_address_bits() // 36; # fixme: what fallback?
+ if ($cpu->{cputype} && $cpu->{cputype} =~ /^(host|max)$/) {
+ $bits = $host_bits;
+ } else {
+ $bits = $host_bits > 40 ? 40 : $host_bits; # take the smaller one
+ }
+ }
+
+ $bits = $bits & ~1; # round down to nearest even as limit is lower with odd bit sizes
+
+ # heuristic: remove 20 bits to get MB and half that as QEMU needs some overhead
+ my $bits_to_max_mem = int(1<<($bits - 21));
+
+ return $bits_to_max_mem > 4*1024*1024 ? 4*1024*1024 : $bits_to_max_mem;
+}
+
+sub get_current_memory {
+ my ($value) = @_;
+
+ my $memory = parse_memory($value);
+ return $memory->{current};
+}
+
sub get_numa_node_list {
my ($conf) = @_;
my @numa_map;
for (my $i = 0; $i < $MAX_NUMA; $i++) {
my $entry = $conf->{"numa$i"} or next;
- my $numa = PVE::QemuServer::parse_numa($entry) or next;
+ my $numa = parse_numa($entry) or next;
push @numa_map, $i;
}
return @numa_map if @numa_map;
return (0..($sockets-1));
}
+sub host_numanode_exists {
+ my ($id) = @_;
+
+ return -d "/sys/devices/system/node/node$id/";
+}
+
# only valid when numa nodes map to a single host node
sub get_numa_guest_to_host_map {
my ($conf) = @_;
my $map = {};
for (my $i = 0; $i < $MAX_NUMA; $i++) {
my $entry = $conf->{"numa$i"} or next;
- my $numa = PVE::QemuServer::parse_numa($entry) or next;
+ my $numa = parse_numa($entry) or next;
$map->{$i} = print_numa_hostnodes($numa->{hostnodes});
}
return $map if %$map;
}
sub foreach_dimm{
- my ($conf, $vmid, $memory, $sockets, $func) = @_;
+ my ($conf, $vmid, $memory, $static_memory, $func) = @_;
my $dimm_id = 0;
- my $current_size = 0;
+ my $current_size = $static_memory;
my $dimm_size = 0;
if($conf->{hugepages} && $conf->{hugepages} == 1024) {
- $current_size = 1024 * $sockets;
$dimm_size = 1024;
} else {
- $current_size = 1024;
$dimm_size = 512;
}
}
}
-sub foreach_reverse_dimm {
- my ($conf, $vmid, $memory, $sockets, $func) = @_;
-
- my $dimm_id = 253;
- my $current_size = 0;
- my $dimm_size = 0;
-
- if($conf->{hugepages} && $conf->{hugepages} == 1024) {
- $current_size = 8355840;
- $dimm_size = 131072;
- } else {
- $current_size = 4177920;
- $dimm_size = 65536;
- }
-
- return if $current_size == $memory;
-
- my @numa_map = get_numa_node_list($conf);
-
- for (my $j = 0; $j < 8; $j++) {
- for (my $i = 0; $i < 32; $i++) {
- my $name = "dimm${dimm_id}";
- $dimm_id--;
- my $numanode = $numa_map[(31-$i) % @numa_map];
- $current_size -= $dimm_size;
- &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
- return $current_size if $current_size <= $memory;
- }
- $dimm_size /= 2;
- }
-}
-
sub qemu_memory_hotplug {
- my ($vmid, $conf, $defaults, $opt, $value) = @_;
+ my ($vmid, $conf, $value) = @_;
+
+ return $value if !PVE::QemuServer::Helpers::vm_running_locally($vmid);
- return $value if !PVE::QemuServer::check_running($vmid);
+ my $oldmem = parse_memory($conf->{memory});
+ my $newmem = parse_memory($value);
- my $sockets = 1;
- $sockets = $conf->{sockets} if $conf->{sockets};
+ return $value if $newmem->{current} == $oldmem->{current};
- my $memory = $conf->{memory} || $defaults->{memory};
- $value = $defaults->{memory} if !$value;
- return $value if $value == $memory;
+ my $memory = $oldmem->{current};
+ $value = $newmem->{current};
+ my $sockets = $conf->{sockets} || 1;
my $static_memory = $STATICMEM;
$static_memory = $static_memory * $sockets if ($conf->{hugepages} && $conf->{hugepages} == 1024);
die "memory can't be lower than $static_memory MB" if $value < $static_memory;
- die "you cannot add more memory than $MAX_MEM MB!\n" if $memory > $MAX_MEM;
+ my $MAX_MEM = get_max_mem($conf);
+ die "you cannot add more memory than max mem $MAX_MEM MB!\n" if $value > $MAX_MEM;
- if($value > $memory) {
+ if ($value > $memory) {
my $numa_hostmap;
- foreach_dimm($conf, $vmid, $value, $sockets, sub {
+ foreach_dimm($conf, $vmid, $value, $static_memory, sub {
my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
- return if $current_size <= $conf->{memory};
+ return if $current_size <= get_current_memory($conf->{memory});
if ($conf->{hugepages}) {
$numa_hostmap = get_numa_guest_to_host_map($conf) if !$numa_hostmap;
my $hugepages_host_topology = hugepages_host_topology();
hugepages_allocate($hugepages_topology, $hugepages_host_topology);
- eval { mon_cmd($vmid, "object-add", 'qom-type' => "memory-backend-file", id => "mem-$name", props => {
- size => int($dimm_size*1024*1024), 'mem-path' => $path, share => JSON::true, prealloc => JSON::true } ); };
+ eval { mon_cmd($vmid, "object-add", 'qom-type' => "memory-backend-file", id => "mem-$name", size => int($dimm_size*1024*1024), 'mem-path' => $path, share => JSON::true, prealloc => JSON::true ) };
if (my $err = $@) {
hugepages_reset($hugepages_host_topology);
die $err;
eval { hugepages_update_locked($code); };
} else {
- eval { mon_cmd($vmid, "object-add", 'qom-type' => "memory-backend-ram", id => "mem-$name", props => { size => int($dimm_size*1024*1024) } ) };
+ eval { mon_cmd($vmid, "object-add", 'qom-type' => "memory-backend-ram", id => "mem-$name", size => int($dimm_size*1024*1024) ) };
}
if (my $err = $@) {
- eval { PVE::QemuServer::qemu_objectdel($vmid, "mem-$name"); };
+ eval { qemu_objectdel($vmid, "mem-$name"); };
die $err;
}
eval { mon_cmd($vmid, "device_add", driver => "pc-dimm", id => "$name", memdev => "mem-$name", node => $numanode) };
if (my $err = $@) {
- eval { PVE::QemuServer::qemu_objectdel($vmid, "mem-$name"); };
+ eval { qemu_objectdel($vmid, "mem-$name"); };
die $err;
}
#update conf after each succesful module hotplug
- $conf->{memory} = $current_size;
+ $newmem->{current} = $current_size;
+ $conf->{memory} = print_memory($newmem);
PVE::QemuConfig->write_config($vmid, $conf);
});
} else {
- foreach_reverse_dimm($conf, $vmid, $value, $sockets, sub {
- my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
+ my $dimms = qemu_memdevices_list($vmid, 'dimm');
- return if $current_size >= $conf->{memory};
- print "try to unplug memory dimm $name\n";
-
- my $retry = 0;
- while (1) {
- eval { PVE::QemuServer::qemu_devicedel($vmid, $name) };
- sleep 3;
- my $dimm_list = qemu_dimm_list($vmid);
- last if !$dimm_list->{$name};
- raise_param_exc({ $name => "error unplug memory module" }) if $retry > 5;
- $retry++;
- }
+ my $current_size = $memory;
+ for my $name (sort { ($b =~ /^dimm(\d+)$/)[0] <=> ($a =~ /^dimm(\d+)$/)[0] } keys %$dimms) {
- #update conf after each succesful module unplug
- $conf->{memory} = $current_size;
+ my $dimm_size = $dimms->{$name}->{size} / 1024 / 1024;
- eval { PVE::QemuServer::qemu_objectdel($vmid, "mem-$name"); };
- PVE::QemuConfig->write_config($vmid, $conf);
- });
+ last if $current_size <= $value;
+
+ print "try to unplug memory dimm $name\n";
+
+ my $retry = 0;
+ while (1) {
+ eval { qemu_devicedel($vmid, $name) };
+ sleep 3;
+ my $dimm_list = qemu_memdevices_list($vmid, 'dimm');
+ last if !$dimm_list->{$name};
+ raise_param_exc({ $name => "error unplug memory module" }) if $retry > 5;
+ $retry++;
+ }
+ $current_size -= $dimm_size;
+ #update conf after each succesful module unplug
+ $newmem->{current} = $current_size;
+ $conf->{memory} = print_memory($newmem);
+
+ eval { qemu_objectdel($vmid, "mem-$name"); };
+ PVE::QemuConfig->write_config($vmid, $conf);
+ }
}
+ return $conf->{memory};
}
-sub qemu_dimm_list {
- my ($vmid) = @_;
+sub qemu_memdevices_list {
+ my ($vmid, $type) = @_;
my $dimmarray = mon_cmd($vmid, "query-memory-devices");
my $dimms = {};
foreach my $dimm (@$dimmarray) {
-
+ next if $type && $dimm->{data}->{id} !~ /^$type(\d+)$/;
$dimms->{$dimm->{data}->{id}}->{id} = $dimm->{data}->{id};
$dimms->{$dimm->{data}->{id}}->{node} = $dimm->{data}->{node};
$dimms->{$dimm->{data}->{id}}->{addr} = $dimm->{data}->{addr};
}
sub config {
- my ($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd) = @_;
+ my ($conf, $vmid, $sockets, $cores, $hotplug, $cmd) = @_;
- my $memory = $conf->{memory} || $defaults->{memory};
+ my $memory = get_current_memory($conf->{memory});
my $static_memory = 0;
- if ($hotplug_features->{memory}) {
+ if ($hotplug) {
die "NUMA needs to be enabled for memory hotplug\n" if !$conf->{numa};
+ my $MAX_MEM = get_max_mem($conf);
die "Total memory is bigger than ${MAX_MEM}MB\n" if $memory > $MAX_MEM;
for (my $i = 0; $i < $MAX_NUMA; $i++) {
if $conf->{"numa$i"};
}
- my $sockets = 1;
- $sockets = $conf->{sockets} if $conf->{sockets};
+ my $sockets = $conf->{sockets} || 1;
$static_memory = $STATICMEM;
$static_memory = $static_memory * $sockets if ($conf->{hugepages} && $conf->{hugepages} == 1024);
my $numa_totalmemory = undef;
for (my $i = 0; $i < $MAX_NUMA; $i++) {
next if !$conf->{"numa$i"};
- my $numa = PVE::QemuServer::parse_numa($conf->{"numa$i"});
+ my $numa = parse_numa($conf->{"numa$i"});
next if !$numa;
# memory
die "missing NUMA node$i memory value\n" if !$numa->{memory};
my $numa_memory = ($static_memory / $sockets);
for (my $i = 0; $i < $sockets; $i++) {
- die "host NUMA node$i doesn't exist\n" if ! -d "/sys/devices/system/node/node$i/" && $conf->{hugepages};
+ die "host NUMA node$i doesn't exist\n"
+ if !host_numanode_exists($i) && $conf->{hugepages};
my $mem_object = print_mem_object($conf, "ram-node$i", $numa_memory);
push @$cmd, '-object', $mem_object;
}
}
- if ($hotplug_features->{memory}) {
- foreach_dimm($conf, $vmid, $memory, $sockets, sub {
+ if ($hotplug) {
+ foreach_dimm($conf, $vmid, $memory, $static_memory, sub {
my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
my $mem_object = print_mem_object($conf, "mem-$name", $dimm_size);
$hostnodes .= "-$end" if defined($end);
$end //= $start;
for (my $i = $start; $i <= $end; ++$i ) {
- die "host NUMA node$i doesn't exist\n" if ! -d "/sys/devices/system/node/node$i/";
+ die "host NUMA node$i doesn't exist\n" if !host_numanode_exists($i);
}
}
return $hostnodes;
return $size / $hugepages_size;
}
+sub hugepages_chunk_size_supported {
+ my ($size) = @_;
+
+ return -d "/sys/kernel/mm/hugepages/hugepages-". ($size * 1024) ."kB";
+}
+
sub hugepages_size {
my ($conf, $size) = @_;
die "hugepages option is not enabled" if !$conf->{hugepages};
die "memory size '$size' is not a positive even integer; cannot use for hugepages\n"
if $size <= 0 || $size & 1;
- my $page_chunk = sub { -d "/sys/kernel/mm/hugepages/hugepages-". ($_[0] * 1024) ."kB" };
- die "your system doesn't support hugepages\n" if !$page_chunk->(2) && !$page_chunk->(1024);
+ die "your system doesn't support hugepages\n"
+ if !hugepages_chunk_size_supported(2) && !hugepages_chunk_size_supported(1024);
if ($conf->{hugepages} eq 'any') {
# try to use 1GB if available && memory size is matching
- if ($page_chunk->(1024) && ($size & 1023) == 0) {
+ if (hugepages_chunk_size_supported(1024) && ($size & 1023) == 0) {
return 1024;
- } elsif ($page_chunk->(2)) {
+ } elsif (hugepages_chunk_size_supported(2)) {
return 2;
} else {
die "host only supports 1024 GB hugepages, but requested size '$size' is not a multiple of 1024 MB\n"
my $hugepagesize = $conf->{hugepages};
- if (!$page_chunk->($hugepagesize)) {
+ if (!hugepages_chunk_size_supported($hugepagesize)) {
die "your system doesn't support hugepages of $hugepagesize MB\n";
} elsif (($size % $hugepagesize) != 0) {
die "Memory size $size is not a multiple of the requested hugepages size $hugepagesize\n";
}
sub hugepages_topology {
- my ($conf) = @_;
+ my ($conf, $hotplug) = @_;
my $hugepages_topology = {};
return if !$conf->{numa};
- my $defaults = PVE::QemuServer::load_defaults();
- my $memory = $conf->{memory} || $defaults->{memory};
+ my $memory = get_current_memory($conf->{memory});
my $static_memory = 0;
- my $sockets = 1;
- $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
- $sockets = $conf->{sockets} if $conf->{sockets};
+ my $sockets = $conf->{sockets} || 1;
my $numa_custom_topology = undef;
- my $hotplug_features = PVE::QemuServer::parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
- if ($hotplug_features->{memory}) {
+ if ($hotplug) {
$static_memory = $STATICMEM;
$static_memory = $static_memory * $sockets if ($conf->{hugepages} && $conf->{hugepages} == 1024);
} else {
#custom numa topology
for (my $i = 0; $i < $MAX_NUMA; $i++) {
next if !$conf->{"numa$i"};
- my $numa = PVE::QemuServer::parse_numa($conf->{"numa$i"});
+ my $numa = parse_numa($conf->{"numa$i"});
next if !$numa;
$numa_custom_topology = 1;
}
}
- if ($hotplug_features->{memory}) {
+ if ($hotplug) {
my $numa_hostmap = get_numa_guest_to_host_map($conf);
- foreach_dimm($conf, undef, $memory, $sockets, sub {
+ foreach_dimm($conf, undef, $memory, $static_memory, sub {
my ($conf, undef, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
$numanode = $numa_hostmap->{$numanode};
my $ovf_name = $xpc->findvalue($xpath_find_name);
if ($ovf_name) {
- ($qm->{name} = $ovf_name) =~ s/[^a-zA-Z0-9\-]//g; # PVE::QemuServer::confdesc requires a valid DNS name
+ # PVE::QemuServer::confdesc requires a valid DNS name
+ ($qm->{name} = $ovf_name) =~ s/[^a-zA-Z0-9\-\.]//g;
} else {
warn "warning: unable to parse the VM name in this OVF manifest, generating a default value\n";
}
die "error parsing $filepath, file seems not to exist at $backing_file_path\n";
}
- my $virtual_size;
- if ( !($virtual_size = PVE::Storage::file_size_info($backing_file_path)) ) {
- die "error parsing $backing_file_path, size seems to be $virtual_size\n";
- }
+ ($backing_file_path) = $backing_file_path =~ m|^(/.*)|; # untaint
+
+ my $virtual_size = PVE::Storage::file_size_info($backing_file_path);
+ die "error parsing $backing_file_path, cannot determine file size\n"
+ if !$virtual_size;
$pve_disk = {
disk_address => $pve_disk_address,
use strict;
use PVE::JSONSchema;
+use PVE::Mapping::PCI;
use PVE::SysFSTools;
+use PVE::Tools;
use base 'Exporter';
our $MAX_HOSTPCI_DEVICES = 16;
-my $PCIRE = qr/([a-f0-9]{4}:)?[a-f0-9]{2}:[a-f0-9]{2}(?:\.[a-f0-9])?/;
+my $PCIRE = qr/(?:[a-f0-9]{4,}:)?[a-f0-9]{2}:[a-f0-9]{2}(?:\.[a-f0-9])?/;
my $hostpci_fmt = {
host => {
default_key => 1,
+ optional => 1,
type => 'string',
pattern => qr/$PCIRE(;$PCIRE)*/,
format_description => 'HOSTPCIID[;HOSTPCIID2...]',
'bus:dev.func' (hexadecimal numbers)
You can us the 'lspci' command to list existing PCI devices.
+
+Either this or the 'mapping' key must be set.
EODESCR
},
+ mapping => {
+ optional => 1,
+ type => 'string',
+ format_description => 'mapping-id',
+ format => 'pve-configid',
+ description => "The ID of a cluster wide mapping. Either this or the default-key 'host'"
+ ." must be set.",
+ },
rombar => {
type => 'boolean',
description => "Specify whether or not the device's ROM will be visible in the"
An instance of this type will be created on startup of the VM and
will be cleaned up when the VM stops.
EODESCR
+ },
+ 'vendor-id' => {
+ type => 'string',
+ pattern => qr/^0x[0-9a-fA-F]{4}$/,
+ format_description => 'hex id',
+ optional => 1,
+ description => "Override PCI vendor ID visible to guest"
+ },
+ 'device-id' => {
+ type => 'string',
+ pattern => qr/^0x[0-9a-fA-F]{4}$/,
+ format_description => 'hex id',
+ optional => 1,
+ description => "Override PCI device ID visible to guest"
+ },
+ 'sub-vendor-id' => {
+ type => 'string',
+ pattern => qr/^0x[0-9a-fA-F]{4}$/,
+ format_description => 'hex id',
+ optional => 1,
+ description => "Override PCI subsystem vendor ID visible to guest"
+ },
+ 'sub-device-id' => {
+ type => 'string',
+ pattern => qr/^0x[0-9a-fA-F]{4}$/,
+ format_description => 'hex id',
+ optional => 1,
+ description => "Override PCI subsystem device ID visible to guest"
}
};
PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
return $pci_addr_map;
}
+sub generate_mdev_uuid {
+ my ($vmid, $index) = @_;
+ return sprintf("%08d-0000-0000-0000-%012d", $index, $vmid);
+}
+
my $get_addr_mapping_from_id = sub {
my ($map, $id) = @_;
return $res;
}
+# returns the parsed pci config but parses the 'host' part into
+# a list if lists into the 'id' property like this:
+#
+# {
+# mdev => 1,
+# rombar => ...
+# ...
+# ids => [
+# # this contains a list of alternative devices,
+# [
+# # which are itself lists of ids for one multifunction device
+# {
+# id => "0000:00:00.0",
+# vendor => "...",
+# },
+# {
+# id => "0000:00:00.1",
+# vendor => "...",
+# },
+# ],
+# [
+# ...
+# ],
+# ...
+# ],
+# }
sub parse_hostpci {
my ($value) = @_;
my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
- my @idlist = split(/;/, $res->{host});
- delete $res->{host};
- foreach my $id (@idlist) {
- my $devs = PVE::SysFSTools::lspci($id);
- die "no PCI device found for '$id'\n" if !scalar(@$devs);
- push @{$res->{pciid}}, @$devs;
+ my $alternatives = [];
+ my $host = delete $res->{host};
+ my $mapping = delete $res->{mapping};
+
+ die "Cannot set both 'host' and 'mapping'.\n" if defined($host) && defined($mapping);
+
+ if ($mapping) {
+ # we have no ordinary pci id, must be a mapping
+ my $devices = PVE::Mapping::PCI::find_on_current_node($mapping);
+ die "PCI device mapping not found for '$mapping'\n" if !$devices || !scalar($devices->@*);
+
+ for my $device ($devices->@*) {
+ eval { PVE::Mapping::PCI::assert_valid($mapping, $device) };
+ die "PCI device mapping invalid (hardware probably changed): $@\n" if $@;
+ push $alternatives->@*, [split(/;/, $device->{path})];
+ }
+ } elsif ($host) {
+ push $alternatives->@*, [split(/;/, $host)];
+ } else {
+ die "Either 'host' or 'mapping' must be set.\n";
+ }
+
+ $res->{ids} = [];
+ for my $alternative ($alternatives->@*) {
+ my $ids = [];
+ foreach my $id ($alternative->@*) {
+ my $devs = PVE::SysFSTools::lspci($id);
+ die "no PCI device found for '$id'\n" if !scalar($devs->@*);
+ push $ids->@*, @$devs;
+ }
+ if (scalar($ids->@*) > 1) {
+ $res->{'has-multifunction'} = 1;
+ die "cannot use mediated device with multifunction device\n" if $res->{mdev};
+ }
+ push $res->{ids}->@*, $ids;
}
+
return $res;
}
+# parses all hostpci devices from a config and does some sanity checks
+# returns a hash like this:
+# {
+# hostpci0 => {
+# # hash from parse_hostpci function
+# },
+# hostpci1 => { ... },
+# ...
+# }
+sub parse_hostpci_devices {
+ my ($conf) = @_;
+
+ my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
+ my $legacy_igd = 0;
+
+ my $parsed_devices = {};
+ for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++) {
+ my $id = "hostpci$i";
+ my $d = parse_hostpci($conf->{$id});
+ next if !$d;
+
+ # check syntax
+ die "q35 machine model is not enabled" if !$q35 && $d->{pcie};
+
+ if ($d->{'legacy-igd'}) {
+ die "only one device can be assigned in legacy-igd mode\n"
+ if $legacy_igd;
+ $legacy_igd = 1;
+
+ die "legacy IGD assignment requires VGA mode to be 'none'\n"
+ if !defined($conf->{'vga'}) || $conf->{'vga'} ne 'none';
+ die "legacy IGD assignment requires rombar to be enabled\n"
+ if defined($d->{rombar}) && !$d->{rombar};
+ die "legacy IGD assignment is not compatible with x-vga\n"
+ if $d->{'x-vga'};
+ die "legacy IGD assignment is not compatible with mdev\n"
+ if $d->{mdev};
+ die "legacy IGD assignment is not compatible with q35\n"
+ if $q35;
+ die "legacy IGD assignment is not compatible with multifunction devices\n"
+ if $d->{'has-multifunction'};
+ die "legacy IGD assignment is not compatible with alternate devices\n"
+ if scalar($d->{ids}->@*) > 1;
+ # check first device for valid id
+ die "legacy IGD assignment only works for devices on host bus 00:02.0\n"
+ if $d->{ids}->[0]->[0]->{id} !~ m/02\.0$/;
+ }
+
+ $parsed_devices->{$id} = $d;
+ }
+
+ return $parsed_devices;
+}
+
+# takes the hash returned by parse_hostpci_devices and for all non mdev gpus,
+# selects one of the given alternatives by trying to reserve it
+#
+# mdev devices must be chosen later when we actually allocate it, but we
+# flatten the inner list since there can only be one device per alternative anyway
+my sub choose_hostpci_devices {
+ my ($devices, $vmid) = @_;
+
+ my $used = {};
+
+ my $add_used_device = sub {
+ my ($devices) = @_;
+ for my $used_device ($devices->@*) {
+ my $used_id = $used_device->{id};
+ die "device '$used_id' assigned more than once\n" if $used->{$used_id};
+ $used->{$used_id} = 1;
+ }
+ };
+
+ for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++) {
+ my $device = $devices->{"hostpci$i"};
+ next if !$device;
+
+ if ($device->{mdev}) {
+ $device->{ids} = [ map { $_->[0] } $device->{ids}->@* ];
+ next;
+ }
+
+ if (scalar($device->{ids}->@* == 1)) {
+ # we only have one alternative, use that
+ $device->{ids} = $device->{ids}->[0];
+ $add_used_device->($device->{ids});
+ next;
+ }
+
+ my $found = 0;
+ for my $alternative ($device->{ids}->@*) {
+ my $ids = [map { $_->{id} } @$alternative];
+
+ next if grep { defined($used->{$_}) } @$ids; # already used
+ eval { reserve_pci_usage($ids, $vmid, 10, undef) };
+ next if $@;
+
+ # found one that is not used or reserved
+ $add_used_device->($alternative);
+ $device->{ids} = $alternative;
+ $found = 1;
+ last;
+ }
+ die "could not find a free device for 'hostpci$i'\n" if !$found;
+ }
+
+ return $devices;
+}
+
sub print_hostpci_devices {
- my ($vmid, $conf, $devices, $vga, $winversion, $q35, $bridges, $arch, $machine_type, $bootorder) = @_;
+ my ($vmid, $conf, $devices, $vga, $winversion, $bridges, $arch, $machine_type, $bootorder) = @_;
my $kvm_off = 0;
my $gpu_passthrough = 0;
my $legacy_igd = 0;
my $pciaddr;
+ my $pci_devices = choose_hostpci_devices(parse_hostpci_devices($conf), $vmid);
+
for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++) {
my $id = "hostpci$i";
- my $d = parse_hostpci($conf->{$id});
+ my $d = $pci_devices->{$id};
next if !$d;
+ $legacy_igd = 1 if $d->{'legacy-igd'};
+
if (my $pcie = $d->{pcie}) {
- die "q35 machine model is not enabled" if !$q35;
# win7 wants to have the pcie devices directly on the pcie bus
# instead of in the root port
if ($winversion == 7) {
$pciaddr = print_pci_addr($pci_name, $bridges, $arch, $machine_type);
}
- my $pcidevices = $d->{pciid};
- my $multifunction = @$pcidevices > 1;
-
- if ($d->{'legacy-igd'}) {
- die "only one device can be assigned in legacy-igd mode\n"
- if $legacy_igd;
- $legacy_igd = 1;
-
- die "legacy IGD assignment requires VGA mode to be 'none'\n"
- if !defined($conf->{'vga'}) || $conf->{'vga'} ne 'none';
- die "legacy IGD assignment requires rombar to be enabled\n"
- if defined($d->{rombar}) && !$d->{rombar};
- die "legacy IGD assignment is not compatible with x-vga\n"
- if $d->{'x-vga'};
- die "legacy IGD assignment is not compatible with mdev\n"
- if $d->{mdev};
- die "legacy IGD assignment is not compatible with q35\n"
- if $q35;
- die "legacy IGD assignment is not compatible with multifunction devices\n"
- if $multifunction;
- die "legacy IGD assignment only works for devices on host bus 00:02.0\n"
- if $pcidevices->[0]->{id} !~ m/02\.0$/;
- }
+ my $num_devices = scalar($d->{ids}->@*);
+ my $multifunction = $num_devices > 1 && !$d->{mdev};
my $xvga = '';
if ($d->{'x-vga'}) {
}
my $sysfspath;
- if ($d->{mdev} && scalar(@$pcidevices) == 1) {
- my $pci_id = $pcidevices->[0]->{id};
- my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
- $sysfspath = "/sys/bus/pci/devices/$pci_id/$uuid";
- } elsif ($d->{mdev}) {
- warn "ignoring mediated device '$id' with multifunction device\n";
+ if ($d->{mdev}) {
+ my $uuid = generate_mdev_uuid($vmid, $i);
+ $sysfspath = "/sys/bus/mdev/devices/$uuid";
}
- my $j = 0;
- foreach my $pcidevice (@$pcidevices) {
+ for (my $j = 0; $j < $num_devices; $j++) {
+ my $pcidevice = $d->{ids}->[$j];
my $devicestr = "vfio-pci";
if ($sysfspath) {
$devicestr .= ",multifunction=on" if $multifunction;
$devicestr .= ",romfile=/usr/share/kvm/$d->{romfile}" if $d->{romfile};
$devicestr .= ",bootindex=$bootorder->{$id}" if $bootorder->{$id};
+ for my $option (qw(vendor-id device-id sub-vendor-id sub-device-id)) {
+ $devicestr .= ",x-pci-$option=$d->{$option}" if $d->{$option};
+ }
}
+
push @$devices, '-device', $devicestr;
- $j++;
+ last if $d->{mdev};
+ }
+ }
+
+ return ($kvm_off, $gpu_passthrough, $legacy_igd, $pci_devices);
+}
+
+sub prepare_pci_device {
+ my ($vmid, $pciid, $index, $mdev) = @_;
+
+ my $info = PVE::SysFSTools::pci_device_info("$pciid");
+ die "cannot prepare PCI pass-through, IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
+ die "no pci device info for device '$pciid'\n" if !$info;
+
+ if ($mdev) {
+ my $uuid = generate_mdev_uuid($vmid, $index);
+ PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $mdev);
+ } else {
+ die "can't unbind/bind PCI group to VFIO '$pciid'\n"
+ if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
+ die "can't reset PCI device '$pciid'\n"
+ if $info->{has_fl_reset} && !PVE::SysFSTools::pci_dev_reset($info);
+ }
+
+ return $info;
+}
+
+my $RUNDIR = '/run/qemu-server';
+my $PCIID_RESERVATION_FILE = "${RUNDIR}/pci-id-reservations";
+my $PCIID_RESERVATION_LOCK = "${PCIID_RESERVATION_FILE}.lock";
+
+# a list of PCI ID to VMID reservations, the validity is protected against leakage by either a PID,
+# for succesfully started VM processes, or a expiration time for the initial time window between
+# reservation and actual VM process start-up.
+my $parse_pci_reservation_unlocked = sub {
+ my $pciids = {};
+ if (my $fh = IO::File->new($PCIID_RESERVATION_FILE, "r")) {
+ while (my $line = <$fh>) {
+ if ($line =~ m/^($PCIRE)\s(\d+)\s(time|pid)\:(\d+)$/) {
+ $pciids->{$1} = {
+ vmid => $2,
+ "$3" => $4,
+ };
+ }
}
}
+ return $pciids;
+};
+
+my $write_pci_reservation_unlocked = sub {
+ my ($reservations) = @_;
- return ($kvm_off, $gpu_passthrough, $legacy_igd);
+ my $data = "";
+ for my $pci_id (sort keys $reservations->%*) {
+ my ($vmid, $pid, $time) = $reservations->{$pci_id}->@{'vmid', 'pid', 'time'};
+ if (defined($pid)) {
+ $data .= "$pci_id $vmid pid:$pid\n";
+ } else {
+ $data .= "$pci_id $vmid time:$time\n";
+ }
+ }
+ PVE::Tools::file_set_contents($PCIID_RESERVATION_FILE, $data);
+};
+
+# removes all PCI device reservations held by the `vmid`
+sub remove_pci_reservation {
+ my ($vmid) = @_;
+
+ PVE::Tools::lock_file($PCIID_RESERVATION_LOCK, 2, sub {
+ my $reservation_list = $parse_pci_reservation_unlocked->();
+ for my $id (keys %$reservation_list) {
+ my $reservation = $reservation_list->{$id};
+ next if $reservation->{vmid} != $vmid;
+ delete $reservation_list->{$id};
+ }
+ $write_pci_reservation_unlocked->($reservation_list);
+ });
+ die $@ if $@;
+}
+
+sub reserve_pci_usage {
+ my ($requested_ids, $vmid, $timeout, $pid) = @_;
+
+ $requested_ids = [ $requested_ids ] if !ref($requested_ids);
+ return if !scalar(@$requested_ids); # do nothing for empty list
+
+ PVE::Tools::lock_file($PCIID_RESERVATION_LOCK, 5, sub {
+ my $reservation_list = $parse_pci_reservation_unlocked->();
+
+ my $ctime = time();
+ for my $id ($requested_ids->@*) {
+ my $reservation = $reservation_list->{$id};
+ if ($reservation && $reservation->{vmid} != $vmid) {
+ # check time based reservation
+ die "PCI device '$id' is currently reserved for use by VMID '$reservation->{vmid}'\n"
+ if defined($reservation->{time}) && $reservation->{time} > $ctime;
+
+ if (my $reserved_pid = $reservation->{pid}) {
+ # check running vm
+ my $running_pid = PVE::QemuServer::Helpers::vm_running_locally($reservation->{vmid});
+ if (defined($running_pid) && $running_pid == $reserved_pid) {
+ die "PCI device '$id' already in use by VMID '$reservation->{vmid}'\n";
+ } else {
+ warn "leftover PCI reservation found for $id, lets take it...\n";
+ }
+ }
+ } elsif ($reservation) {
+ # already reserved by the same vmid
+ if (my $reserved_time = $reservation->{time}) {
+ if (defined($timeout)) {
+ # use the longer timeout
+ my $old_timeout = $reservation->{time} - 5 - $ctime;
+ $timeout = $old_timeout if $old_timeout > $timeout;
+ }
+ } elsif (my $reserved_pid = $reservation->{pid}) {
+ my $running_pid = PVE::QemuServer::Helpers::vm_running_locally($reservation->{vmid});
+ if (defined($running_pid) && $running_pid == $reservation->{pid}) {
+ if (defined($pid)) {
+ die "PCI device '$id' already in use by running VMID '$reservation->{vmid}'\n";
+ } elsif (defined($timeout)) {
+ # ignore timeout reservation for running vms, can happen with e.g.
+ # qm showcmd
+ return;
+ }
+ }
+ }
+ }
+
+ $reservation_list->{$id} = { vmid => $vmid };
+ if (defined($pid)) { # VM started up, we can reserve now with the actual PID
+ $reservation_list->{$id}->{pid} = $pid;
+ } elsif (defined($timeout)) { # tempoaray reserve as we don't now the PID yet
+ $reservation_list->{$id}->{time} = $ctime + $timeout + 5;
+ }
+ }
+ $write_pci_reservation_unlocked->($reservation_list);
+ });
+ die $@ if $@;
}
1;
--- /dev/null
+package PVE::QemuServer::QMPHelpers;
+
+use warnings;
+use strict;
+
+use PVE::QemuServer::Monitor qw(mon_cmd);
+
+use base 'Exporter';
+
+our @EXPORT_OK = qw(
+qemu_deviceadd
+qemu_devicedel
+qemu_objectadd
+qemu_objectdel
+);
+
+sub qemu_deviceadd {
+ my ($vmid, $devicefull) = @_;
+
+ $devicefull = "driver=".$devicefull;
+ my %options = split(/[=,]/, $devicefull);
+
+ mon_cmd($vmid, "device_add" , %options);
+}
+
+sub qemu_devicedel {
+ my ($vmid, $deviceid) = @_;
+
+ my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
+}
+
+sub qemu_objectadd {
+ my ($vmid, $objectid, $qomtype) = @_;
+
+ mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
+
+ return 1;
+}
+
+sub qemu_objectdel {
+ my ($vmid, $objectid) = @_;
+
+ mon_cmd($vmid, "object-del", id => $objectid);
+
+ return 1;
+}
+
+1;
use strict;
use warnings;
use PVE::QemuServer::PCI qw(print_pci_addr);
+use PVE::QemuServer::Machine;
+use PVE::QemuServer::Helpers qw(min_version windows_version);
use PVE::JSONSchema;
+use PVE::Mapping::USB;
use base 'Exporter';
our @EXPORT_OK = qw(
get_usb_devices
);
+my $OLD_MAX_USB = 5;
+our $MAX_USB_DEVICES = 14;
+
+
+my $USB_ID_RE = qr/(0x)?([0-9A-Fa-f]{4}):(0x)?([0-9A-Fa-f]{4})/;
+my $USB_PATH_RE = qr/(\d+)\-(\d+(\.\d+)*)/;
+
+my $usb_fmt = {
+ host => {
+ default_key => 1,
+ optional => 1,
+ type => 'string',
+ pattern => qr/(?:(?:$USB_ID_RE)|(?:$USB_PATH_RE)|[Ss][Pp][Ii][Cc][Ee])/,
+ format_description => 'HOSTUSBDEVICE|spice',
+ description => <<EODESCR,
+The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
+
+ 'bus-port(.port)*' (decimal numbers) or
+ 'vendor_id:product_id' (hexadeciaml numbers) or
+ 'spice'
+
+You can use the 'lsusb -t' command to list existing usb devices.
+
+NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
+machines - use with special care.
+
+The value 'spice' can be used to add a usb redirection devices for spice.
+
+Either this or the 'mapping' key must be set.
+EODESCR
+ },
+ mapping => {
+ optional => 1,
+ type => 'string',
+ format_description => 'mapping-id',
+ format => 'pve-configid',
+ description => "The ID of a cluster wide mapping. Either this or the default-key 'host'"
+ ." must be set.",
+ },
+ usb3 => {
+ optional => 1,
+ type => 'boolean',
+ description => "Specifies whether if given host option is a USB3 device or port."
+ ." For modern guests (machine version >= 7.1 and ostype l26 and windows > 7), this flag"
+ ." is irrelevant (all devices are plugged into a xhci controller).",
+ default => 0,
+ },
+};
+
+PVE::JSONSchema::register_format('pve-qm-usb', $usb_fmt);
+
+our $usbdesc = {
+ optional => 1,
+ type => 'string', format => $usb_fmt,
+ description => "Configure an USB device (n is 0 to 4, for machine version >= 7.1 and ostype"
+ ." l26 or windows > 7, n can be up to 14).",
+};
+PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
+
sub parse_usb_device {
- my ($value) = @_;
+ my ($value, $mapping) = @_;
- return if !$value;
+ return if $value && $mapping; # not a valid configuration
my $res = {};
- if ($value =~ m/^(0x)?([0-9A-Fa-f]{4}):(0x)?([0-9A-Fa-f]{4})$/) {
- $res->{vendorid} = $2;
- $res->{productid} = $4;
- } elsif ($value =~ m/^(\d+)\-(\d+(\.\d+)*)$/) {
- $res->{hostbus} = $1;
- $res->{hostport} = $2;
- } elsif ($value =~ m/^spice$/i) {
- $res->{spice} = 1;
- } else {
- return;
+ if (defined($value)) {
+ if ($value =~ m/^$USB_ID_RE$/) {
+ $res->{vendorid} = $2;
+ $res->{productid} = $4;
+ } elsif ($value =~ m/^$USB_PATH_RE$/) {
+ $res->{hostbus} = $1;
+ $res->{hostport} = $2;
+ } elsif ($value =~ m/^spice$/i) {
+ $res->{spice} = 1;
+ }
+ } elsif (defined($mapping)) {
+ my $devices = PVE::Mapping::USB::find_on_current_node($mapping);
+ die "USB device mapping not found for '$mapping'\n" if !$devices || !scalar($devices->@*);
+ die "More than one USB mapping per host not supported\n" if scalar($devices->@*) > 1;
+ eval {
+ PVE::Mapping::USB::assert_valid($mapping, $devices->[0]);
+ };
+ if (my $err = $@) {
+ die "USB Mapping invalid (hardware probably changed): $err\n";
+ }
+ my $device = $devices->[0];
+
+ if ($device->{path}) {
+ $res = parse_usb_device($device->{path});
+ } else {
+ $res = parse_usb_device($device->{id});
+ }
}
return $res;
}
+my sub assert_usb_index_is_useable {
+ my ($index, $use_qemu_xhci) = @_;
+
+ die "using usb$index is only possible with machine type >= 7.1 and ostype l26 or windows > 7\n"
+ if $index >= $OLD_MAX_USB && !$use_qemu_xhci;
+
+ return undef;
+}
+
sub get_usb_controllers {
- my ($conf, $bridges, $arch, $machine, $format, $max_usb_devices) = @_;
+ my ($conf, $bridges, $arch, $machine, $machine_version) = @_;
my $devices = [];
my $pciaddr = "";
+ my $ostype = $conf->{ostype};
+
+ my $use_qemu_xhci = min_version($machine_version, 7, 1)
+ && defined($ostype) && ($ostype eq 'l26' || windows_version($ostype) > 7);
+ my $is_q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
+
if ($arch eq 'aarch64') {
$pciaddr = print_pci_addr('ehci', $bridges, $arch, $machine);
push @$devices, '-device', "usb-ehci,id=ehci$pciaddr";
- } elsif ($machine !~ /q35/) { # FIXME: combine this and machine_type_is_q35
+ } elsif (!$is_q35) {
$pciaddr = print_pci_addr("piix3", $bridges, $arch, $machine);
push @$devices, '-device', "piix3-usb-uhci,id=uhci$pciaddr.0x2";
-
- my $use_usb2 = 0;
- for (my $i = 0; $i < $max_usb_devices; $i++) {
- next if !$conf->{"usb$i"};
- my $d = eval { PVE::JSONSchema::parse_property_string($format,$conf->{"usb$i"}) };
- next if !$d || $d->{usb3}; # do not add usb2 controller if we have only usb3 devices
- $use_usb2 = 1;
- }
- # include usb device config
- push @$devices, '-readconfig', '/usr/share/qemu-server/pve-usb.cfg' if $use_usb2;
}
- # add usb3 controller if needed
-
- my $use_usb3 = 0;
- for (my $i = 0; $i < $max_usb_devices; $i++) {
+ my ($use_usb2, $use_usb3) = 0;
+ my $any_usb = 0;
+ for (my $i = 0; $i < $MAX_USB_DEVICES; $i++) {
next if !$conf->{"usb$i"};
- my $d = eval { PVE::JSONSchema::parse_property_string($format,$conf->{"usb$i"}) };
- next if !$d || !$d->{usb3};
- $use_usb3 = 1;
+ assert_usb_index_is_useable($i, $use_qemu_xhci);
+ my $d = eval { PVE::JSONSchema::parse_property_string($usb_fmt, $conf->{"usb$i"}) } or next;
+ $any_usb = 1;
+ $use_usb3 = 1 if $d->{usb3};
+ $use_usb2 = 1 if !$d->{usb3};
+ }
+
+ if (!$use_qemu_xhci && !$is_q35 && $use_usb2 && $arch ne 'aarch64') {
+ # include usb device config if still on x86 before-xhci machines and if USB 3 is not used
+ push @$devices, '-readconfig', '/usr/share/qemu-server/pve-usb.cfg';
}
$pciaddr = print_pci_addr("xhci", $bridges, $arch, $machine);
- push @$devices, '-device', "nec-usb-xhci,id=xhci$pciaddr" if $use_usb3;
+ if ($use_qemu_xhci && $any_usb) {
+ push @$devices, '-device', print_qemu_xhci_controller($pciaddr);
+ } elsif ($use_usb3) {
+ push @$devices, '-device', "nec-usb-xhci,id=xhci$pciaddr";
+ }
return @$devices;
}
sub get_usb_devices {
- my ($conf, $format, $max_usb_devices, $features, $bootorder) = @_;
+ my ($conf, $features, $bootorder, $machine_version) = @_;
my $devices = [];
- for (my $i = 0; $i < $max_usb_devices; $i++) {
+ my $ostype = $conf->{ostype};
+ my $use_qemu_xhci = min_version($machine_version, 7, 1)
+ && defined($ostype) && ($ostype eq 'l26' || windows_version($ostype) > 7);
+
+ for (my $i = 0; $i < $MAX_USB_DEVICES; $i++) {
my $devname = "usb$i";
next if !$conf->{$devname};
- my $d = eval { PVE::JSONSchema::parse_property_string($format,$conf->{$devname}) };
+ assert_usb_index_is_useable($i, $use_qemu_xhci);
+ my $d = eval { PVE::JSONSchema::parse_property_string($usb_fmt, $conf->{$devname}) };
next if !$d;
- if (defined($d->{host})) {
- my $hostdevice = parse_usb_device($d->{host});
- $hostdevice->{usb3} = $d->{usb3};
- if ($hostdevice->{spice}) {
- # usb redir support for spice
- my $bus = 'ehci';
- $bus = 'xhci' if $hostdevice->{usb3} && $features->{spice_usb3};
-
- push @$devices, '-chardev', "spicevmc,id=usbredirchardev$i,name=usbredir";
- push @$devices, '-device', "usb-redir,chardev=usbredirchardev$i,id=usbredirdev$i,bus=$bus.0";
-
- warn "warning: spice usb port set as bootdevice, ignoring\n" if $bootorder->{$devname};
- } else {
- push @$devices, '-device', print_usbdevice_full($conf, $devname, $hostdevice, $bootorder);
- }
+ my $port = $use_qemu_xhci ? $i + 1 : undef;
+
+ if ($d->{host} && $d->{host} =~ m/^spice$/) {
+ # usb redir support for spice
+ my $bus = 'ehci';
+ $bus = 'xhci' if ($d->{usb3} && $features->{spice_usb3}) || $use_qemu_xhci;
+
+ push @$devices, '-chardev', "spicevmc,id=usbredirchardev$i,name=usbredir";
+ push @$devices, '-device', print_spice_usbdevice($i, $bus, $port);
+
+ warn "warning: spice usb port set as bootdevice, ignoring\n" if $bootorder->{$devname};
+ } else {
+ push @$devices, '-device', print_usbdevice_full($conf, $devname, $d, $bootorder, $port);
}
}
return @$devices;
}
+sub print_qemu_xhci_controller {
+ my ($pciaddr) = @_;
+ return "qemu-xhci,p2=15,p3=15,id=xhci$pciaddr";
+}
+
+sub print_spice_usbdevice {
+ my ($index, $bus, $port) = @_;
+ my $device = "usb-redir,chardev=usbredirchardev$index,id=usbredirdev$index,bus=$bus.0";
+ if (defined($port)) {
+ $device .= ",port=$port";
+ }
+ return $device;
+}
+
sub print_usbdevice_full {
- my ($conf, $deviceid, $device, $bootorder) = @_;
+ my ($conf, $deviceid, $device, $bootorder, $port) = @_;
return if !$device;
my $usbdevice = "usb-host";
- # if it is a usb3 device, attach it to the xhci controller, else omit the bus option
- if($device->{usb3}) {
+ # if it is a usb3 device or with newer qemu, attach it to the xhci controller, else omit the bus option
+ if ($device->{usb3} || defined($port)) {
$usbdevice .= ",bus=xhci.0";
+ $usbdevice .= ",port=$port" if defined($port);
}
- if (defined($device->{vendorid}) && defined($device->{productid})) {
- $usbdevice .= ",vendorid=0x$device->{vendorid},productid=0x$device->{productid}";
- } elsif (defined($device->{hostbus}) && defined($device->{hostport})) {
- $usbdevice .= ",hostbus=$device->{hostbus},hostport=$device->{hostport}";
+ my $parsed = parse_usb_device($device->{host}, $device->{mapping});
+
+ if (defined($parsed->{vendorid}) && defined($parsed->{productid})) {
+ $usbdevice .= ",vendorid=0x$parsed->{vendorid},productid=0x$parsed->{productid}";
+ } elsif (defined($parsed->{hostbus}) && defined($parsed->{hostport})) {
+ $usbdevice .= ",hostbus=$parsed->{hostbus},hostport=$parsed->{hostport}";
+ } else {
+ die "no usb id or path given\n";
}
$usbdevice .= ",id=$deviceid";
use IO::File;
use IPC::Open3;
use JSON;
+use POSIX qw(EINTR EAGAIN);
use PVE::Cluster qw(cfs_read_file);
use PVE::INotify;
use PVE::IPCC;
use PVE::JSONSchema;
+use PVE::PBSClient;
+use PVE::RESTEnvironment qw(log_warn);
use PVE::QMPClient;
use PVE::Storage::Plugin;
use PVE::Storage::PBSPlugin;
use PVE::Storage;
use PVE::Tools;
use PVE::VZDump;
+use PVE::Format qw(render_duration render_bytes);
use PVE::QemuConfig;
use PVE::QemuServer;
+use PVE::QemuServer::Helpers;
use PVE::QemuServer::Machine;
use PVE::QemuServer::Monitor qw(mon_cmd);
if defined($conf->{name});
$self->{vm_was_running} = 1;
+ $self->{vm_was_paused} = 0;
if (!PVE::QemuServer::check_running($vmid)) {
$self->{vm_was_running} = 0;
+ } elsif (PVE::QemuServer::vm_is_paused($vmid, 0)) {
+ # Do not treat a suspended VM as paused, as it would cause us to skip
+ # fs-freeze even if the VM wakes up before we reach qga_fs_freeze.
+ $self->{vm_was_paused} = 1;
}
$task->{hostname} = $conf->{name};
if (!$volume->{included}) {
$self->loginfo("exclude disk '$name' '$volid' ($volume->{reason})");
next;
- } elsif ($self->{vm_was_running} && $volume_config->{iothread}) {
- if (!PVE::QemuServer::Machine::runs_at_least_qemu_version($vmid, 4, 0, 1)) {
- die "disk '$name' '$volid' (iothread=on) can't use backup feature with running QEMU " .
- "version < 4.0.1! Either set backup=no for this drive or upgrade QEMU and restart VM\n";
- }
+ } elsif ($self->{vm_was_running} && $volume_config->{iothread} &&
+ !PVE::QemuServer::Machine::runs_at_least_qemu_version($vmid, 4, 0, 1)) {
+ die "disk '$name' '$volid' (iothread=on) can't use backup feature with running QEMU " .
+ "version < 4.0.1! Either set backup=no for this drive or upgrade QEMU and restart VM\n";
} else {
my $log = "include disk '$name' '$volid'";
if (defined(my $size = $volume_config->{size})) {
}
next if !$path;
- my ($size, $format) = eval { PVE::Storage::volume_size_info($self->{storecfg}, $volid, 5) };
- die "no such volume '$volid'\n" if $@;
+ my ($size, $format);
+ if ($storeid) {
+ # The call in list context can be expensive for certain plugins like RBD, just get size
+ $size = eval { PVE::Storage::volume_size_info($self->{storecfg}, $volid, 5) };
+ die "cannot determine size of volume '$volid' - $@\n" if $@;
+
+ my $scfg = PVE::Storage::storage_config($self->{storecfg}, $storeid);
+ $format = PVE::QemuServer::qemu_img_format($scfg, $volname);
+ } else {
+ ($size, $format) = eval {
+ PVE::Storage::volume_size_info($self->{storecfg}, $volid, 5);
+ };
+ die "cannot determine size and format of volume '$volid' - $@\n" if $@;
+ }
my $diskinfo = {
path => $path,
volid => $volid,
storeid => $storeid,
+ size => $size,
format => $format,
virtdev => $ds,
qmdevice => "drive-$ds",
};
+ if ($ds eq 'tpmstate0') {
+ # TPM drive only exists for backup, which is reflected in the name
+ $diskinfo->{qmdevice} = 'drive-tpmstate0-backup';
+ $task->{tpmpath} = $path;
+ }
+
if (-b $path) {
$diskinfo->{type} = 'block';
} else {
sub suspend_vm {
my ($self, $task, $vmid) = @_;
+ return if $self->{vm_was_paused};
+
$self->cmd ("qm suspend $vmid --skiplock");
}
sub resume_vm {
my ($self, $task, $vmid) = @_;
+ return if $self->{vm_was_paused};
+
$self->cmd ("qm resume $vmid --skiplock");
}
my $firewall_src = "/etc/pve/firewall/$vmid.fw";
my $firewall_dest = "$task->{tmpdir}/qemu-server.fw";
- my $outfd = IO::File->new (">$outfile") ||
- die "unable to open '$outfile'";
- my $conffd = IO::File->new ($conffile, 'r') ||
- die "unable open '$conffile'";
+ my $outfd = IO::File->new(">$outfile") or die "unable to open '$outfile' - $!\n";
+ my $conffd = IO::File->new($conffile, 'r') or die "unable to open '$conffile' - $!\n";
my $found_snapshot;
my $found_pending;
+ my $found_cloudinit;
while (defined (my $line = <$conffd>)) {
next if $line =~ m/^\#vzdump\#/; # just to be sure
next if $line =~ m/^\#qmdump\#/; # just to be sure
if ($line =~ m/^\[(.*)\]\s*$/) {
if ($1 =~ m/PENDING/i) {
$found_pending = 1;
+ } elsif ($1 =~ m/special:cloudinit/) {
+ $found_cloudinit = 1;
} else {
$found_snapshot = 1;
}
}
- next if $found_snapshot || $found_pending; # skip all snapshots and pending changes config data
+ next if $found_snapshot || $found_pending || $found_cloudinit; # skip all snapshots,pending changes and cloudinit config data
if ($line =~ m/^unused\d+:\s*(\S+)\s*/) {
$self->loginfo("skip unused drive '$1' (not included into backup)");
}
}
-# number, [precision=1]
-my $num2str = sub {
- return sprintf( "%." . ( $_[1] || 1 ) . "f", $_[0] );
-};
-my sub bytes_to_human {
- my ($bytes, $precission) = @_;
-
- return $num2str->($bytes, $precission) . ' B' if $bytes < 1024;
- my $kb = $bytes/1024;
-
- return $num2str->($kb, $precission) . " KiB" if $kb < 1024;
- my $mb = $kb/1024;
-
- return $num2str->($mb, $precission) . " MiB" if $mb < 1024;
- my $gb = $mb/1024;
-
- return $num2str->($gb, $precission) . " GiB" if $gb < 1024;
- my $tb = $gb/1024;
-
- return $num2str->($tb, $precission) . " TiB";
-}
-my sub duration_to_human {
- my ($seconds) = @_;
-
- return sprintf('%2ds', $seconds) if $seconds < 60;
- my $minutes = $seconds / 60;
- $seconds = $seconds % 60;
-
- return sprintf('%2dm %2ds', $minutes, $seconds) if $minutes < 60;
- my $hours = $minutes / 60;
- $minutes = $minutes % 60;
-
- return sprintf('%2dh %2dm %2ds', $hours, $minutes, $seconds) if $hours < 24;
- my $days = $hours / 24;
- $hours = $hours % 24;
-
- return sprintf('%2dd %2dh %2dm', $days, $hours, $minutes);
-}
-
my $bitmap_action_to_human = sub {
my ($self, $info) = @_;
my $action = $info->{action};
if ($action eq "not-used") {
- return "disabled (no support)" if $self->{vm_was_running};
- return "disabled (VM not running)";
+ return "disabled (no support)";
} elsif ($action eq "not-used-removed") {
return "disabled (old bitmap cleared)";
} elsif ($action eq "new") {
if ($info->{dirty} == 0) {
return "OK (drive clean)";
} else {
- my $size = bytes_to_human($info->{size});
- my $dirty = bytes_to_human($info->{dirty});
+ my $size = render_bytes($info->{size}, 1);
+ my $dirty = render_bytes($info->{dirty}, 1);
return "OK ($dirty of $size dirty)";
}
} elsif ($action eq "invalid") {
my ($mb, $delta) = @_;
return "0 B/s" if $mb <= 0;
my $bw = int(($mb / $delta));
- return bytes_to_human($bw) . "/s";
+ return render_bytes($bw, 1) . "/s";
};
my $target = 0;
$last_reused += $info->{size} - $info->{dirty};
}
if ($target < $total) {
- my $total_h = bytes_to_human($total);
- my $target_h = bytes_to_human($target);
+ my $total_h = render_bytes($total, 1);
+ my $target_h = render_bytes($target, 1);
$self->loginfo("using fast incremental mode (dirty-bitmap), $target_h dirty of $total_h total");
}
}
- my $first_round = 1;
my $last_finishing = 0;
while(1) {
my $status = mon_cmd($vmid, 'query-backup');
my $timediff = ($ctime - $last_time) || 1; # fixme
my $mbps_read = $get_mbps->($rbytes, $timediff);
my $mbps_write = $get_mbps->($wbytes, $timediff);
- my $target_h = bytes_to_human($target);
- my $transferred_h = bytes_to_human($transferred);
-
- if (!$has_query_bitmap && $first_round && $target != $total) { # FIXME: remove with PVE 7.0
- my $total_h = bytes_to_human($total);
- $self->loginfo("using fast incremental mode (dirty-bitmap), $target_h dirty of $total_h total");
- }
+ my $target_h = render_bytes($target, 1);
+ my $transferred_h = render_bytes($transferred, 1);
my $statusline = sprintf("%3d%% ($transferred_h of $target_h) in %s"
- .", read: $mbps_read, write: $mbps_write", $percent, duration_to_human($duration));
+ .", read: $mbps_read, write: $mbps_write", $percent, render_duration($duration));
my $res = $status->{status} || 'unknown';
if ($res ne 'active') {
$last_finishing = $status->{finishing};
}
sleep(1);
- $first_round = 0 if $first_round;
}
my $duration = time() - $starttime;
if ($last_zero) {
my $zero_per = $last_target ? int(($last_zero * 100)/$last_target) : 0;
- my $zero_h = bytes_to_human($last_zero, 2);
+ my $zero_h = render_bytes($last_zero);
$self->loginfo("backup is sparse: $zero_h (${zero_per}%) total zero data");
}
if ($reused) {
- my $reused_h = bytes_to_human($reused, 2);
+ my $reused_h = render_bytes($reused);
my $reuse_per = int($reused * 100 / $last_total);
$self->loginfo("backup was done incrementally, reused $reused_h (${reuse_per}%)");
}
if ($transferred) {
- my $transferred_h = bytes_to_human($transferred, 2);
+ my $transferred_h = render_bytes($transferred);
if ($duration) {
my $mbps = $get_mbps->($transferred, $duration);
$self->loginfo("transferred $transferred_h in $duration seconds ($mbps)");
};
};
+my $attach_tpmstate_drive = sub {
+ my ($self, $task, $vmid) = @_;
+
+ return if !$task->{tpmpath};
+
+ # unconditionally try to remove the tpmstate-named drive - it only exists
+ # for backing up, and avoids errors if left over from some previous event
+ eval { PVE::QemuServer::qemu_drivedel($vmid, "tpmstate0-backup"); };
+
+ $self->loginfo('attaching TPM drive to QEMU for backup');
+
+ my $drive = "file=$task->{tpmpath},if=none,read-only=on,id=drive-tpmstate0-backup";
+ $drive =~ s/\\/\\\\/g;
+ my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
+ die "attaching TPM drive failed - $ret\n" if $ret !~ m/OK/s;
+};
+
+my $detach_tpmstate_drive = sub {
+ my ($task, $vmid) = @_;
+ return if !$task->{tpmpath} || !PVE::QemuServer::check_running($vmid);
+ eval { PVE::QemuServer::qemu_drivedel($vmid, "tpmstate0-backup"); };
+};
+
+my sub add_backup_performance_options {
+ my ($qmp_param, $perf, $qemu_support) = @_;
+
+ return if !$perf || scalar(keys $perf->%*) == 0;
+
+ if (!$qemu_support) {
+ my $settings_string = join(', ', sort keys $perf->%*);
+ log_warn("ignoring setting(s): $settings_string - issue checking if supported");
+ return;
+ }
+
+ if (defined($perf->{'max-workers'})) {
+ if ($qemu_support->{'backup-max-workers'}) {
+ $qmp_param->{'max-workers'} = int($perf->{'max-workers'});
+ } else {
+ log_warn("ignoring 'max-workers' setting - not supported by running QEMU");
+ }
+ }
+}
+
+sub get_and_check_pbs_encryption_config {
+ my ($self) = @_;
+
+ my $opts = $self->{vzdump}->{opts};
+ my $scfg = $opts->{scfg};
+
+ my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($scfg, $opts->{storage});
+ my $master_keyfile = PVE::Storage::PBSPlugin::pbs_master_pubkey_file_name($scfg, $opts->{storage});
+
+ if (-e $keyfile) {
+ if (-e $master_keyfile) {
+ $self->loginfo("enabling encryption with master key feature");
+ return ($keyfile, $master_keyfile);
+ } elsif ($scfg->{'master-pubkey'}) {
+ die "master public key configured but no key file found\n";
+ } else {
+ $self->loginfo("enabling encryption");
+ return ($keyfile, undef);
+ }
+ } else {
+ my $encryption_fp = $scfg->{'encryption-key'};
+ die "encryption configured ('$encryption_fp') but no encryption key file found!\n"
+ if $encryption_fp;
+ if (-e $master_keyfile) {
+ $self->log(
+ 'warn',
+ "backup target storage is configured with master-key, but no encryption key set!"
+ ." Ignoring master key settings and creating unencrypted backup."
+ );
+ }
+ return (undef, undef);
+ }
+ die "internal error - unhandled case for getting & checking PBS encryption ($keyfile, $master_keyfile)!";
+}
+
+my sub cleanup_fleecing_images {
+ my ($self, $disks) = @_;
+
+ for my $di ($disks->@*) {
+ if (my $volid = $di->{'fleece-volid'}) {
+ eval { PVE::Storage::vdisk_free($self->{storecfg}, $volid); };
+ $self->log('warn', "error removing fleecing image '$volid' - $@") if $@;
+ }
+ }
+}
+
+my sub allocate_fleecing_images {
+ my ($self, $disks, $vmid, $fleecing_storeid, $format) = @_;
+
+ die "internal error - no fleecing storage specified\n" if !$fleecing_storeid;
+
+ # TODO what about potential left-over images from a failed attempt? Just
+ # auto-remove? While unlikely, could conflict with manually created image from user...
+
+ eval {
+ my $n = 0; # counter for fleecing image names
+
+ for my $di ($disks->@*) {
+ next if $di->{virtdev} =~ m/^(?:tpmstate|efidisk)\d$/; # too small to be worth it
+ if ($di->{type} eq 'block' || $di->{type} eq 'file') {
+ my $scfg = PVE::Storage::storage_config($self->{storecfg}, $fleecing_storeid);
+ my $name = "vm-$vmid-fleece-$n";
+ $name .= ".$format" if $scfg->{path};
+
+ my $size = PVE::Tools::convert_size($di->{size}, 'b' => 'kb');
+
+ $di->{'fleece-volid'} = PVE::Storage::vdisk_alloc(
+ $self->{storecfg}, $fleecing_storeid, $vmid, $format, $name, $size);
+
+ $n++;
+ } else {
+ die "implement me (type '$di->{type}')";
+ }
+ }
+ };
+ if (my $err = $@) {
+ cleanup_fleecing_images($self, $disks);
+ die $err;
+ }
+}
+
+my sub detach_fleecing_images {
+ my ($disks, $vmid) = @_;
+
+ return if !PVE::QemuServer::Helpers::vm_running_locally($vmid);
+
+ for my $di ($disks->@*) {
+ if (my $volid = $di->{'fleece-volid'}) {
+ my $devid = "$di->{qmdevice}-fleecing";
+ $devid =~ s/^drive-//; # re-added by qemu_drivedel()
+ eval { PVE::QemuServer::qemu_drivedel($vmid, $devid) };
+ }
+ }
+}
+
+my sub attach_fleecing_images {
+ my ($self, $disks, $vmid, $format) = @_;
+
+ # unconditionally try to remove potential left-overs from a previous backup
+ detach_fleecing_images($disks, $vmid);
+
+ my $vollist = [ map { $_->{'fleece-volid'} } grep { $_->{'fleece-volid'} } $disks->@* ];
+ PVE::Storage::activate_volumes($self->{storecfg}, $vollist);
+
+ for my $di ($disks->@*) {
+ if (my $volid = $di->{'fleece-volid'}) {
+ $self->loginfo("$di->{qmdevice}: attaching fleecing image $volid to QEMU");
+
+ my $path = PVE::Storage::path($self->{storecfg}, $volid);
+ my $devid = "$di->{qmdevice}-fleecing";
+ my $drive = "file=$path,if=none,id=$devid,format=$format,discard=unmap";
+ # Specify size explicitly, to make it work if storage backend rounded up size for
+ # fleecing image when allocating.
+ $drive .= ",size=$di->{size}" if $format eq 'raw';
+ $drive =~ s/\\/\\\\/g;
+ my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
+ die "attaching fleecing image $volid failed - $ret\n" if $ret !~ m/OK/s;
+ }
+ }
+}
+
+my sub check_and_prepare_fleecing {
+ my ($self, $vmid, $fleecing_opts, $disks, $is_template, $qemu_support) = @_;
+
+ # Even if the VM was started specifically for fleecing, it's possible that the VM is resumed and
+ # then starts doing IO. For VMs that are not resumed the fleecing images will just stay empty,
+ # so there is no big cost.
+
+ my $use_fleecing = $fleecing_opts && $fleecing_opts->{enabled} && !$is_template;
+
+ if ($use_fleecing && !defined($qemu_support->{'backup-fleecing'})) {
+ $self->log(
+ 'warn',
+ "running QEMU version does not support backup fleecing - continuing without",
+ );
+ $use_fleecing = 0;
+ }
+
+ if ($use_fleecing) {
+ my ($default_format, $valid_formats) = PVE::Storage::storage_default_format(
+ $self->{storecfg}, $fleecing_opts->{storage});
+ my $format = scalar(grep { $_ eq 'qcow2' } $valid_formats->@*) ? 'qcow2' : 'raw';
+
+ allocate_fleecing_images($self, $disks, $vmid, $fleecing_opts->{storage}, $format);
+ attach_fleecing_images($self, $disks, $vmid, $format);
+ }
+
+ return $use_fleecing;
+}
+
sub archive_pbs {
my ($self, $task, $vmid) = @_;
my $starttime = time();
- my $server = $scfg->{server};
- my $datastore = $scfg->{datastore};
- my $username = $scfg->{username} // 'root@pam';
my $fingerprint = $scfg->{fingerprint};
-
- my $repo = "$username\@$server:$datastore";
+ my $repo = PVE::PBSClient::get_repository($scfg);
my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $opts->{storage});
- my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($scfg, $opts->{storage});
+ my ($keyfile, $master_keyfile) = $self->get_and_check_pbs_encryption_config();
my $diskcount = scalar(@{$task->{disks}});
- # proxmox-backup-client can only handle raw files and block devs
- # only use it (directly) for disk-less VMs
+ # proxmox-backup-client can only handle raw files and block devs, so only use it (directly) for
+ # disk-less VMs
if (!$diskcount) {
- my @pathlist;
$self->loginfo("backup contains no disks");
local $ENV{PBS_PASSWORD} = $password;
'--backup-id', "$vmid",
'--backup-time', $task->{backup_time},
];
+ if (defined(my $ns = $scfg->{namespace})) {
+ push @$cmd, '--ns', $ns;
+ }
+ if (defined($keyfile)) {
+ push @$cmd, '--keyfile', $keyfile;
+ push @$cmd, '--master-pubkey-file', $master_keyfile if defined($master_keyfile);
+ }
push @$cmd, "qemu-server.conf:$conffile";
push @$cmd, "fw.conf:$firewall" if -e $firewall;
# get list early so we die on unkown drive types before doing anything
my $devlist = _get_task_devlist($task);
+ my $use_fleecing;
$self->enforce_vm_running_for_backup($vmid);
+ $self->{qmeventd_fh} = PVE::QemuServer::register_qmeventd_handle($vmid);
my $backup_job_uuid;
eval {
};
my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
- if (!$qemu_support) {
- die "PBS backups are not supported by the running QEMU version. Please make "
- . "sure you've installed the latest version and the VM has been restarted.\n";
+ my $err = $@;
+ if (!$qemu_support || $err) {
+ die "query-proxmox-support returned empty value\n" if !$err;
+ if ($err =~ m/The command query-proxmox-support has not been found/) {
+ die "PBS backups are not supported by the running QEMU version. Please make "
+ . "sure you've installed the latest version and the VM has been restarted.\n";
+ } else {
+ die "QMP command query-proxmox-support failed - $err\n";
+ }
}
+ # pve-qemu supports it since 5.2.0-1 (PVE 6.4), so safe to die since PVE 8
+ die "master key configured but running QEMU version does not support master keys\n"
+ if !defined($qemu_support->{'pbs-masterkey'}) && defined($master_keyfile);
+
+ $attach_tpmstate_drive->($self, $task, $vmid);
+
+ my $is_template = PVE::QemuConfig->is_template($self->{vmlist}->{$vmid});
+
+ $use_fleecing = check_and_prepare_fleecing(
+ $self, $vmid, $opts->{fleecing}, $task->{disks}, $is_template, $qemu_support);
+
my $fs_frozen = $self->qga_fs_freeze($task, $vmid);
my $params = {
devlist => $devlist,
'config-file' => $conffile,
};
+ $params->{fleecing} = JSON::true if $use_fleecing;
+
+ if (defined(my $ns = $scfg->{namespace})) {
+ $params->{'backup-ns'} = $ns;
+ }
+
$params->{speed} = $opts->{bwlimit}*1024 if $opts->{bwlimit};
+ add_backup_performance_options($params, $opts->{performance}, $qemu_support);
+
$params->{fingerprint} = $fingerprint if defined($fingerprint);
$params->{'firewall-file'} = $firewall if -e $firewall;
- if (-e $keyfile) {
- $self->loginfo("enabling encryption");
+
+ $params->{encrypt} = defined($keyfile) ? JSON::true : JSON::false;
+ if (defined($keyfile)) {
$params->{keyfile} = $keyfile;
- $params->{encrypt} = JSON::true;
- } else {
- $params->{encrypt} = JSON::false;
+ $params->{"master-keyfile"} = $master_keyfile if defined($master_keyfile);
}
- my $is_template = PVE::QemuConfig->is_template($self->{vmlist}->{$vmid});
$params->{'use-dirty-bitmap'} = JSON::true
- if $qemu_support->{'pbs-dirty-bitmap'} && $self->{vm_was_running} && !$is_template;
+ if $qemu_support->{'pbs-dirty-bitmap'} && !$is_template;
- $params->{timeout} = 60; # give some time to connect to the backup server
+ $params->{timeout} = 125; # give some time to connect to the backup server
my $res = eval { mon_cmd($vmid, "backup", %$params) };
my $qmperr = $@;
if ($err) {
$self->logerr($err);
$self->mon_backup_cancel($vmid);
+ $self->resume_vm_after_job_start($task, $vmid);
}
$self->restore_vm_power_state($vmid);
+ if ($use_fleecing) {
+ detach_fleecing_images($task->{disks}, $vmid);
+ cleanup_fleecing_images($self, $task->{disks});
+ }
+
die $err if $err;
}
$speed = $opts->{bwlimit}*1024;
}
+ my $is_template = PVE::QemuConfig->is_template($self->{vmlist}->{$vmid});
+
my $diskcount = scalar(@{$task->{disks}});
- if (PVE::QemuConfig->is_template($self->{vmlist}->{$vmid}) || !$diskcount) {
+ if ($is_template || !$diskcount) {
my @pathlist;
foreach my $di (@{$task->{disks}}) {
if ($di->{type} eq 'block' || $di->{type} eq 'file') {
}
my $devlist = _get_task_devlist($task);
+ my $use_fleecing;
$self->enforce_vm_running_for_backup($vmid);
+ $self->{qmeventd_fh} = PVE::QemuServer::register_qmeventd_handle($vmid);
my $cpid;
my $backup_job_uuid;
die "interrupted by signal\n";
};
+ # Currently, failing to determine Proxmox support is not critical here, because it's only
+ # used for performance settings like 'max-workers'.
+ my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
+ log_warn($@) if $@;
+
+ $attach_tpmstate_drive->($self, $task, $vmid);
+
+ $use_fleecing = check_and_prepare_fleecing(
+ $self, $vmid, $opts->{fleecing}, $task->{disks}, $is_template, $qemu_support);
+
my $outfh;
if ($opts->{stdout}) {
$outfh = $opts->{stdout};
devlist => $devlist
};
$params->{'firewall-file'} = $firewall if -e $firewall;
+ $params->{fleecing} = JSON::true if $use_fleecing;
+ add_backup_performance_options($params, $opts->{performance}, $qemu_support);
$qmpclient->queue_cmd($vmid, $backup_cb, 'backup', %$params);
};
if ($err) {
$self->logerr($err);
$self->mon_backup_cancel($vmid);
+ $self->resume_vm_after_job_start($task, $vmid);
}
$self->restore_vm_power_state($vmid);
+ if ($use_fleecing) {
+ detach_fleecing_images($task->{disks}, $vmid);
+ cleanup_fleecing_images($self, $task->{disks});
+ }
+
if ($err) {
if ($cpid) {
kill(9, $cpid);
sub qga_fs_freeze {
my ($self, $task, $vmid) = @_;
- return if !$self->{vmlist}->{$vmid}->{agent} || $task->{mode} eq 'stop' || !$self->{vm_was_running};
+ return if !$self->{vmlist}->{$vmid}->{agent} || $task->{mode} eq 'stop' || !$self->{vm_was_running} || $self->{vm_was_paused};
if (!PVE::QemuServer::qga_check_running($vmid, 1)) {
$self->loginfo("skipping guest-agent 'fs-freeze', agent configured but not running?");
return;
}
+ my $freeze = PVE::QemuServer::get_qga_key($self->{vmlist}->{$vmid}, 'freeze-fs-on-backup') // 1;
+ if (!$freeze) {
+ $self->loginfo("skipping guest-agent 'fs-freeze', disabled in VM options");
+ return;
+ }
+
$self->loginfo("issuing guest-agent 'fs-freeze' command");
eval { mon_cmd($vmid, "guest-fsfreeze-freeze") };
$self->logerr($@) if $@;
die $@ if $@;
}
-# resume VM againe once we got in a clear state (stop mode backup of running VM)
+# resume VM again once in a clear state (stop mode backup of running VM)
sub resume_vm_after_job_start {
my ($self, $task, $vmid) = @_;
- return if !$self->{vm_was_running};
+ return if !$self->{vm_was_running} || $self->{vm_was_paused};
if (my $stoptime = $task->{vmstoptime}) {
my $delay = time() - $task->{vmstoptime};
} else {
$self->loginfo("resuming VM again");
}
- mon_cmd($vmid, 'cont');
+ mon_cmd($vmid, 'cont', timeout => 45);
}
# stop again if VM was not running before
sub cleanup {
my ($self, $task, $vmid) = @_;
- # nothing to do ?
+ $detach_tpmstate_drive->($task, $vmid);
+
+ if ($self->{qmeventd_fh}) {
+ close($self->{qmeventd_fh});
+ }
}
1;
+qemu-server (8.2.1) bookworm; urgency=medium
+
+ * cpu config: fix get_cpu_bitness always reverting to default cpu type
+
+ -- Proxmox Support Team <support@proxmox.com> Wed, 24 Apr 2024 11:49:03 +0200
+
+qemu-server (8.2.0) bookworm; urgency=medium
+
+ * qmeventd: also treat 'prelaunch' and 'suspended' states as active to avoid
+ issues when backing up VMs that currently are in those states.
+
+ * OS type: add Windows Server 2025 as supported, map it to the same virtual
+ hardware profile as the Windows 11 one.
+
+ -- Proxmox Support Team <support@proxmox.com> Tue, 23 Apr 2024 17:09:20 +0200
+
+qemu-server (8.1.4) bookworm; urgency=medium
+
+ * api: create vm: add missing import for serializing machine type to fix a
+ regression of version 8.1.2.
+
+ -- Proxmox Support Team <support@proxmox.com> Sat, 20 Apr 2024 12:28:35 +0200
+
+qemu-server (8.1.3) bookworm; urgency=medium
+
+ * firewall: add handling for new nftables based firewall implementation,
+ which currently is a opt-in drop-in replacement for the older iptables-
+ based one.
+
+ -- Proxmox Support Team <support@proxmox.com> Fri, 19 Apr 2024 20:23:39 +0200
+
+qemu-server (8.1.2) bookworm; urgency=medium
+
+ * fix #4136: backup: implement fleecing option for improved guest
+ stability when the backup target is slow
+
+ * fix #4474: stop VM: add 'overrule-shutdown' parameter to prevent
+ currently running shutdown tasks from blocking the stop task
+
+ * fix #1905: disk move: allow moving unused disks
+
+ * fix #3784: add vIOMMU parameter to support passthrough of PCI devices to
+ nested virtual machines
+
+ * fix #5363: cloudinit: fix regression to make creation of scsi cloudinit
+ disks possible again
+
+ -- Proxmox Support Team <support@proxmox.com> Fri, 19 Apr 2024 16:09:18 +0200
+
+qemu-server (8.1.1) bookworm; urgency=medium
+
+ * config: pending network: avoid undef-warning on old/new comparison
+
+ * support live-import for 'import-from' disk options on create
+
+ * qm: add 'import' command for importing a VM through a volumeid from a
+ storage that provides the new 'import' content-type.
+
+ * disk import: warn when fallback is used instead of requested format
+
+ * cpu config: die on hotplug of non x86_64 CPUs
+
+ -- Proxmox Support Team <support@proxmox.com> Thu, 14 Mar 2024 14:04:34 +0100
+
+qemu-server (8.1.0) bookworm; urgency=medium
+
+ * migration: do not allow live-migration with VNC clipboard, it's not yet
+ supported by QEMU's vdagent device, which gets used for this feature.
+
+ * cpu config: add QEMU 8.1 cpu models
+
+ * fix #4501: TCP migration: start vm: move port reservation and usage closer
+ together
+
+ * fix #2258: select correct device when removing drive snapshot via QEMU, as
+ the device where the disk is currently attached needs to be targeted, not
+ the one where the disk was attached at the time the snapshot was taken
+
+ * fix #4957: allow one to set the vendor and product information for
+ SCSI-Disks explicitly
+
+ * migration: remember original volume names from the source so that they can
+ get deactivated even if the volume name has to change due to being already
+ in use on the target storage.
+
+ * fix #4085: properly activate the storage(s) of custom cloud-init snippets
+
+ * fix #1734: clone VM: if deactivation of source volume fails demote error
+ to warning. Deactivation can fail if the source is still, or again, in
+ use, e.g., due to parallel clones of the same template.
+
+ * mediated device pass-through: avoid race condition for cleaning up on VM
+ reboot
+
+ * prevent starting a 32-bit VM using a 64-bit OVMF BIOS
+
+ * QMP client: increase default timeout for drive-mirror to 10 minutes like
+ for other block operations.
+
+ -- Proxmox Support Team <support@proxmox.com> Fri, 08 Mar 2024 15:00:25 +0100
+
+qemu-server (8.0.10) bookworm; urgency=medium
+
+ * sdn: pass vmid and hostname to allow requesting a new mapping
+
+ -- Proxmox Support Team <support@proxmox.com> Wed, 22 Nov 2023 14:12:46 +0100
+
+qemu-server (8.0.9) bookworm; urgency=medium
+
+ * add clipboard option to to vga config entry
+
+ * api: add clipboard variable to return at status/current
+
+ * recommend libpve-network-perl for SDN support
+
+ * initial support for dhcp ip allocation in dhcp-enabled SDN zones
+
+ -- Proxmox Support Team <support@proxmox.com> Tue, 21 Nov 2023 15:40:27 +0100
+
+qemu-server (8.0.8) bookworm; urgency=medium
+
+ * fix #2816: restore: remove timeout when allocating disks
+
+ * start: increase maximal timeout if using PCI passthrough
+
+ * fix #4522: api: vncproxy: also set environment variable for the ticket
+ if the websocket option is not set
+
+ * backup, migrate: fix races with suspended VMs that can wake up
+
+ * cpu hotplug: cannot change feature online, so keep these as pending change
+
+ * nbd-stop: increase timeout to 25s
+
+ * start: add warning if a deprecated machine version is configured
+
+ -- Proxmox Support Team <support@proxmox.com> Sun, 12 Nov 2023 18:54:37 +0100
+
+qemu-server (8.0.7) bookworm; urgency=medium
+
+ * fix #4822: vzdump: fix PBS encryption for guests without disks
+
+ * qmeventd: fix parsing of VMID in presence of legacy cgroup entries
+
+ * api: check access for already configured bridge when updating vNIC
+
+ * fix #4620: make 'ide1' and 'ide3' drive keys work for machine type q35
+
+ * cloudinit: fix two checks that were mistakenly restricted to root only,
+ one for setting the ciupgrade option and one for updating the cloudinit
+ drive
+
+ * migration: improve format hint when allocating live-migrated disk on the
+ target to make e.g. remote-migration with qcow2 and LVM-thin target work
+
+ * net: fix setting value for tx_queue_size
+
+ * fix #3963: allow backup of template VM with immutable TPM drive
+
+ -- Proxmox Support Team <support@proxmox.com> Mon, 21 Aug 2023 11:30:45 +0200
+
+qemu-server (8.0.6) bookworm; urgency=medium
+
+ * cloudinit: restore previous default for package upgrades
+
+ * migration: only migrate disks used by the guest, not also those that are
+ owned by them (VMID in name) but not referenced in the config
+
+ * migration: fail when aliased volume are detected, as referencing the same
+ volume multiple times can lead to unexpected behavior in a migration.
+
+ * migration: fix issue with qcow2 cloudinit disk live migration
+
+ -- Proxmox Support Team <support@proxmox.com> Wed, 21 Jun 2023 13:03:01 +0200
+
+qemu-server (8.0.5) bookworm; urgency=medium
+
+ * restore: extend permissions checks
+
+ * vm start: always reset any failed-state of the VM systemd scope to avoid
+ failing a re-start after, e.g., a OOM kill.
+
+ -- Proxmox Support Team <support@proxmox.com> Wed, 21 Jun 2023 09:17:41 +0200
+
+qemu-server (8.0.4) bookworm; urgency=medium
+
+ * vCPU config: add new x86-64-v2, x86-64-v3 and x86-64-v4 models
+
+ * fix #4784: helpers: cope with native versions in manager version check
+
+ * enable cluster mapped USB devices for guests
+
+ * enable cluster mapped PCI devices for guests
+
+ -- Proxmox Support Team <support@proxmox.com> Mon, 19 Jun 2023 07:24:11 +0200
+
+qemu-server (8.0.3) bookworm; urgency=medium
+
+ * qemu: fix permission check call
+
+ -- Proxmox Support Team <support@proxmox.com> Fri, 09 Jun 2023 12:20:40 +0200
+
+qemu-server (8.0.2) bookworm; urgency=medium
+
+ * cfg2cmd: use actual backend names instead of removed tty and paraport
+ aliases
+
+ * cfg2cmd: replace deprecated no-acpi option with acpi=off machine flag
+
+ * cfg2cmd: replace deprecated no-hpet option with hpet=off machine flag
+
+ * schema: avoid using deprecated -no-hpet in example for 'args' property,
+ instead pass thate via new machine option
+
+ * allow setting ipconfigX with VM.Config.Cloudinit
+
+ * fix #3428: cloudinit: add parameter for upgrade on boot
+
+ * cloudinit: fix 'pending' api endpoint
+
+ * fast plug options: add migrate_downtime and migrate_speed for convenience
+
+ * fix #517: api: allow resizing qcow2 disk with snapshots
+
+ * fix #2315: api: have resize endpoint spawn a worker task
+
+ * cloudinit: pass through hostname via fqdn field
+
+ * qmeventd: extract vmid from cgroup file instead of cmdline
+
+ * config: implement method to calculate derived properties from a config
+
+ * api: check bridge access for create, update, clone & restore
+
+ * qm: remote migration: improve error when storage cannot be found
+
+ -- Proxmox Support Team <support@proxmox.com> Fri, 09 Jun 2023 10:26:19 +0200
+
+qemu-server (8.0.1) bookworm; urgency=medium
+
+ * fix #4737: qmeventd: gracefully handle interrupted epoll_wait call
+
+ * handle and warn about VM network interfaces not attached to any bridges
+
+ * block resize: avoid passing zero size to QMP command
+
+ * qmrestore: improve description of bwlimit parameter
+
+ * api: switch agent api call to 'array' type
+
+ * tests: fix invoking migration tests with make
+
+ -- Proxmox Support Team <support@proxmox.com> Wed, 07 Jun 2023 13:50:09 +0200
+
+qemu-server (8.0.0) bookworm; urgency=medium
+
+ * never enable 'smm' flag for the 'virt' machine type (doesn't exist)
+
+ * test: mock calls that can fail in a chroot environment
+
+ * rebuild for Debian Bookworm based releases
+
+ -- Proxmox Support Team <support@proxmox.com> Fri, 19 May 2023 15:07:45 +0200
+
+qemu-server (7.4-3) bullseye; urgency=medium
+
+ * backup prepare: fix format detection for disks without storage ID
+
+ * backup prepare: improve error messages
+
+ -- Proxmox Support Team <support@proxmox.com> Mon, 27 Mar 2023 11:17:16 +0200
+
+qemu-server (7.4-2) bullseye; urgency=medium
+
+ * avoid list context for volume_size_info calls as otherwise we
+ unnecessarily take a slower code path
+
+ -- Proxmox Support Team <support@proxmox.com> Tue, 21 Mar 2023 16:51:01 +0100
+
+qemu-server (7.4-1) bullseye; urgency=medium
+
+ * fix #4553: nvidia vgpu: reuse smbios uuid for '-uuid' parameter
+
+ * pci: workaround nvidia driver issue on mdev cleanup
+
+ * memory: hotplug: sort by numerical ID rather than slot when unplugging
+
+ * memory: use the DIMM list info from QEMU for unplug
+
+ -- Proxmox Support Team <support@proxmox.com> Mon, 20 Mar 2023 17:24:45 +0100
+
+qemu-server (7.3-4) bullseye; urgency=medium
+
+ * fix #4378: standardized error for missing OVMF files
+
+ * schema: memory: be precise that unit is binary prefix
+
+ * close #2792: allow online migration with replicated snapshots
+
+ * schema: OS type: note that the l26 type is also compatible with Linux 6.x
+
+ * hotplug: disk: mark the 'aio' (async IO) as non-hotpluggable to avoid
+ suggesting that it already changed
+
+ * fix #4525: clone disk: disallow mirror if it might cause problems with
+ io_uring using the same heuristics as for start-up
+
+ * start: make not being able to set polling interval for ballooning
+ non-critical
+
+ * swtpm: enable logging to `/run/qemu-server/$vmid-swtpm.log`
+
+ * fix #4140: vzdump: transform the previous hardcoded behavior of issuing a
+ fs-freeze and fs-thaw if QGA is enabled into an overrideable option named
+ 'fs-freeze-on-backup'
+
+ * update network dev: MTU is not hot-pluggable, avoid suggesting so
+
+ * fix #4249: make image clone or conversion respect bandwidth limit
+
+ -- Proxmox Support Team <support@proxmox.com> Thu, 23 Feb 2023 17:12:42 +0100
+
+qemu-server (7.3-3) bullseye; urgency=medium
+
+ * rollback: ignore auto-start request if VM is already running
+
+ * memory hot-plug: check correct value for maximal memory check
+
+ * fix #4435: device list: avoid error for undefined value
+
+ * fix #4358: ignore any suspended lock when destroying a VM
+
+ * migration: log error from query-migrate, if any, upon migration failure
+
+ * cd rom handling: return a clearer error when there is no CD-ROM drive
+
+ * migration: nbd export: switch away from deprecated QMP command
+
+ -- Proxmox Support Team <support@proxmox.com> Mon, 16 Jan 2023 13:52:30 +0100
+
+qemu-server (7.3-2) bullseye; urgency=medium
+
+ * fix #4372: improve edge-case for config-loading on VM resume when
+ migrating
+
+ * ovmf cmd assembly: re-work and re-order arguments assembly
+
+ -- Proxmox Support Team <support@proxmox.com> Fri, 16 Dec 2022 12:54:53 +0100
+
+qemu-server (7.3-1) bullseye; urgency=medium
+
+ * vm resume: improve loading just recently moved config on nocheck/migrate
+ handling
+
+ -- Proxmox Support Team <support@proxmox.com> Mon, 21 Nov 2022 13:43:59 +0100
+
+qemu-server (7.2-12) bullseye; urgency=medium
+
+ * config: only save unique tags when updating them via the API
+
+ * api: create/update vm: fix clamping CPU units function calls
+
+ -- Proxmox Support Team <support@proxmox.com> Mon, 21 Nov 2022 08:36:06 +0100
+
+qemu-server (7.2-11) bullseye; urgency=medium
+
+ * fdb: only manage FDB entries for Linux bridges, ignore OVS for now
+
+ -- Proxmox Support Team <support@proxmox.com> Sun, 20 Nov 2022 16:30:28 +0100
+
+qemu-server (7.2-10) bullseye; urgency=medium
+
+ * fix #4321: properly check cloud-init drive permissions, require both
+ VM.Config.CDROM and VM.Config.Cloudinit, and not VM.Config.Disk, for being
+ able to add a cloud init drive in the first place.
+
+ * api: config update: enforce new tag permission system when setting or
+ removing tags from a guest
+
+
+ * parse config: do not validate informative values in cloud init section
+
+ * fix edge-cases on new cloudinit pending/active recording
+
+ * mtunnel: add API endpoints
+
+ * migrate: add foundation for remote (external cluster) migration, add
+ respective endpoints and qm `remote-migrate` CLI command
+
+ * memory hotplug: make max-memory dynamically calculated from the physicall
+ address bits the VM will use, that is the actual one from the config, if
+ set, the one from the host for CPU type host and 40 bits as fallback for
+ everything else. Calculate the addressable memory (e.g., 40 bits = 1 TiB)
+ and half that for the possible max-memory a VM can use, using the previous
+ hard-coded 4 TiB as overall maximum for backward compat.
+ Admins with inhomogeneous CPUs and thus possible different bit-widths need
+ to take special care themselves to ensure that a VM with memory hot-plug
+ configured can run on other nodes, for example for live-migration.
+
+ -- Proxmox Support Team <support@proxmox.com> Thu, 17 Nov 2022 17:48:03 +0100
+
+qemu-server (7.2-8) bullseye; urgency=medium
+
+ * fix #4296: virtio-net: enable packed queues for qemu 7.1
+
+ * virtio-net: increase defaults rx- and tx-queue-size to 1024
+
+ * fix #4296: virtio-net: enable packed queues for QEMU machines using 7.1 or
+ newer
+
+ * net: increase max queues to 64
+
+ * fix #4284: add read-only to non-hotpluggable disk options
+
+ * delay cloudinit generation in hotplug
+
+ * record cloud-init changes in the cloudinit section
+
+ * rework cloudint config pending handling
+
+ -- Proxmox Support Team <support@proxmox.com> Wed, 16 Nov 2022 18:23:39 +0100
+
+qemu-server (7.2-7) bullseye; urgency=medium
+
+ * api: create/update vm: automatically clamp cpuunit value depending of
+ cgroup version
+
+ * improve cloud init support and add cloudinit hotplug
+
+ * vzdump: skip `special:cloudinit` section
+
+ * fix #3890 - GUI: warn for unlikely iothread config clashes
+
+ * fix #4228: add `start` parameter to snapshot rollback API so that one can
+ automatocally start the VM after rollback finished.
+
+ * vm start/stop: cleanup passed-through pci devices in more situations
+
+ * fix #3593: allow one to configure task set affinity for VMs
+
+ * fix #4324: USB: use qemu-xhci for machine versions >= 7.1
+
+ * usb: increase max USB devices from 5 to 14 for modern 7.1 machine
+ and OS versions (Linux 2.6+ annd Windows 8+)
+
+ * fix #4201: delete cloud-init disk on rollback
+
+ * net devs: register vNIC MAC-Address manually to FDB on start/resume if
+ bridge has learning disabled
+
+ -- Proxmox Support Team <support@proxmox.com> Sun, 13 Nov 2022 15:46:18 +0100
+
+qemu-server (7.2-6) bullseye; urgency=medium
+
+ * schema: move 'pve-targetstorage' to pve-common
+
+ -- Proxmox Support Team <support@proxmox.com> Mon, 07 Nov 2022 16:22:50 +0100
+
+qemu-server (7.2-5) bullseye; urgency=medium
+
+ * qmp client: increase guest fstrim timeout to 10 minutes
+
+ * fix #3577: prevent suspension for VMs with pci passthrough
+
+ * cpu config: map deprecated IceLake-Client CPU type to IceLake-Server
+
+ * snapshot: save VM state: propagate error from QEMU
+
+ * api: create disks: avoid adding secondary cloud-init drives
+
+ * vzdump: TPM state: escape drive string
+
+ * qmp client: increase default fallback timeout to 5s
+
+ * fix regex matching network devices in qm cleanup so that vNICs with double
+ digit IDs are covered too
+
+ * qmeventd: rework 'forced_cleanup' handling and set timeout to 60s
+
+ * qmeventd: send QMP 'quit' command instead of SIGTERM
+
+ * vzdump: set max-workers QMP option when specified and supported
+
+ * fix #4099: disable io_uring for virtual disks on CIFS storages for now
+
+ * qm: move VM-disk related commands to own command group, keep old ones
+ around for backward compatibility
+
+ -- Proxmox Support Team <support@proxmox.com> Mon, 07 Nov 2022 16:15:16 +0100
+
+qemu-server (7.2-4) bullseye; urgency=medium
+
+ * fix #3754: encode JSON as utf8 for CLI
+
+ * cpuconfig: add amd epyc milan model
+
+ * fix #4115: enable option to name QEMU threads after their main purpose
+
+ * fix #4119: give namespace parameter to live-restore
+
+ * automatically add 'uuid' parameter when passing through NVIDIA vGPU
+
+ * vzdump/pbs: die with missing, but configured encryption key
+
+ * vzdump/pbs: die with missing, but configured master key
+
+ -- Proxmox Support Team <support@proxmox.com> Tue, 16 Aug 2022 13:59:20 +0200
+
+qemu-server (7.2-3) bullseye; urgency=medium
+
+ * support pbs namespaces
+
+ -- Proxmox Support Team <support@proxmox.com> Thu, 12 May 2022 15:14:39 +0200
+
+qemu-server (7.2-2) bullseye; urgency=medium
+
+ * api: reassign disk: drop moved disk from boot order
+
+ * explicitly check some prerequisites for virtio-gl display
+
+ -- Proxmox Support Team <support@proxmox.com> Mon, 02 May 2022 17:26:16 +0200
+
+qemu-server (7.2-1) bullseye; urgency=medium
+
+ * migrate: add log for guest fstrim and make a failure noticable
+
+ * migrate: resume initially running VM when failing after convergence
+
+ * parse vm config: remove "\s*" from multi-line comment regex
+
+ * memory: enable balloon free-page-reporting for auto-memory reclaim
+
+ * enable spice also for virtio-gl and virtio-gpu displays and report so in
+ status API
+
+ * api: create: allow overriding non-disk options during restore
+
+ * fix #3861: migrate: fix live migration when cloud-init changes storage
+
+ -- Proxmox Support Team <support@proxmox.com> Thu, 28 Apr 2022 18:35:22 +0200
+
+qemu-server (7.1-5) bullseye; urgency=medium
+
+ * avoid writing the config if there are no pending changes to apply
+
+ * fix #3792: cloudinit: use of uninitialized value
+
+ * pci: allow override of PCI vendor/device ids
+
+ * drive mirror monitor: warn when suspend/resume/freeze/thaw calls fail
+
+ * update config: allow setting boot-order and dev in one go
+
+ * migrate: move tunnel-helpers to pve-guest-common
+
+ * fix #3683: agent file-write: enable user to encode the content themselves
+
+ * cpu units: lower minimum for accessing full cgroupv2 range
+
+ * fix #3845: also clamp cpu units to cgroup dependent valid range on hotplug
+
+ * clone disk: force raw format for TPM state
+
+ * fix #3886: QEMU restore: verify storage allows images before writing
+
+ * fix #3733: bump the timeout used to wait that a for backup started VM is
+ fully stopped (i.e., it's "$vmid.scope vanished) to 20 seconds after the
+ backup has finished to
+
+ * qmp client: increase timeout for thaw to better accommodate the QGA running
+ in Windows VMs
+
+ * api: vm start: 'force-cpu' is for internal migration use only, mark as
+ such
+
+ * device unplug: verify that unplugging SCSI disk completed before
+ continuing with remaining unplug work.
+
+ * clone disk: remove ancient check for min QEMU version 2.7
+
+ * clone disk: pass in efi vars size rather than config
+
+ * clone disk: allow cloning from an unused or unreferenced disk
+
+ * parse ovf: untaint path when getting the file's size info
+
+ * image convert: allow block device as source
+
+ * fix #3424: api: snapshot delete: wait for active replication
+
+ * PCI: allow longer pci domains
+
+ * fix #3957: spell 'occurred' correctly
+
+ * clone disk: also clone EFI disk from snapshot
+
+ * api: add endpoint for parsing .ovf files
+
+ * api: support VM disk import
+
+ * migrate: keep VM paused after migration if it was before
+
+ * vga: add virtio-gl display type for VIRGL
+
+ * restore: cleanup oldconf: also clean up snapshots from kept volumes
+
+ * restore: also deactivate/destroy cloud-init disk upon error
+
+ -- Proxmox Support Team <support@proxmox.com> Mon, 25 Apr 2022 20:15:59 +0200
+
+qemu-server (7.1-4) bullseye; urgency=medium
+
+ * migrate: send updated TPM state volume ID to target node on local-storage
+ migration
+
+ -- Proxmox Support Team <support@proxmox.com> Mon, 22 Nov 2021 17:07:13 +0100
+
+qemu-server (7.1-3) bullseye; urgency=medium
+
+ * replication: do not setup NBD server on VM migrate for the TPM state,
+ QEMU cannot access it directly and we already migrate it via the non-QEMU
+ storage migration anyway.
+
+ -- Proxmox Support Team <support@proxmox.com> Tue, 16 Nov 2021 14:04:45 +0100
+
+qemu-server (7.1-2) bullseye; urgency=medium
+
+ * cfg2cmd: disable SMM when display=none and SeaBIOS is both used
+
+ * pci: do not reserve pci-ids for mediated devices, already handled by sysfs
+ anyway
+
+ * exclude efidisk and tpmstate for boot disk selection heuristic
+
+ -- Proxmox Support Team <support@proxmox.com> Mon, 15 Nov 2021 16:59:23 +0100
+
+qemu-server (7.0-19) bullseye; urgency=medium
+
+ * rollback: improve interaction with snapshot replication
+
+ * cli: qm: rename 'move_disk' command to 'move-disk' with an alias for
+ backward compatibility
+
+ * pi: move-disk: add possibility to reassign a disk to another VM
+
+ * turn SMM off when SeaBIOS and a serial-display are used in combination to
+ avoid a possible boot loop
+
+ -- Proxmox Support Team <support@proxmox.com> Thu, 11 Nov 2021 12:49:10 +0100
+
+qemu-server (7.0-18) bullseye; urgency=medium
+
+ * use non SMM ovmf code file for i440fx machines
+
+ * fix hot-unplugging (removing) a cpulimit on a running VM
+
+ * vm start: only print tpm-related message if there is an actual instance
+
+ * vzdump: increase timeout for QMP 'cont' command after backup started
+
+ * drives: expose readonly flag for SCSI/VirtIO drives as 'ro' property
+
+ * qemu-agent: allow hotplug of the 'fstrim cloned disk' option
+
+ * fix #2429: allow to specify cloud-init vendor snippet via 'cicustom'
+
+ * config: add new meta property with the VM creation time
+
+ * config: meta: also save the QEMU version installed during creation
+
+ * cfg2cmd: switch off ACPI hotplug on bridges for q35 VMs with linux as
+ ostype to avoid changes in network interface naming due to systemd's
+ predicatble naming scheme
+
+ -- Proxmox Support Team <support@proxmox.com> Thu, 04 Nov 2021 15:29:55 +0100
+
+qemu-server (7.0-17) bullseye; urgency=medium
+
+ * fix #3258: block vm start when a PCI(e) device is already in use
+
+ * snapshot: fix TPM state with RBD
+
+ * swtpm: wait for PID file to appear before continuing with VM start
+
+ * OS type: add entry for Windows 11/Server 2022
+
+ -- Proxmox Support Team <support@proxmox.com> Thu, 21 Oct 2021 11:57:09 +0200
+
+qemu-server (7.0-16) bullseye; urgency=medium
+
+ * ovmf: support secure boot enabled code images
+
+ * ovmf: support provisioning an EFI vars template with secureboot by default
+ on and distribution + Microsofts secure-boot key pre-enrolled
+
+ -- Proxmox Support Team <support@proxmox.com> Tue, 05 Oct 2021 20:22:18 +0200
+
+qemu-server (7.0-15) bullseye; urgency=medium
+
+ * api: return task-worker UPID in create template endpoint
+
+ * api: destroy VM: remove pending volumes as well
+
+ * fix #3075: add TPM v1.2 and v2.0 support via swtpm~
+
+ -- Proxmox Support Team <support@proxmox.com> Tue, 05 Oct 2021 07:24:52 +0200
+
+qemu-server (7.0-14) bullseye; urgency=medium
+
+ * fix #3581: pass size via argument for memory-backend-ram QMP call
+
+ * fix #3608: improve removal of the underlying SCSI controller when removing
+ last drive on it
+
+ * migrate: do not suggest that we map shared storages to avoid that
+ subsequent checks could result in false negatives.
+
+ -- Proxmox Support Team <support@proxmox.com> Wed, 22 Sep 2021 09:31:06 +0200
+
+qemu-server (7.0-13) bullseye; urgency=medium
+
+ * fix bootorder regression with implicit default order
+
+ -- Proxmox Support Team <support@proxmox.com> Thu, 5 Aug 2021 14:03:14 +0200
+
+qemu-server (7.0-12) bullseye; urgency=medium
+
+ * fix #3371: import ovf: allow the use of dots in the VM name
+
+ * bootorder: fix double entry on cdrom edit
+
+ -- Proxmox Support Team <support@proxmox.com> Fri, 30 Jul 2021 16:53:44 +0200
+
+qemu-server (7.0-11) bullseye; urgency=medium
+
+ * nic: support the intel e1000e model
+
+ * lvm: avoid the use of io_uring for now
+
+ * live-restore: fail early if target storage doesn't exist
+
+ * api: always add new CD drives to bootorder
+
+ * fix #2563: allow live migration with local cloud-init disk
+
+ -- Proxmox Support Team <support@proxmox.com> Fri, 23 Jul 2021 11:08:48 +0200
+
+qemu-server (7.0-10) bullseye; urgency=medium
+
+ * avoid using io_uring for drives backed by LVM and configured for write-back
+ or write-through cache
+
+ -- Proxmox Support Team <support@proxmox.com> Wed, 07 Jul 2021 14:56:34 +0200
+
+qemu-server (7.0-9) bullseye; urgency=medium
+
+ * cpu weight: always clamp value to lower maximum for cgroup v2 and fix
+ defaults (v1 -> 1024, v2 -> 100)
+
+ * api: improve error handling when applying pending config changes
+
+ -- Proxmox Support Team <support@proxmox.com> Wed, 07 Jul 2021 12:02:13 +0200
+
+qemu-server (7.0-7) bullseye; urgency=medium
+
+ * improve #3329: ensure write-back is used over write-around for EFI disk,
+ as OVMF profits a lot from cached writes due to its frequent
+ read-modify-write operations
+
+ -- Proxmox Support Team <support@proxmox.com> Mon, 05 Jul 2021 20:49:50 +0200
+
+qemu-server (7.0-6) bullseye; urgency=medium
+
+ * live-restore: preload efidisk before starting VM
+
+ * For now do not use io_uring for drives backed by Ceph RBD, with KRBD and
+ write-back or write-through cache enabled, as in that case some polling/IO
+ may hang in QEMU 6.0.
+
+ -- Proxmox Support Team <support@proxmox.com> Fri, 02 Jul 2021 09:45:06 +0200
+
+qemu-server (7.0-5) bullseye; urgency=medium
+
+ * don't default to O_DIRECT (cache=none) on btrfs without nocow
+
+ * fix #2175: api: update VM: check old drive-config for permissions too to
+ ensure a valid transition when limited to CDROM changes.
+
+ -- Proxmox Support Team <support@proxmox.com> Thu, 24 Jun 2021 18:58:19 +0200
+
+qemu-server (7.0-4) bullseye; urgency=medium
+
+ * enable io-uring support by default when running QEMU 6.0 or newer
+
+ * VM start: always check if storages of volumes support correct content-type
+
+ * use KillMode 'process' for systemd scope to cope with depreacation of
+ KillMode=none
+
+ * cli, api: handle new warnings task status
+
+ * improve backup of templates with EFI disks and with SATA and IDE
+ disk controllers in use
+
+ -- Proxmox Support Team <support@proxmox.com> Wed, 23 Jun 2021 12:57:27 +0200
+
+qemu-server (7.0-3) bullseye; urgency=medium
+
+ * vzdump: add master key support
+
+ * vzdump: drop legacy fallback logging for dirty-bitmap
+
+ * vm destroy: do not remove unreferenced disks by default
+
+ * fix #3329: turn on cache=writeback for efidisks on rbd
+
+ * avoid setting LUN number for drives when the `pvscsi` controller is used,
+ as that cannot handle multiple LUNs, increase the `scsi-id` instead
+
+ * config: limit description/comment length to 8 KiB
+
+ * migrate: enforce that image content type is available and configured on
+ target storage
+
+ -- Proxmox Support Team <support@proxmox.com> Mon, 21 Jun 2021 11:17:52 +0200
+
+qemu-server (7.0-2) bullseye; urgency=medium
+
+ * api: clone: sort vm disks to keep numbers consistent
+
+ * api: VM status: make template property optional in return object
+
+ * add compatibility for QEMU 6.0
+
+ * destroy VM: always remove (referenced) VM state volumes
+
+ * destroy VM: also check if unused volumes are base images
+
+ * live-restore: log more similar to regular restore, outputting the user the
+ PBS repo/snapshot and target for each drive.
+
+ -- Proxmox Support Team <support@proxmox.com> Fri, 28 May 2021 12:46:36 +0200
+
+qemu-server (7.0-1) pve; urgency=medium
+
+ * re-build for Proxmox VE 7 / Debian Bullseye
+
+ -- Proxmox Support Team <support@proxmox.com> Thu, 13 May 2021 19:11:18 +0200
+
+qemu-server (6.4-2) pve; urgency=medium
+
+ * fix #2862: allow sata/ide template backups
+
+ * migration: improve speed-limits for >1G connections again
+
+ * fix getting bootdisk size for new bootorder config scheme
+
+ -- Proxmox Support Team <support@proxmox.com> Thu, 29 Apr 2021 16:16:04 +0200
+
+qemu-server (6.4-1) pve; urgency=medium
+
+ * fix the +pveN versioned machine types when PXE is used
+
+ * migration: avoid re-scanning all volumes
+
+ * migration: do not always set default speed limit if none is configured
+
+ * migration: rework logging to more humand friendly format, avoiding to much
+ output noise
+
+ * qmrestore: add live-restore option for CLI tool
+
+ * live-restore: hold 'create' lock during operation
+
+ * live-restore: don't remove VM on error, to allow an VM user to save any new
+ data before retrying the operation.
+
+ * fix #3369: auto-start vm after failed stopmode backup
+
+ -- Proxmox Support Team <support@proxmox.com> Fri, 23 Apr 2021 16:26:54 +0200
+
+qemu-server (6.3-11) pve; urgency=medium
+
+ * enable live-restore tech preview for Proxmox Backup Server hosted backup
+ snapshots.
+
+ * drive mirror: rework periodic status reporting to be human friendlier
+
+ * drive mirror: stop logging progress for a disk once it got ready
+
+ * image convert: use human-readable units in progress report
+
+ -- Proxmox Support Team <support@proxmox.com> Thu, 15 Apr 2021 18:32:06 +0200
+
+qemu-server (6.3-10) pve; urgency=medium
+
+ * increase timeout for block (disk) resize QMP command
+
+ * fix #3314: cloudinit: IPv6 requires type 'static6'
+
+ * fix #2670: cloudinit: enable SLAAC again now that client support is there
+
+ -- Proxmox Support Team <support@proxmox.com> Tue, 30 Mar 2021 18:40:58 +0200
+
+qemu-server (6.3-9) pve; urgency=medium
+
+ * restore vma: fix applying storage-specific bandwidth limit
+
+ * snapshot: set migration caps before savevm-start
+
+ * vzdump: improve error logging for query-proxmox-support to avoid
+ false-positives
+
+ -- Proxmox Support Team <support@proxmox.com> Fri, 26 Mar 2021 09:47:27 +0100
+
+qemu-server (6.3-8) pve; urgency=medium
+
+ * qm status: sort hash keys on verbose output
+
+ * improve windows VM version pinning on VM creation
+
+ -- Proxmox Support Team <support@proxmox.com> Fri, 12 Mar 2021 10:01:09 +0100
+
+qemu-server (6.3-7) pve; urgency=medium
+
+ * vzdump: increase Proxmox Backup Server backup QMP command timeout
+
+ -- Proxmox Support Team <support@proxmox.com> Tue, 09 Mar 2021 08:21:43 +0100
+
+qemu-server (6.3-6) pve; urgency=medium
+
+ * fix #3324: clone disk: use larger blocksize for EFI disk
+
+ * fix #3301: status: add currently running machine and QEMU version to full
+ status
+
+ * api: add endpoint to list all available QEMU machine type and version
+ tuples
+
+ * always pin virtual machines with Windows as ostype to a fixed QEMU machine
+ version by default. For existing VMs with Windows based OS-type use the 5.1
+ machine version (or the next available one, for older QEMU versions) to
+ improve stabillity of the hardware layout from Windows point of view. Linux
+ and other OS types are not as sensitive to those changes, so keep the
+ default to the currently latest available machine versions for non-Windows
+ VMs.
+
+ * update VM: check for CDROM not just drive permissions when removing a
+ device
+
+ -- Proxmox Support Team <support@proxmox.com> Fri, 05 Mar 2021 21:42:59 +0100
+
+qemu-server (6.3-5) pve; urgency=medium
+
+ * cloudinit: add opennebula config format
+
+ * cloudinit: remove pending delete on online regenerate image
+
+ * snapshot/save-vm: periodically print progress and show information about
+ drives during snapshot
+
+ * qmeventd: explicitly close() pidfds
+
+ -- Proxmox Support Team <support@proxmox.com> Thu, 11 Feb 2021 18:05:18 +0100
+
+qemu-server (6.3-4) pve; urgency=medium
+
+ * audio: add the "none" dummy audio backend
+
+ * fix drive-mirror completion with cloudinit
+
+ * vm destroy: allow opt-out of purging unreferenced disks
+
+ * fix #2788: do not resume vms after backup if they were paused before
+
+ * anchor CPU flag regex to avoid arbitrary flag suffixes
+
+ -- Proxmox Support Team <support@proxmox.com> Thu, 28 Jan 2021 17:21:07 +0100
+
+qemu-server (6.3-3) pve; urgency=medium
+
+ * api: adapt VM destroy and purge description
+
+ * clone disk: fix regression in offline clone of efidisk
+
+ * cloudinit: fix cloning/restoring of cloudinit disks in raw format
+
+ -- Proxmox Support Team <support@proxmox.com> Tue, 15 Dec 2020 16:33:01 +0100
+
+qemu-server (6.3-2) pve; urgency=medium
+
+ * PBS: use improved method to assemble repository url, fixing issues when
+ using IPv6 or non-default ports
+
+ -- Proxmox Support Team <support@proxmox.com> Thu, 03 Dec 2020 18:06:25 +0100
+
+qemu-server (6.3-1) pve; urgency=medium
+
+ * deactivate volumes after storage migrate
+
+ * print query-proxmox-support result in 'full' status
+
+ * clone disk: avoid errors after disk was moved by QEMU
+
+ * replace cgroups_write by cgroup change_cpu_shares && change_cpu_quota
+
+ -- Proxmox Support Team <support@proxmox.com> Wed, 25 Nov 2020 14:30:50 +0100
+
+qemu-server (6.2-20) pve; urgency=medium
+
+ * don't migrate replicated VM whose replication job is marked for
+ removal
+
+ * ensure qmeventd service is stopped after pve-guests and pve-ha-lrm service
+ on shutdown
+
+ -- Proxmox Support Team <support@proxmox.com> Thu, 12 Nov 2020 17:08:45 +0100
+
+qemu-server (6.2-19) pve; urgency=medium
+
+ * fix #3113: unbreak drive hotplug
+
+ * qmeventd: add handling for -no-shutdown QEMU instances, to avoid errors if
+ the guest OS shuts down the VM during a backup job.
+
+ -- Proxmox Support Team <support@proxmox.com> Thu, 05 Nov 2020 13:37:00 +0100
+
+qemu-server (6.2-18) pve; urgency=medium
+
+ * migrate: tell QEMU to enable dirty-bitmap migration, if supported
+
+ * partially fix #3056: always try to cancel backups when failed to start job
+
+ -- Proxmox Support Team <support@proxmox.com> Thu, 29 Oct 2020 18:23:13 +0100
+
qemu-server (6.2-17) pve; urgency=medium
* bootorder: don't print empty 'order=' property
Section: admin
Priority: optional
Maintainer: Proxmox Support Team <support@proxmox.com>
-Build-Depends: debhelper (>= 11~),
+Build-Depends: debhelper-compat (= 13),
+ libglib2.0-dev,
libio-multiplex-perl,
libjson-c-dev,
+ libpve-apiclient-perl,
libpve-cluster-perl,
- libpve-common-perl (>= 6.1-5),
- libpve-guest-common-perl (>= 3.0-9),
+ libpve-common-perl (>= 8.0.2),
+ libpve-guest-common-perl (>= 5.1.0),
libpve-storage-perl (>= 6.1-7),
libtest-mockmodule-perl,
libuuid-perl,
pve-doc-generator (>= 6.2-5),
pve-edk2-firmware,
pve-firewall,
- pve-qemu-kvm,
-Standards-Version: 4.3.0
+ pve-ha-manager <!nocheck>,
+ pve-qemu-kvm (>= 7.1~),
+Standards-Version: 4.5.1
Homepage: https://www.proxmox.com
Package: qemu-server
libjson-perl,
libjson-xs-perl,
libnet-ssleay-perl,
- libpve-access-control (>= 5.0-7),
+ libpve-access-control (>= 8.0.0~),
+ libpve-apiclient-perl,
libpve-cluster-perl,
- libpve-common-perl (>= 6.1-5),
- libpve-guest-common-perl (>= 3.0-9),
- libpve-storage-perl (>= 6.2-2),
+ libpve-common-perl (>= 8.0.2),
+ libpve-guest-common-perl (>= 5.1.0),
+ libpve-storage-perl (>= 7.2-10),
libterm-readline-gnu-perl,
libuuid-perl,
libxml-libxml-perl,
perl (>= 5.10.0-19),
+ proxmox-websocket-tunnel,
pve-cluster,
- pve-edk2-firmware (>= 1.20181023-1),
- pve-firewall,
+# TODO: make legacy edk2 optional (suggests) for PVE 9 and warn explicitly about it
+ pve-edk2-firmware-legacy | pve-edk2-firmware (<< 4~),
+ pve-edk2-firmware-ovmf | pve-edk2-firmware (>= 3.20210831-1),
+ pve-firewall (>= 5.0.4),
pve-ha-manager (>= 3.0-9),
- pve-qemu-kvm (>= 3.0.1-62),
+ pve-qemu-kvm (>= 7.1~),
socat,
+ swtpm,
+ swtpm-tools,
${misc:Depends},
${perl:Depends},
${shlibs:Depends},
-Breaks: pve-ha-manager (<= 3.0-4),
- pve-manager (<= 6.0-13),
+Recommends: proxmox-backup-file-restore (>= 2.1.9-2),
+ libpve-network-perl (>= 0.8.3),
+Suggests: pve-edk2-firmware-aarch64, pve-edk2-firmware-riscv,
+Breaks: pve-ha-manager (<< 4.0.1), pve-manager (<= 6.0-13),
Description: Qemu Server Tools
This package contains the Qemu Server tools used by Proxmox VE
-Copyright (C) 2011 - 2020 Proxmox Server Solutions GmbH
+Copyright (C) 2011 - 2024 Proxmox Server Solutions GmbH
This software is written by Proxmox Server Solutions GmbH <support@proxmox.com>
--- /dev/null
+3.0 (native)
CC ?= gcc
CFLAGS += -O2 -Werror -Wall -Wextra -Wpedantic -Wtype-limits -Wl,-z,relro -std=gnu11
-CFLAGS += $(shell pkg-config --cflags json-c)
-LDFLAGS += $(shell pkg-config --libs json-c)
+CFLAGS += $(shell pkg-config --cflags json-c glib-2.0)
+LDFLAGS += $(shell pkg-config --libs json-c glib-2.0)
qmeventd: qmeventd.c
- $(CC) $(CFLAGS) $(LDFLAGS) -o $@ $<
+ $(CC) $(CFLAGS) -o $@ $< $(LDFLAGS)
docs: qmeventd.8
+// SPDX-License-Identifier: AGPL-3.0-or-later
/*
-
- Copyright (C) 2018 Proxmox Server Solutions GmbH
-
- Copyright: qmeventd is under GNU GPL, the GNU General Public License.
-
- This program is free software; you can redistribute it and/or modify
- it under the terms of the GNU General Public License as published by
- the Free Software Foundation; version 2 dated June, 1991.
-
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU General Public License for more details.
-
- You should have received a copy of the GNU General Public License
- along with this program; if not, write to the Free Software
- Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
- 02111-1307, USA.
+ Copyright (C) 2018 - 2021 Proxmox Server Solutions GmbH
Author: Dominik Csapak <d.csapak@proxmox.com>
-
- qmeventd listens on a given socket, and waits for qemu processes
- to connect
-
- it then waits for shutdown events followed by the closing of the socket,
- it then calls /usr/sbin/qm cleanup with following arguments
-
- /usr/sbin/qm cleanup VMID <graceful> <guest>
-
- parameter explanation:
-
- graceful:
- 1|0 depending if it saw a shutdown event before the socket closed
-
- guest:
- 1|0 depending if the shutdown was requested from the guest
-
+ Author: Stefan Reiter <s.reiter@proxmox.com>
+
+ Description:
+
+ qmeventd listens on a given socket, and waits for qemu processes to
+ connect. After accepting a connection qmeventd waits for shutdown events
+ followed by the closing of the socket. Once that happens `qm cleanup` will
+ be executed with following three arguments:
+ VMID <graceful> <guest>
+ Where `graceful` can be `1` or `0` depending if shutdown event was observed
+ before the socket got closed. The second parameter `guest` is also boolean
+ `1` or `0` depending if the shutdown was requested from the guest OS
+ (i.e., the "inside").
*/
#ifndef _GNU_SOURCE
#include <errno.h>
#include <fcntl.h>
+#include <gmodule.h>
#include <json.h>
#include <signal.h>
#include <stdbool.h>
#include <stdio.h>
+#include <stdlib.h>
#include <string.h>
#include <sys/epoll.h>
#include <sys/socket.h>
#include <sys/un.h>
#include <sys/wait.h>
#include <unistd.h>
+#include <time.h>
#include "qmeventd.h"
+#define DEFAULT_KILL_TIMEOUT 60
+
static int verbose = 0;
+static int kill_timeout = DEFAULT_KILL_TIMEOUT;
static int epoll_fd = 0;
static const char *progname;
+GHashTable *vm_clients; // key=vmid (freed on remove), value=*Client (free manually)
+GSList *forced_cleanups;
+static int needs_cleanup = 0;
+
/*
* Helper functions
*/
fprintf(stderr, "Usage: %s [-f] [-v] PATH\n", progname);
fprintf(stderr, " -f run in foreground (default: false)\n");
fprintf(stderr, " -v verbose (default: false)\n");
+ fprintf(stderr, " -t <s> kill timeout (default: %ds)\n", DEFAULT_KILL_TIMEOUT);
fprintf(stderr, " PATH use PATH for socket\n");
}
}
/*
- * reads the vmid from /proc/<pid>/cmdline
- * after the '-id' argument
+ * parses the vmid from the qemu.slice entry of /proc/<pid>/cgroup
*/
static unsigned long
get_vmid_from_pid(pid_t pid)
{
char filename[32] = { 0 };
- int len = snprintf(filename, sizeof(filename), "/proc/%d/cmdline", pid);
+ int len = snprintf(filename, sizeof(filename), "/proc/%d/cgroup", pid);
if (len < 0) {
fprintf(stderr, "error during snprintf for %d: %s\n", pid,
strerror(errno));
}
unsigned long vmid = 0;
- ssize_t rc = 0;
char *buf = NULL;
size_t buflen = 0;
- while ((rc = getdelim(&buf, &buflen, '\0', fp)) >= 0) {
- if (!strcmp(buf, "-id")) {
- break;
+
+ while (getline(&buf, &buflen, fp) >= 0) {
+ char *cgroup_path = strrchr(buf, ':');
+ if (!cgroup_path) {
+ fprintf(stderr, "unexpected cgroup entry %s\n", buf);
+ continue;
}
- }
+ cgroup_path++;
- if (rc < 0) {
- goto err;
- }
+ if (strncmp(cgroup_path, "/qemu.slice/", 12)) {
+ continue;
+ }
+
+ char *vmid_start = strrchr(buf, '/');
+ if (!vmid_start) {
+ fprintf(stderr, "unexpected cgroup entry %s\n", buf);
+ continue;
+ }
+ vmid_start++;
- if (getdelim(&buf, &buflen, '\0', fp) >= 0) {
- if (buf[0] == '-' || buf[0] == '\0') {
- fprintf(stderr, "invalid vmid %s\n", buf);
- goto ret;
+ if (vmid_start[0] == '-' || vmid_start[0] == '\0') {
+ fprintf(stderr, "invalid vmid in cgroup entry %s\n", buf);
+ continue;
}
errno = 0;
char *endptr = NULL;
- vmid = strtoul(buf, &endptr, 10);
- if (errno != 0) {
+ vmid = strtoul(vmid_start, &endptr, 10);
+ if (!endptr || strncmp(endptr, ".scope", 6)) {
+ fprintf(stderr, "unexpected cgroup entry %s\n", buf);
vmid = 0;
- goto err;
- } else if (*endptr != '\0') {
- fprintf(stderr, "invalid vmid %s\n", buf);
+ continue;
+ }
+ if (errno != 0) {
vmid = 0;
}
- goto ret;
+ break;
}
-err:
- fprintf(stderr, "error parsing vmid for %d: %s\n", pid, strerror(errno));
+ if (errno) {
+ fprintf(stderr, "error parsing vmid for %d: %s\n", pid, strerror(errno));
+ } else if (!vmid) {
+ fprintf(stderr, "error parsing vmid for %d: no matching qemu.slice cgroup entry\n", pid);
+ }
-ret:
free(buf);
fclose(fp);
return vmid;
* qmp handling functions
*/
+static void
+send_qmp_cmd(struct Client *client, const char *buf, size_t len)
+{
+ if (!must_write(client->fd, buf, len - 1)) {
+ fprintf(stderr, "%s: cannot send QMP message\n", client->qemu.vmid);
+ cleanup_client(client);
+ }
+}
+
void
handle_qmp_handshake(struct Client *client)
{
- VERBOSE_PRINT("%s: got QMP handshake\n", client->vmid);
- static const char qmp_answer[] = "{\"execute\":\"qmp_capabilities\"}\n";
- if (!must_write(client->fd, qmp_answer, sizeof(qmp_answer) - 1)) {
- fprintf(stderr, "%s: cannot complete handshake\n", client->vmid);
+ VERBOSE_PRINT("pid%d: got QMP handshake, assuming QEMU client\n", client->pid);
+
+ // extract vmid from cmdline, now that we know it's a QEMU process
+ unsigned long vmid = get_vmid_from_pid(client->pid);
+ int res = snprintf(client->qemu.vmid, sizeof(client->qemu.vmid), "%lu", vmid);
+ if (vmid == 0 || res < 0 || res >= (int)sizeof(client->qemu.vmid)) {
+ fprintf(stderr, "could not get vmid from pid %d\n", client->pid);
cleanup_client(client);
+ return;
+ }
+
+ VERBOSE_PRINT("pid%d: assigned VMID: %s\n", client->pid, client->qemu.vmid);
+ client->type = CLIENT_QEMU;
+ if(!g_hash_table_insert(vm_clients, strdup(client->qemu.vmid), client)) {
+ // not fatal, just means backup handling won't work
+ fprintf(stderr, "%s: could not insert client into VMID->client table\n",
+ client->qemu.vmid);
}
+
+ static const char qmp_answer[] = "{\"execute\":\"qmp_capabilities\"}\n";
+ send_qmp_cmd(client, qmp_answer, sizeof(qmp_answer));
}
void
if (!json_object_object_get_ex(obj, "event", &event)) {
return;
}
- VERBOSE_PRINT("%s: got QMP event: %s\n", client->vmid,
- json_object_get_string(event));
+ VERBOSE_PRINT("%s: got QMP event: %s\n", client->qemu.vmid, json_object_get_string(event));
+
+ if (client->state == STATE_TERMINATING) {
+ // QEMU sometimes sends a second SHUTDOWN after SIGTERM, ignore
+ VERBOSE_PRINT("%s: event was after termination, ignoring\n", client->qemu.vmid);
+ return;
+ }
+
// event, check if shutdown and get guest parameter
if (!strcmp(json_object_get_string(event), "SHUTDOWN")) {
- client->graceful = 1;
+ client->qemu.graceful = 1;
struct json_object *data;
struct json_object *guest;
if (json_object_object_get_ex(obj, "data", &data) &&
json_object_object_get_ex(data, "guest", &guest))
{
- client->guest = (unsigned short)json_object_get_boolean(guest);
+ client->qemu.guest = (unsigned short)json_object_get_boolean(guest);
}
+
+ // check if a backup is running and kill QEMU process if not
+ terminate_check(client);
+ }
+}
+
+void
+terminate_check(struct Client *client)
+{
+ if (client->state != STATE_IDLE) {
+ // if we're already in a request, queue this one until after
+ VERBOSE_PRINT("%s: terminate_check queued\n", client->qemu.vmid);
+ client->qemu.term_check_queued = true;
+ return;
+ }
+
+ client->qemu.term_check_queued = false;
+
+ VERBOSE_PRINT("%s: query-status\n", client->qemu.vmid);
+ client->state = STATE_EXPECT_STATUS_RESP;
+ static const char qmp_req[] = "{\"execute\":\"query-status\"}\n";
+ send_qmp_cmd(client, qmp_req, sizeof(qmp_req));
+}
+
+void
+handle_qmp_return(struct Client *client, struct json_object *data, bool error)
+{
+ if (error) {
+ const char *msg = "n/a";
+ struct json_object *desc;
+ if (json_object_object_get_ex(data, "desc", &desc)) {
+ msg = json_object_get_string(desc);
+ }
+ fprintf(stderr, "%s: received error from QMP: %s\n",
+ client->qemu.vmid, msg);
+ client->state = STATE_IDLE;
+ goto out;
+ }
+
+ struct json_object *status;
+ json_bool has_status = data &&
+ json_object_object_get_ex(data, "status", &status);
+
+ bool active = false;
+ if (has_status) {
+ const char *status_str = json_object_get_string(status);
+ active = status_str && (
+ !strcmp(status_str, "running")
+ || !strcmp(status_str, "paused")
+ || !strcmp(status_str, "suspended")
+ || !strcmp(status_str, "prelaunch")
+ );
+ }
+
+ switch (client->state) {
+ case STATE_EXPECT_STATUS_RESP:
+ client->state = STATE_IDLE;
+ if (active) {
+ VERBOSE_PRINT("%s: got status: VM is active\n", client->qemu.vmid);
+ } else if (!client->qemu.backup) {
+ terminate_client(client);
+ } else {
+ // if we're in a backup, don't do anything, vzdump will notify
+ // us when the backup finishes
+ VERBOSE_PRINT("%s: not active, but running backup - keep alive\n",
+ client->qemu.vmid);
+ }
+ break;
+
+ // this means we received the empty return from our handshake answer
+ case STATE_HANDSHAKE:
+ client->state = STATE_IDLE;
+ VERBOSE_PRINT("%s: QMP handshake complete\n", client->qemu.vmid);
+ break;
+
+ // we expect an empty return object after sending quit
+ case STATE_TERMINATING:
+ break;
+ case STATE_IDLE:
+ VERBOSE_PRINT("%s: spurious return value received\n",
+ client->qemu.vmid);
+ break;
+ }
+
+out:
+ if (client->qemu.term_check_queued) {
+ terminate_check(client);
+ }
+}
+
+/*
+ * VZDump specific client functions
+ */
+
+void
+handle_vzdump_handshake(struct Client *client, struct json_object *data)
+{
+ client->state = STATE_IDLE;
+
+ struct json_object *vmid_obj;
+ json_bool has_vmid = data && json_object_object_get_ex(data, "vmid", &vmid_obj);
+
+ if (!has_vmid) {
+ VERBOSE_PRINT("pid%d: invalid vzdump handshake: no vmid\n", client->pid);
+ return;
+ }
+
+ const char *vmid_str = json_object_get_string(vmid_obj);
+
+ if (!vmid_str) {
+ VERBOSE_PRINT("pid%d: invalid vzdump handshake: vmid is not a string\n", client->pid);
+ return;
+ }
+
+ int res = snprintf(client->vzdump.vmid, sizeof(client->vzdump.vmid), "%s", vmid_str);
+ if (res < 0 || res >= (int)sizeof(client->vzdump.vmid)) {
+ VERBOSE_PRINT("pid%d: invalid vzdump handshake: vmid too long or invalid\n", client->pid);
+ return;
+ }
+
+ struct Client *vmc = (struct Client*) g_hash_table_lookup(vm_clients, client->vzdump.vmid);
+ if (vmc) {
+ vmc->qemu.backup = true;
+
+ // only mark as VZDUMP once we have set everything up, otherwise 'cleanup'
+ // might try to access an invalid value
+ client->type = CLIENT_VZDUMP;
+ VERBOSE_PRINT("%s: vzdump backup started\n", client->vzdump.vmid);
+ } else {
+ VERBOSE_PRINT("%s: vzdump requested backup start for unregistered VM\n", client->vzdump.vmid);
}
}
add_new_client(int client_fd)
{
struct Client *client = calloc(sizeof(struct Client), 1);
+ if (client == NULL) {
+ fprintf(stderr, "could not add new client - allocation failed!\n");
+ fflush(stderr);
+ return;
+ }
+ client->state = STATE_HANDSHAKE;
+ client->type = CLIENT_NONE;
client->fd = client_fd;
client->pid = get_pid_from_fd(client_fd);
if (client->pid == 0) {
fprintf(stderr, "could not get pid from client\n");
goto err;
}
- unsigned long vmid = get_vmid_from_pid(client->pid);
- int res = snprintf(client->vmid, sizeof(client->vmid), "%lu", vmid);
- if (vmid == 0 || res < 0 || res >= (int)sizeof(client->vmid)) {
- fprintf(stderr, "could not get vmid from pid %d\n", client->pid);
- goto err;
- }
struct epoll_event ev;
ev.events = EPOLLIN;
ev.data.ptr = client;
- res = epoll_ctl(epoll_fd, EPOLL_CTL_ADD, client_fd, &ev);
+ int res = epoll_ctl(epoll_fd, EPOLL_CTL_ADD, client_fd, &ev);
if (res < 0) {
perror("epoll_ctl client add");
goto err;
}
- VERBOSE_PRINT("added new client, pid: %d, vmid: %s\n", client->pid,
- client->vmid);
+ VERBOSE_PRINT("added new client, pid: %d\n", client->pid);
return;
err:
free(client);
}
-void
-cleanup_client(struct Client *client)
+static void
+cleanup_qemu_client(struct Client *client)
{
- VERBOSE_PRINT("%s: client exited, status: graceful: %d, guest: %d\n",
- client->vmid, client->graceful, client->guest);
- log_neg(epoll_ctl(epoll_fd, EPOLL_CTL_DEL, client->fd, NULL), "epoll del");
- (void)close(client->fd);
-
- unsigned short graceful = client->graceful;
- unsigned short guest = client->guest;
- char vmid[sizeof(client->vmid)];
- strncpy(vmid, client->vmid, sizeof(vmid));
- free(client);
- VERBOSE_PRINT("%s: executing cleanup\n", vmid);
+ unsigned short graceful = client->qemu.graceful;
+ unsigned short guest = client->qemu.guest;
+ char vmid[sizeof(client->qemu.vmid)];
+ strncpy(vmid, client->qemu.vmid, sizeof(vmid));
+ g_hash_table_remove(vm_clients, &vmid); // frees key, ignore errors
+ VERBOSE_PRINT("%s: executing cleanup (graceful: %d, guest: %d)\n",
+ vmid, graceful, guest);
int pid = fork();
if (pid < 0) {
}
}
+void
+cleanup_client(struct Client *client)
+{
+ log_neg(epoll_ctl(epoll_fd, EPOLL_CTL_DEL, client->fd, NULL), "epoll del");
+ (void)close(client->fd);
+
+ struct Client *vmc;
+ switch (client->type) {
+ case CLIENT_QEMU:
+ cleanup_qemu_client(client);
+ break;
+
+ case CLIENT_VZDUMP:
+ vmc = (struct Client*) g_hash_table_lookup(vm_clients, client->vzdump.vmid);
+ if (vmc) {
+ VERBOSE_PRINT("%s: backup ended\n", client->vzdump.vmid);
+ vmc->qemu.backup = false;
+ terminate_check(vmc);
+ }
+ break;
+
+ case CLIENT_NONE:
+ // do nothing, only close socket
+ break;
+ }
+
+ if (client->pidfd > 0) {
+ (void)close(client->pidfd);
+ }
+ VERBOSE_PRINT("removing %s from forced cleanups\n", client->qemu.vmid);
+ forced_cleanups = g_slist_remove(forced_cleanups, client);
+ free(client);
+}
+
+void
+terminate_client(struct Client *client)
+{
+ VERBOSE_PRINT("%s: terminating client (pid %d)\n", client->qemu.vmid, client->pid);
+
+ client->state = STATE_TERMINATING;
+
+ // open a pidfd before kill for later cleanup
+ int pidfd = pidfd_open(client->pid, 0);
+ if (pidfd < 0) {
+ switch (errno) {
+ case ESRCH:
+ // process already dead for some reason, cleanup done
+ VERBOSE_PRINT("%s: failed to open pidfd, process already dead (pid %d)\n",
+ client->qemu.vmid, client->pid);
+ return;
+
+ // otherwise fall back to just using the PID directly, but don't
+ // print if we only failed because we're running on an older kernel
+ case ENOSYS:
+ break;
+ default:
+ perror("failed to open QEMU pidfd for cleanup");
+ break;
+ }
+ }
+
+ // try to send a 'quit' command first, fallback to SIGTERM of the pid
+ static const char qmp_quit_command[] = "{\"execute\":\"quit\"}\n";
+ VERBOSE_PRINT("%s: sending 'quit' via QMP\n", client->qemu.vmid);
+ if (!must_write(client->fd, qmp_quit_command, sizeof(qmp_quit_command) - 1)) {
+ VERBOSE_PRINT("%s: sending 'SIGTERM' to pid %d\n", client->qemu.vmid, client->pid);
+ int err = kill(client->pid, SIGTERM);
+ log_neg(err, "kill");
+ }
+
+ time_t timeout = time(NULL) + kill_timeout;
+
+ client->pidfd = pidfd;
+ client->timeout = timeout;
+
+ forced_cleanups = g_slist_prepend(forced_cleanups, (void *)client);
+ needs_cleanup = 1;
+}
+
void
handle_client(struct Client *client)
{
- VERBOSE_PRINT("%s: entering handle\n", client->vmid);
+ VERBOSE_PRINT("pid%d: entering handle\n", client->pid);
ssize_t len;
do {
len = read(client->fd, (client->buf+client->buflen),
}
return;
} else if (len == 0) {
- VERBOSE_PRINT("%s: got EOF\n", client->vmid);
+ VERBOSE_PRINT("pid%d: got EOF\n", client->pid);
cleanup_client(client);
return;
}
- VERBOSE_PRINT("%s: read %ld bytes\n", client->vmid, len);
+ VERBOSE_PRINT("pid%d: read %ld bytes\n", client->pid, len);
client->buflen += len;
struct json_tokener *tok = json_tokener_new();
handle_qmp_handshake(client);
} else if (json_object_object_get_ex(jobj, "event", &obj)) {
handle_qmp_event(client, jobj);
+ } else if (json_object_object_get_ex(jobj, "return", &obj)) {
+ handle_qmp_return(client, obj, false);
+ } else if (json_object_object_get_ex(jobj, "error", &obj)) {
+ handle_qmp_return(client, obj, true);
+ } else if (json_object_object_get_ex(jobj, "vzdump", &obj)) {
+ handle_vzdump_handshake(client, obj);
} // else ignore message
}
break;
case json_tokener_continue:
if (client->buflen >= sizeof(client->buf)) {
- VERBOSE_PRINT("%s, msg too large, discarding buffer\n",
- client->vmid);
+ VERBOSE_PRINT("pid%d: msg too large, discarding buffer\n", client->pid);
memset(client->buf, 0, sizeof(client->buf));
client->buflen = 0;
} // else we have enough space try again after next read
break;
default:
- VERBOSE_PRINT("%s: parse error: %d, discarding buffer\n",
- client->vmid, jerr);
+ VERBOSE_PRINT("pid%d: parse error: %d, discarding buffer\n", client->pid, jerr);
memset(client->buf, 0, client->buflen);
client->buflen = 0;
break;
json_tokener_free(tok);
}
+static void
+sigkill(void *ptr, void *time_ptr)
+{
+ struct Client *data = ptr;
+ int err;
+
+ if (data->timeout != 0 && data->timeout > *(time_t *)time_ptr) {
+ return;
+ }
+
+ if (data->pidfd > 0) {
+ err = pidfd_send_signal(data->pidfd, SIGKILL, NULL, 0);
+ (void)close(data->pidfd);
+ data->pidfd = -1;
+ } else {
+ err = kill(data->pid, SIGKILL);
+ }
+
+ if (err < 0) {
+ if (errno != ESRCH) {
+ fprintf(stderr, "SIGKILL cleanup of pid '%d' failed - %s\n",
+ data->pid, strerror(errno));
+ }
+ } else {
+ fprintf(stderr, "cleanup failed, terminating pid '%d' with SIGKILL\n",
+ data->pid);
+ }
+
+ data->timeout = 0;
+
+ // remove ourselves from the list
+ forced_cleanups = g_slist_remove(forced_cleanups, ptr);
+}
+
+static void
+handle_forced_cleanup()
+{
+ if (g_slist_length(forced_cleanups) > 0) {
+ VERBOSE_PRINT("clearing forced cleanup backlog\n");
+ time_t cur_time = time(NULL);
+ g_slist_foreach(forced_cleanups, sigkill, &cur_time);
+ }
+ needs_cleanup = g_slist_length(forced_cleanups) > 0;
+}
int
main(int argc, char *argv[])
char *socket_path = NULL;
progname = argv[0];
- while ((opt = getopt(argc, argv, "hfv")) != -1) {
+ while ((opt = getopt(argc, argv, "hfvt:")) != -1) {
switch (opt) {
case 'f':
daemonize = 0;
case 'v':
verbose = 1;
break;
+ case 't':
+ errno = 0;
+ char *endptr = NULL;
+ kill_timeout = strtoul(optarg, &endptr, 10);
+ if (errno != 0 || *endptr != '\0' || kill_timeout == 0) {
+ usage();
+ exit(EXIT_FAILURE);
+ }
+ break;
case 'h':
usage();
exit(EXIT_SUCCESS);
bail_neg(daemon(0, 1), "daemon");
}
+ vm_clients = g_hash_table_new_full(g_str_hash, g_str_equal, free, NULL);
+
int nevents;
for(;;) {
- nevents = epoll_wait(epoll_fd, events, 1, -1);
+ nevents = epoll_wait(epoll_fd, events, 1, needs_cleanup ? 10*1000 : -1);
if (nevents < 0 && errno == EINTR) {
- // signal happened, try again
continue;
}
bail_neg(nevents, "epoll_wait");
for (int n = 0; n < nevents; n++) {
if (events[n].data.fd == sock) {
- int conn_sock = accept4(sock, NULL, NULL,
- SOCK_NONBLOCK | SOCK_CLOEXEC);
+ int conn_sock = accept4(sock, NULL, NULL, SOCK_NONBLOCK | SOCK_CLOEXEC);
log_neg(conn_sock, "accept");
if (conn_sock > -1) {
add_new_client(conn_sock);
handle_client((struct Client *)events[n].data.ptr);
}
}
+ handle_forced_cleanup();
}
}
+// SPDX-License-Identifier: AGPL-3.0-or-later
/*
-
- Copyright (C) 2018 Proxmox Server Solutions GmbH
-
- Copyright: qemumonitor is under GNU GPL, the GNU General Public License.
-
- This program is free software; you can redistribute it and/or modify
- it under the terms of the GNU General Public License as published by
- the Free Software Foundation; version 2 dated June, 1991.
-
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU General Public License for more details.
-
- You should have received a copy of the GNU General Public License
- along with this program; if not, write to the Free Software
- Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
- 02111-1307, USA.
+ Copyright (C) 2018 - 2021 Proxmox Server Solutions GmbH
Author: Dominik Csapak <d.csapak@proxmox.com>
+ Author: Stefan Reiter <s.reiter@proxmox.com>
*/
-#define VERBOSE_PRINT(...) do { if (verbose) { printf(__VA_ARGS__); } } while (0)
+#include <sys/syscall.h>
+#include <time.h>
+
+#ifndef __NR_pidfd_open
+#define __NR_pidfd_open 434
+#endif
+#ifndef __NR_pidfd_send_signal
+#define __NR_pidfd_send_signal 424
+#endif
+
+#define VERBOSE_PRINT(...) do { if (verbose) { printf(__VA_ARGS__); fflush(stdout); } } while (0)
static inline void log_neg(int errval, const char *msg)
{
}
}
+static inline int
+pidfd_open(pid_t pid, unsigned int flags)
+{
+ return syscall(__NR_pidfd_open, pid, flags);
+}
+
+static inline int
+pidfd_send_signal(int pidfd, int sig, siginfo_t *info, unsigned int flags)
+{
+ return syscall(__NR_pidfd_send_signal, pidfd, sig, info, flags);
+}
+
+typedef enum {
+ CLIENT_NONE,
+ CLIENT_QEMU,
+ CLIENT_VZDUMP
+} ClientType;
+
+typedef enum {
+ STATE_HANDSHAKE,
+ STATE_IDLE,
+ STATE_EXPECT_STATUS_RESP,
+ STATE_TERMINATING
+} ClientState;
+
struct Client {
char buf[4096];
- char vmid[16];
+ unsigned int buflen;
+
int fd;
pid_t pid;
- unsigned int buflen;
- unsigned short graceful;
- unsigned short guest;
+ int pidfd;
+ time_t timeout;
+
+ ClientType type;
+ ClientState state;
+
+ // only relevant for type=CLIENT_QEMU
+ struct {
+ char vmid[16];
+ unsigned short graceful;
+ unsigned short guest;
+ bool term_check_queued;
+ bool backup;
+ } qemu;
+
+ // only relevant for type=CLIENT_VZDUMP
+ struct {
+ // vmid of referenced backup
+ char vmid[16];
+ } vzdump;
};
void handle_qmp_handshake(struct Client *client);
void handle_qmp_event(struct Client *client, struct json_object *obj);
+void handle_qmp_return(struct Client *client, struct json_object *data, bool error);
+void handle_vzdump_handshake(struct Client *client, struct json_object *data);
void handle_client(struct Client *client);
void add_new_client(int client_fd);
void cleanup_client(struct Client *client);
+void terminate_client(struct Client *client);
+void terminate_check(struct Client *client);
[Unit]
Description=PVE Qemu Event Daemon
RequiresMountsFor=/var/run
+Before=pve-ha-lrm.service
+Before=pve-guests.service
[Service]
ExecStart=/usr/sbin/qmeventd /var/run/qmeventd.sock
all: test
-test: test_snapshot test_ovf test_cfg_to_cmd test_pci_addr_conflicts test_qemu_img_convert
+test: test_snapshot test_ovf test_cfg_to_cmd test_pci_addr_conflicts test_qemu_img_convert test_migration test_restore_config
test_snapshot: run_snapshot_tests.pl
./run_snapshot_tests.pl
test_pci_addr_conflicts: run_pci_addr_checks.pl
./run_pci_addr_checks.pl
+
+MIGRATION_TEST_TARGETS := $(addprefix test_migration_,$(shell perl -ne 'print "$$1 " if /^\s*name\s*=>\s*["'\'']([^\s"'\'']+)["'\'']\s*,\s*$$/; END { print "\n" }' run_qemu_migrate_tests.pl))
+
+test_migration: run_qemu_migrate_tests.pl MigrationTest/*.pm $(MIGRATION_TEST_TARGETS)
+
+$(MIGRATION_TEST_TARGETS):
+ ./run_qemu_migrate_tests.pl $(@:test_migration_%=%)
+
+test_restore_config: run_qemu_restore_config_tests.pl
+ ./run_qemu_restore_config_tests.pl
+
+.PHONY: clean
+clean:
+ rm -rf MigrationTest/run
--- /dev/null
+package MigrationTest::QemuMigrateMock;
+
+use strict;
+use warnings;
+
+use JSON;
+use Test::MockModule;
+
+use MigrationTest::Shared;
+
+use PVE::API2::Qemu;
+use PVE::Storage;
+use PVE::Tools qw(file_set_contents file_get_contents);
+
+use PVE::CLIHandler;
+use base qw(PVE::CLIHandler);
+
+my $RUN_DIR_PATH = $ENV{RUN_DIR_PATH} or die "no RUN_DIR_PATH set\n";
+my $QM_LIB_PATH = $ENV{QM_LIB_PATH} or die "no QM_LIB_PATH set\n";
+
+my $source_volids = decode_json(file_get_contents("${RUN_DIR_PATH}/source_volids"));
+my $source_vdisks = decode_json(file_get_contents("${RUN_DIR_PATH}/source_vdisks"));
+my $vm_status = decode_json(file_get_contents("${RUN_DIR_PATH}/vm_status"));
+my $expected_calls = decode_json(file_get_contents("${RUN_DIR_PATH}/expected_calls"));
+my $fail_config = decode_json(file_get_contents("${RUN_DIR_PATH}/fail_config"));
+my $storage_migrate_map = decode_json(file_get_contents("${RUN_DIR_PATH}/storage_migrate_map"));
+my $migrate_params = decode_json(file_get_contents("${RUN_DIR_PATH}/migrate_params"));
+
+my $test_vmid = $migrate_params->{vmid};
+my $test_target = $migrate_params->{target};
+my $test_opts = $migrate_params->{opts};
+my $current_log = '';
+
+my $vm_stop_executed = 0;
+
+# mocked modules
+
+my $inotify_module = Test::MockModule->new("PVE::INotify");
+$inotify_module->mock(
+ nodename => sub {
+ return 'pve0';
+ },
+);
+
+$MigrationTest::Shared::qemu_config_module->mock(
+ move_config_to_node => sub {
+ my ($self, $vmid, $target) = @_;
+ die "moving wrong config: '$vmid'\n" if $vmid ne $test_vmid;
+ die "moving config to wrong node: '$target'\n" if $target ne $test_target;
+ delete $expected_calls->{move_config_to_node};
+ },
+);
+
+my $tunnel_module = Test::MockModule->new("PVE::Tunnel");
+$tunnel_module->mock(
+ finish_tunnel => sub {
+ delete $expected_calls->{'finish_tunnel'};
+ return;
+ },
+ write_tunnel => sub {
+ my ($tunnel, $timeout, $command) = @_;
+
+ if ($command =~ m/^resume (\d+)$/) {
+ my $vmid = $1;
+ die "resuming wrong VM '$vmid'\n" if $vmid ne $test_vmid;
+ return;
+ }
+ die "write_tunnel (mocked) - implement me: $command\n";
+ },
+);
+
+my $qemu_migrate_module = Test::MockModule->new("PVE::QemuMigrate");
+$qemu_migrate_module->mock(
+ fork_tunnel => sub {
+ die "fork_tunnel (mocked) - implement me\n"; # currently no call should lead here
+ },
+ read_tunnel => sub {
+ die "read_tunnel (mocked) - implement me\n"; # currently no call should lead here
+ },
+ start_remote_tunnel => sub {
+ my ($self, $raddr, $rport, $ruri, $unix_socket_info) = @_;
+ $expected_calls->{'finish_tunnel'} = 1;
+ $self->{tunnel} = {
+ writer => "mocked",
+ reader => "mocked",
+ pid => 123456,
+ version => 1,
+ };
+ },
+ log => sub {
+ my ($self, $level, $message) = @_;
+ $current_log .= "$level: $message\n";
+ },
+ mon_cmd => sub {
+ my ($vmid, $command, %params) = @_;
+
+ if ($command eq 'nbd-server-start') {
+ return;
+ } elsif ($command eq 'block-dirty-bitmap-add') {
+ my $drive = $params{node};
+ delete $expected_calls->{"block-dirty-bitmap-add-${drive}"};
+ return;
+ } elsif ($command eq 'block-dirty-bitmap-remove') {
+ return;
+ } elsif ($command eq 'query-migrate') {
+ return { status => 'failed' } if $fail_config->{'query-migrate'};
+ return { status => 'completed' };
+ } elsif ($command eq 'migrate') {
+ return;
+ } elsif ($command eq 'migrate-set-parameters') {
+ return;
+ } elsif ($command eq 'migrate_cancel') {
+ return;
+ }
+ die "mon_cmd (mocked) - implement me: $command";
+ },
+ transfer_replication_state => sub {
+ delete $expected_calls->{transfer_replication_state};
+ },
+ switch_replication_job_target => sub {
+ delete $expected_calls->{switch_replication_job_target};
+ },
+);
+
+$MigrationTest::Shared::qemu_server_module->mock(
+ kvm_user_version => sub {
+ return "5.0.0";
+ },
+ qemu_blockjobs_cancel => sub {
+ return;
+ },
+ qemu_drive_mirror => sub {
+ my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
+
+ die "drive_mirror with wrong vmid: '$vmid'\n" if $vmid ne $test_vmid;
+ die "qemu_drive_mirror '$drive' error\n"
+ if $fail_config->{qemu_drive_mirror} && $fail_config->{qemu_drive_mirror} eq $drive;
+
+ my $nbd_info = decode_json(file_get_contents("${RUN_DIR_PATH}/nbd_info"));
+ die "target does not expect drive mirror for '$drive'\n"
+ if !defined($nbd_info->{$drive});
+ delete $nbd_info->{$drive};
+ file_set_contents("${RUN_DIR_PATH}/nbd_info", to_json($nbd_info));
+ },
+ qemu_drive_mirror_monitor => sub {
+ my ($vmid, $vmiddst, $jobs, $completion, $qga) = @_;
+
+ if ($fail_config->{qemu_drive_mirror_monitor}
+ && $fail_config->{qemu_drive_mirror_monitor} eq $completion
+ ) {
+ die "qemu_drive_mirror_monitor '$completion' error\n";
+ }
+ return;
+ },
+ set_migration_caps => sub {
+ return;
+ },
+ vm_stop => sub {
+ $vm_stop_executed = 1;
+ delete $expected_calls->{'vm_stop'};
+ },
+ del_nets_bridge_fdb => sub { return; },
+);
+
+my $qemu_server_cpuconfig_module = Test::MockModule->new("PVE::QemuServer::CPUConfig");
+$qemu_server_cpuconfig_module->mock(
+ get_cpu_from_running_vm => sub {
+ die "invalid test: if you specify a custom CPU model you need to " .
+ "specify runningcpu as well\n" if !defined($vm_status->{runningcpu});
+ return $vm_status->{runningcpu};
+ }
+);
+
+my $qemu_server_helpers_module = Test::MockModule->new("PVE::QemuServer::Helpers");
+$qemu_server_helpers_module->mock(
+ vm_running_locally => sub {
+ return $vm_status->{running} && !$vm_stop_executed;
+ },
+);
+
+my $qemu_server_machine_module = Test::MockModule->new("PVE::QemuServer::Machine");
+$qemu_server_machine_module->mock(
+ qemu_machine_pxe => sub {
+ die "invalid test: no runningmachine specified\n"
+ if !defined($vm_status->{runningmachine});
+ return $vm_status->{runningmachine};
+ },
+);
+
+my $ssh_info_module = Test::MockModule->new("PVE::SSHInfo");
+$ssh_info_module->mock(
+ get_ssh_info => sub {
+ my ($node, $network_cidr) = @_;
+ return {
+ ip => '1.2.3.4',
+ name => $node,
+ network => $network_cidr,
+ };
+ },
+);
+
+$MigrationTest::Shared::storage_module->mock(
+ storage_migrate => sub {
+ my ($cfg, $volid, $target_sshinfo, $target_storeid, $opts, $logfunc) = @_;
+
+ die "storage_migrate '$volid' error\n"
+ if $fail_config->{storage_migrate} && $fail_config->{storage_migrate} eq $volid;
+
+ my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
+
+ die "invalid test: need to add entry for '$volid' to storage_migrate_map\n"
+ if $storeid ne $target_storeid && !defined($storage_migrate_map->{$volid});
+
+ my $target_volname = $storage_migrate_map->{$volid} // $opts->{target_volname} // $volname;
+ my $target_volid = "${target_storeid}:${target_volname}";
+ MigrationTest::Shared::add_target_volid($target_volid);
+
+ return $target_volid;
+ },
+ vdisk_list => sub { # expects vmid to be set
+ my ($cfg, $storeid, $vmid, $vollist) = @_;
+
+ my @storeids = defined($storeid) ? ($storeid) : keys %{$source_vdisks};
+
+ my $res = {};
+ foreach my $storeid (@storeids) {
+ my $list_for_storeid = $source_vdisks->{$storeid};
+ my @list_for_vm = grep { $_->{vmid} eq $vmid } @{$list_for_storeid};
+ $res->{$storeid} = \@list_for_vm;
+ }
+ return $res;
+ },
+ vdisk_free => sub {
+ my ($scfg, $volid) = @_;
+
+ PVE::Storage::parse_volume_id($volid);
+
+ die "vdisk_free '$volid' error\n"
+ if defined($fail_config->{vdisk_free}) && $fail_config->{vdisk_free} eq $volid;
+
+ delete $source_volids->{$volid};
+ },
+ volume_size_info => sub {
+ my ($scfg, $volid) = @_;
+ my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
+
+ for my $v ($source_vdisks->{$storeid}->@*) {
+ return wantarray ? ($v->{size}, $v->{format}, $v->{used}, $v->{parent}) : $v->{size}
+ if $v->{volid} eq $volid;
+ }
+ die "could not find '$volid' in mock 'source_vdisks'\n";
+ },
+);
+
+$MigrationTest::Shared::tools_module->mock(
+ get_host_address_family => sub {
+ die "get_host_address_family (mocked) - implement me\n"; # currently no call should lead here
+ },
+ next_migrate_port => sub {
+ die "next_migrate_port (mocked) - implement me\n"; # currently no call should lead here
+ },
+ run_command => sub {
+ my ($cmd_tail, %param) = @_;
+
+ my $cmd_msg = to_json($cmd_tail);
+
+ my $cmd = shift @{$cmd_tail};
+
+ if ($cmd =~ m@^(?:/usr/bin/)?ssh$@) {
+ while (scalar(@{$cmd_tail})) {
+ $cmd = shift @{$cmd_tail};
+ if ($cmd eq '/bin/true') {
+ return 0;
+ } elsif ($cmd eq 'qm') {
+ $cmd = shift @{$cmd_tail};
+ if ($cmd eq 'start') {
+ delete $expected_calls->{ssh_qm_start};
+
+ delete $vm_status->{runningmachine};
+ delete $vm_status->{runningcpu};
+
+ my @options = ( @{$cmd_tail} );
+ while (scalar(@options)) {
+ my $opt = shift @options;
+ if ($opt eq '--machine') {
+ $vm_status->{runningmachine} = shift @options;
+ } elsif ($opt eq '--force-cpu') {
+ $vm_status->{runningcpu} = shift @options;
+ }
+ }
+
+ return $MigrationTest::Shared::tools_module->original('run_command')->(
+ [
+ '/usr/bin/perl',
+ "-I${QM_LIB_PATH}",
+ "-I${QM_LIB_PATH}/test",
+ "${QM_LIB_PATH}/test/MigrationTest/QmMock.pm",
+ 'start',
+ @{$cmd_tail},
+ ],
+ %param,
+ );
+
+ } elsif ($cmd eq 'nbdstop') {
+ delete $expected_calls->{ssh_nbdstop};
+ return 0;
+ } elsif ($cmd eq 'resume') {
+ return 0;
+ } elsif ($cmd eq 'unlock') {
+ my $vmid = shift @{$cmd_tail};;
+ die "unlocking wrong vmid: $vmid\n" if $vmid ne $test_vmid;
+ PVE::QemuConfig->remove_lock($vmid);
+ return 0;
+ } elsif ($cmd eq 'stop') {
+ return 0;
+ }
+ die "run_command (mocked) ssh qm command - implement me: ${cmd_msg}";
+ } elsif ($cmd eq 'pvesm') {
+ $cmd = shift @{$cmd_tail};
+ if ($cmd eq 'free') {
+ my $volid = shift @{$cmd_tail};
+ PVE::Storage::parse_volume_id($volid);
+ return 1
+ if $fail_config->{ssh_pvesm_free} && $fail_config->{ssh_pvesm_free} eq $volid;
+ MigrationTest::Shared::remove_target_volid($volid);
+ return 0;
+ }
+ die "run_command (mocked) ssh pvesm command - implement me: ${cmd_msg}";
+ }
+ }
+ die "run_command (mocked) ssh command - implement me: ${cmd_msg}";
+ }
+ die "run_command (mocked) - implement me: ${cmd_msg}";
+ },
+);
+
+eval { PVE::QemuMigrate->migrate($test_target, undef, $test_vmid, $test_opts) };
+my $error = $@;
+
+file_set_contents("${RUN_DIR_PATH}/source_volids", to_json($source_volids));
+file_set_contents("${RUN_DIR_PATH}/vm_status", to_json($vm_status));
+file_set_contents("${RUN_DIR_PATH}/expected_calls", to_json($expected_calls));
+file_set_contents("${RUN_DIR_PATH}/log", $current_log);
+
+die $error if $error;
+
+1;
--- /dev/null
+package MigrationTest::QmMock;
+
+use strict;
+use warnings;
+
+use JSON;
+use Test::MockModule;
+
+use MigrationTest::Shared;
+
+use PVE::API2::Qemu;
+use PVE::Storage;
+use PVE::Tools qw(file_set_contents file_get_contents);
+
+use PVE::CLIHandler;
+use base qw(PVE::CLIHandler);
+
+my $RUN_DIR_PATH = $ENV{RUN_DIR_PATH} or die "no RUN_DIR_PATH set\n";
+
+my $target_volids = decode_json(file_get_contents("${RUN_DIR_PATH}/target_volids"));
+my $fail_config = decode_json(file_get_contents("${RUN_DIR_PATH}/fail_config"));
+my $migrate_params = decode_json(file_get_contents("${RUN_DIR_PATH}/migrate_params"));
+my $nodename = $migrate_params->{target};
+
+my $kvm_exectued = 0;
+my $forcemachine;
+
+sub setup_environment {
+ my $rpcenv = PVE::RPCEnvironment::init('MigrationTest::QmMock', 'cli');
+}
+
+# mock RPCEnvironment directly
+
+sub get_user {
+ return 'root@pam';
+}
+
+sub fork_worker {
+ my ($self, $dtype, $id, $user, $function, $background) = @_;
+ $function->(123456);
+ return '123456';
+}
+
+# mocked modules
+
+my $inotify_module = Test::MockModule->new("PVE::INotify");
+$inotify_module->mock(
+ nodename => sub {
+ return $nodename;
+ },
+);
+
+$MigrationTest::Shared::qemu_server_module->mock(
+ nodename => sub {
+ return $nodename;
+ },
+ config_to_command => sub {
+ return [ 'mocked_kvm_command' ];
+ },
+ vm_start_nolock => sub {
+ my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
+ $forcemachine = $params->{forcemachine}
+ or die "mocked vm_start_nolock - expected 'forcemachine' parameter\n";
+ $MigrationTest::Shared::qemu_server_module->original('vm_start_nolock')->(@_);
+ },
+);
+
+my $qemu_server_helpers_module = Test::MockModule->new("PVE::QemuServer::Helpers");
+$qemu_server_helpers_module->mock(
+ vm_running_locally => sub {
+ return $kvm_exectued;
+ },
+);
+
+our $qemu_server_machine_module = Test::MockModule->new("PVE::QemuServer::Machine");
+$qemu_server_machine_module->mock(
+ get_current_qemu_machine => sub {
+ return wantarray ? ($forcemachine, 0) : $forcemachine;
+ },
+);
+
+# to make sure we get valid and predictable names
+my $disk_counter = 10;
+
+$MigrationTest::Shared::storage_module->mock(
+ vdisk_alloc => sub {
+ my ($cfg, $storeid, $vmid, $fmt, $name, $size) = @_;
+
+ die "vdisk_alloc (mocked) - name is not expected to be set - implement me\n"
+ if defined($name);
+
+ my $name_without_extension = "vm-${vmid}-disk-${disk_counter}";
+ $disk_counter++;
+
+ my $volid;
+ my $scfg = PVE::Storage::storage_config($cfg, $storeid);
+ if ($scfg->{path}) {
+ $volid = "${storeid}:${vmid}/${name_without_extension}.${fmt}";
+ } else {
+ $volid = "${storeid}:${name_without_extension}";
+ }
+
+ PVE::Storage::parse_volume_id($volid);
+
+ die "vdisk_alloc '$volid' error\n" if $fail_config->{vdisk_alloc}
+ && $fail_config->{vdisk_alloc} eq $volid;
+
+ MigrationTest::Shared::add_target_volid($volid);
+
+ return $volid;
+ },
+);
+
+$MigrationTest::Shared::qemu_server_module->mock(
+ mon_cmd => sub {
+ my ($vmid, $command, %params) = @_;
+
+ if ($command eq 'nbd-server-start') {
+ return;
+ } elsif ($command eq 'block-export-add') {
+ return;
+ } elsif ($command eq 'query-block') {
+ return [];
+ } elsif ($command eq 'qom-set') {
+ return;
+ }
+ die "mon_cmd (mocked) - implement me: $command";
+ },
+ run_command => sub {
+ my ($cmd_full, %param) = @_;
+
+ my $cmd_msg = to_json($cmd_full);
+
+ my $cmd = shift @{$cmd_full};
+
+ if ($cmd eq '/bin/systemctl') {
+ return;
+ } elsif ($cmd eq 'mocked_kvm_command') {
+ $kvm_exectued = 1;
+ return 0;
+ }
+ die "run_command (mocked) - implement me: ${cmd_msg}";
+ },
+ set_migration_caps => sub {
+ return;
+ },
+ vm_migrate_alloc_nbd_disks => sub{
+ my $nbd = $MigrationTest::Shared::qemu_server_module->original('vm_migrate_alloc_nbd_disks')->(@_);
+ file_set_contents("${RUN_DIR_PATH}/nbd_info", to_json($nbd));
+ return $nbd;
+ },
+);
+
+our $cmddef = {
+ start => [ "PVE::API2::Qemu", 'vm_start', ['vmid'], { node => $nodename } ],
+};
+
+MigrationTest::QmMock->run_cli_handler();
+
+1;
--- /dev/null
+package MigrationTest::Shared;
+
+use strict;
+use warnings;
+
+use JSON;
+use Test::MockModule;
+use Socket qw(AF_INET);
+
+use PVE::QemuConfig;
+use PVE::Tools qw(file_set_contents file_get_contents lock_file_full);
+
+my $RUN_DIR_PATH = $ENV{RUN_DIR_PATH} or die "no RUN_DIR_PATH set\n";
+
+my $storage_config = decode_json(file_get_contents("${RUN_DIR_PATH}/storage_config"));
+my $replication_config = decode_json(file_get_contents("${RUN_DIR_PATH}/replication_config"));
+my $fail_config = decode_json(file_get_contents("${RUN_DIR_PATH}/fail_config"));
+my $migrate_params = decode_json(file_get_contents("${RUN_DIR_PATH}/migrate_params"));
+my $test_vmid = $migrate_params->{vmid};
+
+# helpers
+
+sub add_target_volid {
+ my ($volid) = @_;
+
+ PVE::Storage::parse_volume_id($volid);
+
+ lock_file_full("${RUN_DIR_PATH}/target_volids.lock", undef, 0, sub {
+ my $target_volids = decode_json(file_get_contents("${RUN_DIR_PATH}/target_volids"));
+ die "target volid already present " if defined($target_volids->{$volid});
+ $target_volids->{$volid} = 1;
+ file_set_contents("${RUN_DIR_PATH}/target_volids", to_json($target_volids));
+ });
+ die $@ if $@;
+}
+
+sub remove_target_volid {
+ my ($volid) = @_;
+
+ PVE::Storage::parse_volume_id($volid);
+
+ lock_file_full("${RUN_DIR_PATH}/target_volids.lock", undef, 0, sub {
+ my $target_volids = decode_json(file_get_contents("${RUN_DIR_PATH}/target_volids"));
+ die "target volid does not exist " if !defined($target_volids->{$volid});
+ delete $target_volids->{$volid};
+ file_set_contents("${RUN_DIR_PATH}/target_volids", to_json($target_volids));
+ });
+ die $@ if $@;
+}
+
+my $mocked_cfs_read_file = sub {
+ my ($file) = @_;
+
+ if ($file eq 'datacenter.cfg') {
+ return {};
+ } elsif ($file eq 'replication.cfg') {
+ return $replication_config;
+ }
+ die "cfs_read_file (mocked) - implement me: $file\n";
+};
+
+# mocked modules
+
+our $cgroup_module = Test::MockModule->new("PVE::CGroup");
+$cgroup_module->mock(
+ cgroup_mode => sub {
+ return 2;
+ },
+);
+
+our $cluster_module = Test::MockModule->new("PVE::Cluster");
+$cluster_module->mock(
+ cfs_read_file => $mocked_cfs_read_file,
+ check_cfs_quorum => sub {
+ return 1;
+ },
+);
+
+our $mapping_usb_module = Test::MockModule->new("PVE::Mapping::USB");
+$mapping_usb_module->mock(
+ config => sub {
+ return {};
+ },
+);
+
+our $mapping_pci_module = Test::MockModule->new("PVE::Mapping::PCI");
+$mapping_pci_module->mock(
+ config => sub {
+ return {};
+ },
+);
+
+our $ha_config_module = Test::MockModule->new("PVE::HA::Config");
+$ha_config_module->mock(
+ vm_is_ha_managed => sub {
+ return 0;
+ },
+);
+
+our $qemu_config_module = Test::MockModule->new("PVE::QemuConfig");
+$qemu_config_module->mock(
+ assert_config_exists_on_node => sub {
+ return;
+ },
+ load_config => sub {
+ my ($class, $vmid, $node) = @_;
+ die "trying to load wrong config: '$vmid'\n" if $vmid ne $test_vmid;
+ return decode_json(file_get_contents("${RUN_DIR_PATH}/vm_config"));
+ },
+ lock_config => sub { # no use locking here because lock is local to node
+ my ($self, $vmid, $code, @param) = @_;
+ return $code->(@param);
+ },
+ write_config => sub {
+ my ($class, $vmid, $conf) = @_;
+ die "trying to write wrong config: '$vmid'\n" if $vmid ne $test_vmid;
+ file_set_contents("${RUN_DIR_PATH}/vm_config", to_json($conf));
+ },
+);
+
+our $qemu_server_cloudinit_module = Test::MockModule->new("PVE::QemuServer::Cloudinit");
+$qemu_server_cloudinit_module->mock(
+ generate_cloudinitconfig => sub {
+ return;
+ },
+);
+
+our $qemu_server_module = Test::MockModule->new("PVE::QemuServer");
+$qemu_server_module->mock(
+ clear_reboot_request => sub {
+ return 1;
+ },
+ get_efivars_size => sub {
+ return 128 * 1024;
+ },
+);
+
+our $replication_module = Test::MockModule->new("PVE::Replication");
+$replication_module->mock(
+ run_replication => sub {
+ die "run_replication error" if $fail_config->{run_replication};
+
+ my $vm_config = PVE::QemuConfig->load_config($test_vmid);
+ return PVE::QemuConfig->get_replicatable_volumes(
+ $storage_config,
+ $test_vmid,
+ $vm_config,
+ );
+ },
+);
+
+our $replication_config_module = Test::MockModule->new("PVE::ReplicationConfig");
+$replication_config_module->mock(
+ cfs_read_file => $mocked_cfs_read_file,
+);
+
+our $safe_syslog_module = Test::MockModule->new("PVE::SafeSyslog");
+$safe_syslog_module->mock(
+ initlog => sub {},
+ syslog => sub {},
+);
+
+our $storage_module = Test::MockModule->new("PVE::Storage");
+$storage_module->mock(
+ activate_volumes => sub {
+ return 1;
+ },
+ deactivate_volumes => sub {
+ return 1;
+ },
+ config => sub {
+ return $storage_config;
+ },
+ get_bandwidth_limit => sub {
+ return 123456;
+ },
+ cfs_read_file => $mocked_cfs_read_file,
+);
+
+our $storage_plugin_module = Test::MockModule->new("PVE::Storage::Plugin");
+$storage_plugin_module->mock(
+ cluster_lock_storage => sub {
+ my ($class, $storeid, $shared, $timeout, $func, @param) = @_;
+
+ mkdir "${RUN_DIR_PATH}/lock";
+
+ my $path = "${RUN_DIR_PATH}/lock/pve-storage-${storeid}";
+ return PVE::Tools::lock_file($path, $timeout, $func, @param);
+ },
+);
+
+our $systemd_module = Test::MockModule->new("PVE::Systemd");
+$systemd_module->mock(
+ wait_for_unit_removed => sub {
+ return;
+ },
+ enter_systemd_scope => sub {
+ return;
+ },
+);
+
+my $migrate_port_counter = 60000;
+
+our $tools_module = Test::MockModule->new("PVE::Tools");
+$tools_module->mock(
+ get_host_address_family => sub {
+ return AF_INET;
+ },
+ next_migrate_port => sub {
+ return $migrate_port_counter++;
+ },
+);
+
+1;
/usr/bin/kvm \
-id 8006 \
- -name simple \
- -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server,nowait' \
+ -name 'simple,debug-threads=on' \
+ -no-shutdown \
+ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \
-mon 'chardev=qmp,mode=control' \
-chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' \
-mon 'chardev=qmp-event,mode=control' \
-smp '3,sockets=1,cores=3,maxcpus=3' \
-nodefaults \
-boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \
- -vnc unix:/var/run/qemu-server/8006.vnc,password \
+ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \
-cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \
-m 768 \
-object 'iothread,id=iothread-virtio0' \
/usr/bin/kvm \
-id 8006 \
- -name simple \
- -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server,nowait' \
+ -name 'simple,debug-threads=on' \
+ -no-shutdown \
+ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \
-mon 'chardev=qmp,mode=control' \
-chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' \
-mon 'chardev=qmp-event,mode=control' \
-smp '3,sockets=1,cores=3,maxcpus=3' \
-nodefaults \
-boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \
- -vnc unix:/var/run/qemu-server/8006.vnc,password \
+ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \
-cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \
-m 768 \
-object 'iothread,id=iothread-virtio0' \
/usr/bin/kvm \
-id 8006 \
- -name simple \
- -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server,nowait' \
+ -name 'simple,debug-threads=on' \
+ -no-shutdown \
+ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \
-mon 'chardev=qmp,mode=control' \
-chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' \
-mon 'chardev=qmp-event,mode=control' \
-smp '3,sockets=1,cores=3,maxcpus=3' \
-nodefaults \
-boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \
- -vnc unix:/var/run/qemu-server/8006.vnc,password \
+ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \
-cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \
-m 768 \
-object 'iothread,id=iothread-virtio0' \
--- /dev/null
+# TEST: test CPU type depreacation for Icelake-Client (never existed in the wild)
+# QEMU_VERSION: 7.1
+bootdisk: scsi0
+cores: 2
+cpu: Icelake-Client
+ide2: none,media=cdrom
+memory: 768
+name: simple
+ostype: l26
+scsi0: local:8006/base-8006-disk-0.qcow2,discard=on,size=104858K
+scsihw: virtio-scsi-pci
+smbios1: uuid=7b10d7af-b932-4c66-b2c3-3996152ec465
+sockets: 1
+vmgenid: c773c261-d800-4348-9f5d-167fadd53cf8
--- /dev/null
+/usr/bin/kvm \
+ -id 8006 \
+ -name 'simple,debug-threads=on' \
+ -no-shutdown \
+ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \
+ -mon 'chardev=qmp,mode=control' \
+ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' \
+ -mon 'chardev=qmp-event,mode=control' \
+ -pidfile /var/run/qemu-server/8006.pid \
+ -daemonize \
+ -smbios 'type=1,uuid=7b10d7af-b932-4c66-b2c3-3996152ec465' \
+ -smp '2,sockets=1,cores=2,maxcpus=2' \
+ -nodefaults \
+ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \
+ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \
+ -cpu 'Icelake-Server,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,vendor=GenuineIntel' \
+ -m 768 \
+ -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \
+ -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \
+ -device 'vmgenid,guid=c773c261-d800-4348-9f5d-167fadd53cf8' \
+ -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \
+ -device 'usb-tablet,id=tablet,bus=uhci.0,port=1' \
+ -device 'VGA,id=vga,bus=pci.0,addr=0x2' \
+ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \
+ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \
+ -drive 'if=none,id=drive-ide2,media=cdrom,aio=io_uring' \
+ -device 'ide-cd,bus=ide.1,unit=0,drive=drive-ide2,id=ide2,bootindex=200' \
+ -device 'virtio-scsi-pci,id=scsihw0,bus=pci.0,addr=0x5' \
+ -drive 'file=/var/lib/vz/images/8006/base-8006-disk-0.qcow2,if=none,id=drive-scsi0,discard=on,format=qcow2,cache=none,aio=io_uring,detect-zeroes=unmap' \
+ -device 'scsi-hd,bus=scsihw0.0,channel=0,scsi-id=0,lun=0,drive=drive-scsi0,id=scsi0,bootindex=100' \
+ -machine 'type=pc+pve0'
/usr/bin/kvm \
-id 8006 \
- -name customcpu-defaults \
- -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server,nowait' \
+ -name 'customcpu-defaults,debug-threads=on' \
+ -no-shutdown \
+ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \
-mon 'chardev=qmp,mode=control' \
-chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' \
-mon 'chardev=qmp-event,mode=control' \
-smp '3,sockets=1,cores=3,maxcpus=3' \
-nodefaults \
-boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \
- -vnc unix:/var/run/qemu-server/8006.vnc,password \
+ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \
-cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \
-m 512 \
-device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \
-device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \
-device 'usb-tablet,id=tablet,bus=uhci.0,port=1' \
-device 'VGA,id=vga,bus=pci.0,addr=0x2' \
- -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3' \
+ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \
-iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \
-machine 'type=pc+pve0'
/usr/bin/kvm \
-id 8006 \
- -name customcpu \
- -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server,nowait' \
+ -name 'customcpu,debug-threads=on' \
+ -no-shutdown \
+ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \
-mon 'chardev=qmp,mode=control' \
-chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' \
-mon 'chardev=qmp-event,mode=control' \
-smp '3,sockets=1,cores=3,maxcpus=3' \
-nodefaults \
-boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \
- -vnc unix:/var/run/qemu-server/8006.vnc,password \
- -no-hpet \
+ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \
-cpu 'athlon,+aes,+avx,enforce,hv_ipi,hv_relaxed,hv_reset,hv_runtime,hv_spinlocks=0x1fff,hv_stimer,hv_synic,hv_time,hv_vapic,hv_vendor_id=testvend,hv_vpindex,+kvm_pv_eoi,-kvm_pv_unhalt,vendor=AuthenticAMD,host-phys-bits=true' \
-m 512 \
-device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \
-device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3' \
-iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \
-rtc 'driftfix=slew,base=localtime' \
- -machine 'type=pc+pve0' \
+ -machine 'hpet=off,type=pc-i440fx-5.1+pve0' \
-global 'kvm-pit.lost_tick_policy=discard'
/usr/bin/kvm \
-id 8006 \
- -name customcpu \
- -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server,nowait' \
+ -name 'customcpu,debug-threads=on' \
+ -no-shutdown \
+ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \
-mon 'chardev=qmp,mode=control' \
-chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' \
-mon 'chardev=qmp-event,mode=control' \
-smp '3,sockets=1,cores=3,maxcpus=3' \
-nodefaults \
-boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \
- -vnc unix:/var/run/qemu-server/8006.vnc,password \
- -no-hpet \
+ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \
-cpu 'athlon,+aes,+avx,enforce,hv_ipi,hv_relaxed,hv_reset,hv_runtime,hv_spinlocks=0x1fff,hv_stimer,hv_synic,hv_time,hv_vapic,hv_vendor_id=testvend,hv_vpindex,+kvm_pv_eoi,-kvm_pv_unhalt,vendor=AuthenticAMD,+virt-ssbd,phys-bits=40' \
-m 512 \
-device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \
-device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3' \
-iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \
-rtc 'driftfix=slew,base=localtime' \
- -machine 'type=pc+pve0' \
+ -machine 'hpet=off,type=pc-i440fx-5.1+pve0' \
-global 'kvm-pit.lost_tick_policy=discard'
/usr/bin/kvm \
-id 8006 \
- -name vm8006 \
- -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server,nowait' \
+ -name 'vm8006,debug-threads=on' \
+ -no-shutdown \
+ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \
-mon 'chardev=qmp,mode=control' \
-chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' \
-mon 'chardev=qmp-event,mode=control' \
-pidfile /var/run/qemu-server/8006.pid \
-daemonize \
-smbios 'type=1,uuid=7b10d7af-b932-4c66-b2c3-3996152ec465' \
- -drive 'if=pflash,unit=0,format=raw,readonly,file=/usr/share/pve-edk2-firmware//OVMF_CODE.fd' \
- -drive 'if=pflash,unit=1,format=raw,id=drive-efidisk0,file=/var/lib/vz/images/100/vm-disk-100-0.raw' \
+ -drive 'if=pflash,unit=0,format=raw,readonly=on,file=/usr/share/pve-edk2-firmware//OVMF_CODE.fd' \
+ -drive 'if=pflash,unit=1,id=drive-efidisk0,format=raw,file=/var/lib/vz/images/100/vm-disk-100-0.raw' \
-smp '1,sockets=1,cores=1,maxcpus=1' \
-nodefaults \
-boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \
- -vnc unix:/var/run/qemu-server/8006.vnc,password \
+ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \
-cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \
-m 512 \
-device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \
--- /dev/null
+# TEST: Test raw efidisk size parameter
+smbios1: uuid=7b10d7af-b932-4c66-b2c3-3996152ec465
+bios: ovmf
+efidisk0: local:100/base-disk-100-0.raw
+template: 1
--- /dev/null
+/usr/bin/kvm \
+ -id 8006 \
+ -name 'vm8006,debug-threads=on' \
+ -no-shutdown \
+ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \
+ -mon 'chardev=qmp,mode=control' \
+ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' \
+ -mon 'chardev=qmp-event,mode=control' \
+ -pidfile /var/run/qemu-server/8006.pid \
+ -daemonize \
+ -smbios 'type=1,uuid=7b10d7af-b932-4c66-b2c3-3996152ec465' \
+ -drive 'if=pflash,unit=0,format=raw,readonly=on,file=/usr/share/pve-edk2-firmware//OVMF_CODE.fd' \
+ -drive 'if=pflash,unit=1,id=drive-efidisk0,format=raw,file=/var/lib/vz/images/100/base-disk-100-0.raw,size=131072,readonly=on' \
+ -smp '1,sockets=1,cores=1,maxcpus=1' \
+ -nodefaults \
+ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \
+ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \
+ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \
+ -m 512 \
+ -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \
+ -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \
+ -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \
+ -device 'usb-tablet,id=tablet,bus=uhci.0,port=1' \
+ -device 'VGA,id=vga,bus=pci.0,addr=0x2' \
+ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \
+ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \
+ -machine 'type=pc+pve0' \
+ -snapshot
/usr/bin/kvm \
-id 8006 \
- -name vm8006 \
- -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server,nowait' \
+ -name 'vm8006,debug-threads=on' \
+ -no-shutdown \
+ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \
-mon 'chardev=qmp,mode=control' \
-chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' \
-mon 'chardev=qmp-event,mode=control' \
-pidfile /var/run/qemu-server/8006.pid \
-daemonize \
-smbios 'type=1,uuid=7b10d7af-b932-4c66-b2c3-3996152ec465' \
- -drive 'if=pflash,unit=0,format=raw,readonly,file=/usr/share/pve-edk2-firmware//OVMF_CODE.fd' \
- -drive 'if=pflash,unit=1,format=raw,id=drive-efidisk0,size=131072,file=/var/lib/vz/images/100/vm-disk-100-0.raw' \
+ -drive 'if=pflash,unit=0,format=raw,readonly=on,file=/usr/share/pve-edk2-firmware//OVMF_CODE.fd' \
+ -drive 'if=pflash,unit=1,id=drive-efidisk0,format=raw,file=/var/lib/vz/images/100/vm-disk-100-0.raw,size=131072' \
-smp '1,sockets=1,cores=1,maxcpus=1' \
-nodefaults \
-boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \
- -vnc unix:/var/run/qemu-server/8006.vnc,password \
+ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \
-cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \
-m 512 \
-device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \
-device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \
-device 'usb-tablet,id=tablet,bus=uhci.0,port=1' \
-device 'VGA,id=vga,bus=pci.0,addr=0x2' \
- -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3' \
+ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \
-iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \
-machine 'type=pc+pve0'
--- /dev/null
+# TEST: Test newer 4MB efidisk with secureboot, smm enforce and a TPM device on Q35
+smbios1: uuid=7b10d7af-b932-4c66-b2c3-3996152ec465
+bios: ovmf
+machine: q35
+efidisk0: local:100/vm-disk-100-0.raw,efitype=4m,pre-enrolled-keys=1,size=528K
+tpmstate0: local:108/vm-100-disk-1.raw,size=4M,version=v2.0
--- /dev/null
+/usr/bin/kvm \
+ -id 8006 \
+ -name 'vm8006,debug-threads=on' \
+ -no-shutdown \
+ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \
+ -mon 'chardev=qmp,mode=control' \
+ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' \
+ -mon 'chardev=qmp-event,mode=control' \
+ -pidfile /var/run/qemu-server/8006.pid \
+ -daemonize \
+ -smbios 'type=1,uuid=7b10d7af-b932-4c66-b2c3-3996152ec465' \
+ -drive 'if=pflash,unit=0,format=raw,readonly=on,file=/usr/share/pve-edk2-firmware//OVMF_CODE_4M.secboot.fd' \
+ -drive 'if=pflash,unit=1,id=drive-efidisk0,format=raw,file=/var/lib/vz/images/100/vm-disk-100-0.raw,size=540672' \
+ -smp '1,sockets=1,cores=1,maxcpus=1' \
+ -nodefaults \
+ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \
+ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \
+ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \
+ -m 512 \
+ -readconfig /usr/share/qemu-server/pve-q35-4.0.cfg \
+ -device 'usb-tablet,id=tablet,bus=ehci.0,port=1' \
+ -chardev 'socket,id=tpmchar,path=/var/run/qemu-server/8006.swtpm' \
+ -tpmdev 'emulator,id=tpmdev,chardev=tpmchar' \
+ -device 'tpm-tis,tpmdev=tpmdev' \
+ -device 'VGA,id=vga,bus=pcie.0,addr=0x1' \
+ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \
+ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \
+ -machine 'type=q35+pve0'
--- /dev/null
+# TEST: Test newer 4MB efidisk with secureboot and a TPM device
+smbios1: uuid=7b10d7af-b932-4c66-b2c3-3996152ec465
+bios: ovmf
+efidisk0: local:100/vm-disk-100-0.raw,efitype=4m,pre-enrolled-keys=1,size=528K
+tpmstate0: local:108/vm-100-disk-1.raw,size=4M,version=v2.0
--- /dev/null
+/usr/bin/kvm \
+ -id 8006 \
+ -name 'vm8006,debug-threads=on' \
+ -no-shutdown \
+ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \
+ -mon 'chardev=qmp,mode=control' \
+ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' \
+ -mon 'chardev=qmp-event,mode=control' \
+ -pidfile /var/run/qemu-server/8006.pid \
+ -daemonize \
+ -smbios 'type=1,uuid=7b10d7af-b932-4c66-b2c3-3996152ec465' \
+ -drive 'if=pflash,unit=0,format=raw,readonly=on,file=/usr/share/pve-edk2-firmware//OVMF_CODE_4M.fd' \
+ -drive 'if=pflash,unit=1,id=drive-efidisk0,format=raw,file=/var/lib/vz/images/100/vm-disk-100-0.raw,size=540672' \
+ -smp '1,sockets=1,cores=1,maxcpus=1' \
+ -nodefaults \
+ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \
+ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \
+ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \
+ -m 512 \
+ -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \
+ -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \
+ -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \
+ -device 'usb-tablet,id=tablet,bus=uhci.0,port=1' \
+ -chardev 'socket,id=tpmchar,path=/var/run/qemu-server/8006.swtpm' \
+ -tpmdev 'emulator,id=tpmdev,chardev=tpmchar' \
+ -device 'tpm-tis,tpmdev=tpmdev' \
+ -device 'VGA,id=vga,bus=pci.0,addr=0x2' \
+ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \
+ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \
+ -machine 'type=pc+pve0'
--- /dev/null
+# TEST: Config with efi disk on RBD is very slow without cache - #3329
+bios: ovmf
+bootdisk: scsi0
+cores: 1
+efidisk0: rbd-store:vm-100-disk-1,size=128K
+memory: 512
+net0: virtio=2E:01:68:F9:9C:87,bridge=vmbr0
+numa: 1
+ostype: l26
+smbios1: uuid=3dd750ce-d910-44d0-9493-525c0be4e688
+vmgenid: 54d1c06c-8f5b-440f-b5b2-6eab1380e13a
--- /dev/null
+/usr/bin/kvm \
+ -id 8006 \
+ -name 'vm8006,debug-threads=on' \
+ -no-shutdown \
+ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \
+ -mon 'chardev=qmp,mode=control' \
+ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' \
+ -mon 'chardev=qmp-event,mode=control' \
+ -pidfile /var/run/qemu-server/8006.pid \
+ -daemonize \
+ -smbios 'type=1,uuid=3dd750ce-d910-44d0-9493-525c0be4e688' \
+ -drive 'if=pflash,unit=0,format=raw,readonly=on,file=/usr/share/pve-edk2-firmware//OVMF_CODE.fd' \
+ -drive 'if=pflash,unit=1,id=drive-efidisk0,cache=writeback,format=raw,file=rbd:cpool/vm-100-disk-1:mon_host=127.0.0.42;127.0.0.21;[\:\:1]:auth_supported=none:rbd_cache_policy=writeback,size=131072' \
+ -smp '1,sockets=1,cores=1,maxcpus=1' \
+ -nodefaults \
+ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \
+ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \
+ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \
+ -m 512 \
+ -object 'memory-backend-ram,id=ram-node0,size=512M' \
+ -numa 'node,nodeid=0,cpus=0,memdev=ram-node0' \
+ -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \
+ -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \
+ -device 'vmgenid,guid=54d1c06c-8f5b-440f-b5b2-6eab1380e13a' \
+ -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \
+ -device 'usb-tablet,id=tablet,bus=uhci.0,port=1' \
+ -device 'VGA,id=vga,bus=pci.0,addr=0x2' \
+ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \
+ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \
+ -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/var/lib/qemu-server/pve-bridge,downscript=/var/lib/qemu-server/pve-bridgedown,vhost=on' \
+ -device 'virtio-net-pci,mac=2E:01:68:F9:9C:87,netdev=net0,bus=pci.0,addr=0x12,id=net0,rx_queue_size=1024,tx_queue_size=256,bootindex=300' \
+ -machine 'type=pc+pve0'
--- /dev/null
+# EXPECT_ERROR: to use Intel vIOMMU please set the machine type to q35
+machine: pc,viommu=intel
--- /dev/null
+machine: pc,viommu=virtio
--- /dev/null
+/usr/bin/kvm \
+ -id 8006 \
+ -name 'vm8006,debug-threads=on' \
+ -no-shutdown \
+ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \
+ -mon 'chardev=qmp,mode=control' \
+ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' \
+ -mon 'chardev=qmp-event,mode=control' \
+ -pidfile /var/run/qemu-server/8006.pid \
+ -daemonize \
+ -smp '1,sockets=1,cores=1,maxcpus=1' \
+ -nodefaults \
+ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \
+ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \
+ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \
+ -m 512 \
+ -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \
+ -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \
+ -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \
+ -device 'usb-tablet,id=tablet,bus=uhci.0,port=1' \
+ -device 'VGA,id=vga,bus=pci.0,addr=0x2' \
+ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \
+ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \
+ -device virtio-iommu-pci \
+ -machine 'type=pc+pve0'
/usr/bin/kvm \
-id 8006 \
- -name vm8006 \
- -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server,nowait' \
+ -name 'vm8006,debug-threads=on' \
+ -no-shutdown \
+ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \
-mon 'chardev=qmp,mode=control' \
-chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' \
-mon 'chardev=qmp-event,mode=control' \
-pidfile /var/run/qemu-server/8006.pid \
-daemonize \
-smbios 'type=1,uuid=3dd750ce-d910-44d0-9493-525c0be4e687' \
- -drive 'if=pflash,unit=0,format=raw,readonly,file=/usr/share/pve-edk2-firmware//OVMF_CODE.fd' \
- -drive 'if=pflash,unit=1,format=qcow2,id=drive-efidisk0,file=/var/lib/vz/images/100/vm-100-disk-1.qcow2' \
+ -drive 'if=pflash,unit=0,format=raw,readonly=on,file=/usr/share/pve-edk2-firmware//OVMF_CODE.fd' \
+ -drive 'if=pflash,unit=1,id=drive-efidisk0,format=qcow2,file=/var/lib/vz/images/100/vm-100-disk-1.qcow2' \
-smp '2,sockets=2,cores=1,maxcpus=2' \
-nodefaults \
-boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \
- -vnc unix:/var/run/qemu-server/8006.vnc,password \
- -no-hpet \
+ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \
-cpu 'kvm64,enforce,hv_ipi,hv_relaxed,hv_reset,hv_runtime,hv_spinlocks=0x1fff,hv_stimer,hv_synic,hv_time,hv_vapic,hv_vpindex,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep' \
-m 512 \
-object 'memory-backend-ram,id=ram-node0,size=256M' \
-netdev 'type=tap,id=net0,ifname=tap8006i0,script=/var/lib/qemu-server/pve-bridge,downscript=/var/lib/qemu-server/pve-bridgedown,vhost=on' \
-device 'virtio-net-pci,mac=2E:01:68:F9:9C:87,netdev=net0,bus=pci.0,addr=0x12,id=net0,bootindex=300' \
-rtc 'driftfix=slew,base=localtime' \
- -machine 'type=pc+pve0' \
+ -machine 'hpet=off,type=pc-i440fx-5.1+pve0' \
-global 'kvm-pit.lost_tick_policy=discard'
--- /dev/null
+# TEST: Config with default machine type, Linux & four IDE CD-ROMs
+bootdisk: scsi0
+cores: 2
+ide0: cifs-store:iso/zero.iso,media=cdrom,size=112M
+ide1: cifs-store:iso/one.iso,media=cdrom,size=112M
+ide2: cifs-store:iso/two.iso,media=cdrom,size=112M
+ide3: cifs-store:iso/three.iso,media=cdrom,size=112M
+memory: 512
+net0: virtio=2E:01:68:F9:9C:87,bridge=vmbr0
+ostype: l26
+scsi0: local:100/vm-100-disk-2.qcow2,size=10G
+scsihw: virtio-scsi-pci
+smbios1: uuid=3dd750ce-d910-44d0-9493-525c0be4e687
+vmgenid: 54d1c06c-8f5b-440f-b5b2-6eab1380e13d
--- /dev/null
+/usr/bin/kvm \
+ -id 8006 \
+ -name 'vm8006,debug-threads=on' \
+ -no-shutdown \
+ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \
+ -mon 'chardev=qmp,mode=control' \
+ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' \
+ -mon 'chardev=qmp-event,mode=control' \
+ -pidfile /var/run/qemu-server/8006.pid \
+ -daemonize \
+ -smbios 'type=1,uuid=3dd750ce-d910-44d0-9493-525c0be4e687' \
+ -smp '2,sockets=1,cores=2,maxcpus=2' \
+ -nodefaults \
+ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \
+ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \
+ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \
+ -m 512 \
+ -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \
+ -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \
+ -device 'vmgenid,guid=54d1c06c-8f5b-440f-b5b2-6eab1380e13d' \
+ -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \
+ -device 'usb-tablet,id=tablet,bus=uhci.0,port=1' \
+ -device 'VGA,id=vga,bus=pci.0,addr=0x2' \
+ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \
+ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \
+ -drive 'file=/mnt/pve/cifs-store/template/iso/zero.iso,if=none,id=drive-ide0,media=cdrom,aio=threads' \
+ -device 'ide-cd,bus=ide.0,unit=0,drive=drive-ide0,id=ide0,bootindex=200' \
+ -drive 'file=/mnt/pve/cifs-store/template/iso/one.iso,if=none,id=drive-ide1,media=cdrom,aio=threads' \
+ -device 'ide-cd,bus=ide.0,unit=1,drive=drive-ide1,id=ide1,bootindex=201' \
+ -drive 'file=/mnt/pve/cifs-store/template/iso/two.iso,if=none,id=drive-ide2,media=cdrom,aio=threads' \
+ -device 'ide-cd,bus=ide.1,unit=0,drive=drive-ide2,id=ide2,bootindex=202' \
+ -drive 'file=/mnt/pve/cifs-store/template/iso/three.iso,if=none,id=drive-ide3,media=cdrom,aio=threads' \
+ -device 'ide-cd,bus=ide.1,unit=1,drive=drive-ide3,id=ide3,bootindex=203' \
+ -device 'virtio-scsi-pci,id=scsihw0,bus=pci.0,addr=0x5' \
+ -drive 'file=/var/lib/vz/images/100/vm-100-disk-2.qcow2,if=none,id=drive-scsi0,format=qcow2,cache=none,aio=io_uring,detect-zeroes=on' \
+ -device 'scsi-hd,bus=scsihw0.0,channel=0,scsi-id=0,lun=0,drive=drive-scsi0,id=scsi0,bootindex=100' \
+ -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/var/lib/qemu-server/pve-bridge,downscript=/var/lib/qemu-server/pve-bridgedown,vhost=on' \
+ -device 'virtio-net-pci,mac=2E:01:68:F9:9C:87,netdev=net0,bus=pci.0,addr=0x12,id=net0,rx_queue_size=1024,tx_queue_size=256,bootindex=300' \
+ -machine 'type=pc+pve0'
--- /dev/null
+# TEST: memory hotplug with 1GB hugepage
+# QEMU_VERSION: 3.0
+cores: 2
+memory: 18432
+name: simple
+numa: 1
+ostype: l26
+smbios1: uuid=7b10d7af-b932-4c66-b2c3-3996152ec465
+sockets: 2
+vmgenid: c773c261-d800-4348-9f5d-167fadd53cf8
+hotplug: memory
+hugepages: 1024
\ No newline at end of file
--- /dev/null
+/usr/bin/kvm \
+ -id 8006 \
+ -name 'simple,debug-threads=on' \
+ -no-shutdown \
+ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \
+ -mon 'chardev=qmp,mode=control' \
+ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' \
+ -mon 'chardev=qmp-event,mode=control' \
+ -pidfile /var/run/qemu-server/8006.pid \
+ -daemonize \
+ -smbios 'type=1,uuid=7b10d7af-b932-4c66-b2c3-3996152ec465' \
+ -smp '4,sockets=2,cores=2,maxcpus=4' \
+ -nodefaults \
+ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \
+ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \
+ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \
+ -m 'size=2048,slots=255,maxmem=524288M' \
+ -object 'memory-backend-file,id=ram-node0,size=1024M,mem-path=/run/hugepages/kvm/1048576kB,share=on,prealloc=yes' \
+ -numa 'node,nodeid=0,cpus=0-1,memdev=ram-node0' \
+ -object 'memory-backend-file,id=ram-node1,size=1024M,mem-path=/run/hugepages/kvm/1048576kB,share=on,prealloc=yes' \
+ -numa 'node,nodeid=1,cpus=2-3,memdev=ram-node1' \
+ -object 'memory-backend-file,id=mem-dimm0,size=1024M,mem-path=/run/hugepages/kvm/1048576kB,share=on,prealloc=yes' \
+ -device 'pc-dimm,id=dimm0,memdev=mem-dimm0,node=0' \
+ -object 'memory-backend-file,id=mem-dimm1,size=1024M,mem-path=/run/hugepages/kvm/1048576kB,share=on,prealloc=yes' \
+ -device 'pc-dimm,id=dimm1,memdev=mem-dimm1,node=1' \
+ -object 'memory-backend-file,id=mem-dimm2,size=1024M,mem-path=/run/hugepages/kvm/1048576kB,share=on,prealloc=yes' \
+ -device 'pc-dimm,id=dimm2,memdev=mem-dimm2,node=0' \
+ -object 'memory-backend-file,id=mem-dimm3,size=1024M,mem-path=/run/hugepages/kvm/1048576kB,share=on,prealloc=yes' \
+ -device 'pc-dimm,id=dimm3,memdev=mem-dimm3,node=1' \
+ -object 'memory-backend-file,id=mem-dimm4,size=1024M,mem-path=/run/hugepages/kvm/1048576kB,share=on,prealloc=yes' \
+ -device 'pc-dimm,id=dimm4,memdev=mem-dimm4,node=0' \
+ -object 'memory-backend-file,id=mem-dimm5,size=1024M,mem-path=/run/hugepages/kvm/1048576kB,share=on,prealloc=yes' \
+ -device 'pc-dimm,id=dimm5,memdev=mem-dimm5,node=1' \
+ -object 'memory-backend-file,id=mem-dimm6,size=1024M,mem-path=/run/hugepages/kvm/1048576kB,share=on,prealloc=yes' \
+ -device 'pc-dimm,id=dimm6,memdev=mem-dimm6,node=0' \
+ -object 'memory-backend-file,id=mem-dimm7,size=1024M,mem-path=/run/hugepages/kvm/1048576kB,share=on,prealloc=yes' \
+ -device 'pc-dimm,id=dimm7,memdev=mem-dimm7,node=1' \
+ -object 'memory-backend-file,id=mem-dimm8,size=1024M,mem-path=/run/hugepages/kvm/1048576kB,share=on,prealloc=yes' \
+ -device 'pc-dimm,id=dimm8,memdev=mem-dimm8,node=0' \
+ -object 'memory-backend-file,id=mem-dimm9,size=1024M,mem-path=/run/hugepages/kvm/1048576kB,share=on,prealloc=yes' \
+ -device 'pc-dimm,id=dimm9,memdev=mem-dimm9,node=1' \
+ -object 'memory-backend-file,id=mem-dimm10,size=1024M,mem-path=/run/hugepages/kvm/1048576kB,share=on,prealloc=yes' \
+ -device 'pc-dimm,id=dimm10,memdev=mem-dimm10,node=0' \
+ -object 'memory-backend-file,id=mem-dimm11,size=1024M,mem-path=/run/hugepages/kvm/1048576kB,share=on,prealloc=yes' \
+ -device 'pc-dimm,id=dimm11,memdev=mem-dimm11,node=1' \
+ -object 'memory-backend-file,id=mem-dimm12,size=1024M,mem-path=/run/hugepages/kvm/1048576kB,share=on,prealloc=yes' \
+ -device 'pc-dimm,id=dimm12,memdev=mem-dimm12,node=0' \
+ -object 'memory-backend-file,id=mem-dimm13,size=1024M,mem-path=/run/hugepages/kvm/1048576kB,share=on,prealloc=yes' \
+ -device 'pc-dimm,id=dimm13,memdev=mem-dimm13,node=1' \
+ -object 'memory-backend-file,id=mem-dimm14,size=1024M,mem-path=/run/hugepages/kvm/1048576kB,share=on,prealloc=yes' \
+ -device 'pc-dimm,id=dimm14,memdev=mem-dimm14,node=0' \
+ -object 'memory-backend-file,id=mem-dimm15,size=1024M,mem-path=/run/hugepages/kvm/1048576kB,share=on,prealloc=yes' \
+ -device 'pc-dimm,id=dimm15,memdev=mem-dimm15,node=1' \
+ -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \
+ -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \
+ -device 'vmgenid,guid=c773c261-d800-4348-9f5d-167fadd53cf8' \
+ -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \
+ -device 'usb-tablet,id=tablet,bus=uhci.0,port=1' \
+ -device 'VGA,id=vga,bus=pci.0,addr=0x2' \
+ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3' \
+ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \
+ -machine 'type=pc'
--- /dev/null
+# TEST: basic memory hotplug
+# QEMU_VERSION: 3.0
+cores: 2
+memory: 66560
+name: simple
+numa: 1
+ostype: l26
+smbios1: uuid=7b10d7af-b932-4c66-b2c3-3996152ec465
+sockets: 2
+vmgenid: c773c261-d800-4348-9f5d-167fadd53cf8
+hotplug: memory
--- /dev/null
+/usr/bin/kvm \
+ -id 8006 \
+ -name 'simple,debug-threads=on' \
+ -no-shutdown \
+ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \
+ -mon 'chardev=qmp,mode=control' \
+ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' \
+ -mon 'chardev=qmp-event,mode=control' \
+ -pidfile /var/run/qemu-server/8006.pid \
+ -daemonize \
+ -smbios 'type=1,uuid=7b10d7af-b932-4c66-b2c3-3996152ec465' \
+ -smp '4,sockets=2,cores=2,maxcpus=4' \
+ -nodefaults \
+ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \
+ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \
+ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \
+ -m 'size=1024,slots=255,maxmem=524288M' \
+ -object 'memory-backend-ram,id=ram-node0,size=512M' \
+ -numa 'node,nodeid=0,cpus=0-1,memdev=ram-node0' \
+ -object 'memory-backend-ram,id=ram-node1,size=512M' \
+ -numa 'node,nodeid=1,cpus=2-3,memdev=ram-node1' \
+ -object 'memory-backend-ram,id=mem-dimm0,size=512M' \
+ -device 'pc-dimm,id=dimm0,memdev=mem-dimm0,node=0' \
+ -object 'memory-backend-ram,id=mem-dimm1,size=512M' \
+ -device 'pc-dimm,id=dimm1,memdev=mem-dimm1,node=1' \
+ -object 'memory-backend-ram,id=mem-dimm2,size=512M' \
+ -device 'pc-dimm,id=dimm2,memdev=mem-dimm2,node=0' \
+ -object 'memory-backend-ram,id=mem-dimm3,size=512M' \
+ -device 'pc-dimm,id=dimm3,memdev=mem-dimm3,node=1' \
+ -object 'memory-backend-ram,id=mem-dimm4,size=512M' \
+ -device 'pc-dimm,id=dimm4,memdev=mem-dimm4,node=0' \
+ -object 'memory-backend-ram,id=mem-dimm5,size=512M' \
+ -device 'pc-dimm,id=dimm5,memdev=mem-dimm5,node=1' \
+ -object 'memory-backend-ram,id=mem-dimm6,size=512M' \
+ -device 'pc-dimm,id=dimm6,memdev=mem-dimm6,node=0' \
+ -object 'memory-backend-ram,id=mem-dimm7,size=512M' \
+ -device 'pc-dimm,id=dimm7,memdev=mem-dimm7,node=1' \
+ -object 'memory-backend-ram,id=mem-dimm8,size=512M' \
+ -device 'pc-dimm,id=dimm8,memdev=mem-dimm8,node=0' \
+ -object 'memory-backend-ram,id=mem-dimm9,size=512M' \
+ -device 'pc-dimm,id=dimm9,memdev=mem-dimm9,node=1' \
+ -object 'memory-backend-ram,id=mem-dimm10,size=512M' \
+ -device 'pc-dimm,id=dimm10,memdev=mem-dimm10,node=0' \
+ -object 'memory-backend-ram,id=mem-dimm11,size=512M' \
+ -device 'pc-dimm,id=dimm11,memdev=mem-dimm11,node=1' \
+ -object 'memory-backend-ram,id=mem-dimm12,size=512M' \
+ -device 'pc-dimm,id=dimm12,memdev=mem-dimm12,node=0' \
+ -object 'memory-backend-ram,id=mem-dimm13,size=512M' \
+ -device 'pc-dimm,id=dimm13,memdev=mem-dimm13,node=1' \
+ -object 'memory-backend-ram,id=mem-dimm14,size=512M' \
+ -device 'pc-dimm,id=dimm14,memdev=mem-dimm14,node=0' \
+ -object 'memory-backend-ram,id=mem-dimm15,size=512M' \
+ -device 'pc-dimm,id=dimm15,memdev=mem-dimm15,node=1' \
+ -object 'memory-backend-ram,id=mem-dimm16,size=512M' \
+ -device 'pc-dimm,id=dimm16,memdev=mem-dimm16,node=0' \
+ -object 'memory-backend-ram,id=mem-dimm17,size=512M' \
+ -device 'pc-dimm,id=dimm17,memdev=mem-dimm17,node=1' \
+ -object 'memory-backend-ram,id=mem-dimm18,size=512M' \
+ -device 'pc-dimm,id=dimm18,memdev=mem-dimm18,node=0' \
+ -object 'memory-backend-ram,id=mem-dimm19,size=512M' \
+ -device 'pc-dimm,id=dimm19,memdev=mem-dimm19,node=1' \
+ -object 'memory-backend-ram,id=mem-dimm20,size=512M' \
+ -device 'pc-dimm,id=dimm20,memdev=mem-dimm20,node=0' \
+ -object 'memory-backend-ram,id=mem-dimm21,size=512M' \
+ -device 'pc-dimm,id=dimm21,memdev=mem-dimm21,node=1' \
+ -object 'memory-backend-ram,id=mem-dimm22,size=512M' \
+ -device 'pc-dimm,id=dimm22,memdev=mem-dimm22,node=0' \
+ -object 'memory-backend-ram,id=mem-dimm23,size=512M' \
+ -device 'pc-dimm,id=dimm23,memdev=mem-dimm23,node=1' \
+ -object 'memory-backend-ram,id=mem-dimm24,size=512M' \
+ -device 'pc-dimm,id=dimm24,memdev=mem-dimm24,node=0' \
+ -object 'memory-backend-ram,id=mem-dimm25,size=512M' \
+ -device 'pc-dimm,id=dimm25,memdev=mem-dimm25,node=1' \
+ -object 'memory-backend-ram,id=mem-dimm26,size=512M' \
+ -device 'pc-dimm,id=dimm26,memdev=mem-dimm26,node=0' \
+ -object 'memory-backend-ram,id=mem-dimm27,size=512M' \
+ -device 'pc-dimm,id=dimm27,memdev=mem-dimm27,node=1' \
+ -object 'memory-backend-ram,id=mem-dimm28,size=512M' \
+ -device 'pc-dimm,id=dimm28,memdev=mem-dimm28,node=0' \
+ -object 'memory-backend-ram,id=mem-dimm29,size=512M' \
+ -device 'pc-dimm,id=dimm29,memdev=mem-dimm29,node=1' \
+ -object 'memory-backend-ram,id=mem-dimm30,size=512M' \
+ -device 'pc-dimm,id=dimm30,memdev=mem-dimm30,node=0' \
+ -object 'memory-backend-ram,id=mem-dimm31,size=512M' \
+ -device 'pc-dimm,id=dimm31,memdev=mem-dimm31,node=1' \
+ -object 'memory-backend-ram,id=mem-dimm32,size=1024M' \
+ -device 'pc-dimm,id=dimm32,memdev=mem-dimm32,node=0' \
+ -object 'memory-backend-ram,id=mem-dimm33,size=1024M' \
+ -device 'pc-dimm,id=dimm33,memdev=mem-dimm33,node=1' \
+ -object 'memory-backend-ram,id=mem-dimm34,size=1024M' \
+ -device 'pc-dimm,id=dimm34,memdev=mem-dimm34,node=0' \
+ -object 'memory-backend-ram,id=mem-dimm35,size=1024M' \
+ -device 'pc-dimm,id=dimm35,memdev=mem-dimm35,node=1' \
+ -object 'memory-backend-ram,id=mem-dimm36,size=1024M' \
+ -device 'pc-dimm,id=dimm36,memdev=mem-dimm36,node=0' \
+ -object 'memory-backend-ram,id=mem-dimm37,size=1024M' \
+ -device 'pc-dimm,id=dimm37,memdev=mem-dimm37,node=1' \
+ -object 'memory-backend-ram,id=mem-dimm38,size=1024M' \
+ -device 'pc-dimm,id=dimm38,memdev=mem-dimm38,node=0' \
+ -object 'memory-backend-ram,id=mem-dimm39,size=1024M' \
+ -device 'pc-dimm,id=dimm39,memdev=mem-dimm39,node=1' \
+ -object 'memory-backend-ram,id=mem-dimm40,size=1024M' \
+ -device 'pc-dimm,id=dimm40,memdev=mem-dimm40,node=0' \
+ -object 'memory-backend-ram,id=mem-dimm41,size=1024M' \
+ -device 'pc-dimm,id=dimm41,memdev=mem-dimm41,node=1' \
+ -object 'memory-backend-ram,id=mem-dimm42,size=1024M' \
+ -device 'pc-dimm,id=dimm42,memdev=mem-dimm42,node=0' \
+ -object 'memory-backend-ram,id=mem-dimm43,size=1024M' \
+ -device 'pc-dimm,id=dimm43,memdev=mem-dimm43,node=1' \
+ -object 'memory-backend-ram,id=mem-dimm44,size=1024M' \
+ -device 'pc-dimm,id=dimm44,memdev=mem-dimm44,node=0' \
+ -object 'memory-backend-ram,id=mem-dimm45,size=1024M' \
+ -device 'pc-dimm,id=dimm45,memdev=mem-dimm45,node=1' \
+ -object 'memory-backend-ram,id=mem-dimm46,size=1024M' \
+ -device 'pc-dimm,id=dimm46,memdev=mem-dimm46,node=0' \
+ -object 'memory-backend-ram,id=mem-dimm47,size=1024M' \
+ -device 'pc-dimm,id=dimm47,memdev=mem-dimm47,node=1' \
+ -object 'memory-backend-ram,id=mem-dimm48,size=1024M' \
+ -device 'pc-dimm,id=dimm48,memdev=mem-dimm48,node=0' \
+ -object 'memory-backend-ram,id=mem-dimm49,size=1024M' \
+ -device 'pc-dimm,id=dimm49,memdev=mem-dimm49,node=1' \
+ -object 'memory-backend-ram,id=mem-dimm50,size=1024M' \
+ -device 'pc-dimm,id=dimm50,memdev=mem-dimm50,node=0' \
+ -object 'memory-backend-ram,id=mem-dimm51,size=1024M' \
+ -device 'pc-dimm,id=dimm51,memdev=mem-dimm51,node=1' \
+ -object 'memory-backend-ram,id=mem-dimm52,size=1024M' \
+ -device 'pc-dimm,id=dimm52,memdev=mem-dimm52,node=0' \
+ -object 'memory-backend-ram,id=mem-dimm53,size=1024M' \
+ -device 'pc-dimm,id=dimm53,memdev=mem-dimm53,node=1' \
+ -object 'memory-backend-ram,id=mem-dimm54,size=1024M' \
+ -device 'pc-dimm,id=dimm54,memdev=mem-dimm54,node=0' \
+ -object 'memory-backend-ram,id=mem-dimm55,size=1024M' \
+ -device 'pc-dimm,id=dimm55,memdev=mem-dimm55,node=1' \
+ -object 'memory-backend-ram,id=mem-dimm56,size=1024M' \
+ -device 'pc-dimm,id=dimm56,memdev=mem-dimm56,node=0' \
+ -object 'memory-backend-ram,id=mem-dimm57,size=1024M' \
+ -device 'pc-dimm,id=dimm57,memdev=mem-dimm57,node=1' \
+ -object 'memory-backend-ram,id=mem-dimm58,size=1024M' \
+ -device 'pc-dimm,id=dimm58,memdev=mem-dimm58,node=0' \
+ -object 'memory-backend-ram,id=mem-dimm59,size=1024M' \
+ -device 'pc-dimm,id=dimm59,memdev=mem-dimm59,node=1' \
+ -object 'memory-backend-ram,id=mem-dimm60,size=1024M' \
+ -device 'pc-dimm,id=dimm60,memdev=mem-dimm60,node=0' \
+ -object 'memory-backend-ram,id=mem-dimm61,size=1024M' \
+ -device 'pc-dimm,id=dimm61,memdev=mem-dimm61,node=1' \
+ -object 'memory-backend-ram,id=mem-dimm62,size=1024M' \
+ -device 'pc-dimm,id=dimm62,memdev=mem-dimm62,node=0' \
+ -object 'memory-backend-ram,id=mem-dimm63,size=1024M' \
+ -device 'pc-dimm,id=dimm63,memdev=mem-dimm63,node=1' \
+ -object 'memory-backend-ram,id=mem-dimm64,size=2048M' \
+ -device 'pc-dimm,id=dimm64,memdev=mem-dimm64,node=0' \
+ -object 'memory-backend-ram,id=mem-dimm65,size=2048M' \
+ -device 'pc-dimm,id=dimm65,memdev=mem-dimm65,node=1' \
+ -object 'memory-backend-ram,id=mem-dimm66,size=2048M' \
+ -device 'pc-dimm,id=dimm66,memdev=mem-dimm66,node=0' \
+ -object 'memory-backend-ram,id=mem-dimm67,size=2048M' \
+ -device 'pc-dimm,id=dimm67,memdev=mem-dimm67,node=1' \
+ -object 'memory-backend-ram,id=mem-dimm68,size=2048M' \
+ -device 'pc-dimm,id=dimm68,memdev=mem-dimm68,node=0' \
+ -object 'memory-backend-ram,id=mem-dimm69,size=2048M' \
+ -device 'pc-dimm,id=dimm69,memdev=mem-dimm69,node=1' \
+ -object 'memory-backend-ram,id=mem-dimm70,size=2048M' \
+ -device 'pc-dimm,id=dimm70,memdev=mem-dimm70,node=0' \
+ -object 'memory-backend-ram,id=mem-dimm71,size=2048M' \
+ -device 'pc-dimm,id=dimm71,memdev=mem-dimm71,node=1' \
+ -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \
+ -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \
+ -device 'vmgenid,guid=c773c261-d800-4348-9f5d-167fadd53cf8' \
+ -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \
+ -device 'usb-tablet,id=tablet,bus=uhci.0,port=1' \
+ -device 'VGA,id=vga,bus=pci.0,addr=0x2' \
+ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3' \
+ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \
+ -machine 'type=pc'
--- /dev/null
+# TEST: memory wih 1gb hugepages
+# QEMU_VERSION: 3.0
+cores: 2
+memory: 8192
+name: simple
+numa: 1
+ostype: l26
+smbios1: uuid=7b10d7af-b932-4c66-b2c3-3996152ec465
+sockets: 2
+vmgenid: c773c261-d800-4348-9f5d-167fadd53cf8
+hugepages: 1024
\ No newline at end of file
--- /dev/null
+/usr/bin/kvm \
+ -id 8006 \
+ -name 'simple,debug-threads=on' \
+ -no-shutdown \
+ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \
+ -mon 'chardev=qmp,mode=control' \
+ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' \
+ -mon 'chardev=qmp-event,mode=control' \
+ -pidfile /var/run/qemu-server/8006.pid \
+ -daemonize \
+ -smbios 'type=1,uuid=7b10d7af-b932-4c66-b2c3-3996152ec465' \
+ -smp '4,sockets=2,cores=2,maxcpus=4' \
+ -nodefaults \
+ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \
+ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \
+ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \
+ -m 8192 \
+ -object 'memory-backend-file,id=ram-node0,size=4096M,mem-path=/run/hugepages/kvm/1048576kB,share=on,prealloc=yes' \
+ -numa 'node,nodeid=0,cpus=0-1,memdev=ram-node0' \
+ -object 'memory-backend-file,id=ram-node1,size=4096M,mem-path=/run/hugepages/kvm/1048576kB,share=on,prealloc=yes' \
+ -numa 'node,nodeid=1,cpus=2-3,memdev=ram-node1' \
+ -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \
+ -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \
+ -device 'vmgenid,guid=c773c261-d800-4348-9f5d-167fadd53cf8' \
+ -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \
+ -device 'usb-tablet,id=tablet,bus=uhci.0,port=1' \
+ -device 'VGA,id=vga,bus=pci.0,addr=0x2' \
+ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3' \
+ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \
+ -machine 'type=pc'
--- /dev/null
+# TEST: memory wih 2mb hugepages
+# QEMU_VERSION: 3.0
+cores: 2
+memory: 8192
+name: simple
+numa: 1
+ostype: l26
+smbios1: uuid=7b10d7af-b932-4c66-b2c3-3996152ec465
+sockets: 2
+vmgenid: c773c261-d800-4348-9f5d-167fadd53cf8
+hugepages: 2
\ No newline at end of file
--- /dev/null
+/usr/bin/kvm \
+ -id 8006 \
+ -name 'simple,debug-threads=on' \
+ -no-shutdown \
+ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \
+ -mon 'chardev=qmp,mode=control' \
+ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' \
+ -mon 'chardev=qmp-event,mode=control' \
+ -pidfile /var/run/qemu-server/8006.pid \
+ -daemonize \
+ -smbios 'type=1,uuid=7b10d7af-b932-4c66-b2c3-3996152ec465' \
+ -smp '4,sockets=2,cores=2,maxcpus=4' \
+ -nodefaults \
+ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \
+ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \
+ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \
+ -m 8192 \
+ -object 'memory-backend-file,id=ram-node0,size=4096M,mem-path=/run/hugepages/kvm/2048kB,share=on,prealloc=yes' \
+ -numa 'node,nodeid=0,cpus=0-1,memdev=ram-node0' \
+ -object 'memory-backend-file,id=ram-node1,size=4096M,mem-path=/run/hugepages/kvm/2048kB,share=on,prealloc=yes' \
+ -numa 'node,nodeid=1,cpus=2-3,memdev=ram-node1' \
+ -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \
+ -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \
+ -device 'vmgenid,guid=c773c261-d800-4348-9f5d-167fadd53cf8' \
+ -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \
+ -device 'usb-tablet,id=tablet,bus=uhci.0,port=1' \
+ -device 'VGA,id=vga,bus=pci.0,addr=0x2' \
+ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3' \
+ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \
+ -machine 'type=pc'
/usr/bin/kvm \
-id 8006 \
- -name vm8006 \
- -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server,nowait' \
+ -name 'vm8006,debug-threads=on' \
+ -no-shutdown \
+ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \
-mon 'chardev=qmp,mode=control' \
-chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' \
-mon 'chardev=qmp-event,mode=control' \
-smp '1,sockets=1,cores=1,maxcpus=1' \
-nodefaults \
-boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \
- -vnc unix:/var/run/qemu-server/8006.vnc,password \
+ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \
-cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \
-m 512 \
-device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \
-device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \
-device 'usb-tablet,id=tablet,bus=uhci.0,port=1' \
-device 'VGA,id=vga,bus=pci.0,addr=0x2' \
- -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3' \
+ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \
-iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \
-machine 'type=pc+pve0'
--- /dev/null
+# TEST: Simple test for netdev multi queue on 7.0 machine version
+# QEMU_VERSION: 7.0
+bootdisk: scsi0
+cores: 3
+memory: 768
+name: netdev-multiq
+net0: virtio=A2:C0:43:77:08:A0,bridge=vmbr0,mtu=900,queues=2
+ostype: l26
+smbios1: uuid=7b10d7af-b932-4c66-b2c3-3996152ec465
--- /dev/null
+/usr/bin/kvm \
+ -id 8006 \
+ -name 'netdev-multiq,debug-threads=on' \
+ -no-shutdown \
+ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \
+ -mon 'chardev=qmp,mode=control' \
+ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' \
+ -mon 'chardev=qmp-event,mode=control' \
+ -pidfile /var/run/qemu-server/8006.pid \
+ -daemonize \
+ -smbios 'type=1,uuid=7b10d7af-b932-4c66-b2c3-3996152ec465' \
+ -smp '3,sockets=1,cores=3,maxcpus=3' \
+ -nodefaults \
+ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \
+ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \
+ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \
+ -m 768 \
+ -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \
+ -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \
+ -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \
+ -device 'usb-tablet,id=tablet,bus=uhci.0,port=1' \
+ -device 'VGA,id=vga,bus=pci.0,addr=0x2' \
+ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \
+ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \
+ -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/var/lib/qemu-server/pve-bridge,downscript=/var/lib/qemu-server/pve-bridgedown,vhost=on,queues=2' \
+ -device 'virtio-net-pci,mac=A2:C0:43:77:08:A0,netdev=net0,bus=pci.0,addr=0x12,id=net0,vectors=6,mq=on,bootindex=300,host_mtu=900' \
+ -machine 'type=pc+pve0'
--- /dev/null
+# TEST: Simple test for netdev related stuff
+# QEMU_VERSION: 7.1
+bootdisk: scsi0
+cores: 3
+memory: 768
+name: netdev
+net0: virtio=A2:C0:43:77:08:A0,bridge=vmbr0,mtu=900,queues=2
+ostype: l26
--- /dev/null
+/usr/bin/kvm \
+ -id 8006 \
+ -name 'netdev,debug-threads=on' \
+ -no-shutdown \
+ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \
+ -mon 'chardev=qmp,mode=control' \
+ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' \
+ -mon 'chardev=qmp-event,mode=control' \
+ -pidfile /var/run/qemu-server/8006.pid \
+ -daemonize \
+ -smp '3,sockets=1,cores=3,maxcpus=3' \
+ -nodefaults \
+ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \
+ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \
+ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \
+ -m 768 \
+ -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \
+ -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \
+ -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \
+ -device 'usb-tablet,id=tablet,bus=uhci.0,port=1' \
+ -device 'VGA,id=vga,bus=pci.0,addr=0x2' \
+ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \
+ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \
+ -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/var/lib/qemu-server/pve-bridge,downscript=/var/lib/qemu-server/pve-bridgedown,vhost=on,queues=2' \
+ -device 'virtio-net-pci,mac=A2:C0:43:77:08:A0,netdev=net0,bus=pci.0,addr=0x12,id=net0,vectors=6,mq=on,packed=on,rx_queue_size=1024,tx_queue_size=256,bootindex=300,host_mtu=900' \
+ -machine 'type=pc+pve0'
--- /dev/null
+# TEST: Simple test for netdev related stuff
+# QEMU_VERSION: 7.1
+bootdisk: scsi0
+cores: 3
+memory: 768
+name: netdev
+net0: virtio=A2:C0:43:77:08:A0,bridge=vmbr0,mtu=900
+ostype: l26
--- /dev/null
+/usr/bin/kvm \
+ -id 8006 \
+ -name 'netdev,debug-threads=on' \
+ -no-shutdown \
+ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \
+ -mon 'chardev=qmp,mode=control' \
+ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' \
+ -mon 'chardev=qmp-event,mode=control' \
+ -pidfile /var/run/qemu-server/8006.pid \
+ -daemonize \
+ -smp '3,sockets=1,cores=3,maxcpus=3' \
+ -nodefaults \
+ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \
+ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \
+ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \
+ -m 768 \
+ -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \
+ -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \
+ -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \
+ -device 'usb-tablet,id=tablet,bus=uhci.0,port=1' \
+ -device 'VGA,id=vga,bus=pci.0,addr=0x2' \
+ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \
+ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \
+ -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/var/lib/qemu-server/pve-bridge,downscript=/var/lib/qemu-server/pve-bridgedown,vhost=on' \
+ -device 'virtio-net-pci,mac=A2:C0:43:77:08:A0,netdev=net0,bus=pci.0,addr=0x12,id=net0,rx_queue_size=1024,tx_queue_size=256,bootindex=300,host_mtu=900' \
+ -machine 'type=pc+pve0'
/usr/bin/kvm \
-id 8006 \
- -name netdev \
- -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server,nowait' \
+ -name 'netdev,debug-threads=on' \
+ -no-shutdown \
+ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \
-mon 'chardev=qmp,mode=control' \
-chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' \
-mon 'chardev=qmp-event,mode=control' \
-smp '3,sockets=1,cores=3,maxcpus=3' \
-nodefaults \
-boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \
- -vnc unix:/var/run/qemu-server/8006.vnc,password \
+ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \
-cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \
-m 768 \
-device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \
--- /dev/null
+# TEST: Test error for old ostype type with newer usb config
+# QEMU_VERSION: 7.1.0
+# EXPECT_ERROR: using usb13 is only possible with machine type >= 7.1 and ostype l26 or windows > 7
+cores: 2
+memory: 768
+name: q35-usb3-error
+net0: virtio=A2:C0:43:77:08:A1,bridge=vmbr0
+ostype: w2k
+scsihw: virtio-scsi-pci
+smbios1: uuid=7b10d7af-b932-4c66-b2c3-3996152ec465
+vmgenid: c773c261-d800-4348-9f5d-167fadd53cf8
+vga: qxl
+usb13: spice
--- /dev/null
+# TEST: for a basic configuration with a .pxe machine and +pve pinned
+bootdisk: scsi0
+cores: 3
+ide2: none,media=cdrom
+machine: pc-q35-4.1+pve2.pxe
+memory: 1024
+name: pinned
+net0: virtio=A2:C0:43:77:08:A1,bridge=vmbr0
+numa: 0
+ostype: l26
+scsi0: local:8006/vm-8006-disk-0.raw,discard=on,size=104858K
+scsihw: virtio-scsi-pci
+smbios1: uuid=c7fdd046-fefc-11e9-832e-770e1d5636a0
+sockets: 1
+vmgenid: bdd46b98-fefc-11e9-97b4-d72c378e0f96
+# add rng0 to stress +pve2 version requirement
+rng0: source=/dev/urandom
--- /dev/null
+/usr/bin/kvm \
+ -id 8006 \
+ -name 'pinned,debug-threads=on' \
+ -no-shutdown \
+ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \
+ -mon 'chardev=qmp,mode=control' \
+ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' \
+ -mon 'chardev=qmp-event,mode=control' \
+ -pidfile /var/run/qemu-server/8006.pid \
+ -daemonize \
+ -smbios 'type=1,uuid=c7fdd046-fefc-11e9-832e-770e1d5636a0' \
+ -smp '3,sockets=1,cores=3,maxcpus=3' \
+ -nodefaults \
+ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \
+ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \
+ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \
+ -m 1024 \
+ -readconfig /usr/share/qemu-server/pve-q35-4.0.cfg \
+ -device 'vmgenid,guid=bdd46b98-fefc-11e9-97b4-d72c378e0f96' \
+ -device 'usb-tablet,id=tablet,bus=ehci.0,port=1' \
+ -device 'VGA,id=vga,bus=pcie.0,addr=0x1' \
+ -object 'rng-random,filename=/dev/urandom,id=rng0' \
+ -device 'virtio-rng-pci,rng=rng0,max-bytes=1024,period=1000,bus=pci.1,addr=0x1d' \
+ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3' \
+ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \
+ -drive 'if=none,id=drive-ide2,media=cdrom,aio=io_uring' \
+ -device 'ide-cd,bus=ide.1,unit=0,drive=drive-ide2,id=ide2,bootindex=200' \
+ -device 'virtio-scsi-pci,id=scsihw0,bus=pci.0,addr=0x5' \
+ -drive 'file=/var/lib/vz/images/8006/vm-8006-disk-0.raw,if=none,id=drive-scsi0,discard=on,format=raw,cache=none,aio=io_uring,detect-zeroes=unmap' \
+ -device 'scsi-hd,bus=scsihw0.0,channel=0,scsi-id=0,lun=0,drive=drive-scsi0,id=scsi0,bootindex=100' \
+ -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/var/lib/qemu-server/pve-bridge,downscript=/var/lib/qemu-server/pve-bridgedown,vhost=on' \
+ -device 'virtio-net-pci,mac=A2:C0:43:77:08:A1,netdev=net0,bus=pci.0,addr=0x12,id=net0,bootindex=300,romfile=pxe-virtio.rom' \
+ -machine 'type=pc-q35-4.1+pve2'
--- /dev/null
+# TEST: for a basic configuration with a .pxe machine
+bootdisk: scsi0
+cores: 3
+ide2: none,media=cdrom
+machine: pc-q35-5.1.pxe
+memory: 1024
+name: pinned
+net0: virtio=A2:C0:43:77:08:A1,bridge=vmbr0
+numa: 0
+ostype: l26
+scsi0: local:8006/vm-8006-disk-0.raw,discard=on,size=104858K
+scsihw: virtio-scsi-pci
+smbios1: uuid=c7fdd046-fefc-11e9-832e-770e1d5636a0
+sockets: 1
+vmgenid: bdd46b98-fefc-11e9-97b4-d72c378e0f96
--- /dev/null
+/usr/bin/kvm \
+ -id 8006 \
+ -name 'pinned,debug-threads=on' \
+ -no-shutdown \
+ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \
+ -mon 'chardev=qmp,mode=control' \
+ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' \
+ -mon 'chardev=qmp-event,mode=control' \
+ -pidfile /var/run/qemu-server/8006.pid \
+ -daemonize \
+ -smbios 'type=1,uuid=c7fdd046-fefc-11e9-832e-770e1d5636a0' \
+ -smp '3,sockets=1,cores=3,maxcpus=3' \
+ -nodefaults \
+ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \
+ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \
+ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \
+ -m 1024 \
+ -readconfig /usr/share/qemu-server/pve-q35-4.0.cfg \
+ -device 'vmgenid,guid=bdd46b98-fefc-11e9-97b4-d72c378e0f96' \
+ -device 'usb-tablet,id=tablet,bus=ehci.0,port=1' \
+ -device 'VGA,id=vga,bus=pcie.0,addr=0x1' \
+ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3' \
+ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \
+ -drive 'if=none,id=drive-ide2,media=cdrom,aio=io_uring' \
+ -device 'ide-cd,bus=ide.1,unit=0,drive=drive-ide2,id=ide2,bootindex=200' \
+ -device 'virtio-scsi-pci,id=scsihw0,bus=pci.0,addr=0x5' \
+ -drive 'file=/var/lib/vz/images/8006/vm-8006-disk-0.raw,if=none,id=drive-scsi0,discard=on,format=raw,cache=none,aio=io_uring,detect-zeroes=unmap' \
+ -device 'scsi-hd,bus=scsihw0.0,channel=0,scsi-id=0,lun=0,drive=drive-scsi0,id=scsi0,bootindex=100' \
+ -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/var/lib/qemu-server/pve-bridge,downscript=/var/lib/qemu-server/pve-bridgedown,vhost=on' \
+ -device 'virtio-net-pci,mac=A2:C0:43:77:08:A1,netdev=net0,bus=pci.0,addr=0x12,id=net0,bootindex=300,romfile=pxe-virtio.rom' \
+ -machine 'type=pc-q35-5.1+pve0'
/usr/bin/kvm \
-id 8006 \
- -name pinned \
- -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server,nowait' \
+ -name 'pinned,debug-threads=on' \
+ -no-shutdown \
+ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \
-mon 'chardev=qmp,mode=control' \
-chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' \
-mon 'chardev=qmp-event,mode=control' \
-smp '3,sockets=1,cores=3,maxcpus=3' \
-nodefaults \
-boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \
- -vnc unix:/var/run/qemu-server/8006.vnc,password \
+ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \
-cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \
-m 1024 \
-readconfig /usr/share/qemu-server/pve-q35.cfg \
-device 'VGA,id=vga,bus=pcie.0,addr=0x1' \
-device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3' \
-iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \
- -drive 'if=none,id=drive-ide2,media=cdrom,aio=threads' \
+ -drive 'if=none,id=drive-ide2,media=cdrom,aio=io_uring' \
-device 'ide-cd,bus=ide.1,unit=0,drive=drive-ide2,id=ide2,bootindex=200' \
-device 'virtio-scsi-pci,id=scsihw0,bus=pci.0,addr=0x5' \
- -drive 'file=/var/lib/vz/images/8006/vm-8006-disk-0.raw,if=none,id=drive-scsi0,discard=on,format=raw,cache=none,aio=native,detect-zeroes=unmap' \
+ -drive 'file=/var/lib/vz/images/8006/vm-8006-disk-0.raw,if=none,id=drive-scsi0,discard=on,format=raw,cache=none,aio=io_uring,detect-zeroes=unmap' \
-device 'scsi-hd,bus=scsihw0.0,channel=0,scsi-id=0,lun=0,drive=drive-scsi0,id=scsi0,bootindex=100' \
-netdev 'type=tap,id=net0,ifname=tap8006i0,script=/var/lib/qemu-server/pve-bridge,downscript=/var/lib/qemu-server/pve-bridgedown,vhost=on' \
-device 'virtio-net-pci,mac=A2:C0:43:77:08:A1,netdev=net0,bus=pci.0,addr=0x12,id=net0,bootindex=300' \
--- /dev/null
+# TEST: Config with q35, Linux & four IDE CD-ROMs
+bootdisk: scsi0
+cores: 2
+ide0: cifs-store:iso/zero.iso,media=cdrom,size=112M
+ide1: cifs-store:iso/one.iso,media=cdrom,size=112M
+ide2: cifs-store:iso/two.iso,media=cdrom,size=112M
+ide3: cifs-store:iso/three.iso,media=cdrom,size=112M
+machine: q35
+memory: 512
+net0: virtio=2E:01:68:F9:9C:87,bridge=vmbr0
+ostype: l26
+scsi0: local:100/vm-100-disk-2.qcow2,size=10G
+scsihw: virtio-scsi-pci
+smbios1: uuid=3dd750ce-d910-44d0-9493-525c0be4e687
+vmgenid: 54d1c06c-8f5b-440f-b5b2-6eab1380e13d
--- /dev/null
+/usr/bin/kvm \
+ -id 8006 \
+ -name 'vm8006,debug-threads=on' \
+ -no-shutdown \
+ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \
+ -mon 'chardev=qmp,mode=control' \
+ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' \
+ -mon 'chardev=qmp-event,mode=control' \
+ -pidfile /var/run/qemu-server/8006.pid \
+ -daemonize \
+ -smbios 'type=1,uuid=3dd750ce-d910-44d0-9493-525c0be4e687' \
+ -global 'ICH9-LPC.acpi-pci-hotplug-with-bridge-support=off' \
+ -smp '2,sockets=1,cores=2,maxcpus=2' \
+ -nodefaults \
+ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \
+ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \
+ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \
+ -m 512 \
+ -readconfig /usr/share/qemu-server/pve-q35-4.0.cfg \
+ -device 'vmgenid,guid=54d1c06c-8f5b-440f-b5b2-6eab1380e13d' \
+ -device 'usb-tablet,id=tablet,bus=ehci.0,port=1' \
+ -device 'VGA,id=vga,bus=pcie.0,addr=0x1' \
+ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \
+ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \
+ -drive 'file=/mnt/pve/cifs-store/template/iso/zero.iso,if=none,id=drive-ide0,media=cdrom,aio=threads' \
+ -device 'ide-cd,bus=ide.0,unit=0,drive=drive-ide0,id=ide0,bootindex=200' \
+ -drive 'file=/mnt/pve/cifs-store/template/iso/one.iso,if=none,id=drive-ide1,media=cdrom,aio=threads' \
+ -device 'ide-cd,bus=ide.2,unit=0,drive=drive-ide1,id=ide1,bootindex=201' \
+ -drive 'file=/mnt/pve/cifs-store/template/iso/two.iso,if=none,id=drive-ide2,media=cdrom,aio=threads' \
+ -device 'ide-cd,bus=ide.1,unit=0,drive=drive-ide2,id=ide2,bootindex=202' \
+ -drive 'file=/mnt/pve/cifs-store/template/iso/three.iso,if=none,id=drive-ide3,media=cdrom,aio=threads' \
+ -device 'ide-cd,bus=ide.3,unit=0,drive=drive-ide3,id=ide3,bootindex=203' \
+ -device 'virtio-scsi-pci,id=scsihw0,bus=pci.0,addr=0x5' \
+ -drive 'file=/var/lib/vz/images/100/vm-100-disk-2.qcow2,if=none,id=drive-scsi0,format=qcow2,cache=none,aio=io_uring,detect-zeroes=on' \
+ -device 'scsi-hd,bus=scsihw0.0,channel=0,scsi-id=0,lun=0,drive=drive-scsi0,id=scsi0,bootindex=100' \
+ -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/var/lib/qemu-server/pve-bridge,downscript=/var/lib/qemu-server/pve-bridgedown,vhost=on' \
+ -device 'virtio-net-pci,mac=2E:01:68:F9:9C:87,netdev=net0,bus=pci.0,addr=0x12,id=net0,rx_queue_size=1024,tx_queue_size=256,bootindex=300' \
+ -machine 'type=q35+pve0'
--- /dev/null
+# TEST: Config with q35, NUMA, hostpci mapping passthrough, EFI & Linux
+bios: ovmf
+bootdisk: scsi0
+cores: 1
+efidisk0: local:100/vm-100-disk-1.qcow2,size=128K
+hostpci0: mapping=someNic
+hostpci1: mapping=someGpu,mdev=some-model
+hostpci2: mapping=someNic
+machine: q35
+memory: 512
+net0: virtio=2E:01:68:F9:9C:87,bridge=vmbr0
+numa: 1
+ostype: l26
+scsihw: virtio-scsi-pci
+smbios1: uuid=3dd750ce-d910-44d0-9493-525c0be4e687
+sockets: 2
+vmgenid: 54d1c06c-8f5b-440f-b5b2-6eab1380e13d
--- /dev/null
+/usr/bin/kvm \
+ -id 8006 \
+ -name 'vm8006,debug-threads=on' \
+ -no-shutdown \
+ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \
+ -mon 'chardev=qmp,mode=control' \
+ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' \
+ -mon 'chardev=qmp-event,mode=control' \
+ -pidfile /var/run/qemu-server/8006.pid \
+ -daemonize \
+ -smbios 'type=1,uuid=3dd750ce-d910-44d0-9493-525c0be4e687' \
+ -drive 'if=pflash,unit=0,format=raw,readonly=on,file=/usr/share/pve-edk2-firmware//OVMF_CODE.fd' \
+ -drive 'if=pflash,unit=1,id=drive-efidisk0,format=qcow2,file=/var/lib/vz/images/100/vm-100-disk-1.qcow2' \
+ -global 'ICH9-LPC.acpi-pci-hotplug-with-bridge-support=off' \
+ -smp '2,sockets=2,cores=1,maxcpus=2' \
+ -nodefaults \
+ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \
+ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \
+ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \
+ -m 512 \
+ -object 'memory-backend-ram,id=ram-node0,size=256M' \
+ -numa 'node,nodeid=0,cpus=0,memdev=ram-node0' \
+ -object 'memory-backend-ram,id=ram-node1,size=256M' \
+ -numa 'node,nodeid=1,cpus=1,memdev=ram-node1' \
+ -readconfig /usr/share/qemu-server/pve-q35-4.0.cfg \
+ -device 'vmgenid,guid=54d1c06c-8f5b-440f-b5b2-6eab1380e13d' \
+ -device 'usb-tablet,id=tablet,bus=ehci.0,port=1' \
+ -device 'vfio-pci,host=0000:07:10.0,id=hostpci0,bus=pci.0,addr=0x10' \
+ -device 'vfio-pci,sysfsdev=/sys/bus/mdev/devices/00000001-0000-0000-0000-000000008006,id=hostpci1,bus=pci.0,addr=0x11' \
+ -device 'vfio-pci,host=0000:07:10.4,id=hostpci2,bus=pci.0,addr=0x1b' \
+ -device 'VGA,id=vga,bus=pcie.0,addr=0x1' \
+ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \
+ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \
+ -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/var/lib/qemu-server/pve-bridge,downscript=/var/lib/qemu-server/pve-bridgedown,vhost=on' \
+ -device 'virtio-net-pci,mac=2E:01:68:F9:9C:87,netdev=net0,bus=pci.0,addr=0x12,id=net0,rx_queue_size=1024,tx_queue_size=256,bootindex=300' \
+ -machine 'type=q35+pve0'
/usr/bin/kvm \
-id 8006 \
- -name vm8006 \
- -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server,nowait' \
+ -name 'vm8006,debug-threads=on' \
+ -no-shutdown \
+ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \
-mon 'chardev=qmp,mode=control' \
-chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' \
-mon 'chardev=qmp-event,mode=control' \
-pidfile /var/run/qemu-server/8006.pid \
-daemonize \
-smbios 'type=1,uuid=3dd750ce-d910-44d0-9493-525c0be4e687' \
- -drive 'if=pflash,unit=0,format=raw,readonly,file=/usr/share/pve-edk2-firmware//OVMF_CODE.fd' \
- -drive 'if=pflash,unit=1,format=qcow2,id=drive-efidisk0,file=/var/lib/vz/images/100/vm-100-disk-1.qcow2' \
+ -drive 'if=pflash,unit=0,format=raw,readonly=on,file=/usr/share/pve-edk2-firmware//OVMF_CODE.fd' \
+ -drive 'if=pflash,unit=1,id=drive-efidisk0,format=qcow2,file=/var/lib/vz/images/100/vm-100-disk-1.qcow2' \
+ -global 'ICH9-LPC.acpi-pci-hotplug-with-bridge-support=off' \
-smp '2,sockets=2,cores=1,maxcpus=2' \
-nodefaults \
-boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \
- -vnc unix:/var/run/qemu-server/8006.vnc,password \
+ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \
-cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \
-m 512 \
-object 'memory-backend-ram,id=ram-node0,size=256M' \
-device 'vfio-pci,host=0000:f0:43.1,id=hostpci0.1,bus=pci.0,addr=0x10.1' \
-device 'vfio-pci,host=1234:f0:43.1,id=hostpci1,bus=pci.0,addr=0x11' \
-device 'VGA,id=vga,bus=pcie.0,addr=0x1' \
- -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3' \
+ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \
-iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \
-netdev 'type=tap,id=net0,ifname=tap8006i0,script=/var/lib/qemu-server/pve-bridge,downscript=/var/lib/qemu-server/pve-bridgedown,vhost=on' \
- -device 'virtio-net-pci,mac=2E:01:68:F9:9C:87,netdev=net0,bus=pci.0,addr=0x12,id=net0,bootindex=300' \
+ -device 'virtio-net-pci,mac=2E:01:68:F9:9C:87,netdev=net0,bus=pci.0,addr=0x12,id=net0,rx_queue_size=1024,tx_queue_size=256,bootindex=300' \
-machine 'type=q35+pve0'
--- /dev/null
+# TEST: Overriding PCI vendor/device IDs reported to guest
+bios: ovmf
+bootdisk: scsi0
+cores: 1
+efidisk0: local:100/vm-100-disk-1.qcow2,size=128K
+hostpci0: 00:ff.1,vendor-id=0x1234,device-id=0x5678,sub-vendor-id=0x2233,sub-device-id=0x0000
+hostpci1: d0:13.0,pcie=1,vendor-id=0x1234,device-id=0x5678
+machine: q35
+memory: 512
+net0: virtio=2E:01:68:F9:9C:87,bridge=vmbr0
+numa: 1
+ostype: l26
+scsihw: virtio-scsi-pci
+smbios1: uuid=3dd750ce-d910-44d0-9493-525c0be4e687
+sockets: 2
+vmgenid: 54d1c06c-8f5b-440f-b5b2-6eab1380e13d
--- /dev/null
+/usr/bin/kvm \
+ -id 8006 \
+ -name 'vm8006,debug-threads=on' \
+ -no-shutdown \
+ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \
+ -mon 'chardev=qmp,mode=control' \
+ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' \
+ -mon 'chardev=qmp-event,mode=control' \
+ -pidfile /var/run/qemu-server/8006.pid \
+ -daemonize \
+ -smbios 'type=1,uuid=3dd750ce-d910-44d0-9493-525c0be4e687' \
+ -drive 'if=pflash,unit=0,format=raw,readonly=on,file=/usr/share/pve-edk2-firmware//OVMF_CODE.fd' \
+ -drive 'if=pflash,unit=1,id=drive-efidisk0,format=qcow2,file=/var/lib/vz/images/100/vm-100-disk-1.qcow2' \
+ -global 'ICH9-LPC.acpi-pci-hotplug-with-bridge-support=off' \
+ -smp '2,sockets=2,cores=1,maxcpus=2' \
+ -nodefaults \
+ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \
+ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \
+ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \
+ -m 512 \
+ -object 'memory-backend-ram,id=ram-node0,size=256M' \
+ -numa 'node,nodeid=0,cpus=0,memdev=ram-node0' \
+ -object 'memory-backend-ram,id=ram-node1,size=256M' \
+ -numa 'node,nodeid=1,cpus=1,memdev=ram-node1' \
+ -readconfig /usr/share/qemu-server/pve-q35-4.0.cfg \
+ -device 'vmgenid,guid=54d1c06c-8f5b-440f-b5b2-6eab1380e13d' \
+ -device 'usb-tablet,id=tablet,bus=ehci.0,port=1' \
+ -device 'vfio-pci,host=0000:00:ff.1,id=hostpci0,bus=pci.0,addr=0x10,x-pci-vendor-id=0x1234,x-pci-device-id=0x5678,x-pci-sub-vendor-id=0x2233,x-pci-sub-device-id=0x0000' \
+ -device 'vfio-pci,host=0000:d0:13.0,id=hostpci1,bus=ich9-pcie-port-2,addr=0x0,x-pci-vendor-id=0x1234,x-pci-device-id=0x5678' \
+ -device 'VGA,id=vga,bus=pcie.0,addr=0x1' \
+ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \
+ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \
+ -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/var/lib/qemu-server/pve-bridge,downscript=/var/lib/qemu-server/pve-bridgedown,vhost=on' \
+ -device 'virtio-net-pci,mac=2E:01:68:F9:9C:87,netdev=net0,bus=pci.0,addr=0x12,id=net0,rx_queue_size=1024,tx_queue_size=256,bootindex=300' \
+ -machine 'type=q35+pve0'
hostpci2: 00:f4.0
hostpci3: d0:15.1,pcie=1
hostpci4: d0:17.0,pcie=1,rombar=0
-hostpci7: d0:15.1,pcie=1
+hostpci7: d0:15.2,pcie=1
machine: q35
memory: 512
net0: virtio=2E:01:68:F9:9C:87,bridge=vmbr0
/usr/bin/kvm \
-id 8006 \
- -name vm8006 \
- -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server,nowait' \
+ -name 'vm8006,debug-threads=on' \
+ -no-shutdown \
+ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \
-mon 'chardev=qmp,mode=control' \
-chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' \
-mon 'chardev=qmp-event,mode=control' \
-pidfile /var/run/qemu-server/8006.pid \
-daemonize \
-smbios 'type=1,uuid=3dd750ce-d910-44d0-9493-525c0be4e687' \
- -drive 'if=pflash,unit=0,format=raw,readonly,file=/usr/share/pve-edk2-firmware//OVMF_CODE.fd' \
- -drive 'if=pflash,unit=1,format=qcow2,id=drive-efidisk0,file=/var/lib/vz/images/100/vm-100-disk-1.qcow2' \
+ -drive 'if=pflash,unit=0,format=raw,readonly=on,file=/usr/share/pve-edk2-firmware//OVMF_CODE.fd' \
+ -drive 'if=pflash,unit=1,id=drive-efidisk0,format=qcow2,file=/var/lib/vz/images/100/vm-100-disk-1.qcow2' \
+ -global 'ICH9-LPC.acpi-pci-hotplug-with-bridge-support=off' \
-smp '2,sockets=2,cores=1,maxcpus=2' \
-nodefaults \
-boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \
- -vnc unix:/var/run/qemu-server/8006.vnc,password \
+ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \
-cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \
-m 512 \
-object 'memory-backend-ram,id=ram-node0,size=256M' \
-device 'pcie-root-port,id=ich9-pcie-port-5,addr=10.0,x-speed=16,x-width=32,multifunction=on,bus=pcie.0,port=5,chassis=5' \
-device 'vfio-pci,host=0000:d0:17.0,id=hostpci4,bus=ich9-pcie-port-5,addr=0x0,rombar=0' \
-device 'pcie-root-port,id=ich9-pcie-port-8,addr=10.3,x-speed=16,x-width=32,multifunction=on,bus=pcie.0,port=8,chassis=8' \
- -device 'vfio-pci,host=0000:d0:15.1,id=hostpci7,bus=ich9-pcie-port-8,addr=0x0' \
+ -device 'vfio-pci,host=0000:d0:15.2,id=hostpci7,bus=ich9-pcie-port-8,addr=0x0' \
-device 'VGA,id=vga,bus=pcie.0,addr=0x1' \
- -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3' \
+ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \
-iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \
-netdev 'type=tap,id=net0,ifname=tap8006i0,script=/var/lib/qemu-server/pve-bridge,downscript=/var/lib/qemu-server/pve-bridgedown,vhost=on' \
- -device 'virtio-net-pci,mac=2E:01:68:F9:9C:87,netdev=net0,bus=pci.0,addr=0x12,id=net0,bootindex=300' \
+ -device 'virtio-net-pci,mac=2E:01:68:F9:9C:87,netdev=net0,bus=pci.0,addr=0x12,id=net0,rx_queue_size=1024,tx_queue_size=256,bootindex=300' \
-machine 'type=q35+pve0'
--- /dev/null
+# TEST: Config with q35, Linux & nothing much else but on 6.0
+# QEMU_VERSION: 6.0.0
+bios: ovmf
+bootdisk: scsi0
+cores: 2
+efidisk0: local:100/vm-100-disk-1.qcow2,size=128K
+machine: q35
+memory: 512
+net0: virtio=2E:01:68:F9:9C:87,bridge=vmbr0
+ostype: l26
+scsihw: virtio-scsi-pci
+smbios1: uuid=3dd750ce-d910-44d0-9493-525c0be4e687
+vmgenid: 54d1c06c-8f5b-440f-b5b2-6eab1380e13d
--- /dev/null
+/usr/bin/kvm \
+ -id 8006 \
+ -name 'vm8006,debug-threads=on' \
+ -no-shutdown \
+ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \
+ -mon 'chardev=qmp,mode=control' \
+ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' \
+ -mon 'chardev=qmp-event,mode=control' \
+ -pidfile /var/run/qemu-server/8006.pid \
+ -daemonize \
+ -smbios 'type=1,uuid=3dd750ce-d910-44d0-9493-525c0be4e687' \
+ -drive 'if=pflash,unit=0,format=raw,readonly=on,file=/usr/share/pve-edk2-firmware//OVMF_CODE.fd' \
+ -drive 'if=pflash,unit=1,id=drive-efidisk0,format=qcow2,file=/var/lib/vz/images/100/vm-100-disk-1.qcow2' \
+ -smp '2,sockets=1,cores=2,maxcpus=2' \
+ -nodefaults \
+ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \
+ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \
+ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \
+ -m 512 \
+ -readconfig /usr/share/qemu-server/pve-q35-4.0.cfg \
+ -device 'vmgenid,guid=54d1c06c-8f5b-440f-b5b2-6eab1380e13d' \
+ -device 'usb-tablet,id=tablet,bus=ehci.0,port=1' \
+ -device 'VGA,id=vga,bus=pcie.0,addr=0x1' \
+ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3' \
+ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \
+ -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/var/lib/qemu-server/pve-bridge,downscript=/var/lib/qemu-server/pve-bridgedown,vhost=on' \
+ -device 'virtio-net-pci,mac=2E:01:68:F9:9C:87,netdev=net0,bus=pci.0,addr=0x12,id=net0,bootindex=300' \
+ -machine 'type=q35+pve0'
--- /dev/null
+# TEST: Config with q35, Linux & nothing much else but on 7.0
+# QEMU_VERSION: 7.0.0
+bios: ovmf
+bootdisk: scsi0
+cores: 2
+efidisk0: local:100/vm-100-disk-1.qcow2,size=128K
+machine: q35
+meta: creation-qemu=6.1
+memory: 512
+net0: virtio=2E:01:68:F9:9C:87,bridge=vmbr0
+ostype: l26
+scsihw: virtio-scsi-pci
+smbios1: uuid=3dd750ce-d910-44d0-9493-525c0be4e687
+vmgenid: 54d1c06c-8f5b-440f-b5b2-6eab1380e13d
--- /dev/null
+/usr/bin/kvm \
+ -id 8006 \
+ -name 'vm8006,debug-threads=on' \
+ -no-shutdown \
+ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \
+ -mon 'chardev=qmp,mode=control' \
+ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' \
+ -mon 'chardev=qmp-event,mode=control' \
+ -pidfile /var/run/qemu-server/8006.pid \
+ -daemonize \
+ -smbios 'type=1,uuid=3dd750ce-d910-44d0-9493-525c0be4e687' \
+ -drive 'if=pflash,unit=0,format=raw,readonly=on,file=/usr/share/pve-edk2-firmware//OVMF_CODE.fd' \
+ -drive 'if=pflash,unit=1,id=drive-efidisk0,format=qcow2,file=/var/lib/vz/images/100/vm-100-disk-1.qcow2' \
+ -smp '2,sockets=1,cores=2,maxcpus=2' \
+ -nodefaults \
+ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \
+ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \
+ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \
+ -m 512 \
+ -readconfig /usr/share/qemu-server/pve-q35-4.0.cfg \
+ -device 'vmgenid,guid=54d1c06c-8f5b-440f-b5b2-6eab1380e13d' \
+ -device 'usb-tablet,id=tablet,bus=ehci.0,port=1' \
+ -device 'VGA,id=vga,bus=pcie.0,addr=0x1' \
+ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \
+ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \
+ -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/var/lib/qemu-server/pve-bridge,downscript=/var/lib/qemu-server/pve-bridgedown,vhost=on' \
+ -device 'virtio-net-pci,mac=2E:01:68:F9:9C:87,netdev=net0,bus=pci.0,addr=0x12,id=net0,bootindex=300' \
+ -machine 'type=q35+pve0'
--- /dev/null
+# TEST: Config with q35, Linux & nothing much else
+#
+bios: ovmf
+bootdisk: scsi0
+cores: 2
+efidisk0: local:100/vm-100-disk-1.qcow2,size=128K
+machine: pc-q35-6.1
+memory: 512
+net0: virtio=2E:01:68:F9:9C:87,bridge=vmbr0
+ostype: l26
+scsihw: virtio-scsi-pci
+smbios1: uuid=3dd750ce-d910-44d0-9493-525c0be4e687
+vmgenid: 54d1c06c-8f5b-440f-b5b2-6eab1380e13d
--- /dev/null
+/usr/bin/kvm \
+ -id 8006 \
+ -name 'vm8006,debug-threads=on' \
+ -no-shutdown \
+ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \
+ -mon 'chardev=qmp,mode=control' \
+ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' \
+ -mon 'chardev=qmp-event,mode=control' \
+ -pidfile /var/run/qemu-server/8006.pid \
+ -daemonize \
+ -smbios 'type=1,uuid=3dd750ce-d910-44d0-9493-525c0be4e687' \
+ -drive 'if=pflash,unit=0,format=raw,readonly=on,file=/usr/share/pve-edk2-firmware//OVMF_CODE.fd' \
+ -drive 'if=pflash,unit=1,id=drive-efidisk0,format=qcow2,file=/var/lib/vz/images/100/vm-100-disk-1.qcow2' \
+ -smp '2,sockets=1,cores=2,maxcpus=2' \
+ -nodefaults \
+ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \
+ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \
+ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \
+ -m 512 \
+ -readconfig /usr/share/qemu-server/pve-q35-4.0.cfg \
+ -device 'vmgenid,guid=54d1c06c-8f5b-440f-b5b2-6eab1380e13d' \
+ -device 'usb-tablet,id=tablet,bus=ehci.0,port=1' \
+ -device 'VGA,id=vga,bus=pcie.0,addr=0x1' \
+ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3' \
+ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \
+ -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/var/lib/qemu-server/pve-bridge,downscript=/var/lib/qemu-server/pve-bridgedown,vhost=on' \
+ -device 'virtio-net-pci,mac=2E:01:68:F9:9C:87,netdev=net0,bus=pci.0,addr=0x12,id=net0,bootindex=300' \
+ -machine 'type=pc-q35-6.1+pve0'
--- /dev/null
+# TEST: Config with q35, Linux & nothing much else
+#
+bios: ovmf
+bootdisk: scsi0
+cores: 2
+efidisk0: local:100/vm-100-disk-1.qcow2,size=128K
+machine: q35
+memory: 512
+net0: virtio=2E:01:68:F9:9C:87,bridge=vmbr0
+ostype: l26
+scsihw: virtio-scsi-pci
+smbios1: uuid=3dd750ce-d910-44d0-9493-525c0be4e687
+vmgenid: 54d1c06c-8f5b-440f-b5b2-6eab1380e13d
--- /dev/null
+/usr/bin/kvm \
+ -id 8006 \
+ -name 'vm8006,debug-threads=on' \
+ -no-shutdown \
+ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \
+ -mon 'chardev=qmp,mode=control' \
+ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' \
+ -mon 'chardev=qmp-event,mode=control' \
+ -pidfile /var/run/qemu-server/8006.pid \
+ -daemonize \
+ -smbios 'type=1,uuid=3dd750ce-d910-44d0-9493-525c0be4e687' \
+ -drive 'if=pflash,unit=0,format=raw,readonly=on,file=/usr/share/pve-edk2-firmware//OVMF_CODE.fd' \
+ -drive 'if=pflash,unit=1,id=drive-efidisk0,format=qcow2,file=/var/lib/vz/images/100/vm-100-disk-1.qcow2' \
+ -global 'ICH9-LPC.acpi-pci-hotplug-with-bridge-support=off' \
+ -smp '2,sockets=1,cores=2,maxcpus=2' \
+ -nodefaults \
+ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \
+ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \
+ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \
+ -m 512 \
+ -readconfig /usr/share/qemu-server/pve-q35-4.0.cfg \
+ -device 'vmgenid,guid=54d1c06c-8f5b-440f-b5b2-6eab1380e13d' \
+ -device 'usb-tablet,id=tablet,bus=ehci.0,port=1' \
+ -device 'VGA,id=vga,bus=pcie.0,addr=0x1' \
+ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \
+ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \
+ -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/var/lib/qemu-server/pve-bridge,downscript=/var/lib/qemu-server/pve-bridgedown,vhost=on' \
+ -device 'virtio-net-pci,mac=2E:01:68:F9:9C:87,netdev=net0,bus=pci.0,addr=0x12,id=net0,rx_queue_size=1024,tx_queue_size=256,bootindex=300' \
+ -machine 'type=q35+pve0'
--- /dev/null
+# TEST: Test usb error for q35 and older machine type
+# QEMU_VERSION: 4.0.0
+# EXPECT_ERROR: using usb13 is only possible with machine type >= 7.1 and ostype l26 or windows > 7
+cores: 2
+memory: 768
+name: q35-usb3-error
+net0: virtio=A2:C0:43:77:08:A1,bridge=vmbr0
+ostype: l26
+machine: q35
+scsihw: virtio-scsi-pci
+smbios1: uuid=7b10d7af-b932-4c66-b2c3-3996152ec465
+vmgenid: c773c261-d800-4348-9f5d-167fadd53cf8
+vga: qxl
+usb13: spice
--- /dev/null
+# TEST: Test Q35 USB2 passthrough combination
+# QEMU_VERSION: 4.0.0
+cores: 2
+memory: 768
+name: q35-usb2
+net0: virtio=A2:C0:43:77:08:A1,bridge=vmbr0
+ostype: l26
+machine: q35
+scsihw: virtio-scsi-pci
+smbios1: uuid=7b10d7af-b932-4c66-b2c3-3996152ec465
+vmgenid: c773c261-d800-4348-9f5d-167fadd53cf8
+vga: qxl
+usb1: spice
--- /dev/null
+/usr/bin/kvm \
+ -id 8006 \
+ -name 'q35-usb2,debug-threads=on' \
+ -no-shutdown \
+ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \
+ -mon 'chardev=qmp,mode=control' \
+ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' \
+ -mon 'chardev=qmp-event,mode=control' \
+ -pidfile /var/run/qemu-server/8006.pid \
+ -daemonize \
+ -smbios 'type=1,uuid=7b10d7af-b932-4c66-b2c3-3996152ec465' \
+ -smp '2,sockets=1,cores=2,maxcpus=2' \
+ -nodefaults \
+ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \
+ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \
+ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \
+ -m 768 \
+ -readconfig /usr/share/qemu-server/pve-q35-4.0.cfg \
+ -device 'vmgenid,guid=c773c261-d800-4348-9f5d-167fadd53cf8' \
+ -chardev 'spicevmc,id=usbredirchardev1,name=usbredir' \
+ -device 'usb-redir,chardev=usbredirchardev1,id=usbredirdev1,bus=ehci.0' \
+ -device 'qxl-vga,id=vga,bus=pcie.0,addr=0x1' \
+ -device 'virtio-serial,id=spice,bus=pci.0,addr=0x9' \
+ -chardev 'spicevmc,id=vdagent,name=vdagent' \
+ -device 'virtserialport,chardev=vdagent,name=com.redhat.spice.0' \
+ -spice 'tls-port=61000,addr=127.0.0.1,tls-ciphers=HIGH,seamless-migration=on' \
+ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3' \
+ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \
+ -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/var/lib/qemu-server/pve-bridge,downscript=/var/lib/qemu-server/pve-bridgedown,vhost=on' \
+ -device 'virtio-net-pci,mac=A2:C0:43:77:08:A1,netdev=net0,bus=pci.0,addr=0x12,id=net0,bootindex=300' \
+ -machine 'type=q35'
--- /dev/null
+# TEST: Test Q35 USB3 passthrough combination
+# QEMU_VERSION: 4.0.0
+cores: 2
+memory: 768
+name: q35-usb3
+net0: virtio=A2:C0:43:77:08:A1,bridge=vmbr0
+ostype: l26
+machine: q35
+scsihw: virtio-scsi-pci
+smbios1: uuid=7b10d7af-b932-4c66-b2c3-3996152ec465
+vmgenid: c773c261-d800-4348-9f5d-167fadd53cf8
+vga: qxl
+usb1: spice,usb3=1
--- /dev/null
+/usr/bin/kvm \
+ -id 8006 \
+ -name 'q35-usb3,debug-threads=on' \
+ -no-shutdown \
+ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \
+ -mon 'chardev=qmp,mode=control' \
+ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' \
+ -mon 'chardev=qmp-event,mode=control' \
+ -pidfile /var/run/qemu-server/8006.pid \
+ -daemonize \
+ -smbios 'type=1,uuid=7b10d7af-b932-4c66-b2c3-3996152ec465' \
+ -smp '2,sockets=1,cores=2,maxcpus=2' \
+ -nodefaults \
+ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \
+ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \
+ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \
+ -m 768 \
+ -readconfig /usr/share/qemu-server/pve-q35-4.0.cfg \
+ -device 'vmgenid,guid=c773c261-d800-4348-9f5d-167fadd53cf8' \
+ -device 'nec-usb-xhci,id=xhci,bus=pci.1,addr=0x1b' \
+ -chardev 'spicevmc,id=usbredirchardev1,name=usbredir' \
+ -device 'usb-redir,chardev=usbredirchardev1,id=usbredirdev1,bus=xhci.0' \
+ -device 'qxl-vga,id=vga,bus=pcie.0,addr=0x1' \
+ -device 'virtio-serial,id=spice,bus=pci.0,addr=0x9' \
+ -chardev 'spicevmc,id=vdagent,name=vdagent' \
+ -device 'virtserialport,chardev=vdagent,name=com.redhat.spice.0' \
+ -spice 'tls-port=61000,addr=127.0.0.1,tls-ciphers=HIGH,seamless-migration=on' \
+ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3' \
+ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \
+ -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/var/lib/qemu-server/pve-bridge,downscript=/var/lib/qemu-server/pve-bridgedown,vhost=on' \
+ -device 'virtio-net-pci,mac=A2:C0:43:77:08:A1,netdev=net0,bus=pci.0,addr=0x12,id=net0,bootindex=300' \
+ -machine 'type=q35'
--- /dev/null
+machine: q35,viommu=intel
--- /dev/null
+/usr/bin/kvm \
+ -id 8006 \
+ -name 'vm8006,debug-threads=on' \
+ -no-shutdown \
+ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \
+ -mon 'chardev=qmp,mode=control' \
+ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' \
+ -mon 'chardev=qmp-event,mode=control' \
+ -pidfile /var/run/qemu-server/8006.pid \
+ -daemonize \
+ -smp '1,sockets=1,cores=1,maxcpus=1' \
+ -nodefaults \
+ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \
+ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \
+ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \
+ -m 512 \
+ -device 'intel-iommu,intremap=on,caching-mode=on' \
+ -readconfig /usr/share/qemu-server/pve-q35-4.0.cfg \
+ -device 'usb-tablet,id=tablet,bus=ehci.0,port=1' \
+ -device 'VGA,id=vga,bus=pcie.0,addr=0x1' \
+ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \
+ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \
+ -machine 'type=q35+pve0,kernel-irqchip=split'
--- /dev/null
+machine: type=q35,viommu=virtio
--- /dev/null
+/usr/bin/kvm \
+ -id 8006 \
+ -name 'vm8006,debug-threads=on' \
+ -no-shutdown \
+ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \
+ -mon 'chardev=qmp,mode=control' \
+ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' \
+ -mon 'chardev=qmp-event,mode=control' \
+ -pidfile /var/run/qemu-server/8006.pid \
+ -daemonize \
+ -smp '1,sockets=1,cores=1,maxcpus=1' \
+ -nodefaults \
+ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \
+ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \
+ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \
+ -m 512 \
+ -readconfig /usr/share/qemu-server/pve-q35-4.0.cfg \
+ -device 'usb-tablet,id=tablet,bus=ehci.0,port=1' \
+ -device 'VGA,id=vga,bus=pcie.0,addr=0x1' \
+ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \
+ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \
+ -device virtio-iommu-pci \
+ -machine 'type=q35+pve0'
/usr/bin/kvm \
-id 8006 \
- -name vm8006 \
- -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server,nowait' \
+ -name 'vm8006,debug-threads=on' \
+ -no-shutdown \
+ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \
-mon 'chardev=qmp,mode=control' \
-chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' \
-mon 'chardev=qmp-event,mode=control' \
-pidfile /var/run/qemu-server/8006.pid \
-daemonize \
-smbios 'type=1,uuid=3dd750ce-d910-44d0-9493-525c0be4e687' \
- -drive 'if=pflash,unit=0,format=raw,readonly,file=/usr/share/pve-edk2-firmware//OVMF_CODE.fd' \
- -drive 'if=pflash,unit=1,format=qcow2,id=drive-efidisk0,file=/var/lib/vz/images/100/vm-100-disk-1.qcow2' \
+ -drive 'if=pflash,unit=0,format=raw,readonly=on,file=/usr/share/pve-edk2-firmware//OVMF_CODE.fd' \
+ -drive 'if=pflash,unit=1,id=drive-efidisk0,format=qcow2,file=/var/lib/vz/images/100/vm-100-disk-1.qcow2' \
-smp '2,sockets=2,cores=1,maxcpus=2' \
-nodefaults \
-boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \
- -vnc unix:/var/run/qemu-server/8006.vnc,password \
- -no-hpet \
+ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \
-cpu 'kvm64,enforce,hv_ipi,hv_relaxed,hv_reset,hv_runtime,hv_spinlocks=0x1fff,hv_stimer,hv_synic,hv_time,hv_vapic,hv_vpindex,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep' \
-m 512 \
-object 'memory-backend-ram,id=ram-node0,size=256M' \
-netdev 'type=tap,id=net0,ifname=tap8006i0,script=/var/lib/qemu-server/pve-bridge,downscript=/var/lib/qemu-server/pve-bridgedown,vhost=on' \
-device 'virtio-net-pci,mac=2E:01:68:F9:9C:87,netdev=net0,bus=pci.0,addr=0x12,id=net0,bootindex=300' \
-rtc 'driftfix=slew,base=localtime' \
- -machine 'type=q35+pve0' \
+ -machine 'hpet=off,type=pc-q35-5.1+pve0' \
-global 'kvm-pit.lost_tick_policy=discard'
--- /dev/null
+# TEST: Test for new xhci controller with new machine version
+# QEMU_VERSION: 7.1.0
+cores: 2
+memory: 768
+name: spiceusb3
+net0: virtio=A2:C0:43:77:08:A1,bridge=vmbr0
+ostype: l26
+scsihw: virtio-scsi-pci
+smbios1: uuid=7b10d7af-b932-4c66-b2c3-3996152ec465
+vmgenid: c773c261-d800-4348-9f5d-167fadd53cf8
+vga: qxl
+usb1: spice
+usb5: spice
+usb13: host=1-14
--- /dev/null
+/usr/bin/kvm \
+ -id 8006 \
+ -name 'spiceusb3,debug-threads=on' \
+ -no-shutdown \
+ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \
+ -mon 'chardev=qmp,mode=control' \
+ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' \
+ -mon 'chardev=qmp-event,mode=control' \
+ -pidfile /var/run/qemu-server/8006.pid \
+ -daemonize \
+ -smbios 'type=1,uuid=7b10d7af-b932-4c66-b2c3-3996152ec465' \
+ -smp '2,sockets=1,cores=2,maxcpus=2' \
+ -nodefaults \
+ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \
+ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \
+ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \
+ -m 768 \
+ -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \
+ -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \
+ -device 'vmgenid,guid=c773c261-d800-4348-9f5d-167fadd53cf8' \
+ -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \
+ -device 'qemu-xhci,p2=15,p3=15,id=xhci,bus=pci.1,addr=0x1b' \
+ -chardev 'spicevmc,id=usbredirchardev1,name=usbredir' \
+ -device 'usb-redir,chardev=usbredirchardev1,id=usbredirdev1,bus=xhci.0,port=2' \
+ -chardev 'spicevmc,id=usbredirchardev5,name=usbredir' \
+ -device 'usb-redir,chardev=usbredirchardev5,id=usbredirdev5,bus=xhci.0,port=6' \
+ -device 'usb-host,bus=xhci.0,port=14,hostbus=1,hostport=14,id=usb13' \
+ -device 'qxl-vga,id=vga,max_outputs=4,bus=pci.0,addr=0x2' \
+ -device 'virtio-serial,id=spice,bus=pci.0,addr=0x9' \
+ -chardev 'spicevmc,id=vdagent,name=vdagent' \
+ -device 'virtserialport,chardev=vdagent,name=com.redhat.spice.0' \
+ -spice 'tls-port=61000,addr=127.0.0.1,tls-ciphers=HIGH,seamless-migration=on' \
+ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \
+ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \
+ -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/var/lib/qemu-server/pve-bridge,downscript=/var/lib/qemu-server/pve-bridgedown,vhost=on' \
+ -device 'virtio-net-pci,mac=A2:C0:43:77:08:A1,netdev=net0,bus=pci.0,addr=0x12,id=net0,rx_queue_size=1024,tx_queue_size=256,bootindex=300' \
+ -machine 'type=pc+pve0'
--- /dev/null
+# TEST: Test Q35 USB passthrough combination with qemu-xhci
+# QEMU_VERSION: 7.1.0
+cores: 2
+memory: 768
+name: q35-qemu-xhci
+net0: virtio=A2:C0:43:77:08:A1,bridge=vmbr0
+ostype: l26
+machine: pc-q35-7.1
+scsihw: virtio-scsi-pci
+smbios1: uuid=7b10d7af-b932-4c66-b2c3-3996152ec465
+vmgenid: c773c261-d800-4348-9f5d-167fadd53cf8
+usb1: spice
--- /dev/null
+/usr/bin/kvm \
+ -id 8006 \
+ -name 'q35-qemu-xhci,debug-threads=on' \
+ -no-shutdown \
+ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \
+ -mon 'chardev=qmp,mode=control' \
+ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' \
+ -mon 'chardev=qmp-event,mode=control' \
+ -pidfile /var/run/qemu-server/8006.pid \
+ -daemonize \
+ -smbios 'type=1,uuid=7b10d7af-b932-4c66-b2c3-3996152ec465' \
+ -smp '2,sockets=1,cores=2,maxcpus=2' \
+ -nodefaults \
+ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \
+ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \
+ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \
+ -m 768 \
+ -readconfig /usr/share/qemu-server/pve-q35-4.0.cfg \
+ -device 'vmgenid,guid=c773c261-d800-4348-9f5d-167fadd53cf8' \
+ -device 'qemu-xhci,p2=15,p3=15,id=xhci,bus=pci.1,addr=0x1b' \
+ -device 'usb-tablet,id=tablet,bus=ehci.0,port=1' \
+ -chardev 'spicevmc,id=usbredirchardev1,name=usbredir' \
+ -device 'usb-redir,chardev=usbredirchardev1,id=usbredirdev1,bus=xhci.0,port=2' \
+ -device 'VGA,id=vga,bus=pcie.0,addr=0x1' \
+ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \
+ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \
+ -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/var/lib/qemu-server/pve-bridge,downscript=/var/lib/qemu-server/pve-bridgedown,vhost=on' \
+ -device 'virtio-net-pci,mac=A2:C0:43:77:08:A1,netdev=net0,bus=pci.0,addr=0x12,id=net0,rx_queue_size=1024,tx_queue_size=256,bootindex=300' \
+ -machine 'type=pc-q35-7.1+pve0'
--- /dev/null
+# TEST: Test for smm-related regression with SeaBIOS and serial display
+bootdisk: scsi0
+cores: 3
+ide2: none,media=cdrom
+memory: 768
+name: seabiosserial
+net0: virtio=A2:C0:43:77:08:A0,bridge=vmbr0
+numa: 0
+ostype: l26
+scsi0: local:8006/vm-8006-disk-0.qcow2,discard=on,size=104858K
+scsihw: virtio-scsi-pci
+serial0: socket
+smbios1: uuid=7b10d7af-b932-4c66-b2c3-3996152ec465
+sockets: 1
+vga: serial0
+vmgenid: c773c261-d800-4348-9f5d-167fadd53cf8
--- /dev/null
+/usr/bin/kvm \
+ -id 8006 \
+ -name 'seabiosserial,debug-threads=on' \
+ -no-shutdown \
+ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \
+ -mon 'chardev=qmp,mode=control' \
+ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' \
+ -mon 'chardev=qmp-event,mode=control' \
+ -pidfile /var/run/qemu-server/8006.pid \
+ -daemonize \
+ -smbios 'type=1,uuid=7b10d7af-b932-4c66-b2c3-3996152ec465' \
+ -smp '3,sockets=1,cores=3,maxcpus=3' \
+ -nodefaults \
+ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \
+ -nographic \
+ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \
+ -m 768 \
+ -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \
+ -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \
+ -device 'vmgenid,guid=c773c261-d800-4348-9f5d-167fadd53cf8' \
+ -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \
+ -chardev 'socket,id=serial0,path=/var/run/qemu-server/8006.serial0,server=on,wait=off' \
+ -device 'isa-serial,chardev=serial0' \
+ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \
+ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \
+ -drive 'if=none,id=drive-ide2,media=cdrom,aio=io_uring' \
+ -device 'ide-cd,bus=ide.1,unit=0,drive=drive-ide2,id=ide2,bootindex=200' \
+ -device 'virtio-scsi-pci,id=scsihw0,bus=pci.0,addr=0x5' \
+ -drive 'file=/var/lib/vz/images/8006/vm-8006-disk-0.qcow2,if=none,id=drive-scsi0,discard=on,format=qcow2,cache=none,aio=io_uring,detect-zeroes=unmap' \
+ -device 'scsi-hd,bus=scsihw0.0,channel=0,scsi-id=0,lun=0,drive=drive-scsi0,id=scsi0,bootindex=100' \
+ -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/var/lib/qemu-server/pve-bridge,downscript=/var/lib/qemu-server/pve-bridgedown,vhost=on' \
+ -device 'virtio-net-pci,mac=A2:C0:43:77:08:A0,netdev=net0,bus=pci.0,addr=0x12,id=net0,rx_queue_size=1024,tx_queue_size=256,bootindex=300' \
+ -machine 'smm=off,type=pc+pve0'
--- /dev/null
+# TEST: Simple test for balloon free page reporting enabled by default on 6.2
+# QEMU_VERSION: 6.2
+bootdisk: scsi0
+cores: 3
+ide2: none,media=cdrom
+memory: 768
+name: simple
+net0: virtio=A2:C0:43:77:08:A0,bridge=vmbr0
+numa: 0
+ostype: l26
+scsi0: local:8006/vm-8006-disk-0.qcow2,discard=on,size=104858K
+scsihw: virtio-scsi-pci
+smbios1: uuid=7b10d7af-b932-4c66-b2c3-3996152ec465
+sockets: 1
+vmgenid: c773c261-d800-4348-1010-1010add53cf8
--- /dev/null
+/usr/bin/kvm \
+ -id 8006 \
+ -name 'simple,debug-threads=on' \
+ -no-shutdown \
+ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \
+ -mon 'chardev=qmp,mode=control' \
+ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' \
+ -mon 'chardev=qmp-event,mode=control' \
+ -pidfile /var/run/qemu-server/8006.pid \
+ -daemonize \
+ -smbios 'type=1,uuid=7b10d7af-b932-4c66-b2c3-3996152ec465' \
+ -smp '3,sockets=1,cores=3,maxcpus=3' \
+ -nodefaults \
+ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \
+ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \
+ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \
+ -m 768 \
+ -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \
+ -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \
+ -device 'vmgenid,guid=c773c261-d800-4348-1010-1010add53cf8' \
+ -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \
+ -device 'usb-tablet,id=tablet,bus=uhci.0,port=1' \
+ -device 'VGA,id=vga,bus=pci.0,addr=0x2' \
+ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \
+ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \
+ -drive 'if=none,id=drive-ide2,media=cdrom,aio=io_uring' \
+ -device 'ide-cd,bus=ide.1,unit=0,drive=drive-ide2,id=ide2,bootindex=200' \
+ -device 'virtio-scsi-pci,id=scsihw0,bus=pci.0,addr=0x5' \
+ -drive 'file=/var/lib/vz/images/8006/vm-8006-disk-0.qcow2,if=none,id=drive-scsi0,discard=on,format=qcow2,cache=none,aio=io_uring,detect-zeroes=unmap' \
+ -device 'scsi-hd,bus=scsihw0.0,channel=0,scsi-id=0,lun=0,drive=drive-scsi0,id=scsi0,bootindex=100' \
+ -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/var/lib/qemu-server/pve-bridge,downscript=/var/lib/qemu-server/pve-bridgedown,vhost=on' \
+ -device 'virtio-net-pci,mac=A2:C0:43:77:08:A0,netdev=net0,bus=pci.0,addr=0x12,id=net0,bootindex=300' \
+ -machine 'type=pc+pve0'
--- /dev/null
+# TEST: Simple test for a BTRFS backed VM, which shouldn't use cache=none like other storages
+# QEMU_VERSION: 6.0
+bootdisk: scsi0
+cores: 3
+ide2: none,media=cdrom
+memory: 768
+name: simple
+net0: virtio=A2:C0:43:77:08:A0,bridge=vmbr0
+numa: 0
+ostype: l26
+scsi0: btrfs-store:8006/vm-8006-disk-0.raw,discard=on,size=104858K
+scsihw: virtio-scsi-pci
+smbios1: uuid=7b10d7af-b932-4c66-b2c3-3996152ec465
+sockets: 1
+vmgenid: c773c261-d800-4348-1010-1010add53cf8
--- /dev/null
+/usr/bin/kvm \
+ -id 8006 \
+ -name 'simple,debug-threads=on' \
+ -no-shutdown \
+ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \
+ -mon 'chardev=qmp,mode=control' \
+ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' \
+ -mon 'chardev=qmp-event,mode=control' \
+ -pidfile /var/run/qemu-server/8006.pid \
+ -daemonize \
+ -smbios 'type=1,uuid=7b10d7af-b932-4c66-b2c3-3996152ec465' \
+ -smp '3,sockets=1,cores=3,maxcpus=3' \
+ -nodefaults \
+ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \
+ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \
+ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \
+ -m 768 \
+ -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \
+ -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \
+ -device 'vmgenid,guid=c773c261-d800-4348-1010-1010add53cf8' \
+ -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \
+ -device 'usb-tablet,id=tablet,bus=uhci.0,port=1' \
+ -device 'VGA,id=vga,bus=pci.0,addr=0x2' \
+ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3' \
+ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \
+ -drive 'if=none,id=drive-ide2,media=cdrom,aio=io_uring' \
+ -device 'ide-cd,bus=ide.1,unit=0,drive=drive-ide2,id=ide2,bootindex=200' \
+ -device 'virtio-scsi-pci,id=scsihw0,bus=pci.0,addr=0x5' \
+ -drive 'file=/butter/bread/images/8006/vm-8006-disk-0/disk.raw,if=none,id=drive-scsi0,discard=on,format=raw,aio=io_uring,detect-zeroes=unmap' \
+ -device 'scsi-hd,bus=scsihw0.0,channel=0,scsi-id=0,lun=0,drive=drive-scsi0,id=scsi0,bootindex=100' \
+ -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/var/lib/qemu-server/pve-bridge,downscript=/var/lib/qemu-server/pve-bridgedown,vhost=on' \
+ -device 'virtio-net-pci,mac=A2:C0:43:77:08:A0,netdev=net0,bus=pci.0,addr=0x12,id=net0,bootindex=300' \
+ -machine 'type=pc+pve0'
/usr/bin/kvm \
-id 8006 \
- -name simple \
- -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server,nowait' \
+ -name 'simple,debug-threads=on' \
+ -no-shutdown \
+ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \
-mon 'chardev=qmp,mode=control' \
-chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' \
-mon 'chardev=qmp-event,mode=control' \
-smp '3,sockets=1,cores=3,maxcpus=3' \
-nodefaults \
-boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \
- -vnc unix:/var/run/qemu-server/8006.vnc,password \
+ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \
-cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \
-m 768 \
-object 'iothread,id=iothread-virtio0' \
--- /dev/null
+# TEST: Simple test for a basic template configuration
+# QEMU_VERSION: 3.0
+bootdisk: scsi0
+cores: 3
+ide2: none,media=cdrom
+memory: 768
+name: simple
+net0: virtio=A2:C0:43:77:08:A0,bridge=vmbr0
+numa: 0
+ostype: l26
+sata0: local:8006/base-8006-disk-0.qcow2,discard=on,size=104858K
+scsi0: local:8006/base-8006-disk-1.qcow2,discard=on,size=104858K
+scsihw: virtio-scsi-pci
+smbios1: uuid=7b10d7af-b932-4c66-b2c3-3996152ec465
+sockets: 1
+template: 1
+vmgenid: c773c261-d800-4348-9f5d-167fadd53cf8
--- /dev/null
+/usr/bin/kvm \
+ -id 8006 \
+ -name 'simple,debug-threads=on' \
+ -no-shutdown \
+ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \
+ -mon 'chardev=qmp,mode=control' \
+ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' \
+ -mon 'chardev=qmp-event,mode=control' \
+ -pidfile /var/run/qemu-server/8006.pid \
+ -daemonize \
+ -smbios 'type=1,uuid=7b10d7af-b932-4c66-b2c3-3996152ec465' \
+ -smp '3,sockets=1,cores=3,maxcpus=3' \
+ -nodefaults \
+ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \
+ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \
+ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \
+ -m 768 \
+ -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \
+ -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \
+ -device 'vmgenid,guid=c773c261-d800-4348-9f5d-167fadd53cf8' \
+ -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \
+ -device 'usb-tablet,id=tablet,bus=uhci.0,port=1' \
+ -device 'VGA,id=vga,bus=pci.0,addr=0x2' \
+ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3' \
+ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \
+ -drive 'if=none,id=drive-ide2,media=cdrom,aio=threads' \
+ -device 'ide-cd,bus=ide.1,unit=0,drive=drive-ide2,id=ide2,bootindex=200' \
+ -device 'virtio-scsi-pci,id=scsihw0,bus=pci.0,addr=0x5' \
+ -drive 'file=/var/lib/vz/images/8006/base-8006-disk-1.qcow2,if=none,id=drive-scsi0,discard=on,format=qcow2,cache=none,aio=native,detect-zeroes=unmap,readonly=on' \
+ -device 'scsi-hd,bus=scsihw0.0,channel=0,scsi-id=0,lun=0,drive=drive-scsi0,id=scsi0,bootindex=100' \
+ -device 'ahci,id=ahci0,multifunction=on,bus=pci.0,addr=0x7' \
+ -drive 'file=/var/lib/vz/images/8006/base-8006-disk-0.qcow2,if=none,id=drive-sata0,discard=on,format=qcow2,cache=none,aio=native,detect-zeroes=unmap' \
+ -device 'ide-hd,bus=ahci0.0,drive=drive-sata0,id=sata0' \
+ -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/var/lib/qemu-server/pve-bridge,downscript=/var/lib/qemu-server/pve-bridgedown,vhost=on' \
+ -device 'virtio-net-pci,mac=A2:C0:43:77:08:A0,netdev=net0,bus=pci.0,addr=0x12,id=net0,bootindex=300' \
+ -machine 'type=pc' \
+ -snapshot
/usr/bin/kvm \
-id 8006 \
- -name simple \
- -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server,nowait' \
+ -name 'simple,debug-threads=on' \
+ -no-shutdown \
+ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \
-mon 'chardev=qmp,mode=control' \
-chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' \
-mon 'chardev=qmp-event,mode=control' \
-smp '3,sockets=1,cores=3,maxcpus=3' \
-nodefaults \
-boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \
- -vnc unix:/var/run/qemu-server/8006.vnc,password \
+ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \
-cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \
-m 768 \
-device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \
/usr/bin/kvm \
-id 8006 \
- -name vm8006 \
- -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server,nowait' \
+ -name 'vm8006,debug-threads=on' \
+ -no-shutdown \
+ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \
-mon 'chardev=qmp,mode=control' \
-chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' \
-mon 'chardev=qmp-event,mode=control' \
-smp '1,sockets=1,cores=1,maxcpus=1' \
-nodefaults \
-boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \
- -vnc unix:/var/run/qemu-server/8006.vnc,password \
+ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \
-cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \
-m 512 \
-device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \
/usr/bin/kvm \
-id 8006 \
- -name spicelinux \
- -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server,nowait' \
+ -name 'spicelinux,debug-threads=on' \
+ -no-shutdown \
+ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \
-mon 'chardev=qmp,mode=control' \
-chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' \
-mon 'chardev=qmp-event,mode=control' \
-smp '2,sockets=1,cores=2,maxcpus=2' \
-nodefaults \
-boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \
- -vnc unix:/var/run/qemu-server/8006.vnc,password \
+ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \
-cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \
-m 768 \
-device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \
/usr/bin/kvm \
-id 8006 \
- -name spiceusb3 \
- -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server,nowait' \
+ -name 'spiceusb3,debug-threads=on' \
+ -no-shutdown \
+ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \
-mon 'chardev=qmp,mode=control' \
-chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' \
-mon 'chardev=qmp-event,mode=control' \
-smp '2,sockets=1,cores=2,maxcpus=2' \
-nodefaults \
-boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \
- -vnc unix:/var/run/qemu-server/8006.vnc,password \
+ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \
-cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \
-m 768 \
-device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \
/usr/bin/kvm \
-id 8006 \
- -name spiceusb3 \
- -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server,nowait' \
+ -name 'spiceusb3,debug-threads=on' \
+ -no-shutdown \
+ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \
-mon 'chardev=qmp,mode=control' \
-chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' \
-mon 'chardev=qmp-event,mode=control' \
-smp '2,sockets=1,cores=2,maxcpus=2' \
-nodefaults \
-boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \
- -vnc unix:/var/run/qemu-server/8006.vnc,password \
- -no-hpet \
+ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \
-cpu 'kvm64,enforce,hv_ipi,hv_relaxed,hv_reset,hv_runtime,hv_spinlocks=0x1fff,hv_stimer,hv_synic,hv_time,hv_vapic,hv_vpindex,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep' \
-m 768 \
-device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \
-netdev 'type=tap,id=net0,ifname=tap8006i0,script=/var/lib/qemu-server/pve-bridge,downscript=/var/lib/qemu-server/pve-bridgedown,vhost=on' \
-device 'virtio-net-pci,mac=A2:C0:43:77:08:A1,netdev=net0,bus=pci.0,addr=0x12,id=net0,bootindex=300' \
-rtc 'driftfix=slew,base=localtime' \
- -machine 'type=pc' \
+ -machine 'hpet=off,type=pc-i440fx-4.0' \
-global 'kvm-pit.lost_tick_policy=discard'
--- /dev/null
+# TEST: Test error for old machine type with newer usb config
+# QEMU_VERSION: 4.0.0
+# EXPECT_ERROR: using usb13 is only possible with machine type >= 7.1 and ostype l26 or windows > 7
+cores: 2
+memory: 768
+name: q35-usb3-error
+net0: virtio=A2:C0:43:77:08:A1,bridge=vmbr0
+ostype: l26
+scsihw: virtio-scsi-pci
+smbios1: uuid=7b10d7af-b932-4c66-b2c3-3996152ec465
+vmgenid: c773c261-d800-4348-9f5d-167fadd53cf8
+vga: qxl
+usb13: spice
--- /dev/null
+# TEST: Test for a VNC clipboard with a SPICE QXL display
+vga: qxl,clipboard=vnc
--- /dev/null
+/usr/bin/kvm \
+ -id 8006 \
+ -name 'vm8006,debug-threads=on' \
+ -no-shutdown \
+ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \
+ -mon 'chardev=qmp,mode=control' \
+ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' \
+ -mon 'chardev=qmp-event,mode=control' \
+ -pidfile /var/run/qemu-server/8006.pid \
+ -daemonize \
+ -smp '1,sockets=1,cores=1,maxcpus=1' \
+ -nodefaults \
+ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \
+ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \
+ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \
+ -m 512 \
+ -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \
+ -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \
+ -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \
+ -device 'qxl-vga,id=vga,max_outputs=4,bus=pci.0,addr=0x2' \
+ -device 'virtio-serial,id=spice,bus=pci.0,addr=0x9' \
+ -chardev 'qemu-vdagent,id=vdagent,name=vdagent,clipboard=on' \
+ -device 'virtserialport,chardev=vdagent,name=com.redhat.spice.0' \
+ -spice 'tls-port=61000,addr=127.0.0.1,tls-ciphers=HIGH,seamless-migration=on' \
+ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \
+ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \
+ -machine 'type=pc+pve0'
--- /dev/null
+# TEST: Test for a VNC clipboard with a std display
+vga: std,clipboard=vnc
--- /dev/null
+/usr/bin/kvm \
+ -id 8006 \
+ -name 'vm8006,debug-threads=on' \
+ -no-shutdown \
+ -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \
+ -mon 'chardev=qmp,mode=control' \
+ -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' \
+ -mon 'chardev=qmp-event,mode=control' \
+ -pidfile /var/run/qemu-server/8006.pid \
+ -daemonize \
+ -smp '1,sockets=1,cores=1,maxcpus=1' \
+ -nodefaults \
+ -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \
+ -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \
+ -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \
+ -m 512 \
+ -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \
+ -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \
+ -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \
+ -device 'usb-tablet,id=tablet,bus=uhci.0,port=1' \
+ -device 'VGA,id=vga,bus=pci.0,addr=0x2' \
+ -device 'virtio-serial,id=spice,bus=pci.0,addr=0x9' \
+ -chardev 'qemu-vdagent,id=vdagent,name=vdagent,clipboard=on' \
+ -device 'virtserialport,chardev=vdagent,name=com.redhat.spice.0' \
+ -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \
+ -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \
+ -machine 'type=pc+pve0'
--- /dev/null
+# regular VM with an EFI disk
+bios: ovmf
+boot: order=scsi0;ide2;net0
+cores: 1
+efidisk0: target:139/vm-139-disk-0.qcow2,size=128K
+ide2: local:iso/debian-10.6.0-amd64-netinst.iso,media=cdrom
+memory: 2048
+name: eficloneclone
+net0: virtio=7A:6C:A5:8B:11:93,bridge=vmbr0,firewall=1
+numa: 0
+ostype: l26
+scsi0: target:139/vm-139-disk-1.raw,size=4G
+scsihw: virtio-scsi-pci
+smbios1: uuid=21a7e7bc-3cd2-4232-a009-a41f4ee992ae
+sockets: 1
+vmgenid: 0
--- /dev/null
+# plain VM
+bootdisk: scsi0
+cores: 1
+ide2: none,media=cdrom
+memory: 512
+name: apache
+net0: virtio=92:38:11:FD:ED:87,bridge=vmbr0,firewall=1
+numa: 0
+ostype: l26
+scsi0: target:142/vm-142-disk-0.qcow2,size=4G
+scsihw: virtio-scsi-pci
+smbios1: uuid=ddf91b3f-a597-42be-9a7e-fb6421dcd5cd
+sockets: 1
+vmgenid: 0
+tags: foo bar
--- /dev/null
+# some properties should be filtered
+bootdisk: scsi0
+cores: 1
+ide2: none,media=cdrom
+memory: 512
+name: apache
+net0: virtio=92:38:11:FD:ED:87,bridge=vmbr0,firewall=1
+numa: 0
+ostype: l26
+scsi0: target:1422/vm-1422-disk-0.qcow2,size=4G
+scsihw: virtio-scsi-pci
+smbios1: uuid=ddf91b3f-a597-42be-9a7e-fb6421dcd5cd
+sockets: 1
+vmgenid: 0
--- /dev/null
+# many disks
+boot: order=scsi0;ide2;net0
+cores: 1
+ide2: none,media=cdrom
+memory: 2048
+net0: virtio=26:15:5B:73:3F:7C,bridge=vmbr0,firewall=1
+numa: 0
+ostype: l26
+scsi0: target:179/vm-179-disk-0.qcow2,cache=none,discard=on,size=32G,ssd=1
+scsi1: target:179/vm-179-disk-1.qcow2,cache=writethrough,size=32G
+scsi2: target:179/vm-179-disk-2.qcow2,mbps_rd=7,mbps_wr=7,replicate=0,size=32G
+scsi3: target:179/vm-179-disk-3.vmdk,size=32G
+#scsi4: myfs:179/vm-179-disk-1.qcow2,backup=0,size=32G
+scsihw: virtio-scsi-pci
+smbios1: uuid=1819ead7-a55d-4544-8d38-29ca94869a9c
+sockets: 1
+vmgenid: 0
--- /dev/null
+# regular VM with an EFI disk
+bios: ovmf
+boot: order=scsi0;ide2;net0
+cores: 1
+efidisk0: mydir:139/vm-139-disk-0.qcow2,size=128K
+ide2: local:iso/debian-10.6.0-amd64-netinst.iso,media=cdrom
+memory: 2048
+name: eficloneclone
+net0: virtio=7A:6C:A5:8B:11:93,bridge=vmbr0,firewall=1
+numa: 0
+ostype: l26
+scsi0: rbdkvm:vm-139-disk-1,size=4G
+scsihw: virtio-scsi-pci
+smbios1: uuid=21a7e7bc-3cd2-4232-a009-a41f4ee992ae
+sockets: 1
+vmgenid: 0
+#qmdump#map:efidisk0:drive-efidisk0:mydir:qcow2:
+#qmdump#map:scsi0:drive-scsi0:rbdkvm::
--- /dev/null
+# plain VM
+bootdisk: scsi0
+cores: 1
+ide2: none,media=cdrom
+memory: 512
+name: apache
+net0: virtio=92:38:11:FD:ED:87,bridge=vmbr0,firewall=1
+numa: 0
+ostype: l26
+scsi0: mydir:142/vm-142-disk-0.qcow2,size=4G
+scsihw: virtio-scsi-pci
+smbios1: uuid=ddf91b3f-a597-42be-9a7e-fb6421dcd5cd
+sockets: 1
+vmgenid: 0
+tags: foo bar
+#qmdump#map:scsi0:drive-scsi0:mydir:qcow2:
--- /dev/null
+# some properties should be filtered
+bootdisk: scsi0
+cores: 1
+ide2: none,media=cdrom
+memory: 512
+name: apache
+net0: virtio=92:38:11:FD:ED:87,bridge=vmbr0,firewall=1
+numa: 0
+ostype: l26
+scsi0: mydir:1422/vm-1422-disk-0.qcow2,size=4G
+unused7: mydir:1422/vm-1422-disk-8.qcow2
+parent: snap
+lock: backup
+scsihw: virtio-scsi-pci
+smbios1: uuid=ddf91b3f-a597-42be-9a7e-fb6421dcd5cd
+sockets: 1
+vmgenid: 0
+#qmdump#map:scsi0:drive-scsi0:mydir:qcow2:
--- /dev/null
+# many disks
+boot: order=scsi0;ide2;net0
+cores: 1
+ide2: none,media=cdrom
+memory: 2048
+net0: virtio=26:15:5B:73:3F:7C,bridge=vmbr0,firewall=1
+numa: 0
+ostype: l26
+scsi0: myfs:179/vm-179-disk-4.qcow2,cache=none,discard=on,size=32G,ssd=1
+scsi1: myfs:179/vm-179-disk-0.qcow2,cache=writethrough,size=32G
+scsi2: myfs:179/vm-179-disk-2.qcow2,mbps_rd=7,mbps_wr=7,replicate=0,size=32G
+scsi3: myfs:179/vm-179-disk-3.vmdk,size=32G
+scsi4: myfs:179/vm-179-disk-1.qcow2,backup=0,size=32G
+scsihw: virtio-scsi-pci
+smbios1: uuid=1819ead7-a55d-4544-8d38-29ca94869a9c
+sockets: 1
+vmgenid: 0
+#qmdump#map:scsi0:drive-scsi0:myfs:qcow2:
+#qmdump#map:scsi1:drive-scsi1:myfs:qcow2:
+#qmdump#map:scsi2:drive-scsi2:myfs:qcow2:
+#qmdump#map:scsi3:drive-scsi3:myfs:vmdk:
type => 'dir',
shared => 0,
},
+ 'btrfs-store' => {
+ content => {
+ images => 1,
+ },
+ path => '/butter/bread',
+ type => 'btrfs',
+ },
'cifs-store' => {
shared => 1,
path => '/mnt/pve/cifs-store',
type => 'cifs',
share => 'CIFShare',
content => {
- images => 1
+ images => 1,
+ iso => 1,
},
},
'rbd-store' => {
monhost => '127.0.0.42,127.0.0.21,::1',
+ fsid => 'fc4181a6-56eb-4f68-b452-8ba1f381ca2a',
content => {
images => 1
},
"0000:0f:f2.0",
"0000:d0:13.0",
"0000:d0:15.1",
+ "0000:d0:15.2",
"0000:d0:17.0",
"0000:f0:42.0",
"0000:f0:43.0",
"0000:f0:43.1",
"1234:f0:43.1",
+ "0000:01:00.4",
+ "0000:01:00.5",
+ "0000:01:00.6",
+ "0000:07:10.0",
+ "0000:07:10.1",
+ "0000:07:10.4",
];
+my $pci_map_config = {
+ ids => {
+ someGpu => {
+ type => 'pci',
+ map => [
+ 'node=localhost,path=0000:01:00.4,id=10de:2231,iommugroup=1',
+ 'node=localhost,path=0000:01:00.5,id=10de:2231,iommugroup=1',
+ 'node=localhost,path=0000:01:00.6,id=10de:2231,iommugroup=1',
+ ],
+ },
+ someNic => {
+ type => 'pci',
+ map => [
+ 'node=localhost,path=0000:07:10.0,id=8086:1520,iommugroup=2',
+ 'node=localhost,path=0000:07:10.1,id=8086:1520,iommugroup=2',
+ 'node=localhost,path=0000:07:10.4,id=8086:1520,iommugroup=2',
+ ],
+ },
+ },
+};
+
+my $usb_map_config = {},
+
my $current_test; # = {
# description => 'Test description', # if available
# qemu_version => '2.12',
},
);
+my $qemu_server_memory;
+$qemu_server_memory = Test::MockModule->new('PVE::QemuServer::Memory');
+$qemu_server_memory->mock(
+ hugepages_chunk_size_supported => sub {
+ return 1;
+ },
+ host_numanode_exists => sub {
+ my ($id) = @_;
+ return 1;
+ },
+ get_host_phys_address_bits => sub {
+ return 46;
+ }
+);
+
my $pve_common_tools;
$pve_common_tools = Test::MockModule->new('PVE::Tools');
$pve_common_tools->mock(
} sort @$pci_devs
];
},
+ pci_device_info => sub {
+ my ($path, $noerr) = @_;
+
+ if ($path =~ m/^0000:01:00/) {
+ return {
+ mdev => 1,
+ iommugroup => 1,
+ mdev => 1,
+ vendor => "0x10de",
+ device => "0x2231",
+ };
+ } elsif ($path =~ m/^0000:07:10/) {
+ return {
+ iommugroup => 2,
+ mdev => 0,
+ vendor => "0x8086",
+ device => "0x1520",
+ };
+ } else {
+ return {};
+ }
+ },
);
my $qemu_monitor_module;
);
$qemu_monitor_module->mock('qmp_cmd', \&qmp_cmd);
+my $mapping_usb_module = Test::MockModule->new("PVE::Mapping::USB");
+$mapping_usb_module->mock(
+ config => sub {
+ return $usb_map_config;
+ },
+);
+
+my $mapping_pci_module = Test::MockModule->new("PVE::Mapping::PCI");
+$mapping_pci_module->mock(
+ config => sub {
+ return $pci_map_config;
+ },
+);
+
+my $pci_module = Test::MockModule->new("PVE::QemuServer::PCI");
+$pci_module->mock(
+ reserve_pci_usage => sub {
+ my ($ids, $vmid, $timeout, $pid, $dryrun) = @_;
+
+ $ids = [$ids] if !ref($ids);
+
+ for my $id (@$ids) {
+ if ($id eq "0000:07:10.1") {
+ die "reserved";
+ }
+ }
+
+ return undef;
+ },
+);
+
sub diff($$) {
my ($a, $b) = @_;
return if $a eq $b;
},
"rbd-store" => {
monhost => "127.0.0.42,127.0.0.21,::1",
+ fsid => 'fc4181a6-56eb-4f68-b452-8ba1f381ca2a',
content => {
images => 1
},
my $tests = [
{
name => 'qcow2raw',
- parameters => [ "local:$vmid/vm-$vmid-disk-0.qcow2", "local:$vmid/vm-$vmid-disk-0.raw", 1024*10, undef, 0 ],
+ parameters => [ "local:$vmid/vm-$vmid-disk-0.qcow2", "local:$vmid/vm-$vmid-disk-0.raw", 1024*10, undef, 0, undef ],
expected => [
"/usr/bin/qemu-img", "convert", "-p", "-n", "-f", "qcow2", "-O", "raw",
"/var/lib/vz/images/$vmid/vm-$vmid-disk-0.qcow2", "/var/lib/vz/images/$vmid/vm-$vmid-disk-0.raw"
},
{
name => "raw2qcow2",
- parameters => [ "local:$vmid/vm-$vmid-disk-0.raw", "local:$vmid/vm-$vmid-disk-0.qcow2", 1024*10, undef, 0 ],
+ parameters => [ "local:$vmid/vm-$vmid-disk-0.raw", "local:$vmid/vm-$vmid-disk-0.qcow2", 1024*10, undef, 0, undef ],
expected => [
"/usr/bin/qemu-img", "convert", "-p", "-n", "-f", "raw", "-O", "qcow2",
"/var/lib/vz/images/$vmid/vm-$vmid-disk-0.raw", "/var/lib/vz/images/$vmid/vm-$vmid-disk-0.qcow2"
},
{
name => "local2rbd",
- parameters => [ "local:$vmid/vm-$vmid-disk-0.raw", "rbd-store:vm-$vmid-disk-0", 1024*10, undef, 0 ],
+ parameters => [ "local:$vmid/vm-$vmid-disk-0.raw", "rbd-store:vm-$vmid-disk-0", 1024*10, undef, 0, undef ],
expected => [
"/usr/bin/qemu-img", "convert", "-p", "-n", "-f", "raw", "-O", "raw",
"/var/lib/vz/images/$vmid/vm-$vmid-disk-0.raw", "rbd:cpool/vm-$vmid-disk-0:mon_host=127.0.0.42;127.0.0.21;[\\:\\:1]:auth_supported=none"
},
{
name => "rbd2local",
- parameters => [ "rbd-store:vm-$vmid-disk-0", "local:$vmid/vm-$vmid-disk-0.raw", 1024*10, undef, 0 ],
+ parameters => [ "rbd-store:vm-$vmid-disk-0", "local:$vmid/vm-$vmid-disk-0.raw", 1024*10, undef, 0, undef ],
expected => [
"/usr/bin/qemu-img", "convert", "-p", "-n", "-f", "raw", "-O", "raw",
"rbd:cpool/vm-$vmid-disk-0:mon_host=127.0.0.42;127.0.0.21;[\\:\\:1]:auth_supported=none", "/var/lib/vz/images/$vmid/vm-$vmid-disk-0.raw"
},
{
name => "local2zos",
- parameters => [ "local:$vmid/vm-$vmid-disk-0.raw", "zfs-over-iscsi:vm-$vmid-disk-0", 1024*10, undef, 0 ],
+ parameters => [ "local:$vmid/vm-$vmid-disk-0.raw", "zfs-over-iscsi:vm-$vmid-disk-0", 1024*10, undef, 0, undef ],
expected => [
"/usr/bin/qemu-img", "convert", "-p", "-n", "-f", "raw", "--target-image-opts",
"/var/lib/vz/images/$vmid/vm-$vmid-disk-0.raw",
},
{
name => "zos2local",
- parameters => [ "zfs-over-iscsi:vm-$vmid-disk-0", "local:$vmid/vm-$vmid-disk-0.raw", 1024*10, undef, 0 ],
+ parameters => [ "zfs-over-iscsi:vm-$vmid-disk-0", "local:$vmid/vm-$vmid-disk-0.raw", 1024*10, undef, 0, undef ],
expected => [
"/usr/bin/qemu-img", "convert", "-p", "-n", "--image-opts", "-O", "raw",
"file.driver=iscsi,file.transport=tcp,file.initiator-name=foobar,file.portal=127.0.0.1,file.target=iqn.2019-10.org.test:foobar,file.lun=1,driver=raw",
},
{
name => "zos2rbd",
- parameters => [ "zfs-over-iscsi:vm-$vmid-disk-0", "rbd-store:vm-$vmid-disk-0", 1024*10, undef, 0 ],
+ parameters => [ "zfs-over-iscsi:vm-$vmid-disk-0", "rbd-store:vm-$vmid-disk-0", 1024*10, undef, 0, undef ],
expected => [
"/usr/bin/qemu-img", "convert", "-p", "-n", "--image-opts", "-O", "raw",
"file.driver=iscsi,file.transport=tcp,file.initiator-name=foobar,file.portal=127.0.0.1,file.target=iqn.2019-10.org.test:foobar,file.lun=1,driver=raw",
},
{
name => "rbd2zos",
- parameters => [ "rbd-store:vm-$vmid-disk-0", "zfs-over-iscsi:vm-$vmid-disk-0", 1024*10, undef, 0 ],
+ parameters => [ "rbd-store:vm-$vmid-disk-0", "zfs-over-iscsi:vm-$vmid-disk-0", 1024*10, undef, 0, undef ],
expected => [
"/usr/bin/qemu-img", "convert", "-p", "-n", "-f", "raw", "--target-image-opts",
"rbd:cpool/vm-$vmid-disk-0:mon_host=127.0.0.42;127.0.0.21;[\\:\\:1]:auth_supported=none",
},
{
name => "local2lvmthin",
- parameters => [ "local:$vmid/vm-$vmid-disk-0.raw", "local-lvm:vm-$vmid-disk-0", 1024*10, undef, 0 ],
+ parameters => [ "local:$vmid/vm-$vmid-disk-0.raw", "local-lvm:vm-$vmid-disk-0", 1024*10, undef, 0, undef ],
expected => [
"/usr/bin/qemu-img", "convert", "-p", "-n", "-f", "raw", "-O", "raw",
"/var/lib/vz/images/$vmid/vm-$vmid-disk-0.raw",
},
{
name => "lvmthin2local",
- parameters => [ "local-lvm:vm-$vmid-disk-0", "local:$vmid/vm-$vmid-disk-0.raw", 1024*10, undef, 0 ],
+ parameters => [ "local-lvm:vm-$vmid-disk-0", "local:$vmid/vm-$vmid-disk-0.raw", 1024*10, undef, 0, undef ],
expected => [
"/usr/bin/qemu-img", "convert", "-p", "-n", "-f", "raw", "-O", "raw",
"/dev/pve/vm-$vmid-disk-0",
},
{
name => "zeroinit",
- parameters => [ "local-lvm:vm-$vmid-disk-0", "local:$vmid/vm-$vmid-disk-0.raw", 1024*10, undef, 1 ],
+ parameters => [ "local-lvm:vm-$vmid-disk-0", "local:$vmid/vm-$vmid-disk-0.raw", 1024*10, undef, 1, undef ],
expected => [
"/usr/bin/qemu-img", "convert", "-p", "-n", "-f", "raw", "-O", "raw",
"/dev/pve/vm-$vmid-disk-0",
},
{
name => "notexistingstorage",
- parameters => [ "local-lvm:vm-$vmid-disk-0", "not-existing:$vmid/vm-$vmid-disk-0.raw", 1024*10, undef, 1 ],
+ parameters => [ "local-lvm:vm-$vmid-disk-0", "not-existing:$vmid/vm-$vmid-disk-0.raw", 1024*10, undef, 1, undef ],
expected => "storage 'not-existing' does not exist\n",
},
{
name => "vmdkfile",
- parameters => [ "./test.vmdk", "local:$vmid/vm-$vmid-disk-0.raw", 1024*10, undef, 0 ],
+ parameters => [ "./test.vmdk", "local:$vmid/vm-$vmid-disk-0.raw", 1024*10, undef, 0, undef ],
expected => [
"/usr/bin/qemu-img", "convert", "-p", "-n", "-f", "vmdk", "-O", "raw",
"./test.vmdk",
},
{
name => "notexistingfile",
- parameters => [ "/foo/bar", "local:$vmid/vm-$vmid-disk-0.raw", 1024*10, undef, 0 ],
+ parameters => [ "/foo/bar", "local:$vmid/vm-$vmid-disk-0.raw", 1024*10, undef, 0, undef ],
expected => "source '/foo/bar' is not a valid volid nor path for qemu-img convert\n",
},
{
name => "efidisk",
- parameters => [ "/usr/share/kvm/OVMF_VARS-pure-efi.fd", "local:$vmid/vm-$vmid-disk-0.raw", 1024*10, undef, 0 ],
+ parameters => [ "/usr/share/kvm/OVMF_VARS-pure-efi.fd", "local:$vmid/vm-$vmid-disk-0.raw", 1024*10, undef, 0, undef ],
expected => [
"/usr/bin/qemu-img", "convert", "-p", "-n", "-O", "raw",
"/usr/share/kvm/OVMF_VARS-pure-efi.fd",
},
{
name => "efi2zos",
- parameters => [ "/usr/share/kvm/OVMF_VARS-pure-efi.fd", "zfs-over-iscsi:vm-$vmid-disk-0", 1024*10, undef, 0 ],
+ parameters => [ "/usr/share/kvm/OVMF_VARS-pure-efi.fd", "zfs-over-iscsi:vm-$vmid-disk-0", 1024*10, undef, 0, undef ],
expected => [
"/usr/bin/qemu-img", "convert", "-p", "-n", "--target-image-opts",
"/usr/share/kvm/OVMF_VARS-pure-efi.fd",
"file.driver=iscsi,file.transport=tcp,file.initiator-name=foobar,file.portal=127.0.0.1,file.target=iqn.2019-10.org.test:foobar,file.lun=1,driver=raw",
]
- }
+ },
+ {
+ name => "bwlimit",
+ parameters => [ "local-lvm:vm-$vmid-disk-0", "local:$vmid/vm-$vmid-disk-0.raw", 1024*10, undef, 0, 1024 ],
+ expected => [
+ "/usr/bin/qemu-img", "convert", "-p", "-n", "-r", "1024K", "-f", "raw", "-O", "raw",
+ "/dev/pve/vm-$vmid-disk-0",
+ "/var/lib/vz/images/$vmid/vm-$vmid-disk-0.raw",
+ ]
+ },
];
my $command;
--- /dev/null
+#!/usr/bin/perl
+
+use strict;
+use warnings;
+
+use JSON;
+use Test::More;
+use Test::MockModule;
+
+use PVE::JSONSchema;
+use PVE::Tools qw(file_set_contents file_get_contents run_command);
+
+my $QM_LIB_PATH = '..';
+my $MIGRATE_LIB_PATH = '..';
+my $RUN_DIR_PATH = './MigrationTest/run/';
+
+# test configuration shared by all tests
+
+my $replication_config = {
+ 'ids' => {
+ '105-0' => {
+ 'guest' => '105',
+ 'id' => '105-0',
+ 'jobnum' => '0',
+ 'source' => 'pve0',
+ 'target' => 'pve2',
+ 'type' => 'local'
+ },
+ },
+ 'order' => {
+ '105-0' => 1,
+ }
+};
+
+my $storage_config = {
+ ids => {
+ local => {
+ content => {
+ images => 1,
+ },
+ path => "/var/lib/vz",
+ type => "dir",
+ shared => 0,
+ },
+ "local-lvm" => {
+ content => {
+ images => 1,
+ },
+ nodes => {
+ pve0 => 1,
+ pve1 => 1,
+ },
+ type => "lvmthin",
+ thinpool => "data",
+ vgname => "pve",
+ },
+ "local-zfs" => {
+ content => {
+ images => 1,
+ rootdir => 1,
+ },
+ pool => "rpool/data",
+ sparse => 1,
+ type => "zfspool",
+ },
+ "rbd-store" => {
+ monhost => "127.0.0.42,127.0.0.21,::1",
+ fsid => 'fc4181a6-56eb-4f68-b452-8ba1f381ca2a',
+ content => {
+ images => 1,
+ },
+ type => "rbd",
+ pool => "cpool",
+ username => "admin",
+ shared => 1,
+ },
+ "local-dir" => {
+ content => {
+ images => 1,
+ },
+ path => "/some/dir/",
+ type => "dir",
+ },
+ "other-dir" => {
+ content => {
+ images => 1,
+ },
+ path => "/some/other/dir/",
+ type => "dir",
+ },
+ "zfs-alias-1" => {
+ content => {
+ images => 1,
+ rootdir => 1,
+ },
+ pool => "aliaspool",
+ sparse => 1,
+ type => "zfspool",
+ },
+ "zfs-alias-2" => {
+ content => {
+ images => 1,
+ rootdir => 1,
+ },
+ pool => "aliaspool",
+ sparse => 1,
+ type => "zfspool",
+ },
+ },
+};
+
+my $vm_configs = {
+ 105 => {
+ 'bootdisk' => 'scsi0',
+ 'cores' => 1,
+ 'ide0' => 'local-zfs:vm-105-disk-1,size=103M',
+ 'ide2' => 'none,media=cdrom',
+ 'memory' => 512,
+ 'name' => 'Copy-of-VM-newapache',
+ 'net0' => 'virtio=4A:A3:E4:4C:CF:F0,bridge=vmbr0,firewall=1',
+ 'numa' => 0,
+ 'ostype' => 'l26',
+ 'parent' => 'ohsnap',
+ 'pending' => {},
+ 'scsi0' => 'local-zfs:vm-105-disk-0,size=4G',
+ 'scsihw' => 'virtio-scsi-pci',
+ 'smbios1' => 'uuid=1ddfe18b-77e0-47f6-a4bd-f1761bf6d763',
+ 'snapshots' => {
+ 'ohsnap' => {
+ 'bootdisk' => 'scsi0',
+ 'cores' => 1,
+ 'ide2' => 'none,media=cdrom',
+ 'memory' => 512,
+ 'name' => 'Copy-of-VM-newapache',
+ 'net0' => 'virtio=4A:A3:E4:4C:CF:F0,bridge=vmbr0,firewall=1',
+ 'numa' => 0,
+ 'ostype' => 'l26',
+ 'scsi0' => 'local-zfs:vm-105-disk-0,size=4G',
+ 'scsihw' => 'virtio-scsi-pci',
+ 'smbios1' => 'uuid=1ddfe18b-77e0-47f6-a4bd-f1761bf6d763',
+ 'snaptime' => 1580976924,
+ 'sockets' => 1,
+ 'startup' => 'order=2',
+ 'vmgenid' => '4eb1d535-9381-4ddc-a8aa-af50c4d9177b'
+ },
+ },
+ 'sockets' => 1,
+ 'startup' => 'order=2',
+ 'vmgenid' => '4eb1d535-9381-4ddc-a8aa-af50c4d9177b',
+ },
+ 111 => {
+ 'bootdisk' => 'scsi0',
+ 'cores' => 1,
+ 'ide0' => 'local-lvm:vm-111-disk-0,size=4096M',
+ 'ide2' => 'none,media=cdrom',
+ 'memory' => 512,
+ 'name' => 'pending-test',
+ 'net0' => 'virtio=4A:A3:E4:4C:CF:F0,bridge=vmbr0,firewall=1',
+ 'numa' => 0,
+ 'ostype' => 'l26',
+ 'pending' => {
+ 'scsi0' => 'local-zfs:vm-111-disk-0,size=103M',
+ },
+ 'scsihw' => 'virtio-scsi-pci',
+ 'snapshots' => {},
+ 'smbios1' => 'uuid=5ad71d4d-8f73-4377-853e-2d22c10c96a5',
+ 'sockets' => 1,
+ 'vmgenid' => '2c00c030-0b5b-4988-a371-6ab259893f22',
+ },
+ 123 => {
+ 'bootdisk' => 'scsi0',
+ 'cores' => 1,
+ 'scsi0' => 'zfs-alias-1:vm-123-disk-0,size=4096M',
+ 'scsi1' => 'zfs-alias-2:vm-123-disk-0,size=4096M',
+ 'ide2' => 'none,media=cdrom',
+ 'memory' => 512,
+ 'name' => 'alias-test',
+ 'net0' => 'virtio=4A:A3:E4:4C:CF:F0,bridge=vmbr0,firewall=1',
+ 'numa' => 0,
+ 'ostype' => 'l26',
+ 'pending' => {},
+ 'scsihw' => 'virtio-scsi-pci',
+ 'snapshots' => {},
+ 'smbios1' => 'uuid=5ad71d4d-8f73-4377-853e-2d22c10c96a5',
+ 'sockets' => 1,
+ 'vmgenid' => '2c00c030-0b5b-4988-a371-6ab259893f22',
+ },
+ 149 => {
+ 'agent' => '0',
+ 'bootdisk' => 'scsi0',
+ 'cores' => 1,
+ 'hotplug' => 'disk,network,usb,memory,cpu',
+ 'ide2' => 'none,media=cdrom',
+ 'memory' => 4096,
+ 'name' => 'asdf',
+ 'net0' => 'virtio=52:5D:7E:62:85:97,bridge=vmbr1',
+ 'numa' => 1,
+ 'ostype' => 'l26',
+ 'scsi0' => 'local-lvm:vm-149-disk-0,format=raw,size=4G',
+ 'scsi1' => 'local-dir:149/vm-149-disk-0.qcow2,format=qcow2,size=1G',
+ 'scsihw' => 'virtio-scsi-pci',
+ 'snapshots' => {},
+ 'smbios1' => 'uuid=e980bd43-a405-42e2-b5f4-31efe6517460',
+ 'sockets' => 1,
+ 'startup' => 'order=2',
+ 'vmgenid' => '36c6c50c-6ef5-4adc-9b6f-6ba9c8071db0',
+ },
+ 341 => {
+ 'arch' => 'aarch64',
+ 'bootdisk' => 'scsi0',
+ 'cores' => 1,
+ 'efidisk0' => 'local-lvm:vm-341-disk-0',
+ 'ide2' => 'none,media=cdrom',
+ 'ipconfig0' => 'ip=103.214.69.10/25,gw=103.214.69.1',
+ 'memory' => 4096,
+ 'name' => 'VM1033',
+ 'net0' => 'virtio=4E:F1:82:6D:D7:4B,bridge=vmbr0,firewall=1,rate=10',
+ 'numa' => 0,
+ 'ostype' => 'l26',
+ 'scsi0' => 'rbd-store:vm-341-disk-0,size=1G',
+ 'scsihw' => 'virtio-scsi-pci',
+ 'snapshots' => {},
+ 'smbios1' => 'uuid=e01e4c73-46f1-47c8-af79-288fdf6b7462',
+ 'sockets' => 2,
+ 'vmgenid' => 'af47c000-eb0c-48e8-8991-ca4593cd6916',
+ },
+ 1033 => {
+ 'bootdisk' => 'scsi0',
+ 'cores' => 1,
+ 'ide0' => 'rbd-store:vm-1033-cloudinit,media=cdrom,size=4M',
+ 'ide2' => 'none,media=cdrom',
+ 'ipconfig0' => 'ip=103.214.69.10/25,gw=103.214.69.1',
+ 'memory' => 4096,
+ 'name' => 'VM1033',
+ 'net0' => 'virtio=4E:F1:82:6D:D7:4B,bridge=vmbr0,firewall=1,rate=10',
+ 'numa' => 0,
+ 'ostype' => 'l26',
+ 'scsi0' => 'rbd-store:vm-1033-disk-1,size=1G',
+ 'scsihw' => 'virtio-scsi-pci',
+ 'snapshots' => {},
+ 'smbios1' => 'uuid=e01e4c73-46f1-47c8-af79-288fdf6b7462',
+ 'sockets' => 2,
+ 'vmgenid' => 'af47c000-eb0c-48e8-8991-ca4593cd6916',
+ },
+ 4567 => {
+ 'bootdisk' => 'scsi0',
+ 'cores' => 1,
+ 'ide2' => 'none,media=cdrom',
+ 'memory' => 512,
+ 'name' => 'snapme',
+ 'net0' => 'virtio=A6:D1:F1:EB:7B:C2,bridge=vmbr0,firewall=1',
+ 'numa' => 0,
+ 'ostype' => 'l26',
+ 'parent' => 'snap1',
+ 'pending' => {},
+ 'scsi0' => 'local-dir:4567/vm-4567-disk-0.qcow2,size=4G',
+ 'scsihw' => 'virtio-scsi-pci',
+ 'smbios1' => 'uuid=2925fdec-a066-4228-b46b-eef8662f5e74',
+ 'snapshots' => {
+ 'snap1' => {
+ 'bootdisk' => 'scsi0',
+ 'cores' => 1,
+ 'ide2' => 'none,media=cdrom',
+ 'memory' => 512,
+ 'name' => 'snapme',
+ 'net0' => 'virtio=A6:D1:F1:EB:7B:C2,bridge=vmbr0,firewall=1',
+ 'numa' => 0,
+ 'ostype' => 'l26',
+ 'runningcpu' => 'kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep',
+ 'runningmachine' => 'pc-i440fx-5.0+pve0',
+ 'scsi0' => 'local-dir:4567/vm-4567-disk-0.qcow2,size=4G',
+ 'scsihw' => 'virtio-scsi-pci',
+ 'smbios1' => 'uuid=2925fdec-a066-4228-b46b-eef8662f5e74',
+ 'snaptime' => 1595928799,
+ 'sockets' => 1,
+ 'startup' => 'order=2',
+ 'vmgenid' => '932b227a-8a39-4ede-955a-dbd4bc4385ed',
+ 'vmstate' => 'local-dir:4567/vm-4567-state-snap1.raw',
+ },
+ 'snap2' => {
+ 'bootdisk' => 'scsi0',
+ 'cores' => 1,
+ 'ide2' => 'none,media=cdrom',
+ 'memory' => 512,
+ 'name' => 'snapme',
+ 'net0' => 'virtio=A6:D1:F1:EB:7B:C2,bridge=vmbr0,firewall=1',
+ 'numa' => 0,
+ 'ostype' => 'l26',
+ 'parent' => 'snap1',
+ 'runningcpu' => 'kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep',
+ 'runningmachine' => 'pc-i440fx-5.0+pve0',
+ 'scsi0' => 'local-dir:4567/vm-4567-disk-0.qcow2,size=4G',
+ 'scsi1' => 'local-zfs:vm-4567-disk-0,size=1G',
+ 'scsihw' => 'virtio-scsi-pci',
+ 'smbios1' => 'uuid=2925fdec-a066-4228-b46b-eef8662f5e74',
+ 'snaptime' => 1595928871,
+ 'sockets' => 1,
+ 'startup' => 'order=2',
+ 'vmgenid' => '932b227a-8a39-4ede-955a-dbd4bc4385ed',
+ 'vmstate' => 'local-dir:4567/vm-4567-state-snap2.raw',
+ },
+ },
+ 'sockets' => 1,
+ 'startup' => 'order=2',
+ 'unused0' => 'local-zfs:vm-4567-disk-0',
+ 'vmgenid' => 'e698e60c-9278-4dd9-941f-416075383f2a',
+ },
+};
+
+my $source_vdisks = {
+ 'local-dir' => [
+ {
+ 'ctime' => 1589439681,
+ 'format' => 'qcow2',
+ 'parent' => undef,
+ 'size' => 1073741824,
+ 'used' => 335872,
+ 'vmid' => '149',
+ 'volid' => 'local-dir:149/vm-149-disk-0.qcow2',
+ },
+ {
+ 'ctime' => 1595928898,
+ 'format' => 'qcow2',
+ 'parent' => undef,
+ 'size' => 4294967296,
+ 'used' => 1811664896,
+ 'vmid' => '4567',
+ 'volid' => 'local-dir:4567/vm-4567-disk-0.qcow2',
+ },
+ {
+ 'ctime' => 1595928800,
+ 'format' => 'raw',
+ 'parent' => undef,
+ 'size' => 274666496,
+ 'used' => 274669568,
+ 'vmid' => '4567',
+ 'volid' => 'local-dir:4567/vm-4567-state-snap1.raw',
+ },
+ {
+ 'ctime' => 1595928872,
+ 'format' => 'raw',
+ 'parent' => undef,
+ 'size' => 273258496,
+ 'used' => 273260544,
+ 'vmid' => '4567',
+ 'volid' => 'local-dir:4567/vm-4567-state-snap2.raw',
+ },
+ ],
+ 'local-lvm' => [
+ {
+ 'ctime' => '1589277334',
+ 'format' => 'raw',
+ 'size' => 4294967296,
+ 'vmid' => '149',
+ 'volid' => 'local-lvm:vm-149-disk-0',
+ },
+ {
+ 'ctime' => '1589277334',
+ 'format' => 'raw',
+ 'size' => 4194304,
+ 'vmid' => '341',
+ 'volid' => 'local-lvm:vm-341-disk-0',
+ },
+ {
+ 'ctime' => '1589277334',
+ 'format' => 'raw',
+ 'size' => 4294967296,
+ 'vmid' => '111',
+ 'volid' => 'local-lvm:vm-111-disk-0',
+ },
+ ],
+ 'local-zfs' => [
+ {
+ 'ctime' => '1589277334',
+ 'format' => 'raw',
+ 'size' => 4294967296,
+ 'vmid' => '105',
+ 'volid' => 'local-zfs:vm-105-disk-0',
+ },
+ {
+ 'ctime' => '1589277334',
+ 'format' => 'raw',
+ 'size' => 108003328,
+ 'vmid' => '105',
+ 'volid' => 'local-zfs:vm-105-disk-1',
+ },
+ {
+ 'ctime' => '1589277334',
+ 'format' => 'raw',
+ 'size' => 108003328,
+ 'vmid' => '111',
+ 'volid' => 'local-zfs:vm-111-disk-0',
+ },
+ {
+ 'format' => 'raw',
+ 'name' => 'vm-4567-disk-0',
+ 'parent' => undef,
+ 'size' => 1073741824,
+ 'vmid' => '4567',
+ 'volid' => 'local-zfs:vm-4567-disk-0',
+ },
+ ],
+ 'rbd-store' => [
+ {
+ 'ctime' => '1589277334',
+ 'format' => 'raw',
+ 'size' => 1073741824,
+ 'vmid' => '1033',
+ 'volid' => 'rbd-store:vm-1033-disk-1',
+ },
+ {
+ 'ctime' => '1589277334',
+ 'format' => 'raw',
+ 'size' => 1073741824,
+ 'vmid' => '1033',
+ 'volid' => 'rbd-store:vm-1033-cloudinit',
+ },
+ ],
+ 'zfs-alias-1' => [
+ {
+ 'ctime' => '1589277334',
+ 'format' => 'raw',
+ 'size' => 4294967296,
+ 'vmid' => '123',
+ 'volid' => 'zfs-alias-1:vm-123-disk-0',
+ },
+ ],
+ 'zfs-alias-2' => [
+ {
+ 'ctime' => '1589277334',
+ 'format' => 'raw',
+ 'size' => 4294967296,
+ 'vmid' => '123',
+ 'volid' => 'zfs-alias-2:vm-123-disk-0',
+ },
+ ],
+};
+
+my $default_expected_calls_online = {
+ move_config_to_node => 1,
+ ssh_qm_start => 1,
+ vm_stop => 1,
+};
+
+my $default_expected_calls_offline = {
+ move_config_to_node => 1,
+};
+
+my $replicated_expected_calls_online = {
+ %{$default_expected_calls_online},
+ transfer_replication_state => 1,
+ switch_replication_job_target => 1,
+};
+
+my $replicated_expected_calls_offline = {
+ %{$default_expected_calls_offline},
+ transfer_replication_state => 1,
+ switch_replication_job_target => 1,
+};
+
+# helpers
+
+sub get_patched_config {
+ my ($vmid, $patch) = @_;
+
+ my $new_config = { %{$vm_configs->{$vmid}} };
+ patch_config($new_config, $patch) if defined($patch);
+
+ return $new_config;
+}
+
+sub patch_config {
+ my ($config, $patch) = @_;
+
+ foreach my $key (keys %{$patch}) {
+ if ($key eq 'snapshots' && defined($patch->{$key})) {
+ my $new_snapshot_configs = {};
+ foreach my $snap (keys %{$patch->{snapshots}}) {
+ my $new_snapshot_config = { %{$config->{snapshots}->{$snap}} };
+ patch_config($new_snapshot_config, $patch->{snapshots}->{$snap});
+ $new_snapshot_configs->{$snap} = $new_snapshot_config;
+ }
+ $config->{snapshots} = $new_snapshot_configs;
+ } elsif (defined($patch->{$key})) {
+ $config->{$key} = $patch->{$key};
+ } else { # use undef value for deletion
+ delete $config->{$key};
+ }
+ }
+}
+
+sub local_volids_for_vm {
+ my ($vmid) = @_;
+
+ my $res = {};
+ foreach my $storeid (keys %{$source_vdisks}) {
+ next if $storage_config->{ids}->{$storeid}->{shared};
+ $res = {
+ %{$res},
+ map { $_->{vmid} eq $vmid ? ($_->{volid} => 1) : () } @{$source_vdisks->{$storeid}}
+ };
+ }
+ return $res;
+}
+
+my $tests = [
+# each test consists of the following:
+# name - unique name for the test which also serves as a dir name.
+# NOTE: gets passed to make, so don't use whitespace or slash
+# and adapt buildsys (regex) on code structure changes
+# target - hostname of target node
+# vmid - ID of the VM to migrate
+# opts - options for the migrate() call
+# target_volids - hash of volids on the target at the beginning
+# vm_status - hash with running, runningmachine and optionally runningcpu
+# expected_calls - hash whose keys are calls which are required
+# to be made if the migration gets far enough
+# expect_die - expect the migration call to fail, and an error message
+# matching the specified text in the log
+# expected - hash consisting of:
+# source_volids - hash of volids expected on the source
+# target_volids - hash of volids expected on the target
+# vm_config - vm configuration hash
+# vm_status - hash with running, runningmachine and optionally runningcpu
+ {
+ # NOTE get_efivars_size is mocked and returns 128K
+ name => '341_running_efidisk_targetstorage_dir',
+ target => 'pve1',
+ vmid => 341,
+ vm_status => {
+ running => 1,
+ runningmachine => 'pc-q35-5.0+pve0',
+ },
+ opts => {
+ online => 1,
+ 'with-local-disks' => 1,
+ targetstorage => 'local-dir',
+ },
+ expected_calls => $default_expected_calls_online,
+ expected => {
+ source_volids => {},
+ target_volids => {
+ 'local-dir:341/vm-341-disk-10.raw' => 1,
+ },
+ vm_config => get_patched_config(341, {
+ efidisk0 => 'local-dir:341/vm-341-disk-10.raw,format=raw,size=128K',
+ }),
+ vm_status => {
+ running => 1,
+ runningmachine => 'pc-q35-5.0+pve0',
+ },
+ },
+ },
+ {
+ # NOTE get_efivars_size is mocked and returns 128K
+ name => '341_running_efidisk',
+ target => 'pve1',
+ vmid => 341,
+ vm_status => {
+ running => 1,
+ runningmachine => 'pc-q35-5.0+pve0',
+ },
+ opts => {
+ online => 1,
+ 'with-local-disks' => 1,
+ },
+ expected_calls => $default_expected_calls_online,
+ expected => {
+ source_volids => {},
+ target_volids => {
+ 'local-lvm:vm-341-disk-10' => 1,
+ },
+ vm_config => get_patched_config(341, {
+ efidisk0 => 'local-lvm:vm-341-disk-10,format=raw,size=128K',
+ }),
+ vm_status => {
+ running => 1,
+ runningmachine => 'pc-q35-5.0+pve0',
+ },
+ },
+ },
+ {
+ name => '149_running_vdisk_alloc_and_pvesm_free_fail',
+ target => 'pve1',
+ vmid => 149,
+ vm_status => {
+ running => 1,
+ runningmachine => 'pc-q35-5.0+pve0',
+ },
+ opts => {
+ online => 1,
+ 'with-local-disks' => 1,
+ },
+ fail_config => {
+ vdisk_alloc => 'local-dir:149/vm-149-disk-11.qcow2',
+ pvesm_free => 'local-lvm:vm-149-disk-10',
+ },
+ expected_calls => {},
+ expect_die => "remote command failed with exit code",
+ expected => {
+ source_volids => local_volids_for_vm(149),
+ target_volids => {
+ 'local-lvm:vm-149-disk-10' => 1,
+ },
+ vm_config => $vm_configs->{149},
+ vm_status => {
+ running => 1,
+ runningmachine => 'pc-q35-5.0+pve0',
+ },
+ },
+ },
+ {
+ name => '149_running_vdisk_alloc_fail',
+ target => 'pve1',
+ vmid => 149,
+ vm_status => {
+ running => 1,
+ runningmachine => 'pc-q35-5.0+pve0',
+ },
+ opts => {
+ online => 1,
+ 'with-local-disks' => 1,
+ },
+ fail_config => {
+ vdisk_alloc => 'local-lvm:vm-149-disk-10',
+ },
+ expected_calls => {},
+ expect_die => "remote command failed with exit code",
+ expected => {
+ source_volids => local_volids_for_vm(149),
+ target_volids => {},
+ vm_config => $vm_configs->{149},
+ vm_status => {
+ running => 1,
+ runningmachine => 'pc-q35-5.0+pve0',
+ },
+ },
+ },
+ {
+ name => '149_vdisk_free_fail',
+ target => 'pve1',
+ vmid => 149,
+ vm_status => {
+ running => 0,
+ },
+ opts => {
+ 'with-local-disks' => 1,
+ },
+ fail_config => {
+ 'vdisk_free' => 'local-lvm:vm-149-disk-0',
+ },
+ expected_calls => $default_expected_calls_offline,
+ expect_die => "vdisk_free 'local-lvm:vm-149-disk-0' error",
+ expected => {
+ source_volids => {
+ 'local-lvm:vm-149-disk-0' => 1,
+ },
+ target_volids => local_volids_for_vm(149),
+ vm_config => $vm_configs->{149},
+ vm_status => {
+ running => 0,
+ },
+ },
+ },
+ {
+ name => '105_replicated_run_replication_fail',
+ target => 'pve2',
+ vmid => 105,
+ vm_status => {
+ running => 0,
+ },
+ target_volids => local_volids_for_vm(105),
+ fail_config => {
+ run_replication => 1,
+ },
+ expected_calls => {},
+ expect_die => 'run_replication error',
+ expected => {
+ source_volids => local_volids_for_vm(105),
+ target_volids => local_volids_for_vm(105),
+ vm_config => $vm_configs->{105},
+ vm_status => {
+ running => 0,
+ },
+ },
+ },
+ {
+ name => '1033_running_query_migrate_fail',
+ target => 'pve2',
+ vmid => 1033,
+ vm_status => {
+ running => 1,
+ runningmachine => 'pc-q35-5.0+pve0',
+ },
+ opts => {
+ online => 1,
+ },
+ fail_config => {
+ 'query-migrate' => 1,
+ },
+ expected_calls => {},
+ expect_die => 'online migrate failure - aborting',
+ expected => {
+ source_volids => {},
+ target_volids => {},
+ vm_config => $vm_configs->{1033},
+ vm_status => {
+ running => 1,
+ runningmachine => 'pc-q35-5.0+pve0',
+ },
+ },
+ },
+ {
+ name => '4567_targetstorage_dirotherdir',
+ target => 'pve1',
+ vmid => 4567,
+ vm_status => {
+ running => 0,
+ },
+ opts => {
+ targetstorage => 'local-dir:other-dir,local-zfs:local-zfs',
+ },
+ storage_migrate_map => {
+ 'local-dir:4567/vm-4567-disk-0.qcow2' => '4567/vm-4567-disk-0.qcow2',
+ 'local-dir:4567/vm-4567-state-snap1.raw' => '4567/vm-4567-state-snap1.raw',
+ 'local-dir:4567/vm-4567-state-snap2.raw' => '4567/vm-4567-state-snap2.raw',
+ },
+ expected_calls => $default_expected_calls_offline,
+ expected => {
+ source_volids => {},
+ target_volids => {
+ 'other-dir:4567/vm-4567-disk-0.qcow2' => 1,
+ 'other-dir:4567/vm-4567-state-snap1.raw' => 1,
+ 'other-dir:4567/vm-4567-state-snap2.raw' => 1,
+ 'local-zfs:vm-4567-disk-0' => 1,
+ },
+ vm_config => get_patched_config(4567, {
+ 'scsi0' => 'other-dir:4567/vm-4567-disk-0.qcow2,size=4G',
+ snapshots => {
+ snap1 => {
+ 'scsi0' => 'other-dir:4567/vm-4567-disk-0.qcow2,size=4G',
+ 'vmstate' => 'other-dir:4567/vm-4567-state-snap1.raw',
+ },
+ snap2 => {
+ 'scsi0' => 'other-dir:4567/vm-4567-disk-0.qcow2,size=4G',
+ 'scsi1' => 'local-zfs:vm-4567-disk-0,size=1G',
+ 'vmstate' => 'other-dir:4567/vm-4567-state-snap2.raw',
+ },
+ },
+ }),
+ vm_status => {
+ running => 0,
+ },
+ },
+ },
+ {
+ name => '4567_running',
+ target => 'pve1',
+ vmid => 4567,
+ vm_status => {
+ running => 1,
+ runningmachine => 'pc-i440fx-5.0+pve0',
+ },
+ opts => {
+ online => 1,
+ 'with-local-disks' => 1,
+ },
+ expected_calls => {},
+ expect_die => 'online storage migration not possible if non-replicated snapshot exists',
+ expected => {
+ source_volids => local_volids_for_vm(4567),
+ target_volids => {},
+ vm_config => $vm_configs->{4567},
+ vm_status => {
+ running => 1,
+ runningmachine => 'pc-i440fx-5.0+pve0',
+ },
+ },
+ },
+ {
+ name => '4567_offline',
+ target => 'pve1',
+ vmid => 4567,
+ vm_status => {
+ running => 0,
+ },
+ expected_calls => $default_expected_calls_offline,
+ expected => {
+ source_volids => {},
+ target_volids => local_volids_for_vm(4567),
+ vm_config => $vm_configs->{4567},
+ vm_status => {
+ running => 0,
+ },
+ },
+ },
+ {
+ name => '149_running_orphaned_disk_targetstorage_zfs',
+ target => 'pve1',
+ vmid => 149,
+ vm_status => {
+ running => 1,
+ runningmachine => 'pc-q35-5.0+pve0',
+ },
+ opts => {
+ online => 1,
+ 'with-local-disks' => 1,
+ targetstorage => 'local-zfs',
+ },
+ config_patch => {
+ scsi1 => undef,
+ },
+ storage_migrate_map => {
+ 'local-dir:149/vm-149-disk-0.qcow2' => 'vm-149-disk-0',
+ },
+ expected_calls => $default_expected_calls_online,
+ expected => {
+ source_volids => {
+ 'local-dir:149/vm-149-disk-0.qcow2' => 1,
+ },
+ target_volids => {
+ 'local-zfs:vm-149-disk-10' => 1,
+ },
+ vm_config => get_patched_config(149, {
+ scsi0 => 'local-zfs:vm-149-disk-10,format=raw,size=4G',
+ scsi1 => undef,
+ }),
+ vm_status => {
+ running => 1,
+ runningmachine => 'pc-q35-5.0+pve0',
+ },
+ },
+ },
+ {
+ name => '149_running_orphaned_disk',
+ target => 'pve1',
+ vmid => 149,
+ vm_status => {
+ running => 1,
+ runningmachine => 'pc-q35-5.0+pve0',
+ },
+ opts => {
+ online => 1,
+ 'with-local-disks' => 1,
+ },
+ config_patch => {
+ scsi1 => undef,
+ },
+ storage_migrate_map => {
+ 'local-dir:149/vm-149-disk-0.qcow2' => '149/vm-149-disk-0.qcow2',
+ },
+ expected_calls => $default_expected_calls_online,
+ expected => {
+ source_volids => {
+ 'local-dir:149/vm-149-disk-0.qcow2' => 1,
+ },
+ target_volids => {
+ 'local-lvm:vm-149-disk-10' => 1,
+ },
+ vm_config => get_patched_config(149, {
+ scsi0 => 'local-lvm:vm-149-disk-10,format=raw,size=4G',
+ scsi1 => undef,
+ }),
+ vm_status => {
+ running => 1,
+ runningmachine => 'pc-q35-5.0+pve0',
+ },
+ },
+ },
+ {
+ # FIXME: This test is not (yet) a realistic situation, because
+ # storage_migrate currently never changes the format (AFAICT)
+ # But if such migrations become possible, we need to either update
+ # the 'format' property or simply remove it for drives migrated
+ # with storage_migrate (the property is optional, so it shouldn't be a problem)
+ name => '149_targetstorage_map_lvmzfs_defaultlvm',
+ target => 'pve1',
+ vmid => 149,
+ vm_status => {
+ running => 0,
+ },
+ opts => {
+ targetstorage => 'local-lvm:local-zfs,local-lvm',
+ },
+ storage_migrate_map => {
+ 'local-lvm:vm-149-disk-0' => 'vm-149-disk-0',
+ 'local-dir:149/vm-149-disk-0.qcow2' => 'vm-149-disk-0',
+ },
+ expected_calls => $default_expected_calls_offline,
+ expected => {
+ source_volids => {},
+ target_volids => {
+ 'local-zfs:vm-149-disk-0' => 1,
+ 'local-lvm:vm-149-disk-0' => 1,
+ },
+ vm_config => get_patched_config(149, {
+ scsi0 => 'local-zfs:vm-149-disk-0,format=raw,size=4G',
+ scsi1 => 'local-lvm:vm-149-disk-0,format=qcow2,size=1G',
+ }),
+ vm_status => {
+ running => 0,
+ },
+ },
+ },
+ {
+ # FIXME same as for the previous test
+ name => '149_targetstorage_map_dirzfs_lvmdir',
+ target => 'pve1',
+ vmid => 149,
+ vm_status => {
+ running => 0,
+ },
+ opts => {
+ online => 1,
+ 'with-local-disks' => 1,
+ targetstorage => 'local-dir:local-zfs,local-lvm:local-dir',
+ },
+ storage_migrate_map => {
+ 'local-lvm:vm-149-disk-0' => '149/vm-149-disk-0.raw',
+ 'local-dir:149/vm-149-disk-0.qcow2' => 'vm-149-disk-0',
+ },
+ expected_calls => $default_expected_calls_offline,
+ expected => {
+ source_volids => {},
+ target_volids => {
+ 'local-dir:149/vm-149-disk-0.raw' => 1,
+ 'local-zfs:vm-149-disk-0' => 1,
+ },
+ vm_config => get_patched_config(149, {
+ scsi0 => 'local-dir:149/vm-149-disk-0.raw,format=raw,size=4G',
+ scsi1 => 'local-zfs:vm-149-disk-0,format=qcow2,size=1G',
+ }),
+ vm_status => {
+ running => 0,
+ },
+ },
+ },
+ {
+ name => '149_running_targetstorage_map_lvmzfs_defaultlvm',
+ target => 'pve1',
+ vmid => 149,
+ vm_status => {
+ running => 1,
+ runningmachine => 'pc-q35-5.0+pve0',
+ },
+ opts => {
+ online => 1,
+ 'with-local-disks' => 1,
+ targetstorage => 'local-lvm:local-zfs,local-lvm',
+ },
+ expected_calls => $default_expected_calls_online,
+ expected => {
+ source_volids => {},
+ target_volids => {
+ 'local-zfs:vm-149-disk-10' => 1,
+ 'local-lvm:vm-149-disk-11' => 1,
+ },
+ vm_config => get_patched_config(149, {
+ scsi0 => 'local-zfs:vm-149-disk-10,format=raw,size=4G',
+ scsi1 => 'local-lvm:vm-149-disk-11,format=raw,size=1G',
+ }),
+ vm_status => {
+ running => 1,
+ runningmachine => 'pc-q35-5.0+pve0',
+ },
+ },
+ },
+ {
+ name => '149_running_targetstorage_map_lvmzfs_dirdir',
+ target => 'pve1',
+ vmid => 149,
+ vm_status => {
+ running => 1,
+ runningmachine => 'pc-q35-5.0+pve0',
+ },
+ opts => {
+ online => 1,
+ 'with-local-disks' => 1,
+ targetstorage => 'local-lvm:local-zfs,local-dir:local-dir',
+ },
+ expected_calls => $default_expected_calls_online,
+ expected => {
+ source_volids => {},
+ target_volids => {
+ 'local-zfs:vm-149-disk-10' => 1,
+ 'local-dir:149/vm-149-disk-11.qcow2' => 1,
+ },
+ vm_config => get_patched_config(149, {
+ scsi0 => 'local-zfs:vm-149-disk-10,format=raw,size=4G',
+ scsi1 => 'local-dir:149/vm-149-disk-11.qcow2,format=qcow2,size=1G',
+ }),
+ vm_status => {
+ running => 1,
+ runningmachine => 'pc-q35-5.0+pve0',
+ },
+ },
+ },
+ {
+ name => '149_running_targetstorage_zfs',
+ target => 'pve1',
+ vmid => 149,
+ vm_status => {
+ running => 1,
+ runningmachine => 'pc-q35-5.0+pve0',
+ },
+ opts => {
+ online => 1,
+ 'with-local-disks' => 1,
+ targetstorage => 'local-zfs',
+ },
+ expected_calls => $default_expected_calls_online,
+ expected => {
+ source_volids => {},
+ target_volids => {
+ 'local-zfs:vm-149-disk-10' => 1,
+ 'local-zfs:vm-149-disk-11' => 1,
+ },
+ vm_config => get_patched_config(149, {
+ scsi0 => 'local-zfs:vm-149-disk-10,format=raw,size=4G',
+ scsi1 => 'local-zfs:vm-149-disk-11,format=raw,size=1G',
+ }),
+ vm_status => {
+ running => 1,
+ runningmachine => 'pc-q35-5.0+pve0',
+ },
+ },
+ },
+ {
+ name => '149_running_wrong_size',
+ target => 'pve1',
+ vmid => 149,
+ vm_status => {
+ running => 1,
+ runningmachine => 'pc-q35-5.0+pve0',
+ },
+ opts => {
+ online => 1,
+ 'with-local-disks' => 1,
+ },
+ config_patch => {
+ scsi0 => 'local-lvm:vm-149-disk-0,size=123T',
+ },
+ expected_calls => $default_expected_calls_online,
+ expected => {
+ source_volids => {},
+ target_volids => {
+ 'local-lvm:vm-149-disk-10' => 1,
+ 'local-dir:149/vm-149-disk-11.qcow2' => 1,
+ },
+ vm_config => get_patched_config(149, {
+ scsi0 => 'local-lvm:vm-149-disk-10,format=raw,size=4G',
+ scsi1 => 'local-dir:149/vm-149-disk-11.qcow2,format=qcow2,size=1G',
+ }),
+ vm_status => {
+ running => 1,
+ runningmachine => 'pc-q35-5.0+pve0',
+ },
+ },
+ },
+ {
+ name => '149_running_missing_size',
+ target => 'pve1',
+ vmid => 149,
+ vm_status => {
+ running => 1,
+ runningmachine => 'pc-q35-5.0+pve0',
+ },
+ opts => {
+ online => 1,
+ 'with-local-disks' => 1,
+ },
+ config_patch => {
+ scsi0 => 'local-lvm:vm-149-disk-0',
+ },
+ expected_calls => $default_expected_calls_online,
+ expected => {
+ source_volids => {},
+ target_volids => {
+ 'local-lvm:vm-149-disk-10' => 1,
+ 'local-dir:149/vm-149-disk-11.qcow2' => 1,
+ },
+ vm_config => get_patched_config(149, {
+ scsi0 => 'local-lvm:vm-149-disk-10,format=raw,size=4G',
+ scsi1 => 'local-dir:149/vm-149-disk-11.qcow2,format=qcow2,size=1G',
+ }),
+ vm_status => {
+ running => 1,
+ runningmachine => 'pc-q35-5.0+pve0',
+ },
+ },
+ },
+ {
+ name => '105_local_device_shared',
+ target => 'pve1',
+ vmid => 105,
+ vm_status => {
+ running => 0,
+ },
+ config_patch => {
+ ide2 => '/dev/sde,shared=1',
+ },
+ expected_calls => $default_expected_calls_offline,
+ expected => {
+ source_volids => {},
+ target_volids => local_volids_for_vm(105),
+ vm_config => get_patched_config(105, {
+ ide2 => '/dev/sde,shared=1',
+ }),
+ vm_status => {
+ running => 0,
+ },
+ },
+ },
+ {
+ name => '105_local_device_in_snapshot',
+ target => 'pve1',
+ vmid => 105,
+ vm_status => {
+ running => 0,
+ },
+ config_patch => {
+ snapshots => {
+ ohsnap => {
+ ide2 => '/dev/sde',
+ },
+ },
+ },
+ expected_calls => {},
+ expect_die => "can't migrate local disk '/dev/sde': local file/device",
+ expected => {
+ source_volids => local_volids_for_vm(105),
+ target_volids => {},
+ vm_config => get_patched_config(105, {
+ snapshots => {
+ ohsnap => {
+ ide2 => '/dev/sde',
+ },
+ },
+ }),
+ vm_status => {
+ running => 0,
+ },
+ },
+ },
+ {
+ name => '105_local_device',
+ target => 'pve1',
+ vmid => 105,
+ vm_status => {
+ running => 0,
+ },
+ config_patch => {
+ ide2 => '/dev/sde',
+ },
+ expected_calls => {},
+ expect_die => "can't migrate local disk '/dev/sde': local file/device",
+ expected => {
+ source_volids => local_volids_for_vm(105),
+ target_volids => {},
+ vm_config => get_patched_config(105, {
+ ide2 => '/dev/sde',
+ }),
+ vm_status => {
+ running => 0,
+ },
+ },
+ },
+ {
+ name => '105_cdrom_in_snapshot',
+ target => 'pve1',
+ vmid => 105,
+ vm_status => {
+ running => 0,
+ },
+ config_patch => {
+ snapshots => {
+ ohsnap => {
+ ide2 => 'cdrom,media=cdrom',
+ },
+ },
+ },
+ expected_calls => {},
+ expect_die => "can't migrate local cdrom drive (referenced in snapshot - ohsnap",
+ expected => {
+ source_volids => local_volids_for_vm(105),
+ target_volids => {},
+ vm_config => get_patched_config(105, {
+ snapshots => {
+ ohsnap => {
+ ide2 => 'cdrom,media=cdrom',
+ },
+ },
+ }),
+ vm_status => {
+ running => 0,
+ },
+ },
+ },
+ {
+ name => '105_cdrom',
+ target => 'pve1',
+ vmid => 105,
+ vm_status => {
+ running => 0,
+ },
+ config_patch => {
+ ide2 => 'cdrom,media=cdrom',
+ },
+ expected_calls => {},
+ expect_die => "can't migrate local cdrom drive",
+ expected => {
+ source_volids => local_volids_for_vm(105),
+ target_volids => {},
+ vm_config => get_patched_config(105, {
+ ide2 => 'cdrom,media=cdrom',
+ }),
+ vm_status => {
+ running => 0,
+ },
+ },
+ },
+ {
+ name => '149_running_missing_option_withlocaldisks',
+ target => 'pve1',
+ vmid => 149,
+ vm_status => {
+ running => 1,
+ runningmachine => 'pc-q35-5.0+pve0',
+ },
+ opts => {
+ online => 1,
+ },
+ expected_calls => {},
+ expect_die => "can't live migrate attached local disks without with-local-disks option",
+ expected => {
+ source_volids => local_volids_for_vm(149),
+ target_volids => {},
+ vm_config => $vm_configs->{149},
+ vm_status => {
+ running => 1,
+ runningmachine => 'pc-q35-5.0+pve0',
+ },
+ },
+ },
+ {
+ name => '149_running_missing_option_online',
+ target => 'pve1',
+ vmid => 149,
+ vm_status => {
+ running => 1,
+ runningmachine => 'pc-q35-5.0+pve0',
+ },
+ opts => {
+ 'with-local-disks' => 1,
+ },
+ expected_calls => {},
+ expect_die => "can't migrate running VM without --online",
+ expected => {
+ source_volids => local_volids_for_vm(149),
+ target_volids => {},
+ vm_config => $vm_configs->{149},
+ vm_status => {
+ running => 1,
+ runningmachine => 'pc-q35-5.0+pve0',
+ },
+ },
+ },
+ {
+ name => '1033_running_customcpu',
+ target => 'pve1',
+ vmid => 1033,
+ vm_status => {
+ running => 1,
+ runningmachine => 'pc-q35-5.0+pve0',
+ runningcpu => 'host,+kvm_pv_eoi,+kvm_pv_unhalt',
+ },
+ opts => {
+ online => 1,
+ },
+ config_patch => {
+ cpu => 'custom-mycpu',
+ },
+ expected_calls => $default_expected_calls_online,
+ expected => {
+ source_volids => {},
+ target_volids => {},
+ vm_config => get_patched_config(1033, {
+ cpu => 'custom-mycpu',
+ }),
+ vm_status => {
+ running => 1,
+ runningmachine => 'pc-q35-5.0+pve0',
+ runningcpu => 'host,+kvm_pv_eoi,+kvm_pv_unhalt',
+ },
+ },
+ },
+ {
+ name => '105_replicated_to_non_replication_target',
+ target => 'pve1',
+ vmid => 105,
+ vm_status => {
+ running => 0,
+ },
+ target_volids => {},
+ expected_calls => $replicated_expected_calls_offline,
+ expected => {
+ source_volids => {},
+ target_volids => local_volids_for_vm(105),
+ vm_config => $vm_configs->{105},
+ vm_status => {
+ running => 0,
+ },
+ },
+ },
+ {
+ name => '105_running_replicated',
+ target => 'pve2',
+ vmid => 105,
+ vm_status => {
+ running => 1,
+ runningmachine => 'pc-i440fx-5.0+pve0',
+ },
+ opts => {
+ online => 1,
+ 'with-local-disks' => 1,
+ },
+ target_volids => local_volids_for_vm(105),
+ expected_calls => {
+ %{$replicated_expected_calls_online},
+ 'block-dirty-bitmap-add-drive-scsi0' => 1,
+ 'block-dirty-bitmap-add-drive-ide0' => 1,
+ },
+ expected => {
+ source_volids => local_volids_for_vm(105),
+ target_volids => local_volids_for_vm(105),
+ vm_config => $vm_configs->{105},
+ vm_status => {
+ running => 1,
+ runningmachine => 'pc-i440fx-5.0+pve0',
+ },
+ },
+ },
+ {
+ name => '105_replicated',
+ target => 'pve2',
+ vmid => 105,
+ vm_status => {
+ running => 0,
+ },
+ target_volids => local_volids_for_vm(105),
+ expected_calls => $replicated_expected_calls_offline,
+ expected => {
+ source_volids => local_volids_for_vm(105),
+ target_volids => local_volids_for_vm(105),
+ vm_config => $vm_configs->{105},
+ vm_status => {
+ running => 0,
+ },
+ },
+ },
+ {
+ name => '105_running_replicated_without_snapshot',
+ target => 'pve2',
+ vmid => 105,
+ vm_status => {
+ running => 1,
+ runningmachine => 'pc-i440fx-5.0+pve0',
+ },
+ config_patch => {
+ snapshots => undef,
+ },
+ opts => {
+ online => 1,
+ 'with-local-disks' => 1,
+ },
+ target_volids => local_volids_for_vm(105),
+ expected_calls => {
+ %{$replicated_expected_calls_online},
+ 'block-dirty-bitmap-add-drive-scsi0' => 1,
+ 'block-dirty-bitmap-add-drive-ide0' => 1,
+ },
+ expected => {
+ source_volids => local_volids_for_vm(105),
+ target_volids => local_volids_for_vm(105),
+ vm_config => get_patched_config(105, {
+ snapshots => {},
+ }),
+ vm_status => {
+ running => 1,
+ runningmachine => 'pc-i440fx-5.0+pve0',
+ },
+ },
+ },
+ {
+ name => '105_replicated_without_snapshot',
+ target => 'pve2',
+ vmid => 105,
+ vm_status => {
+ running => 0,
+ },
+ config_patch => {
+ snapshots => undef,
+ },
+ opts => {
+ online => 1,
+ },
+ target_volids => local_volids_for_vm(105),
+ expected_calls => $replicated_expected_calls_offline,
+ expected => {
+ source_volids => local_volids_for_vm(105),
+ target_volids => local_volids_for_vm(105),
+ vm_config => get_patched_config(105, {
+ snapshots => {},
+ }),
+ vm_status => {
+ running => 0,
+ },
+ },
+ },
+ {
+ name => '1033_running',
+ target => 'pve2',
+ vmid => 1033,
+ vm_status => {
+ running => 1,
+ runningmachine => 'pc-q35-5.0+pve0',
+ },
+ opts => {
+ online => 1,
+ },
+ expected_calls => $default_expected_calls_online,
+ expected => {
+ source_volids => {},
+ target_volids => {},
+ vm_config => $vm_configs->{1033},
+ vm_status => {
+ running => 1,
+ runningmachine => 'pc-q35-5.0+pve0',
+ },
+ },
+ },
+ {
+ name => '149_locked',
+ target => 'pve2',
+ vmid => 149,
+ vm_status => {
+ running => 0,
+ },
+ config_patch => {
+ lock => 'locked',
+ },
+ expected_calls => {},
+ expect_die => "VM is locked",
+ expected => {
+ source_volids => local_volids_for_vm(149),
+ target_volids => {},
+ vm_config => get_patched_config(149, {
+ lock => 'locked',
+ }),
+ vm_status => {
+ running => 0,
+ },
+ },
+ },
+ {
+ name => '149_storage_not_available',
+ target => 'pve2',
+ vmid => 149,
+ vm_status => {
+ running => 0,
+ },
+ expected_calls => {},
+ expect_die => "storage 'local-lvm' is not available on node 'pve2'",
+ expected => {
+ source_volids => local_volids_for_vm(149),
+ target_volids => {},
+ vm_config => $vm_configs->{149},
+ vm_status => {
+ running => 0,
+ },
+ },
+ },
+ {
+ name => '149_running',
+ target => 'pve1',
+ vmid => 149,
+ vm_status => {
+ running => 1,
+ runningmachine => 'pc-q35-5.0+pve0',
+ },
+ opts => {
+ online => 1,
+ 'with-local-disks' => 1,
+ },
+ expected_calls => $default_expected_calls_online,
+ expected => {
+ source_volids => {},
+ target_volids => {
+ 'local-lvm:vm-149-disk-10' => 1,
+ 'local-dir:149/vm-149-disk-11.qcow2' => 1,
+ },
+ vm_config => get_patched_config(149, {
+ scsi0 => 'local-lvm:vm-149-disk-10,format=raw,size=4G',
+ scsi1 => 'local-dir:149/vm-149-disk-11.qcow2,format=qcow2,size=1G',
+ }),
+ vm_status => {
+ running => 1,
+ runningmachine => 'pc-q35-5.0+pve0',
+ },
+ },
+ },
+ {
+ name => '149_running_drive_mirror_fail',
+ target => 'pve1',
+ vmid => 149,
+ vm_status => {
+ running => 1,
+ runningmachine => 'pc-q35-5.0+pve0',
+ },
+ opts => {
+ online => 1,
+ 'with-local-disks' => 1,
+ },
+ expected_calls => {},
+ expect_die => "qemu_drive_mirror 'scsi1' error",
+ fail_config => {
+ 'qemu_drive_mirror' => 'scsi1',
+ },
+ expected => {
+ source_volids => local_volids_for_vm(149),
+ target_volids => {},
+ vm_config => $vm_configs->{149},
+ vm_status => {
+ running => 1,
+ runningmachine => 'pc-q35-5.0+pve0',
+ },
+ },
+ },
+ {
+ name => '149_running_unused_block_job_cancel_fail',
+ target => 'pve1',
+ vmid => 149,
+ vm_status => {
+ running => 1,
+ runningmachine => 'pc-q35-5.0+pve0',
+ },
+ opts => {
+ online => 1,
+ 'with-local-disks' => 1,
+ },
+ config_patch => {
+ scsi1 => undef,
+ unused0 => 'local-dir:149/vm-149-disk-0.qcow2',
+ },
+ expected_calls => {},
+ expect_die => "qemu_drive_mirror_monitor 'cancel' error",
+ # note that 'cancel' is also used to finish and that's what this test is about
+ fail_config => {
+ 'qemu_drive_mirror_monitor' => 'cancel',
+ },
+ expected => {
+ source_volids => local_volids_for_vm(149),
+ target_volids => {},
+ vm_config => get_patched_config(149, {
+ scsi1 => undef,
+ unused0 => 'local-dir:149/vm-149-disk-0.qcow2',
+ }),
+ vm_status => {
+ running => 1,
+ runningmachine => 'pc-q35-5.0+pve0',
+ },
+ },
+ },
+ {
+ name => '149_offline',
+ target => 'pve1',
+ vmid => 149,
+ vm_status => {
+ running => 0,
+ },
+ opts => {
+ 'with-local-disks' => 1,
+ },
+ expected_calls => $default_expected_calls_offline,
+ expected => {
+ source_volids => {},
+ target_volids => local_volids_for_vm(149),
+ vm_config => $vm_configs->{149},
+ vm_status => {
+ running => 0,
+ },
+ },
+ },
+ {
+ name => '149_storage_migrate_fail',
+ target => 'pve1',
+ vmid => 149,
+ vm_status => {
+ running => 0,
+ },
+ opts => {
+ 'with-local-disks' => 1,
+ },
+ fail_config => {
+ 'storage_migrate' => 'local-lvm:vm-149-disk-0',
+ },
+ expected_calls => {},
+ expect_die => "storage_migrate 'local-lvm:vm-149-disk-0' error",
+ expected => {
+ source_volids => local_volids_for_vm(149),
+ target_volids => {},
+ vm_config => $vm_configs->{149},
+ vm_status => {
+ running => 0,
+ },
+ },
+ },
+ {
+ name => '111_running_pending',
+ target => 'pve1',
+ vmid => 111,
+ vm_status => {
+ running => 1,
+ runningmachine => 'pc-q35-5.0+pve0',
+ },
+ opts => {
+ online => 1,
+ 'with-local-disks' => 1,
+ },
+ expected_calls => $default_expected_calls_online,
+ expected => {
+ source_volids => {},
+ target_volids => {
+ 'local-zfs:vm-111-disk-0' => 1,
+ 'local-lvm:vm-111-disk-10' => 1,
+ },
+ vm_config => get_patched_config(111, {
+ ide0 => 'local-lvm:vm-111-disk-10,format=raw,size=4G',
+ pending => {
+ scsi0 => 'local-zfs:vm-111-disk-0,size=103M',
+ },
+ }),
+ vm_status => {
+ running => 1,
+ runningmachine => 'pc-q35-5.0+pve0',
+ },
+ },
+ },
+ {
+ name => '123_alias_fail',
+ target => 'pve1',
+ vmid => 123,
+ vm_status => {
+ running => 0,
+ },
+ opts => {
+ 'with-local-disks' => 1,
+ },
+ expected_calls => {},
+ expect_die => "detected not supported aliased volumes",
+ expected => {
+ source_volids => local_volids_for_vm(123),
+ target_volids => {},
+ vm_config => $vm_configs->{123},
+ vm_status => {
+ running => 0,
+ },
+ },
+ },
+];
+
+my $single_test_name = shift;
+
+mkdir $RUN_DIR_PATH;
+
+foreach my $test (@{$tests}) {
+ my $name = $test->{name};
+ next if defined($single_test_name) && $name ne $single_test_name;
+
+ my $run_dir = "${RUN_DIR_PATH}/${name}";
+
+ mkdir $run_dir;
+ file_set_contents("${run_dir}/replication_config", to_json($replication_config));
+ file_set_contents("${run_dir}/storage_config", to_json($storage_config));
+ file_set_contents("${run_dir}/source_vdisks", to_json($source_vdisks));
+
+ my $expect_die = $test->{expect_die};
+ my $expected = $test->{expected};
+
+ my $source_volids = local_volids_for_vm($test->{vmid});
+ my $target_volids = $test->{target_volids} // {};
+
+ my $config_patch = $test->{config_patch};
+ my $vm_config = get_patched_config($test->{vmid}, $test->{config_patch});
+
+ my $fail_config = $test->{fail_config} // {};
+ my $storage_migrate_map = $test->{storage_migrate_map} // {};
+
+ if (my $targetstorage = $test->{opts}->{targetstorage}) {
+ $test->{opts}->{storagemap} = PVE::JSONSchema::parse_idmap($targetstorage, 'pve-storage-id');
+ }
+
+ my $migrate_params = {
+ target => $test->{target},
+ vmid => $test->{vmid},
+ opts => $test->{opts},
+ };
+
+ file_set_contents("${run_dir}/nbd_info", to_json({}));
+ file_set_contents("${run_dir}/source_volids", to_json($source_volids));
+ file_set_contents("${run_dir}/target_volids", to_json($target_volids));
+ file_set_contents("${run_dir}/vm_config", to_json($vm_config));
+ file_set_contents("${run_dir}/vm_status", to_json($test->{vm_status}));
+ file_set_contents("${run_dir}/expected_calls", to_json($test->{expected_calls}));
+ file_set_contents("${run_dir}/fail_config", to_json($fail_config));
+ file_set_contents("${run_dir}/storage_migrate_map", to_json($storage_migrate_map));
+ file_set_contents("${run_dir}/migrate_params", to_json($migrate_params));
+
+ $ENV{QM_LIB_PATH} = $QM_LIB_PATH;
+ $ENV{RUN_DIR_PATH} = $run_dir;
+ my $exitcode = run_command([
+ '/usr/bin/perl',
+ "-I${MIGRATE_LIB_PATH}",
+ "-I${MIGRATE_LIB_PATH}/test",
+ "${MIGRATE_LIB_PATH}/test/MigrationTest/QemuMigrateMock.pm",
+ ], noerr => 1, errfunc => sub {print "#$name - $_[0]\n"} );
+
+ if (defined($expect_die) && $exitcode) {
+ my $log = file_get_contents("${run_dir}/log");
+ my @lines = split /\n/, $log;
+
+ my $matched = 0;
+ foreach my $line (@lines) {
+ $matched = 1 if $line =~ m/^err:.*\Q${expect_die}\E/;
+ $matched = 1 if $line =~ m/^warn:.*\Q${expect_die}\E/;
+ }
+ if (!$matched) {
+ fail($name);
+ note("expected error message is not present in log");
+ }
+ } elsif (defined($expect_die) && !$exitcode) {
+ fail($name);
+ note("mocked migrate call didn't fail, but it was expected to - check log");
+ } elsif (!defined($expect_die) && $exitcode) {
+ fail($name);
+ note("mocked migrate call failed, but it was not expected - check log");
+ }
+
+ my $expected_calls = decode_json(file_get_contents("${run_dir}/expected_calls"));
+ foreach my $call (keys %{$expected_calls}) {
+ fail($name);
+ note("expected call '$call' was not made");
+ }
+
+ if (!defined($expect_die)) {
+ my $nbd_info = decode_json(file_get_contents("${run_dir}/nbd_info"));
+ foreach my $drive (keys %{$nbd_info}) {
+ fail($name);
+ note("drive '$drive' was not mirrored");
+ }
+ }
+
+ my $actual = {
+ source_volids => decode_json(file_get_contents("${run_dir}/source_volids")),
+ target_volids => decode_json(file_get_contents("${run_dir}/target_volids")),
+ vm_config => decode_json(file_get_contents("${run_dir}/vm_config")),
+ vm_status => decode_json(file_get_contents("${run_dir}/vm_status")),
+ };
+
+ is_deeply($actual, $expected, $name);
+}
+
+done_testing();
--- /dev/null
+#!/usr/bin/perl
+
+use strict;
+use warnings;
+
+use lib qw(..);
+
+use Test::MockModule;
+use Test::More;
+use Test::MockModule;
+
+use File::Basename;
+
+use PVE::QemuServer;
+use PVE::Tools qw(dir_glob_foreach file_get_contents);
+
+my $INPUT_DIR = './restore-config-input';
+my $EXPECTED_DIR = './restore-config-expected';
+
+my $pve_cluster_module = Test::MockModule->new('PVE::Cluster');
+$pve_cluster_module->mock(
+ cfs_read_file => sub {
+ return {};
+ },
+);
+
+# NOTE update when you add/remove tests
+plan tests => 4;
+
+my $cfs_mock = Test::MockModule->new("PVE::Cluster");
+$cfs_mock->mock(
+ cfs_read_file => sub {
+ my ($file) = @_;
+
+ if ($file eq 'datacenter.cfg') {
+ return {};
+ } else {
+ die "'cfs_read_file' called - missing mock?\n";
+ }
+ },
+);
+
+dir_glob_foreach('./restore-config-input', '[0-9]+.conf', sub {
+ my ($file) = @_;
+
+ my $vmid = basename($file, ('.conf'));
+
+ my $fh = IO::File->new("${INPUT_DIR}/${file}", "r") or
+ die "unable to read '$file' - $!\n";
+
+ my $map = {};
+ my $disknum = 0;
+
+ # NOTE For now, the map is hardcoded to a file-based 'target' storage.
+ # In the future, the test could be extended to include parse_backup_hints
+ # and restore_allocate_devices. Even better if the config-related logic from
+ # the restore_XYZ_archive functions could become a separate function.
+ while (defined(my $line = <$fh>)) {
+ if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
+ my ($drive, undef, $storeid, $fmt) = ($1, $2, $3, $4);
+
+ $fmt ||= 'raw';
+
+ $map->{$drive} = "target:${vmid}/vm-${vmid}-disk-${disknum}.${fmt}";
+ $disknum++;
+ }
+ }
+
+ $fh->seek(0, 0) or die "seek failed - $!\n";
+
+ my $got = '';
+ my $cookie = { netcount => 0 };
+
+ while (defined(my $line = <$fh>)) {
+ $got .= PVE::QemuServer::restore_update_config_line(
+ $cookie,
+ $map,
+ $line,
+ 0,
+ );
+ }
+
+ my $expected = file_get_contents("${EXPECTED_DIR}/${file}");
+
+ is_deeply($got, $expected, $file);
+});
+
+done_testing();
cores: 4
ide0: somestore:somedisk,discard=on,size=32G
ide2: none,media=cdrom
-machine: somemachine
+machine: q35
memory: 8192
name: win
net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1
cores: 4
ide0: somestore:somedisk,discard=on,size=32G
ide2: none,media=cdrom
-machine: somemachine
+machine: q35
memory: 8192
name: win
net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1
cores: 4
ide0: somestore:somedisk,discard=on,size=32G
ide2: none,media=cdrom
-machine: somemachine
+machine: q35
memory: 8192
name: win
net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1
cores: 4
ide0: somestore:somedisk,discard=on,size=32G
ide2: none,media=cdrom
-machine: somemachine
+machine: q35
memory: 8192
name: win
net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1
cores: 4
ide0: somestore:somedisk,discard=on,size=32G
ide2: none,media=cdrom
-machine: somemachine
+machine: q35
memory: 8192
name: win
net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1
net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1
numa: 0
ostype: win7
-runningmachine: somemachine
+runningmachine: q35
smbios1: uuid=01234567-890a-bcde-f012-34567890abcd
snaptime: 1234567890
sockets: 1
cores: 4
ide0: local:snapshotable-disk-1,discard=on,size=32G
ide2: none,media=cdrom
-machine: somemachine
+machine: q35
memory: 8192
name: win
net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1
numa: 0
ostype: win7
parent: test
-runningmachine: somemachine
+runningmachine: q35
smbios1: uuid=01234567-890a-bcde-f012-34567890abcd
snaptime: 1234567890
sockets: 1
net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1
numa: 0
ostype: win7
-runningmachine: somemachine
+runningmachine: q35
smbios1: uuid=01234567-890a-bcde-f012-34567890abcd
snaptime: 1234567890
sockets: 1
net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1
numa: 0
ostype: win7
-runningmachine: somemachine
+runningmachine: q35
smbios1: uuid=01234567-890a-bcde-f012-34567890abcd
snaptime: 1234567890
sockets: 1
net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1
numa: 0
ostype: win7
-runningmachine: somemachine
+runningmachine: q35
smbios1: uuid=01234567-890a-bcde-f012-34567890abcd
snaptime: 1234567890
sockets: 1
--- /dev/null
+bootdisk: ide0
+cores: 4
+ide0: local:snapshotable-disk-1,discard=on,size=32G
+ide2: none,media=cdrom
+machine: q35
+memory: 8192
+name: win
+net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1
+numa: 0
+ostype: win7
+smbios1: uuid=01234567-890a-bcde-f012-34567890abcd
+sockets: 1
+vga: qxl
cores: 4
ide0: local:snapshotable-disk-1,discard=on,size=32G
ide2: none,media=cdrom
-machine: somemachine
+machine: q35
memory: 8192
name: win
net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1
--- /dev/null
+agent: 1
+bootdisk: ide0
+cores: 4
+ide0: local:snapshotable-disk-1,discard=on,size=32G
+ide2: none,media=cdrom
+memory: 8192
+name: win
+net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1
+numa: 0
+ostype: win7
+parent: test
+smbios1: uuid=01234567-890a-bcde-f012-34567890abcd
+sockets: 1
+vga: qxl
+
+[test]
+#test comment
+agent: 1
+bootdisk: ide0
+cores: 4
+ide0: local:snapshotable-disk-1,discard=on,size=32G
+ide2: none,media=cdrom
+machine: q35
+memory: 8192
+name: win
+net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1
+numa: 0
+ostype: win7
+smbios1: uuid=01234567-890a-bcde-f012-34567890abcd
+snaptime: 1234567890
+sockets: 1
+vga: qxl
+vmstate: somestorage:state-volume
net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1
numa: 0
ostype: win7
-runningmachine: somemachine
+runningmachine: q35
smbios1: uuid=01234567-890a-bcde-f012-34567890abcd
snapstate: prepare
snaptime: 1234567890
numa: 0
ostype: win7
parent: test
-runningmachine: somemachine
+runningmachine: q35
smbios1: uuid=01234567-890a-bcde-f012-34567890abcd
snapstate: prepare
snaptime: 1234567890
cores: 4
ide0: local:snapshotable-disk-1,discard=on,size=32G
ide2: none,media=cdrom
-machine: somemachine
+machine: q35
memory: 8192
name: win
net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1
cores: 4
ide0: local:snapshotable-disk-1,discard=on,size=32G
ide2: none,media=cdrom
-machine: somemachine
+machine: q35
memory: 8192
name: win
net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1
cores: 4
ide0: local:snapshotable-disk-1,discard=on,size=32G
ide2: none,media=cdrom
-machine: somemachine
+machine: q35
memory: 8192
name: win
net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1
cores: 4
ide0: local:snapshotable-disk-1,discard=on,size=32G
ide2: none,media=cdrom
-machine: somemachine
+machine: q35
memory: 8192
name: win
net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1
cores: 4
ide0: local:unsnapshotable-disk-1,discard=on,size=32G
ide2: none,media=cdrom
-machine: somemachine
+machine: q35
memory: 8192
name: win
net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1
cores: 4
ide0: local:snapshotable-disk-1,discard=on,size=32G
ide2: none,media=cdrom
-machine: somemachine
+machine: q35
memory: 8192
name: win
net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1
cores: 4
ide0: local:snapshotable-disk-1,discard=on,size=32G
ide2: none,media=cdrom
-machine: somemachine
+machine: q35
memory: 8192
name: win
net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1
cores: 4
ide0: local:snapshotable-disk-1,discard=on,size=32G
ide2: none,media=cdrom
-machine: somemachine
+machine: q35
memory: 8192
name: win
net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1
cores: 4
ide0: local:snapshotable-disk-1,discard=on,size=32G
ide2: none,media=cdrom
-machine: somemachine
+machine: q35
memory: 8192
name: win
net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1
net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1
numa: 0
ostype: win7
-runningmachine: somemachine
+runningmachine: q35
smbios1: uuid=01234567-890a-bcde-f012-34567890abcd
snaptime: 1234567890
sockets: 1
--- /dev/null
+agent: 1
+bootdisk: ide0
+cores: 4
+ide0: local:snapshotable-disk-1,discard=on,size=32G
+ide2: none,media=cdrom
+memory: 8192
+name: win
+net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1
+numa: 0
+ostype: win7
+parent: test
+smbios1: uuid=01234567-890a-bcde-f012-34567890abcd
+sockets: 1
+vga: qxl
+
+[test]
+#test comment
+agent: 1
+bootdisk: ide0
+cores: 4
+ide0: local:snapshotable-disk-1,discard=on,size=32G
+ide2: none,media=cdrom
+machine: q35
+memory: 8192
+name: win
+net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1
+numa: 0
+ostype: win7
+runningmachine: q35
+smbios1: uuid=01234567-890a-bcde-f012-34567890abcd
+snaptime: 1234567890
+sockets: 1
+vga: qxl
+vmstate: somestorage:state-volume
cores: 4
ide0: somestore:somedisk,discard=on,size=32G
ide2: none,media=cdrom
-machine: somemachine
+machine: q35
memory: 8192
name: win
net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1
cores: 4
ide0: somestore:somedisk,discard=on,size=32G
ide2: none,media=cdrom
-machine: somemachine
+machine: q35
memory: 8192
name: win
net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1
cores: 4
ide0: somestore:somedisk,discard=on,size=32G
ide2: none,media=cdrom
-machine: somemachine
+machine: q35
memory: 8192
name: win
net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1
cores: 4
ide0: somestore:somedisk,discard=on,size=32G
ide2: none,media=cdrom
-machine: somemachine
+machine: q35
memory: 8192
name: win
net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1
cores: 4
ide0: somestore:somedisk,discard=on,size=32G
ide2: none,media=cdrom
-machine: somemachine
+machine: q35
memory: 8192
name: win
net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1
cores: 4
ide0: local:snapshotable-disk-1,discard=on,size=32G
ide2: none,media=cdrom
-machine: somemachine
+machine: q35
memory: 8192
name: win
net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1
--- /dev/null
+bootdisk: ide0
+cores: 4
+ide0: local:snapshotable-disk-1,discard=on,size=32G
+ide2: none,media=cdrom
+machine: q35
+memory: 8192
+name: win
+net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1
+numa: 0
+ostype: win7
+smbios1: uuid=01234567-890a-bcde-f012-34567890abcd
+sockets: 1
+vga: qxl
cores: 4
ide0: local:snapshotable-disk-1,discard=on,size=32G
ide2: none,media=cdrom
-machine: somemachine
+machine: q35
memory: 8192
name: win
net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1
cores: 4
ide0: local:snapshotable-disk-1,discard=on,size=32G
ide2: none,media=cdrom
-machine: somemachine
+machine: q35
memory: 8192
name: win
net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1
--- /dev/null
+agent: 1
+bootdisk: ide0
+cores: 4
+ide0: local:snapshotable-disk-1,discard=on,size=32G
+ide2: none,media=cdrom
+memory: 8192
+name: win
+net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1
+numa: 0
+ostype: win7
+parent: test
+smbios1: uuid=01234567-890a-bcde-f012-34567890abcd
+sockets: 1
+vga: qxl
+
+[test]
+#test comment
+agent: 1
+bootdisk: ide0
+cores: 4
+ide0: local:snapshotable-disk-1,discard=on,size=32G
+ide2: none,media=cdrom
+machine: q35
+memory: 8192
+name: win
+net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1
+numa: 0
+ostype: win7
+smbios1: uuid=01234567-890a-bcde-f012-34567890abcd
+snaptime: 1234567890
+sockets: 1
+vga: qxl
+vmstate: somestorage:state-volume
cores: 4
ide0: local:snapshotable-disk-1,discard=on,size=32G
ide2: none,media=cdrom
-machine: somemachine
+machine: q35
memory: 8192
name: win
net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1
cores: 2
ide0: local:snapshotable-disk-1,size=32G
ide2: none,media=cdrom
-machine: someothermachine
+machine: pc
memory: 4096
name: win
net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1
cores: 2
ide0: local:snapshotable-disk-1,size=32G
ide2: none,media=cdrom
-machine: someothermachine
+machine: pc
memory: 4096
name: win
net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1
cores: 2
ide0: local:snapshotable-disk-1,size=32G
ide2: none,media=cdrom
-machine: someothermachine
+machine: pc
memory: 4096
name: win
net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1
cores: 4
ide0: local:snapshotable-disk-1,discard=on,size=32G
ide2: none,media=cdrom
-machine: someothermachine
+machine: pc
memory: 8192
name: win
net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1
cores: 4
ide0: local:snapshotable-disk-1,discard=on,size=32G
ide2: none,media=cdrom
-machine: somemachine
+machine: q35
memory: 8192
name: win
net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1
cores: 4
ide0: local:snapshotable-disk-1,discard=on,size=32G
ide2: none,media=cdrom
-machine: somemachine
+machine: q35
memory: 8192
name: win
net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1
cores: 4
ide0: local:unsnapshotable-disk-1,discard=on,size=32G
ide2: none,media=cdrom
-machine: somemachine
+machine: q35
memory: 8192
name: win
net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1
cores: 4
ide0: local:snapshotable-disk-1,discard=on,size=32G
ide2: none,media=cdrom
-machine: somemachine
+machine: q35
memory: 8192
name: win
net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1
cores: 4
ide0: local:snapshotable-disk-1,discard=on,size=32G
ide2: none,media=cdrom
-machine: somemachine
+machine: q35
memory: 8192
name: win
net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1
cores: 4
ide0: local:snapshotable-disk-1,discard=on,size=32G
ide2: none,media=cdrom
-machine: somemachine
+machine: q35
memory: 8192
name: win
net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1
cores: 4
ide0: local:snapshotable-disk-1,discard=on,size=32G
ide2: none,media=cdrom
-machine: somemachine
+machine: q35
memory: 8192
name: win
net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1
net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1
numa: 0
ostype: win7
-runningmachine: somemachine
+runningmachine: q35
smbios1: uuid=01234567-890a-bcde-f012-34567890abcd
snaptime: 1234567890
sockets: 1
--- /dev/null
+agent: 1
+bootdisk: ide0
+cores: 4
+ide0: local:snapshotable-disk-1,discard=on,size=32G
+ide2: none,media=cdrom
+memory: 8192
+name: win
+net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1
+numa: 0
+ostype: win7
+parent: test
+smbios1: uuid=01234567-890a-bcde-f012-34567890abcd
+sockets: 1
+vga: qxl
+
+[test]
+#test comment
+agent: 1
+bootdisk: ide0
+cores: 4
+ide0: local:snapshotable-disk-1,discard=on,size=32G
+ide2: none,media=cdrom
+machine: q35
+memory: 8192
+name: win
+net0: e1000=12:34:56:78:90:12,bridge=somebr0,firewall=1
+numa: 0
+ostype: win7
+runningmachine: q35
+smbios1: uuid=01234567-890a-bcde-f012-34567890abcd
+snaptime: 1234567890
+sockets: 1
+vga: qxl
+vmstate: somestorage:state-volume
use Test::MockModule;
use Test::More;
+my $activate_storage_possible = 1;
my $nodename;
my $snapshot_possible;
my $vol_snapshot_possible = {};
die "volume_rollback_is_possible failed\n";
}
+sub mocked_activate_storage {
+ my ($storecfg, $storeid) = @_;
+ die "Storage config not mocked! aborting\n"
+ if defined($storecfg);
+ die "storage activation failed\n"
+ if !$activate_storage_possible;
+ return;
+}
+
sub mocked_activate_volumes {
my ($storecfg, $volumes) = @_;
die "Storage config not mocked! aborting\n"
my $snap = $conf->{snapshots}->{$snapname};
$snap->{vmstate} = "somestorage:state-volume";
- $snap->{runningmachine} = "somemachine"
+ $snap->{runningmachine} = "q35"
}
sub assert_config_exists_on_node {
my ($vmid, $cmd) = @_;
my $exec = $cmd->{execute};
- if ($exec eq "delete-drive-snapshot") {
- return;
- }
if ($exec eq "guest-ping") {
die "guest-ping disabled\n"
if !$vm_mon->{guest_ping};
return;
}
if ($exec eq "query-savevm") {
- return { "status" => "completed" };
+ return {
+ "status" => "completed",
+ "bytes" => 1024*1024*1024,
+ "total-time" => 5000,
+ };
}
die "unexpected vm_qmp_command!\n";
}
return;
}
+sub set_migration_caps {} # ignored
+
# END redefine PVE::QemuServer methods
PVE::Tools::run_command("rm -rf snapshot-working");
my $storage_module = Test::MockModule->new('PVE::Storage');
$storage_module->mock('config', sub { return; });
$storage_module->mock('path', sub { return "/some/store/statefile/path"; });
+$storage_module->mock('activate_storage', \&mocked_activate_storage);
$storage_module->mock('activate_volumes', \&mocked_activate_volumes);
$storage_module->mock('deactivate_volumes', \&mocked_deactivate_volumes);
$storage_module->mock('vdisk_free', \&mocked_vdisk_free);
testcase_create("202", "test", 0, "test comment", "volume snapshot disabled\n\n", { "local:snapshotable-disk-1" => "test" }, { "local:snapshotable-disk-1" => "test" });
$vm_mon->{savevm_start} = 0;
-printf("Expected error for snapshot_create when Qemu mon command 'savevm-start' fails\n");
+printf("Expected error for snapshot_create when QEMU mon command 'savevm-start' fails\n");
testcase_create("203", "test", 0, "test comment", "savevm-start disabled\n\n");
$vm_mon->{savevm_start} = 1;
printf("Successful snapshot_create with no existing snapshots but set machine type\n");
testcase_create("301", "test", 1, "test comment", "", { "local:snapshotable-disk-1" => "test" });
+$activate_storage_possible = 0;
+
+printf("Expected error for snapshot_create when storage activation is not possible\n");
+testcase_create("303", "test", 1, "test comment", "storage activation failed\n\n");
+
+$activate_storage_possible = 1;
+
$nodename = "delete";
printf("\n");
printf("Running delete tests\n");
printf("Expected error for snapshot_delete with locked config\n");
testcase_delete("203", "test", 0, "VM is locked (backup)\n");
+$activate_storage_possible = 0;
+
+printf("Expected error for snapshot_delete when storage activation is not possible\n");
+testcase_delete("204", "test", 0, "storage activation failed\n");
+
+$activate_storage_possible = 1;
+
$nodename = "rollback";
printf("\n");
printf("Running rollback tests\n");
printf("Successful snapshot_rollback with saved vmstate and machine config and runningmachine \n");
testcase_rollback("302", "test", "", { "local:snapshotable-disk-1" => "test" });
+$activate_storage_possible = 0;
+
+printf("Expected error for snapshot_rollback when storage activation is not possible\n");
+testcase_rollback("303", "test", "storage activation failed\n");
+
+$activate_storage_possible = 1;
+
done_testing();
1;
use PVE::QemuServer;
use PVE::Tools qw(run_command);
use PVE::Network;
+use PVE::Firewall;
my $have_sdn;
eval {
require PVE::Network::SDN::Zones;
+ require PVE::Network::SDN::Vnets;
$have_sdn = 1;
};
my $netconf = $conf->{$netid};
$netconf = $conf->{pending}->{$netid} if !$migratedfrom && defined($conf->{pending}->{$netid});
-
+
die "unable to get network config '$netid'\n"
if !defined($netconf);
my $net = PVE::QemuServer::parse_net($netconf);
die "unable to parse network config '$netid'\n" if !$net;
+# The nftable-based implementation from the newer proxmox-firewall does not requires FW bridges
+my $create_firewall_bridges = $net->{firewall} && !PVE::Firewall::is_nftables();
+
if ($have_sdn) {
+ PVE::Network::SDN::Vnets::add_dhcp_mapping($net->{bridge}, $net->{macaddr}, $vmid, $conf->{name});
PVE::Network::SDN::Zones::tap_create($iface, $net->{bridge});
- PVE::Network::SDN::Zones::tap_plug($iface, $net->{bridge}, $net->{tag}, $net->{firewall}, $net->{trunks}, $net->{rate});
+ PVE::Network::SDN::Zones::tap_plug($iface, $net->{bridge}, $net->{tag}, $create_firewall_bridges, $net->{trunks}, $net->{rate});
} else {
PVE::Network::tap_create($iface, $net->{bridge});
- PVE::Network::tap_plug($iface, $net->{bridge}, $net->{tag}, $net->{firewall}, $net->{trunks}, $net->{rate});
+ PVE::Network::tap_plug($iface, $net->{bridge}, $net->{tag}, $create_firewall_bridges, $net->{trunks}, $net->{rate});
}
exit 0;
projects / qemu-server.git / commitdiff