use PVE::JSONSchema qw(get_standard_option parse_property_string);
use PVE::ProcFSTools;
use PVE::PBSClient;
+use PVE::RESTEnvironment qw(log_warn);
use PVE::RPCEnvironment;
use PVE::Storage;
use PVE::SysFSTools;
optional => 1,
});
-PVE::JSONSchema::register_standard_option('pve-targetstorage', {
- description => "Mapping from source to target storages. Providing only a single storage ID maps all source storages to that storage. Providing the special value '1' will map each source storage to itself.",
- type => 'string',
- format => 'storage-pair-list',
- optional => 1,
-});
-
#no warnings 'redefine';
my $nodename_cache;
searchdomain => {
optional => 1,
type => 'string',
- description => "cloud-init: Sets DNS search domains for a container. Create will'
+ description => 'cloud-init: Sets DNS search domains for a container. Create will'
.' automatically use the setting from the host if neither searchdomain nor nameserver'
- .' are set.",
+ .' are set.',
},
nameserver => {
optional => 1,
type => 'string', format => 'address-list',
- description => "cloud-init: Sets DNS server IP address for a container. Create will'
+ description => 'cloud-init: Sets DNS server IP address for a container. Create will'
.' automatically use the setting from the host if neither searchdomain nor nameserver'
- .' are set.",
+ .' are set.',
},
sshkeys => {
optional => 1,
# sometimes, just plain disable...
my $lvm_no_io_uring = $scfg && $scfg->{type} eq 'lvm';
+ # io_uring causes problems when used with CIFS since kernel 5.15
+ # Some discussion: https://www.spinics.net/lists/linux-cifs/msg26734.html
+ my $cifs_no_io_uring = $scfg && $scfg->{type} eq 'cifs';
+
if (!$drive->{aio}) {
- if ($io_uring && !$rbd_no_io_uring && !$lvm_no_io_uring) {
+ if ($io_uring && !$rbd_no_io_uring && !$lvm_no_io_uring && !$cifs_no_io_uring) {
# io_uring supports all cache modes
$opts .= ",aio=io_uring";
} else {
my ($pbs_conf, $pbs_name) = @_;
my $blockdev = "driver=pbs,node-name=$pbs_name,read-only=on";
$blockdev .= ",repository=$pbs_conf->{repository}";
+ $blockdev .= ",namespace=$pbs_conf->{namespace}" if $pbs_conf->{namespace};
$blockdev .= ",snapshot=$pbs_conf->{snapshot}";
$blockdev .= ",archive=$pbs_conf->{archive}";
$blockdev .= ",keyfile=$pbs_conf->{keyfile}" if $pbs_conf->{keyfile};
$pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
}
+ if ($vga->{type} eq 'virtio-gl') {
+ my $base = '/usr/lib/x86_64-linux-gnu/lib';
+ die "missing libraries for '$vga->{type}' detected! Please install 'libgl1' and 'libegl1'\n"
+ if !-e "${base}EGL.so.1" || !-e "${base}GL.so.1";
+
+ die "no DRM render node detected (/dev/dri/renderD*), no GPU? - needed for '$vga->{type}' display\n"
+ if !PVE::Tools::dir_glob_regex('/dev/dri/', "renderD.*");
+ }
+
return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
}
if (drive_is_cloudinit($drive)) {
eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
warn $@ if $@;
+ delete $conf->{cloudinit};
} elsif (!drive_is_cdrom($drive)) {
my $volid = $drive->{file};
if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
digest => Digest::SHA::sha1_hex($raw),
snapshots => {},
pending => {},
+ cloudinit => {},
};
my $handle_error = sub {
$descr = undef;
$conf = $res->{$section} = {};
next;
+ } elsif ($line =~ m/^\[special:cloudinit\]\s*$/i) {
+ $section = 'cloudinit';
+ $descr = undef;
+ $conf = $res->{$section} = {};
+ next;
} elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
$section = $1;
next;
}
- if ($line =~ m/^\#(.*)\s*$/) {
+ if ($line =~ m/^\#(.*)$/) {
$descr = '' if !defined($descr);
$descr .= PVE::Tools::decode_text($1) . "\n";
next;
foreach my $key (keys %$cref) {
next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
- $key eq 'snapstate' || $key eq 'pending';
+ $key eq 'snapstate' || $key eq 'pending' || $key eq 'cloudinit';
my $value = $cref->{$key};
if ($key eq 'delete') {
die "propertry 'delete' is only allowed in [PENDING]\n"
&$cleanup_config($conf->{pending}, 1);
+ &$cleanup_config($conf->{cloudinit});
+
foreach my $snapname (keys %{$conf->{snapshots}}) {
die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
&$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
}
foreach my $key (sort keys %$conf) {
- next if $key =~ /^(digest|description|pending|snapshots)$/;
+ next if $key =~ /^(digest|description|pending|cloudinit|snapshots)$/;
$raw .= "$key: $conf->{$key}\n";
}
return $raw;
$raw .= &$generate_raw_config($conf->{pending}, 1);
}
+ if (scalar(keys %{$conf->{cloudinit}})){
+ $raw .= "\n[special:cloudinit]\n";
+ $raw .= &$generate_raw_config($conf->{cloudinit});
+ }
+
foreach my $snapname (sort keys %{$conf->{snapshots}}) {
$raw .= "\n[$snapname]\n";
$raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
return \@flags;
}
-my sub get_cpuunits {
- my ($conf) = @_;
- my $is_cgroupv2 = PVE::CGroup::cgroup_mode() == 2;
-
- my $cpuunits = $conf->{cpuunits};
- return $is_cgroupv2 ? 100 : 1024 if !defined($cpuunits);
-
- if ($is_cgroupv2) {
- $cpuunits = 10000 if $cpuunits >= 10000; # v1 can be higher, so clamp v2 there
- } else {
- $cpuunits = 2 if $cpuunits < 2; # v2 can be lower, so clamp v1 there
- }
- return $cpuunits;
-}
-
# Since commit 277d33454f77ec1d1e0bc04e37621e4dd2424b67 in pve-qemu, smm is not off by default
# anymore. But smm=off seems to be required when using SeaBIOS and serial display.
my sub should_disable_smm {
my $vmname = $conf->{name} || "vm$vmid";
- push @$cmd, '-name', $vmname;
+ push @$cmd, '-name', "$vmname,debug-threads=on";
push @$cmd, '-no-shutdown';
$read_only_str = ',readonly=on' if drive_is_read_only($conf, $d);
} else {
- warn "no efidisk configured! Using temporary efivars disk.\n";
+ log_warn("no efidisk configured! Using temporary efivars disk.");
$path = "/tmp/$vmid-ovmf.fd";
PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
$format = 'raw';
my $spice_port;
- if ($qxlnum) {
+ if ($qxlnum || $vga->{type} =~ /^virtio/) {
if ($qxlnum > 1) {
if ($winversion){
for (my $i = 1; $i < $qxlnum; $i++){
# enable balloon by default, unless explicitly disabled
if (!defined($conf->{balloon}) || $conf->{balloon}) {
my $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
- push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
+ my $ballooncmd = "virtio-balloon-pci,id=balloon0$pciaddr";
+ $ballooncmd .= ",free-page-reporting=on" if min_version($machine_version, 6, 2);
+ push @$devices, '-device', $ballooncmd;
}
if ($conf->{watchdog}) {
die "skip\n" if !$hotplug_features->{memory};
PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
} elsif ($opt eq 'cpuunits') {
- $cgroup->change_cpu_shares(undef, 1024);
+ $cgroup->change_cpu_shares(undef);
} elsif ($opt eq 'cpulimit') {
$cgroup->change_cpu_quota(undef, undef); # reset, cgroup module can better decide values
} else {
die "skip\n" if !$hotplug_features->{memory};
$value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
} elsif ($opt eq 'cpuunits') {
- my $new_cpuunits = get_cpuunits({ $opt => $conf->{pending}->{$opt} }); # to clamp
- $cgroup->change_cpu_shares($new_cpuunits, 1024);
+ my $new_cpuunits = PVE::CGroup::clamp_cpu_shares($conf->{pending}->{$opt}); #clamp
+ $cgroup->change_cpu_shares($new_cpuunits);
} elsif ($opt eq 'cpulimit') {
my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
$cgroup->change_cpu_quota($cpulimit, 100000);
# type => secure/insecure - tunnel over encrypted connection or plain-text
# nbd_proto_version => int, 0 for TCP, 1 for UNIX
# replicated_volumes => which volids should be re-used with bitmaps for nbd migration
-# tpmstate_vol => new volid of tpmstate0, not yet contained in config
+# offline_volumes => new volids of offline migrated disks like tpmstate and cloudinit, not yet
+# contained in config
sub vm_start_nolock {
my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
# this way we can reuse the old ISO with the correct config
PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid) if !$migratedfrom;
- # override TPM state vol if migrated, conf is out of date still
- if (my $tpmvol = $migrate_opts->{tpmstate_vol}) {
- my $parsed = parse_drive("tpmstate0", $conf->{tpmstate0});
- $parsed->{file} = $tpmvol;
- $conf->{tpmstate0} = print_drive($parsed);
+ # override offline migrated volumes, conf is out of date still
+ if (my $offline_volumes = $migrate_opts->{offline_volumes}) {
+ for my $key (sort keys $offline_volumes->%*) {
+ my $parsed = parse_drive($key, $conf->{$key});
+ $parsed->{file} = $offline_volumes->{$key};
+ $conf->{$key} = print_drive($parsed);
+ }
}
my $defaults = load_defaults();
PVE::QemuServer::PCI::reserve_pci_usage($pci_id_list, $vmid, $start_timeout);
eval {
+ my $uuid;
for my $id (sort keys %$pci_devices) {
my $d = $pci_devices->{$id};
for my $dev ($d->{pciid}->@*) {
- PVE::QemuServer::PCI::prepare_pci_device($vmid, $dev->{id}, $id, $d->{mdev});
+ my $info = PVE::QemuServer::PCI::prepare_pci_device($vmid, $dev->{id}, $id, $d->{mdev});
+
+ # nvidia grid needs the uuid of the mdev as qemu parameter
+ if ($d->{mdev} && !defined($uuid) && $info->{vendor} eq '10de') {
+ $uuid = PVE::QemuServer::PCI::generate_mdev_uuid($vmid, $id);
+ }
}
}
+ push @$cmd, '-uuid', $uuid if defined($uuid);
};
if (my $err = $@) {
eval { PVE::QemuServer::PCI::remove_pci_reservation($pci_id_list) };
# timeout should be more than enough here...
PVE::Systemd::wait_for_unit_removed("$vmid.scope", 20);
- my $cpuunits = get_cpuunits($conf);
+ my $cpuunits = PVE::CGroup::clamp_cpu_shares($conf->{cpuunits});
my %run_params = (
timeout => $statefile ? undef : $start_timeout,
my $restore_cleanup_oldconf = sub {
my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
+ my $kept_disks = {};
+
PVE::QemuConfig->foreach_volume($oldconf, sub {
my ($ds, $drive) = @_;
if (my $err = $@) {
warn $err;
}
+ } else {
+ $kept_disks->{$volid} = 1;
}
});
- # delete vmstate files, after the restore we have no snapshots anymore
- foreach my $snapname (keys %{$oldconf->{snapshots}}) {
+ # after the restore we have no snapshots anymore
+ for my $snapname (keys $oldconf->{snapshots}->%*) {
my $snap = $oldconf->{snapshots}->{$snapname};
if ($snap->{vmstate}) {
eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
warn $err;
}
}
+
+ for my $volid (keys $kept_disks->%*) {
+ eval { PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snapname); };
+ warn $@ if $@;
+ }
}
};
}
my $restore_deactivate_volumes = sub {
- my ($storecfg, $devinfo) = @_;
+ my ($storecfg, $virtdev_hash) = @_;
my $vollist = [];
- foreach my $devname (keys %$devinfo) {
- my $volid = $devinfo->{$devname}->{volid};
- push @$vollist, $volid if $volid;
+ for my $dev (values $virtdev_hash->%*) {
+ push $vollist->@*, $dev->{volid} if $dev->{volid};
}
- PVE::Storage::deactivate_volumes($storecfg, $vollist);
+ eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
+ print STDERR $@ if $@;
};
my $restore_destroy_volumes = sub {
- my ($storecfg, $devinfo) = @_;
+ my ($storecfg, $virtdev_hash) = @_;
- foreach my $devname (keys %$devinfo) {
- my $volid = $devinfo->{$devname}->{volid};
- next if !$volid;
+ for my $dev (values $virtdev_hash->%*) {
+ my $volid = $dev->{volid} or next;
eval {
- if ($volid =~ m|^/|) {
- unlink $volid || die 'unlink failed\n';
- } else {
- PVE::Storage::vdisk_free($storecfg, $volid);
- }
+ PVE::Storage::vdisk_free($storecfg, $volid);
print STDERR "temporary volume '$volid' sucessfuly removed\n";
};
print STDERR "unable to cleanup '$volid' - $@" if $@;
}
};
+my $restore_merge_config = sub {
+ my ($filename, $backup_conf_raw, $override_conf) = @_;
+
+ my $backup_conf = parse_vm_config($filename, $backup_conf_raw);
+ for my $key (keys $override_conf->%*) {
+ $backup_conf->{$key} = $override_conf->{$key};
+ }
+
+ return $backup_conf;
+};
+
sub scan_volids {
my ($cfg, $vmid) = @_;
my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($storecfg, $storeid);
my $repo = PVE::PBSClient::get_repository($scfg);
+ my $namespace = $scfg->{namespace};
# This is only used for `pbs-restore` and the QEMU PBS driver (live-restore)
my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
my $new_conf_raw = '';
my $rpcenv = PVE::RPCEnvironment::get();
- my $devinfo = {};
+ my $devinfo = {}; # info about drives included in backup
+ my $virtdev_hash = {}; # info about allocated drives
eval {
# enable interrupts
my $fh = IO::File->new($cfgfn, "r") ||
die "unable to read qemu-server.conf - $!\n";
- my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
+ $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
# fixme: rate limit?
# for live-restore we only want to preload the efidisk and TPM state
next if $options->{live} && $virtdev ne 'efidisk0' && $virtdev ne 'tpmstate0';
+ my @ns_arg;
+ if (defined(my $ns = $scfg->{namespace})) {
+ @ns_arg = ('--ns', $ns);
+ }
+
my $pbs_restore_cmd = [
'/usr/bin/pbs-restore',
'--repository', $repo,
+ @ns_arg,
$pbs_backup_name,
"$d->{devname}.img.fidx",
$path,
my $err = $@;
if ($err || !$options->{live}) {
- $restore_deactivate_volumes->($storecfg, $devinfo);
+ $restore_deactivate_volumes->($storecfg, $virtdev_hash);
}
rmtree $tmpdir;
if ($err) {
- $restore_destroy_volumes->($storecfg, $devinfo);
+ $restore_destroy_volumes->($storecfg, $virtdev_hash);
die $err;
}
$new_conf_raw .= "\nlock: create";
}
- PVE::Tools::file_set_contents($conffile, $new_conf_raw);
-
- PVE::Cluster::cfs_update(); # make sure we read new file
+ my $new_conf = $restore_merge_config->($conffile, $new_conf_raw, $options->{override_conf});
+ PVE::QemuConfig->write_config($vmid, $new_conf);
eval { rescan($vmid, 1); };
warn $@ if $@;
# these special drives are already restored before start
delete $devinfo->{'drive-efidisk0'};
delete $devinfo->{'drive-tpmstate0-backup'};
- pbs_live_restore($vmid, $conf, $storecfg, $devinfo, $repo, $keyfile, $pbs_backup_name);
+
+ my $pbs_opts = {
+ repo => $repo,
+ keyfile => $keyfile,
+ snapshot => $pbs_backup_name,
+ namespace => $namespace,
+ };
+ pbs_live_restore($vmid, $conf, $storecfg, $devinfo, $pbs_opts);
PVE::QemuConfig->remove_lock($vmid, "create");
}
}
sub pbs_live_restore {
- my ($vmid, $conf, $storecfg, $restored_disks, $repo, $keyfile, $snap) = @_;
+ my ($vmid, $conf, $storecfg, $restored_disks, $opts) = @_;
print "starting VM for live-restore\n";
- print "repository: '$repo', snapshot: '$snap'\n";
+ print "repository: '$opts->{repo}', snapshot: '$opts->{snapshot}'\n";
my $pbs_backing = {};
for my $ds (keys %$restored_disks) {
$ds =~ m/^drive-(.*)$/;
my $confname = $1;
$pbs_backing->{$confname} = {
- repository => $repo,
- snapshot => $snap,
+ repository => $opts->{repo},
+ snapshot => $opts->{snapshot},
archive => "$ds.img.fidx",
};
- $pbs_backing->{$confname}->{keyfile} = $keyfile if -e $keyfile;
+ $pbs_backing->{$confname}->{keyfile} = $opts->{keyfile} if -e $opts->{keyfile};
+ $pbs_backing->{$confname}->{namespace} = $opts->{namespace} if defined($opts->{namespace});
my $drive = parse_drive($confname, $conf->{$confname});
print "restoring '$ds' to '$drive->{file}'\n";
my $oldtimeout;
my $timeout = 5;
- my $devinfo = {};
+ my $devinfo = {}; # info about drives included in backup
+ my $virtdev_hash = {}; # info about allocated drives
my $rpcenv = PVE::RPCEnvironment::get();
PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
}
- my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
+ $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
foreach my $info (values %{$virtdev_hash}) {
my $storeid = $info->{storeid};
alarm($oldtimeout) if $oldtimeout;
- $restore_deactivate_volumes->($cfg, $devinfo);
+ $restore_deactivate_volumes->($cfg, $virtdev_hash);
close($fifofh) if $fifofh;
unlink $mapfifo;
rmtree $tmpdir;
if ($err) {
- $restore_destroy_volumes->($cfg, $devinfo);
+ $restore_destroy_volumes->($cfg, $virtdev_hash);
die $err;
}
- PVE::Tools::file_set_contents($conffile, $new_conf_raw);
-
- PVE::Cluster::cfs_update(); # make sure we read new file
+ my $new_conf = $restore_merge_config->($conffile, $new_conf_raw, $opts->{override_conf});
+ PVE::QemuConfig->write_config($vmid, $new_conf);
eval { rescan($vmid, 1); };
warn $@ if $@;
sub restore_tar_archive {
my ($archive, $vmid, $user, $opts) = @_;
+ if (scalar(keys $opts->{override_conf}->%*) > 0) {
+ my $keystring = join(' ', keys $opts->{override_conf}->%*);
+ die "cannot pass along options ($keystring) when restoring from tar archive\n";
+ }
+
if ($archive ne '-') {
my $firstfile = tar_archive_read_firstfile($archive);
die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"