# 'backup', 'snapshot' or 'rollback'. Most actions are not allowed when such lock is set.
# But you can ignore this kind of lock with the --skiplock flag.
-cfs_register_file('/qemu-server/',
- \&parse_vm_config,
- \&write_vm_config);
+cfs_register_file(
+ '/qemu-server/',
+ \&parse_vm_config,
+ \&write_vm_config
+);
PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
description => "Some command save/restore state from this location.",
optional => 1,
});
-#no warnings 'redefine';
-
+# FIXME: remove in favor of just using the INotify one, it's cached there exactly the same way
my $nodename_cache;
sub nodename {
$nodename_cache //= PVE::INotify::nodename();
},
affinity => {
type => 'string', format => 'pve-cpuset',
- description => "List of host cores used to execute guest processes.",
+ description => "List of host cores used to execute guest processes, for example: 0,5,8-11",
optional => 1,
},
};
}),
queues => {
type => 'integer',
- minimum => 0, maximum => 16,
+ minimum => 0, maximum => 64,
description => 'Number of packet queues to be used on the device.',
optional => 1,
},
}
sub print_netdevice_full {
- my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
+ my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type, $machine_version) = @_;
my $device = $net->{model};
if ($net->{model} eq 'virtio') {
# and out of each queue plus one config interrupt and control vector queue
my $vectors = $net->{queues} * 2 + 2;
$tmpstr .= ",vectors=$vectors,mq=on";
+ if (min_version($machine_version, 7, 1)) {
+ $tmpstr .= ",packed=on";
+ }
+ }
+
+ if (min_version($machine_version, 7, 1) && $net->{model} eq 'virtio'){
+ $tmpstr .= ",rx_queue_size=1024,tx_queue_size=1024";
}
+
$tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
if (my $mtu = $net->{mtu}) {
return dclone($confdesc_cloudinit);
}
+sub cloudinit_pending_properties {
+ my $p = {
+ map { $_ => 1 } keys $confdesc_cloudinit->%*,
+ name => 1,
+ };
+ $p->{"net$_"} = 1 for 0..($MAX_NETS-1);
+ return $p;
+}
+
sub check_type {
my ($key, $value) = @_;
my $conf = $res;
my $descr;
+ my $finish_description = sub {
+ if (defined($descr)) {
+ $descr =~ s/\s+$//;
+ $conf->{description} = $descr;
+ }
+ $descr = undef;
+ };
my $section = '';
my @lines = split(/\n/, $raw);
if ($line =~ m/^\[PENDING\]\s*$/i) {
$section = 'pending';
- if (defined($descr)) {
- $descr =~ s/\s+$//;
- $conf->{description} = $descr;
- }
- $descr = undef;
+ $finish_description->();
$conf = $res->{$section} = {};
next;
} elsif ($line =~ m/^\[special:cloudinit\]\s*$/i) {
$section = 'cloudinit';
- $descr = undef;
+ $finish_description->();
$conf = $res->{$section} = {};
next;
} elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
$section = $1;
- if (defined($descr)) {
- $descr =~ s/\s+$//;
- $conf->{description} = $descr;
- }
- $descr = undef;
+ $finish_description->();
$conf = $res->{snapshots}->{$section} = {};
next;
}
} elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
my $key = $1;
my $value = $2;
+ if ($section eq 'cloudinit') {
+ # ignore validation only used for informative purpose
+ $conf->{$key} = $value;
+ next;
+ }
eval { $value = check_type($key, $value); };
if ($@) {
$handle_error->("vm $vmid - unable to parse value of '$key' - $@");
}
}
- if (defined($descr)) {
- $descr =~ s/\s+$//;
- $conf->{description} = $descr;
- }
+ $finish_description->();
delete $res->{snapstate}; # just to be sure
return $res;
&$cleanup_config($conf->{pending}, 1);
- &$cleanup_config($conf->{cloudinit});
-
foreach my $snapname (keys %{$conf->{snapshots}}) {
die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
&$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
$raw .= &$generate_raw_config($conf->{pending}, 1);
}
- if (scalar(keys %{$conf->{cloudinit}})){
+ if (scalar(keys %{$conf->{cloudinit}}) && PVE::QemuConfig->has_cloudinit($conf)){
$raw .= "\n[special:cloudinit]\n";
$raw .= &$generate_raw_config($conf->{cloudinit});
}
sub check_running {
my ($vmid, $nocheck, $node) = @_;
+ # $nocheck is set when called during a migration, in which case the config
+ # file might still or already reside on the *other* node
+ # - because rename has already happened, and current node is source
+ # - because rename hasn't happened yet, and current node is target
+ # - because rename has happened, current node is target, but hasn't yet
+ # processed it yet
PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
return PVE::QemuServer::Helpers::vm_running_locally($vmid);
}
my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu,
$pbs_backing) = @_;
- my $cmd = [];
my ($globalFlags, $machineFlags, $rtcFlags) = ([], [], []);
my $devices = [];
my $bridges = {};
my $use_old_bios_files = undef;
($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
+ my $cmd = [];
if ($conf->{affinity}) {
- push @$cmd, "/usr/bin/taskset";
- push @$cmd, "--cpu-list";
- push @$cmd, "--all-tasks";
- push @$cmd, $conf->{affinity};
+ push @$cmd, '/usr/bin/taskset', '--cpu-list', '--all-tasks', $conf->{affinity};
}
push @$cmd, $kvm_binary;
push @$devices, '-netdev', $netdevfull;
my $netdevicefull = print_netdevice_full(
- $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type);
+ $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type, $machine_version);
push @$devices, '-device', $netdevicefull;
}
return if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
+ my $machine_version = PVE::QemuServer::Machine::extract_version($machine_type);
my $use_old_bios_files = undef;
($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
my $netdevicefull = print_netdevice_full(
- $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
+ $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type, $machine_version);
qemu_deviceadd($vmid, $netdevicefull);
eval {
qemu_deviceaddverify($vmid, $deviceid);
$errors->{$opt} = "hotplug problem - $msg";
};
+ my $cloudinit_pending_properties = PVE::QemuServer::cloudinit_pending_properties();
+
+ my $cloudinit_record_changed = sub {
+ my ($conf, $opt, $old, $new) = @_;
+ return if !$cloudinit_pending_properties->{$opt};
+
+ my $ci = ($conf->{cloudinit} //= {});
+
+ my $recorded = $ci->{$opt};
+ my %added = map { $_ => 1 } PVE::Tools::split_list(delete($ci->{added}) // '');
+
+ if (defined($new)) {
+ if (defined($old)) {
+ # an existing value is being modified
+ if (defined($recorded)) {
+ # the value was already not in sync
+ if ($new eq $recorded) {
+ # a value is being reverted to the cloud-init state:
+ delete $ci->{$opt};
+ delete $added{$opt};
+ } else {
+ # the value was changed multiple times, do nothing
+ }
+ } elsif ($added{$opt}) {
+ # the value had been marked as added and is being changed, do nothing
+ } else {
+ # the value is new, record it:
+ $ci->{$opt} = $old;
+ }
+ } else {
+ # a new value is being added
+ if (defined($recorded)) {
+ # it was already not in sync
+ if ($new eq $recorded) {
+ # a value is being reverted to the cloud-init state:
+ delete $ci->{$opt};
+ delete $added{$opt};
+ } else {
+ # the value had temporarily been removed, do nothing
+ }
+ } elsif ($added{$opt}) {
+ # the value had been marked as added already, do nothing
+ } else {
+ # the value is new, add it
+ $added{$opt} = 1;
+ }
+ }
+ } elsif (!defined($old)) {
+ # a non-existent value is being removed? ignore...
+ } else {
+ # a value is being deleted
+ if (defined($recorded)) {
+ # a value was already recorded, just keep it
+ } elsif ($added{$opt}) {
+ # the value was marked as added, remove it
+ delete $added{$opt};
+ } else {
+ # a previously unrecorded value is being removed, record the old value:
+ $ci->{$opt} = $old;
+ }
+ }
+
+ my $added = join(',', sort keys %added);
+ $ci->{added} = $added if length($added);
+ };
+
my $changes = 0;
foreach my $opt (keys %{$conf->{pending}}) { # add/change
if ($fast_plug_option->{$opt}) {
- $conf->{$opt} = $conf->{pending}->{$opt};
- delete $conf->{pending}->{$opt};
+ my $new = delete $conf->{pending}->{$opt};
+ $cloudinit_record_changed->($conf, $opt, $conf->{$opt}, $new);
+ $conf->{$opt} = $new;
$changes = 1;
}
}
my $cgroup = PVE::QemuServer::CGroup->new($vmid);
my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
+
foreach my $opt (sort keys %$pending_delete_hash) {
next if $selection && !$selection->{$opt};
my $force = $pending_delete_hash->{$opt}->{force};
if (my $err = $@) {
&$add_error($opt, $err) if $err ne "skip\n";
} else {
- delete $conf->{$opt};
+ my $old = delete $conf->{$opt};
+ $cloudinit_record_changed->($conf, $opt, $old, undef);
PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
}
}
+ my $cloudinit_opt;
foreach my $opt (keys %{$conf->{pending}}) {
next if $selection && !$selection->{$opt};
my $value = $conf->{pending}->{$opt};
# some changes can be done without hotplug
my $drive = parse_drive($opt, $value);
if (drive_is_cloudinit($drive)) {
- PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
+ $cloudinit_opt = [$opt, $drive];
+ # apply all the other changes first, then generate the cloudinit disk
+ die "skip\n";
}
vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
$vmid, $opt, $value, $arch, $machine_type);
die "skip\n"; # skip non-hot-pluggable options
}
};
+ if (my $err = $@) {
+ &$add_error($opt, $err) if $err ne "skip\n";
+ } else {
+ $cloudinit_record_changed->($conf, $opt, $conf->{$opt}, $value);
+ $conf->{$opt} = $value;
+ delete $conf->{pending}->{$opt};
+ }
+ }
+
+ if (defined($cloudinit_opt)) {
+ my ($opt, $drive) = @$cloudinit_opt;
+ my $value = $conf->{pending}->{$opt};
+ eval {
+ my $temp = {%$conf, $opt => $value};
+ PVE::QemuServer::Cloudinit::apply_cloudinit_config($temp, $vmid);
+ vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
+ $vmid, $opt, $value, $arch, $machine_type);
+ };
if (my $err = $@) {
&$add_error($opt, $err) if $err ne "skip\n";
} else {
PVE::QemuConfig->write_config($vmid, $conf);
- if($hotplug_features->{cloudinit}) {
- my $pending = PVE::QemuServer::Cloudinit::get_pending_config($conf, $vmid);
- my $regenerate = undef;
- for my $item (@$pending) {
- $regenerate = 1 if defined($item->{delete}) or defined($item->{pending});
- }
- PVE::QemuServer::vmconfig_update_cloudinit_drive($storecfg, $conf, $vmid) if $regenerate;
+ if ($hotplug_features->{cloudinit} && PVE::QemuServer::Cloudinit::has_changes($conf)) {
+ PVE::QemuServer::vmconfig_update_cloudinit_drive($storecfg, $conf, $vmid);
}
}
sub vmconfig_apply_pending {
- my ($vmid, $conf, $storecfg, $errors) = @_;
+ my ($vmid, $conf, $storecfg, $errors, $skip_cloud_init) = @_;
return if !scalar(keys %{$conf->{pending}});
PVE::QemuConfig->cleanup_pending($conf);
- my $generate_cloudnit = undef;
+ my $generate_cloudinit = $skip_cloud_init ? 0 : undef;
foreach my $opt (keys %{$conf->{pending}}) { # add/change
next if $opt eq 'delete'; # just to be sure
if (is_valid_drivename($opt)) {
my $drive = parse_drive($opt, $conf->{pending}->{$opt});
- $generate_cloudnit = 1 if drive_is_cloudinit($drive);
+ $generate_cloudinit //= 1 if drive_is_cloudinit($drive);
}
$conf->{$opt} = delete $conf->{pending}->{$opt};
# write all changes at once to avoid unnecessary i/o
PVE::QemuConfig->write_config($vmid, $conf);
- PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid) if $generate_cloudnit;
+ if ($generate_cloudinit) {
+ if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) {
+ # After successful generation and if there were changes to be applied, update the
+ # config to drop the {cloudinit} entry.
+ PVE::QemuConfig->write_config($vmid, $conf);
+ }
+ }
}
sub vmconfig_update_net {
safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
- safe_string_ne($drive->{ssd}, $old_drive->{ssd})) {
+ safe_string_ne($drive->{ssd}, $old_drive->{ssd}) ||
+ safe_string_ne($drive->{ro}, $old_drive->{ro})) {
die "skip\n";
}
return if !$cloudinit_drive;
- PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
+ if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) {
+ PVE::QemuConfig->write_config($vmid, $conf);
+ }
+
my $running = PVE::QemuServer::check_running($vmid);
if ($running) {
# don't regenerate the ISO if the VM is started as part of a live migration
# this way we can reuse the old ISO with the correct config
- PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid) if !$migratedfrom;
+ if (!$migratedfrom) {
+ if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) {
+ # FIXME: apply_cloudinit_config updates $conf in this case, and it would only drop
+ # $conf->{cloudinit}, so we could just not do this?
+ # But we do it above, so for now let's be consistent.
+ $conf = PVE::QemuConfig->load_config($vmid); # update/reload
+ }
+ }
# override offline migrated volumes, conf is out of date still
if (my $offline_volumes = $migrate_opts->{offline_volumes}) {
my $defaults = load_defaults();
# set environment variable useful inside network script
- $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
+ # for remote migration the config is available on the target node!
+ if (!$migrate_opts->{remote_node}) {
+ $ENV{PVE_MIGRATED_FROM} = $migratedfrom;
+ }
PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
return $migration_ip;
};
- my $migrate_uri;
if ($statefile) {
if ($statefile eq 'tcp') {
- my $localip = "localhost";
+ my $migrate = $res->{migrate} = { proto => 'tcp' };
+ $migrate->{addr} = "localhost";
my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
my $nodename = nodename();
}
if ($migration_type eq 'insecure') {
- $localip = $get_migration_ip->($nodename);
- $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
+ $migrate->{addr} = $get_migration_ip->($nodename);
+ $migrate->{addr} = "[$migrate->{addr}]" if Net::IP::ip_is_ipv6($migrate->{addr});
}
my $pfamily = PVE::Tools::get_host_address_family($nodename);
- my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
- $migrate_uri = "tcp:${localip}:${migrate_port}";
- push @$cmd, '-incoming', $migrate_uri;
+ $migrate->{port} = PVE::Tools::next_migrate_port($pfamily);
+ $migrate->{uri} = "tcp:$migrate->{addr}:$migrate->{port}";
+ push @$cmd, '-incoming', $migrate->{uri};
push @$cmd, '-S';
} elsif ($statefile eq 'unix') {
# should be default for secure migrations as a ssh TCP forward
# tunnel is not deterministic reliable ready and fails regurarly
# to set up in time, so use UNIX socket forwards
- my $socket_addr = "/run/qemu-server/$vmid.migrate";
- unlink $socket_addr;
-
- $migrate_uri = "unix:$socket_addr";
+ my $migrate = $res->{migrate} = { proto => 'unix' };
+ $migrate->{addr} = "/run/qemu-server/$vmid.migrate";
+ unlink $migrate->{addr};
- push @$cmd, '-incoming', $migrate_uri;
+ $migrate->{uri} = "unix:$migrate->{addr}";
+ push @$cmd, '-incoming', $migrate->{uri};
push @$cmd, '-S';
} elsif (-e $statefile) {
eval { PVE::QemuServer::PCI::reserve_pci_usage($pci_id_list, $vmid, undef, $pid) };
warn $@ if $@;
- print "migration listens on $migrate_uri\n" if $migrate_uri;
- $res->{migrate_uri} = $migrate_uri;
-
- if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix') {
+ if (defined($res->{migrate})) {
+ print "migration listens on $res->{migrate}->{uri}\n";
+ } elsif ($statefile) {
eval { mon_cmd($vmid, "cont"); };
warn $@ if $@;
}
my $migrate_storage_uri;
# nbd_protocol_version > 0 for unix socket support
- if ($nbd_protocol_version > 0 && $migration_type eq 'secure') {
+ if ($nbd_protocol_version > 0 && ($migration_type eq 'secure' || $migration_type eq 'websocket')) {
my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
$migrate_storage_uri = "nbd:unix:$socket_path";
+ $res->{migrate}->{unix_sockets} = [$socket_path];
} else {
my $nodename = nodename();
my $localip = $get_migration_ip->($nodename);
$migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
}
- $res->{migrate_storage_uri} = $migrate_storage_uri;
-
foreach my $opt (sort keys %$nbd) {
my $drivestr = $nbd->{$opt}->{drivestr};
my $volid = $nbd->{$opt}->{volid};
}
}
+# $nocheck is set when called as part of a migration - in this context the
+# location of the config file (source or target node) is not deterministic,
+# since migration cannot wait for pmxcfs to process the rename
sub vm_resume {
my ($vmid, $skiplock, $nocheck) = @_;
my $res = mon_cmd($vmid, 'query-status');
my $resume_cmd = 'cont';
my $reset = 0;
- my $conf = PVE::QemuConfig->load_config($vmid);
+ my $conf;
+ if ($nocheck) {
+ $conf = eval { PVE::QemuConfig->load_config($vmid) }; # try on target node
+ if ($@) {
+ my $vmlist = PVE::Cluster::get_vmlist();
+ if (exists($vmlist->{ids}->{$vmid})) {
+ my $node = $vmlist->{ids}->{$vmid}->{node};
+ $conf = eval { PVE::QemuConfig->load_config($vmid, $node) }; # try on source node
+ }
+ if (!$conf) {
+ PVE::Cluster::cfs_update(); # vmlist was wrong, invalidate cache
+ $conf = PVE::QemuConfig->load_config($vmid); # last try on target node again
+ }
+ }
+ } else {
+ $conf = PVE::QemuConfig->load_config($vmid);
+ }
if ($res->{status}) {
return if $res->{status} eq 'running'; # job done, go home
}
if (!$nocheck) {
-
PVE::QemuConfig->check_lock($conf)
if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
}
next;
}
+ my $bridge = $net->{bridge};
if ($have_sdn) {
- PVE::Network::SDN::Zones::add_bridge_fdb($iface, $mac, $net->{bridge}, $net->{firewall});
- } else {
+ PVE::Network::SDN::Zones::add_bridge_fdb($iface, $mac, $bridge, $net->{firewall});
+ } elsif (-d "/sys/class/net/$bridge/bridge") { # avoid fdb management with OVS for now
PVE::Network::add_bridge_fdb($iface, $mac, $net->{firewall});
}
}
}
+sub del_nets_bridge_fdb {
+ my ($conf, $vmid) = @_;
+
+ for my $opt (keys %$conf) {
+ next if $opt !~ m/^net(\d+)$/;
+ my $iface = "tap${vmid}i$1";
+
+ my $net = parse_net($conf->{$opt}) or next;
+ my $mac = $net->{macaddr} or next;
+
+ my $bridge = $net->{bridge};
+ if ($have_sdn) {
+ PVE::Network::SDN::Zones::del_bridge_fdb($iface, $mac, $bridge, $net->{firewall});
+ } elsif (-d "/sys/class/net/$bridge/bridge") { # avoid fdb management with OVS for now
+ PVE::Network::del_bridge_fdb($iface, $mac, $net->{firewall});
+ }
+ }
+}
+
1;