use strict;
use warnings;
-use POSIX;
-use IO::Handle;
-use IO::Select;
-use IO::File;
-use IO::Dir;
-use IO::Socket::UNIX;
+use Cwd 'abs_path';
+use Digest::SHA;
+use Fcntl ':flock';
+use Fcntl;
use File::Basename;
+use File::Copy qw(copy);
use File::Path;
use File::stat;
use Getopt::Long;
-use Digest::SHA;
-use Fcntl ':flock';
-use Cwd 'abs_path';
+use IO::Dir;
+use IO::File;
+use IO::Handle;
+use IO::Select;
+use IO::Socket::UNIX;
use IPC::Open3;
use JSON;
-use Fcntl;
-use PVE::SafeSyslog;
-use Storable qw(dclone);
use MIME::Base64;
-use PVE::Exception qw(raise raise_param_exc);
-use PVE::Storage;
-use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach $IPV6RE);
-use PVE::JSONSchema qw(get_standard_option);
+use POSIX;
+use Storable qw(dclone);
+use Time::HiRes qw(gettimeofday);
+use URI::Escape;
+
use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
+use PVE::Exception qw(raise raise_param_exc);
+use PVE::GuestHelpers;
use PVE::INotify;
+use PVE::JSONSchema qw(get_standard_option);
use PVE::ProcFSTools;
-use PVE::QemuConfig;
-use PVE::QMPClient;
use PVE::RPCEnvironment;
-use PVE::GuestHelpers;
-use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
-use PVE::QemuServer::Memory;
-use PVE::QemuServer::USB qw(parse_usb_device);
-use PVE::QemuServer::Cloudinit;
+use PVE::SafeSyslog;
+use PVE::Storage;
use PVE::SysFSTools;
use PVE::Systemd;
-use Time::HiRes qw(gettimeofday);
-use File::Copy qw(copy);
-use URI::Escape;
+use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach get_host_arch $IPV6RE);
+
+use PVE::QMPClient;
+use PVE::QemuConfig;
+use PVE::QemuServer::Cloudinit;
+use PVE::QemuServer::Memory;
+use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port);
+use PVE::QemuServer::USB qw(parse_usb_device);
my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
my $OVMF = {
PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
description => "Specifies the Qemu machine type.",
type => 'string',
- pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?|virt(?:-\d+\.\d+)?)',
+ pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\.pxe)?|virt(?:-\d+(\.\d+)+)?)',
maxLength => 40,
optional => 1,
});
'Skylake-Client-IBRS' => 'GenuineIntel',
'Skylake-Server' => 'GenuineIntel',
'Skylake-Server-IBRS' => 'GenuineIntel',
+ 'Cascadelake-Server' => 'GenuineIntel',
+ KnightsMill => 'GenuineIntel',
+
# AMD CPUs
athlon => 'AuthenticAMD',
'pdpe1gb',
'md-clear',
'hv-tlbflush',
- 'hv-evmcs'
+ 'hv-evmcs',
+ 'aes'
);
my $cpu_flag = qr/[+-](@{[join('|', @supported_cpu_flags)]})/;
},
};
+my $spice_enhancements_fmt = {
+ foldersharing => {
+ type => 'boolean',
+ optional => 1,
+ default => '0',
+ description => "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
+ },
+ videostreaming => {
+ type => 'string',
+ enum => ['off', 'all', 'filter'],
+ default => 'off',
+ optional => 1,
+ description => "Enable video streaming. Uses compression for detected video streams."
+ },
+};
+
my $confdesc = {
onboot => {
optional => 1,
description => "Configure a audio device, useful in combination with QXL/Spice.",
optional => 1
},
+ spice_enhancements => {
+ type => 'string',
+ format => $spice_enhancements_fmt,
+ description => "Configure additional enhancements for SPICE.",
+ optional => 1
+ },
};
my $cicustom_fmt = {
my $MAX_USB_DEVICES = 5;
my $MAX_NETS = 32;
my $MAX_UNUSED_DISKS = 256;
-my $MAX_HOSTPCI_DEVICES = 4;
+my $MAX_HOSTPCI_DEVICES = 16;
my $MAX_SERIAL_PORTS = 4;
my $MAX_PARALLEL_PORTS = 3;
my $MAX_NUMA = 8;
usb3 => {
optional => 1,
type => 'boolean',
- description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
+ description => "Specifies whether if given host option is a USB3 device or port.",
default => 0,
},
};
return $kvm_api_version;
}
-my $kvm_user_version;
+my $kvm_user_version = {};
+my $kvm_mtime = {};
sub kvm_user_version {
+ my ($binary) = @_;
+
+ $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
+ my $st = stat($binary);
- return $kvm_user_version if $kvm_user_version;
+ my $cachedmtime = $kvm_mtime->{$binary} // -1;
+ return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
+ $cachedmtime == $st->mtime;
- $kvm_user_version = 'unknown';
+ $kvm_user_version->{$binary} = 'unknown';
+ $kvm_mtime->{$binary} = $st->mtime;
my $code = sub {
my $line = shift;
if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
- $kvm_user_version = $2;
+ $kvm_user_version->{$binary} = $2;
}
};
- eval { run_command("kvm -version", outfunc => $code); };
+ eval { run_command([$binary, '--version'], outfunc => $code); };
warn $@ if $@;
- return $kvm_user_version;
+ return $kvm_user_version->{$binary};
}
$path = PVE::Storage::path($storecfg, $drive->{file});
}
- if($path =~ m/^iscsi\:\/\//){
+ # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
+ if ($path =~ m/^iscsi\:\/\// &&
+ !qemu_machine_feature_enabled($machine_type, undef, 4, 1)) {
$devicetype = 'generic';
}
}
return undef;
}
-sub split_flagged_list {
- my $text = shift || '';
- $text =~ s/[,;]/ /g;
- $text =~ s/^\s+//;
- return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
-}
-
-sub join_flagged_list {
- my ($how, $lst) = @_;
- join $how, map { $lst->{$_} . $_ } keys %$lst;
-}
-
-sub vmconfig_delete_pending_option {
- my ($conf, $key, $force) = @_;
-
- delete $conf->{pending}->{$key};
- my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
- $pending_delete_hash->{$key} = $force ? '!' : '';
- $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
-}
-
-sub vmconfig_undelete_pending_option {
- my ($conf, $key) = @_;
-
- my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
- delete $pending_delete_hash->{$key};
-
- if (%$pending_delete_hash) {
- $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
- } else {
- delete $conf->{pending}->{delete};
- }
-}
-
sub vmconfig_register_unused_drive {
my ($storecfg, $vmid, $conf, $drive) = @_;
}
}
-sub vmconfig_cleanup_pending {
- my ($conf) = @_;
-
- # remove pending changes when nothing changed
- my $changes;
- foreach my $opt (keys %{$conf->{pending}}) {
- if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq $conf->{$opt})) {
- $changes = 1;
- delete $conf->{pending}->{$opt};
- }
- }
-
- my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
- my $pending_delete_hash = {};
- while (my ($opt, $force) = each %$current_delete_hash) {
- if (defined($conf->{$opt})) {
- $pending_delete_hash->{$opt} = $force;
- } else {
- $changes = 1;
- }
- }
-
- if (%$pending_delete_hash) {
- $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
- } else {
- delete $conf->{pending}->{delete};
- }
-
- return $changes;
-}
-
# smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
my $smbios1_fmt = {
uuid => {
}
}
-sub touch_config {
- my ($vmid) = @_;
-
- my $conf = PVE::QemuConfig->config_file($vmid);
- utime undef, undef, $conf;
-}
-
sub destroy_vm {
my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
- my $conffile = PVE::QemuConfig->config_file($vmid);
-
my $conf = PVE::QemuConfig->load_config($vmid);
PVE::QemuConfig->check_lock($conf) if !$skiplock;
});
- if ($keep_empty_config) {
- PVE::Tools::file_set_contents($conffile, "memory: 128\n");
- } else {
- unlink $conffile;
- }
-
# also remove unused disk
eval {
my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
};
warn $@ if $@;
+
+ if ($keep_empty_config) {
+ PVE::QemuConfig->write_config($vmid, { memory => 128 });
+ } else {
+ PVE::QemuConfig->destroy_config($vmid);
+ }
}
sub parse_vm_config {
push @loc_res, "ivshmem" if $conf->{ivshmem};
foreach my $k (keys %$conf) {
- next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
+ next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
# sockets are safe: they will recreated be on the target side post-migrate
next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
foreach my $vmid (keys %$list) {
next if $opt_vmid && ($vmid ne $opt_vmid);
- my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
- my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
+ my $conf = PVE::QemuConfig->load_config($vmid);
my $d = { vmid => $vmid };
$d->{pid} = $list->{$vmid}->{pid};
return {
dev => $audioproperties->{device},
- dev_id => "audio$id",
+ dev_id => "audiodev$id",
backend => $audiodriver,
backend_id => "$audiodriver-backend${id}",
};
return $1 || 1;
}
-my $host_arch; # FIXME: fix PVE::Tools::get_host_arch
-sub get_host_arch() {
- $host_arch = (POSIX::uname())[4] if !$host_arch;
- return $host_arch;
-}
-
sub is_native($) {
my ($arch) = @_;
return get_host_arch() eq $arch;
my $devices = [];
my $pciaddr = '';
my $bridges = {};
- my $kvmver = kvm_user_version();
my $vernum = 0; # unknown
my $ostype = $conf->{ostype};
my $winversion = windows_version($ostype);
my $kvm = $conf->{kvm};
my ($arch, $machine_type) = get_basic_machine_info($conf, $forcemachine);
+ my $kvm_binary = get_command_for_arch($arch);
+ my $kvmver = kvm_user_version($kvm_binary);
$kvm //= 1 if is_native($arch);
if ($kvm) {
die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
- my $have_ovz = -f '/proc/vz/vestat';
-
my $q35 = machine_type_is_q35($conf);
my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
my $use_old_bios_files = undef;
my $cpuunits = defined($conf->{cpuunits}) ?
$conf->{cpuunits} : $defaults->{cpuunits};
- push @$cmd, get_command_for_arch($arch);
+ push @$cmd, $kvm_binary;
push @$cmd, '-id', $vmid;
# host pci devices
for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++) {
- my $d = parse_hostpci($conf->{"hostpci$i"});
+ my $id = "hostpci$i";
+ my $d = parse_hostpci($conf->{$id});
next if !$d;
- my $pcie = $d->{pcie};
- if ($pcie) {
+ if (my $pcie = $d->{pcie}) {
die "q35 machine model is not enabled" if !$q35;
# win7 wants to have the pcie devices directly on the pcie bus
# instead of in the root port
if ($winversion == 7) {
- $pciaddr = print_pcie_addr("hostpci${i}bus0");
+ $pciaddr = print_pcie_addr("${id}bus0");
} else {
- $pciaddr = print_pcie_addr("hostpci$i");
+ # add more root ports if needed, 4 are present by default
+ # by pve-q35 cfgs, rest added here on demand.
+ if ($i > 3) {
+ push @$devices, '-device', print_pcie_root_port($i);
+ }
+ $pciaddr = print_pcie_addr($id);
}
} else {
- $pciaddr = print_pci_addr("hostpci$i", $bridges, $arch, $machine_type);
+ $pciaddr = print_pci_addr($id, $bridges, $arch, $machine_type);
}
- my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
- my $romfile = $d->{romfile};
-
my $xvga = '';
if ($d->{'x-vga'}) {
- $xvga = ',x-vga=on';
+ $xvga = ',x-vga=on' if !($conf->{bios} && $conf->{bios} eq 'ovmf');
$kvm_off = 1;
$vga->{type} = 'none' if !defined($conf->{vga});
$gpu_passthrough = 1;
-
- if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
- $xvga = "";
- }
}
+
my $pcidevices = $d->{pciid};
my $multifunction = 1 if @$pcidevices > 1;
+
my $sysfspath;
if ($d->{mdev} && scalar(@$pcidevices) == 1) {
- my $id = $pcidevices->[0]->{id};
+ my $pci_id = $pcidevices->[0]->{id};
my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
- $sysfspath = "/sys/bus/pci/devices/0000:$id/$uuid";
+ $sysfspath = "/sys/bus/pci/devices/0000:$pci_id/$uuid";
} elsif ($d->{mdev}) {
- warn "ignoring mediated device with multifunction device\n";
+ warn "ignoring mediated device '$id' with multifunction device\n";
}
my $j=0;
- foreach my $pcidevice (@$pcidevices) {
-
- my $id = "hostpci$i";
- $id .= ".$j" if $multifunction;
- my $addr = $pciaddr;
- $addr .= ".$j" if $multifunction;
+ foreach my $pcidevice (@$pcidevices) {
my $devicestr = "vfio-pci";
+
if ($sysfspath) {
$devicestr .= ",sysfsdev=$sysfspath";
} else {
$devicestr .= ",host=$pcidevice->{id}";
}
- $devicestr .= ",id=$id$addr";
- if($j == 0){
- $devicestr .= "$rombar$xvga";
+ my $mf_addr = $multifunction ? ".$j" : '';
+ $devicestr .= ",id=${id}${mf_addr}${pciaddr}${mf_addr}";
+
+ if ($j == 0) {
+ $devicestr .= ',rombar=0' if defined($d->{rombar}) && !$d->{rombar};
+ $devicestr .= "$xvga";
$devicestr .= ",multifunction=on" if $multifunction;
- $devicestr .= ",romfile=/usr/share/kvm/$romfile" if $romfile;
+ $devicestr .= ",romfile=/usr/share/kvm/$d->{romfile}" if $d->{romfile};
}
push @$devices, '-device', $devicestr;
}
# usb devices
- my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES);
+ my $usb_dev_features = {};
+ $usb_dev_features->{spice_usb3} = 1 if qemu_machine_feature_enabled($machine_type, $kvmver, 4, 0);
+
+ my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features);
push @$devices, @usbdevices if @usbdevices;
# serial devices
for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
my $pfamily = PVE::Tools::get_host_address_family($nodename);
my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
- my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
- $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
-
- push @$devices, '-spice', "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
+
+ my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
+ $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
+
+ my $spice_enhancement = PVE::JSONSchema::parse_property_string($spice_enhancements_fmt, $conf->{spice_enhancements} // '');
+ if ($spice_enhancement->{foldersharing}) {
+ push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
+ push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
+ }
+
+ my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
+ $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}" if $spice_enhancement->{videostreaming};
+ push @$devices, '-spice', "$spice_opts";
}
# enable balloon by default, unless explicitly disabled
$bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
- while (my ($k, $v) = each %$bridges) {
+ for my $k (sort {$b cmp $a} keys %$bridges) {
$pciaddr = print_pci_addr("pci.$k", undef, $arch, $machine_type);
unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
}
if (my $vmstate = $conf->{vmstate}) {
my $statepath = PVE::Storage::path($storecfg, $vmstate);
- PVE::Storage::activate_volumes($storecfg, [$vmstate]);
+ push @$vollist, $vmstate;
push @$cmd, '-loadstate', $statepath;
}
}
sub qmp_socket {
- my ($vmid, $qga, $name) = @_;
+ my ($vmid, $qga) = @_;
my $sockettype = $qga ? 'qga' : 'qmp';
- my $ext = $name ? '-'.$name : '';
- return "${var_run_tmpdir}/$vmid$ext.$sockettype";
+ return "${var_run_tmpdir}/$vmid.$sockettype";
}
sub pidfile_name {
my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
- my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
- while (my ($opt, $force) = each %$pending_delete_hash) {
+ my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
+ foreach my $opt (sort keys %$pending_delete_hash) {
next if $selection && !$selection->{$opt};
+ my $force = $pending_delete_hash->{$opt}->{force};
eval {
if ($opt eq 'hotplug') {
die "skip\n" if ($conf->{hotplug} =~ /memory/);
} else {
# save new config if hotplug was successful
delete $conf->{$opt};
- vmconfig_undelete_pending_option($conf, $opt);
+ PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
PVE::QemuConfig->write_config($vmid, $conf);
$conf = PVE::QemuConfig->load_config($vmid); # update/reload
}
}
}
+
+
sub vmconfig_apply_pending {
my ($vmid, $conf, $storecfg) = @_;
# cold plug
- my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
- while (my ($opt, $force) = each %$pending_delete_hash) {
+ my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
+ foreach my $opt (sort keys %$pending_delete_hash) {
die "internal error" if $opt =~ m/^unused/;
+ my $force = $pending_delete_hash->{$opt}->{force};
$conf = PVE::QemuConfig->load_config($vmid); # update/reload
if (!defined($conf->{$opt})) {
- vmconfig_undelete_pending_option($conf, $opt);
+ PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
PVE::QemuConfig->write_config($vmid, $conf);
} elsif (is_valid_drivename($opt)) {
vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
- vmconfig_undelete_pending_option($conf, $opt);
+ PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
delete $conf->{$opt};
PVE::QemuConfig->write_config($vmid, $conf);
} else {
- vmconfig_undelete_pending_option($conf, $opt);
+ PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
delete $conf->{$opt};
PVE::QemuConfig->write_config($vmid, $conf);
}
die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
+ # clean up leftover reboot request files
+ eval { clear_reboot_request($vmid); };
+ warn $@ if $@;
+
if (!$statefile && scalar(keys %{$conf->{pending}})) {
vmconfig_apply_pending($vmid, $conf, $storecfg);
$conf = PVE::QemuConfig->load_config($vmid); # update/reload
my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
- my $migrate_port = 0;
my $migrate_uri;
if ($statefile) {
if ($statefile eq 'tcp') {
}
my $pfamily = PVE::Tools::get_host_address_family($nodename);
- $migrate_port = PVE::Tools::next_migrate_port($pfamily);
+ my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
$migrate_uri = "tcp:${localip}:${migrate_port}";
push @$cmd, '-incoming', $migrate_uri;
push @$cmd, '-S';
push @$cmd, '-incoming', $migrate_uri;
push @$cmd, '-S';
- } else {
+ } elsif (-e $statefile) {
push @$cmd, '-loadstate', $statefile;
+ } else {
+ my $statepath = PVE::Storage::path($storecfg, $statefile);
+ push @$vollist, $statefile;
+ push @$cmd, '-loadstate', $statepath;
}
} elsif ($paused) {
push @$cmd, '-S';
my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
my $pfamily = PVE::Tools::get_host_address_family($nodename);
- $migrate_port = PVE::Tools::next_migrate_port($pfamily);
+ my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
- vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${migrate_port}" } } );
+ vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${storage_migrate_port}" } } );
$localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
foreach my $opt (sort keys %$local_volumes) {
my $volid = $local_volumes->{$opt};
vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
- my $migrate_storage_uri = "nbd:${localip}:${migrate_port}:exportname=drive-$opt";
+ my $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}:exportname=drive-$opt";
print "storage migration listens on $migrate_storage_uri volume:$volid\n";
}
}
warn $@ if $@; # avoid errors - just warn
}
-# Note: use $nockeck to skip tests if VM configuration file exists.
-# We need that when migration VMs to other nodes (files already moved)
-# Note: we set $keepActive in vzdump stop mode - volumes need to stay active
-sub vm_stop {
- my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
-
- $force = 1 if !defined($force) && !$shutdown;
-
- if ($migratedfrom){
- my $pid = check_running($vmid, $nocheck, $migratedfrom);
- kill 15, $pid if $pid;
- my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
- vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
- return;
- }
-
- PVE::QemuConfig->lock_config($vmid, sub {
+# call only in locked context
+sub _do_vm_stop {
+ my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
- my $pid = check_running($vmid, $nocheck);
- return if !$pid;
+ my $pid = check_running($vmid, $nocheck);
+ return if !$pid;
- my $conf;
- if (!$nocheck) {
- $conf = PVE::QemuConfig->load_config($vmid);
- PVE::QemuConfig->check_lock($conf) if !$skiplock;
- if (!defined($timeout) && $shutdown && $conf->{startup}) {
- my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
- $timeout = $opts->{down} if $opts->{down};
- }
- PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
+ my $conf;
+ if (!$nocheck) {
+ $conf = PVE::QemuConfig->load_config($vmid);
+ PVE::QemuConfig->check_lock($conf) if !$skiplock;
+ if (!defined($timeout) && $shutdown && $conf->{startup}) {
+ my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
+ $timeout = $opts->{down} if $opts->{down};
}
+ PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
+ }
- eval {
- if ($shutdown) {
- if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
- vm_qmp_command($vmid, {
+ eval {
+ if ($shutdown) {
+ if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
+ vm_qmp_command($vmid, {
execute => "guest-shutdown",
arguments => { timeout => $timeout }
}, $nocheck);
- } else {
- vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
- }
} else {
- vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
- }
- };
- my $err = $@;
-
- if (!$err) {
- $timeout = 60 if !defined($timeout);
-
- my $count = 0;
- while (($count < $timeout) && check_running($vmid, $nocheck)) {
- $count++;
- sleep 1;
- }
-
- if ($count >= $timeout) {
- if ($force) {
- warn "VM still running - terminating now with SIGTERM\n";
- kill 15, $pid;
- } else {
- die "VM quit/powerdown failed - got timeout\n";
- }
- } else {
- vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
- return;
+ vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
}
} else {
- if ($force) {
- warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
- kill 15, $pid;
- } else {
- die "VM quit/powerdown failed\n";
- }
+ vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
}
+ };
+ my $err = $@;
- # wait again
- $timeout = 10;
+ if (!$err) {
+ $timeout = 60 if !defined($timeout);
my $count = 0;
while (($count < $timeout) && check_running($vmid, $nocheck)) {
}
if ($count >= $timeout) {
- warn "VM still running - terminating now with SIGKILL\n";
- kill 9, $pid;
- sleep 1;
+ if ($force) {
+ warn "VM still running - terminating now with SIGTERM\n";
+ kill 15, $pid;
+ } else {
+ die "VM quit/powerdown failed - got timeout\n";
+ }
+ } else {
+ vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
+ return;
}
+ } else {
+ if ($force) {
+ warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
+ kill 15, $pid;
+ } else {
+ die "VM quit/powerdown failed\n";
+ }
+ }
+
+ # wait again
+ $timeout = 10;
+
+ my $count = 0;
+ while (($count < $timeout) && check_running($vmid, $nocheck)) {
+ $count++;
+ sleep 1;
+ }
+
+ if ($count >= $timeout) {
+ warn "VM still running - terminating now with SIGKILL\n";
+ kill 9, $pid;
+ sleep 1;
+ }
+
+ vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
+}
+
+# Note: use $nocheck to skip tests if VM configuration file exists.
+# We need that when migration VMs to other nodes (files already moved)
+# Note: we set $keepActive in vzdump stop mode - volumes need to stay active
+sub vm_stop {
+ my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
+
+ $force = 1 if !defined($force) && !$shutdown;
+
+ if ($migratedfrom){
+ my $pid = check_running($vmid, $nocheck, $migratedfrom);
+ kill 15, $pid if $pid;
+ my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
+ vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
+ return;
+ }
+
+ PVE::QemuConfig->lock_config($vmid, sub {
+ _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
+ });
+}
+
+sub vm_reboot {
+ my ($vmid, $timeout) = @_;
+
+ PVE::QemuConfig->lock_config($vmid, sub {
+
+ # only reboot if running, as qmeventd starts it again on a stop event
+ return if !check_running($vmid);
+
+ create_reboot_request($vmid);
+
+ my $storecfg = PVE::Storage::config();
+ _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
- vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
});
}
});
}
-sub vm_destroy {
- my ($storecfg, $vmid, $skiplock) = @_;
-
- PVE::QemuConfig->lock_config($vmid, sub {
-
- my $conf = PVE::QemuConfig->load_config($vmid);
-
- if (!check_running($vmid)) {
- destroy_vm($storecfg, $vmid, undef, $skiplock);
- } else {
- die "VM $vmid is running - destroy failed\n";
- }
- });
-}
-
# vzdump restore implementaion
sub tar_archive_read_firstfile {
foreach_drive($oldconf, sub {
my ($ds, $drive) = @_;
- return if !$drive->{is_cloudinit} && drive_is_cdrom($drive);
+ return if drive_is_cdrom($drive, 1);
my $volid = $drive->{file};
return if !$volid || $volid =~ m|^/|;
my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
- if ($src_storeid && $dst_storeid) {
+ die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
- PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
+ my $cachemode;
+ my $src_path;
+ my $src_is_iscsi = 0;
+ my $src_format = 'raw';
+ if ($src_storeid) {
+ PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
- my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
+ $src_format = qemu_img_format($src_scfg, $src_volname);
+ $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
+ $src_is_iscsi = ($src_path =~ m|^iscsi://|);
+ $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
+ } elsif (-f $src_volid) {
+ $src_path = $src_volid;
+ if ($src_path =~ m/\.($QEMU_FORMAT_RE)$/) {
+ $src_format = $1;
+ }
+ }
- my $src_format = qemu_img_format($src_scfg, $src_volname);
- my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
+ die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
- my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
- my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
+ my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
+ my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
+ my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
+ my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
- my $src_is_iscsi = ($src_path =~ m|^iscsi://|);
- my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
+ my $cmd = [];
+ push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
+ push @$cmd, '-l', "snapshot.name=$snapname" if($snapname && $src_format eq "qcow2");
+ push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
+ push @$cmd, '-T', $cachemode if defined($cachemode);
+
+ if ($src_is_iscsi) {
+ push @$cmd, '--image-opts';
+ $src_path = convert_iscsi_path($src_path);
+ } else {
+ push @$cmd, '-f', $src_format;
+ }
- my $cmd = [];
- push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
- push @$cmd, '-l', "snapshot.name=$snapname" if($snapname && $src_format eq "qcow2");
- push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
- push @$cmd, '-T', 'none' if $src_scfg->{type} eq 'zfspool';
+ if ($dst_is_iscsi) {
+ push @$cmd, '--target-image-opts';
+ $dst_path = convert_iscsi_path($dst_path);
+ } else {
+ push @$cmd, '-O', $dst_format;
+ }
- if ($src_is_iscsi) {
- push @$cmd, '--image-opts';
- $src_path = convert_iscsi_path($src_path);
- } else {
- push @$cmd, '-f', $src_format;
- }
+ push @$cmd, $src_path;
- if ($dst_is_iscsi) {
- push @$cmd, '--target-image-opts';
- $dst_path = convert_iscsi_path($dst_path);
- } else {
- push @$cmd, '-O', $dst_format;
- }
+ if (!$dst_is_iscsi && $is_zero_initialized) {
+ push @$cmd, "zeroinit:$dst_path";
+ } else {
+ push @$cmd, $dst_path;
+ }
- push @$cmd, $src_path;
+ my $parser = sub {
+ my $line = shift;
+ if($line =~ m/\((\S+)\/100\%\)/){
+ my $percent = $1;
+ my $transferred = int($size * $percent / 100);
+ my $remaining = $size - $transferred;
- if (!$dst_is_iscsi && $is_zero_initialized) {
- push @$cmd, "zeroinit:$dst_path";
- } else {
- push @$cmd, $dst_path;
+ print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
}
- my $parser = sub {
- my $line = shift;
- if($line =~ m/\((\S+)\/100\%\)/){
- my $percent = $1;
- my $transferred = int($size * $percent / 100);
- my $remaining = $size - $transferred;
-
- print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
- }
-
- };
+ };
- eval { run_command($cmd, timeout => undef, outfunc => $parser); };
- my $err = $@;
- die "copy failed: $err" if $err;
- }
+ eval { run_command($cmd, timeout => undef, outfunc => $parser); };
+ my $err = $@;
+ die "copy failed: $err" if $err;
}
sub qemu_img_format {
print "create full clone of drive $drivename ($drive->{file})\n";
my $name = undef;
- if (drive_is_cloudinit($drive)) {
- $name = "vm-$newvmid-cloudinit";
- $snapname = undef;
- # we only get here if it's supported by QEMU_FORMAT_RE, so just accept
- if ($dst_format ne 'raw') {
- $name .= ".$dst_format";
- }
- }
$newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
push @$newvollist, $newvolid;
$current_minor = $2;
}
- return 1 if $current_major > $version_major ||
- ($current_major == $version_major &&
- $current_minor >= $version_minor);
+ return 1 if version_cmp($current_major, $version_major, $current_minor, $version_minor) >= 0;
+}
+
+# gets in pairs the versions you want to compares, i.e.:
+# ($a-major, $b-major, $a-minor, $b-minor, $a-extra, $b-extra, ...)
+# returns 0 if same, -1 if $a is older than $b, +1 if $a is newer than $b
+sub version_cmp {
+ my @versions = @_;
+
+ my $size = scalar(@versions);
+
+ return 0 if $size == 0;
+ die "cannot compare odd count of versions" if $size & 1;
+
+ for (my $i = 0; $i < $size; $i += 2) {
+ my ($a, $b) = splice(@versions, 0, 2);
+ $a //= 0;
+ $b //= 0;
+
+ return 1 if $a > $b;
+ return -1 if $a < $b;
+ }
+ return 0;
+}
+
+# dies if a) VM not running or not exisiting b) Version query failed
+# So, any defined return value is valid, any invalid state can be caught by eval
+sub runs_at_least_qemu_version {
+ my ($vmid, $major, $minor, $extra) = @_;
+
+ my $v = vm_qmp_command($vmid, { execute => 'query-version' });
+ die "could not query currently running version for VM $vmid\n" if !defined($v);
+ $v = $v->{qemu};
+
+ return version_cmp($v->{major}, $major, $v->{minor}, $minor, $v->{micro}, $extra) >= 0;
}
sub qemu_machine_pxe {
- my ($vmid, $conf, $machine) = @_;
+ my ($vmid, $conf) = @_;
- $machine = PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
+ my $machine = PVE::QemuServer::get_current_qemu_machine($vmid);
if ($conf->{machine} && $conf->{machine} =~ m/\.pxe$/) {
$machine .= '.pxe';
my (undef, $ovmf_vars) = get_ovmf_files($arch);
die "EFI vars default image not found\n" if ! -f $ovmf_vars;
- my $vars_size = PVE::Tools::convert_size(-s $ovmf_vars, 'b' => 'kb');
+ my $vars_size_b = -s $ovmf_vars;
+ my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
PVE::Storage::activate_volumes($storecfg, [$volid]);
- my $path = PVE::Storage::path($storecfg, $volid);
- eval {
- run_command(['/usr/bin/qemu-img', 'convert', '-n', '-f', 'raw', '-O', $fmt, $ovmf_vars, $path]);
- };
- die "Copying EFI vars image failed: $@" if $@;
+ qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
return ($volid, $vars_size);
}
vm_mon_cmd($vmid, 'nbd-server-stop');
}
+sub create_reboot_request {
+ my ($vmid) = @_;
+ open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
+ or die "failed to create reboot trigger file: $!\n";
+ close($fh);
+}
+
+sub clear_reboot_request {
+ my ($vmid) = @_;
+ my $path = "/run/qemu-server/$vmid.reboot";
+ my $res = 0;
+
+ $res = unlink($path);
+ die "could not remove reboot request for $vmid: $!"
+ if !$res && $! != POSIX::ENOENT;
+
+ return $res;
+}
+
# bash completion helper
sub complete_backup_archives {