use strict;
use warnings;
+
use POSIX;
use IO::Handle;
use IO::Select;
use Fcntl;
use PVE::SafeSyslog;
use Storable qw(dclone);
+use MIME::Base64;
use PVE::Exception qw(raise raise_param_exc);
use PVE::Storage;
use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach $IPV6RE);
use PVE::QMPClient;
use PVE::RPCEnvironment;
use PVE::GuestHelpers;
-use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
+use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port);
use PVE::QemuServer::Memory;
use PVE::QemuServer::USB qw(parse_usb_device);
use PVE::QemuServer::Cloudinit;
],
};
-my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
+my $qemu_snap_storage = { rbd => 1 };
my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
optional => 1,
});
-PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
- description => "The name of the snapshot.",
- type => 'string', format => 'pve-configid',
- maxLength => 40,
-});
-
PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
type => 'string',
enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
max => 'default',
};
-my $cpu_flag = qr/[+-](pcid|spec-ctrl|ibpb|ssbd|virt-ssbd|amd-ssbd|amd-no-ssb|pdpe1gb)/;
+my @supported_cpu_flags = (
+ 'pcid',
+ 'spec-ctrl',
+ 'ibpb',
+ 'ssbd',
+ 'virt-ssbd',
+ 'amd-ssbd',
+ 'amd-no-ssb',
+ 'pdpe1gb',
+ 'md-clear',
+ 'hv-tlbflush',
+ 'hv-evmcs',
+ 'aes'
+);
+my $cpu_flag = qr/[+-](@{[join('|', @supported_cpu_flags)]})/;
my $cpu_fmt = {
cputype => {
flags => {
description => "List of additional CPU flags separated by ';'."
. " Use '+FLAG' to enable, '-FLAG' to disable a flag."
- . " Currently supported flags: 'pcid', 'spec-ctrl', 'ibpb', 'ssbd', 'virt-ssbd', 'amd-ssbd', 'amd-no-ssb', 'pdpe1gb'.",
+ . " Currently supported flags: @{[join(', ', @supported_cpu_flags)]}.",
format_description => '+FLAG[;-FLAG...]',
type => 'string',
pattern => qr/$cpu_flag(;$cpu_flag)*/,
},
};
+my $audio_fmt = {
+ device => {
+ type => 'string',
+ enum => [qw(ich9-intel-hda intel-hda AC97)],
+ description => "Configure an audio device."
+ },
+ driver => {
+ type => 'string',
+ enum => ['spice'],
+ default => 'spice',
+ optional => 1,
+ description => "Driver backend for the audio device."
+ },
+};
+
+my $spice_enhancements_fmt = {
+ foldersharing => {
+ type => 'boolean',
+ optional => 1,
+ default => '0',
+ description => "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
+ },
+ videostreaming => {
+ type => 'string',
+ enum => ['off', 'all', 'filter'],
+ default => 'off',
+ optional => 1,
+ description => "Enable video streaming. Uses compression for detected video streams."
+ },
+};
+
my $confdesc = {
onboot => {
optional => 1,
optional => 1,
type => 'string',
description => "Lock/unlock the VM.",
- enum => [qw(backup clone create migrate rollback snapshot snapshot-delete)],
+ enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
},
cpulimit => {
optional => 1,
smbios1 => {
description => "Specify SMBIOS type 1 fields.",
type => 'string', format => 'pve-qm-smbios1',
- maxLength => 256,
+ maxLength => 512,
optional => 1,
},
protection => {
format => $ivshmem_fmt,
description => "Inter-VM shared memory. Useful for direct communication between VMs, or to the host.",
optional => 1,
- }
+ },
+ audio0 => {
+ type => 'string',
+ format => $audio_fmt,
+ description => "Configure a audio device, useful in combination with QXL/Spice.",
+ optional => 1
+ },
+ spice_enhancements => {
+ type => 'string',
+ format => $spice_enhancements_fmt,
+ description => "Configure additional enhancements for SPICE.",
+ optional => 1
+ },
};
my $cicustom_fmt = {
my $MAX_USB_DEVICES = 5;
my $MAX_NETS = 32;
my $MAX_UNUSED_DISKS = 256;
-my $MAX_HOSTPCI_DEVICES = 4;
+my $MAX_HOSTPCI_DEVICES = 16;
my $MAX_SERIAL_PORTS = 4;
my $MAX_PARALLEL_PORTS = 3;
my $MAX_NUMA = 8;
__EOD__
my $net_fmt = {
- macaddr => {
- type => 'string',
- pattern => qr/[0-9a-f]{2}(?::[0-9a-f]{2}){5}/i,
+ macaddr => get_standard_option('mac-addr', {
description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
- format_description => "XX:XX:XX:XX:XX:XX",
- optional => 1,
- },
+ }),
model => {
type => 'string',
description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
usb3 => {
optional => 1,
type => 'boolean',
- description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
+ description => "Specifies whether if given host option is a USB3 device or port.",
default => 0,
},
};
pattern => qr/$PCIRE(;$PCIRE)*/,
format_description => 'HOSTPCIID[;HOSTPCIID2...]',
description => <<EODESCR,
-Host PCI device pass through. The PCI ID of a host's PCI device or a list
+Host PCI device pass through. The PCI ID of a host's PCI device or a list
of PCI virtual functions of the host. HOSTPCIID syntax is:
'bus:dev.func' (hexadecimal numbers)
verbose_description => <<EODESCR,
Map host PCI devices into guest.
-NOTE: This option allows direct access to host hardware. So it is no longer
+NOTE: This option allows direct access to host hardware. So it is no longer
possible to migrate such machines - use with special care.
CAUTION: Experimental! User reported problems with this option.
return $kvm_api_version;
}
-my $kvm_user_version;
+my $kvm_user_version = {};
+my $kvm_mtime = {};
sub kvm_user_version {
+ my ($binary) = @_;
- return $kvm_user_version if $kvm_user_version;
+ $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
+ my $st = stat($binary);
- $kvm_user_version = 'unknown';
+ my $cachedmtime = $kvm_mtime->{$binary} // -1;
+ return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
+ $cachedmtime == $st->mtime;
+
+ $kvm_user_version->{$binary} = 'unknown';
+ $kvm_mtime->{$binary} = $st->mtime;
my $code = sub {
my $line = shift;
if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
- $kvm_user_version = $2;
+ $kvm_user_version->{$binary} = $2;
}
};
- eval { run_command("kvm -version", outfunc => $code); };
+ eval { run_command([$binary, '--version'], outfunc => $code); };
warn $@ if $@;
- return $kvm_user_version;
+ return $kvm_user_version->{$binary};
}
my $path;
my $volid = $drive->{file};
my $format;
-
+
if (drive_is_cdrom($drive)) {
$path = get_iso_path($storecfg, $vmid, $volid);
} else {
return $changes;
}
-# smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
+# smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
my $smbios1_fmt = {
uuid => {
type => 'string',
},
version => {
type => 'string',
- pattern => '\S+',
- format_description => 'string',
+ pattern => '[A-Za-z0-9+\/]+={0,2}',
+ format_description => 'Base64 encoded string',
description => "Set SMBIOS1 version.",
optional => 1,
},
serial => {
type => 'string',
- pattern => '\S+',
- format_description => 'string',
+ pattern => '[A-Za-z0-9+\/]+={0,2}',
+ format_description => 'Base64 encoded string',
description => "Set SMBIOS1 serial number.",
optional => 1,
},
manufacturer => {
type => 'string',
- pattern => '\S+',
- format_description => 'string',
+ pattern => '[A-Za-z0-9+\/]+={0,2}',
+ format_description => 'Base64 encoded string',
description => "Set SMBIOS1 manufacturer.",
optional => 1,
},
product => {
type => 'string',
- pattern => '\S+',
- format_description => 'string',
+ pattern => '[A-Za-z0-9+\/]+={0,2}',
+ format_description => 'Base64 encoded string',
description => "Set SMBIOS1 product ID.",
optional => 1,
},
sku => {
type => 'string',
- pattern => '\S+',
- format_description => 'string',
+ pattern => '[A-Za-z0-9+\/]+={0,2}',
+ format_description => 'Base64 encoded string',
description => "Set SMBIOS1 SKU string.",
optional => 1,
},
family => {
type => 'string',
- pattern => '\S+',
- format_description => 'string',
+ pattern => '[A-Za-z0-9+\/]+={0,2}',
+ format_description => 'Base64 encoded string',
description => "Set SMBIOS1 family string.",
optional => 1,
},
+ base64 => {
+ type => 'boolean',
+ description => 'Flag to indicate that the SMBIOS values are base64 encoded',
+ optional => 1,
+ },
};
sub parse_smbios1 {
sub check_local_resources {
my ($conf, $noerr) = @_;
- my $loc_res = 0;
+ my @loc_res = ();
- $loc_res = 1 if $conf->{hostusb}; # old syntax
- $loc_res = 1 if $conf->{hostpci}; # old syntax
+ push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
+ push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
- $loc_res = 1 if $conf->{ivshmem};
+ push @loc_res, "ivshmem" if $conf->{ivshmem};
foreach my $k (keys %$conf) {
- next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
+ next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
# sockets are safe: they will recreated be on the target side post-migrate
next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
- $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
+ push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
}
- die "VM uses local resources\n" if $loc_res && !$noerr;
+ die "VM uses local resources\n" if scalar @loc_res && !$noerr;
- return $loc_res;
+ return \@loc_res;
}
# check if used storages are available on all nodes (use by migrate)
return $nodehash
}
+sub check_local_storage_availability {
+ my ($conf, $storecfg) = @_;
+
+ my $nodelist = PVE::Cluster::get_nodelist();
+ my $nodehash = { map { $_ => {} } @$nodelist };
+
+ foreach_drive($conf, sub {
+ my ($ds, $drive) = @_;
+
+ my $volid = $drive->{file};
+ return if !$volid;
+
+ my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
+ if ($storeid) {
+ my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
+
+ if ($scfg->{disable}) {
+ foreach my $node (keys %$nodehash) {
+ $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
+ }
+ } elsif (my $avail = $scfg->{nodes}) {
+ foreach my $node (keys %$nodehash) {
+ if (!$avail->{$node}) {
+ $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
+ }
+ }
+ }
+ }
+ });
+
+ foreach my $node (values %$nodehash) {
+ if (my $unavail = $node->{unavailable_storages}) {
+ $node->{unavailable_storages} = [ sort keys %$unavail ];
+ }
+ }
+
+ return $nodehash
+}
+
sub check_cmdline {
my ($pidfile, $pid) = @_;
type => 'number',
optional => 1,
},
+ lock => {
+ description => "The current config lock, if any.",
+ type => 'string',
+ optional => 1,
+ }
};
my $last_proc_pid_stat;
$d->{template} = PVE::QemuConfig->is_template($conf);
$d->{serial} = 1 if conf_has_serial($conf);
+ $d->{lock} = $conf->{lock} if $conf->{lock};
$res->{$vmid} = $d;
}
my $volhash = {};
my $test_volid = sub {
- my ($volid, $is_cdrom, $replicate, $shared, $snapname) = @_;
+ my ($volid, $is_cdrom, $replicate, $shared, $snapname, $size) = @_;
return if !$volid;
$volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
if defined($snapname);
+ $volhash->{$volid}->{size} = $size if $size;
};
foreach_drive($conf, sub {
my ($ds, $drive) = @_;
- $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, undef);
+ $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, undef, $drive->{size});
});
foreach my $snapname (keys %{$conf->{snapshots}}) {
return 0;
}
+sub conf_has_audio {
+ my ($conf, $id) = @_;
+
+ $id //= 0;
+ my $audio = $conf->{"audio$id"};
+ return undef if !defined($audio);
+
+ my $audioproperties = PVE::JSONSchema::parse_property_string($audio_fmt, $audio);
+ my $audiodriver = $audioproperties->{driver} // 'spice';
+
+ return {
+ dev => $audioproperties->{device},
+ dev_id => "audiodev$id",
+ backend => $audiodriver,
+ backend_id => "$audiodriver-backend${id}",
+ };
+}
+
sub vga_conf_has_spice {
my ($vga) = @_;
my $devices = [];
my $pciaddr = '';
my $bridges = {};
- my $kvmver = kvm_user_version();
my $vernum = 0; # unknown
my $ostype = $conf->{ostype};
my $winversion = windows_version($ostype);
my $kvm = $conf->{kvm};
my ($arch, $machine_type) = get_basic_machine_info($conf, $forcemachine);
+ my $kvm_binary = get_command_for_arch($arch);
+ my $kvmver = kvm_user_version($kvm_binary);
$kvm //= 1 if is_native($arch);
if ($kvm) {
my $cpuunits = defined($conf->{cpuunits}) ?
$conf->{cpuunits} : $defaults->{cpuunits};
- push @$cmd, get_command_for_arch($arch);
+ push @$cmd, $kvm_binary;
push @$cmd, '-id', $vmid;
push @$cmd, '-daemonize';
if ($conf->{smbios1}) {
- push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
+ my $smbios_conf = parse_smbios1($conf->{smbios1});
+ if ($smbios_conf->{base64}) {
+ # Do not pass base64 flag to qemu
+ delete $smbios_conf->{base64};
+ my $smbios_string = "";
+ foreach my $key (keys %$smbios_conf) {
+ my $value;
+ if ($key eq "uuid") {
+ $value = $smbios_conf->{uuid}
+ } else {
+ $value = decode_base64($smbios_conf->{$key});
+ }
+ # qemu accepts any binary data, only commas need escaping by double comma
+ $value =~ s/,/,,/g;
+ $smbios_string .= "," . $key . "=" . $value if $value;
+ }
+ push @$cmd, '-smbios', "type=1" . $smbios_string;
+ } else {
+ push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
+ }
}
if ($conf->{vmgenid}) {
push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
}
+ # load q35 config
+ if ($q35) {
+ # we use different pcie-port hardware for qemu >= 4.0 for passthrough
+ if (qemu_machine_feature_enabled($machine_type, $kvmver, 4, 0)) {
+ push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
+ } else {
+ push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
+ }
+ }
# add usb controllers
my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
# host pci devices
for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++) {
- my $d = parse_hostpci($conf->{"hostpci$i"});
+ my $id = "hostpci$i";
+ my $d = parse_hostpci($conf->{$id});
next if !$d;
- my $pcie = $d->{pcie};
- if ($pcie) {
+ if (my $pcie = $d->{pcie}) {
die "q35 machine model is not enabled" if !$q35;
# win7 wants to have the pcie devices directly on the pcie bus
# instead of in the root port
if ($winversion == 7) {
- $pciaddr = print_pcie_addr("hostpci${i}bus0");
+ $pciaddr = print_pcie_addr("${id}bus0");
} else {
- $pciaddr = print_pcie_addr("hostpci$i");
+ # add more root ports if needed, 4 are present by default
+ # by pve-q35 cfgs, rest added here on demand.
+ if ($i > 3) {
+ push @$devices, '-device', print_pcie_root_port($i);
+ }
+ $pciaddr = print_pcie_addr($id);
}
} else {
- $pciaddr = print_pci_addr("hostpci$i", $bridges, $arch, $machine_type);
+ $pciaddr = print_pci_addr($id, $bridges, $arch, $machine_type);
}
- my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
- my $romfile = $d->{romfile};
-
my $xvga = '';
if ($d->{'x-vga'}) {
- $xvga = ',x-vga=on';
+ $xvga = ',x-vga=on' if !($conf->{bios} && $conf->{bios} eq 'ovmf');
$kvm_off = 1;
$vga->{type} = 'none' if !defined($conf->{vga});
$gpu_passthrough = 1;
-
- if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
- $xvga = "";
- }
}
+
my $pcidevices = $d->{pciid};
my $multifunction = 1 if @$pcidevices > 1;
+
my $sysfspath;
if ($d->{mdev} && scalar(@$pcidevices) == 1) {
- my $id = $pcidevices->[0]->{id};
+ my $pci_id = $pcidevices->[0]->{id};
my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
- $sysfspath = "/sys/bus/pci/devices/0000:$id/$uuid";
+ $sysfspath = "/sys/bus/pci/devices/0000:$pci_id/$uuid";
} elsif ($d->{mdev}) {
- warn "ignoring mediated device with multifunction device\n";
+ warn "ignoring mediated device '$id' with multifunction device\n";
}
my $j=0;
- foreach my $pcidevice (@$pcidevices) {
-
- my $id = "hostpci$i";
- $id .= ".$j" if $multifunction;
- my $addr = $pciaddr;
- $addr .= ".$j" if $multifunction;
+ foreach my $pcidevice (@$pcidevices) {
my $devicestr = "vfio-pci";
+
if ($sysfspath) {
$devicestr .= ",sysfsdev=$sysfspath";
} else {
$devicestr .= ",host=$pcidevice->{id}";
}
- $devicestr .= ",id=$id$addr";
- if($j == 0){
- $devicestr .= "$rombar$xvga";
+ my $mf_addr = $multifunction ? ".$j" : '';
+ $devicestr .= ",id=${id}${mf_addr}${pciaddr}${mf_addr}";
+
+ if ($j == 0) {
+ $devicestr .= ',rombar=0' if defined($d->{rombar}) && !$d->{rombar};
+ $devicestr .= "$xvga";
$devicestr .= ",multifunction=on" if $multifunction;
- $devicestr .= ",romfile=/usr/share/kvm/$romfile" if $romfile;
+ $devicestr .= ",romfile=/usr/share/kvm/$d->{romfile}" if $d->{romfile};
}
push @$devices, '-device', $devicestr;
}
# usb devices
- my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES);
+ my $usb_dev_features = {};
+ $usb_dev_features->{spice_usb3} = 1 if qemu_machine_feature_enabled($machine_type, $kvmver, 4, 1);
+
+ my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features);
push @$devices, @usbdevices if @usbdevices;
# serial devices
for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
}
}
+ if (my $audio = conf_has_audio($conf)) {
+
+ my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
+
+ my $id = $audio->{dev_id};
+ if ($audio->{dev} eq 'AC97') {
+ push @$devices, '-device', "AC97,id=${id}${audiopciaddr}";
+ } elsif ($audio->{dev} =~ /intel\-hda$/) {
+ push @$devices, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
+ push @$devices, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0";
+ push @$devices, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1";
+ } else {
+ die "unkown audio device '$audio->{dev}', implement me!";
+ }
+
+ push @$devices, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
+ }
my $sockets = 1;
$sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_type, undef, $qxlnum, $bridges);
my $socket = vnc_socket($vmid);
- push @$cmd, '-vnc', "unix:$socket,x509,password";
+ push @$cmd, '-vnc', "unix:$socket,password";
} else {
push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
push @$cmd, '-nographic';
push @$machineFlags, "type=${machine_type}";
}
- if ($conf->{startdate}) {
+ if (($conf->{startdate}) && ($conf->{startdate} ne 'now')) {
push @$rtcFlags, "base=$conf->{startdate}";
} elsif ($useLocaltime) {
push @$rtcFlags, 'base=localtime';
push @$cmd, get_cpu_options($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough);
PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
-
+
push @$cmd, '-S' if $conf->{freeze};
push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
- # enable sound
- #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
- #push @$cmd, '-soundhw', 'es1370';
- #push @$cmd, '-soundhw', $soundhw if $soundhw;
-
if (parse_guest_agent($conf)->{enabled}) {
my $qgasocket = qmp_socket($vmid, 1);
my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
$spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
- push @$devices, '-spice', "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
+ my $spice_enhancement = PVE::JSONSchema::parse_property_string($spice_enhancements_fmt, $conf->{spice_enhancements} // '');
+ if ($spice_enhancement->{foldersharing}) {
+ push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
+ push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
+ }
+
+ my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
+ $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}" if $spice_enhancement->{videostreaming};
+ push @$devices, '-spice', "$spice_opts";
push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
+
}
# enable balloon by default, unless explicitly disabled
my $queues = '';
if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
$queues = ",num_queues=$drive->{queues}";
- }
+ }
push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
$scsicontroller->{$controller}=1;
push @$cmd, '-global', join(',', @$globalFlags)
if scalar(@$globalFlags);
+ if (my $vmstate = $conf->{vmstate}) {
+ my $statepath = PVE::Storage::path($storecfg, $vmstate);
+ PVE::Storage::activate_volumes($storecfg, [$vmstate]);
+ push @$cmd, '-loadstate', $statepath;
+ }
+
# add custom args
if ($conf->{args}) {
my $aa = PVE::Tools::split_args($conf->{args});
die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
- PVE::QemuConfig->check_lock($conf) if !$skiplock;
+ my $is_suspended = PVE::QemuConfig->has_lock($conf, 'suspended');
+
+ PVE::QemuConfig->check_lock($conf)
+ if !($skiplock || $is_suspended);
die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
+ # clean up leftover reboot request files
+ eval { clear_reboot_request($vmid); };
+ warn $@ if $@;
+
if (!$statefile && scalar(keys %{$conf->{pending}})) {
vmconfig_apply_pending($vmid, $conf, $storecfg);
$conf = PVE::QemuConfig->load_config($vmid); # update/reload
PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
+ if ($is_suspended) {
+ # enforce machine type on suspended vm to ensure HW compatibility
+ $forcemachine = $conf->{runningmachine};
+ print "Resuming suspended VM\n";
+ }
+
my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
my $migrate_port = 0;
PVE::Storage::activate_volumes($storecfg, $vollist);
- if (-d "/sys/fs/cgroup/systemd/qemu.slice/$vmid.scope") {
- eval {
- run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
- outfunc => sub {}, errfunc => sub {});
- };
- }
+ eval {
+ run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
+ outfunc => sub {}, errfunc => sub {});
+ };
+ # Issues with the above 'stop' not being fully completed are extremely rare, a very low
+ # timeout should be more than enough here...
+ PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
: $defaults->{cpuunits};
- my $start_timeout = $conf->{hugepages} ? 300 : 30;
+ my $start_timeout = ($conf->{hugepages} || $is_suspended) ? 300 : 30;
my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077);
my %properties = (
property => "guest-stats-polling-interval",
value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
+ if ($is_suspended && (my $vmstate = $conf->{vmstate})) {
+ print "Resumed VM, removing state\n";
+ delete $conf->@{qw(lock vmstate runningmachine)};
+ PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
+ PVE::Storage::vdisk_free($storecfg, $vmstate);
+ PVE::QemuConfig->write_config($vmid, $conf);
+ }
+
PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
});
}
my $res;
my $timeout;
- if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
- $timeout = $cmd->{arguments}->{timeout};
- delete $cmd->{arguments}->{timeout};
+ if ($cmd->{arguments}) {
+ $timeout = delete $cmd->{arguments}->{timeout};
}
eval {
sub vm_human_monitor_command {
my ($vmid, $cmdline) = @_;
- my $res;
-
my $cmd = {
execute => 'human-monitor-command',
arguments => { 'command-line' => $cmdline},
warn $@ if $@; # avoid errors - just warn
}
-# Note: use $nockeck to skip tests if VM configuration file exists.
+# call only in locked context
+sub _do_vm_stop {
+ my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
+
+ my $pid = check_running($vmid, $nocheck);
+ return if !$pid;
+
+ my $conf;
+ if (!$nocheck) {
+ $conf = PVE::QemuConfig->load_config($vmid);
+ PVE::QemuConfig->check_lock($conf) if !$skiplock;
+ if (!defined($timeout) && $shutdown && $conf->{startup}) {
+ my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
+ $timeout = $opts->{down} if $opts->{down};
+ }
+ PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
+ }
+
+ eval {
+ if ($shutdown) {
+ if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
+ vm_qmp_command($vmid, {
+ execute => "guest-shutdown",
+ arguments => { timeout => $timeout }
+ }, $nocheck);
+ } else {
+ vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
+ }
+ } else {
+ vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
+ }
+ };
+ my $err = $@;
+
+ if (!$err) {
+ $timeout = 60 if !defined($timeout);
+
+ my $count = 0;
+ while (($count < $timeout) && check_running($vmid, $nocheck)) {
+ $count++;
+ sleep 1;
+ }
+
+ if ($count >= $timeout) {
+ if ($force) {
+ warn "VM still running - terminating now with SIGTERM\n";
+ kill 15, $pid;
+ } else {
+ die "VM quit/powerdown failed - got timeout\n";
+ }
+ } else {
+ vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
+ return;
+ }
+ } else {
+ if ($force) {
+ warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
+ kill 15, $pid;
+ } else {
+ die "VM quit/powerdown failed\n";
+ }
+ }
+
+ # wait again
+ $timeout = 10;
+
+ my $count = 0;
+ while (($count < $timeout) && check_running($vmid, $nocheck)) {
+ $count++;
+ sleep 1;
+ }
+
+ if ($count >= $timeout) {
+ warn "VM still running - terminating now with SIGKILL\n";
+ kill 9, $pid;
+ sleep 1;
+ }
+
+ vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
+}
+
+# Note: use $nocheck to skip tests if VM configuration file exists.
# We need that when migration VMs to other nodes (files already moved)
# Note: we set $keepActive in vzdump stop mode - volumes need to stay active
sub vm_stop {
}
PVE::QemuConfig->lock_config($vmid, sub {
+ _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
+ });
+}
- my $pid = check_running($vmid, $nocheck);
- return if !$pid;
-
- my $conf;
- if (!$nocheck) {
- $conf = PVE::QemuConfig->load_config($vmid);
- PVE::QemuConfig->check_lock($conf) if !$skiplock;
- if (!defined($timeout) && $shutdown && $conf->{startup}) {
- my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
- $timeout = $opts->{down} if $opts->{down};
- }
- PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
- }
-
- $timeout = 60 if !defined($timeout);
-
- eval {
- if ($shutdown) {
- if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
- vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
- } else {
- vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
- }
- } else {
- vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
- }
- };
- my $err = $@;
-
- if (!$err) {
- my $count = 0;
- while (($count < $timeout) && check_running($vmid, $nocheck)) {
- $count++;
- sleep 1;
- }
+sub vm_reboot {
+ my ($vmid, $timeout) = @_;
- if ($count >= $timeout) {
- if ($force) {
- warn "VM still running - terminating now with SIGTERM\n";
- kill 15, $pid;
- } else {
- die "VM quit/powerdown failed - got timeout\n";
- }
- } else {
- vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
- return;
- }
- } else {
- if ($force) {
- warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
- kill 15, $pid;
- } else {
- die "VM quit/powerdown failed\n";
- }
- }
+ PVE::QemuConfig->lock_config($vmid, sub {
- # wait again
- $timeout = 10;
+ # only reboot if running, as qmeventd starts it again on a stop event
+ return if !check_running($vmid);
- my $count = 0;
- while (($count < $timeout) && check_running($vmid, $nocheck)) {
- $count++;
- sleep 1;
- }
+ create_reboot_request($vmid);
- if ($count >= $timeout) {
- warn "VM still running - terminating now with SIGKILL\n";
- kill 9, $pid;
- sleep 1;
- }
+ my $storecfg = PVE::Storage::config();
+ _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
- vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
});
}
sub vm_suspend {
- my ($vmid, $skiplock) = @_;
+ my ($vmid, $skiplock, $includestate, $statestorage) = @_;
+
+ my $conf;
+ my $path;
+ my $storecfg;
+ my $vmstate;
PVE::QemuConfig->lock_config($vmid, sub {
- my $conf = PVE::QemuConfig->load_config($vmid);
+ $conf = PVE::QemuConfig->load_config($vmid);
+ my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
PVE::QemuConfig->check_lock($conf)
- if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
+ if !($skiplock || $is_backing_up);
- vm_mon_cmd($vmid, "stop");
+ die "cannot suspend to disk during backup\n"
+ if $is_backing_up && $includestate;
+
+ if ($includestate) {
+ $conf->{lock} = 'suspending';
+ my $date = strftime("%Y-%m-%d", localtime(time()));
+ $storecfg = PVE::Storage::config();
+ $vmstate = PVE::QemuConfig->__snapshot_save_vmstate($vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
+ $path = PVE::Storage::path($storecfg, $vmstate);
+ PVE::QemuConfig->write_config($vmid, $conf);
+ } else {
+ vm_mon_cmd($vmid, "stop");
+ }
});
+
+ if ($includestate) {
+ # save vm state
+ PVE::Storage::activate_volumes($storecfg, [$vmstate]);
+
+ eval {
+ vm_mon_cmd($vmid, "savevm-start", statefile => $path);
+ for(;;) {
+ my $state = vm_mon_cmd_nocheck($vmid, "query-savevm");
+ if (!$state->{status}) {
+ die "savevm not active\n";
+ } elsif ($state->{status} eq 'active') {
+ sleep(1);
+ next;
+ } elsif ($state->{status} eq 'completed') {
+ print "State saved, quitting\n";
+ last;
+ } elsif ($state->{status} eq 'failed' && $state->{error}) {
+ die "query-savevm failed with error '$state->{error}'\n"
+ } else {
+ die "query-savevm returned status '$state->{status}'\n";
+ }
+ }
+ };
+ my $err = $@;
+
+ PVE::QemuConfig->lock_config($vmid, sub {
+ $conf = PVE::QemuConfig->load_config($vmid);
+ if ($err) {
+ # cleanup, but leave suspending lock, to indicate something went wrong
+ eval {
+ vm_mon_cmd($vmid, "savevm-end");
+ PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
+ PVE::Storage::vdisk_free($storecfg, $vmstate);
+ delete $conf->@{qw(vmstate runningmachine)};
+ PVE::QemuConfig->write_config($vmid, $conf);
+ };
+ warn $@ if $@;
+ die $err;
+ }
+
+ die "lock changed unexpectedly\n"
+ if !PVE::QemuConfig->has_lock($conf, 'suspending');
+
+ vm_qmp_command($vmid, { execute => "quit" });
+ $conf->{lock} = 'suspended';
+ PVE::QemuConfig->write_config($vmid, $conf);
+ });
+ }
}
sub vm_resume {
my ($vmid, $skiplock, $nocheck) = @_;
PVE::QemuConfig->lock_config($vmid, sub {
-
- my $res = vm_mon_cmd($vmid, 'query-status');
+ my $vm_mon_cmd = $nocheck ? \&vm_mon_cmd_nocheck : \&vm_mon_cmd;
+ my $res = $vm_mon_cmd->($vmid, 'query-status');
my $resume_cmd = 'cont';
if ($res->{status} && $res->{status} eq 'suspended') {
PVE::QemuConfig->check_lock($conf)
if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
-
- vm_mon_cmd($vmid, $resume_cmd);
-
- } else {
- vm_mon_cmd_nocheck($vmid, $resume_cmd);
}
+
+ $vm_mon_cmd->($vmid, $resume_cmd);
});
}
my $conf = PVE::QemuConfig->load_config($vmid);
# there is no qmp command, so we use the human monitor command
- vm_human_monitor_command($vmid, "sendkey $key");
+ my $res = vm_human_monitor_command($vmid, "sendkey $key");
+ die $res if $res ne '';
});
}
return if $line =~ m/^lock:/;
return if $line =~ m/^unused\d+:/;
return if $line =~ m/^parent:/;
- return if $line =~ m/^template:/; # restored VM is never a template
my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
$storage_limits{$storeid} = $bwlimit;
$virtdev_hash->{$virtdev} = $devinfo->{$devname};
+ } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
+ my $virtdev = $1;
+ my $drive = parse_drive($virtdev, $2);
+ if (drive_is_cloudinit($drive)) {
+ my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
+ my $scfg = PVE::Storage::storage_config($cfg, $storeid);
+ my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
+
+ my $d = {
+ format => $format,
+ storeid => $opts->{storage} // $storeid,
+ size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
+ file => $drive->{file}, # to make drive_is_cloudinit check possible
+ name => "vm-$vmid-cloudinit",
+ is_cloudinit => 1,
+ };
+ $virtdev_hash->{$virtdev} = $d;
+ }
}
}
foreach_drive($oldconf, sub {
my ($ds, $drive) = @_;
- return if drive_is_cdrom($drive);
+ return if !$drive->{is_cloudinit} && drive_is_cdrom($drive);
my $volid = $drive->{file};
-
return if !$volid || $volid =~ m|^/|;
my ($path, $owner) = PVE::Storage::path($cfg, $volid);
}
});
- # delete vmstate files
- # since after the restore we have no snapshots anymore
+ # delete vmstate files, after the restore we have no snapshots anymore
foreach my $snapname (keys %{$oldconf->{snapshots}}) {
my $snap = $oldconf->{snapshots}->{$snapname};
if ($snap->{vmstate}) {
my $supported = grep { $_ eq $d->{format} } @$validFormats;
$d->{format} = $defFormat if !$supported;
- my $volid = PVE::Storage::vdisk_alloc($cfg, $storeid, $vmid,
- $d->{format}, undef, $alloc_size);
+ my $name;
+ if ($d->{is_cloudinit}) {
+ $name = $d->{name};
+ $name .= ".$d->{format}" if $d->{format} ne 'raw';
+ }
+
+ my $volid = PVE::Storage::vdisk_alloc($cfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
print STDERR "new volume ID is '$volid'\n";
$d->{volid} = $volid;
- my $path = PVE::Storage::path($cfg, $volid);
- PVE::Storage::activate_volumes($cfg,[$volid]);
+ PVE::Storage::activate_volumes($cfg, [$volid]);
my $write_zeros = 1;
if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
$write_zeros = 0;
}
- print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
+ if (!$d->{is_cloudinit}) {
+ my $path = PVE::Storage::path($cfg, $volid);
+
+ print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
- print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
+ print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
+ }
$map->{$virtdev} = $volid;
}
my ($storecfg, $volid) = @_;
my $storage_name = PVE::Storage::parse_volume_id($volid);
+ my $scfg = $storecfg->{ids}->{$storage_name};
- if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
- && !$storecfg->{ids}->{$storage_name}->{krbd}){
+ if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
return 1;
}
}
sub qemu_drive_mirror {
- my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga) = @_;
+ my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
$jobs = {} if !$jobs;
my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
$opts->{format} = $format if $format;
- print "drive mirror is starting for drive-$drive\n";
-
- eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); }; #if a job already run for this device,it's throw an error
+ if (defined($bwlimit)) {
+ $opts->{speed} = $bwlimit * 1024;
+ print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
+ } else {
+ print "drive mirror is starting for drive-$drive\n";
+ }
+ # if a job already runs for this device we get an error, catch it for cleanup
+ eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); };
if (my $err = $@) {
eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
- die "mirroring error: $err";
+ warn "$@\n" if $@;
+ die "mirroring error: $err\n";
}
qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
sub clone_disk {
my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
- $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga) = @_;
+ $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
my $newvolid;
if (drive_is_cloudinit($drive)) {
$name = "vm-$newvmid-cloudinit";
$snapname = undef;
- # cloudinit only supports raw and qcow2 atm:
- if ($dst_format eq 'qcow2') {
- $name .= '.qcow2';
- } elsif ($dst_format ne 'raw') {
- die "clone: unhandled format for cloudinit image\n";
+ # we only get here if it's supported by QEMU_FORMAT_RE, so just accept
+ if ($dst_format ne 'raw') {
+ $name .= ".$dst_format";
}
}
$newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
if (!$running || $snapname) {
+ # TODO: handle bwlimits
qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
} else {
if $drive->{iothread};
}
- qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga);
+ qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga, $bwlimit);
}
}
}
sub qemu_machine_pxe {
- my ($vmid, $conf, $machine) = @_;
+ my ($vmid, $conf) = @_;
- $machine = PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
+ my $machine = PVE::QemuServer::get_current_qemu_machine($vmid);
if ($conf->{machine} && $conf->{machine} =~ m/\.pxe$/) {
$machine .= '.pxe';
push @$cpuFlags , 'hv_synic';
push @$cpuFlags , 'hv_stimer';
}
+
+ if (qemu_machine_feature_enabled ($machine_type, $kvmver, 3, 1)) {
+ push @$cpuFlags , 'hv_ipi';
+ }
}
}
vm_mon_cmd($vmid, 'nbd-server-stop');
}
+sub create_reboot_request {
+ my ($vmid) = @_;
+ open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
+ or die "failed to create reboot trigger file: $!\n";
+ close($fh);
+}
+
+sub clear_reboot_request {
+ my ($vmid) = @_;
+ my $path = "/run/qemu-server/$vmid.reboot";
+ my $res = 0;
+
+ $res = unlink($path);
+ die "could not remove reboot request for $vmid: $!"
+ if !$res && $! != POSIX::ENOENT;
+
+ return $res;
+}
+
# bash completion helper
sub complete_backup_archives {