X-Git-Url: https://git.proxmox.com/?a=blobdiff_plain;f=PVE%2FQemuServer.pm;h=c346af2cc1c5105d7288cca52acab0b0990dc898;hb=d5535a00c45b63fe4245abbd5949371560464e99;hp=dacb8d1c50983ed51e4526cfc62ab11b3eaca69b;hpb=6dbcb073674b7a8b09ef720bbc75115f3c172fc4;p=qemu-server.git diff --git a/PVE/QemuServer.pm b/PVE/QemuServer.pm index dacb8d1..c346af2 100644 --- a/PVE/QemuServer.pm +++ b/PVE/QemuServer.pm @@ -2,6 +2,7 @@ package PVE::QemuServer; use strict; use warnings; + use POSIX; use IO::Handle; use IO::Select; @@ -20,6 +21,7 @@ use JSON; use Fcntl; use PVE::SafeSyslog; use Storable qw(dclone); +use MIME::Base64; use PVE::Exception qw(raise raise_param_exc); use PVE::Storage; use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach $IPV6RE); @@ -53,7 +55,7 @@ my $OVMF = { ], }; -my $qemu_snap_storage = {rbd => 1, sheepdog => 1}; +my $qemu_snap_storage = { rbd => 1 }; my $cpuinfo = PVE::ProcFSTools::read_cpuinfo(); @@ -77,12 +79,6 @@ PVE::JSONSchema::register_standard_option('pve-qm-stateuri', { optional => 1, }); -PVE::JSONSchema::register_standard_option('pve-snapshot-name', { - description => "The name of the snapshot.", - type => 'string', format => 'pve-configid', - maxLength => 40, -}); - PVE::JSONSchema::register_standard_option('pve-qm-image-format', { type => 'string', enum => [qw(raw cow qcow qed qcow2 vmdk cloop)], @@ -171,7 +167,7 @@ my $cpu_vendor_list = { max => 'default', }; -my $cpu_flag = qr/[+-](pcid|spec-ctrl|ibpb|ssbd|virt-ssbd|amd-ssbd|amd-no-ssb|pdpe1gb)/; +my $cpu_flag = qr/[+-](pcid|spec-ctrl|ibpb|ssbd|virt-ssbd|amd-ssbd|amd-no-ssb|pdpe1gb|md-clear)/; my $cpu_fmt = { cputype => { @@ -197,7 +193,7 @@ my $cpu_fmt = { flags => { description => "List of additional CPU flags separated by ';'." . " Use '+FLAG' to enable, '-FLAG' to disable a flag." - . " Currently supported flags: 'pcid', 'spec-ctrl', 'ibpb', 'ssbd', 'virt-ssbd', 'amd-ssbd', 'amd-no-ssb', 'pdpe1gb'.", + . " Currently supported flags: 'pcid', 'spec-ctrl', 'ibpb', 'ssbd', 'virt-ssbd', 'amd-ssbd', 'amd-no-ssb', 'pdpe1gb', 'md-clear'.", format_description => '+FLAG[;-FLAG...]', type => 'string', pattern => qr/$cpu_flag(;$cpu_flag)*/, @@ -271,6 +267,21 @@ my $ivshmem_fmt = { }, }; +my $audio_fmt = { + device => { + type => 'string', + enum => [qw(ich9-intel-hda intel-hda AC97)], + description => "Configure an audio device." + }, + driver => { + type => 'string', + enum => ['spice'], + default => 'spice', + optional => 1, + description => "Driver backend for the audio device." + }, +}; + my $confdesc = { onboot => { optional => 1, @@ -300,7 +311,7 @@ my $confdesc = { optional => 1, type => 'string', description => "Lock/unlock the VM.", - enum => [qw(backup clone create migrate rollback snapshot snapshot-delete)], + enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)], }, cpulimit => { optional => 1, @@ -596,7 +607,7 @@ EODESCR smbios1 => { description => "Specify SMBIOS type 1 fields.", type => 'string', format => 'pve-qm-smbios1', - maxLength => 256, + maxLength => 512, optional => 1, }, protection => { @@ -641,9 +652,40 @@ EODESCR format => $ivshmem_fmt, description => "Inter-VM shared memory. Useful for direct communication between VMs, or to the host.", optional => 1, - } + }, + audio0 => { + type => 'string', + format => $audio_fmt, + description => "Configure a audio device, useful in combination with QXL/Spice.", + optional => 1 + }, }; +my $cicustom_fmt = { + meta => { + type => 'string', + optional => 1, + description => 'Specify a custom file containing all meta data passed to the VM via cloud-init. This is provider specific meaning configdrive2 and nocloud differ.', + format => 'pve-volume-id', + format_description => 'volume', + }, + network => { + type => 'string', + optional => 1, + description => 'Specify a custom file containing all network data passed to the VM via cloud-init.', + format => 'pve-volume-id', + format_description => 'volume', + }, + user => { + type => 'string', + optional => 1, + description => 'Specify a custom file containing all user data passed to the VM via cloud-init.', + format => 'pve-volume-id', + format_description => 'volume', + }, +}; +PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt); + my $confdesc_cloudinit = { citype => { optional => 1, @@ -661,6 +703,12 @@ my $confdesc_cloudinit = { type => 'string', description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.', }, + cicustom => { + optional => 1, + type => 'string', + description => 'cloud-init: Specify custom files to replace the automatically generated ones at start.', + format => 'pve-qm-cicustom', + }, searchdomain => { optional => 1, type => 'string', @@ -771,13 +819,9 @@ The DHCP server assign addresses to the guest starting from 10.0.2.15. __EOD__ my $net_fmt = { - macaddr => { - type => 'string', - pattern => qr/[0-9a-f]{2}(?::[0-9a-f]{2}){5}/i, + macaddr => get_standard_option('mac-addr', { description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.", - format_description => "XX:XX:XX:XX:XX:XX", - optional => 1, - }, + }), model => { type => 'string', description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.", @@ -1259,7 +1303,7 @@ my $hostpci_fmt = { pattern => qr/$PCIRE(;$PCIRE)*/, format_description => 'HOSTPCIID[;HOSTPCIID2...]', description => < <{file}; my $format; - + if (drive_is_cdrom($drive)) { $path = get_iso_path($storecfg, $vmid, $volid); } else { @@ -2336,7 +2380,7 @@ sub vmconfig_cleanup_pending { return $changes; } -# smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str] +# smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool] my $smbios1_fmt = { uuid => { type => 'string', @@ -2347,46 +2391,51 @@ my $smbios1_fmt = { }, version => { type => 'string', - pattern => '\S+', - format_description => 'string', + pattern => '[A-Za-z0-9+\/]+={0,2}', + format_description => 'Base64 encoded string', description => "Set SMBIOS1 version.", optional => 1, }, serial => { type => 'string', - pattern => '\S+', - format_description => 'string', + pattern => '[A-Za-z0-9+\/]+={0,2}', + format_description => 'Base64 encoded string', description => "Set SMBIOS1 serial number.", optional => 1, }, manufacturer => { type => 'string', - pattern => '\S+', - format_description => 'string', + pattern => '[A-Za-z0-9+\/]+={0,2}', + format_description => 'Base64 encoded string', description => "Set SMBIOS1 manufacturer.", optional => 1, }, product => { type => 'string', - pattern => '\S+', - format_description => 'string', + pattern => '[A-Za-z0-9+\/]+={0,2}', + format_description => 'Base64 encoded string', description => "Set SMBIOS1 product ID.", optional => 1, }, sku => { type => 'string', - pattern => '\S+', - format_description => 'string', + pattern => '[A-Za-z0-9+\/]+={0,2}', + format_description => 'Base64 encoded string', description => "Set SMBIOS1 SKU string.", optional => 1, }, family => { type => 'string', - pattern => '\S+', - format_description => 'string', + pattern => '[A-Za-z0-9+\/]+={0,2}', + format_description => 'Base64 encoded string', description => "Set SMBIOS1 family string.", optional => 1, }, + base64 => { + type => 'boolean', + description => 'Flag to indicate that the SMBIOS values are base64 encoded', + optional => 1, + }, }; sub parse_smbios1 { @@ -2824,23 +2873,23 @@ sub config_list { sub check_local_resources { my ($conf, $noerr) = @_; - my $loc_res = 0; + my @loc_res = (); - $loc_res = 1 if $conf->{hostusb}; # old syntax - $loc_res = 1 if $conf->{hostpci}; # old syntax + push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax + push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax - $loc_res = 1 if $conf->{ivshmem}; + push @loc_res, "ivshmem" if $conf->{ivshmem}; foreach my $k (keys %$conf) { next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice'); # sockets are safe: they will recreated be on the target side post-migrate next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket'); - $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/; + push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/; } - die "VM uses local resources\n" if $loc_res && !$noerr; + die "VM uses local resources\n" if scalar @loc_res && !$noerr; - return $loc_res; + return \@loc_res; } # check if used storages are available on all nodes (use by migrate) @@ -2896,6 +2945,45 @@ sub shared_nodes { return $nodehash } +sub check_local_storage_availability { + my ($conf, $storecfg) = @_; + + my $nodelist = PVE::Cluster::get_nodelist(); + my $nodehash = { map { $_ => {} } @$nodelist }; + + foreach_drive($conf, sub { + my ($ds, $drive) = @_; + + my $volid = $drive->{file}; + return if !$volid; + + my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1); + if ($storeid) { + my $scfg = PVE::Storage::storage_config($storecfg, $storeid); + + if ($scfg->{disable}) { + foreach my $node (keys %$nodehash) { + $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1; + } + } elsif (my $avail = $scfg->{nodes}) { + foreach my $node (keys %$nodehash) { + if (!$avail->{$node}) { + $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1; + } + } + } + } + }); + + foreach my $node (values %$nodehash) { + if (my $unavail = $node->{unavailable_storages}) { + $node->{unavailable_storages} = [ sort keys %$unavail ]; + } + } + + return $nodehash +} + sub check_cmdline { my ($pidfile, $pid) = @_; @@ -3038,6 +3126,11 @@ our $vmstatus_return_properties = { type => 'number', optional => 1, }, + lock => { + description => "The current config lock, if any.", + type => 'string', + optional => 1, + } }; my $last_proc_pid_stat; @@ -3108,6 +3201,7 @@ sub vmstatus { $d->{template} = PVE::QemuConfig->is_template($conf); $d->{serial} = 1 if conf_has_serial($conf); + $d->{lock} = $conf->{lock} if $conf->{lock}; $res->{$vmid} = $d; } @@ -3267,7 +3361,7 @@ sub foreach_volid { my $volhash = {}; my $test_volid = sub { - my ($volid, $is_cdrom, $replicate, $shared, $snapname) = @_; + my ($volid, $is_cdrom, $replicate, $shared, $snapname, $size) = @_; return if !$volid; @@ -3285,11 +3379,12 @@ sub foreach_volid { $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1 if defined($snapname); + $volhash->{$volid}->{size} = $size if $size; }; foreach_drive($conf, sub { my ($ds, $drive) = @_; - $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, undef); + $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, undef, $drive->{size}); }); foreach my $snapname (keys %{$conf->{snapshots}}) { @@ -3318,6 +3413,24 @@ sub conf_has_serial { return 0; } +sub conf_has_audio { + my ($conf, $id) = @_; + + $id //= 0; + my $audio = $conf->{"audio$id"}; + return undef if !defined($audio); + + my $audioproperties = PVE::JSONSchema::parse_property_string($audio_fmt, $audio); + my $audiodriver = $audioproperties->{driver} // 'spice'; + + return { + dev => $audioproperties->{device}, + dev_id => "audio$id", + backend => $audiodriver, + backend_id => "$audiodriver-backend${id}", + }; +} + sub vga_conf_has_spice { my ($vga) = @_; @@ -3496,7 +3609,26 @@ sub config_to_command { push @$cmd, '-daemonize'; if ($conf->{smbios1}) { - push @$cmd, '-smbios', "type=1,$conf->{smbios1}"; + my $smbios_conf = parse_smbios1($conf->{smbios1}); + if ($smbios_conf->{base64}) { + # Do not pass base64 flag to qemu + delete $smbios_conf->{base64}; + my $smbios_string = ""; + foreach my $key (keys %$smbios_conf) { + my $value; + if ($key eq "uuid") { + $value = $smbios_conf->{uuid} + } else { + $value = decode_base64($smbios_conf->{$key}); + } + # qemu accepts any binary data, only commas need escaping by double comma + $value =~ s/,/,,/g; + $smbios_string .= "," . $key . "=" . $value if $value; + } + push @$cmd, '-smbios', "type=1" . $smbios_string; + } else { + push @$cmd, '-smbios', "type=1,$conf->{smbios1}"; + } } if ($conf->{vmgenid}) { @@ -3535,6 +3667,15 @@ sub config_to_command { push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path"; } + # load q35 config + if ($q35) { + # we use different pcie-port hardware for qemu >= 4.0 for passthrough + if (qemu_machine_feature_enabled($machine_type, $kvmver, 4, 0)) { + push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg'; + } else { + push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg'; + } + } # add usb controllers my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES); @@ -3579,7 +3720,7 @@ sub config_to_command { next if !$d; my $pcie = $d->{pcie}; - if($pcie){ + if ($pcie) { die "q35 machine model is not enabled" if !$q35; # win7 wants to have the pcie devices directly on the pcie bus # instead of in the root port @@ -3588,7 +3729,7 @@ sub config_to_command { } else { $pciaddr = print_pcie_addr("hostpci$i"); } - }else{ + } else { $pciaddr = print_pci_addr("hostpci$i", $bridges, $arch, $machine_type); } @@ -3678,6 +3819,23 @@ sub config_to_command { } } + if (my $audio = conf_has_audio($conf)) { + + my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type); + + my $id = $audio->{dev_id}; + if ($audio->{dev} eq 'AC97') { + push @$devices, '-device', "AC97,id=${id}${audiopciaddr}"; + } elsif ($audio->{dev} =~ /intel\-hda$/) { + push @$devices, '-device', "$audio->{dev},id=${id}${audiopciaddr}"; + push @$devices, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0"; + push @$devices, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1"; + } else { + die "unkown audio device '$audio->{dev}', implement me!"; + } + + push @$devices, '-audiodev', "$audio->{backend},id=$audio->{backend_id}"; + } my $sockets = 1; $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused @@ -3726,7 +3884,7 @@ sub config_to_command { if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){ push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_type, undef, $qxlnum, $bridges); my $socket = vnc_socket($vmid); - push @$cmd, '-vnc', "unix:$socket,x509,password"; + push @$cmd, '-vnc', "unix:$socket,password"; } else { push @$cmd, '-vga', 'none' if $vga->{type} eq 'none'; push @$cmd, '-nographic'; @@ -3761,7 +3919,7 @@ sub config_to_command { push @$machineFlags, "type=${machine_type}"; } - if ($conf->{startdate}) { + if (($conf->{startdate}) && ($conf->{startdate} ne 'now')) { push @$rtcFlags, "base=$conf->{startdate}"; } elsif ($useLocaltime) { push @$rtcFlags, 'base=localtime'; @@ -3770,16 +3928,11 @@ sub config_to_command { push @$cmd, get_cpu_options($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough); PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd); - + push @$cmd, '-S' if $conf->{freeze}; push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard}); - # enable sound - #my $soundhw = $conf->{soundhw} || $defaults->{soundhw}; - #push @$cmd, '-soundhw', 'es1370'; - #push @$cmd, '-soundhw', $soundhw if $soundhw; - if (parse_guest_agent($conf)->{enabled}) { my $qgasocket = qmp_socket($vmid, 1); my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type); @@ -3894,7 +4047,7 @@ sub config_to_command { my $queues = ''; if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){ $queues = ",num_queues=$drive->{queues}"; - } + } push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller}; $scsicontroller->{$controller}=1; @@ -3933,18 +4086,17 @@ sub config_to_command { if ($conf->{ivshmem}) { my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem}); + my $bus; if ($q35) { $bus = print_pcie_addr("ivshmem"); } else { $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type); } - my $path = '/dev/shm/pve-shm-'; - if ($ivshmem->{name}) { - $path .= $ivshmem->{name}; - } else { - $path .= $vmid; - } + + my $ivshmem_name = $ivshmem->{name} // $vmid; + my $path = '/dev/shm/pve-shm-' . $ivshmem_name; + push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,"; push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path,size=$ivshmem->{size}M"; } @@ -3972,6 +4124,12 @@ sub config_to_command { push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags); + if (my $vmstate = $conf->{vmstate}) { + my $statepath = PVE::Storage::path($storecfg, $vmstate); + PVE::Storage::activate_volumes($storecfg, [$vmstate]); + push @$cmd, '-loadstate', $statepath; + } + # add custom args if ($conf->{args}) { my $aa = PVE::Tools::split_args($conf->{args}); @@ -4133,20 +4291,23 @@ sub vm_deviceplug { } elsif ($deviceid =~ m/^(net)(\d+)$/) { - return undef if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid); + return undef if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid); - my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf); - my $use_old_bios_files = undef; - ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type); + my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf); + my $use_old_bios_files = undef; + ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type); - my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type); - qemu_deviceadd($vmid, $netdevicefull); - eval { qemu_deviceaddverify($vmid, $deviceid); }; + my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type); + qemu_deviceadd($vmid, $netdevicefull); + eval { + qemu_deviceaddverify($vmid, $deviceid); + qemu_set_link_status($vmid, $deviceid, !$device->{link_down}); + }; if (my $err = $@) { eval { qemu_netdevdel($vmid, $deviceid); }; warn $@ if $@; die $err; - } + } } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) { @@ -4245,7 +4406,11 @@ sub qemu_iothread_add { sub qemu_iothread_del { my($conf, $vmid, $deviceid) = @_; - my $device = parse_drive($deviceid, $conf->{$deviceid}); + my $confid = $deviceid; + if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) { + $confid = 'scsi' . $1; + } + my $device = parse_drive($confid, $conf->{$confid}); if ($device->{iothread}) { my $iothreads = vm_iothreads_list($vmid); qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"}; @@ -5115,7 +5280,10 @@ sub vm_start { die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf); - PVE::QemuConfig->check_lock($conf) if !$skiplock; + my $is_suspended = PVE::QemuConfig->has_lock($conf, 'suspended'); + + PVE::QemuConfig->check_lock($conf) + if !($skiplock || $is_suspended); die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom); @@ -5181,6 +5349,12 @@ sub vm_start { PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1); + if ($is_suspended) { + # enforce machine type on suspended vm to ensure HW compatibility + $forcemachine = $conf->{runningmachine}; + print "Resuming suspended VM\n"; + } + my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine); my $migrate_port = 0; @@ -5261,17 +5435,18 @@ sub vm_start { PVE::Storage::activate_volumes($storecfg, $vollist); - if (-d "/sys/fs/cgroup/systemd/qemu.slice/$vmid.scope") { - eval { - run_command(['/bin/systemctl', 'stop', "$vmid.scope"], - outfunc => sub {}, errfunc => sub {}); - }; - } + eval { + run_command(['/bin/systemctl', 'stop', "$vmid.scope"], + outfunc => sub {}, errfunc => sub {}); + }; + # Issues with the above 'stop' not being fully completed are extremely rare, a very low + # timeout should be more than enough here... + PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5); my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits} : $defaults->{cpuunits}; - my $start_timeout = $conf->{hugepages} ? 300 : 30; + my $start_timeout = ($conf->{hugepages} || $is_suspended) ? 300 : 30; my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077); my %properties = ( @@ -5378,6 +5553,14 @@ sub vm_start { property => "guest-stats-polling-interval", value => 2) if (!defined($conf->{balloon}) || $conf->{balloon}); + if ($is_suspended && (my $vmstate = $conf->{vmstate})) { + print "Resumed VM, removing state\n"; + delete $conf->@{qw(lock vmstate runningmachine)}; + PVE::Storage::deactivate_volumes($storecfg, [$vmstate]); + PVE::Storage::vdisk_free($storecfg, $vmstate); + PVE::QemuConfig->write_config($vmid, $conf); + } + PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start'); }); } @@ -5402,9 +5585,8 @@ sub vm_qmp_command { my $res; my $timeout; - if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) { - $timeout = $cmd->{arguments}->{timeout}; - delete $cmd->{arguments}->{timeout}; + if ($cmd->{arguments}) { + $timeout = delete $cmd->{arguments}->{timeout}; } eval { @@ -5429,8 +5611,6 @@ sub vm_qmp_command { sub vm_human_monitor_command { my ($vmid, $cmdline) = @_; - my $res; - my $cmd = { execute => 'human-monitor-command', arguments => { 'command-line' => $cmdline}, @@ -5507,6 +5687,10 @@ sub vm_stop_cleanup { if ($conf->{ivshmem}) { my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem}); + # just delete it for now, VMs which have this already open do not + # are affected, but new VMs will get a separated one. If this + # becomes an issue we either add some sort of ref-counting or just + # add a "don't delete on stop" flag to the ivshmem format. unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid); } @@ -5559,12 +5743,13 @@ sub vm_stop { PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop'); } - $timeout = 60 if !defined($timeout); - eval { if ($shutdown) { if (defined($conf) && parse_guest_agent($conf)->{enabled}) { - vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck); + vm_qmp_command($vmid, { + execute => "guest-shutdown", + arguments => { timeout => $timeout } + }, $nocheck); } else { vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck); } @@ -5575,6 +5760,8 @@ sub vm_stop { my $err = $@; if (!$err) { + $timeout = 60 if !defined($timeout); + my $count = 0; while (($count < $timeout) && check_running($vmid, $nocheck)) { $count++; @@ -5621,25 +5808,92 @@ sub vm_stop { } sub vm_suspend { - my ($vmid, $skiplock) = @_; + my ($vmid, $skiplock, $includestate, $statestorage) = @_; + + my $conf; + my $path; + my $storecfg; + my $vmstate; PVE::QemuConfig->lock_config($vmid, sub { - my $conf = PVE::QemuConfig->load_config($vmid); + $conf = PVE::QemuConfig->load_config($vmid); + my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup'); PVE::QemuConfig->check_lock($conf) - if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup')); + if !($skiplock || $is_backing_up); + + die "cannot suspend to disk during backup\n" + if $is_backing_up && $includestate; - vm_mon_cmd($vmid, "stop"); + if ($includestate) { + $conf->{lock} = 'suspending'; + my $date = strftime("%Y-%m-%d", localtime(time())); + $storecfg = PVE::Storage::config(); + $vmstate = PVE::QemuConfig->__snapshot_save_vmstate($vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1); + $path = PVE::Storage::path($storecfg, $vmstate); + PVE::QemuConfig->write_config($vmid, $conf); + } else { + vm_mon_cmd($vmid, "stop"); + } }); + + if ($includestate) { + # save vm state + PVE::Storage::activate_volumes($storecfg, [$vmstate]); + + eval { + vm_mon_cmd($vmid, "savevm-start", statefile => $path); + for(;;) { + my $state = vm_mon_cmd_nocheck($vmid, "query-savevm"); + if (!$state->{status}) { + die "savevm not active\n"; + } elsif ($state->{status} eq 'active') { + sleep(1); + next; + } elsif ($state->{status} eq 'completed') { + print "State saved, quitting\n"; + last; + } elsif ($state->{status} eq 'failed' && $state->{error}) { + die "query-savevm failed with error '$state->{error}'\n" + } else { + die "query-savevm returned status '$state->{status}'\n"; + } + } + }; + my $err = $@; + + PVE::QemuConfig->lock_config($vmid, sub { + $conf = PVE::QemuConfig->load_config($vmid); + if ($err) { + # cleanup, but leave suspending lock, to indicate something went wrong + eval { + vm_mon_cmd($vmid, "savevm-end"); + PVE::Storage::deactivate_volumes($storecfg, [$vmstate]); + PVE::Storage::vdisk_free($storecfg, $vmstate); + delete $conf->@{qw(vmstate runningmachine)}; + PVE::QemuConfig->write_config($vmid, $conf); + }; + warn $@ if $@; + die $err; + } + + die "lock changed unexpectedly\n" + if !PVE::QemuConfig->has_lock($conf, 'suspending'); + + vm_qmp_command($vmid, { execute => "quit" }); + $conf->{lock} = 'suspended'; + PVE::QemuConfig->write_config($vmid, $conf); + }); + } } sub vm_resume { my ($vmid, $skiplock, $nocheck) = @_; PVE::QemuConfig->lock_config($vmid, sub { - - my $res = vm_mon_cmd($vmid, 'query-status'); + my $vm_mon_cmd = $nocheck ? \&vm_mon_cmd_nocheck : \&vm_mon_cmd; + my $res = $vm_mon_cmd->($vmid, 'query-status'); my $resume_cmd = 'cont'; if ($res->{status} && $res->{status} eq 'suspended') { @@ -5652,12 +5906,9 @@ sub vm_resume { PVE::QemuConfig->check_lock($conf) if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup')); - - vm_mon_cmd($vmid, $resume_cmd); - - } else { - vm_mon_cmd_nocheck($vmid, $resume_cmd); } + + $vm_mon_cmd->($vmid, $resume_cmd); }); } @@ -5669,7 +5920,8 @@ sub vm_sendkey { my $conf = PVE::QemuConfig->load_config($vmid); # there is no qmp command, so we use the human monitor command - vm_human_monitor_command($vmid, "sendkey $key"); + my $res = vm_human_monitor_command($vmid, "sendkey $key"); + die $res if $res ne ''; }); } @@ -5776,7 +6028,6 @@ sub restore_update_config_line { return if $line =~ m/^lock:/; return if $line =~ m/^unused\d+:/; return if $line =~ m/^parent:/; - return if $line =~ m/^template:/; # restored VM is never a template my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg'); if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) { @@ -6137,6 +6388,24 @@ sub restore_vma_archive { $storage_limits{$storeid} = $bwlimit; $virtdev_hash->{$virtdev} = $devinfo->{$devname}; + } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) { + my $virtdev = $1; + my $drive = parse_drive($virtdev, $2); + if (drive_is_cloudinit($drive)) { + my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}); + my $scfg = PVE::Storage::storage_config($cfg, $storeid); + my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback + + my $d = { + format => $format, + storeid => $opts->{storage} // $storeid, + size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE, + file => $drive->{file}, # to make drive_is_cloudinit check possible + name => "vm-$vmid-cloudinit", + is_cloudinit => 1, + }; + $virtdev_hash->{$virtdev} = $d; + } } } @@ -6158,10 +6427,9 @@ sub restore_vma_archive { foreach_drive($oldconf, sub { my ($ds, $drive) = @_; - return if drive_is_cdrom($drive); + return if !$drive->{is_cloudinit} && drive_is_cdrom($drive); my $volid = $drive->{file}; - return if !$volid || $volid =~ m|^/|; my ($path, $owner) = PVE::Storage::path($cfg, $volid); @@ -6177,8 +6445,7 @@ sub restore_vma_archive { } }); - # delete vmstate files - # since after the restore we have no snapshots anymore + # delete vmstate files, after the restore we have no snapshots anymore foreach my $snapname (keys %{$oldconf->{snapshots}}) { my $snap = $oldconf->{snapshots}->{$snapname}; if ($snap->{vmstate}) { @@ -6207,22 +6474,30 @@ sub restore_vma_archive { my $supported = grep { $_ eq $d->{format} } @$validFormats; $d->{format} = $defFormat if !$supported; - my $volid = PVE::Storage::vdisk_alloc($cfg, $storeid, $vmid, - $d->{format}, undef, $alloc_size); + my $name; + if ($d->{is_cloudinit}) { + $name = $d->{name}; + $name .= ".$d->{format}" if $d->{format} ne 'raw'; + } + + my $volid = PVE::Storage::vdisk_alloc($cfg, $storeid, $vmid, $d->{format}, $name, $alloc_size); print STDERR "new volume ID is '$volid'\n"; $d->{volid} = $volid; - my $path = PVE::Storage::path($cfg, $volid); - PVE::Storage::activate_volumes($cfg,[$volid]); + PVE::Storage::activate_volumes($cfg, [$volid]); my $write_zeros = 1; if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) { $write_zeros = 0; } - print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n"; + if (!$d->{is_cloudinit}) { + my $path = PVE::Storage::path($cfg, $volid); - print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n"; + print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n"; + + print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n"; + } $map->{$virtdev} = $volid; } @@ -6458,9 +6733,9 @@ sub do_snapshots_with_qemu { my ($storecfg, $volid) = @_; my $storage_name = PVE::Storage::parse_volume_id($volid); + my $scfg = $storecfg->{ids}->{$storage_name}; - if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}} - && !$storecfg->{ids}->{$storage_name}->{krbd}){ + if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){ return 1; } @@ -6503,6 +6778,23 @@ sub template_create { }); } +sub convert_iscsi_path { + my ($path) = @_; + + if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) { + my $portal = $1; + my $target = $2; + my $lun = $3; + + my $initiator_name = get_initiator_name(); + + return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,". + "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw"; + } + + die "cannot convert iscsi path '$path', unkown format\n"; +} + sub qemu_img_convert { my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_; @@ -6523,13 +6815,32 @@ sub qemu_img_convert { my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname); my $dst_path = PVE::Storage::path($storecfg, $dst_volid); + my $src_is_iscsi = ($src_path =~ m|^iscsi://|); + my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|); + my $cmd = []; push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n'; push @$cmd, '-l', "snapshot.name=$snapname" if($snapname && $src_format eq "qcow2"); push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool'; push @$cmd, '-T', 'none' if $src_scfg->{type} eq 'zfspool'; - push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path; - if ($is_zero_initialized) { + + if ($src_is_iscsi) { + push @$cmd, '--image-opts'; + $src_path = convert_iscsi_path($src_path); + } else { + push @$cmd, '-f', $src_format; + } + + if ($dst_is_iscsi) { + push @$cmd, '--target-image-opts'; + $dst_path = convert_iscsi_path($dst_path); + } else { + push @$cmd, '-O', $dst_format; + } + + push @$cmd, $src_path; + + if (!$dst_is_iscsi && $is_zero_initialized) { push @$cmd, "zeroinit:$dst_path"; } else { push @$cmd, $dst_path; @@ -6564,7 +6875,7 @@ sub qemu_img_format { } sub qemu_drive_mirror { - my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga) = @_; + my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga, $bwlimit) = @_; $jobs = {} if !$jobs; @@ -6591,13 +6902,19 @@ sub qemu_drive_mirror { my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target }; $opts->{format} = $format if $format; - print "drive mirror is starting for drive-$drive\n"; - - eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); }; #if a job already run for this device,it's throw an error + if (defined($bwlimit)) { + $opts->{speed} = $bwlimit * 1024; + print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n"; + } else { + print "drive mirror is starting for drive-$drive\n"; + } + # if a job already runs for this device we get an error, catch it for cleanup + eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); }; if (my $err = $@) { eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) }; - die "mirroring error: $err"; + warn "$@\n" if $@; + die "mirroring error: $err\n"; } qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga); @@ -6735,7 +7052,7 @@ sub qemu_blockjobs_cancel { sub clone_disk { my ($storecfg, $vmid, $running, $drivename, $drive, $snapname, - $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga) = @_; + $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga, $bwlimit) = @_; my $newvolid; @@ -6756,11 +7073,9 @@ sub clone_disk { if (drive_is_cloudinit($drive)) { $name = "vm-$newvmid-cloudinit"; $snapname = undef; - # cloudinit only supports raw and qcow2 atm: - if ($dst_format eq 'qcow2') { - $name .= '.qcow2'; - } elsif ($dst_format ne 'raw') { - die "clone: unhandled format for cloudinit image\n"; + # we only get here if it's supported by QEMU_FORMAT_RE, so just accept + if ($dst_format ne 'raw') { + $name .= ".$dst_format"; } } $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024)); @@ -6770,6 +7085,7 @@ sub clone_disk { my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid); if (!$running || $snapname) { + # TODO: handle bwlimits qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit); } else { @@ -6779,7 +7095,7 @@ sub clone_disk { if $drive->{iothread}; } - qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga); + qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga, $bwlimit); } } @@ -6956,6 +7272,13 @@ sub add_hyperv_enlightenments { push @$cpuFlags , 'hv_synic'; push @$cpuFlags , 'hv_stimer'; } + + if (qemu_machine_feature_enabled ($machine_type, $kvmver, 3, 1)) { + push @$cpuFlags , 'hv_tlbflush'; + push @$cpuFlags , 'hv_ipi'; + # FIXME: AMD does not supports this currently, only add with special flag?? + #push @$cpuFlags , 'hv_evmcs'; + } } }