X-Git-Url: https://git.proxmox.com/?a=blobdiff_plain;f=PVE%2FVZDump%2FQemuServer.pm;h=7429508241e716cc8ffd092230b963e094192f34;hb=21a9ec2ad312693b0485223a9f3f9946f79c8f4e;hp=c919aa9c92c4c190e1a91abae8e15ec5a7ed7c15;hpb=0f6c69814b18fc4a68044b2bcd13ac48bd9e41d7;p=qemu-server.git diff --git a/PVE/VZDump/QemuServer.pm b/PVE/VZDump/QemuServer.pm index c919aa9..7429508 100644 --- a/PVE/VZDump/QemuServer.pm +++ b/PVE/VZDump/QemuServer.pm @@ -8,17 +8,20 @@ use File::Path; use IO::File; use IPC::Open3; use JSON; +use POSIX qw(EINTR EAGAIN); use PVE::Cluster qw(cfs_read_file); use PVE::INotify; use PVE::IPCC; use PVE::JSONSchema; +use PVE::PBSClient; use PVE::QMPClient; use PVE::Storage::Plugin; use PVE::Storage::PBSPlugin; use PVE::Storage; use PVE::Tools; use PVE::VZDump; +use PVE::Format qw(render_duration render_bytes); use PVE::QemuConfig; use PVE::QemuServer; @@ -60,8 +63,11 @@ sub prepare { if defined($conf->{name}); $self->{vm_was_running} = 1; + $self->{vm_was_paused} = 0; if (!PVE::QemuServer::check_running($vmid)) { $self->{vm_was_running} = 0; + } elsif (PVE::QemuServer::vm_is_paused($vmid)) { + $self->{vm_was_paused} = 1; } $task->{hostname} = $conf->{name}; @@ -80,11 +86,10 @@ sub prepare { if (!$volume->{included}) { $self->loginfo("exclude disk '$name' '$volid' ($volume->{reason})"); next; - } elsif ($self->{vm_was_running} && $volume_config->{iothread}) { - if (!PVE::QemuServer::Machine::runs_at_least_qemu_version($vmid, 4, 0, 1)) { - die "disk '$name' '$volid' (iothread=on) can't use backup feature with running QEMU " . - "version < 4.0.1! Either set backup=no for this drive or upgrade QEMU and restart VM\n"; - } + } elsif ($self->{vm_was_running} && $volume_config->{iothread} && + !PVE::QemuServer::Machine::runs_at_least_qemu_version($vmid, 4, 0, 1)) { + die "disk '$name' '$volid' (iothread=on) can't use backup feature with running QEMU " . + "version < 4.0.1! Either set backup=no for this drive or upgrade QEMU and restart VM\n"; } else { my $log = "include disk '$name' '$volid'"; if (defined(my $size = $volume_config->{size})) { @@ -125,6 +130,12 @@ sub prepare { qmdevice => "drive-$ds", }; + if ($ds eq 'tpmstate0') { + # TPM drive only exists for backup, which is reflected in the name + $diskinfo->{qmdevice} = 'drive-tpmstate0-backup'; + $task->{tpmpath} = $path; + } + if (-b $path) { $diskinfo->{type} = 'block'; } else { @@ -174,12 +185,16 @@ sub start_vm { sub suspend_vm { my ($self, $task, $vmid) = @_; + return if $self->{vm_was_paused}; + $self->cmd ("qm suspend $vmid --skiplock"); } sub resume_vm { my ($self, $task, $vmid) = @_; + return if $self->{vm_was_paused}; + $self->cmd ("qm resume $vmid --skiplock"); } @@ -253,46 +268,24 @@ sub archive { } } -# number, [precision=1] -my $num2str = sub { - return sprintf( "%." . ( $_[1] || 1 ) . "f", $_[0] ); -}; -sub bytes_to_human { - my ($bytes, $precission) = @_; - - return $num2str->($bytes, $precission) . ' B' if $bytes < 1024; - my $kb = $bytes/1024; - - return $num2str->($kb, $precission) . " KiB" if $kb < 1024; - my $mb = $kb/1024; - - return $num2str->($mb, $precission) . " MiB" if $mb < 1024; - my $gb = $mb/1024; - - return $num2str->($gb, $precission) . " GiB" if $gb < 1024; - my $tb = $gb/1024; - - return $num2str->($tb, $precission) . " TiB"; -}; - my $bitmap_action_to_human = sub { - my ($info) = @_; + my ($self, $info) = @_; my $action = $info->{action}; if ($action eq "not-used") { - return "disabled"; + return "disabled (no support)"; } elsif ($action eq "not-used-removed") { return "disabled (old bitmap cleared)"; } elsif ($action eq "new") { - return "created new bitmap"; + return "created new"; } elsif ($action eq "used") { if ($info->{dirty} == 0) { - return "OK, drive clean"; + return "OK (drive clean)"; } else { - my $size = bytes_to_human($info->{size}); - my $dirty = bytes_to_human($info->{dirty}); - return "OK, $dirty of $size dirty"; + my $size = render_bytes($info->{size}, 1); + my $dirty = render_bytes($info->{dirty}, 1); + return "OK ($dirty of $size dirty)"; } } elsif ($action eq "invalid") { return "existing bitmap was invalid and has been cleared"; @@ -302,7 +295,7 @@ my $bitmap_action_to_human = sub { }; my $query_backup_status_loop = sub { - my ($self, $vmid, $job_uuid, $pbs_features) = @_; + my ($self, $vmid, $job_uuid, $qemu_support) = @_; my $starttime = time (); my $last_time = $starttime; @@ -313,31 +306,41 @@ my $query_backup_status_loop = sub { my ($mb, $delta) = @_; return "0 B/s" if $mb <= 0; my $bw = int(($mb / $delta)); - return bytes_to_human($bw) . "/s"; + return render_bytes($bw, 1) . "/s"; }; my $target = 0; my $last_reused = 0; - my $has_query_bitmap = 0; - if (defined($pbs_features) && $pbs_features->{'query-bitmap-info'}) { - $has_query_bitmap = 1; + my $has_query_bitmap = $qemu_support && $qemu_support->{'query-bitmap-info'}; + my $is_template = PVE::QemuConfig->is_template($self->{vmlist}->{$vmid}); + if ($has_query_bitmap) { + my $total = 0; my $bitmap_info = mon_cmd($vmid, 'query-pbs-bitmap-info'); - $self->loginfo("Fast incremental status:"); - foreach my $info (@$bitmap_info) { - my $text = $bitmap_action_to_human->($info); - my $drive = $info->{drive}; - $drive =~ s/^drive-//; # for consistency - $self->loginfo("$drive: $text"); + for my $info (sort { $a->{drive} cmp $b->{drive} } @$bitmap_info) { + if (!$is_template) { + my $text = $bitmap_action_to_human->($self, $info); + my $drive = $info->{drive}; + $drive =~ s/^drive-//; # for consistency + $self->loginfo("$drive: dirty-bitmap status: $text"); + } $target += $info->{dirty}; + $total += $info->{size}; $last_reused += $info->{size} - $info->{dirty}; } + if ($target < $total) { + my $total_h = render_bytes($total, 1); + my $target_h = render_bytes($target, 1); + $self->loginfo("using fast incremental mode (dirty-bitmap), $target_h dirty of $total_h total"); + } } + my $last_finishing = 0; while(1) { my $status = mon_cmd($vmid, 'query-backup'); my $total = $status->{total} || 0; - $target = $total if !$has_query_bitmap; + my $dirty = $status->{dirty}; + $target = (defined($dirty) && $dirty < $total) ? $dirty : $total if !$has_query_bitmap; $transferred = $status->{transferred} || 0; $reused = $status->{reused}; my $percent = $target ? int(($transferred * 100)/$target) : 100; @@ -360,15 +363,17 @@ my $query_backup_status_loop = sub { my $timediff = ($ctime - $last_time) || 1; # fixme my $mbps_read = $get_mbps->($rbytes, $timediff); my $mbps_write = $get_mbps->($wbytes, $timediff); - my $target_h = bytes_to_human($target); - my $transferred_h = bytes_to_human($transferred); + my $target_h = render_bytes($target, 1); + my $transferred_h = render_bytes($transferred, 1); - my $statusline = "status: $percent% ($transferred_h of $target_h), duration $duration" - .", read: $mbps_read, write: $mbps_write"; + my $statusline = sprintf("%3d%% ($transferred_h of $target_h) in %s" + .", read: $mbps_read, write: $mbps_write", $percent, render_duration($duration)); my $res = $status->{status} || 'unknown'; if ($res ne 'active') { - $self->loginfo($statusline); + if ($last_percent < 100) { + $self->loginfo($statusline); + } if ($res ne 'done') { die (($status->{errmsg} || "unknown error") . "\n") if $res eq 'error'; die "got unexpected status '$res'\n"; @@ -388,26 +393,35 @@ my $query_backup_status_loop = sub { $last_transferred = $transferred if $transferred; $last_time = $ctime; $last_reused = $reused; + + if (!$last_finishing && $status->{finishing}) { + $self->loginfo("Waiting for server to finish backup validation..."); + } + $last_finishing = $status->{finishing}; } sleep(1); } my $duration = time() - $starttime; - if ($transferred && $duration) { - my $transferred_h = bytes_to_human($transferred, 2); - my $mbps = $get_mbps->($transferred, $duration); - if ($reused) { - my $reused_h = bytes_to_human($reused, 2); - my $reuse_per = int($reused * 100 / $last_total); - $self->loginfo("backup was done incrementally, reused $reused_h (${reuse_per}%)"); - } - $self->loginfo("transferred $transferred_h in $duration seconds ($mbps)"); - } - if (!defined($pbs_features) && $last_zero) { + if ($last_zero) { my $zero_per = $last_target ? int(($last_zero * 100)/$last_target) : 0; - my $zero_h = bytes_to_human($last_zero, 2); - $self->loginfo("Backup is sparse: ${zero_per}% ($zero_h) zero data"); + my $zero_h = render_bytes($last_zero); + $self->loginfo("backup is sparse: $zero_h (${zero_per}%) total zero data"); + } + if ($reused) { + my $reused_h = render_bytes($reused); + my $reuse_per = int($reused * 100 / $last_total); + $self->loginfo("backup was done incrementally, reused $reused_h (${reuse_per}%)"); + } + if ($transferred) { + my $transferred_h = render_bytes($transferred); + if ($duration) { + my $mbps = $get_mbps->($transferred, $duration); + $self->loginfo("transferred $transferred_h in $duration seconds ($mbps)"); + } else { + $self->loginfo("transferred $transferred_h in <1 seconds"); + } } return { @@ -416,6 +430,28 @@ my $query_backup_status_loop = sub { }; }; +my $attach_tpmstate_drive = sub { + my ($self, $task, $vmid) = @_; + + return if !$task->{tpmpath}; + + # unconditionally try to remove the tpmstate-named drive - it only exists + # for backing up, and avoids errors if left over from some previous event + eval { PVE::QemuServer::qemu_drivedel($vmid, "tpmstate0-backup"); }; + + $self->loginfo('attaching TPM drive to QEMU for backup'); + + my $drive = "file=$task->{tpmpath},if=none,read-only=on,id=drive-tpmstate0-backup"; + my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\""); + die "attaching TPM drive failed\n" if $ret !~ m/OK/s; +}; + +my $detach_tpmstate_drive = sub { + my ($task, $vmid) = @_; + return if !$task->{tpmpath} || !PVE::QemuServer::check_running($vmid); + eval { PVE::QemuServer::qemu_drivedel($vmid, "tpmstate0-backup"); }; +}; + sub archive_pbs { my ($self, $task, $vmid) = @_; @@ -427,30 +463,18 @@ sub archive_pbs { my $starttime = time(); - my $server = $scfg->{server}; - my $datastore = $scfg->{datastore}; - my $username = $scfg->{username} // 'root@pam'; my $fingerprint = $scfg->{fingerprint}; - - my $repo = "$username\@$server:$datastore"; + my $repo = PVE::PBSClient::get_repository($scfg); my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $opts->{storage}); my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($scfg, $opts->{storage}); + my $master_keyfile = PVE::Storage::PBSPlugin::pbs_master_pubkey_file_name($scfg, $opts->{storage}); my $diskcount = scalar(@{$task->{disks}}); - if (PVE::QemuConfig->is_template($self->{vmlist}->{$vmid}) || !$diskcount) { + # proxmox-backup-client can only handle raw files and block devs + # only use it (directly) for disk-less VMs + if (!$diskcount) { my @pathlist; - # FIXME: accumulate disk sizes to use for backup job (email) log - foreach my $di (@{$task->{disks}}) { - if ($di->{type} eq 'block' || $di->{type} eq 'file') { - push @pathlist, "$di->{qmdevice}.img:$di->{path}"; - } else { - die "implement me (type $di->{type})"; - } - } - - if (!$diskcount) { - $self->loginfo("backup contains no disks"); - } + $self->loginfo("backup contains no disks"); local $ENV{PBS_PASSWORD} = $password; local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint); @@ -462,10 +486,12 @@ sub archive_pbs { '--backup-id', "$vmid", '--backup-time', $task->{backup_time}, ]; + if (defined(my $ns = $scfg->{namespace})) { + push @$cmd, '--ns', $ns; + } push @$cmd, "qemu-server.conf:$conffile"; push @$cmd, "fw.conf:$firewall" if -e $firewall; - push @$cmd, @pathlist if scalar(@pathlist); $self->loginfo("starting template backup"); $self->loginfo(join(' ', @$cmd)); @@ -479,6 +505,7 @@ sub archive_pbs { my $devlist = _get_task_devlist($task); $self->enforce_vm_running_for_backup($vmid); + $self->{qmeventd_fh} = PVE::QemuServer::register_qmeventd_handle($vmid); my $backup_job_uuid; eval { @@ -487,11 +514,25 @@ sub archive_pbs { }; my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") }; - if (!$qemu_support) { - die "PBS backups are not supported by the running QEMU version. Please make " - . "sure you've installed the latest version and the VM has been restarted.\n"; + my $err = $@; + if (!$qemu_support || $err) { + die "query-proxmox-support returned empty value\n" if !$err; + if ($err =~ m/The command query-proxmox-support has not been found/) { + die "PBS backups are not supported by the running QEMU version. Please make " + . "sure you've installed the latest version and the VM has been restarted.\n"; + } else { + die "QMP command query-proxmox-support failed - $err\n"; + } } + if (!defined($qemu_support->{"pbs-masterkey"}) && -e $master_keyfile) { + $self->loginfo("WARNING: backup target is configured with master key, but running QEMU version does not support master keys."); + $self->loginfo("Please make sure you've installed the latest version and the VM has been restarted to use master key feature."); + $master_keyfile = undef; # skip rest of master key handling below + } + + $attach_tpmstate_drive->($self, $task, $vmid); + my $fs_frozen = $self->qga_fs_freeze($task, $vmid); my $params = { @@ -503,19 +544,31 @@ sub archive_pbs { devlist => $devlist, 'config-file' => $conffile, }; + if (defined(my $ns = $scfg->{namespace})) { + $params->{'backup-ns'} = $ns; + } + $params->{speed} = $opts->{bwlimit}*1024 if $opts->{bwlimit}; $params->{fingerprint} = $fingerprint if defined($fingerprint); $params->{'firewall-file'} = $firewall if -e $firewall; if (-e $keyfile) { $self->loginfo("enabling encryption"); $params->{keyfile} = $keyfile; $params->{encrypt} = JSON::true; + if (defined($master_keyfile) && -e $master_keyfile) { + $self->loginfo("enabling master key feature"); + $params->{"master-keyfile"} = $master_keyfile; + } } else { + $self->loginfo("WARNING: backup target is configured with master key, but this backup is not encrypted - master key settings will be ignored!") + if defined($master_keyfile) && -e $master_keyfile; $params->{encrypt} = JSON::false; } - $params->{'use-dirty-bitmap'} = JSON::true if $qemu_support->{'pbs-dirty-bitmap'}; + my $is_template = PVE::QemuConfig->is_template($self->{vmlist}->{$vmid}); + $params->{'use-dirty-bitmap'} = JSON::true + if $qemu_support->{'pbs-dirty-bitmap'} && !$is_template; - $params->{timeout} = 60; # give some time to connect to the backup server + $params->{timeout} = 125; # give some time to connect to the backup server my $res = eval { mon_cmd($vmid, "backup", %$params) }; my $qmperr = $@; @@ -538,7 +591,8 @@ sub archive_pbs { my $err = $@; if ($err) { $self->logerr($err); - $self->mon_backup_cancel($vmid) if defined($backup_job_uuid); + $self->mon_backup_cancel($vmid); + $self->resume_vm_after_job_start($task, $vmid); } $self->restore_vm_power_state($vmid); @@ -644,6 +698,7 @@ sub archive_vma { my $devlist = _get_task_devlist($task); $self->enforce_vm_running_for_backup($vmid); + $self->{qmeventd_fh} = PVE::QemuServer::register_qmeventd_handle($vmid); my $cpid; my $backup_job_uuid; @@ -653,6 +708,8 @@ sub archive_vma { die "interrupted by signal\n"; }; + $attach_tpmstate_drive->($self, $task, $vmid); + my $outfh; if ($opts->{stdout}) { $outfh = $opts->{stdout}; @@ -715,7 +772,8 @@ sub archive_vma { my $err = $@; if ($err) { $self->logerr($err); - $self->mon_backup_cancel($vmid) if defined($backup_job_uuid); + $self->mon_backup_cancel($vmid); + $self->resume_vm_after_job_start($task, $vmid); } $self->restore_vm_power_state($vmid); @@ -756,7 +814,7 @@ sub _get_task_devlist { sub qga_fs_freeze { my ($self, $task, $vmid) = @_; - return if !$self->{vmlist}->{$vmid}->{agent} || $task->{mode} eq 'stop' || !$self->{vm_was_running}; + return if !$self->{vmlist}->{$vmid}->{agent} || $task->{mode} eq 'stop' || !$self->{vm_was_running} || $self->{vm_was_paused}; if (!PVE::QemuServer::qga_check_running($vmid, 1)) { $self->loginfo("skipping guest-agent 'fs-freeze', agent configured but not running?"); @@ -794,6 +852,7 @@ sub enforce_vm_running_for_backup { # start with skiplock my $params = { skiplock => 1, + skiptemplate => 1, paused => 1, }; PVE::QemuServer::vm_start($self->{storecfg}, $vmid, $params); @@ -801,11 +860,11 @@ sub enforce_vm_running_for_backup { die $@ if $@; } -# resume VM againe once we got in a clear state (stop mode backup of running VM) +# resume VM again once in a clear state (stop mode backup of running VM) sub resume_vm_after_job_start { my ($self, $task, $vmid) = @_; - return if !$self->{vm_was_running}; + return if !$self->{vm_was_running} || $self->{vm_was_paused}; if (my $stoptime = $task->{vmstoptime}) { my $delay = time() - $task->{vmstoptime}; @@ -814,7 +873,7 @@ sub resume_vm_after_job_start { } else { $self->loginfo("resuming VM again"); } - mon_cmd($vmid, 'cont'); + mon_cmd($vmid, 'cont', timeout => 45); } # stop again if VM was not running before @@ -854,7 +913,11 @@ sub snapshot { sub cleanup { my ($self, $task, $vmid) = @_; - # nothing to do ? + $detach_tpmstate_drive->($task, $vmid); + + if ($self->{qmeventd_fh}) { + close($self->{qmeventd_fh}); + } } 1;