]> git.proxmox.com Git - qemu-server.git/blame - PVE/VZDump/QemuServer.pm
suspend: continue cleanup even if savevm-end QMP command fails
[qemu-server.git] / PVE / VZDump / QemuServer.pm
CommitLineData
1e3baf05
DM
1package PVE::VZDump::QemuServer;
2
1e3baf05
DM
3use strict;
4use warnings;
d610b145 5
1e3baf05 6use File::Basename;
d610b145
TL
7use File::Path;
8use IO::File;
9use IPC::Open3;
69e62894 10use JSON;
4ac842cb 11use POSIX qw(EINTR EAGAIN);
d610b145
TL
12
13use PVE::Cluster qw(cfs_read_file);
66ab1d91 14use PVE::INotify;
91bd6c90 15use PVE::IPCC;
d610b145 16use PVE::JSONSchema;
fbec3f89 17use PVE::PBSClient;
4de4eefc 18use PVE::RESTEnvironment qw(log_warn);
0a13e08e 19use PVE::QMPClient;
f5bdefa4 20use PVE::Storage::Plugin;
c5983223 21use PVE::Storage::PBSPlugin;
1e3baf05 22use PVE::Storage;
d610b145
TL
23use PVE::Tools;
24use PVE::VZDump;
f1aca33d 25use PVE::Format qw(render_duration render_bytes);
d610b145 26
912792e2 27use PVE::QemuConfig;
1e3baf05 28use PVE::QemuServer;
5d857282 29use PVE::QemuServer::Helpers;
3392d6ca 30use PVE::QemuServer::Machine;
0a13e08e 31use PVE::QemuServer::Monitor qw(mon_cmd);
1e3baf05
DM
32
33use base qw (PVE::VZDump::Plugin);
34
35sub new {
36 my ($class, $vzdump) = @_;
874a096e 37
66ab1d91 38 PVE::VZDump::check_bin('qm');
1e3baf05 39
e2812738 40 my $self = bless { vzdump => $vzdump }, $class;
1e3baf05
DM
41
42 $self->{vmlist} = PVE::QemuServer::vzlist();
43 $self->{storecfg} = PVE::Storage::config();
44
45 return $self;
46};
47
1e3baf05
DM
48sub type {
49 return 'qemu';
50}
51
52sub vmlist {
53 my ($self) = @_;
1e3baf05
DM
54 return [ keys %{$self->{vmlist}} ];
55}
56
57sub prepare {
58 my ($self, $task, $vmid, $mode) = @_;
59
60 $task->{disks} = [];
61
ffda963f 62 my $conf = $self->{vmlist}->{$vmid} = PVE::QemuConfig->load_config($vmid);
1e3baf05 63
85b84b7b
WL
64 $self->loginfo("VM Name: $conf->{name}")
65 if defined($conf->{name});
66
91bd6c90 67 $self->{vm_was_running} = 1;
b08c37c3 68 $self->{vm_was_paused} = 0;
91bd6c90
DM
69 if (!PVE::QemuServer::check_running($vmid)) {
70 $self->{vm_was_running} = 0;
6f0627d4
FS
71 } elsif (PVE::QemuServer::vm_is_paused($vmid, 0)) {
72 # Do not treat a suspended VM as paused, as it would cause us to skip
73 # fs-freeze even if the VM wakes up before we reach qga_fs_freeze.
b08c37c3 74 $self->{vm_was_paused} = 1;
585b6e28
DM
75 }
76
1e3baf05
DM
77 $task->{hostname} = $conf->{name};
78
874a096e 79 my $hostname = PVE::INotify::nodename();
1e3baf05 80
b969cc68
DM
81 my $vollist = [];
82 my $drivehash = {};
185df962 83 my $backup_volumes = PVE::QemuConfig->get_backup_volumes($conf);
1e3baf05 84
185df962
AL
85 foreach my $volume (@{$backup_volumes}) {
86 my $name = $volume->{key};
5a92276e 87 my $volume_config = $volume->{volume_config};
185df962 88 my $volid = $volume_config->{file};
1e3baf05 89
185df962
AL
90 if (!$volume->{included}) {
91 $self->loginfo("exclude disk '$name' '$volid' ($volume->{reason})");
92 next;
f9dde219
SR
93 } elsif ($self->{vm_was_running} && $volume_config->{iothread} &&
94 !PVE::QemuServer::Machine::runs_at_least_qemu_version($vmid, 4, 0, 1)) {
95 die "disk '$name' '$volid' (iothread=on) can't use backup feature with running QEMU " .
96 "version < 4.0.1! Either set backup=no for this drive or upgrade QEMU and restart VM\n";
b53b958b 97 } else {
185df962
AL
98 my $log = "include disk '$name' '$volid'";
99 if (defined(my $size = $volume_config->{size})) {
100 my $readable_size = PVE::JSONSchema::format_size($size);
0db93c2d 101 $log .= " $readable_size";
185df962 102 }
0db93c2d 103 $self->loginfo($log);
874a096e 104 }
b969cc68 105
f5bdefa4 106 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
b969cc68 107 push @$vollist, $volid if $storeid;
185df962
AL
108 $drivehash->{$name} = $volume->{volume_config};
109 }
b969cc68
DM
110
111 PVE::Storage::activate_volumes($self->{storecfg}, $vollist);
112
075b417a
DM
113 foreach my $ds (sort keys %$drivehash) {
114 my $drive = $drivehash->{$ds};
874a096e 115
1e3baf05 116 my $volid = $drive->{file};
f5bdefa4 117 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
a113a58a
TL
118
119 my $path = $volid;
1e3baf05 120 if ($storeid) {
b969cc68 121 $path = PVE::Storage::path($self->{storecfg}, $volid);
1e3baf05 122 }
b969cc68 123 next if !$path;
1e3baf05 124
670f947e
FE
125 my ($size, $format);
126 if ($storeid) {
127 # The call in list context can be expensive for certain plugins like RBD, just get size
128 $size = eval { PVE::Storage::volume_size_info($self->{storecfg}, $volid, 5) };
857e7b85 129 die "cannot determine size of volume '$volid' - $@\n" if $@;
91bd6c90 130
670f947e
FE
131 my $scfg = PVE::Storage::storage_config($self->{storecfg}, $storeid);
132 $format = PVE::QemuServer::qemu_img_format($scfg, $volname);
133 } else {
134 ($size, $format) = eval {
135 PVE::Storage::volume_size_info($self->{storecfg}, $volid, 5);
136 };
857e7b85 137 die "cannot determine size and format of volume '$volid' - $@\n" if $@;
670f947e 138 }
efa3aa24 139
a113a58a
TL
140 my $diskinfo = {
141 path => $path,
142 volid => $volid,
143 storeid => $storeid,
36377acf 144 size => $size,
a113a58a
TL
145 format => $format,
146 virtdev => $ds,
147 qmdevice => "drive-$ds",
148 };
1e3baf05 149
f9dde219
SR
150 if ($ds eq 'tpmstate0') {
151 # TPM drive only exists for backup, which is reflected in the name
152 $diskinfo->{qmdevice} = 'drive-tpmstate0-backup';
153 $task->{tpmpath} = $path;
154 }
155
1e3baf05 156 if (-b $path) {
1e3baf05 157 $diskinfo->{type} = 'block';
1e3baf05 158 } else {
1e3baf05 159 $diskinfo->{type} = 'file';
1e3baf05
DM
160 }
161
162 push @{$task->{disks}}, $diskinfo;
b969cc68 163 }
1e3baf05
DM
164}
165
166sub vm_status {
167 my ($self, $vmid) = @_;
168
66ab1d91 169 my $running = PVE::QemuServer::check_running($vmid) ? 1 : 0;
874a096e
DM
170
171 return wantarray ? ($running, $running ? 'running' : 'stopped') : $running;
1e3baf05
DM
172}
173
174sub lock_vm {
175 my ($self, $vmid) = @_;
176
f301bc0d 177 PVE::QemuConfig->set_lock($vmid, 'backup');
1e3baf05
DM
178}
179
180sub unlock_vm {
181 my ($self, $vmid) = @_;
182
f301bc0d 183 PVE::QemuConfig->remove_lock($vmid, 'backup');
1e3baf05
DM
184}
185
186sub stop_vm {
187 my ($self, $task, $vmid) = @_;
188
189 my $opts = $self->{vzdump}->{opts};
190
191 my $wait = $opts->{stopwait} * 60;
192 # send shutdown and wait
254575e9 193 $self->cmd ("qm shutdown $vmid --skiplock --keepActive --timeout $wait");
1e3baf05
DM
194}
195
196sub start_vm {
197 my ($self, $task, $vmid) = @_;
198
66ab1d91 199 $self->cmd ("qm start $vmid --skiplock");
1e3baf05
DM
200}
201
202sub suspend_vm {
203 my ($self, $task, $vmid) = @_;
204
b08c37c3
DC
205 return if $self->{vm_was_paused};
206
66ab1d91 207 $self->cmd ("qm suspend $vmid --skiplock");
1e3baf05
DM
208}
209
210sub resume_vm {
211 my ($self, $task, $vmid) = @_;
212
b08c37c3
DC
213 return if $self->{vm_was_paused};
214
66ab1d91 215 $self->cmd ("qm resume $vmid --skiplock");
1e3baf05
DM
216}
217
1e3baf05
DM
218sub assemble {
219 my ($self, $task, $vmid) = @_;
220
04096e7b 221 my $conffile = PVE::QemuConfig->config_file($vmid);
1e3baf05
DM
222
223 my $outfile = "$task->{tmpdir}/qemu-server.conf";
c05f7f3f
WL
224 my $firewall_src = "/etc/pve/firewall/$vmid.fw";
225 my $firewall_dest = "$task->{tmpdir}/qemu-server.fw";
226
b7bbeff3
FE
227 my $outfd = IO::File->new(">$outfile") or die "unable to open '$outfile' - $!\n";
228 my $conffd = IO::File->new($conffile, 'r') or die "unable to open '$conffile' - $!\n";
c05f7f3f
WL
229
230 my $found_snapshot;
391c2230 231 my $found_pending;
31280b4b 232 my $found_cloudinit;
c05f7f3f
WL
233 while (defined (my $line = <$conffd>)) {
234 next if $line =~ m/^\#vzdump\#/; # just to be sure
235 next if $line =~ m/^\#qmdump\#/; # just to be sure
391c2230
FG
236 if ($line =~ m/^\[(.*)\]\s*$/) {
237 if ($1 =~ m/PENDING/i) {
238 $found_pending = 1;
31280b4b
AD
239 } elsif ($1 =~ m/special:cloudinit/) {
240 $found_cloudinit = 1;
391c2230
FG
241 } else {
242 $found_snapshot = 1;
243 }
1e3baf05 244 }
31280b4b 245 next if $found_snapshot || $found_pending || $found_cloudinit; # skip all snapshots,pending changes and cloudinit config data
391c2230 246
c05f7f3f
WL
247 if ($line =~ m/^unused\d+:\s*(\S+)\s*/) {
248 $self->loginfo("skip unused drive '$1' (not included into backup)");
249 next;
1e3baf05 250 }
c05f7f3f 251 next if $line =~ m/^lock:/ || $line =~ m/^parent:/;
91bd6c90 252
c05f7f3f
WL
253 print $outfd $line;
254 }
255
256 foreach my $di (@{$task->{disks}}) {
257 if ($di->{type} eq 'block' || $di->{type} eq 'file') {
258 my $storeid = $di->{storeid} || '';
259 my $format = $di->{format} || '';
260 print $outfd "#qmdump#map:$di->{virtdev}:$di->{qmdevice}:$storeid:$format:\n";
261 } else {
262 die "internal error";
91bd6c90 263 }
c05f7f3f 264 }
1e3baf05 265
c05f7f3f
WL
266 if ($found_snapshot) {
267 $self->loginfo("snapshots found (not included into backup)");
268 }
391c2230
FG
269 if ($found_pending) {
270 $self->loginfo("pending configuration changes found (not included into backup)");
271 }
272
c05f7f3f 273 PVE::Tools::file_copy($firewall_src, $firewall_dest) if -f $firewall_src;
1e3baf05
DM
274}
275
276sub archive {
fad02a16 277 my ($self, $task, $vmid, $filename, $comp) = @_;
1e3baf05 278
c5983223 279 my $opts = $self->{vzdump}->{opts};
c5983223
DM
280 my $scfg = $opts->{scfg};
281
69074863 282 if ($self->{vzdump}->{opts}->{pbs}) {
c5983223
DM
283 $self->archive_pbs($task, $vmid);
284 } else {
285 $self->archive_vma($task, $vmid, $filename, $comp);
286 }
287}
288
3a44897f 289my $bitmap_action_to_human = sub {
13ddc7eb 290 my ($self, $info) = @_;
3a44897f
SR
291
292 my $action = $info->{action};
293
294 if ($action eq "not-used") {
962d4d64 295 return "disabled (no support)";
3a44897f
SR
296 } elsif ($action eq "not-used-removed") {
297 return "disabled (old bitmap cleared)";
298 } elsif ($action eq "new") {
13ddc7eb 299 return "created new";
3a44897f
SR
300 } elsif ($action eq "used") {
301 if ($info->{dirty} == 0) {
13ddc7eb 302 return "OK (drive clean)";
3a44897f 303 } else {
f1aca33d
SR
304 my $size = render_bytes($info->{size}, 1);
305 my $dirty = render_bytes($info->{dirty}, 1);
13ddc7eb 306 return "OK ($dirty of $size dirty)";
3a44897f
SR
307 }
308 } elsif ($action eq "invalid") {
309 return "existing bitmap was invalid and has been cleared";
310 } else {
311 return "unknown";
312 }
313};
314
c5983223 315my $query_backup_status_loop = sub {
f8566714 316 my ($self, $vmid, $job_uuid, $qemu_support) = @_;
c5983223 317
c5983223 318 my $starttime = time ();
09eb196b 319 my $last_time = $starttime;
8825248c 320 my ($last_percent, $last_total, $last_target, $last_zero, $last_transferred) = (-1, 0, 0, 0, 0);
4d159c24 321 my ($transferred, $reused);
c5983223 322
09eb196b
TL
323 my $get_mbps = sub {
324 my ($mb, $delta) = @_;
4d159c24
TL
325 return "0 B/s" if $mb <= 0;
326 my $bw = int(($mb / $delta));
f1aca33d 327 return render_bytes($bw, 1) . "/s";
09eb196b
TL
328 };
329
3a44897f 330 my $target = 0;
8825248c 331 my $last_reused = 0;
f8566714 332 my $has_query_bitmap = $qemu_support && $qemu_support->{'query-bitmap-info'};
5749c383 333 my $is_template = PVE::QemuConfig->is_template($self->{vmlist}->{$vmid});
f8566714 334 if ($has_query_bitmap) {
13ddc7eb 335 my $total = 0;
3a44897f 336 my $bitmap_info = mon_cmd($vmid, 'query-pbs-bitmap-info');
5749c383
TL
337 for my $info (sort { $a->{drive} cmp $b->{drive} } @$bitmap_info) {
338 if (!$is_template) {
339 my $text = $bitmap_action_to_human->($self, $info);
340 my $drive = $info->{drive};
341 $drive =~ s/^drive-//; # for consistency
342 $self->loginfo("$drive: dirty-bitmap status: $text");
343 }
3a44897f 344 $target += $info->{dirty};
13ddc7eb 345 $total += $info->{size};
8825248c 346 $last_reused += $info->{size} - $info->{dirty};
3a44897f 347 }
13ddc7eb 348 if ($target < $total) {
f1aca33d
SR
349 my $total_h = render_bytes($total, 1);
350 my $target_h = render_bytes($target, 1);
13ddc7eb
TL
351 $self->loginfo("using fast incremental mode (dirty-bitmap), $target_h dirty of $total_h total");
352 }
3a44897f
SR
353 }
354
1eb7e590 355 my $last_finishing = 0;
c5983223 356 while(1) {
09eb196b
TL
357 my $status = mon_cmd($vmid, 'query-backup');
358
c5983223 359 my $total = $status->{total} || 0;
f8566714
TL
360 my $dirty = $status->{dirty};
361 $target = (defined($dirty) && $dirty < $total) ? $dirty : $total if !$has_query_bitmap;
c5983223 362 $transferred = $status->{transferred} || 0;
4d159c24 363 $reused = $status->{reused};
0f6c6981 364 my $percent = $target ? int(($transferred * 100)/$target) : 100;
c5983223 365 my $zero = $status->{'zero-bytes'} || 0;
c5983223
DM
366
367 die "got unexpected uuid\n" if !$status->{uuid} || ($status->{uuid} ne $job_uuid);
368
369 my $ctime = time();
370 my $duration = $ctime - $starttime;
371
372 my $rbytes = $transferred - $last_transferred;
8825248c
SR
373 my $wbytes;
374 if ($reused) {
375 # reused includes zero bytes for PBS
376 $wbytes = $rbytes - ($reused - $last_reused);
377 } else {
378 $wbytes = $rbytes - ($zero - $last_zero);
379 }
c5983223
DM
380
381 my $timediff = ($ctime - $last_time) || 1; # fixme
09eb196b
TL
382 my $mbps_read = $get_mbps->($rbytes, $timediff);
383 my $mbps_write = $get_mbps->($wbytes, $timediff);
f1aca33d
SR
384 my $target_h = render_bytes($target, 1);
385 my $transferred_h = render_bytes($transferred, 1);
4d159c24 386
a40295b6 387 my $statusline = sprintf("%3d%% ($transferred_h of $target_h) in %s"
f1aca33d 388 .", read: $mbps_read, write: $mbps_write", $percent, render_duration($duration));
09eb196b 389
c5983223
DM
390 my $res = $status->{status} || 'unknown';
391 if ($res ne 'active') {
1eb7e590
SR
392 if ($last_percent < 100) {
393 $self->loginfo($statusline);
394 }
09eb196b
TL
395 if ($res ne 'done') {
396 die (($status->{errmsg} || "unknown error") . "\n") if $res eq 'error';
397 die "got unexpected status '$res'\n";
09eb196b 398 }
6cdb568c
TL
399 $last_target = $target if $target;
400 $last_total = $total if $total;
401 $last_zero = $zero if $zero;
402 $last_transferred = $transferred if $transferred;
c5983223
DM
403 last;
404 }
09eb196b 405 if ($percent != $last_percent && ($timediff > 2)) {
c5983223 406 $self->loginfo($statusline);
09eb196b 407 $last_percent = $percent;
4d159c24 408 $last_target = $target if $target;
c5983223
DM
409 $last_total = $total if $total;
410 $last_zero = $zero if $zero;
411 $last_transferred = $transferred if $transferred;
412 $last_time = $ctime;
8825248c 413 $last_reused = $reused;
1eb7e590
SR
414
415 if (!$last_finishing && $status->{finishing}) {
d5b0cfb1 416 $self->loginfo("Waiting for server to finish backup validation...");
1eb7e590
SR
417 }
418 $last_finishing = $status->{finishing};
c5983223
DM
419 }
420 sleep(1);
421 }
422
423 my $duration = time() - $starttime;
d35412a3
TL
424
425 if ($last_zero) {
426 my $zero_per = $last_target ? int(($last_zero * 100)/$last_target) : 0;
f1aca33d 427 my $zero_h = render_bytes($last_zero);
d35412a3
TL
428 $self->loginfo("backup is sparse: $zero_h (${zero_per}%) total zero data");
429 }
1ca43f1c 430 if ($reused) {
f1aca33d 431 my $reused_h = render_bytes($reused);
1ca43f1c
SR
432 my $reuse_per = int($reused * 100 / $last_total);
433 $self->loginfo("backup was done incrementally, reused $reused_h (${reuse_per}%)");
434 }
bafae3ec 435 if ($transferred) {
f1aca33d 436 my $transferred_h = render_bytes($transferred);
bafae3ec
TL
437 if ($duration) {
438 my $mbps = $get_mbps->($transferred, $duration);
439 $self->loginfo("transferred $transferred_h in $duration seconds ($mbps)");
440 } else {
441 $self->loginfo("transferred $transferred_h in <1 seconds");
442 }
4d159c24
TL
443 }
444
b4be9c02
TL
445 return {
446 total => $last_total,
447 reused => $reused,
448 };
c5983223
DM
449};
450
f9dde219
SR
451my $attach_tpmstate_drive = sub {
452 my ($self, $task, $vmid) = @_;
453
454 return if !$task->{tpmpath};
455
456 # unconditionally try to remove the tpmstate-named drive - it only exists
457 # for backing up, and avoids errors if left over from some previous event
458 eval { PVE::QemuServer::qemu_drivedel($vmid, "tpmstate0-backup"); };
459
460 $self->loginfo('attaching TPM drive to QEMU for backup');
461
462 my $drive = "file=$task->{tpmpath},if=none,read-only=on,id=drive-tpmstate0-backup";
93e3f756 463 $drive =~ s/\\/\\\\/g;
f9dde219 464 my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
df28f8cc 465 die "attaching TPM drive failed - $ret\n" if $ret !~ m/OK/s;
f9dde219
SR
466};
467
468my $detach_tpmstate_drive = sub {
469 my ($task, $vmid) = @_;
470 return if !$task->{tpmpath} || !PVE::QemuServer::check_running($vmid);
471 eval { PVE::QemuServer::qemu_drivedel($vmid, "tpmstate0-backup"); };
472};
473
4de4eefc
FE
474my sub add_backup_performance_options {
475 my ($qmp_param, $perf, $qemu_support) = @_;
476
477 return if !$perf || scalar(keys $perf->%*) == 0;
478
479 if (!$qemu_support) {
480 my $settings_string = join(', ', sort keys $perf->%*);
481 log_warn("ignoring setting(s): $settings_string - issue checking if supported");
482 return;
483 }
484
485 if (defined($perf->{'max-workers'})) {
486 if ($qemu_support->{'backup-max-workers'}) {
487 $qmp_param->{'max-workers'} = int($perf->{'max-workers'});
488 } else {
489 log_warn("ignoring 'max-workers' setting - not supported by running QEMU");
490 }
491 }
492}
493
3234f563
TL
494sub get_and_check_pbs_encryption_config {
495 my ($self) = @_;
496
497 my $opts = $self->{vzdump}->{opts};
498 my $scfg = $opts->{scfg};
499
500 my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($scfg, $opts->{storage});
501 my $master_keyfile = PVE::Storage::PBSPlugin::pbs_master_pubkey_file_name($scfg, $opts->{storage});
502
503 if (-e $keyfile) {
504 if (-e $master_keyfile) {
505 $self->loginfo("enabling encryption with master key feature");
506 return ($keyfile, $master_keyfile);
507 } elsif ($scfg->{'master-pubkey'}) {
508 die "master public key configured but no key file found\n";
509 } else {
510 $self->loginfo("enabling encryption");
511 return ($keyfile, undef);
512 }
513 } else {
514 my $encryption_fp = $scfg->{'encryption-key'};
515 die "encryption configured ('$encryption_fp') but no encryption key file found!\n"
516 if $encryption_fp;
517 if (-e $master_keyfile) {
518 $self->log(
519 'warn',
520 "backup target storage is configured with master-key, but no encryption key set!"
521 ." Ignoring master key settings and creating unencrypted backup."
522 );
523 }
524 return (undef, undef);
525 }
526 die "internal error - unhandled case for getting & checking PBS encryption ($keyfile, $master_keyfile)!";
527}
528
5d857282
FE
529my sub cleanup_fleecing_images {
530 my ($self, $disks) = @_;
531
532 for my $di ($disks->@*) {
533 if (my $volid = $di->{'fleece-volid'}) {
534 eval { PVE::Storage::vdisk_free($self->{storecfg}, $volid); };
535 $self->log('warn', "error removing fleecing image '$volid' - $@") if $@;
536 }
537 }
538}
539
540my sub allocate_fleecing_images {
541 my ($self, $disks, $vmid, $fleecing_storeid, $format) = @_;
542
543 die "internal error - no fleecing storage specified\n" if !$fleecing_storeid;
544
545 # TODO what about potential left-over images from a failed attempt? Just
546 # auto-remove? While unlikely, could conflict with manually created image from user...
547
548 eval {
549 my $n = 0; # counter for fleecing image names
550
551 for my $di ($disks->@*) {
552 next if $di->{virtdev} =~ m/^(?:tpmstate|efidisk)\d$/; # too small to be worth it
553 if ($di->{type} eq 'block' || $di->{type} eq 'file') {
554 my $scfg = PVE::Storage::storage_config($self->{storecfg}, $fleecing_storeid);
555 my $name = "vm-$vmid-fleece-$n";
556 $name .= ".$format" if $scfg->{path};
557
558 my $size = PVE::Tools::convert_size($di->{size}, 'b' => 'kb');
559
560 $di->{'fleece-volid'} = PVE::Storage::vdisk_alloc(
561 $self->{storecfg}, $fleecing_storeid, $vmid, $format, $name, $size);
562
563 $n++;
564 } else {
565 die "implement me (type '$di->{type}')";
566 }
567 }
568 };
569 if (my $err = $@) {
570 cleanup_fleecing_images($self, $disks);
571 die $err;
572 }
573}
574
575my sub detach_fleecing_images {
576 my ($disks, $vmid) = @_;
577
578 return if !PVE::QemuServer::Helpers::vm_running_locally($vmid);
579
580 for my $di ($disks->@*) {
581 if (my $volid = $di->{'fleece-volid'}) {
582 my $devid = "$di->{qmdevice}-fleecing";
583 $devid =~ s/^drive-//; # re-added by qemu_drivedel()
584 eval { PVE::QemuServer::qemu_drivedel($vmid, $devid) };
585 }
586 }
587}
588
589my sub attach_fleecing_images {
590 my ($self, $disks, $vmid, $format) = @_;
591
592 # unconditionally try to remove potential left-overs from a previous backup
593 detach_fleecing_images($disks, $vmid);
594
595 my $vollist = [ map { $_->{'fleece-volid'} } grep { $_->{'fleece-volid'} } $disks->@* ];
596 PVE::Storage::activate_volumes($self->{storecfg}, $vollist);
597
598 for my $di ($disks->@*) {
599 if (my $volid = $di->{'fleece-volid'}) {
600 $self->loginfo("$di->{qmdevice}: attaching fleecing image $volid to QEMU");
601
602 my $path = PVE::Storage::path($self->{storecfg}, $volid);
603 my $devid = "$di->{qmdevice}-fleecing";
604 my $drive = "file=$path,if=none,id=$devid,format=$format,discard=unmap";
605 # Specify size explicitly, to make it work if storage backend rounded up size for
606 # fleecing image when allocating.
607 $drive .= ",size=$di->{size}" if $format eq 'raw';
608 $drive =~ s/\\/\\\\/g;
609 my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
610 die "attaching fleecing image $volid failed - $ret\n" if $ret !~ m/OK/s;
611 }
612 }
613}
614
615my sub check_and_prepare_fleecing {
616 my ($self, $vmid, $fleecing_opts, $disks, $is_template, $qemu_support) = @_;
617
618 # Even if the VM was started specifically for fleecing, it's possible that the VM is resumed and
619 # then starts doing IO. For VMs that are not resumed the fleecing images will just stay empty,
620 # so there is no big cost.
621
622 my $use_fleecing = $fleecing_opts && $fleecing_opts->{enabled} && !$is_template;
623
624 if ($use_fleecing && !defined($qemu_support->{'backup-fleecing'})) {
625 $self->log(
626 'warn',
627 "running QEMU version does not support backup fleecing - continuing without",
628 );
629 $use_fleecing = 0;
630 }
631
632 if ($use_fleecing) {
633 my ($default_format, $valid_formats) = PVE::Storage::storage_default_format(
634 $self->{storecfg}, $fleecing_opts->{storage});
635 my $format = scalar(grep { $_ eq 'qcow2' } $valid_formats->@*) ? 'qcow2' : 'raw';
636
637 allocate_fleecing_images($self, $disks, $vmid, $fleecing_opts->{storage}, $format);
638 attach_fleecing_images($self, $disks, $vmid, $format);
639 }
640
641 return $use_fleecing;
642}
643
c5983223
DM
644sub archive_pbs {
645 my ($self, $task, $vmid) = @_;
646
1e3baf05 647 my $conffile = "$task->{tmpdir}/qemu-server.conf";
c05f7f3f 648 my $firewall = "$task->{tmpdir}/qemu-server.fw";
1e3baf05
DM
649
650 my $opts = $self->{vzdump}->{opts};
c5983223
DM
651 my $scfg = $opts->{scfg};
652
653 my $starttime = time();
654
c5983223 655 my $fingerprint = $scfg->{fingerprint};
fbec3f89 656 my $repo = PVE::PBSClient::get_repository($scfg);
c5983223 657 my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $opts->{storage});
3234f563 658 my ($keyfile, $master_keyfile) = $self->get_and_check_pbs_encryption_config();
c5983223 659
d11e91d2 660 my $diskcount = scalar(@{$task->{disks}});
b241ddbf
TL
661 # proxmox-backup-client can only handle raw files and block devs, so only use it (directly) for
662 # disk-less VMs
4ef13a7f 663 if (!$diskcount) {
4ef13a7f 664 $self->loginfo("backup contains no disks");
c5983223
DM
665
666 local $ENV{PBS_PASSWORD} = $password;
77b24c62 667 local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
c5983223
DM
668 my $cmd = [
669 '/usr/bin/proxmox-backup-client',
670 'backup',
671 '--repository', $repo,
672 '--backup-type', 'vm',
673 '--backup-id', "$vmid",
674 '--backup-time', $task->{backup_time},
d11e91d2 675 ];
21a9ec2a
WB
676 if (defined(my $ns = $scfg->{namespace})) {
677 push @$cmd, '--ns', $ns;
678 }
3234f563 679 if (defined($keyfile)) {
fbd3dde7 680 push @$cmd, '--keyfile', $keyfile;
3234f563 681 push @$cmd, '--master-pubkey-file', $master_keyfile if defined($master_keyfile);
fbd3dde7 682 }
c5983223
DM
683
684 push @$cmd, "qemu-server.conf:$conffile";
685 push @$cmd, "fw.conf:$firewall" if -e $firewall;
c5983223
DM
686
687 $self->loginfo("starting template backup");
688 $self->loginfo(join(' ', @$cmd));
689
690 $self->cmd($cmd);
691
692 return;
693 }
694
81dcd479
TL
695 # get list early so we die on unkown drive types before doing anything
696 my $devlist = _get_task_devlist($task);
5d857282 697 my $use_fleecing;
c5983223 698
0b2f574b 699 $self->enforce_vm_running_for_backup($vmid);
65911545 700 $self->{qmeventd_fh} = PVE::QemuServer::register_qmeventd_handle($vmid);
c5983223 701
f6168f1a 702 my $backup_job_uuid;
c5983223
DM
703 eval {
704 $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
d11e91d2 705 die "interrupted by signal\n";
c5983223
DM
706 };
707
fb9f512c 708 my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
2cfb0905
SR
709 my $err = $@;
710 if (!$qemu_support || $err) {
711 die "query-proxmox-support returned empty value\n" if !$err;
712 if ($err =~ m/The command query-proxmox-support has not been found/) {
713 die "PBS backups are not supported by the running QEMU version. Please make "
714 . "sure you've installed the latest version and the VM has been restarted.\n";
715 } else {
716 die "QMP command query-proxmox-support failed - $err\n";
717 }
fb9f512c
SR
718 }
719
781fb803
TL
720 # pve-qemu supports it since 5.2.0-1 (PVE 6.4), so safe to die since PVE 8
721 die "master key configured but running QEMU version does not support master keys\n"
3234f563 722 if !defined($qemu_support->{'pbs-masterkey'}) && defined($master_keyfile);
9bf522bc 723
f9dde219
SR
724 $attach_tpmstate_drive->($self, $task, $vmid);
725
5d857282
FE
726 my $is_template = PVE::QemuConfig->is_template($self->{vmlist}->{$vmid});
727
728 $use_fleecing = check_and_prepare_fleecing(
729 $self, $vmid, $opts->{fleecing}, $task->{disks}, $is_template, $qemu_support);
730
cb521f2d 731 my $fs_frozen = $self->qga_fs_freeze($task, $vmid);
c5983223 732
d11e91d2
TL
733 my $params = {
734 format => "pbs",
735 'backup-file' => $repo,
736 'backup-id' => "$vmid",
737 'backup-time' => $task->{backup_time},
738 password => $password,
739 devlist => $devlist,
740 'config-file' => $conffile,
c5983223 741 };
5d857282
FE
742 $params->{fleecing} = JSON::true if $use_fleecing;
743
21a9ec2a 744 if (defined(my $ns = $scfg->{namespace})) {
0c9a94d2 745 $params->{'backup-ns'} = $ns;
21a9ec2a 746 }
4de4eefc 747
2790636a 748 $params->{speed} = $opts->{bwlimit}*1024 if $opts->{bwlimit};
4de4eefc
FE
749 add_backup_performance_options($params, $opts->{performance}, $qemu_support);
750
d11e91d2
TL
751 $params->{fingerprint} = $fingerprint if defined($fingerprint);
752 $params->{'firewall-file'} = $firewall if -e $firewall;
3234f563
TL
753
754 $params->{encrypt} = defined($keyfile) ? JSON::true : JSON::false;
755 if (defined($keyfile)) {
ece74030 756 $params->{keyfile} = $keyfile;
3234f563 757 $params->{"master-keyfile"} = $master_keyfile if defined($master_keyfile);
ece74030 758 }
c5983223 759
78179bda 760 $params->{'use-dirty-bitmap'} = JSON::true
962d4d64 761 if $qemu_support->{'pbs-dirty-bitmap'} && !$is_template;
fb9f512c 762
46b676c0 763 $params->{timeout} = 125; # give some time to connect to the backup server
f57666e9 764
d11e91d2 765 my $res = eval { mon_cmd($vmid, "backup", %$params) };
c5983223 766 my $qmperr = $@;
d11e91d2 767 $backup_job_uuid = $res->{UUID} if $res;
c5983223 768
1ece829a
TL
769 if ($fs_frozen) {
770 $self->qga_fs_thaw($vmid);
c5983223
DM
771 }
772
773 die $qmperr if $qmperr;
f6168f1a 774 die "got no uuid for backup task\n" if !defined($backup_job_uuid);
c5983223 775
f6168f1a 776 $self->loginfo("started backup task '$backup_job_uuid'");
c5983223 777
0b2f574b 778 $self->resume_vm_after_job_start($task, $vmid);
c5983223 779
3a44897f 780 my $stat = $query_backup_status_loop->($self, $vmid, $backup_job_uuid, $qemu_support);
2098f2ff 781 $task->{size} = $stat->{total};
c5983223
DM
782 };
783 my $err = $@;
c5983223
DM
784 if ($err) {
785 $self->logerr($err);
edae1718 786 $self->mon_backup_cancel($vmid);
ebce5239 787 $self->resume_vm_after_job_start($task, $vmid);
c5983223 788 }
0b2f574b 789 $self->restore_vm_power_state($vmid);
c5983223 790
5d857282
FE
791 if ($use_fleecing) {
792 detach_fleecing_images($task->{disks}, $vmid);
793 cleanup_fleecing_images($self, $task->{disks});
794 }
795
c5983223
DM
796 die $err if $err;
797}
798
02da0c65
TL
799my $fork_compressor_pipe = sub {
800 my ($self, $comp, $outfileno) = @_;
801
802 my @pipefd = POSIX::pipe();
803 my $cpid = fork();
804 die "unable to fork worker - $!" if !defined($cpid) || $cpid < 0;
805 if ($cpid == 0) {
806 eval {
807 POSIX::close($pipefd[1]);
808 # redirect STDIN
809 my $fd = fileno(STDIN);
810 close STDIN;
811 POSIX::close(0) if $fd != 0;
812 die "unable to redirect STDIN - $!"
813 if !open(STDIN, "<&", $pipefd[0]);
814
815 # redirect STDOUT
816 $fd = fileno(STDOUT);
817 close STDOUT;
818 POSIX::close (1) if $fd != 1;
819
820 die "unable to redirect STDOUT - $!"
821 if !open(STDOUT, ">&", $outfileno);
822
823 exec($comp);
824 die "fork compressor '$comp' failed\n";
825 };
826 if (my $err = $@) {
827 $self->logerr($err);
828 POSIX::_exit(1);
829 }
830 POSIX::_exit(0);
831 kill(-9, $$);
832 } else {
833 POSIX::close($pipefd[0]);
834 $outfileno = $pipefd[1];
835 }
836
837 return ($cpid, $outfileno);
838};
839
c5983223
DM
840sub archive_vma {
841 my ($self, $task, $vmid, $filename, $comp) = @_;
842
843 my $conffile = "$task->{tmpdir}/qemu-server.conf";
844 my $firewall = "$task->{tmpdir}/qemu-server.fw";
845
846 my $opts = $self->{vzdump}->{opts};
847
848 my $starttime = time();
1e3baf05 849
91bd6c90
DM
850 my $speed = 0;
851 if ($opts->{bwlimit}) {
874a096e 852 $speed = $opts->{bwlimit}*1024;
91bd6c90 853 }
1e3baf05 854
5d857282
FE
855 my $is_template = PVE::QemuConfig->is_template($self->{vmlist}->{$vmid});
856
c82935e9 857 my $diskcount = scalar(@{$task->{disks}});
5d857282 858 if ($is_template || !$diskcount) {
23b4120b
DM
859 my @pathlist;
860 foreach my $di (@{$task->{disks}}) {
861 if ($di->{type} eq 'block' || $di->{type} eq 'file') {
862 push @pathlist, "$di->{qmdevice}=$di->{path}";
863 } else {
864 die "implement me";
865 }
866 }
867
c82935e9
DM
868 if (!$diskcount) {
869 $self->loginfo("backup contains no disks");
870 }
871
23b4120b
DM
872 my $outcmd;
873 if ($comp) {
874a096e 874 $outcmd = "exec:$comp";
23b4120b 875 } else {
874a096e 876 $outcmd = "exec:cat";
23b4120b
DM
877 }
878
a2fab11a 879 $outcmd .= " > $filename" if !$opts->{stdout};
23b4120b 880
c05f7f3f
WL
881 my $cmd = ['/usr/bin/vma', 'create', '-v', '-c', $conffile];
882 push @$cmd, '-c', $firewall if -e $firewall;
883 push @$cmd, $outcmd, @pathlist;
23b4120b
DM
884
885 $self->loginfo("starting template backup");
886 $self->loginfo(join(' ', @$cmd));
887
888 if ($opts->{stdout}) {
6bb12239 889 $self->cmd($cmd, output => ">&" . fileno($opts->{stdout}));
23b4120b
DM
890 } else {
891 $self->cmd($cmd);
892 }
893
894 return;
895 }
896
81dcd479 897 my $devlist = _get_task_devlist($task);
5d857282 898 my $use_fleecing;
1e3baf05 899
0b2f574b 900 $self->enforce_vm_running_for_backup($vmid);
65911545 901 $self->{qmeventd_fh} = PVE::QemuServer::register_qmeventd_handle($vmid);
91bd6c90
DM
902
903 my $cpid;
d2cc2cbe
DM
904 my $backup_job_uuid;
905
91bd6c90
DM
906 eval {
907 $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
d11e91d2 908 die "interrupted by signal\n";
91bd6c90
DM
909 };
910
4de4eefc
FE
911 # Currently, failing to determine Proxmox support is not critical here, because it's only
912 # used for performance settings like 'max-workers'.
913 my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
914 log_warn($@) if $@;
915
f9dde219
SR
916 $attach_tpmstate_drive->($self, $task, $vmid);
917
5d857282
FE
918 $use_fleecing = check_and_prepare_fleecing(
919 $self, $vmid, $opts->{fleecing}, $task->{disks}, $is_template, $qemu_support);
920
91bd6c90
DM
921 my $outfh;
922 if ($opts->{stdout}) {
923 $outfh = $opts->{stdout};
924 } else {
925 $outfh = IO::File->new($filename, "w") ||
926 die "unable to open file '$filename' - $!\n";
927 }
02da0c65 928 my $outfileno = fileno($outfh);
91bd6c90 929
91bd6c90 930 if ($comp) {
02da0c65 931 ($cpid, $outfileno) = $fork_compressor_pipe->($self, $comp, $outfileno);
91bd6c90
DM
932 }
933
e8705fc5
TL
934 my $qmpclient = PVE::QMPClient->new();
935 my $backup_cb = sub {
936 my ($vmid, $resp) = @_;
937 $backup_job_uuid = $resp->{return}->{UUID};
938 };
939 my $add_fd_cb = sub {
91bd6c90
DM
940 my ($vmid, $resp) = @_;
941
c05f7f3f
WL
942 my $params = {
943 'backup-file' => "/dev/fdname/backup",
944 speed => $speed,
945 'config-file' => $conffile,
946 devlist => $devlist
947 };
c05f7f3f 948 $params->{'firewall-file'} = $firewall if -e $firewall;
5d857282 949 $params->{fleecing} = JSON::true if $use_fleecing;
4de4eefc 950 add_backup_performance_options($params, $opts->{performance}, $qemu_support);
d11e91d2 951
c05f7f3f 952 $qmpclient->queue_cmd($vmid, $backup_cb, 'backup', %$params);
91bd6c90
DM
953 };
954
d11e91d2 955 $qmpclient->queue_cmd($vmid, $add_fd_cb, 'getfd', fd => $outfileno, fdname => "backup");
ab6a9a0c 956
cb521f2d 957 my $fs_frozen = $self->qga_fs_freeze($task, $vmid);
874a096e 958
c5983223 959 eval { $qmpclient->queue_execute(30) };
f0f30448 960 my $qmperr = $@;
91bd6c90 961
1ece829a
TL
962 if ($fs_frozen) {
963 $self->qga_fs_thaw($vmid);
ab6a9a0c 964 }
d11e91d2 965
f0f30448 966 die $qmperr if $qmperr;
874a096e 967 die $qmpclient->{errors}->{$vmid} if $qmpclient->{errors}->{$vmid};
91bd6c90
DM
968
969 if ($cpid) {
874a096e 970 POSIX::close($outfileno) == 0 ||
91bd6c90
DM
971 die "close output file handle failed\n";
972 }
973
d2cc2cbe 974 die "got no uuid for backup task\n" if !defined($backup_job_uuid);
91bd6c90 975
d2cc2cbe 976 $self->loginfo("started backup task '$backup_job_uuid'");
91bd6c90 977
0b2f574b 978 $self->resume_vm_after_job_start($task, $vmid);
91bd6c90 979
d2cc2cbe 980 $query_backup_status_loop->($self, $vmid, $backup_job_uuid);
91bd6c90
DM
981 };
982 my $err = $@;
19599cd9 983 if ($err) {
60635a57 984 $self->logerr($err);
edae1718 985 $self->mon_backup_cancel($vmid);
ebce5239 986 $self->resume_vm_after_job_start($task, $vmid);
19599cd9
DM
987 }
988
0b2f574b 989 $self->restore_vm_power_state($vmid);
91bd6c90 990
5d857282
FE
991 if ($use_fleecing) {
992 detach_fleecing_images($task->{disks}, $vmid);
993 cleanup_fleecing_images($self, $task->{disks});
994 }
995
91bd6c90 996 if ($err) {
874a096e
DM
997 if ($cpid) {
998 kill(9, $cpid);
91bd6c90
DM
999 waitpid($cpid, 0);
1000 }
1001 die $err;
1002 }
1003
1004 if ($cpid && (waitpid($cpid, 0) > 0)) {
1005 my $stat = $?;
1006 my $ec = $stat >> 8;
1007 my $signal = $stat & 127;
1008 if ($ec || $signal) {
874a096e 1009 die "$comp failed - wrong exit status $ec" .
91bd6c90
DM
1010 ($signal ? " (signal $signal)\n" : "\n");
1011 }
1012 }
1013}
1014
81dcd479
TL
1015sub _get_task_devlist {
1016 my ($task) = @_;
1017
1018 my $devlist = '';
1019 foreach my $di (@{$task->{disks}}) {
1020 if ($di->{type} eq 'block' || $di->{type} eq 'file') {
1021 $devlist .= ',' if $devlist;
1022 $devlist .= $di->{qmdevice};
1023 } else {
1024 die "implement me (type '$di->{type}')";
1025 }
1026 }
1027 return $devlist;
1028}
1029
1ece829a 1030sub qga_fs_freeze {
cb521f2d 1031 my ($self, $task, $vmid) = @_;
b08c37c3 1032 return if !$self->{vmlist}->{$vmid}->{agent} || $task->{mode} eq 'stop' || !$self->{vm_was_running} || $self->{vm_was_paused};
1ece829a
TL
1033
1034 if (!PVE::QemuServer::qga_check_running($vmid, 1)) {
1035 $self->loginfo("skipping guest-agent 'fs-freeze', agent configured but not running?");
1036 return;
1037 }
1038
93e21fd2
CH
1039 my $freeze = PVE::QemuServer::get_qga_key($self->{vmlist}->{$vmid}, 'freeze-fs-on-backup') // 1;
1040 if (!$freeze) {
1041 $self->loginfo("skipping guest-agent 'fs-freeze', disabled in VM options");
1042 return;
1043 }
1044
1ece829a
TL
1045 $self->loginfo("issuing guest-agent 'fs-freeze' command");
1046 eval { mon_cmd($vmid, "guest-fsfreeze-freeze") };
1047 $self->logerr($@) if $@;
1048
1049 return 1; # even on mon command error, ensure we always thaw again
1050}
1051
1052# only call if fs_freeze return 1
1053sub qga_fs_thaw {
1054 my ($self, $vmid) = @_;
1055
1056 $self->loginfo("issuing guest-agent 'fs-thaw' command");
1057 eval { mon_cmd($vmid, "guest-fsfreeze-thaw") };
1058 $self->logerr($@) if $@;
1059}
1060
0b2f574b
TL
1061# we need a running QEMU/KVM process for backup, starts a paused (prelaunch)
1062# one if VM isn't already running
1063sub enforce_vm_running_for_backup {
1064 my ($self, $vmid) = @_;
1065
1066 if (PVE::QemuServer::check_running($vmid)) {
1067 $self->{vm_was_running} = 1;
1068 return;
1069 }
1070
1071 eval {
1072 $self->loginfo("starting kvm to execute backup task");
1073 # start with skiplock
0c498cca
FG
1074 my $params = {
1075 skiplock => 1,
4ef13a7f 1076 skiptemplate => 1,
0c498cca
FG
1077 paused => 1,
1078 };
1079 PVE::QemuServer::vm_start($self->{storecfg}, $vmid, $params);
0b2f574b
TL
1080 };
1081 die $@ if $@;
1082}
1083
ebce5239 1084# resume VM again once in a clear state (stop mode backup of running VM)
0b2f574b
TL
1085sub resume_vm_after_job_start {
1086 my ($self, $task, $vmid) = @_;
1087
b08c37c3 1088 return if !$self->{vm_was_running} || $self->{vm_was_paused};
0b2f574b
TL
1089
1090 if (my $stoptime = $task->{vmstoptime}) {
1091 my $delay = time() - $task->{vmstoptime};
1092 $task->{vmstoptime} = undef; # avoid printing 'online after ..' twice
1093 $self->loginfo("resuming VM again after $delay seconds");
1094 } else {
1095 $self->loginfo("resuming VM again");
1096 }
6a558901 1097 mon_cmd($vmid, 'cont', timeout => 45);
0b2f574b
TL
1098}
1099
1100# stop again if VM was not running before
1101sub restore_vm_power_state {
1102 my ($self, $vmid) = @_;
1103
1104 # we always let VMs keep running
1105 return if $self->{vm_was_running};
1106
1107 eval {
1108 my $resp = mon_cmd($vmid, 'query-status');
1109 my $status = $resp && $resp->{status} ? $resp->{status} : 'unknown';
1110 if ($status eq 'prelaunch') {
1111 $self->loginfo("stopping kvm after backup task");
1112 PVE::QemuServer::vm_stop($self->{storecfg}, $vmid, 1);
1113 } else {
1114 $self->loginfo("kvm status changed after backup ('$status') - keep VM running");
1115 }
1116 };
1117 warn $@ if $@;
1118}
1119
1120sub mon_backup_cancel {
1121 my ($self, $vmid) = @_;
1122
1123 $self->loginfo("aborting backup job");
1124 eval { mon_cmd($vmid, 'backup-cancel') };
1125 $self->logerr($@) if $@;
1126}
1127
91bd6c90
DM
1128sub snapshot {
1129 my ($self, $task, $vmid) = @_;
1130
1131 # nothing to do
1e3baf05
DM
1132}
1133
1134sub cleanup {
1135 my ($self, $task, $vmid) = @_;
1136
f9dde219
SR
1137 $detach_tpmstate_drive->($task, $vmid);
1138
4ac842cb
SR
1139 if ($self->{qmeventd_fh}) {
1140 close($self->{qmeventd_fh});
1141 }
1e3baf05
DM
1142}
1143
11441;