]>
Commit | Line | Data |
---|---|---|
1e3baf05 DM |
1 | package PVE::VZDump::QemuServer; |
2 | ||
1e3baf05 DM |
3 | use strict; |
4 | use warnings; | |
d610b145 | 5 | |
1e3baf05 | 6 | use File::Basename; |
d610b145 TL |
7 | use File::Path; |
8 | use IO::File; | |
9 | use IPC::Open3; | |
69e62894 | 10 | use JSON; |
4ac842cb | 11 | use POSIX qw(EINTR EAGAIN); |
d610b145 TL |
12 | |
13 | use PVE::Cluster qw(cfs_read_file); | |
66ab1d91 | 14 | use PVE::INotify; |
91bd6c90 | 15 | use PVE::IPCC; |
d610b145 | 16 | use PVE::JSONSchema; |
fbec3f89 | 17 | use PVE::PBSClient; |
4de4eefc | 18 | use PVE::RESTEnvironment qw(log_warn); |
0a13e08e | 19 | use PVE::QMPClient; |
f5bdefa4 | 20 | use PVE::Storage::Plugin; |
c5983223 | 21 | use PVE::Storage::PBSPlugin; |
1e3baf05 | 22 | use PVE::Storage; |
d610b145 TL |
23 | use PVE::Tools; |
24 | use PVE::VZDump; | |
f1aca33d | 25 | use PVE::Format qw(render_duration render_bytes); |
d610b145 | 26 | |
912792e2 | 27 | use PVE::QemuConfig; |
1e3baf05 | 28 | use PVE::QemuServer; |
5d857282 | 29 | use PVE::QemuServer::Helpers; |
3392d6ca | 30 | use PVE::QemuServer::Machine; |
0a13e08e | 31 | use PVE::QemuServer::Monitor qw(mon_cmd); |
1e3baf05 DM |
32 | |
33 | use base qw (PVE::VZDump::Plugin); | |
34 | ||
35 | sub new { | |
36 | my ($class, $vzdump) = @_; | |
874a096e | 37 | |
66ab1d91 | 38 | PVE::VZDump::check_bin('qm'); |
1e3baf05 | 39 | |
e2812738 | 40 | my $self = bless { vzdump => $vzdump }, $class; |
1e3baf05 DM |
41 | |
42 | $self->{vmlist} = PVE::QemuServer::vzlist(); | |
43 | $self->{storecfg} = PVE::Storage::config(); | |
44 | ||
45 | return $self; | |
46 | }; | |
47 | ||
1e3baf05 DM |
48 | sub type { |
49 | return 'qemu'; | |
50 | } | |
51 | ||
52 | sub vmlist { | |
53 | my ($self) = @_; | |
1e3baf05 DM |
54 | return [ keys %{$self->{vmlist}} ]; |
55 | } | |
56 | ||
57 | sub prepare { | |
58 | my ($self, $task, $vmid, $mode) = @_; | |
59 | ||
60 | $task->{disks} = []; | |
61 | ||
ffda963f | 62 | my $conf = $self->{vmlist}->{$vmid} = PVE::QemuConfig->load_config($vmid); |
1e3baf05 | 63 | |
85b84b7b WL |
64 | $self->loginfo("VM Name: $conf->{name}") |
65 | if defined($conf->{name}); | |
66 | ||
91bd6c90 | 67 | $self->{vm_was_running} = 1; |
b08c37c3 | 68 | $self->{vm_was_paused} = 0; |
91bd6c90 DM |
69 | if (!PVE::QemuServer::check_running($vmid)) { |
70 | $self->{vm_was_running} = 0; | |
6f0627d4 FS |
71 | } elsif (PVE::QemuServer::vm_is_paused($vmid, 0)) { |
72 | # Do not treat a suspended VM as paused, as it would cause us to skip | |
73 | # fs-freeze even if the VM wakes up before we reach qga_fs_freeze. | |
b08c37c3 | 74 | $self->{vm_was_paused} = 1; |
585b6e28 DM |
75 | } |
76 | ||
1e3baf05 DM |
77 | $task->{hostname} = $conf->{name}; |
78 | ||
874a096e | 79 | my $hostname = PVE::INotify::nodename(); |
1e3baf05 | 80 | |
b969cc68 DM |
81 | my $vollist = []; |
82 | my $drivehash = {}; | |
185df962 | 83 | my $backup_volumes = PVE::QemuConfig->get_backup_volumes($conf); |
1e3baf05 | 84 | |
185df962 AL |
85 | foreach my $volume (@{$backup_volumes}) { |
86 | my $name = $volume->{key}; | |
5a92276e | 87 | my $volume_config = $volume->{volume_config}; |
185df962 | 88 | my $volid = $volume_config->{file}; |
1e3baf05 | 89 | |
185df962 AL |
90 | if (!$volume->{included}) { |
91 | $self->loginfo("exclude disk '$name' '$volid' ($volume->{reason})"); | |
92 | next; | |
f9dde219 SR |
93 | } elsif ($self->{vm_was_running} && $volume_config->{iothread} && |
94 | !PVE::QemuServer::Machine::runs_at_least_qemu_version($vmid, 4, 0, 1)) { | |
95 | die "disk '$name' '$volid' (iothread=on) can't use backup feature with running QEMU " . | |
96 | "version < 4.0.1! Either set backup=no for this drive or upgrade QEMU and restart VM\n"; | |
b53b958b | 97 | } else { |
185df962 AL |
98 | my $log = "include disk '$name' '$volid'"; |
99 | if (defined(my $size = $volume_config->{size})) { | |
100 | my $readable_size = PVE::JSONSchema::format_size($size); | |
0db93c2d | 101 | $log .= " $readable_size"; |
185df962 | 102 | } |
0db93c2d | 103 | $self->loginfo($log); |
874a096e | 104 | } |
b969cc68 | 105 | |
f5bdefa4 | 106 | my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1); |
b969cc68 | 107 | push @$vollist, $volid if $storeid; |
185df962 AL |
108 | $drivehash->{$name} = $volume->{volume_config}; |
109 | } | |
b969cc68 DM |
110 | |
111 | PVE::Storage::activate_volumes($self->{storecfg}, $vollist); | |
112 | ||
075b417a DM |
113 | foreach my $ds (sort keys %$drivehash) { |
114 | my $drive = $drivehash->{$ds}; | |
874a096e | 115 | |
1e3baf05 | 116 | my $volid = $drive->{file}; |
f5bdefa4 | 117 | my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1); |
a113a58a TL |
118 | |
119 | my $path = $volid; | |
1e3baf05 | 120 | if ($storeid) { |
b969cc68 | 121 | $path = PVE::Storage::path($self->{storecfg}, $volid); |
1e3baf05 | 122 | } |
b969cc68 | 123 | next if !$path; |
1e3baf05 | 124 | |
670f947e FE |
125 | my ($size, $format); |
126 | if ($storeid) { | |
127 | # The call in list context can be expensive for certain plugins like RBD, just get size | |
128 | $size = eval { PVE::Storage::volume_size_info($self->{storecfg}, $volid, 5) }; | |
857e7b85 | 129 | die "cannot determine size of volume '$volid' - $@\n" if $@; |
91bd6c90 | 130 | |
670f947e FE |
131 | my $scfg = PVE::Storage::storage_config($self->{storecfg}, $storeid); |
132 | $format = PVE::QemuServer::qemu_img_format($scfg, $volname); | |
133 | } else { | |
134 | ($size, $format) = eval { | |
135 | PVE::Storage::volume_size_info($self->{storecfg}, $volid, 5); | |
136 | }; | |
857e7b85 | 137 | die "cannot determine size and format of volume '$volid' - $@\n" if $@; |
670f947e | 138 | } |
efa3aa24 | 139 | |
a113a58a TL |
140 | my $diskinfo = { |
141 | path => $path, | |
142 | volid => $volid, | |
143 | storeid => $storeid, | |
36377acf | 144 | size => $size, |
a113a58a TL |
145 | format => $format, |
146 | virtdev => $ds, | |
147 | qmdevice => "drive-$ds", | |
148 | }; | |
1e3baf05 | 149 | |
f9dde219 SR |
150 | if ($ds eq 'tpmstate0') { |
151 | # TPM drive only exists for backup, which is reflected in the name | |
152 | $diskinfo->{qmdevice} = 'drive-tpmstate0-backup'; | |
153 | $task->{tpmpath} = $path; | |
154 | } | |
155 | ||
1e3baf05 | 156 | if (-b $path) { |
1e3baf05 | 157 | $diskinfo->{type} = 'block'; |
1e3baf05 | 158 | } else { |
1e3baf05 | 159 | $diskinfo->{type} = 'file'; |
1e3baf05 DM |
160 | } |
161 | ||
162 | push @{$task->{disks}}, $diskinfo; | |
b969cc68 | 163 | } |
1e3baf05 DM |
164 | } |
165 | ||
166 | sub vm_status { | |
167 | my ($self, $vmid) = @_; | |
168 | ||
66ab1d91 | 169 | my $running = PVE::QemuServer::check_running($vmid) ? 1 : 0; |
874a096e DM |
170 | |
171 | return wantarray ? ($running, $running ? 'running' : 'stopped') : $running; | |
1e3baf05 DM |
172 | } |
173 | ||
174 | sub lock_vm { | |
175 | my ($self, $vmid) = @_; | |
176 | ||
f301bc0d | 177 | PVE::QemuConfig->set_lock($vmid, 'backup'); |
1e3baf05 DM |
178 | } |
179 | ||
180 | sub unlock_vm { | |
181 | my ($self, $vmid) = @_; | |
182 | ||
f301bc0d | 183 | PVE::QemuConfig->remove_lock($vmid, 'backup'); |
1e3baf05 DM |
184 | } |
185 | ||
186 | sub stop_vm { | |
187 | my ($self, $task, $vmid) = @_; | |
188 | ||
189 | my $opts = $self->{vzdump}->{opts}; | |
190 | ||
191 | my $wait = $opts->{stopwait} * 60; | |
192 | # send shutdown and wait | |
254575e9 | 193 | $self->cmd ("qm shutdown $vmid --skiplock --keepActive --timeout $wait"); |
1e3baf05 DM |
194 | } |
195 | ||
196 | sub start_vm { | |
197 | my ($self, $task, $vmid) = @_; | |
198 | ||
66ab1d91 | 199 | $self->cmd ("qm start $vmid --skiplock"); |
1e3baf05 DM |
200 | } |
201 | ||
202 | sub suspend_vm { | |
203 | my ($self, $task, $vmid) = @_; | |
204 | ||
b08c37c3 DC |
205 | return if $self->{vm_was_paused}; |
206 | ||
66ab1d91 | 207 | $self->cmd ("qm suspend $vmid --skiplock"); |
1e3baf05 DM |
208 | } |
209 | ||
210 | sub resume_vm { | |
211 | my ($self, $task, $vmid) = @_; | |
212 | ||
b08c37c3 DC |
213 | return if $self->{vm_was_paused}; |
214 | ||
66ab1d91 | 215 | $self->cmd ("qm resume $vmid --skiplock"); |
1e3baf05 DM |
216 | } |
217 | ||
1e3baf05 DM |
218 | sub assemble { |
219 | my ($self, $task, $vmid) = @_; | |
220 | ||
04096e7b | 221 | my $conffile = PVE::QemuConfig->config_file($vmid); |
1e3baf05 DM |
222 | |
223 | my $outfile = "$task->{tmpdir}/qemu-server.conf"; | |
c05f7f3f WL |
224 | my $firewall_src = "/etc/pve/firewall/$vmid.fw"; |
225 | my $firewall_dest = "$task->{tmpdir}/qemu-server.fw"; | |
226 | ||
b7bbeff3 FE |
227 | my $outfd = IO::File->new(">$outfile") or die "unable to open '$outfile' - $!\n"; |
228 | my $conffd = IO::File->new($conffile, 'r') or die "unable to open '$conffile' - $!\n"; | |
c05f7f3f WL |
229 | |
230 | my $found_snapshot; | |
391c2230 | 231 | my $found_pending; |
31280b4b | 232 | my $found_cloudinit; |
c05f7f3f WL |
233 | while (defined (my $line = <$conffd>)) { |
234 | next if $line =~ m/^\#vzdump\#/; # just to be sure | |
235 | next if $line =~ m/^\#qmdump\#/; # just to be sure | |
391c2230 FG |
236 | if ($line =~ m/^\[(.*)\]\s*$/) { |
237 | if ($1 =~ m/PENDING/i) { | |
238 | $found_pending = 1; | |
31280b4b AD |
239 | } elsif ($1 =~ m/special:cloudinit/) { |
240 | $found_cloudinit = 1; | |
391c2230 FG |
241 | } else { |
242 | $found_snapshot = 1; | |
243 | } | |
1e3baf05 | 244 | } |
31280b4b | 245 | next if $found_snapshot || $found_pending || $found_cloudinit; # skip all snapshots,pending changes and cloudinit config data |
391c2230 | 246 | |
c05f7f3f WL |
247 | if ($line =~ m/^unused\d+:\s*(\S+)\s*/) { |
248 | $self->loginfo("skip unused drive '$1' (not included into backup)"); | |
249 | next; | |
1e3baf05 | 250 | } |
c05f7f3f | 251 | next if $line =~ m/^lock:/ || $line =~ m/^parent:/; |
91bd6c90 | 252 | |
c05f7f3f WL |
253 | print $outfd $line; |
254 | } | |
255 | ||
256 | foreach my $di (@{$task->{disks}}) { | |
257 | if ($di->{type} eq 'block' || $di->{type} eq 'file') { | |
258 | my $storeid = $di->{storeid} || ''; | |
259 | my $format = $di->{format} || ''; | |
260 | print $outfd "#qmdump#map:$di->{virtdev}:$di->{qmdevice}:$storeid:$format:\n"; | |
261 | } else { | |
262 | die "internal error"; | |
91bd6c90 | 263 | } |
c05f7f3f | 264 | } |
1e3baf05 | 265 | |
c05f7f3f WL |
266 | if ($found_snapshot) { |
267 | $self->loginfo("snapshots found (not included into backup)"); | |
268 | } | |
391c2230 FG |
269 | if ($found_pending) { |
270 | $self->loginfo("pending configuration changes found (not included into backup)"); | |
271 | } | |
272 | ||
c05f7f3f | 273 | PVE::Tools::file_copy($firewall_src, $firewall_dest) if -f $firewall_src; |
1e3baf05 DM |
274 | } |
275 | ||
276 | sub archive { | |
fad02a16 | 277 | my ($self, $task, $vmid, $filename, $comp) = @_; |
1e3baf05 | 278 | |
c5983223 | 279 | my $opts = $self->{vzdump}->{opts}; |
c5983223 DM |
280 | my $scfg = $opts->{scfg}; |
281 | ||
69074863 | 282 | if ($self->{vzdump}->{opts}->{pbs}) { |
c5983223 DM |
283 | $self->archive_pbs($task, $vmid); |
284 | } else { | |
285 | $self->archive_vma($task, $vmid, $filename, $comp); | |
286 | } | |
287 | } | |
288 | ||
3a44897f | 289 | my $bitmap_action_to_human = sub { |
13ddc7eb | 290 | my ($self, $info) = @_; |
3a44897f SR |
291 | |
292 | my $action = $info->{action}; | |
293 | ||
294 | if ($action eq "not-used") { | |
962d4d64 | 295 | return "disabled (no support)"; |
3a44897f SR |
296 | } elsif ($action eq "not-used-removed") { |
297 | return "disabled (old bitmap cleared)"; | |
298 | } elsif ($action eq "new") { | |
13ddc7eb | 299 | return "created new"; |
3a44897f SR |
300 | } elsif ($action eq "used") { |
301 | if ($info->{dirty} == 0) { | |
13ddc7eb | 302 | return "OK (drive clean)"; |
3a44897f | 303 | } else { |
f1aca33d SR |
304 | my $size = render_bytes($info->{size}, 1); |
305 | my $dirty = render_bytes($info->{dirty}, 1); | |
13ddc7eb | 306 | return "OK ($dirty of $size dirty)"; |
3a44897f SR |
307 | } |
308 | } elsif ($action eq "invalid") { | |
309 | return "existing bitmap was invalid and has been cleared"; | |
310 | } else { | |
311 | return "unknown"; | |
312 | } | |
313 | }; | |
314 | ||
c5983223 | 315 | my $query_backup_status_loop = sub { |
f8566714 | 316 | my ($self, $vmid, $job_uuid, $qemu_support) = @_; |
c5983223 | 317 | |
c5983223 | 318 | my $starttime = time (); |
09eb196b | 319 | my $last_time = $starttime; |
8825248c | 320 | my ($last_percent, $last_total, $last_target, $last_zero, $last_transferred) = (-1, 0, 0, 0, 0); |
4d159c24 | 321 | my ($transferred, $reused); |
c5983223 | 322 | |
09eb196b TL |
323 | my $get_mbps = sub { |
324 | my ($mb, $delta) = @_; | |
4d159c24 TL |
325 | return "0 B/s" if $mb <= 0; |
326 | my $bw = int(($mb / $delta)); | |
f1aca33d | 327 | return render_bytes($bw, 1) . "/s"; |
09eb196b TL |
328 | }; |
329 | ||
3a44897f | 330 | my $target = 0; |
8825248c | 331 | my $last_reused = 0; |
f8566714 | 332 | my $has_query_bitmap = $qemu_support && $qemu_support->{'query-bitmap-info'}; |
5749c383 | 333 | my $is_template = PVE::QemuConfig->is_template($self->{vmlist}->{$vmid}); |
f8566714 | 334 | if ($has_query_bitmap) { |
13ddc7eb | 335 | my $total = 0; |
3a44897f | 336 | my $bitmap_info = mon_cmd($vmid, 'query-pbs-bitmap-info'); |
5749c383 TL |
337 | for my $info (sort { $a->{drive} cmp $b->{drive} } @$bitmap_info) { |
338 | if (!$is_template) { | |
339 | my $text = $bitmap_action_to_human->($self, $info); | |
340 | my $drive = $info->{drive}; | |
341 | $drive =~ s/^drive-//; # for consistency | |
342 | $self->loginfo("$drive: dirty-bitmap status: $text"); | |
343 | } | |
3a44897f | 344 | $target += $info->{dirty}; |
13ddc7eb | 345 | $total += $info->{size}; |
8825248c | 346 | $last_reused += $info->{size} - $info->{dirty}; |
3a44897f | 347 | } |
13ddc7eb | 348 | if ($target < $total) { |
f1aca33d SR |
349 | my $total_h = render_bytes($total, 1); |
350 | my $target_h = render_bytes($target, 1); | |
13ddc7eb TL |
351 | $self->loginfo("using fast incremental mode (dirty-bitmap), $target_h dirty of $total_h total"); |
352 | } | |
3a44897f SR |
353 | } |
354 | ||
1eb7e590 | 355 | my $last_finishing = 0; |
c5983223 | 356 | while(1) { |
09eb196b TL |
357 | my $status = mon_cmd($vmid, 'query-backup'); |
358 | ||
c5983223 | 359 | my $total = $status->{total} || 0; |
f8566714 TL |
360 | my $dirty = $status->{dirty}; |
361 | $target = (defined($dirty) && $dirty < $total) ? $dirty : $total if !$has_query_bitmap; | |
c5983223 | 362 | $transferred = $status->{transferred} || 0; |
4d159c24 | 363 | $reused = $status->{reused}; |
0f6c6981 | 364 | my $percent = $target ? int(($transferred * 100)/$target) : 100; |
c5983223 | 365 | my $zero = $status->{'zero-bytes'} || 0; |
c5983223 DM |
366 | |
367 | die "got unexpected uuid\n" if !$status->{uuid} || ($status->{uuid} ne $job_uuid); | |
368 | ||
369 | my $ctime = time(); | |
370 | my $duration = $ctime - $starttime; | |
371 | ||
372 | my $rbytes = $transferred - $last_transferred; | |
8825248c SR |
373 | my $wbytes; |
374 | if ($reused) { | |
375 | # reused includes zero bytes for PBS | |
376 | $wbytes = $rbytes - ($reused - $last_reused); | |
377 | } else { | |
378 | $wbytes = $rbytes - ($zero - $last_zero); | |
379 | } | |
c5983223 DM |
380 | |
381 | my $timediff = ($ctime - $last_time) || 1; # fixme | |
09eb196b TL |
382 | my $mbps_read = $get_mbps->($rbytes, $timediff); |
383 | my $mbps_write = $get_mbps->($wbytes, $timediff); | |
f1aca33d SR |
384 | my $target_h = render_bytes($target, 1); |
385 | my $transferred_h = render_bytes($transferred, 1); | |
4d159c24 | 386 | |
a40295b6 | 387 | my $statusline = sprintf("%3d%% ($transferred_h of $target_h) in %s" |
f1aca33d | 388 | .", read: $mbps_read, write: $mbps_write", $percent, render_duration($duration)); |
09eb196b | 389 | |
c5983223 DM |
390 | my $res = $status->{status} || 'unknown'; |
391 | if ($res ne 'active') { | |
1eb7e590 SR |
392 | if ($last_percent < 100) { |
393 | $self->loginfo($statusline); | |
394 | } | |
09eb196b TL |
395 | if ($res ne 'done') { |
396 | die (($status->{errmsg} || "unknown error") . "\n") if $res eq 'error'; | |
397 | die "got unexpected status '$res'\n"; | |
09eb196b | 398 | } |
6cdb568c TL |
399 | $last_target = $target if $target; |
400 | $last_total = $total if $total; | |
401 | $last_zero = $zero if $zero; | |
402 | $last_transferred = $transferred if $transferred; | |
c5983223 DM |
403 | last; |
404 | } | |
09eb196b | 405 | if ($percent != $last_percent && ($timediff > 2)) { |
c5983223 | 406 | $self->loginfo($statusline); |
09eb196b | 407 | $last_percent = $percent; |
4d159c24 | 408 | $last_target = $target if $target; |
c5983223 DM |
409 | $last_total = $total if $total; |
410 | $last_zero = $zero if $zero; | |
411 | $last_transferred = $transferred if $transferred; | |
412 | $last_time = $ctime; | |
8825248c | 413 | $last_reused = $reused; |
1eb7e590 SR |
414 | |
415 | if (!$last_finishing && $status->{finishing}) { | |
d5b0cfb1 | 416 | $self->loginfo("Waiting for server to finish backup validation..."); |
1eb7e590 SR |
417 | } |
418 | $last_finishing = $status->{finishing}; | |
c5983223 DM |
419 | } |
420 | sleep(1); | |
421 | } | |
422 | ||
423 | my $duration = time() - $starttime; | |
d35412a3 TL |
424 | |
425 | if ($last_zero) { | |
426 | my $zero_per = $last_target ? int(($last_zero * 100)/$last_target) : 0; | |
f1aca33d | 427 | my $zero_h = render_bytes($last_zero); |
d35412a3 TL |
428 | $self->loginfo("backup is sparse: $zero_h (${zero_per}%) total zero data"); |
429 | } | |
1ca43f1c | 430 | if ($reused) { |
f1aca33d | 431 | my $reused_h = render_bytes($reused); |
1ca43f1c SR |
432 | my $reuse_per = int($reused * 100 / $last_total); |
433 | $self->loginfo("backup was done incrementally, reused $reused_h (${reuse_per}%)"); | |
434 | } | |
bafae3ec | 435 | if ($transferred) { |
f1aca33d | 436 | my $transferred_h = render_bytes($transferred); |
bafae3ec TL |
437 | if ($duration) { |
438 | my $mbps = $get_mbps->($transferred, $duration); | |
439 | $self->loginfo("transferred $transferred_h in $duration seconds ($mbps)"); | |
440 | } else { | |
441 | $self->loginfo("transferred $transferred_h in <1 seconds"); | |
442 | } | |
4d159c24 TL |
443 | } |
444 | ||
b4be9c02 TL |
445 | return { |
446 | total => $last_total, | |
447 | reused => $reused, | |
448 | }; | |
c5983223 DM |
449 | }; |
450 | ||
f9dde219 SR |
451 | my $attach_tpmstate_drive = sub { |
452 | my ($self, $task, $vmid) = @_; | |
453 | ||
454 | return if !$task->{tpmpath}; | |
455 | ||
456 | # unconditionally try to remove the tpmstate-named drive - it only exists | |
457 | # for backing up, and avoids errors if left over from some previous event | |
458 | eval { PVE::QemuServer::qemu_drivedel($vmid, "tpmstate0-backup"); }; | |
459 | ||
460 | $self->loginfo('attaching TPM drive to QEMU for backup'); | |
461 | ||
462 | my $drive = "file=$task->{tpmpath},if=none,read-only=on,id=drive-tpmstate0-backup"; | |
93e3f756 | 463 | $drive =~ s/\\/\\\\/g; |
f9dde219 | 464 | my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\""); |
df28f8cc | 465 | die "attaching TPM drive failed - $ret\n" if $ret !~ m/OK/s; |
f9dde219 SR |
466 | }; |
467 | ||
468 | my $detach_tpmstate_drive = sub { | |
469 | my ($task, $vmid) = @_; | |
470 | return if !$task->{tpmpath} || !PVE::QemuServer::check_running($vmid); | |
471 | eval { PVE::QemuServer::qemu_drivedel($vmid, "tpmstate0-backup"); }; | |
472 | }; | |
473 | ||
4de4eefc FE |
474 | my sub add_backup_performance_options { |
475 | my ($qmp_param, $perf, $qemu_support) = @_; | |
476 | ||
477 | return if !$perf || scalar(keys $perf->%*) == 0; | |
478 | ||
479 | if (!$qemu_support) { | |
480 | my $settings_string = join(', ', sort keys $perf->%*); | |
481 | log_warn("ignoring setting(s): $settings_string - issue checking if supported"); | |
482 | return; | |
483 | } | |
484 | ||
485 | if (defined($perf->{'max-workers'})) { | |
486 | if ($qemu_support->{'backup-max-workers'}) { | |
487 | $qmp_param->{'max-workers'} = int($perf->{'max-workers'}); | |
488 | } else { | |
489 | log_warn("ignoring 'max-workers' setting - not supported by running QEMU"); | |
490 | } | |
491 | } | |
492 | } | |
493 | ||
3234f563 TL |
494 | sub get_and_check_pbs_encryption_config { |
495 | my ($self) = @_; | |
496 | ||
497 | my $opts = $self->{vzdump}->{opts}; | |
498 | my $scfg = $opts->{scfg}; | |
499 | ||
500 | my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($scfg, $opts->{storage}); | |
501 | my $master_keyfile = PVE::Storage::PBSPlugin::pbs_master_pubkey_file_name($scfg, $opts->{storage}); | |
502 | ||
503 | if (-e $keyfile) { | |
504 | if (-e $master_keyfile) { | |
505 | $self->loginfo("enabling encryption with master key feature"); | |
506 | return ($keyfile, $master_keyfile); | |
507 | } elsif ($scfg->{'master-pubkey'}) { | |
508 | die "master public key configured but no key file found\n"; | |
509 | } else { | |
510 | $self->loginfo("enabling encryption"); | |
511 | return ($keyfile, undef); | |
512 | } | |
513 | } else { | |
514 | my $encryption_fp = $scfg->{'encryption-key'}; | |
515 | die "encryption configured ('$encryption_fp') but no encryption key file found!\n" | |
516 | if $encryption_fp; | |
517 | if (-e $master_keyfile) { | |
518 | $self->log( | |
519 | 'warn', | |
520 | "backup target storage is configured with master-key, but no encryption key set!" | |
521 | ." Ignoring master key settings and creating unencrypted backup." | |
522 | ); | |
523 | } | |
524 | return (undef, undef); | |
525 | } | |
526 | die "internal error - unhandled case for getting & checking PBS encryption ($keyfile, $master_keyfile)!"; | |
527 | } | |
528 | ||
5d857282 FE |
529 | my sub cleanup_fleecing_images { |
530 | my ($self, $disks) = @_; | |
531 | ||
532 | for my $di ($disks->@*) { | |
533 | if (my $volid = $di->{'fleece-volid'}) { | |
534 | eval { PVE::Storage::vdisk_free($self->{storecfg}, $volid); }; | |
535 | $self->log('warn', "error removing fleecing image '$volid' - $@") if $@; | |
536 | } | |
537 | } | |
538 | } | |
539 | ||
540 | my sub allocate_fleecing_images { | |
541 | my ($self, $disks, $vmid, $fleecing_storeid, $format) = @_; | |
542 | ||
543 | die "internal error - no fleecing storage specified\n" if !$fleecing_storeid; | |
544 | ||
545 | # TODO what about potential left-over images from a failed attempt? Just | |
546 | # auto-remove? While unlikely, could conflict with manually created image from user... | |
547 | ||
548 | eval { | |
549 | my $n = 0; # counter for fleecing image names | |
550 | ||
551 | for my $di ($disks->@*) { | |
552 | next if $di->{virtdev} =~ m/^(?:tpmstate|efidisk)\d$/; # too small to be worth it | |
553 | if ($di->{type} eq 'block' || $di->{type} eq 'file') { | |
554 | my $scfg = PVE::Storage::storage_config($self->{storecfg}, $fleecing_storeid); | |
555 | my $name = "vm-$vmid-fleece-$n"; | |
556 | $name .= ".$format" if $scfg->{path}; | |
557 | ||
558 | my $size = PVE::Tools::convert_size($di->{size}, 'b' => 'kb'); | |
559 | ||
560 | $di->{'fleece-volid'} = PVE::Storage::vdisk_alloc( | |
561 | $self->{storecfg}, $fleecing_storeid, $vmid, $format, $name, $size); | |
562 | ||
563 | $n++; | |
564 | } else { | |
565 | die "implement me (type '$di->{type}')"; | |
566 | } | |
567 | } | |
568 | }; | |
569 | if (my $err = $@) { | |
570 | cleanup_fleecing_images($self, $disks); | |
571 | die $err; | |
572 | } | |
573 | } | |
574 | ||
575 | my sub detach_fleecing_images { | |
576 | my ($disks, $vmid) = @_; | |
577 | ||
578 | return if !PVE::QemuServer::Helpers::vm_running_locally($vmid); | |
579 | ||
580 | for my $di ($disks->@*) { | |
581 | if (my $volid = $di->{'fleece-volid'}) { | |
582 | my $devid = "$di->{qmdevice}-fleecing"; | |
583 | $devid =~ s/^drive-//; # re-added by qemu_drivedel() | |
584 | eval { PVE::QemuServer::qemu_drivedel($vmid, $devid) }; | |
585 | } | |
586 | } | |
587 | } | |
588 | ||
589 | my sub attach_fleecing_images { | |
590 | my ($self, $disks, $vmid, $format) = @_; | |
591 | ||
592 | # unconditionally try to remove potential left-overs from a previous backup | |
593 | detach_fleecing_images($disks, $vmid); | |
594 | ||
595 | my $vollist = [ map { $_->{'fleece-volid'} } grep { $_->{'fleece-volid'} } $disks->@* ]; | |
596 | PVE::Storage::activate_volumes($self->{storecfg}, $vollist); | |
597 | ||
598 | for my $di ($disks->@*) { | |
599 | if (my $volid = $di->{'fleece-volid'}) { | |
600 | $self->loginfo("$di->{qmdevice}: attaching fleecing image $volid to QEMU"); | |
601 | ||
602 | my $path = PVE::Storage::path($self->{storecfg}, $volid); | |
603 | my $devid = "$di->{qmdevice}-fleecing"; | |
604 | my $drive = "file=$path,if=none,id=$devid,format=$format,discard=unmap"; | |
605 | # Specify size explicitly, to make it work if storage backend rounded up size for | |
606 | # fleecing image when allocating. | |
607 | $drive .= ",size=$di->{size}" if $format eq 'raw'; | |
608 | $drive =~ s/\\/\\\\/g; | |
609 | my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\""); | |
610 | die "attaching fleecing image $volid failed - $ret\n" if $ret !~ m/OK/s; | |
611 | } | |
612 | } | |
613 | } | |
614 | ||
615 | my sub check_and_prepare_fleecing { | |
616 | my ($self, $vmid, $fleecing_opts, $disks, $is_template, $qemu_support) = @_; | |
617 | ||
618 | # Even if the VM was started specifically for fleecing, it's possible that the VM is resumed and | |
619 | # then starts doing IO. For VMs that are not resumed the fleecing images will just stay empty, | |
620 | # so there is no big cost. | |
621 | ||
622 | my $use_fleecing = $fleecing_opts && $fleecing_opts->{enabled} && !$is_template; | |
623 | ||
624 | if ($use_fleecing && !defined($qemu_support->{'backup-fleecing'})) { | |
625 | $self->log( | |
626 | 'warn', | |
627 | "running QEMU version does not support backup fleecing - continuing without", | |
628 | ); | |
629 | $use_fleecing = 0; | |
630 | } | |
631 | ||
632 | if ($use_fleecing) { | |
633 | my ($default_format, $valid_formats) = PVE::Storage::storage_default_format( | |
634 | $self->{storecfg}, $fleecing_opts->{storage}); | |
635 | my $format = scalar(grep { $_ eq 'qcow2' } $valid_formats->@*) ? 'qcow2' : 'raw'; | |
636 | ||
637 | allocate_fleecing_images($self, $disks, $vmid, $fleecing_opts->{storage}, $format); | |
638 | attach_fleecing_images($self, $disks, $vmid, $format); | |
639 | } | |
640 | ||
641 | return $use_fleecing; | |
642 | } | |
643 | ||
c5983223 DM |
644 | sub archive_pbs { |
645 | my ($self, $task, $vmid) = @_; | |
646 | ||
1e3baf05 | 647 | my $conffile = "$task->{tmpdir}/qemu-server.conf"; |
c05f7f3f | 648 | my $firewall = "$task->{tmpdir}/qemu-server.fw"; |
1e3baf05 DM |
649 | |
650 | my $opts = $self->{vzdump}->{opts}; | |
c5983223 DM |
651 | my $scfg = $opts->{scfg}; |
652 | ||
653 | my $starttime = time(); | |
654 | ||
c5983223 | 655 | my $fingerprint = $scfg->{fingerprint}; |
fbec3f89 | 656 | my $repo = PVE::PBSClient::get_repository($scfg); |
c5983223 | 657 | my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $opts->{storage}); |
3234f563 | 658 | my ($keyfile, $master_keyfile) = $self->get_and_check_pbs_encryption_config(); |
c5983223 | 659 | |
d11e91d2 | 660 | my $diskcount = scalar(@{$task->{disks}}); |
b241ddbf TL |
661 | # proxmox-backup-client can only handle raw files and block devs, so only use it (directly) for |
662 | # disk-less VMs | |
4ef13a7f | 663 | if (!$diskcount) { |
4ef13a7f | 664 | $self->loginfo("backup contains no disks"); |
c5983223 DM |
665 | |
666 | local $ENV{PBS_PASSWORD} = $password; | |
77b24c62 | 667 | local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint); |
c5983223 DM |
668 | my $cmd = [ |
669 | '/usr/bin/proxmox-backup-client', | |
670 | 'backup', | |
671 | '--repository', $repo, | |
672 | '--backup-type', 'vm', | |
673 | '--backup-id', "$vmid", | |
674 | '--backup-time', $task->{backup_time}, | |
d11e91d2 | 675 | ]; |
21a9ec2a WB |
676 | if (defined(my $ns = $scfg->{namespace})) { |
677 | push @$cmd, '--ns', $ns; | |
678 | } | |
3234f563 | 679 | if (defined($keyfile)) { |
fbd3dde7 | 680 | push @$cmd, '--keyfile', $keyfile; |
3234f563 | 681 | push @$cmd, '--master-pubkey-file', $master_keyfile if defined($master_keyfile); |
fbd3dde7 | 682 | } |
c5983223 DM |
683 | |
684 | push @$cmd, "qemu-server.conf:$conffile"; | |
685 | push @$cmd, "fw.conf:$firewall" if -e $firewall; | |
c5983223 DM |
686 | |
687 | $self->loginfo("starting template backup"); | |
688 | $self->loginfo(join(' ', @$cmd)); | |
689 | ||
690 | $self->cmd($cmd); | |
691 | ||
692 | return; | |
693 | } | |
694 | ||
81dcd479 TL |
695 | # get list early so we die on unkown drive types before doing anything |
696 | my $devlist = _get_task_devlist($task); | |
5d857282 | 697 | my $use_fleecing; |
c5983223 | 698 | |
0b2f574b | 699 | $self->enforce_vm_running_for_backup($vmid); |
65911545 | 700 | $self->{qmeventd_fh} = PVE::QemuServer::register_qmeventd_handle($vmid); |
c5983223 | 701 | |
f6168f1a | 702 | my $backup_job_uuid; |
c5983223 DM |
703 | eval { |
704 | $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub { | |
d11e91d2 | 705 | die "interrupted by signal\n"; |
c5983223 DM |
706 | }; |
707 | ||
fb9f512c | 708 | my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") }; |
2cfb0905 SR |
709 | my $err = $@; |
710 | if (!$qemu_support || $err) { | |
711 | die "query-proxmox-support returned empty value\n" if !$err; | |
712 | if ($err =~ m/The command query-proxmox-support has not been found/) { | |
713 | die "PBS backups are not supported by the running QEMU version. Please make " | |
714 | . "sure you've installed the latest version and the VM has been restarted.\n"; | |
715 | } else { | |
716 | die "QMP command query-proxmox-support failed - $err\n"; | |
717 | } | |
fb9f512c SR |
718 | } |
719 | ||
781fb803 TL |
720 | # pve-qemu supports it since 5.2.0-1 (PVE 6.4), so safe to die since PVE 8 |
721 | die "master key configured but running QEMU version does not support master keys\n" | |
3234f563 | 722 | if !defined($qemu_support->{'pbs-masterkey'}) && defined($master_keyfile); |
9bf522bc | 723 | |
f9dde219 SR |
724 | $attach_tpmstate_drive->($self, $task, $vmid); |
725 | ||
5d857282 FE |
726 | my $is_template = PVE::QemuConfig->is_template($self->{vmlist}->{$vmid}); |
727 | ||
728 | $use_fleecing = check_and_prepare_fleecing( | |
729 | $self, $vmid, $opts->{fleecing}, $task->{disks}, $is_template, $qemu_support); | |
730 | ||
cb521f2d | 731 | my $fs_frozen = $self->qga_fs_freeze($task, $vmid); |
c5983223 | 732 | |
d11e91d2 TL |
733 | my $params = { |
734 | format => "pbs", | |
735 | 'backup-file' => $repo, | |
736 | 'backup-id' => "$vmid", | |
737 | 'backup-time' => $task->{backup_time}, | |
738 | password => $password, | |
739 | devlist => $devlist, | |
740 | 'config-file' => $conffile, | |
c5983223 | 741 | }; |
5d857282 FE |
742 | $params->{fleecing} = JSON::true if $use_fleecing; |
743 | ||
21a9ec2a | 744 | if (defined(my $ns = $scfg->{namespace})) { |
0c9a94d2 | 745 | $params->{'backup-ns'} = $ns; |
21a9ec2a | 746 | } |
4de4eefc | 747 | |
2790636a | 748 | $params->{speed} = $opts->{bwlimit}*1024 if $opts->{bwlimit}; |
4de4eefc FE |
749 | add_backup_performance_options($params, $opts->{performance}, $qemu_support); |
750 | ||
d11e91d2 TL |
751 | $params->{fingerprint} = $fingerprint if defined($fingerprint); |
752 | $params->{'firewall-file'} = $firewall if -e $firewall; | |
3234f563 TL |
753 | |
754 | $params->{encrypt} = defined($keyfile) ? JSON::true : JSON::false; | |
755 | if (defined($keyfile)) { | |
ece74030 | 756 | $params->{keyfile} = $keyfile; |
3234f563 | 757 | $params->{"master-keyfile"} = $master_keyfile if defined($master_keyfile); |
ece74030 | 758 | } |
c5983223 | 759 | |
78179bda | 760 | $params->{'use-dirty-bitmap'} = JSON::true |
962d4d64 | 761 | if $qemu_support->{'pbs-dirty-bitmap'} && !$is_template; |
fb9f512c | 762 | |
46b676c0 | 763 | $params->{timeout} = 125; # give some time to connect to the backup server |
f57666e9 | 764 | |
d11e91d2 | 765 | my $res = eval { mon_cmd($vmid, "backup", %$params) }; |
c5983223 | 766 | my $qmperr = $@; |
d11e91d2 | 767 | $backup_job_uuid = $res->{UUID} if $res; |
c5983223 | 768 | |
1ece829a TL |
769 | if ($fs_frozen) { |
770 | $self->qga_fs_thaw($vmid); | |
c5983223 DM |
771 | } |
772 | ||
773 | die $qmperr if $qmperr; | |
f6168f1a | 774 | die "got no uuid for backup task\n" if !defined($backup_job_uuid); |
c5983223 | 775 | |
f6168f1a | 776 | $self->loginfo("started backup task '$backup_job_uuid'"); |
c5983223 | 777 | |
0b2f574b | 778 | $self->resume_vm_after_job_start($task, $vmid); |
c5983223 | 779 | |
3a44897f | 780 | my $stat = $query_backup_status_loop->($self, $vmid, $backup_job_uuid, $qemu_support); |
2098f2ff | 781 | $task->{size} = $stat->{total}; |
c5983223 DM |
782 | }; |
783 | my $err = $@; | |
c5983223 DM |
784 | if ($err) { |
785 | $self->logerr($err); | |
edae1718 | 786 | $self->mon_backup_cancel($vmid); |
ebce5239 | 787 | $self->resume_vm_after_job_start($task, $vmid); |
c5983223 | 788 | } |
0b2f574b | 789 | $self->restore_vm_power_state($vmid); |
c5983223 | 790 | |
5d857282 FE |
791 | if ($use_fleecing) { |
792 | detach_fleecing_images($task->{disks}, $vmid); | |
793 | cleanup_fleecing_images($self, $task->{disks}); | |
794 | } | |
795 | ||
c5983223 DM |
796 | die $err if $err; |
797 | } | |
798 | ||
02da0c65 TL |
799 | my $fork_compressor_pipe = sub { |
800 | my ($self, $comp, $outfileno) = @_; | |
801 | ||
802 | my @pipefd = POSIX::pipe(); | |
803 | my $cpid = fork(); | |
804 | die "unable to fork worker - $!" if !defined($cpid) || $cpid < 0; | |
805 | if ($cpid == 0) { | |
806 | eval { | |
807 | POSIX::close($pipefd[1]); | |
808 | # redirect STDIN | |
809 | my $fd = fileno(STDIN); | |
810 | close STDIN; | |
811 | POSIX::close(0) if $fd != 0; | |
812 | die "unable to redirect STDIN - $!" | |
813 | if !open(STDIN, "<&", $pipefd[0]); | |
814 | ||
815 | # redirect STDOUT | |
816 | $fd = fileno(STDOUT); | |
817 | close STDOUT; | |
818 | POSIX::close (1) if $fd != 1; | |
819 | ||
820 | die "unable to redirect STDOUT - $!" | |
821 | if !open(STDOUT, ">&", $outfileno); | |
822 | ||
823 | exec($comp); | |
824 | die "fork compressor '$comp' failed\n"; | |
825 | }; | |
826 | if (my $err = $@) { | |
827 | $self->logerr($err); | |
828 | POSIX::_exit(1); | |
829 | } | |
830 | POSIX::_exit(0); | |
831 | kill(-9, $$); | |
832 | } else { | |
833 | POSIX::close($pipefd[0]); | |
834 | $outfileno = $pipefd[1]; | |
835 | } | |
836 | ||
837 | return ($cpid, $outfileno); | |
838 | }; | |
839 | ||
c5983223 DM |
840 | sub archive_vma { |
841 | my ($self, $task, $vmid, $filename, $comp) = @_; | |
842 | ||
843 | my $conffile = "$task->{tmpdir}/qemu-server.conf"; | |
844 | my $firewall = "$task->{tmpdir}/qemu-server.fw"; | |
845 | ||
846 | my $opts = $self->{vzdump}->{opts}; | |
847 | ||
848 | my $starttime = time(); | |
1e3baf05 | 849 | |
91bd6c90 DM |
850 | my $speed = 0; |
851 | if ($opts->{bwlimit}) { | |
874a096e | 852 | $speed = $opts->{bwlimit}*1024; |
91bd6c90 | 853 | } |
1e3baf05 | 854 | |
5d857282 FE |
855 | my $is_template = PVE::QemuConfig->is_template($self->{vmlist}->{$vmid}); |
856 | ||
c82935e9 | 857 | my $diskcount = scalar(@{$task->{disks}}); |
5d857282 | 858 | if ($is_template || !$diskcount) { |
23b4120b DM |
859 | my @pathlist; |
860 | foreach my $di (@{$task->{disks}}) { | |
861 | if ($di->{type} eq 'block' || $di->{type} eq 'file') { | |
862 | push @pathlist, "$di->{qmdevice}=$di->{path}"; | |
863 | } else { | |
864 | die "implement me"; | |
865 | } | |
866 | } | |
867 | ||
c82935e9 DM |
868 | if (!$diskcount) { |
869 | $self->loginfo("backup contains no disks"); | |
870 | } | |
871 | ||
23b4120b DM |
872 | my $outcmd; |
873 | if ($comp) { | |
874a096e | 874 | $outcmd = "exec:$comp"; |
23b4120b | 875 | } else { |
874a096e | 876 | $outcmd = "exec:cat"; |
23b4120b DM |
877 | } |
878 | ||
a2fab11a | 879 | $outcmd .= " > $filename" if !$opts->{stdout}; |
23b4120b | 880 | |
c05f7f3f WL |
881 | my $cmd = ['/usr/bin/vma', 'create', '-v', '-c', $conffile]; |
882 | push @$cmd, '-c', $firewall if -e $firewall; | |
883 | push @$cmd, $outcmd, @pathlist; | |
23b4120b DM |
884 | |
885 | $self->loginfo("starting template backup"); | |
886 | $self->loginfo(join(' ', @$cmd)); | |
887 | ||
888 | if ($opts->{stdout}) { | |
6bb12239 | 889 | $self->cmd($cmd, output => ">&" . fileno($opts->{stdout})); |
23b4120b DM |
890 | } else { |
891 | $self->cmd($cmd); | |
892 | } | |
893 | ||
894 | return; | |
895 | } | |
896 | ||
81dcd479 | 897 | my $devlist = _get_task_devlist($task); |
5d857282 | 898 | my $use_fleecing; |
1e3baf05 | 899 | |
0b2f574b | 900 | $self->enforce_vm_running_for_backup($vmid); |
65911545 | 901 | $self->{qmeventd_fh} = PVE::QemuServer::register_qmeventd_handle($vmid); |
91bd6c90 DM |
902 | |
903 | my $cpid; | |
d2cc2cbe DM |
904 | my $backup_job_uuid; |
905 | ||
91bd6c90 DM |
906 | eval { |
907 | $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub { | |
d11e91d2 | 908 | die "interrupted by signal\n"; |
91bd6c90 DM |
909 | }; |
910 | ||
4de4eefc FE |
911 | # Currently, failing to determine Proxmox support is not critical here, because it's only |
912 | # used for performance settings like 'max-workers'. | |
913 | my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") }; | |
914 | log_warn($@) if $@; | |
915 | ||
f9dde219 SR |
916 | $attach_tpmstate_drive->($self, $task, $vmid); |
917 | ||
5d857282 FE |
918 | $use_fleecing = check_and_prepare_fleecing( |
919 | $self, $vmid, $opts->{fleecing}, $task->{disks}, $is_template, $qemu_support); | |
920 | ||
91bd6c90 DM |
921 | my $outfh; |
922 | if ($opts->{stdout}) { | |
923 | $outfh = $opts->{stdout}; | |
924 | } else { | |
925 | $outfh = IO::File->new($filename, "w") || | |
926 | die "unable to open file '$filename' - $!\n"; | |
927 | } | |
02da0c65 | 928 | my $outfileno = fileno($outfh); |
91bd6c90 | 929 | |
91bd6c90 | 930 | if ($comp) { |
02da0c65 | 931 | ($cpid, $outfileno) = $fork_compressor_pipe->($self, $comp, $outfileno); |
91bd6c90 DM |
932 | } |
933 | ||
e8705fc5 TL |
934 | my $qmpclient = PVE::QMPClient->new(); |
935 | my $backup_cb = sub { | |
936 | my ($vmid, $resp) = @_; | |
937 | $backup_job_uuid = $resp->{return}->{UUID}; | |
938 | }; | |
939 | my $add_fd_cb = sub { | |
91bd6c90 DM |
940 | my ($vmid, $resp) = @_; |
941 | ||
c05f7f3f WL |
942 | my $params = { |
943 | 'backup-file' => "/dev/fdname/backup", | |
944 | speed => $speed, | |
945 | 'config-file' => $conffile, | |
946 | devlist => $devlist | |
947 | }; | |
c05f7f3f | 948 | $params->{'firewall-file'} = $firewall if -e $firewall; |
5d857282 | 949 | $params->{fleecing} = JSON::true if $use_fleecing; |
4de4eefc | 950 | add_backup_performance_options($params, $opts->{performance}, $qemu_support); |
d11e91d2 | 951 | |
c05f7f3f | 952 | $qmpclient->queue_cmd($vmid, $backup_cb, 'backup', %$params); |
91bd6c90 DM |
953 | }; |
954 | ||
d11e91d2 | 955 | $qmpclient->queue_cmd($vmid, $add_fd_cb, 'getfd', fd => $outfileno, fdname => "backup"); |
ab6a9a0c | 956 | |
cb521f2d | 957 | my $fs_frozen = $self->qga_fs_freeze($task, $vmid); |
874a096e | 958 | |
c5983223 | 959 | eval { $qmpclient->queue_execute(30) }; |
f0f30448 | 960 | my $qmperr = $@; |
91bd6c90 | 961 | |
1ece829a TL |
962 | if ($fs_frozen) { |
963 | $self->qga_fs_thaw($vmid); | |
ab6a9a0c | 964 | } |
d11e91d2 | 965 | |
f0f30448 | 966 | die $qmperr if $qmperr; |
874a096e | 967 | die $qmpclient->{errors}->{$vmid} if $qmpclient->{errors}->{$vmid}; |
91bd6c90 DM |
968 | |
969 | if ($cpid) { | |
874a096e | 970 | POSIX::close($outfileno) == 0 || |
91bd6c90 DM |
971 | die "close output file handle failed\n"; |
972 | } | |
973 | ||
d2cc2cbe | 974 | die "got no uuid for backup task\n" if !defined($backup_job_uuid); |
91bd6c90 | 975 | |
d2cc2cbe | 976 | $self->loginfo("started backup task '$backup_job_uuid'"); |
91bd6c90 | 977 | |
0b2f574b | 978 | $self->resume_vm_after_job_start($task, $vmid); |
91bd6c90 | 979 | |
d2cc2cbe | 980 | $query_backup_status_loop->($self, $vmid, $backup_job_uuid); |
91bd6c90 DM |
981 | }; |
982 | my $err = $@; | |
19599cd9 | 983 | if ($err) { |
60635a57 | 984 | $self->logerr($err); |
edae1718 | 985 | $self->mon_backup_cancel($vmid); |
ebce5239 | 986 | $self->resume_vm_after_job_start($task, $vmid); |
19599cd9 DM |
987 | } |
988 | ||
0b2f574b | 989 | $self->restore_vm_power_state($vmid); |
91bd6c90 | 990 | |
5d857282 FE |
991 | if ($use_fleecing) { |
992 | detach_fleecing_images($task->{disks}, $vmid); | |
993 | cleanup_fleecing_images($self, $task->{disks}); | |
994 | } | |
995 | ||
91bd6c90 | 996 | if ($err) { |
874a096e DM |
997 | if ($cpid) { |
998 | kill(9, $cpid); | |
91bd6c90 DM |
999 | waitpid($cpid, 0); |
1000 | } | |
1001 | die $err; | |
1002 | } | |
1003 | ||
1004 | if ($cpid && (waitpid($cpid, 0) > 0)) { | |
1005 | my $stat = $?; | |
1006 | my $ec = $stat >> 8; | |
1007 | my $signal = $stat & 127; | |
1008 | if ($ec || $signal) { | |
874a096e | 1009 | die "$comp failed - wrong exit status $ec" . |
91bd6c90 DM |
1010 | ($signal ? " (signal $signal)\n" : "\n"); |
1011 | } | |
1012 | } | |
1013 | } | |
1014 | ||
81dcd479 TL |
1015 | sub _get_task_devlist { |
1016 | my ($task) = @_; | |
1017 | ||
1018 | my $devlist = ''; | |
1019 | foreach my $di (@{$task->{disks}}) { | |
1020 | if ($di->{type} eq 'block' || $di->{type} eq 'file') { | |
1021 | $devlist .= ',' if $devlist; | |
1022 | $devlist .= $di->{qmdevice}; | |
1023 | } else { | |
1024 | die "implement me (type '$di->{type}')"; | |
1025 | } | |
1026 | } | |
1027 | return $devlist; | |
1028 | } | |
1029 | ||
1ece829a | 1030 | sub qga_fs_freeze { |
cb521f2d | 1031 | my ($self, $task, $vmid) = @_; |
b08c37c3 | 1032 | return if !$self->{vmlist}->{$vmid}->{agent} || $task->{mode} eq 'stop' || !$self->{vm_was_running} || $self->{vm_was_paused}; |
1ece829a TL |
1033 | |
1034 | if (!PVE::QemuServer::qga_check_running($vmid, 1)) { | |
1035 | $self->loginfo("skipping guest-agent 'fs-freeze', agent configured but not running?"); | |
1036 | return; | |
1037 | } | |
1038 | ||
93e21fd2 CH |
1039 | my $freeze = PVE::QemuServer::get_qga_key($self->{vmlist}->{$vmid}, 'freeze-fs-on-backup') // 1; |
1040 | if (!$freeze) { | |
1041 | $self->loginfo("skipping guest-agent 'fs-freeze', disabled in VM options"); | |
1042 | return; | |
1043 | } | |
1044 | ||
1ece829a TL |
1045 | $self->loginfo("issuing guest-agent 'fs-freeze' command"); |
1046 | eval { mon_cmd($vmid, "guest-fsfreeze-freeze") }; | |
1047 | $self->logerr($@) if $@; | |
1048 | ||
1049 | return 1; # even on mon command error, ensure we always thaw again | |
1050 | } | |
1051 | ||
1052 | # only call if fs_freeze return 1 | |
1053 | sub qga_fs_thaw { | |
1054 | my ($self, $vmid) = @_; | |
1055 | ||
1056 | $self->loginfo("issuing guest-agent 'fs-thaw' command"); | |
1057 | eval { mon_cmd($vmid, "guest-fsfreeze-thaw") }; | |
1058 | $self->logerr($@) if $@; | |
1059 | } | |
1060 | ||
0b2f574b TL |
1061 | # we need a running QEMU/KVM process for backup, starts a paused (prelaunch) |
1062 | # one if VM isn't already running | |
1063 | sub enforce_vm_running_for_backup { | |
1064 | my ($self, $vmid) = @_; | |
1065 | ||
1066 | if (PVE::QemuServer::check_running($vmid)) { | |
1067 | $self->{vm_was_running} = 1; | |
1068 | return; | |
1069 | } | |
1070 | ||
1071 | eval { | |
1072 | $self->loginfo("starting kvm to execute backup task"); | |
1073 | # start with skiplock | |
0c498cca FG |
1074 | my $params = { |
1075 | skiplock => 1, | |
4ef13a7f | 1076 | skiptemplate => 1, |
0c498cca FG |
1077 | paused => 1, |
1078 | }; | |
1079 | PVE::QemuServer::vm_start($self->{storecfg}, $vmid, $params); | |
0b2f574b TL |
1080 | }; |
1081 | die $@ if $@; | |
1082 | } | |
1083 | ||
ebce5239 | 1084 | # resume VM again once in a clear state (stop mode backup of running VM) |
0b2f574b TL |
1085 | sub resume_vm_after_job_start { |
1086 | my ($self, $task, $vmid) = @_; | |
1087 | ||
b08c37c3 | 1088 | return if !$self->{vm_was_running} || $self->{vm_was_paused}; |
0b2f574b TL |
1089 | |
1090 | if (my $stoptime = $task->{vmstoptime}) { | |
1091 | my $delay = time() - $task->{vmstoptime}; | |
1092 | $task->{vmstoptime} = undef; # avoid printing 'online after ..' twice | |
1093 | $self->loginfo("resuming VM again after $delay seconds"); | |
1094 | } else { | |
1095 | $self->loginfo("resuming VM again"); | |
1096 | } | |
6a558901 | 1097 | mon_cmd($vmid, 'cont', timeout => 45); |
0b2f574b TL |
1098 | } |
1099 | ||
1100 | # stop again if VM was not running before | |
1101 | sub restore_vm_power_state { | |
1102 | my ($self, $vmid) = @_; | |
1103 | ||
1104 | # we always let VMs keep running | |
1105 | return if $self->{vm_was_running}; | |
1106 | ||
1107 | eval { | |
1108 | my $resp = mon_cmd($vmid, 'query-status'); | |
1109 | my $status = $resp && $resp->{status} ? $resp->{status} : 'unknown'; | |
1110 | if ($status eq 'prelaunch') { | |
1111 | $self->loginfo("stopping kvm after backup task"); | |
1112 | PVE::QemuServer::vm_stop($self->{storecfg}, $vmid, 1); | |
1113 | } else { | |
1114 | $self->loginfo("kvm status changed after backup ('$status') - keep VM running"); | |
1115 | } | |
1116 | }; | |
1117 | warn $@ if $@; | |
1118 | } | |
1119 | ||
1120 | sub mon_backup_cancel { | |
1121 | my ($self, $vmid) = @_; | |
1122 | ||
1123 | $self->loginfo("aborting backup job"); | |
1124 | eval { mon_cmd($vmid, 'backup-cancel') }; | |
1125 | $self->logerr($@) if $@; | |
1126 | } | |
1127 | ||
91bd6c90 DM |
1128 | sub snapshot { |
1129 | my ($self, $task, $vmid) = @_; | |
1130 | ||
1131 | # nothing to do | |
1e3baf05 DM |
1132 | } |
1133 | ||
1134 | sub cleanup { | |
1135 | my ($self, $task, $vmid) = @_; | |
1136 | ||
f9dde219 SR |
1137 | $detach_tpmstate_drive->($task, $vmid); |
1138 | ||
4ac842cb SR |
1139 | if ($self->{qmeventd_fh}) { |
1140 | close($self->{qmeventd_fh}); | |
1141 | } | |
1e3baf05 DM |
1142 | } |
1143 | ||
1144 | 1; |