]>
Commit | Line | Data |
---|---|---|
1e3baf05 DM |
1 | package PVE::VZDump::QemuServer; |
2 | ||
1e3baf05 DM |
3 | use strict; |
4 | use warnings; | |
d610b145 | 5 | |
1e3baf05 | 6 | use File::Basename; |
d610b145 TL |
7 | use File::Path; |
8 | use IO::File; | |
9 | use IPC::Open3; | |
69e62894 | 10 | use JSON; |
4ac842cb | 11 | use POSIX qw(EINTR EAGAIN); |
d610b145 TL |
12 | |
13 | use PVE::Cluster qw(cfs_read_file); | |
66ab1d91 | 14 | use PVE::INotify; |
91bd6c90 | 15 | use PVE::IPCC; |
d610b145 | 16 | use PVE::JSONSchema; |
fbec3f89 | 17 | use PVE::PBSClient; |
0a13e08e | 18 | use PVE::QMPClient; |
f5bdefa4 | 19 | use PVE::Storage::Plugin; |
c5983223 | 20 | use PVE::Storage::PBSPlugin; |
1e3baf05 | 21 | use PVE::Storage; |
d610b145 TL |
22 | use PVE::Tools; |
23 | use PVE::VZDump; | |
f1aca33d | 24 | use PVE::Format qw(render_duration render_bytes); |
d610b145 | 25 | |
912792e2 | 26 | use PVE::QemuConfig; |
1e3baf05 | 27 | use PVE::QemuServer; |
3392d6ca | 28 | use PVE::QemuServer::Machine; |
0a13e08e | 29 | use PVE::QemuServer::Monitor qw(mon_cmd); |
1e3baf05 DM |
30 | |
31 | use base qw (PVE::VZDump::Plugin); | |
32 | ||
33 | sub new { | |
34 | my ($class, $vzdump) = @_; | |
874a096e | 35 | |
66ab1d91 | 36 | PVE::VZDump::check_bin('qm'); |
1e3baf05 | 37 | |
e2812738 | 38 | my $self = bless { vzdump => $vzdump }, $class; |
1e3baf05 DM |
39 | |
40 | $self->{vmlist} = PVE::QemuServer::vzlist(); | |
41 | $self->{storecfg} = PVE::Storage::config(); | |
42 | ||
43 | return $self; | |
44 | }; | |
45 | ||
1e3baf05 DM |
46 | sub type { |
47 | return 'qemu'; | |
48 | } | |
49 | ||
50 | sub vmlist { | |
51 | my ($self) = @_; | |
1e3baf05 DM |
52 | return [ keys %{$self->{vmlist}} ]; |
53 | } | |
54 | ||
55 | sub prepare { | |
56 | my ($self, $task, $vmid, $mode) = @_; | |
57 | ||
58 | $task->{disks} = []; | |
59 | ||
ffda963f | 60 | my $conf = $self->{vmlist}->{$vmid} = PVE::QemuConfig->load_config($vmid); |
1e3baf05 | 61 | |
85b84b7b WL |
62 | $self->loginfo("VM Name: $conf->{name}") |
63 | if defined($conf->{name}); | |
64 | ||
91bd6c90 | 65 | $self->{vm_was_running} = 1; |
b08c37c3 | 66 | $self->{vm_was_paused} = 0; |
91bd6c90 DM |
67 | if (!PVE::QemuServer::check_running($vmid)) { |
68 | $self->{vm_was_running} = 0; | |
b08c37c3 DC |
69 | } elsif (PVE::QemuServer::vm_is_paused($vmid)) { |
70 | $self->{vm_was_paused} = 1; | |
585b6e28 DM |
71 | } |
72 | ||
1e3baf05 DM |
73 | $task->{hostname} = $conf->{name}; |
74 | ||
874a096e | 75 | my $hostname = PVE::INotify::nodename(); |
1e3baf05 | 76 | |
b969cc68 DM |
77 | my $vollist = []; |
78 | my $drivehash = {}; | |
185df962 | 79 | my $backup_volumes = PVE::QemuConfig->get_backup_volumes($conf); |
1e3baf05 | 80 | |
185df962 AL |
81 | foreach my $volume (@{$backup_volumes}) { |
82 | my $name = $volume->{key}; | |
5a92276e | 83 | my $volume_config = $volume->{volume_config}; |
185df962 | 84 | my $volid = $volume_config->{file}; |
1e3baf05 | 85 | |
185df962 AL |
86 | if (!$volume->{included}) { |
87 | $self->loginfo("exclude disk '$name' '$volid' ($volume->{reason})"); | |
88 | next; | |
f9dde219 SR |
89 | } elsif ($self->{vm_was_running} && $volume_config->{iothread} && |
90 | !PVE::QemuServer::Machine::runs_at_least_qemu_version($vmid, 4, 0, 1)) { | |
91 | die "disk '$name' '$volid' (iothread=on) can't use backup feature with running QEMU " . | |
92 | "version < 4.0.1! Either set backup=no for this drive or upgrade QEMU and restart VM\n"; | |
b53b958b | 93 | } else { |
185df962 AL |
94 | my $log = "include disk '$name' '$volid'"; |
95 | if (defined(my $size = $volume_config->{size})) { | |
96 | my $readable_size = PVE::JSONSchema::format_size($size); | |
0db93c2d | 97 | $log .= " $readable_size"; |
185df962 | 98 | } |
0db93c2d | 99 | $self->loginfo($log); |
874a096e | 100 | } |
b969cc68 | 101 | |
f5bdefa4 | 102 | my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1); |
b969cc68 | 103 | push @$vollist, $volid if $storeid; |
185df962 AL |
104 | $drivehash->{$name} = $volume->{volume_config}; |
105 | } | |
b969cc68 DM |
106 | |
107 | PVE::Storage::activate_volumes($self->{storecfg}, $vollist); | |
108 | ||
075b417a DM |
109 | foreach my $ds (sort keys %$drivehash) { |
110 | my $drive = $drivehash->{$ds}; | |
874a096e | 111 | |
1e3baf05 | 112 | my $volid = $drive->{file}; |
f5bdefa4 | 113 | my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1); |
a113a58a TL |
114 | |
115 | my $path = $volid; | |
1e3baf05 | 116 | if ($storeid) { |
b969cc68 | 117 | $path = PVE::Storage::path($self->{storecfg}, $volid); |
1e3baf05 | 118 | } |
b969cc68 | 119 | next if !$path; |
1e3baf05 | 120 | |
a113a58a | 121 | my ($size, $format) = eval { PVE::Storage::volume_size_info($self->{storecfg}, $volid, 5) }; |
daca220d | 122 | die "no such volume '$volid'\n" if $@; |
91bd6c90 | 123 | |
a113a58a TL |
124 | my $diskinfo = { |
125 | path => $path, | |
126 | volid => $volid, | |
127 | storeid => $storeid, | |
128 | format => $format, | |
129 | virtdev => $ds, | |
130 | qmdevice => "drive-$ds", | |
131 | }; | |
1e3baf05 | 132 | |
f9dde219 SR |
133 | if ($ds eq 'tpmstate0') { |
134 | # TPM drive only exists for backup, which is reflected in the name | |
135 | $diskinfo->{qmdevice} = 'drive-tpmstate0-backup'; | |
136 | $task->{tpmpath} = $path; | |
137 | } | |
138 | ||
1e3baf05 | 139 | if (-b $path) { |
1e3baf05 | 140 | $diskinfo->{type} = 'block'; |
1e3baf05 | 141 | } else { |
1e3baf05 | 142 | $diskinfo->{type} = 'file'; |
1e3baf05 DM |
143 | } |
144 | ||
145 | push @{$task->{disks}}, $diskinfo; | |
b969cc68 | 146 | } |
1e3baf05 DM |
147 | } |
148 | ||
149 | sub vm_status { | |
150 | my ($self, $vmid) = @_; | |
151 | ||
66ab1d91 | 152 | my $running = PVE::QemuServer::check_running($vmid) ? 1 : 0; |
874a096e DM |
153 | |
154 | return wantarray ? ($running, $running ? 'running' : 'stopped') : $running; | |
1e3baf05 DM |
155 | } |
156 | ||
157 | sub lock_vm { | |
158 | my ($self, $vmid) = @_; | |
159 | ||
f301bc0d | 160 | PVE::QemuConfig->set_lock($vmid, 'backup'); |
1e3baf05 DM |
161 | } |
162 | ||
163 | sub unlock_vm { | |
164 | my ($self, $vmid) = @_; | |
165 | ||
f301bc0d | 166 | PVE::QemuConfig->remove_lock($vmid, 'backup'); |
1e3baf05 DM |
167 | } |
168 | ||
169 | sub stop_vm { | |
170 | my ($self, $task, $vmid) = @_; | |
171 | ||
172 | my $opts = $self->{vzdump}->{opts}; | |
173 | ||
174 | my $wait = $opts->{stopwait} * 60; | |
175 | # send shutdown and wait | |
254575e9 | 176 | $self->cmd ("qm shutdown $vmid --skiplock --keepActive --timeout $wait"); |
1e3baf05 DM |
177 | } |
178 | ||
179 | sub start_vm { | |
180 | my ($self, $task, $vmid) = @_; | |
181 | ||
66ab1d91 | 182 | $self->cmd ("qm start $vmid --skiplock"); |
1e3baf05 DM |
183 | } |
184 | ||
185 | sub suspend_vm { | |
186 | my ($self, $task, $vmid) = @_; | |
187 | ||
b08c37c3 DC |
188 | return if $self->{vm_was_paused}; |
189 | ||
66ab1d91 | 190 | $self->cmd ("qm suspend $vmid --skiplock"); |
1e3baf05 DM |
191 | } |
192 | ||
193 | sub resume_vm { | |
194 | my ($self, $task, $vmid) = @_; | |
195 | ||
b08c37c3 DC |
196 | return if $self->{vm_was_paused}; |
197 | ||
66ab1d91 | 198 | $self->cmd ("qm resume $vmid --skiplock"); |
1e3baf05 DM |
199 | } |
200 | ||
1e3baf05 DM |
201 | sub assemble { |
202 | my ($self, $task, $vmid) = @_; | |
203 | ||
04096e7b | 204 | my $conffile = PVE::QemuConfig->config_file($vmid); |
1e3baf05 DM |
205 | |
206 | my $outfile = "$task->{tmpdir}/qemu-server.conf"; | |
c05f7f3f WL |
207 | my $firewall_src = "/etc/pve/firewall/$vmid.fw"; |
208 | my $firewall_dest = "$task->{tmpdir}/qemu-server.fw"; | |
209 | ||
210 | my $outfd = IO::File->new (">$outfile") || | |
211 | die "unable to open '$outfile'"; | |
212 | my $conffd = IO::File->new ($conffile, 'r') || | |
213 | die "unable open '$conffile'"; | |
214 | ||
215 | my $found_snapshot; | |
391c2230 | 216 | my $found_pending; |
c05f7f3f WL |
217 | while (defined (my $line = <$conffd>)) { |
218 | next if $line =~ m/^\#vzdump\#/; # just to be sure | |
219 | next if $line =~ m/^\#qmdump\#/; # just to be sure | |
391c2230 FG |
220 | if ($line =~ m/^\[(.*)\]\s*$/) { |
221 | if ($1 =~ m/PENDING/i) { | |
222 | $found_pending = 1; | |
223 | } else { | |
224 | $found_snapshot = 1; | |
225 | } | |
1e3baf05 | 226 | } |
1bd6fe13 | 227 | next if $found_snapshot || $found_pending; # skip all snapshots and pending changes config data |
391c2230 | 228 | |
c05f7f3f WL |
229 | if ($line =~ m/^unused\d+:\s*(\S+)\s*/) { |
230 | $self->loginfo("skip unused drive '$1' (not included into backup)"); | |
231 | next; | |
1e3baf05 | 232 | } |
c05f7f3f | 233 | next if $line =~ m/^lock:/ || $line =~ m/^parent:/; |
91bd6c90 | 234 | |
c05f7f3f WL |
235 | print $outfd $line; |
236 | } | |
237 | ||
238 | foreach my $di (@{$task->{disks}}) { | |
239 | if ($di->{type} eq 'block' || $di->{type} eq 'file') { | |
240 | my $storeid = $di->{storeid} || ''; | |
241 | my $format = $di->{format} || ''; | |
242 | print $outfd "#qmdump#map:$di->{virtdev}:$di->{qmdevice}:$storeid:$format:\n"; | |
243 | } else { | |
244 | die "internal error"; | |
91bd6c90 | 245 | } |
c05f7f3f | 246 | } |
1e3baf05 | 247 | |
c05f7f3f WL |
248 | if ($found_snapshot) { |
249 | $self->loginfo("snapshots found (not included into backup)"); | |
250 | } | |
391c2230 FG |
251 | if ($found_pending) { |
252 | $self->loginfo("pending configuration changes found (not included into backup)"); | |
253 | } | |
254 | ||
c05f7f3f | 255 | PVE::Tools::file_copy($firewall_src, $firewall_dest) if -f $firewall_src; |
1e3baf05 DM |
256 | } |
257 | ||
258 | sub archive { | |
fad02a16 | 259 | my ($self, $task, $vmid, $filename, $comp) = @_; |
1e3baf05 | 260 | |
c5983223 | 261 | my $opts = $self->{vzdump}->{opts}; |
c5983223 DM |
262 | my $scfg = $opts->{scfg}; |
263 | ||
69074863 | 264 | if ($self->{vzdump}->{opts}->{pbs}) { |
c5983223 DM |
265 | $self->archive_pbs($task, $vmid); |
266 | } else { | |
267 | $self->archive_vma($task, $vmid, $filename, $comp); | |
268 | } | |
269 | } | |
270 | ||
3a44897f | 271 | my $bitmap_action_to_human = sub { |
13ddc7eb | 272 | my ($self, $info) = @_; |
3a44897f SR |
273 | |
274 | my $action = $info->{action}; | |
275 | ||
276 | if ($action eq "not-used") { | |
962d4d64 | 277 | return "disabled (no support)"; |
3a44897f SR |
278 | } elsif ($action eq "not-used-removed") { |
279 | return "disabled (old bitmap cleared)"; | |
280 | } elsif ($action eq "new") { | |
13ddc7eb | 281 | return "created new"; |
3a44897f SR |
282 | } elsif ($action eq "used") { |
283 | if ($info->{dirty} == 0) { | |
13ddc7eb | 284 | return "OK (drive clean)"; |
3a44897f | 285 | } else { |
f1aca33d SR |
286 | my $size = render_bytes($info->{size}, 1); |
287 | my $dirty = render_bytes($info->{dirty}, 1); | |
13ddc7eb | 288 | return "OK ($dirty of $size dirty)"; |
3a44897f SR |
289 | } |
290 | } elsif ($action eq "invalid") { | |
291 | return "existing bitmap was invalid and has been cleared"; | |
292 | } else { | |
293 | return "unknown"; | |
294 | } | |
295 | }; | |
296 | ||
c5983223 | 297 | my $query_backup_status_loop = sub { |
f8566714 | 298 | my ($self, $vmid, $job_uuid, $qemu_support) = @_; |
c5983223 | 299 | |
c5983223 | 300 | my $starttime = time (); |
09eb196b | 301 | my $last_time = $starttime; |
8825248c | 302 | my ($last_percent, $last_total, $last_target, $last_zero, $last_transferred) = (-1, 0, 0, 0, 0); |
4d159c24 | 303 | my ($transferred, $reused); |
c5983223 | 304 | |
09eb196b TL |
305 | my $get_mbps = sub { |
306 | my ($mb, $delta) = @_; | |
4d159c24 TL |
307 | return "0 B/s" if $mb <= 0; |
308 | my $bw = int(($mb / $delta)); | |
f1aca33d | 309 | return render_bytes($bw, 1) . "/s"; |
09eb196b TL |
310 | }; |
311 | ||
3a44897f | 312 | my $target = 0; |
8825248c | 313 | my $last_reused = 0; |
f8566714 | 314 | my $has_query_bitmap = $qemu_support && $qemu_support->{'query-bitmap-info'}; |
5749c383 | 315 | my $is_template = PVE::QemuConfig->is_template($self->{vmlist}->{$vmid}); |
f8566714 | 316 | if ($has_query_bitmap) { |
13ddc7eb | 317 | my $total = 0; |
3a44897f | 318 | my $bitmap_info = mon_cmd($vmid, 'query-pbs-bitmap-info'); |
5749c383 TL |
319 | for my $info (sort { $a->{drive} cmp $b->{drive} } @$bitmap_info) { |
320 | if (!$is_template) { | |
321 | my $text = $bitmap_action_to_human->($self, $info); | |
322 | my $drive = $info->{drive}; | |
323 | $drive =~ s/^drive-//; # for consistency | |
324 | $self->loginfo("$drive: dirty-bitmap status: $text"); | |
325 | } | |
3a44897f | 326 | $target += $info->{dirty}; |
13ddc7eb | 327 | $total += $info->{size}; |
8825248c | 328 | $last_reused += $info->{size} - $info->{dirty}; |
3a44897f | 329 | } |
13ddc7eb | 330 | if ($target < $total) { |
f1aca33d SR |
331 | my $total_h = render_bytes($total, 1); |
332 | my $target_h = render_bytes($target, 1); | |
13ddc7eb TL |
333 | $self->loginfo("using fast incremental mode (dirty-bitmap), $target_h dirty of $total_h total"); |
334 | } | |
3a44897f SR |
335 | } |
336 | ||
1eb7e590 | 337 | my $last_finishing = 0; |
c5983223 | 338 | while(1) { |
09eb196b TL |
339 | my $status = mon_cmd($vmid, 'query-backup'); |
340 | ||
c5983223 | 341 | my $total = $status->{total} || 0; |
f8566714 TL |
342 | my $dirty = $status->{dirty}; |
343 | $target = (defined($dirty) && $dirty < $total) ? $dirty : $total if !$has_query_bitmap; | |
c5983223 | 344 | $transferred = $status->{transferred} || 0; |
4d159c24 | 345 | $reused = $status->{reused}; |
0f6c6981 | 346 | my $percent = $target ? int(($transferred * 100)/$target) : 100; |
c5983223 | 347 | my $zero = $status->{'zero-bytes'} || 0; |
c5983223 DM |
348 | |
349 | die "got unexpected uuid\n" if !$status->{uuid} || ($status->{uuid} ne $job_uuid); | |
350 | ||
351 | my $ctime = time(); | |
352 | my $duration = $ctime - $starttime; | |
353 | ||
354 | my $rbytes = $transferred - $last_transferred; | |
8825248c SR |
355 | my $wbytes; |
356 | if ($reused) { | |
357 | # reused includes zero bytes for PBS | |
358 | $wbytes = $rbytes - ($reused - $last_reused); | |
359 | } else { | |
360 | $wbytes = $rbytes - ($zero - $last_zero); | |
361 | } | |
c5983223 DM |
362 | |
363 | my $timediff = ($ctime - $last_time) || 1; # fixme | |
09eb196b TL |
364 | my $mbps_read = $get_mbps->($rbytes, $timediff); |
365 | my $mbps_write = $get_mbps->($wbytes, $timediff); | |
f1aca33d SR |
366 | my $target_h = render_bytes($target, 1); |
367 | my $transferred_h = render_bytes($transferred, 1); | |
4d159c24 | 368 | |
a40295b6 | 369 | my $statusline = sprintf("%3d%% ($transferred_h of $target_h) in %s" |
f1aca33d | 370 | .", read: $mbps_read, write: $mbps_write", $percent, render_duration($duration)); |
09eb196b | 371 | |
c5983223 DM |
372 | my $res = $status->{status} || 'unknown'; |
373 | if ($res ne 'active') { | |
1eb7e590 SR |
374 | if ($last_percent < 100) { |
375 | $self->loginfo($statusline); | |
376 | } | |
09eb196b TL |
377 | if ($res ne 'done') { |
378 | die (($status->{errmsg} || "unknown error") . "\n") if $res eq 'error'; | |
379 | die "got unexpected status '$res'\n"; | |
09eb196b | 380 | } |
6cdb568c TL |
381 | $last_target = $target if $target; |
382 | $last_total = $total if $total; | |
383 | $last_zero = $zero if $zero; | |
384 | $last_transferred = $transferred if $transferred; | |
c5983223 DM |
385 | last; |
386 | } | |
09eb196b | 387 | if ($percent != $last_percent && ($timediff > 2)) { |
c5983223 | 388 | $self->loginfo($statusline); |
09eb196b | 389 | $last_percent = $percent; |
4d159c24 | 390 | $last_target = $target if $target; |
c5983223 DM |
391 | $last_total = $total if $total; |
392 | $last_zero = $zero if $zero; | |
393 | $last_transferred = $transferred if $transferred; | |
394 | $last_time = $ctime; | |
8825248c | 395 | $last_reused = $reused; |
1eb7e590 SR |
396 | |
397 | if (!$last_finishing && $status->{finishing}) { | |
d5b0cfb1 | 398 | $self->loginfo("Waiting for server to finish backup validation..."); |
1eb7e590 SR |
399 | } |
400 | $last_finishing = $status->{finishing}; | |
c5983223 DM |
401 | } |
402 | sleep(1); | |
403 | } | |
404 | ||
405 | my $duration = time() - $starttime; | |
d35412a3 TL |
406 | |
407 | if ($last_zero) { | |
408 | my $zero_per = $last_target ? int(($last_zero * 100)/$last_target) : 0; | |
f1aca33d | 409 | my $zero_h = render_bytes($last_zero); |
d35412a3 TL |
410 | $self->loginfo("backup is sparse: $zero_h (${zero_per}%) total zero data"); |
411 | } | |
1ca43f1c | 412 | if ($reused) { |
f1aca33d | 413 | my $reused_h = render_bytes($reused); |
1ca43f1c SR |
414 | my $reuse_per = int($reused * 100 / $last_total); |
415 | $self->loginfo("backup was done incrementally, reused $reused_h (${reuse_per}%)"); | |
416 | } | |
bafae3ec | 417 | if ($transferred) { |
f1aca33d | 418 | my $transferred_h = render_bytes($transferred); |
bafae3ec TL |
419 | if ($duration) { |
420 | my $mbps = $get_mbps->($transferred, $duration); | |
421 | $self->loginfo("transferred $transferred_h in $duration seconds ($mbps)"); | |
422 | } else { | |
423 | $self->loginfo("transferred $transferred_h in <1 seconds"); | |
424 | } | |
4d159c24 TL |
425 | } |
426 | ||
b4be9c02 TL |
427 | return { |
428 | total => $last_total, | |
429 | reused => $reused, | |
430 | }; | |
c5983223 DM |
431 | }; |
432 | ||
f9dde219 SR |
433 | my $attach_tpmstate_drive = sub { |
434 | my ($self, $task, $vmid) = @_; | |
435 | ||
436 | return if !$task->{tpmpath}; | |
437 | ||
438 | # unconditionally try to remove the tpmstate-named drive - it only exists | |
439 | # for backing up, and avoids errors if left over from some previous event | |
440 | eval { PVE::QemuServer::qemu_drivedel($vmid, "tpmstate0-backup"); }; | |
441 | ||
442 | $self->loginfo('attaching TPM drive to QEMU for backup'); | |
443 | ||
444 | my $drive = "file=$task->{tpmpath},if=none,read-only=on,id=drive-tpmstate0-backup"; | |
445 | my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\""); | |
446 | die "attaching TPM drive failed\n" if $ret !~ m/OK/s; | |
447 | }; | |
448 | ||
449 | my $detach_tpmstate_drive = sub { | |
450 | my ($task, $vmid) = @_; | |
451 | return if !$task->{tpmpath} || !PVE::QemuServer::check_running($vmid); | |
452 | eval { PVE::QemuServer::qemu_drivedel($vmid, "tpmstate0-backup"); }; | |
453 | }; | |
454 | ||
c5983223 DM |
455 | sub archive_pbs { |
456 | my ($self, $task, $vmid) = @_; | |
457 | ||
1e3baf05 | 458 | my $conffile = "$task->{tmpdir}/qemu-server.conf"; |
c05f7f3f | 459 | my $firewall = "$task->{tmpdir}/qemu-server.fw"; |
1e3baf05 DM |
460 | |
461 | my $opts = $self->{vzdump}->{opts}; | |
c5983223 DM |
462 | my $scfg = $opts->{scfg}; |
463 | ||
464 | my $starttime = time(); | |
465 | ||
c5983223 | 466 | my $fingerprint = $scfg->{fingerprint}; |
fbec3f89 | 467 | my $repo = PVE::PBSClient::get_repository($scfg); |
c5983223 | 468 | my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $opts->{storage}); |
ece74030 | 469 | my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($scfg, $opts->{storage}); |
9bf522bc | 470 | my $master_keyfile = PVE::Storage::PBSPlugin::pbs_master_pubkey_file_name($scfg, $opts->{storage}); |
c5983223 | 471 | |
d11e91d2 | 472 | my $diskcount = scalar(@{$task->{disks}}); |
4ef13a7f FG |
473 | # proxmox-backup-client can only handle raw files and block devs |
474 | # only use it (directly) for disk-less VMs | |
475 | if (!$diskcount) { | |
c5983223 | 476 | my @pathlist; |
4ef13a7f | 477 | $self->loginfo("backup contains no disks"); |
c5983223 DM |
478 | |
479 | local $ENV{PBS_PASSWORD} = $password; | |
77b24c62 | 480 | local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint); |
c5983223 DM |
481 | my $cmd = [ |
482 | '/usr/bin/proxmox-backup-client', | |
483 | 'backup', | |
484 | '--repository', $repo, | |
485 | '--backup-type', 'vm', | |
486 | '--backup-id', "$vmid", | |
487 | '--backup-time', $task->{backup_time}, | |
d11e91d2 | 488 | ]; |
c5983223 DM |
489 | |
490 | push @$cmd, "qemu-server.conf:$conffile"; | |
491 | push @$cmd, "fw.conf:$firewall" if -e $firewall; | |
c5983223 DM |
492 | |
493 | $self->loginfo("starting template backup"); | |
494 | $self->loginfo(join(' ', @$cmd)); | |
495 | ||
496 | $self->cmd($cmd); | |
497 | ||
498 | return; | |
499 | } | |
500 | ||
81dcd479 TL |
501 | # get list early so we die on unkown drive types before doing anything |
502 | my $devlist = _get_task_devlist($task); | |
c5983223 | 503 | |
0b2f574b | 504 | $self->enforce_vm_running_for_backup($vmid); |
65911545 | 505 | $self->{qmeventd_fh} = PVE::QemuServer::register_qmeventd_handle($vmid); |
c5983223 | 506 | |
f6168f1a | 507 | my $backup_job_uuid; |
c5983223 DM |
508 | eval { |
509 | $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub { | |
d11e91d2 | 510 | die "interrupted by signal\n"; |
c5983223 DM |
511 | }; |
512 | ||
fb9f512c | 513 | my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") }; |
2cfb0905 SR |
514 | my $err = $@; |
515 | if (!$qemu_support || $err) { | |
516 | die "query-proxmox-support returned empty value\n" if !$err; | |
517 | if ($err =~ m/The command query-proxmox-support has not been found/) { | |
518 | die "PBS backups are not supported by the running QEMU version. Please make " | |
519 | . "sure you've installed the latest version and the VM has been restarted.\n"; | |
520 | } else { | |
521 | die "QMP command query-proxmox-support failed - $err\n"; | |
522 | } | |
fb9f512c SR |
523 | } |
524 | ||
9bf522bc FG |
525 | if (!defined($qemu_support->{"pbs-masterkey"}) && -e $master_keyfile) { |
526 | $self->loginfo("WARNING: backup target is configured with master key, but running QEMU version does not support master keys."); | |
527 | $self->loginfo("Please make sure you've installed the latest version and the VM has been restarted to use master key feature."); | |
528 | $master_keyfile = undef; # skip rest of master key handling below | |
529 | } | |
530 | ||
f9dde219 SR |
531 | $attach_tpmstate_drive->($self, $task, $vmid); |
532 | ||
cb521f2d | 533 | my $fs_frozen = $self->qga_fs_freeze($task, $vmid); |
c5983223 | 534 | |
d11e91d2 TL |
535 | my $params = { |
536 | format => "pbs", | |
537 | 'backup-file' => $repo, | |
538 | 'backup-id' => "$vmid", | |
539 | 'backup-time' => $task->{backup_time}, | |
540 | password => $password, | |
541 | devlist => $devlist, | |
542 | 'config-file' => $conffile, | |
c5983223 | 543 | }; |
2790636a | 544 | $params->{speed} = $opts->{bwlimit}*1024 if $opts->{bwlimit}; |
d11e91d2 TL |
545 | $params->{fingerprint} = $fingerprint if defined($fingerprint); |
546 | $params->{'firewall-file'} = $firewall if -e $firewall; | |
ece74030 WB |
547 | if (-e $keyfile) { |
548 | $self->loginfo("enabling encryption"); | |
549 | $params->{keyfile} = $keyfile; | |
550 | $params->{encrypt} = JSON::true; | |
9bf522bc FG |
551 | if (defined($master_keyfile) && -e $master_keyfile) { |
552 | $self->loginfo("enabling master key feature"); | |
553 | $params->{"master-keyfile"} = $master_keyfile; | |
554 | } | |
ece74030 | 555 | } else { |
9bf522bc FG |
556 | $self->loginfo("WARNING: backup target is configured with master key, but this backup is not encrypted - master key settings will be ignored!") |
557 | if defined($master_keyfile) && -e $master_keyfile; | |
ece74030 WB |
558 | $params->{encrypt} = JSON::false; |
559 | } | |
c5983223 | 560 | |
4ef13a7f | 561 | my $is_template = PVE::QemuConfig->is_template($self->{vmlist}->{$vmid}); |
78179bda | 562 | $params->{'use-dirty-bitmap'} = JSON::true |
962d4d64 | 563 | if $qemu_support->{'pbs-dirty-bitmap'} && !$is_template; |
fb9f512c | 564 | |
46b676c0 | 565 | $params->{timeout} = 125; # give some time to connect to the backup server |
f57666e9 | 566 | |
d11e91d2 | 567 | my $res = eval { mon_cmd($vmid, "backup", %$params) }; |
c5983223 | 568 | my $qmperr = $@; |
d11e91d2 | 569 | $backup_job_uuid = $res->{UUID} if $res; |
c5983223 | 570 | |
1ece829a TL |
571 | if ($fs_frozen) { |
572 | $self->qga_fs_thaw($vmid); | |
c5983223 DM |
573 | } |
574 | ||
575 | die $qmperr if $qmperr; | |
f6168f1a | 576 | die "got no uuid for backup task\n" if !defined($backup_job_uuid); |
c5983223 | 577 | |
f6168f1a | 578 | $self->loginfo("started backup task '$backup_job_uuid'"); |
c5983223 | 579 | |
0b2f574b | 580 | $self->resume_vm_after_job_start($task, $vmid); |
c5983223 | 581 | |
3a44897f | 582 | my $stat = $query_backup_status_loop->($self, $vmid, $backup_job_uuid, $qemu_support); |
2098f2ff | 583 | $task->{size} = $stat->{total}; |
c5983223 DM |
584 | }; |
585 | my $err = $@; | |
c5983223 DM |
586 | if ($err) { |
587 | $self->logerr($err); | |
edae1718 | 588 | $self->mon_backup_cancel($vmid); |
ebce5239 | 589 | $self->resume_vm_after_job_start($task, $vmid); |
c5983223 | 590 | } |
0b2f574b | 591 | $self->restore_vm_power_state($vmid); |
c5983223 DM |
592 | |
593 | die $err if $err; | |
594 | } | |
595 | ||
02da0c65 TL |
596 | my $fork_compressor_pipe = sub { |
597 | my ($self, $comp, $outfileno) = @_; | |
598 | ||
599 | my @pipefd = POSIX::pipe(); | |
600 | my $cpid = fork(); | |
601 | die "unable to fork worker - $!" if !defined($cpid) || $cpid < 0; | |
602 | if ($cpid == 0) { | |
603 | eval { | |
604 | POSIX::close($pipefd[1]); | |
605 | # redirect STDIN | |
606 | my $fd = fileno(STDIN); | |
607 | close STDIN; | |
608 | POSIX::close(0) if $fd != 0; | |
609 | die "unable to redirect STDIN - $!" | |
610 | if !open(STDIN, "<&", $pipefd[0]); | |
611 | ||
612 | # redirect STDOUT | |
613 | $fd = fileno(STDOUT); | |
614 | close STDOUT; | |
615 | POSIX::close (1) if $fd != 1; | |
616 | ||
617 | die "unable to redirect STDOUT - $!" | |
618 | if !open(STDOUT, ">&", $outfileno); | |
619 | ||
620 | exec($comp); | |
621 | die "fork compressor '$comp' failed\n"; | |
622 | }; | |
623 | if (my $err = $@) { | |
624 | $self->logerr($err); | |
625 | POSIX::_exit(1); | |
626 | } | |
627 | POSIX::_exit(0); | |
628 | kill(-9, $$); | |
629 | } else { | |
630 | POSIX::close($pipefd[0]); | |
631 | $outfileno = $pipefd[1]; | |
632 | } | |
633 | ||
634 | return ($cpid, $outfileno); | |
635 | }; | |
636 | ||
c5983223 DM |
637 | sub archive_vma { |
638 | my ($self, $task, $vmid, $filename, $comp) = @_; | |
639 | ||
640 | my $conffile = "$task->{tmpdir}/qemu-server.conf"; | |
641 | my $firewall = "$task->{tmpdir}/qemu-server.fw"; | |
642 | ||
643 | my $opts = $self->{vzdump}->{opts}; | |
644 | ||
645 | my $starttime = time(); | |
1e3baf05 | 646 | |
91bd6c90 DM |
647 | my $speed = 0; |
648 | if ($opts->{bwlimit}) { | |
874a096e | 649 | $speed = $opts->{bwlimit}*1024; |
91bd6c90 | 650 | } |
1e3baf05 | 651 | |
c82935e9 | 652 | my $diskcount = scalar(@{$task->{disks}}); |
ffda963f | 653 | if (PVE::QemuConfig->is_template($self->{vmlist}->{$vmid}) || !$diskcount) { |
23b4120b DM |
654 | my @pathlist; |
655 | foreach my $di (@{$task->{disks}}) { | |
656 | if ($di->{type} eq 'block' || $di->{type} eq 'file') { | |
657 | push @pathlist, "$di->{qmdevice}=$di->{path}"; | |
658 | } else { | |
659 | die "implement me"; | |
660 | } | |
661 | } | |
662 | ||
c82935e9 DM |
663 | if (!$diskcount) { |
664 | $self->loginfo("backup contains no disks"); | |
665 | } | |
666 | ||
23b4120b DM |
667 | my $outcmd; |
668 | if ($comp) { | |
874a096e | 669 | $outcmd = "exec:$comp"; |
23b4120b | 670 | } else { |
874a096e | 671 | $outcmd = "exec:cat"; |
23b4120b DM |
672 | } |
673 | ||
a2fab11a | 674 | $outcmd .= " > $filename" if !$opts->{stdout}; |
23b4120b | 675 | |
c05f7f3f WL |
676 | my $cmd = ['/usr/bin/vma', 'create', '-v', '-c', $conffile]; |
677 | push @$cmd, '-c', $firewall if -e $firewall; | |
678 | push @$cmd, $outcmd, @pathlist; | |
23b4120b DM |
679 | |
680 | $self->loginfo("starting template backup"); | |
681 | $self->loginfo(join(' ', @$cmd)); | |
682 | ||
683 | if ($opts->{stdout}) { | |
6bb12239 | 684 | $self->cmd($cmd, output => ">&" . fileno($opts->{stdout})); |
23b4120b DM |
685 | } else { |
686 | $self->cmd($cmd); | |
687 | } | |
688 | ||
689 | return; | |
690 | } | |
691 | ||
81dcd479 | 692 | my $devlist = _get_task_devlist($task); |
1e3baf05 | 693 | |
0b2f574b | 694 | $self->enforce_vm_running_for_backup($vmid); |
65911545 | 695 | $self->{qmeventd_fh} = PVE::QemuServer::register_qmeventd_handle($vmid); |
91bd6c90 DM |
696 | |
697 | my $cpid; | |
d2cc2cbe DM |
698 | my $backup_job_uuid; |
699 | ||
91bd6c90 DM |
700 | eval { |
701 | $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub { | |
d11e91d2 | 702 | die "interrupted by signal\n"; |
91bd6c90 DM |
703 | }; |
704 | ||
f9dde219 SR |
705 | $attach_tpmstate_drive->($self, $task, $vmid); |
706 | ||
91bd6c90 DM |
707 | my $outfh; |
708 | if ($opts->{stdout}) { | |
709 | $outfh = $opts->{stdout}; | |
710 | } else { | |
711 | $outfh = IO::File->new($filename, "w") || | |
712 | die "unable to open file '$filename' - $!\n"; | |
713 | } | |
02da0c65 | 714 | my $outfileno = fileno($outfh); |
91bd6c90 | 715 | |
91bd6c90 | 716 | if ($comp) { |
02da0c65 | 717 | ($cpid, $outfileno) = $fork_compressor_pipe->($self, $comp, $outfileno); |
91bd6c90 DM |
718 | } |
719 | ||
e8705fc5 TL |
720 | my $qmpclient = PVE::QMPClient->new(); |
721 | my $backup_cb = sub { | |
722 | my ($vmid, $resp) = @_; | |
723 | $backup_job_uuid = $resp->{return}->{UUID}; | |
724 | }; | |
725 | my $add_fd_cb = sub { | |
91bd6c90 DM |
726 | my ($vmid, $resp) = @_; |
727 | ||
c05f7f3f WL |
728 | my $params = { |
729 | 'backup-file' => "/dev/fdname/backup", | |
730 | speed => $speed, | |
731 | 'config-file' => $conffile, | |
732 | devlist => $devlist | |
733 | }; | |
c05f7f3f | 734 | $params->{'firewall-file'} = $firewall if -e $firewall; |
d11e91d2 | 735 | |
c05f7f3f | 736 | $qmpclient->queue_cmd($vmid, $backup_cb, 'backup', %$params); |
91bd6c90 DM |
737 | }; |
738 | ||
d11e91d2 | 739 | $qmpclient->queue_cmd($vmid, $add_fd_cb, 'getfd', fd => $outfileno, fdname => "backup"); |
ab6a9a0c | 740 | |
cb521f2d | 741 | my $fs_frozen = $self->qga_fs_freeze($task, $vmid); |
874a096e | 742 | |
c5983223 | 743 | eval { $qmpclient->queue_execute(30) }; |
f0f30448 | 744 | my $qmperr = $@; |
91bd6c90 | 745 | |
1ece829a TL |
746 | if ($fs_frozen) { |
747 | $self->qga_fs_thaw($vmid); | |
ab6a9a0c | 748 | } |
d11e91d2 | 749 | |
f0f30448 | 750 | die $qmperr if $qmperr; |
874a096e | 751 | die $qmpclient->{errors}->{$vmid} if $qmpclient->{errors}->{$vmid}; |
91bd6c90 DM |
752 | |
753 | if ($cpid) { | |
874a096e | 754 | POSIX::close($outfileno) == 0 || |
91bd6c90 DM |
755 | die "close output file handle failed\n"; |
756 | } | |
757 | ||
d2cc2cbe | 758 | die "got no uuid for backup task\n" if !defined($backup_job_uuid); |
91bd6c90 | 759 | |
d2cc2cbe | 760 | $self->loginfo("started backup task '$backup_job_uuid'"); |
91bd6c90 | 761 | |
0b2f574b | 762 | $self->resume_vm_after_job_start($task, $vmid); |
91bd6c90 | 763 | |
d2cc2cbe | 764 | $query_backup_status_loop->($self, $vmid, $backup_job_uuid); |
91bd6c90 DM |
765 | }; |
766 | my $err = $@; | |
19599cd9 | 767 | if ($err) { |
60635a57 | 768 | $self->logerr($err); |
edae1718 | 769 | $self->mon_backup_cancel($vmid); |
ebce5239 | 770 | $self->resume_vm_after_job_start($task, $vmid); |
19599cd9 DM |
771 | } |
772 | ||
0b2f574b | 773 | $self->restore_vm_power_state($vmid); |
91bd6c90 DM |
774 | |
775 | if ($err) { | |
874a096e DM |
776 | if ($cpid) { |
777 | kill(9, $cpid); | |
91bd6c90 DM |
778 | waitpid($cpid, 0); |
779 | } | |
780 | die $err; | |
781 | } | |
782 | ||
783 | if ($cpid && (waitpid($cpid, 0) > 0)) { | |
784 | my $stat = $?; | |
785 | my $ec = $stat >> 8; | |
786 | my $signal = $stat & 127; | |
787 | if ($ec || $signal) { | |
874a096e | 788 | die "$comp failed - wrong exit status $ec" . |
91bd6c90 DM |
789 | ($signal ? " (signal $signal)\n" : "\n"); |
790 | } | |
791 | } | |
792 | } | |
793 | ||
81dcd479 TL |
794 | sub _get_task_devlist { |
795 | my ($task) = @_; | |
796 | ||
797 | my $devlist = ''; | |
798 | foreach my $di (@{$task->{disks}}) { | |
799 | if ($di->{type} eq 'block' || $di->{type} eq 'file') { | |
800 | $devlist .= ',' if $devlist; | |
801 | $devlist .= $di->{qmdevice}; | |
802 | } else { | |
803 | die "implement me (type '$di->{type}')"; | |
804 | } | |
805 | } | |
806 | return $devlist; | |
807 | } | |
808 | ||
1ece829a | 809 | sub qga_fs_freeze { |
cb521f2d | 810 | my ($self, $task, $vmid) = @_; |
b08c37c3 | 811 | return if !$self->{vmlist}->{$vmid}->{agent} || $task->{mode} eq 'stop' || !$self->{vm_was_running} || $self->{vm_was_paused}; |
1ece829a TL |
812 | |
813 | if (!PVE::QemuServer::qga_check_running($vmid, 1)) { | |
814 | $self->loginfo("skipping guest-agent 'fs-freeze', agent configured but not running?"); | |
815 | return; | |
816 | } | |
817 | ||
818 | $self->loginfo("issuing guest-agent 'fs-freeze' command"); | |
819 | eval { mon_cmd($vmid, "guest-fsfreeze-freeze") }; | |
820 | $self->logerr($@) if $@; | |
821 | ||
822 | return 1; # even on mon command error, ensure we always thaw again | |
823 | } | |
824 | ||
825 | # only call if fs_freeze return 1 | |
826 | sub qga_fs_thaw { | |
827 | my ($self, $vmid) = @_; | |
828 | ||
829 | $self->loginfo("issuing guest-agent 'fs-thaw' command"); | |
830 | eval { mon_cmd($vmid, "guest-fsfreeze-thaw") }; | |
831 | $self->logerr($@) if $@; | |
832 | } | |
833 | ||
0b2f574b TL |
834 | # we need a running QEMU/KVM process for backup, starts a paused (prelaunch) |
835 | # one if VM isn't already running | |
836 | sub enforce_vm_running_for_backup { | |
837 | my ($self, $vmid) = @_; | |
838 | ||
839 | if (PVE::QemuServer::check_running($vmid)) { | |
840 | $self->{vm_was_running} = 1; | |
841 | return; | |
842 | } | |
843 | ||
844 | eval { | |
845 | $self->loginfo("starting kvm to execute backup task"); | |
846 | # start with skiplock | |
0c498cca FG |
847 | my $params = { |
848 | skiplock => 1, | |
4ef13a7f | 849 | skiptemplate => 1, |
0c498cca FG |
850 | paused => 1, |
851 | }; | |
852 | PVE::QemuServer::vm_start($self->{storecfg}, $vmid, $params); | |
0b2f574b TL |
853 | }; |
854 | die $@ if $@; | |
855 | } | |
856 | ||
ebce5239 | 857 | # resume VM again once in a clear state (stop mode backup of running VM) |
0b2f574b TL |
858 | sub resume_vm_after_job_start { |
859 | my ($self, $task, $vmid) = @_; | |
860 | ||
b08c37c3 | 861 | return if !$self->{vm_was_running} || $self->{vm_was_paused}; |
0b2f574b TL |
862 | |
863 | if (my $stoptime = $task->{vmstoptime}) { | |
864 | my $delay = time() - $task->{vmstoptime}; | |
865 | $task->{vmstoptime} = undef; # avoid printing 'online after ..' twice | |
866 | $self->loginfo("resuming VM again after $delay seconds"); | |
867 | } else { | |
868 | $self->loginfo("resuming VM again"); | |
869 | } | |
6a558901 | 870 | mon_cmd($vmid, 'cont', timeout => 45); |
0b2f574b TL |
871 | } |
872 | ||
873 | # stop again if VM was not running before | |
874 | sub restore_vm_power_state { | |
875 | my ($self, $vmid) = @_; | |
876 | ||
877 | # we always let VMs keep running | |
878 | return if $self->{vm_was_running}; | |
879 | ||
880 | eval { | |
881 | my $resp = mon_cmd($vmid, 'query-status'); | |
882 | my $status = $resp && $resp->{status} ? $resp->{status} : 'unknown'; | |
883 | if ($status eq 'prelaunch') { | |
884 | $self->loginfo("stopping kvm after backup task"); | |
885 | PVE::QemuServer::vm_stop($self->{storecfg}, $vmid, 1); | |
886 | } else { | |
887 | $self->loginfo("kvm status changed after backup ('$status') - keep VM running"); | |
888 | } | |
889 | }; | |
890 | warn $@ if $@; | |
891 | } | |
892 | ||
893 | sub mon_backup_cancel { | |
894 | my ($self, $vmid) = @_; | |
895 | ||
896 | $self->loginfo("aborting backup job"); | |
897 | eval { mon_cmd($vmid, 'backup-cancel') }; | |
898 | $self->logerr($@) if $@; | |
899 | } | |
900 | ||
91bd6c90 DM |
901 | sub snapshot { |
902 | my ($self, $task, $vmid) = @_; | |
903 | ||
904 | # nothing to do | |
1e3baf05 DM |
905 | } |
906 | ||
907 | sub cleanup { | |
908 | my ($self, $task, $vmid) = @_; | |
909 | ||
f9dde219 SR |
910 | $detach_tpmstate_drive->($task, $vmid); |
911 | ||
4ac842cb SR |
912 | if ($self->{qmeventd_fh}) { |
913 | close($self->{qmeventd_fh}); | |
914 | } | |
1e3baf05 DM |
915 | } |
916 | ||
917 | 1; |