]>
Commit | Line | Data |
---|---|---|
1 | package PVE::VZDump; | |
2 | ||
3 | use strict; | |
4 | use warnings; | |
5 | ||
6 | use Clone; | |
7 | use Fcntl ':flock'; | |
8 | use File::Basename; | |
9 | use File::Path; | |
10 | use IO::File; | |
11 | use IO::Select; | |
12 | use IPC::Open3; | |
13 | use POSIX qw(strftime); | |
14 | use Time::Local; | |
15 | ||
16 | use PVE::Cluster qw(cfs_read_file); | |
17 | use PVE::DataCenterConfig; | |
18 | use PVE::Exception qw(raise_param_exc); | |
19 | use PVE::HA::Config; | |
20 | use PVE::HA::Env::PVE2; | |
21 | use PVE::JSONSchema qw(get_standard_option); | |
22 | use PVE::RPCEnvironment; | |
23 | use PVE::Storage; | |
24 | use PVE::VZDump::Common; | |
25 | use PVE::VZDump::Plugin; | |
26 | use PVE::Tools qw(extract_param split_list); | |
27 | use PVE::API2Tools; | |
28 | ||
29 | my @posix_filesystems = qw(ext3 ext4 nfs nfs4 reiserfs xfs); | |
30 | ||
31 | my $lockfile = '/var/run/vzdump.lock'; | |
32 | my $pidfile = '/var/run/vzdump.pid'; | |
33 | my $logdir = '/var/log/vzdump'; | |
34 | ||
35 | my @plugins = qw(); | |
36 | ||
37 | my $confdesc = PVE::VZDump::Common::get_confdesc(); | |
38 | ||
39 | my $confdesc_for_defaults = Clone::clone($confdesc); | |
40 | delete $confdesc_for_defaults->{$_}->{requires} for qw(notes-template protected); | |
41 | ||
42 | # Load available plugins | |
43 | my @pve_vzdump_classes = qw(PVE::VZDump::QemuServer PVE::VZDump::LXC); | |
44 | foreach my $plug (@pve_vzdump_classes) { | |
45 | my $filename = "/usr/share/perl5/$plug.pm"; | |
46 | $filename =~ s!::!/!g; | |
47 | if (-f $filename) { | |
48 | eval { require $filename; }; | |
49 | if (!$@) { | |
50 | $plug->import (); | |
51 | push @plugins, $plug; | |
52 | } else { | |
53 | die $@; | |
54 | } | |
55 | } | |
56 | } | |
57 | ||
58 | sub get_storage_param { | |
59 | my ($param) = @_; | |
60 | ||
61 | return if $param->{dumpdir}; | |
62 | return $param->{storage} || 'local'; | |
63 | } | |
64 | ||
65 | # helper functions | |
66 | ||
67 | sub debugmsg { | |
68 | my ($mtype, $msg, $logfd, $syslog) = @_; | |
69 | ||
70 | PVE::VZDump::Plugin::debugmsg(@_); | |
71 | } | |
72 | ||
73 | sub run_command { | |
74 | my ($logfd, $cmdstr, %param) = @_; | |
75 | ||
76 | my $logfunc = sub { | |
77 | my $line = shift; | |
78 | debugmsg ('info', $line, $logfd); | |
79 | }; | |
80 | ||
81 | PVE::Tools::run_command($cmdstr, %param, logfunc => $logfunc); | |
82 | } | |
83 | ||
84 | my $verify_notes_template = sub { | |
85 | my ($template) = @_; | |
86 | ||
87 | die "contains a line feed\n" if $template =~ /\n/; | |
88 | ||
89 | my @problematic = (); | |
90 | while ($template =~ /\\(.)/g) { | |
91 | my $char = $1; | |
92 | push @problematic, "escape sequence '\\$char' at char " . (pos($template) - 2) | |
93 | if $char !~ /^[n\\]$/; | |
94 | } | |
95 | ||
96 | while ($template =~ /\{\{([^\s{}]+)\}\}/g) { | |
97 | my $var = $1; | |
98 | push @problematic, "variable '$var' at char " . (pos($template) - length($var)) | |
99 | if $var !~ /^(cluster|guestname|node|vmid)$/; | |
100 | } | |
101 | ||
102 | die "found unknown: " . join(', ', @problematic) . "\n" if scalar(@problematic); | |
103 | }; | |
104 | ||
105 | my $generate_notes = sub { | |
106 | my ($notes_template, $task) = @_; | |
107 | ||
108 | $verify_notes_template->($notes_template); | |
109 | ||
110 | my $info = { | |
111 | cluster => PVE::Cluster::get_clinfo()->{cluster}->{name} // 'standalone node', | |
112 | guestname => $task->{hostname} // "VM $task->{vmid}", # is always set for CTs | |
113 | node => PVE::INotify::nodename(), | |
114 | vmid => $task->{vmid}, | |
115 | }; | |
116 | ||
117 | my $unescape = sub { | |
118 | my ($char) = @_; | |
119 | return '\\' if $char eq '\\'; | |
120 | return "\n" if $char eq 'n'; | |
121 | die "unexpected escape character '$char'\n"; | |
122 | }; | |
123 | ||
124 | $notes_template =~ s/\\(.)/$unescape->($1)/eg; | |
125 | ||
126 | my $vars = join('|', keys $info->%*); | |
127 | $notes_template =~ s/\{\{($vars)\}\}/$info->{$1}/g; | |
128 | ||
129 | return $notes_template; | |
130 | }; | |
131 | ||
132 | my sub parse_performance { | |
133 | my ($param) = @_; | |
134 | ||
135 | if (defined(my $perf = $param->{performance})) { | |
136 | return if ref($perf) eq 'HASH'; # already parsed | |
137 | $param->{performance} = PVE::JSONSchema::parse_property_string('backup-performance', $perf); | |
138 | } | |
139 | } | |
140 | ||
141 | my $parse_prune_backups_maxfiles = sub { | |
142 | my ($param, $kind) = @_; | |
143 | ||
144 | my $maxfiles = delete $param->{maxfiles}; | |
145 | my $prune_backups = $param->{'prune-backups'}; | |
146 | ||
147 | debugmsg('warn', "both 'maxfiles' and 'prune-backups' defined as ${kind} - ignoring 'maxfiles'") | |
148 | if defined($maxfiles) && defined($prune_backups); | |
149 | ||
150 | if (defined($prune_backups)) { | |
151 | return if ref($prune_backups) eq 'HASH'; # already parsed | |
152 | $param->{'prune-backups'} = PVE::JSONSchema::parse_property_string( | |
153 | 'prune-backups', | |
154 | $prune_backups | |
155 | ); | |
156 | } elsif (defined($maxfiles)) { | |
157 | if ($maxfiles) { | |
158 | $param->{'prune-backups'} = { 'keep-last' => $maxfiles }; | |
159 | } else { | |
160 | $param->{'prune-backups'} = { 'keep-all' => 1 }; | |
161 | } | |
162 | } | |
163 | }; | |
164 | ||
165 | sub storage_info { | |
166 | my $storage = shift; | |
167 | ||
168 | my $cfg = PVE::Storage::config(); | |
169 | my $scfg = PVE::Storage::storage_config($cfg, $storage); | |
170 | my $type = $scfg->{type}; | |
171 | ||
172 | die "can't use storage '$storage' for backups - wrong content type\n" | |
173 | if (!$scfg->{content}->{backup}); | |
174 | ||
175 | my $info = { | |
176 | scfg => $scfg, | |
177 | }; | |
178 | ||
179 | $info->{'prune-backups'} = PVE::JSONSchema::parse_property_string('prune-backups', $scfg->{'prune-backups'}) | |
180 | if defined($scfg->{'prune-backups'}); | |
181 | ||
182 | if ($type eq 'pbs') { | |
183 | $info->{pbs} = 1; | |
184 | } else { | |
185 | $info->{dumpdir} = PVE::Storage::get_backup_dir($cfg, $storage); | |
186 | } | |
187 | ||
188 | return $info; | |
189 | } | |
190 | ||
191 | sub format_size { | |
192 | my $size = shift; | |
193 | ||
194 | my $kb = $size / 1024; | |
195 | ||
196 | if ($kb < 1024) { | |
197 | return int ($kb) . "KB"; | |
198 | } | |
199 | ||
200 | my $mb = $size / (1024*1024); | |
201 | if ($mb < 1024) { | |
202 | return int ($mb) . "MB"; | |
203 | } | |
204 | my $gb = $mb / 1024; | |
205 | if ($gb < 1024) { | |
206 | return sprintf ("%.2fGB", $gb); | |
207 | } | |
208 | my $tb = $gb / 1024; | |
209 | return sprintf ("%.2fTB", $tb); | |
210 | } | |
211 | ||
212 | sub format_time { | |
213 | my $seconds = shift; | |
214 | ||
215 | my $hours = int ($seconds/3600); | |
216 | $seconds = $seconds - $hours*3600; | |
217 | my $min = int ($seconds/60); | |
218 | $seconds = $seconds - $min*60; | |
219 | ||
220 | return sprintf ("%02d:%02d:%02d", $hours, $min, $seconds); | |
221 | } | |
222 | ||
223 | sub encode8bit { | |
224 | my ($str) = @_; | |
225 | ||
226 | $str =~ s/^(.{990})/$1\n/mg; # reduce line length | |
227 | ||
228 | return $str; | |
229 | } | |
230 | ||
231 | sub escape_html { | |
232 | my ($str) = @_; | |
233 | ||
234 | $str =~ s/&/&/g; | |
235 | $str =~ s/</</g; | |
236 | $str =~ s/>/>/g; | |
237 | ||
238 | return $str; | |
239 | } | |
240 | ||
241 | sub check_bin { | |
242 | my ($bin) = @_; | |
243 | ||
244 | foreach my $p (split (/:/, $ENV{PATH})) { | |
245 | my $fn = "$p/$bin"; | |
246 | if (-x $fn) { | |
247 | return $fn; | |
248 | } | |
249 | } | |
250 | ||
251 | die "unable to find command '$bin'\n"; | |
252 | } | |
253 | ||
254 | sub check_vmids { | |
255 | my (@vmids) = @_; | |
256 | ||
257 | my $res = []; | |
258 | for my $vmid (sort {$a <=> $b} @vmids) { | |
259 | die "ERROR: strange VM ID '${vmid}'\n" if $vmid !~ m/^\d+$/; | |
260 | $vmid = int ($vmid); # remove leading zeros | |
261 | next if !$vmid; | |
262 | push @$res, $vmid; | |
263 | } | |
264 | ||
265 | return $res; | |
266 | } | |
267 | ||
268 | ||
269 | sub read_vzdump_defaults { | |
270 | ||
271 | my $fn = "/etc/vzdump.conf"; | |
272 | ||
273 | my $defaults = { | |
274 | map { | |
275 | my $default = $confdesc->{$_}->{default}; | |
276 | defined($default) ? ($_ => $default) : () | |
277 | } keys %$confdesc_for_defaults | |
278 | }; | |
279 | $parse_prune_backups_maxfiles->($defaults, "defaults in VZDump schema"); | |
280 | parse_performance($defaults); | |
281 | ||
282 | my $raw; | |
283 | eval { $raw = PVE::Tools::file_get_contents($fn); }; | |
284 | return $defaults if $@; | |
285 | ||
286 | my $conf_schema = { type => 'object', properties => $confdesc_for_defaults }; | |
287 | my $res = PVE::JSONSchema::parse_config($conf_schema, $fn, $raw); | |
288 | if (my $excludes = $res->{'exclude-path'}) { | |
289 | if (ref($excludes) eq 'ARRAY') { | |
290 | my $list = []; | |
291 | for my $path ($excludes->@*) { | |
292 | # We still use `split_args` here to be compatible with old configs where one line | |
293 | # still has multiple space separated entries. | |
294 | push $list->@*, PVE::Tools::split_args($path)->@*; | |
295 | } | |
296 | $res->{'exclude-path'} = $list; | |
297 | } else { | |
298 | $res->{'exclude-path'} = PVE::Tools::split_args($excludes); | |
299 | } | |
300 | } | |
301 | if (defined($res->{mailto})) { | |
302 | my @mailto = split_list($res->{mailto}); | |
303 | $res->{mailto} = [ @mailto ]; | |
304 | } | |
305 | $parse_prune_backups_maxfiles->($res, "options in '$fn'"); | |
306 | parse_performance($res); | |
307 | ||
308 | foreach my $key (keys %$defaults) { | |
309 | $res->{$key} = $defaults->{$key} if !defined($res->{$key}); | |
310 | } | |
311 | ||
312 | if (defined($res->{storage}) && defined($res->{dumpdir})) { | |
313 | debugmsg('warn', "both 'storage' and 'dumpdir' defined in '$fn' - ignoring 'dumpdir'"); | |
314 | delete $res->{dumpdir}; | |
315 | } | |
316 | ||
317 | return $res; | |
318 | } | |
319 | ||
320 | use constant MAX_MAIL_SIZE => 1024*1024; | |
321 | sub sendmail { | |
322 | my ($self, $tasklist, $totaltime, $err, $detail_pre, $detail_post) = @_; | |
323 | ||
324 | my $opts = $self->{opts}; | |
325 | ||
326 | my $mailto = $opts->{mailto}; | |
327 | ||
328 | return if !($mailto && scalar(@$mailto)); | |
329 | ||
330 | my $cmdline = $self->{cmdline}; | |
331 | ||
332 | my $ecount = 0; | |
333 | foreach my $task (@$tasklist) { | |
334 | $ecount++ if $task->{state} ne 'ok'; | |
335 | chomp $task->{msg} if $task->{msg}; | |
336 | $task->{backuptime} = 0 if !$task->{backuptime}; | |
337 | $task->{size} = 0 if !$task->{size}; | |
338 | $task->{target} = 'unknown' if !$task->{target}; | |
339 | $task->{hostname} = "VM $task->{vmid}" if !$task->{hostname}; | |
340 | ||
341 | if ($task->{state} eq 'todo') { | |
342 | $task->{msg} = 'aborted'; | |
343 | } | |
344 | } | |
345 | ||
346 | my $notify = $opts->{mailnotification} || 'always'; | |
347 | return if (!$ecount && !$err && ($notify eq 'failure')); | |
348 | ||
349 | my $stat = ($ecount || $err) ? 'backup failed' : 'backup successful'; | |
350 | if ($err) { | |
351 | if ($err =~ /\n/) { | |
352 | $stat .= ": multiple problems"; | |
353 | } else { | |
354 | $stat .= ": $err"; | |
355 | $err = undef; | |
356 | } | |
357 | } | |
358 | ||
359 | my $hostname = `hostname -f` || PVE::INotify::nodename(); | |
360 | chomp $hostname; | |
361 | ||
362 | # text part | |
363 | my $text = $err ? "$err\n\n" : ''; | |
364 | my $namelength = 20; | |
365 | $text .= sprintf ( | |
366 | "%-10s %-${namelength}s %-6s %10s %10s %s\n", | |
367 | qw(VMID NAME STATUS TIME SIZE FILENAME) | |
368 | ); | |
369 | foreach my $task (@$tasklist) { | |
370 | my $name = substr($task->{hostname}, 0, $namelength); | |
371 | my $successful = $task->{state} eq 'ok'; | |
372 | my $size = $successful ? format_size ($task->{size}) : 0; | |
373 | my $filename = $successful ? $task->{target} : '-'; | |
374 | my $size_fmt = $successful ? "%10s": "%8.2fMB"; | |
375 | $text .= sprintf( | |
376 | "%-10s %-${namelength}s %-6s %10s $size_fmt %s\n", | |
377 | $task->{vmid}, | |
378 | $name, | |
379 | $task->{state}, | |
380 | format_time($task->{backuptime}), | |
381 | $size, | |
382 | $filename, | |
383 | ); | |
384 | } | |
385 | ||
386 | my $text_log_part; | |
387 | $text_log_part .= "\nDetailed backup logs:\n\n"; | |
388 | $text_log_part .= "$cmdline\n\n"; | |
389 | ||
390 | $text_log_part .= $detail_pre . "\n" if defined($detail_pre); | |
391 | foreach my $task (@$tasklist) { | |
392 | my $vmid = $task->{vmid}; | |
393 | my $log = $task->{tmplog}; | |
394 | if (!$log) { | |
395 | $text_log_part .= "$vmid: no log available\n\n"; | |
396 | next; | |
397 | } | |
398 | if (open (my $TMP, '<', "$log")) { | |
399 | while (my $line = <$TMP>) { | |
400 | next if $line =~ /^status: \d+/; # not useful in mails | |
401 | $text_log_part .= encode8bit ("$vmid: $line"); | |
402 | } | |
403 | close ($TMP); | |
404 | } else { | |
405 | $text_log_part .= "$vmid: Could not open log file\n\n"; | |
406 | } | |
407 | $text_log_part .= "\n"; | |
408 | } | |
409 | $text_log_part .= $detail_post if defined($detail_post); | |
410 | ||
411 | # html part | |
412 | my $html = "<html><body>\n"; | |
413 | $html .= "<p>" . (escape_html($err) =~ s/\n/<br>/gr) . "</p>\n" if $err; | |
414 | $html .= "<table border=1 cellpadding=3>\n"; | |
415 | $html .= "<tr><td>VMID<td>NAME<td>STATUS<td>TIME<td>SIZE<td>FILENAME</tr>\n"; | |
416 | ||
417 | my $ssize = 0; | |
418 | foreach my $task (@$tasklist) { | |
419 | my $vmid = $task->{vmid}; | |
420 | my $name = $task->{hostname}; | |
421 | ||
422 | if ($task->{state} eq 'ok') { | |
423 | $ssize += $task->{size}; | |
424 | ||
425 | $html .= sprintf ( | |
426 | "<tr><td>%s<td>%s<td>OK<td>%s<td align=right>%s<td>%s</tr>\n", | |
427 | $vmid, | |
428 | $name, | |
429 | format_time($task->{backuptime}), | |
430 | format_size ($task->{size}), | |
431 | escape_html ($task->{target}), | |
432 | ); | |
433 | } else { | |
434 | $html .= sprintf ( | |
435 | "<tr><td>%s<td>%s<td><font color=red>FAILED<td>%s<td colspan=2>%s</tr>\n", | |
436 | $vmid, | |
437 | $name, | |
438 | format_time($task->{backuptime}), | |
439 | escape_html ($task->{msg}), | |
440 | ); | |
441 | } | |
442 | } | |
443 | ||
444 | $html .= sprintf ("<tr><td align=left colspan=3>TOTAL<td>%s<td>%s<td></tr>", | |
445 | format_time ($totaltime), format_size ($ssize)); | |
446 | ||
447 | $html .= "\n</table><br><br>\n"; | |
448 | my $html_log_part; | |
449 | $html_log_part .= "Detailed backup logs:<br /><br />\n"; | |
450 | $html_log_part .= "<pre>\n"; | |
451 | $html_log_part .= escape_html($cmdline) . "\n\n"; | |
452 | ||
453 | $html_log_part .= escape_html($detail_pre) . "\n" if defined($detail_pre); | |
454 | foreach my $task (@$tasklist) { | |
455 | my $vmid = $task->{vmid}; | |
456 | my $log = $task->{tmplog}; | |
457 | if (!$log) { | |
458 | $html_log_part .= "$vmid: no log available\n\n"; | |
459 | next; | |
460 | } | |
461 | if (open (my $TMP, '<', "$log")) { | |
462 | while (my $line = <$TMP>) { | |
463 | next if $line =~ /^status: \d+/; # not useful in mails | |
464 | if ($line =~ m/^\S+\s\d+\s+\d+:\d+:\d+\s+(ERROR|WARN):/) { | |
465 | $html_log_part .= encode8bit ("$vmid: <font color=red>". | |
466 | escape_html ($line) . "</font>"); | |
467 | } else { | |
468 | $html_log_part .= encode8bit ("$vmid: " . escape_html ($line)); | |
469 | } | |
470 | } | |
471 | close ($TMP); | |
472 | } else { | |
473 | $html_log_part .= "$vmid: Could not open log file\n\n"; | |
474 | } | |
475 | $html_log_part .= "\n"; | |
476 | } | |
477 | $html_log_part .= escape_html($detail_post) if defined($detail_post); | |
478 | $html_log_part .= "</pre>"; | |
479 | my $html_end = "\n</body></html>\n"; | |
480 | # end html part | |
481 | ||
482 | if (length($text) + length($text_log_part) + | |
483 | length($html) + length($html_log_part) + | |
484 | length($html_end) < MAX_MAIL_SIZE) | |
485 | { | |
486 | $html .= $html_log_part; | |
487 | $html .= $html_end; | |
488 | $text .= $text_log_part; | |
489 | } else { | |
490 | my $msg = "Log output was too long to be sent by mail. ". | |
491 | "See Task History for details!\n"; | |
492 | $text .= $msg; | |
493 | $html .= "<p>$msg</p>"; | |
494 | $html .= $html_end; | |
495 | } | |
496 | ||
497 | my $subject = "vzdump backup status ($hostname) : $stat"; | |
498 | ||
499 | my $dcconf = PVE::Cluster::cfs_read_file('datacenter.cfg'); | |
500 | my $mailfrom = $dcconf->{email_from} || "root"; | |
501 | ||
502 | PVE::Tools::sendmail($mailto, $subject, $text, $html, $mailfrom, "vzdump backup tool"); | |
503 | }; | |
504 | ||
505 | sub new { | |
506 | my ($class, $cmdline, $opts, $skiplist) = @_; | |
507 | ||
508 | mkpath $logdir; | |
509 | ||
510 | check_bin ('cp'); | |
511 | check_bin ('df'); | |
512 | check_bin ('sendmail'); | |
513 | check_bin ('rsync'); | |
514 | check_bin ('tar'); | |
515 | check_bin ('mount'); | |
516 | check_bin ('umount'); | |
517 | check_bin ('cstream'); | |
518 | check_bin ('ionice'); | |
519 | ||
520 | if ($opts->{mode} && $opts->{mode} eq 'snapshot') { | |
521 | check_bin ('lvcreate'); | |
522 | check_bin ('lvs'); | |
523 | check_bin ('lvremove'); | |
524 | } | |
525 | ||
526 | my $defaults = read_vzdump_defaults(); | |
527 | ||
528 | foreach my $k (keys %$defaults) { | |
529 | next if $k eq 'exclude-path' || $k eq 'prune-backups'; # dealt with separately | |
530 | if ($k eq 'dumpdir' || $k eq 'storage') { | |
531 | $opts->{$k} = $defaults->{$k} if !defined ($opts->{dumpdir}) && | |
532 | !defined ($opts->{storage}); | |
533 | } else { | |
534 | $opts->{$k} = $defaults->{$k} if !defined ($opts->{$k}); | |
535 | } | |
536 | } | |
537 | ||
538 | $opts->{dumpdir} =~ s|/+$|| if ($opts->{dumpdir}); | |
539 | $opts->{tmpdir} =~ s|/+$|| if ($opts->{tmpdir}); | |
540 | ||
541 | $skiplist = [] if !$skiplist; | |
542 | my $self = bless { | |
543 | cmdline => $cmdline, | |
544 | opts => $opts, | |
545 | skiplist => $skiplist, | |
546 | }, $class; | |
547 | ||
548 | my $findexcl = $self->{findexcl} = []; | |
549 | if ($defaults->{'exclude-path'}) { | |
550 | push @$findexcl, @{$defaults->{'exclude-path'}}; | |
551 | } | |
552 | ||
553 | if ($opts->{'exclude-path'}) { | |
554 | push @$findexcl, @{$opts->{'exclude-path'}}; | |
555 | } | |
556 | ||
557 | if ($opts->{stdexcludes}) { | |
558 | push @$findexcl, | |
559 | '/tmp/?*', | |
560 | '/var/tmp/?*', | |
561 | '/var/run/?*.pid', | |
562 | ; | |
563 | } | |
564 | ||
565 | foreach my $p (@plugins) { | |
566 | my $pd = $p->new($self); | |
567 | ||
568 | push @{$self->{plugins}}, $pd; | |
569 | } | |
570 | ||
571 | if (defined($opts->{storage}) && $opts->{stdout}) { | |
572 | die "cannot use options 'storage' and 'stdout' at the same time\n"; | |
573 | } elsif (defined($opts->{storage}) && defined($opts->{dumpdir})) { | |
574 | die "cannot use options 'storage' and 'dumpdir' at the same time\n"; | |
575 | } | |
576 | ||
577 | if (my $storage = get_storage_param($opts)) { | |
578 | $opts->{storage} = $storage; | |
579 | } | |
580 | ||
581 | # Enforced by the API too, but these options might come in via defaults. Drop them if necessary. | |
582 | if (!$opts->{storage}) { | |
583 | delete $opts->{$_} for qw(notes-template protected); | |
584 | } | |
585 | ||
586 | my $errors = ''; | |
587 | my $add_error = sub { | |
588 | my ($error) = @_; | |
589 | $errors .= "\n" if $errors; | |
590 | chomp($error); | |
591 | $errors .= $error; | |
592 | }; | |
593 | ||
594 | eval { | |
595 | $self->{job_init_log} = ''; | |
596 | open my $job_init_fd, '>', \$self->{job_init_log}; | |
597 | $self->run_hook_script('job-init', undef, $job_init_fd); | |
598 | close $job_init_fd; | |
599 | ||
600 | PVE::Cluster::cfs_update(); # Pick up possible changes made by the hook script. | |
601 | }; | |
602 | $add_error->($@) if $@; | |
603 | ||
604 | if ($opts->{storage}) { | |
605 | my $storage_cfg = PVE::Storage::config(); | |
606 | eval { PVE::Storage::activate_storage($storage_cfg, $opts->{storage}) }; | |
607 | $add_error->("could not activate storage '$opts->{storage}': $@") if $@; | |
608 | ||
609 | my $info = eval { storage_info ($opts->{storage}) }; | |
610 | if (my $err = $@) { | |
611 | $add_error->("could not get storage information for '$opts->{storage}': $err"); | |
612 | } else { | |
613 | $opts->{dumpdir} = $info->{dumpdir}; | |
614 | $opts->{scfg} = $info->{scfg}; | |
615 | $opts->{pbs} = $info->{pbs}; | |
616 | $opts->{'prune-backups'} //= $info->{'prune-backups'}; | |
617 | } | |
618 | } elsif ($opts->{dumpdir}) { | |
619 | $add_error->("dumpdir '$opts->{dumpdir}' does not exist") | |
620 | if ! -d $opts->{dumpdir}; | |
621 | } else { | |
622 | die "internal error"; | |
623 | } | |
624 | ||
625 | $opts->{'prune-backups'} //= $defaults->{'prune-backups'}; | |
626 | ||
627 | # avoid triggering any remove code path if keep-all is set | |
628 | $opts->{remove} = 0 if $opts->{'prune-backups'}->{'keep-all'}; | |
629 | ||
630 | if ($opts->{tmpdir} && ! -d $opts->{tmpdir}) { | |
631 | $add_error->("tmpdir '$opts->{tmpdir}' does not exist"); | |
632 | } | |
633 | ||
634 | if ($errors) { | |
635 | eval { $self->sendmail([], 0, $errors); }; | |
636 | debugmsg ('err', $@) if $@; | |
637 | die "$errors\n"; | |
638 | } | |
639 | ||
640 | return $self; | |
641 | } | |
642 | ||
643 | sub get_mount_info { | |
644 | my ($dir) = @_; | |
645 | ||
646 | # Note: df 'available' can be negative, and percentage set to '-' | |
647 | ||
648 | my $cmd = [ 'df', '-P', '-T', '-B', '1', $dir]; | |
649 | ||
650 | my $res; | |
651 | ||
652 | my $parser = sub { | |
653 | my $line = shift; | |
654 | if (my ($fsid, $fstype, undef, $mp) = $line =~ | |
655 | m!(\S+.*)\s+(\S+)\s+\d+\s+\-?\d+\s+\d+\s+(\d+%|-)\s+(/.*)$!) { | |
656 | $res = { | |
657 | device => $fsid, | |
658 | fstype => $fstype, | |
659 | mountpoint => $mp, | |
660 | }; | |
661 | } | |
662 | }; | |
663 | ||
664 | eval { PVE::Tools::run_command($cmd, errfunc => sub {}, outfunc => $parser); }; | |
665 | warn $@ if $@; | |
666 | ||
667 | return $res; | |
668 | } | |
669 | ||
670 | sub getlock { | |
671 | my ($self, $upid) = @_; | |
672 | ||
673 | my $fh; | |
674 | ||
675 | my $maxwait = $self->{opts}->{lockwait} || $self->{lockwait}; | |
676 | ||
677 | die "missing UPID" if !$upid; # should not happen | |
678 | ||
679 | my $SERVER_FLCK; | |
680 | if (!open ($SERVER_FLCK, '>>', "$lockfile")) { | |
681 | debugmsg ('err', "can't open lock on file '$lockfile' - $!", undef, 1); | |
682 | die "can't open lock on file '$lockfile' - $!"; | |
683 | } | |
684 | ||
685 | if (!flock ($SERVER_FLCK, LOCK_EX|LOCK_NB)) { | |
686 | if (!$maxwait) { | |
687 | debugmsg ('err', "can't acquire lock '$lockfile' (wait = 0)", undef, 1); | |
688 | die "can't acquire lock '$lockfile' (wait = 0)"; | |
689 | } | |
690 | ||
691 | debugmsg('info', "trying to get global lock - waiting...", undef, 1); | |
692 | eval { | |
693 | alarm ($maxwait * 60); | |
694 | ||
695 | local $SIG{ALRM} = sub { alarm (0); die "got timeout\n"; }; | |
696 | ||
697 | if (!flock ($SERVER_FLCK, LOCK_EX)) { | |
698 | my $err = $!; | |
699 | close ($SERVER_FLCK); | |
700 | alarm (0); | |
701 | die "$err\n"; | |
702 | } | |
703 | alarm (0); | |
704 | }; | |
705 | alarm (0); | |
706 | ||
707 | my $err = $@; | |
708 | ||
709 | if ($err) { | |
710 | debugmsg ('err', "can't acquire lock '$lockfile' - $err", undef, 1); | |
711 | die "can't acquire lock '$lockfile' - $err"; | |
712 | } | |
713 | ||
714 | debugmsg('info', "got global lock", undef, 1); | |
715 | } | |
716 | ||
717 | PVE::Tools::file_set_contents($pidfile, $upid); | |
718 | ||
719 | return $SERVER_FLCK; | |
720 | } | |
721 | ||
722 | sub run_hook_script { | |
723 | my ($self, $phase, $task, $logfd) = @_; | |
724 | ||
725 | my $opts = $self->{opts}; | |
726 | ||
727 | my $script = $opts->{script}; | |
728 | return if !$script; | |
729 | ||
730 | die "Error: The hook script '$script' does not exist.\n" if ! -f $script; | |
731 | die "Error: The hook script '$script' is not executable.\n" if ! -x $script; | |
732 | ||
733 | my $cmd = [$script, $phase]; | |
734 | ||
735 | if ($task) { | |
736 | push @$cmd, $task->{mode}; | |
737 | push @$cmd, $task->{vmid}; | |
738 | } | |
739 | ||
740 | local %ENV; | |
741 | # set immutable opts directly (so they are available in all phases) | |
742 | $ENV{STOREID} = $opts->{storage} if $opts->{storage}; | |
743 | $ENV{DUMPDIR} = $opts->{dumpdir} if $opts->{dumpdir}; | |
744 | ||
745 | foreach my $ek (qw(vmtype hostname target logfile)) { | |
746 | $ENV{uc($ek)} = $task->{$ek} if $task->{$ek}; | |
747 | } | |
748 | ||
749 | run_command ($logfd, $cmd); | |
750 | } | |
751 | ||
752 | sub compressor_info { | |
753 | my ($opts) = @_; | |
754 | my $opt_compress = $opts->{compress}; | |
755 | ||
756 | if (!$opt_compress || $opt_compress eq '0') { | |
757 | return undef; | |
758 | } elsif ($opt_compress eq '1' || $opt_compress eq 'lzo') { | |
759 | return ('lzop', 'lzo'); | |
760 | } elsif ($opt_compress eq 'gzip') { | |
761 | if ($opts->{pigz} > 0) { | |
762 | my $pigz_threads = $opts->{pigz}; | |
763 | if ($pigz_threads == 1) { | |
764 | my $cpuinfo = PVE::ProcFSTools::read_cpuinfo(); | |
765 | $pigz_threads = int(($cpuinfo->{cpus} + 1)/2); | |
766 | } | |
767 | return ("pigz -p ${pigz_threads} --rsyncable", 'gz'); | |
768 | } else { | |
769 | return ('gzip --rsyncable', 'gz'); | |
770 | } | |
771 | } elsif ($opt_compress eq 'zstd') { | |
772 | my $zstd_threads = $opts->{zstd} // 1; | |
773 | if ($zstd_threads == 0) { | |
774 | my $cpuinfo = PVE::ProcFSTools::read_cpuinfo(); | |
775 | $zstd_threads = int(($cpuinfo->{cpus} + 1)/2); | |
776 | } | |
777 | return ("zstd --threads=${zstd_threads}", 'zst'); | |
778 | } else { | |
779 | die "internal error - unknown compression option '$opt_compress'"; | |
780 | } | |
781 | } | |
782 | ||
783 | sub get_backup_file_list { | |
784 | my ($dir, $bkname) = @_; | |
785 | ||
786 | my $bklist = []; | |
787 | foreach my $fn (<$dir/${bkname}-*>) { | |
788 | my $archive_info = eval { PVE::Storage::archive_info($fn) } // {}; | |
789 | if ($archive_info->{is_std_name}) { | |
790 | my $path = "$dir/$archive_info->{filename}"; | |
791 | my $backup = { | |
792 | 'path' => $path, | |
793 | 'ctime' => $archive_info->{ctime}, | |
794 | }; | |
795 | $backup->{mark} = "protected" | |
796 | if -e PVE::Storage::protection_file_path($path); | |
797 | push @{$bklist}, $backup; | |
798 | } | |
799 | } | |
800 | ||
801 | return $bklist; | |
802 | } | |
803 | ||
804 | sub exec_backup_task { | |
805 | my ($self, $task) = @_; | |
806 | ||
807 | my $opts = $self->{opts}; | |
808 | ||
809 | my $cfg = PVE::Storage::config(); | |
810 | my $vmid = $task->{vmid}; | |
811 | my $plugin = $task->{plugin}; | |
812 | ||
813 | $task->{backup_time} = time(); | |
814 | ||
815 | my $pbs_group_name; | |
816 | my $pbs_snapshot_name; | |
817 | ||
818 | my $vmstarttime = time (); | |
819 | ||
820 | my $logfd; | |
821 | ||
822 | my $cleanup = {}; | |
823 | ||
824 | my $log_vm_online_again = sub { | |
825 | return if !defined($task->{vmstoptime}); | |
826 | $task->{vmconttime} //= time(); | |
827 | my $delay = $task->{vmconttime} - $task->{vmstoptime}; | |
828 | $delay = '<1' if $delay < 1; | |
829 | debugmsg ('info', "guest is online again after $delay seconds", $logfd); | |
830 | }; | |
831 | ||
832 | eval { | |
833 | die "unable to find VM '$vmid'\n" if !$plugin; | |
834 | ||
835 | my $vmtype = $plugin->type(); | |
836 | ||
837 | if ($self->{opts}->{pbs}) { | |
838 | if ($vmtype eq 'lxc') { | |
839 | $pbs_group_name = "ct/$vmid"; | |
840 | } elsif ($vmtype eq 'qemu') { | |
841 | $pbs_group_name = "vm/$vmid"; | |
842 | } else { | |
843 | die "pbs backup not implemented for plugin type '$vmtype'\n"; | |
844 | } | |
845 | my $btime = strftime("%FT%TZ", gmtime($task->{backup_time})); | |
846 | $pbs_snapshot_name = "$pbs_group_name/$btime"; | |
847 | } | |
848 | ||
849 | # for now we deny backups of a running ha managed service in *stop* mode | |
850 | # as it interferes with the HA stack (started services should not stop). | |
851 | if ($opts->{mode} eq 'stop' && | |
852 | PVE::HA::Config::vm_is_ha_managed($vmid, 'started')) | |
853 | { | |
854 | die "Cannot execute a backup with stop mode on a HA managed and". | |
855 | " enabled Service. Use snapshot mode or disable the Service.\n"; | |
856 | } | |
857 | ||
858 | my $tmplog = "$logdir/$vmtype-$vmid.log"; | |
859 | ||
860 | my $bkname = "vzdump-$vmtype-$vmid"; | |
861 | my $basename = $bkname . strftime("-%Y_%m_%d-%H_%M_%S", localtime($task->{backup_time})); | |
862 | ||
863 | my $prune_options = $opts->{'prune-backups'}; | |
864 | ||
865 | my $backup_limit = 0; | |
866 | if (!$prune_options->{'keep-all'}) { | |
867 | foreach my $keep (values %{$prune_options}) { | |
868 | $backup_limit += $keep; | |
869 | } | |
870 | } | |
871 | ||
872 | if (($backup_limit && !$opts->{remove}) || $opts->{protected}) { | |
873 | my $count; | |
874 | my $protected_count; | |
875 | if (my $storeid = $opts->{storage}) { | |
876 | my @backups = grep { | |
877 | !$_->{subtype} || $_->{subtype} eq $vmtype | |
878 | } PVE::Storage::volume_list($cfg, $storeid, $vmid, 'backup')->@*; | |
879 | ||
880 | $count = grep { !$_->{protected} } @backups; | |
881 | $protected_count = scalar(@backups) - $count; | |
882 | } else { | |
883 | $count = grep { !$_->{mark} || $_->{mark} ne "protected" } get_backup_file_list($opts->{dumpdir}, $bkname)->@*; | |
884 | } | |
885 | ||
886 | if ($opts->{protected}) { | |
887 | my $max_protected = PVE::Storage::get_max_protected_backups( | |
888 | $opts->{scfg}, | |
889 | $opts->{storage}, | |
890 | ); | |
891 | if ($max_protected > -1 && $protected_count >= $max_protected) { | |
892 | die "The number of protected backups per guest is limited to $max_protected ". | |
893 | "on storage '$opts->{storage}'\n"; | |
894 | } | |
895 | } elsif ($count >= $backup_limit) { | |
896 | die "There is a max backup limit of $backup_limit enforced by the target storage ". | |
897 | "or the vzdump parameters. Either increase the limit or delete old backups.\n"; | |
898 | } | |
899 | } | |
900 | ||
901 | if (!$self->{opts}->{pbs}) { | |
902 | $task->{logfile} = "$opts->{dumpdir}/$basename.log"; | |
903 | } | |
904 | ||
905 | my $ext = $vmtype eq 'qemu' ? '.vma' : '.tar'; | |
906 | my ($comp, $comp_ext) = compressor_info($opts); | |
907 | if ($comp && $comp_ext) { | |
908 | $ext .= ".${comp_ext}"; | |
909 | } | |
910 | ||
911 | if ($self->{opts}->{pbs}) { | |
912 | die "unable to pipe backup to stdout\n" if $opts->{stdout}; | |
913 | $task->{target} = $pbs_snapshot_name; | |
914 | } else { | |
915 | if ($opts->{stdout}) { | |
916 | $task->{target} = '-'; | |
917 | } else { | |
918 | $task->{target} = $task->{tmptar} = "$opts->{dumpdir}/$basename$ext"; | |
919 | $task->{tmptar} =~ s/\.[^\.]+$/\.dat/; | |
920 | unlink $task->{tmptar}; | |
921 | } | |
922 | } | |
923 | ||
924 | $task->{vmtype} = $vmtype; | |
925 | ||
926 | my $pid = $$; | |
927 | if ($opts->{tmpdir}) { | |
928 | $task->{tmpdir} = "$opts->{tmpdir}/vzdumptmp${pid}_$vmid/"; | |
929 | } elsif ($self->{opts}->{pbs}) { | |
930 | $task->{tmpdir} = "/var/tmp/vzdumptmp${pid}_$vmid"; | |
931 | } else { | |
932 | # dumpdir is posix? then use it as temporary dir | |
933 | my $info = get_mount_info($opts->{dumpdir}); | |
934 | if ($vmtype eq 'qemu' || | |
935 | grep ($_ eq $info->{fstype}, @posix_filesystems)) { | |
936 | $task->{tmpdir} = "$opts->{dumpdir}/$basename.tmp"; | |
937 | } else { | |
938 | $task->{tmpdir} = "/var/tmp/vzdumptmp${pid}_$vmid"; | |
939 | debugmsg ('info', "filesystem type on dumpdir is '$info->{fstype}' -" . | |
940 | "using $task->{tmpdir} for temporary files", $logfd); | |
941 | } | |
942 | } | |
943 | ||
944 | rmtree $task->{tmpdir}; | |
945 | mkdir $task->{tmpdir}; | |
946 | -d $task->{tmpdir} || | |
947 | die "unable to create temporary directory '$task->{tmpdir}'"; | |
948 | ||
949 | $logfd = IO::File->new (">$tmplog") || | |
950 | die "unable to create log file '$tmplog'"; | |
951 | ||
952 | $task->{dumpdir} = $opts->{dumpdir}; | |
953 | $task->{storeid} = $opts->{storage}; | |
954 | $task->{scfg} = $opts->{scfg}; | |
955 | $task->{tmplog} = $tmplog; | |
956 | ||
957 | unlink $task->{logfile} if defined($task->{logfile}); | |
958 | ||
959 | debugmsg ('info', "Starting Backup of VM $vmid ($vmtype)", $logfd, 1); | |
960 | debugmsg ('info', "Backup started at " . strftime("%F %H:%M:%S", localtime())); | |
961 | ||
962 | $plugin->set_logfd ($logfd); | |
963 | ||
964 | # test is VM is running | |
965 | my ($running, $status_text) = $plugin->vm_status ($vmid); | |
966 | ||
967 | debugmsg ('info', "status = ${status_text}", $logfd); | |
968 | ||
969 | # lock VM (prevent config changes) | |
970 | $plugin->lock_vm ($vmid); | |
971 | ||
972 | $cleanup->{unlock} = 1; | |
973 | ||
974 | # prepare | |
975 | ||
976 | my $mode = $running ? $task->{mode} : 'stop'; | |
977 | ||
978 | if ($mode eq 'snapshot') { | |
979 | my %saved_task = %$task; | |
980 | eval { $plugin->prepare ($task, $vmid, $mode); }; | |
981 | if (my $err = $@) { | |
982 | die $err if $err !~ m/^mode failure/; | |
983 | debugmsg ('info', $err, $logfd); | |
984 | debugmsg ('info', "trying 'suspend' mode instead", $logfd); | |
985 | $mode = 'suspend'; # so prepare is called again below | |
986 | %$task = %saved_task; | |
987 | } | |
988 | } | |
989 | ||
990 | $cleanup->{prepared} = 1; | |
991 | ||
992 | $task->{mode} = $mode; | |
993 | ||
994 | debugmsg ('info', "backup mode: $mode", $logfd); | |
995 | debugmsg ('info', "bandwidth limit: $opts->{bwlimit} KB/s", $logfd) if $opts->{bwlimit}; | |
996 | debugmsg ('info', "ionice priority: $opts->{ionice}", $logfd); | |
997 | ||
998 | if ($mode eq 'stop') { | |
999 | $plugin->prepare ($task, $vmid, $mode); | |
1000 | ||
1001 | $self->run_hook_script ('backup-start', $task, $logfd); | |
1002 | ||
1003 | if ($running) { | |
1004 | debugmsg ('info', "stopping virtual guest", $logfd); | |
1005 | $task->{vmstoptime} = time(); | |
1006 | $self->run_hook_script ('pre-stop', $task, $logfd); | |
1007 | $plugin->stop_vm ($task, $vmid); | |
1008 | $cleanup->{restart} = 1; | |
1009 | } | |
1010 | ||
1011 | ||
1012 | } elsif ($mode eq 'suspend') { | |
1013 | $plugin->prepare ($task, $vmid, $mode); | |
1014 | ||
1015 | $self->run_hook_script ('backup-start', $task, $logfd); | |
1016 | ||
1017 | if ($vmtype eq 'lxc') { | |
1018 | # pre-suspend rsync | |
1019 | $plugin->copy_data_phase1($task, $vmid); | |
1020 | } | |
1021 | ||
1022 | debugmsg ('info', "suspending guest", $logfd); | |
1023 | $task->{vmstoptime} = time (); | |
1024 | $self->run_hook_script ('pre-stop', $task, $logfd); | |
1025 | $plugin->suspend_vm ($task, $vmid); | |
1026 | $cleanup->{resume} = 1; | |
1027 | ||
1028 | if ($vmtype eq 'lxc') { | |
1029 | # post-suspend rsync | |
1030 | $plugin->copy_data_phase2($task, $vmid); | |
1031 | ||
1032 | debugmsg ('info', "resuming guest", $logfd); | |
1033 | $cleanup->{resume} = 0; | |
1034 | $self->run_hook_script('pre-restart', $task, $logfd); | |
1035 | $plugin->resume_vm($task, $vmid); | |
1036 | $self->run_hook_script('post-restart', $task, $logfd); | |
1037 | $log_vm_online_again->(); | |
1038 | } | |
1039 | ||
1040 | } elsif ($mode eq 'snapshot') { | |
1041 | $self->run_hook_script ('backup-start', $task, $logfd); | |
1042 | ||
1043 | my $snapshot_count = $task->{snapshot_count} || 0; | |
1044 | ||
1045 | $self->run_hook_script ('pre-stop', $task, $logfd); | |
1046 | ||
1047 | if ($snapshot_count > 1) { | |
1048 | debugmsg ('info', "suspend vm to make snapshot", $logfd); | |
1049 | $task->{vmstoptime} = time (); | |
1050 | $plugin->suspend_vm ($task, $vmid); | |
1051 | $cleanup->{resume} = 1; | |
1052 | } | |
1053 | ||
1054 | $plugin->snapshot ($task, $vmid); | |
1055 | ||
1056 | $self->run_hook_script ('pre-restart', $task, $logfd); | |
1057 | ||
1058 | if ($snapshot_count > 1) { | |
1059 | debugmsg ('info', "resume vm", $logfd); | |
1060 | $cleanup->{resume} = 0; | |
1061 | $plugin->resume_vm ($task, $vmid); | |
1062 | $log_vm_online_again->(); | |
1063 | } | |
1064 | ||
1065 | $self->run_hook_script ('post-restart', $task, $logfd); | |
1066 | ||
1067 | } else { | |
1068 | die "internal error - unknown mode '$mode'\n"; | |
1069 | } | |
1070 | ||
1071 | # assemble archive image | |
1072 | $plugin->assemble ($task, $vmid); | |
1073 | ||
1074 | # produce archive | |
1075 | ||
1076 | if ($opts->{stdout}) { | |
1077 | debugmsg ('info', "sending archive to stdout", $logfd); | |
1078 | $plugin->archive($task, $vmid, $task->{tmptar}, $comp); | |
1079 | $self->run_hook_script ('backup-end', $task, $logfd); | |
1080 | return; | |
1081 | } | |
1082 | ||
1083 | my $archive_txt = $self->{opts}->{pbs} ? 'Proxmox Backup Server' : 'vzdump'; | |
1084 | debugmsg('info', "creating $archive_txt archive '$task->{target}'", $logfd); | |
1085 | $plugin->archive($task, $vmid, $task->{tmptar}, $comp); | |
1086 | ||
1087 | if ($self->{opts}->{pbs}) { | |
1088 | # size is added to task struct in guest vzdump plugins | |
1089 | } else { | |
1090 | rename ($task->{tmptar}, $task->{target}) || | |
1091 | die "unable to rename '$task->{tmptar}' to '$task->{target}'\n"; | |
1092 | ||
1093 | # determine size | |
1094 | $task->{size} = (-s $task->{target}) || 0; | |
1095 | my $cs = format_size ($task->{size}); | |
1096 | debugmsg ('info', "archive file size: $cs", $logfd); | |
1097 | } | |
1098 | ||
1099 | # Mark as protected before pruning. | |
1100 | if (my $storeid = $opts->{storage}) { | |
1101 | my $volname = $opts->{pbs} ? $task->{target} : basename($task->{target}); | |
1102 | my $volid = "${storeid}:backup/${volname}"; | |
1103 | ||
1104 | if ($opts->{'notes-template'} && $opts->{'notes-template'} ne '') { | |
1105 | debugmsg('info', "adding notes to backup", $logfd); | |
1106 | my $notes = eval { $generate_notes->($opts->{'notes-template'}, $task); }; | |
1107 | if (my $err = $@) { | |
1108 | debugmsg('warn', "unable to add notes - $err", $logfd); | |
1109 | } else { | |
1110 | eval { PVE::Storage::update_volume_attribute($cfg, $volid, 'notes', $notes) }; | |
1111 | debugmsg('warn', "unable to add notes - $@", $logfd) if $@; | |
1112 | } | |
1113 | } | |
1114 | ||
1115 | if ($opts->{protected}) { | |
1116 | debugmsg('info', "marking backup as protected", $logfd); | |
1117 | eval { PVE::Storage::update_volume_attribute($cfg, $volid, 'protected', 1) }; | |
1118 | die "unable to set protected flag - $@\n" if $@; | |
1119 | } | |
1120 | } | |
1121 | ||
1122 | if ($opts->{remove}) { | |
1123 | my $keepstr = join(', ', map { "$_=$prune_options->{$_}" } sort keys %$prune_options); | |
1124 | debugmsg ('info', "prune older backups with retention: $keepstr", $logfd); | |
1125 | my $pruned = 0; | |
1126 | if (!defined($opts->{storage})) { | |
1127 | my $bklist = get_backup_file_list($opts->{dumpdir}, $bkname); | |
1128 | ||
1129 | PVE::Storage::prune_mark_backup_group($bklist, $prune_options); | |
1130 | ||
1131 | foreach my $prune_entry (@{$bklist}) { | |
1132 | next if $prune_entry->{mark} ne 'remove'; | |
1133 | $pruned++; | |
1134 | my $archive_path = $prune_entry->{path}; | |
1135 | debugmsg ('info', "delete old backup '$archive_path'", $logfd); | |
1136 | PVE::Storage::archive_remove($archive_path); | |
1137 | } | |
1138 | } else { | |
1139 | my $pruned_list = PVE::Storage::prune_backups( | |
1140 | $cfg, | |
1141 | $opts->{storage}, | |
1142 | $prune_options, | |
1143 | $vmid, | |
1144 | $vmtype, | |
1145 | 0, | |
1146 | sub { debugmsg($_[0], $_[1], $logfd) }, | |
1147 | ); | |
1148 | $pruned = scalar(grep { $_->{mark} eq 'remove' } $pruned_list->@*); | |
1149 | } | |
1150 | my $log_pruned_extra = $pruned > 0 ? " not covered by keep-retention policy" : ""; | |
1151 | debugmsg ('info', "pruned $pruned backup(s)${log_pruned_extra}", $logfd); | |
1152 | } | |
1153 | ||
1154 | $self->run_hook_script ('backup-end', $task, $logfd); | |
1155 | }; | |
1156 | my $err = $@; | |
1157 | ||
1158 | if ($plugin) { | |
1159 | # clean-up | |
1160 | ||
1161 | if ($cleanup->{unlock}) { | |
1162 | eval { $plugin->unlock_vm ($vmid); }; | |
1163 | warn $@ if $@; | |
1164 | } | |
1165 | ||
1166 | if ($cleanup->{prepared}) { | |
1167 | # only call cleanup when necessary (when prepare was executed) | |
1168 | eval { $plugin->cleanup ($task, $vmid) }; | |
1169 | warn $@ if $@; | |
1170 | } | |
1171 | ||
1172 | eval { $plugin->set_logfd (undef); }; | |
1173 | warn $@ if $@; | |
1174 | ||
1175 | if ($cleanup->{resume} || $cleanup->{restart}) { | |
1176 | eval { | |
1177 | $self->run_hook_script ('pre-restart', $task, $logfd); | |
1178 | if ($cleanup->{resume}) { | |
1179 | debugmsg ('info', "resume vm", $logfd); | |
1180 | $plugin->resume_vm ($task, $vmid); | |
1181 | } else { | |
1182 | my $running = $plugin->vm_status($vmid); | |
1183 | if (!$running) { | |
1184 | debugmsg ('info', "restarting vm", $logfd); | |
1185 | $plugin->start_vm ($task, $vmid); | |
1186 | } | |
1187 | } | |
1188 | $self->run_hook_script ('post-restart', $task, $logfd); | |
1189 | }; | |
1190 | my $err = $@; | |
1191 | if ($err) { | |
1192 | warn $err; | |
1193 | } else { | |
1194 | $log_vm_online_again->(); | |
1195 | } | |
1196 | } | |
1197 | } | |
1198 | ||
1199 | eval { unlink $task->{tmptar} if $task->{tmptar} && -f $task->{tmptar}; }; | |
1200 | warn $@ if $@; | |
1201 | ||
1202 | eval { rmtree $task->{tmpdir} if $task->{tmpdir} && -d $task->{tmpdir}; }; | |
1203 | warn $@ if $@; | |
1204 | ||
1205 | my $delay = $task->{backuptime} = time () - $vmstarttime; | |
1206 | ||
1207 | if ($err) { | |
1208 | $task->{state} = 'err'; | |
1209 | $task->{msg} = $err; | |
1210 | debugmsg ('err', "Backup of VM $vmid failed - $err", $logfd, 1); | |
1211 | debugmsg ('info', "Failed at " . strftime("%F %H:%M:%S", localtime())); | |
1212 | ||
1213 | eval { $self->run_hook_script ('backup-abort', $task, $logfd); }; | |
1214 | ||
1215 | } else { | |
1216 | $task->{state} = 'ok'; | |
1217 | my $tstr = format_time ($delay); | |
1218 | debugmsg ('info', "Finished Backup of VM $vmid ($tstr)", $logfd, 1); | |
1219 | debugmsg ('info', "Backup finished at " . strftime("%F %H:%M:%S", localtime())); | |
1220 | } | |
1221 | ||
1222 | close ($logfd) if $logfd; | |
1223 | ||
1224 | if ($task->{tmplog}) { | |
1225 | if ($self->{opts}->{pbs}) { | |
1226 | if ($task->{state} eq 'ok') { | |
1227 | eval { | |
1228 | PVE::Storage::PBSPlugin::run_raw_client_cmd( | |
1229 | $opts->{scfg}, | |
1230 | $opts->{storage}, | |
1231 | 'upload-log', | |
1232 | [ $pbs_snapshot_name, $task->{tmplog} ], | |
1233 | errmsg => "uploading backup task log failed", | |
1234 | outfunc => sub {}, | |
1235 | ); | |
1236 | }; | |
1237 | debugmsg('warn', "$@") if $@; # $@ contains already error prefix | |
1238 | } | |
1239 | } elsif ($task->{logfile}) { | |
1240 | system {'cp'} 'cp', $task->{tmplog}, $task->{logfile}; | |
1241 | } | |
1242 | } | |
1243 | ||
1244 | eval { $self->run_hook_script ('log-end', $task); }; | |
1245 | ||
1246 | die $err if $err && $err =~ m/^interrupted by signal$/; | |
1247 | } | |
1248 | ||
1249 | sub exec_backup { | |
1250 | my ($self, $rpcenv, $authuser) = @_; | |
1251 | ||
1252 | my $opts = $self->{opts}; | |
1253 | ||
1254 | debugmsg ('info', "starting new backup job: $self->{cmdline}", undef, 1); | |
1255 | ||
1256 | if (scalar(@{$self->{skiplist}})) { | |
1257 | my $skip_string = join(', ', sort { $a <=> $b } @{$self->{skiplist}}); | |
1258 | debugmsg ('info', "skip external VMs: $skip_string"); | |
1259 | } | |
1260 | ||
1261 | my $tasklist = []; | |
1262 | my $vzdump_plugins = {}; | |
1263 | foreach my $plugin (@{$self->{plugins}}) { | |
1264 | my $type = $plugin->type(); | |
1265 | next if exists $vzdump_plugins->{$type}; | |
1266 | $vzdump_plugins->{$type} = $plugin; | |
1267 | } | |
1268 | ||
1269 | my $vmlist = PVE::Cluster::get_vmlist(); | |
1270 | my $vmids = [ sort { $a <=> $b } @{$opts->{vmids}} ]; | |
1271 | foreach my $vmid (@{$vmids}) { | |
1272 | my $plugin; | |
1273 | if (defined($vmlist->{ids}->{$vmid})) { | |
1274 | my $guest_type = $vmlist->{ids}->{$vmid}->{type}; | |
1275 | $plugin = $vzdump_plugins->{$guest_type}; | |
1276 | next if !$rpcenv->check($authuser, "/vms/$vmid", [ 'VM.Backup' ], $opts->{all}); | |
1277 | } | |
1278 | push @$tasklist, { | |
1279 | mode => $opts->{mode}, | |
1280 | plugin => $plugin, | |
1281 | state => 'todo', | |
1282 | vmid => $vmid, | |
1283 | }; | |
1284 | } | |
1285 | ||
1286 | # Use in-memory files for the outer hook logs to pass them to sendmail. | |
1287 | my $job_start_log = ''; | |
1288 | my $job_end_log = ''; | |
1289 | open my $job_start_fd, '>', \$job_start_log; | |
1290 | open my $job_end_fd, '>', \$job_end_log; | |
1291 | ||
1292 | my $starttime = time(); | |
1293 | my $errcount = 0; | |
1294 | eval { | |
1295 | ||
1296 | $self->run_hook_script ('job-start', undef, $job_start_fd); | |
1297 | ||
1298 | foreach my $task (@$tasklist) { | |
1299 | $self->exec_backup_task ($task); | |
1300 | $errcount += 1 if $task->{state} ne 'ok'; | |
1301 | } | |
1302 | ||
1303 | $self->run_hook_script ('job-end', undef, $job_end_fd); | |
1304 | }; | |
1305 | my $err = $@; | |
1306 | ||
1307 | if ($err) { | |
1308 | eval { $self->run_hook_script ('job-abort', undef, $job_end_fd); }; | |
1309 | $err .= $@ if $@; | |
1310 | debugmsg ('err', "Backup job failed - $err", undef, 1); | |
1311 | } else { | |
1312 | if ($errcount) { | |
1313 | debugmsg ('info', "Backup job finished with errors", undef, 1); | |
1314 | } else { | |
1315 | debugmsg ('info', "Backup job finished successfully", undef, 1); | |
1316 | } | |
1317 | } | |
1318 | ||
1319 | close $job_start_fd; | |
1320 | close $job_end_fd; | |
1321 | ||
1322 | my $totaltime = time() - $starttime; | |
1323 | ||
1324 | eval { | |
1325 | # otherwise $self->sendmail() will interpret it as multiple problems | |
1326 | my $chomped_err = $err; | |
1327 | chomp($chomped_err) if $chomped_err; | |
1328 | ||
1329 | $self->sendmail( | |
1330 | $tasklist, | |
1331 | $totaltime, | |
1332 | $chomped_err, | |
1333 | $self->{job_init_log} . $job_start_log, | |
1334 | $job_end_log, | |
1335 | ); | |
1336 | }; | |
1337 | debugmsg ('err', $@) if $@; | |
1338 | ||
1339 | die $err if $err; | |
1340 | ||
1341 | die "job errors\n" if $errcount; | |
1342 | ||
1343 | unlink $pidfile; | |
1344 | } | |
1345 | ||
1346 | ||
1347 | sub option_exists { | |
1348 | my $key = shift; | |
1349 | return defined($confdesc->{$key}); | |
1350 | } | |
1351 | ||
1352 | # NOTE it might make sense to merge this and verify_vzdump_parameters(), but one | |
1353 | # needs to adapt command_line() in guest-common's PVE/VZDump/Common.pm and detect | |
1354 | # a second parsing attempt, because verify_vzdump_parameters() is called twice | |
1355 | # during the update_job API call. | |
1356 | sub parse_mailto_exclude_path { | |
1357 | my ($param) = @_; | |
1358 | ||
1359 | # exclude-path list need to be 0 separated or be an array | |
1360 | if (defined($param->{'exclude-path'})) { | |
1361 | my $expaths; | |
1362 | if (ref($param->{'exclude-path'}) eq 'ARRAY') { | |
1363 | $expaths = $param->{'exclude-path'}; | |
1364 | } else { | |
1365 | $expaths = [split(/\0/, $param->{'exclude-path'} || '')]; | |
1366 | } | |
1367 | $param->{'exclude-path'} = $expaths; | |
1368 | } | |
1369 | ||
1370 | if (defined($param->{mailto})) { | |
1371 | my @mailto = PVE::Tools::split_list(extract_param($param, 'mailto')); | |
1372 | $param->{mailto} = [ @mailto ]; | |
1373 | } | |
1374 | ||
1375 | return; | |
1376 | } | |
1377 | ||
1378 | sub verify_vzdump_parameters { | |
1379 | my ($param, $check_missing) = @_; | |
1380 | ||
1381 | raise_param_exc({ all => "option conflicts with option 'vmid'"}) | |
1382 | if $param->{all} && $param->{vmid}; | |
1383 | ||
1384 | raise_param_exc({ exclude => "option conflicts with option 'vmid'"}) | |
1385 | if $param->{exclude} && $param->{vmid}; | |
1386 | ||
1387 | raise_param_exc({ pool => "option conflicts with option 'vmid'"}) | |
1388 | if $param->{pool} && $param->{vmid}; | |
1389 | ||
1390 | raise_param_exc({ 'prune-backups' => "option conflicts with option 'maxfiles'"}) | |
1391 | if defined($param->{'prune-backups'}) && defined($param->{maxfiles}); | |
1392 | ||
1393 | $parse_prune_backups_maxfiles->($param, 'CLI parameters'); | |
1394 | parse_performance($param); | |
1395 | ||
1396 | if (my $template = $param->{'notes-template'}) { | |
1397 | eval { $verify_notes_template->($template); }; | |
1398 | raise_param_exc({'notes-template' => $@}) if $@; | |
1399 | } | |
1400 | ||
1401 | $param->{all} = 1 if (defined($param->{exclude}) && !$param->{pool}); | |
1402 | ||
1403 | return if !$check_missing; | |
1404 | ||
1405 | raise_param_exc({ vmid => "property is missing"}) | |
1406 | if !($param->{all} || $param->{stop} || $param->{pool}) && !$param->{vmid}; | |
1407 | ||
1408 | } | |
1409 | ||
1410 | sub stop_running_backups { | |
1411 | my($self) = @_; | |
1412 | ||
1413 | my $upid = PVE::Tools::file_read_firstline($pidfile); | |
1414 | return if !$upid; | |
1415 | ||
1416 | my $task = PVE::Tools::upid_decode($upid); | |
1417 | ||
1418 | if (PVE::ProcFSTools::check_process_running($task->{pid}, $task->{pstart}) && | |
1419 | PVE::ProcFSTools::read_proc_starttime($task->{pid}) == $task->{pstart}) { | |
1420 | kill(15, $task->{pid}); | |
1421 | # wait max 15 seconds to shut down (else, do nothing for now) | |
1422 | my $i; | |
1423 | for ($i = 15; $i > 0; $i--) { | |
1424 | last if !PVE::ProcFSTools::check_process_running(($task->{pid}, $task->{pstart})); | |
1425 | sleep (1); | |
1426 | } | |
1427 | die "stopping backup process $task->{pid} failed\n" if $i == 0; | |
1428 | } | |
1429 | } | |
1430 | ||
1431 | sub get_included_guests { | |
1432 | my ($job) = @_; | |
1433 | ||
1434 | my $vmids = []; | |
1435 | my $vmids_per_node = {}; | |
1436 | ||
1437 | my $vmlist = PVE::Cluster::get_vmlist(); | |
1438 | ||
1439 | if ($job->{pool}) { | |
1440 | $vmids = PVE::API2Tools::get_resource_pool_guest_members($job->{pool}); | |
1441 | } elsif ($job->{vmid}) { | |
1442 | $vmids = [ split_list($job->{vmid}) ]; | |
1443 | } elsif ($job->{all}) { | |
1444 | # all or exclude | |
1445 | my $exclude = check_vmids(split_list($job->{exclude})); | |
1446 | my $excludehash = { map { $_ => 1 } @$exclude }; | |
1447 | ||
1448 | for my $id (keys %{$vmlist->{ids}}) { | |
1449 | next if $excludehash->{$id}; | |
1450 | push @$vmids, $id; | |
1451 | } | |
1452 | } else { | |
1453 | return $vmids_per_node; | |
1454 | } | |
1455 | $vmids = check_vmids(@$vmids); | |
1456 | ||
1457 | for my $vmid (@$vmids) { | |
1458 | if (defined($vmlist->{ids}->{$vmid})) { | |
1459 | my $node = $vmlist->{ids}->{$vmid}->{node}; | |
1460 | next if (defined $job->{node} && $job->{node} ne $node); | |
1461 | ||
1462 | push @{$vmids_per_node->{$node}}, $vmid; | |
1463 | } else { | |
1464 | push @{$vmids_per_node->{''}}, $vmid; | |
1465 | } | |
1466 | } | |
1467 | ||
1468 | return $vmids_per_node; | |
1469 | } | |
1470 | ||
1471 | 1; |