]>
Commit | Line | Data |
---|---|---|
aaeeeebe DM |
1 | package PVE::VZDump; |
2 | ||
aaeeeebe DM |
3 | use strict; |
4 | use warnings; | |
df2d3636 | 5 | |
aaeeeebe | 6 | use Fcntl ':flock'; |
df2d3636 | 7 | use File::Path; |
aaeeeebe DM |
8 | use IO::File; |
9 | use IO::Select; | |
10 | use IPC::Open3; | |
e2b5eb29 | 11 | use POSIX qw(strftime); |
aaeeeebe | 12 | use Time::Local; |
df2d3636 TL |
13 | |
14 | use PVE::Cluster qw(cfs_read_file); | |
15 | use PVE::DataCenterConfig; | |
16 | use PVE::Exception qw(raise_param_exc); | |
2de3ee58 | 17 | use PVE::HA::Config; |
df2d3636 TL |
18 | use PVE::HA::Env::PVE2; |
19 | use PVE::JSONSchema qw(get_standard_option); | |
20 | use PVE::RPCEnvironment; | |
21 | use PVE::Storage; | |
2424074e | 22 | use PVE::VZDump::Common; |
df2d3636 | 23 | use PVE::VZDump::Plugin; |
05447e04 | 24 | use PVE::Tools qw(extract_param split_list); |
5c4da4c3 | 25 | use PVE::API2Tools; |
aaeeeebe DM |
26 | |
27 | my @posix_filesystems = qw(ext3 ext4 nfs nfs4 reiserfs xfs); | |
28 | ||
29 | my $lockfile = '/var/run/vzdump.lock'; | |
8682f6fc | 30 | my $pidfile = '/var/run/vzdump.pid'; |
aaeeeebe DM |
31 | my $logdir = '/var/log/vzdump'; |
32 | ||
dafb6246 | 33 | my @plugins = qw(); |
aaeeeebe | 34 | |
2424074e | 35 | my $confdesc = PVE::VZDump::Common::get_confdesc(); |
cc61ea36 | 36 | |
aaeeeebe | 37 | # Load available plugins |
8187f6b0 DM |
38 | my @pve_vzdump_classes = qw(PVE::VZDump::QemuServer PVE::VZDump::LXC); |
39 | foreach my $plug (@pve_vzdump_classes) { | |
40 | my $filename = "/usr/share/perl5/$plug.pm"; | |
41 | $filename =~ s!::!/!g; | |
42 | if (-f $filename) { | |
43 | eval { require $filename; }; | |
44 | if (!$@) { | |
45 | $plug->import (); | |
46 | push @plugins, $plug; | |
47 | } else { | |
c76d0106 | 48 | die $@; |
8187f6b0 DM |
49 | } |
50 | } | |
aaeeeebe DM |
51 | } |
52 | ||
53 | # helper functions | |
54 | ||
aaeeeebe DM |
55 | sub debugmsg { |
56 | my ($mtype, $msg, $logfd, $syslog) = @_; | |
57 | ||
a5c94797 | 58 | PVE::VZDump::Plugin::debugmsg(@_); |
aaeeeebe DM |
59 | } |
60 | ||
61 | sub run_command { | |
62 | my ($logfd, $cmdstr, %param) = @_; | |
63 | ||
7f910306 | 64 | my $logfunc = sub { |
4a4051d8 | 65 | my $line = shift; |
4a4051d8 | 66 | debugmsg ('info', $line, $logfd); |
aaeeeebe DM |
67 | }; |
68 | ||
7f910306 | 69 | PVE::Tools::run_command($cmdstr, %param, logfunc => $logfunc); |
aaeeeebe DM |
70 | } |
71 | ||
72 | sub storage_info { | |
73 | my $storage = shift; | |
74 | ||
bbcfdc08 | 75 | my $cfg = PVE::Storage::config(); |
4a4051d8 | 76 | my $scfg = PVE::Storage::storage_config($cfg, $storage); |
aaeeeebe | 77 | my $type = $scfg->{type}; |
60e049c2 TM |
78 | |
79 | die "can't use storage type '$type' for backup\n" | |
6d09915c | 80 | if (!($type eq 'dir' || $type eq 'nfs' || $type eq 'glusterfs' |
1a87db9e | 81 | || $type eq 'cifs' || $type eq 'cephfs' || $type eq 'pbs')); |
60e049c2 | 82 | die "can't use storage '$storage' for backups - wrong content type\n" |
aaeeeebe DM |
83 | if (!$scfg->{content}->{backup}); |
84 | ||
4a4051d8 | 85 | PVE::Storage::activate_storage($cfg, $storage); |
aaeeeebe | 86 | |
3a805f8d FE |
87 | my $info = { |
88 | scfg => $scfg, | |
89 | maxfiles => $scfg->{maxfiles}, | |
90 | }; | |
91 | ||
e6946086 FE |
92 | $info->{'prune-backups'} = PVE::JSONSchema::parse_property_string('prune-backups', $scfg->{'prune-backups'}) |
93 | if defined($scfg->{'prune-backups'}); | |
94 | ||
95 | die "cannot have 'maxfiles' and 'prune-backups' configured at the same time\n" | |
96 | if defined($info->{'prune-backups'}) && defined($info->{maxfiles}); | |
97 | ||
1a87db9e | 98 | if ($type eq 'pbs') { |
3a805f8d | 99 | $info->{pbs} = 1; |
1a87db9e | 100 | } else { |
3a805f8d | 101 | $info->{dumpdir} = PVE::Storage::get_backup_dir($cfg, $storage); |
1a87db9e | 102 | } |
3a805f8d FE |
103 | |
104 | return $info; | |
aaeeeebe DM |
105 | } |
106 | ||
107 | sub format_size { | |
108 | my $size = shift; | |
109 | ||
110 | my $kb = $size / 1024; | |
111 | ||
112 | if ($kb < 1024) { | |
113 | return int ($kb) . "KB"; | |
114 | } | |
115 | ||
116 | my $mb = $size / (1024*1024); | |
aaeeeebe DM |
117 | if ($mb < 1024) { |
118 | return int ($mb) . "MB"; | |
9c3a51ee TL |
119 | } |
120 | my $gb = $mb / 1024; | |
121 | if ($gb < 1024) { | |
aaeeeebe | 122 | return sprintf ("%.2fGB", $gb); |
60e049c2 | 123 | } |
9c3a51ee TL |
124 | my $tb = $gb / 1024; |
125 | return sprintf ("%.2fTB", $tb); | |
aaeeeebe DM |
126 | } |
127 | ||
128 | sub format_time { | |
129 | my $seconds = shift; | |
130 | ||
131 | my $hours = int ($seconds/3600); | |
132 | $seconds = $seconds - $hours*3600; | |
133 | my $min = int ($seconds/60); | |
134 | $seconds = $seconds - $min*60; | |
135 | ||
136 | return sprintf ("%02d:%02d:%02d", $hours, $min, $seconds); | |
137 | } | |
138 | ||
139 | sub encode8bit { | |
140 | my ($str) = @_; | |
141 | ||
142 | $str =~ s/^(.{990})/$1\n/mg; # reduce line length | |
143 | ||
144 | return $str; | |
145 | } | |
146 | ||
147 | sub escape_html { | |
148 | my ($str) = @_; | |
149 | ||
150 | $str =~ s/&/&/g; | |
151 | $str =~ s/</</g; | |
152 | $str =~ s/>/>/g; | |
153 | ||
154 | return $str; | |
155 | } | |
156 | ||
157 | sub check_bin { | |
158 | my ($bin) = @_; | |
159 | ||
160 | foreach my $p (split (/:/, $ENV{PATH})) { | |
161 | my $fn = "$p/$bin"; | |
162 | if (-x $fn) { | |
163 | return $fn; | |
164 | } | |
165 | } | |
166 | ||
167 | die "unable to find command '$bin'\n"; | |
168 | } | |
169 | ||
170 | sub check_vmids { | |
171 | my (@vmids) = @_; | |
172 | ||
173 | my $res = []; | |
50ba40ec | 174 | for my $vmid (sort {$a <=> $b} @vmids) { |
aaeeeebe DM |
175 | die "ERROR: strange VM ID '${vmid}'\n" if $vmid !~ m/^\d+$/; |
176 | $vmid = int ($vmid); # remove leading zeros | |
4a4051d8 | 177 | next if !$vmid; |
aaeeeebe DM |
178 | push @$res, $vmid; |
179 | } | |
180 | ||
181 | return $res; | |
182 | } | |
183 | ||
184 | ||
185 | sub read_vzdump_defaults { | |
186 | ||
187 | my $fn = "/etc/vzdump.conf"; | |
188 | ||
cc61ea36 | 189 | my $defaults = { |
1ab98f05 WB |
190 | map { |
191 | my $default = $confdesc->{$_}->{default}; | |
192 | defined($default) ? ($_ => $default) : () | |
193 | } keys %$confdesc | |
aaeeeebe DM |
194 | }; |
195 | ||
cc61ea36 TL |
196 | my $raw; |
197 | eval { $raw = PVE::Tools::file_get_contents($fn); }; | |
198 | return $defaults if $@; | |
aaeeeebe | 199 | |
cc61ea36 TL |
200 | my $conf_schema = { type => 'object', properties => $confdesc, }; |
201 | my $res = PVE::JSONSchema::parse_config($conf_schema, $fn, $raw); | |
6368bbd4 WB |
202 | if (my $excludes = $res->{'exclude-path'}) { |
203 | $res->{'exclude-path'} = PVE::Tools::split_args($excludes); | |
204 | } | |
74b47fd8 | 205 | if (defined($res->{mailto})) { |
05447e04 | 206 | my @mailto = split_list($res->{mailto}); |
74b47fd8 FG |
207 | $res->{mailto} = [ @mailto ]; |
208 | } | |
cc61ea36 TL |
209 | |
210 | foreach my $key (keys %$defaults) { | |
e94fa5cb | 211 | $res->{$key} = $defaults->{$key} if !defined($res->{$key}); |
aaeeeebe | 212 | } |
aaeeeebe DM |
213 | |
214 | return $res; | |
215 | } | |
216 | ||
ce84a950 | 217 | use constant MAX_MAIL_SIZE => 1024*1024; |
6ec9de44 | 218 | sub sendmail { |
44b21574 | 219 | my ($self, $tasklist, $totaltime, $err, $detail_pre, $detail_post) = @_; |
aaeeeebe DM |
220 | |
221 | my $opts = $self->{opts}; | |
222 | ||
223 | my $mailto = $opts->{mailto}; | |
224 | ||
4a4051d8 | 225 | return if !($mailto && scalar(@$mailto)); |
aaeeeebe DM |
226 | |
227 | my $cmdline = $self->{cmdline}; | |
228 | ||
229 | my $ecount = 0; | |
230 | foreach my $task (@$tasklist) { | |
231 | $ecount++ if $task->{state} ne 'ok'; | |
232 | chomp $task->{msg} if $task->{msg}; | |
233 | $task->{backuptime} = 0 if !$task->{backuptime}; | |
234 | $task->{size} = 0 if !$task->{size}; | |
6cba1855 | 235 | $task->{target} = 'unknown' if !$task->{target}; |
aaeeeebe DM |
236 | $task->{hostname} = "VM $task->{vmid}" if !$task->{hostname}; |
237 | ||
238 | if ($task->{state} eq 'todo') { | |
239 | $task->{msg} = 'aborted'; | |
240 | } | |
241 | } | |
242 | ||
02e59759 DM |
243 | my $notify = $opts->{mailnotification} || 'always'; |
244 | return if (!$ecount && !$err && ($notify eq 'failure')); | |
8b4b4860 | 245 | |
6ec9de44 | 246 | my $stat = ($ecount || $err) ? 'backup failed' : 'backup successful'; |
403761c4 WB |
247 | if ($err) { |
248 | if ($err =~ /\n/) { | |
249 | $stat .= ": multiple problems"; | |
250 | } else { | |
251 | $stat .= ": $err"; | |
252 | $err = undef; | |
253 | } | |
254 | } | |
aaeeeebe | 255 | |
4a4051d8 | 256 | my $hostname = `hostname -f` || PVE::INotify::nodename(); |
aaeeeebe DM |
257 | chomp $hostname; |
258 | ||
aaeeeebe | 259 | # text part |
403761c4 | 260 | my $text = $err ? "$err\n\n" : ''; |
2cc64d0e DJ |
261 | my $namelength = 20; |
262 | $text .= sprintf ( | |
263 | "%-10s %-${namelength}s %-6s %10s %10s %s\n", | |
264 | qw(VMID NAME STATUS TIME SIZE FILENAME) | |
265 | ); | |
aaeeeebe | 266 | foreach my $task (@$tasklist) { |
2cc64d0e DJ |
267 | my $name = substr($task->{hostname}, 0, $namelength); |
268 | my $successful = $task->{state} eq 'ok'; | |
269 | my $size = $successful ? format_size ($task->{size}) : 0; | |
270 | my $filename = $successful ? $task->{target} : '-'; | |
271 | my $size_fmt = $successful ? "%10s": "%8.2fMB"; | |
272 | $text .= sprintf( | |
273 | "%-10s %-${namelength}s %-6s %10s $size_fmt %s\n", | |
274 | $task->{vmid}, | |
275 | $name, | |
276 | $task->{state}, | |
277 | format_time($task->{backuptime}), | |
278 | $size, | |
279 | $filename, | |
280 | ); | |
aaeeeebe | 281 | } |
7a0a8e67 | 282 | |
ce84a950 DJ |
283 | my $text_log_part; |
284 | $text_log_part .= "\nDetailed backup logs:\n\n"; | |
285 | $text_log_part .= "$cmdline\n\n"; | |
aaeeeebe | 286 | |
ce84a950 | 287 | $text_log_part .= $detail_pre . "\n" if defined($detail_pre); |
aaeeeebe DM |
288 | foreach my $task (@$tasklist) { |
289 | my $vmid = $task->{vmid}; | |
290 | my $log = $task->{tmplog}; | |
291 | if (!$log) { | |
ce84a950 | 292 | $text_log_part .= "$vmid: no log available\n\n"; |
aaeeeebe DM |
293 | next; |
294 | } | |
ce84a950 DJ |
295 | if (open (TMP, "$log")) { |
296 | while (my $line = <TMP>) { | |
297 | next if $line =~ /^status: \d+/; # not useful in mails | |
298 | $text_log_part .= encode8bit ("$vmid: $line"); | |
299 | } | |
300 | } else { | |
301 | $text_log_part .= "$vmid: Could not open log file\n\n"; | |
302 | } | |
aaeeeebe | 303 | close (TMP); |
ce84a950 | 304 | $text_log_part .= "\n"; |
aaeeeebe | 305 | } |
ce84a950 | 306 | $text_log_part .= $detail_post if defined($detail_post); |
aaeeeebe | 307 | |
aaeeeebe | 308 | # html part |
7a0a8e67 | 309 | my $html = "<html><body>\n"; |
403761c4 | 310 | $html .= "<p>" . (escape_html($err) =~ s/\n/<br>/gr) . "</p>\n" if $err; |
7a0a8e67 TL |
311 | $html .= "<table border=1 cellpadding=3>\n"; |
312 | $html .= "<tr><td>VMID<td>NAME<td>STATUS<td>TIME<td>SIZE<td>FILENAME</tr>\n"; | |
aaeeeebe DM |
313 | |
314 | my $ssize = 0; | |
315 | ||
316 | foreach my $task (@$tasklist) { | |
317 | my $vmid = $task->{vmid}; | |
318 | my $name = $task->{hostname}; | |
319 | ||
320 | if ($task->{state} eq 'ok') { | |
321 | ||
322 | $ssize += $task->{size}; | |
323 | ||
7a0a8e67 | 324 | $html .= sprintf ("<tr><td>%s<td>%s<td>OK<td>%s<td align=right>%s<td>%s</tr>\n", |
aaeeeebe DM |
325 | $vmid, $name, |
326 | format_time($task->{backuptime}), | |
327 | format_size ($task->{size}), | |
6cba1855 | 328 | escape_html ($task->{target})); |
aaeeeebe | 329 | } else { |
7a0a8e67 TL |
330 | $html .= sprintf ("<tr><td>%s<td>%s<td><font color=red>FAILED<td>%s<td colspan=2>%s</tr>\n", |
331 | $vmid, $name, format_time($task->{backuptime}), | |
aaeeeebe DM |
332 | escape_html ($task->{msg})); |
333 | } | |
334 | } | |
335 | ||
7a0a8e67 | 336 | $html .= sprintf ("<tr><td align=left colspan=3>TOTAL<td>%s<td>%s<td></tr>", |
aaeeeebe DM |
337 | format_time ($totaltime), format_size ($ssize)); |
338 | ||
ce84a950 DJ |
339 | $html .= "\n</table><br><br>\n"; |
340 | my $html_log_part; | |
341 | $html_log_part .= "Detailed backup logs:<br /><br />\n"; | |
342 | $html_log_part .= "<pre>\n"; | |
343 | $html_log_part .= escape_html($cmdline) . "\n\n"; | |
aaeeeebe | 344 | |
ce84a950 | 345 | $html_log_part .= escape_html($detail_pre) . "\n" if defined($detail_pre); |
aaeeeebe DM |
346 | foreach my $task (@$tasklist) { |
347 | my $vmid = $task->{vmid}; | |
348 | my $log = $task->{tmplog}; | |
349 | if (!$log) { | |
ce84a950 | 350 | $html_log_part .= "$vmid: no log available\n\n"; |
aaeeeebe DM |
351 | next; |
352 | } | |
ce84a950 DJ |
353 | if (open (TMP, "$log")) { |
354 | while (my $line = <TMP>) { | |
355 | next if $line =~ /^status: \d+/; # not useful in mails | |
356 | if ($line =~ m/^\S+\s\d+\s+\d+:\d+:\d+\s+(ERROR|WARN):/) { | |
357 | $html_log_part .= encode8bit ("$vmid: <font color=red>". | |
358 | escape_html ($line) . "</font>"); | |
359 | } else { | |
360 | $html_log_part .= encode8bit ("$vmid: " . escape_html ($line)); | |
361 | } | |
aaeeeebe | 362 | } |
ce84a950 DJ |
363 | } else { |
364 | $html_log_part .= "$vmid: Could not open log file\n\n"; | |
aaeeeebe DM |
365 | } |
366 | close (TMP); | |
ce84a950 | 367 | $html_log_part .= "\n"; |
aaeeeebe | 368 | } |
ce84a950 DJ |
369 | $html_log_part .= escape_html($detail_post) if defined($detail_post); |
370 | $html_log_part .= "</pre>"; | |
371 | my $html_end .= "\n</body></html>\n"; | |
7a0a8e67 | 372 | # end html part |
aaeeeebe | 373 | |
ce84a950 DJ |
374 | if (length($text) + length($text_log_part) + |
375 | length($html) + length($html_log_part) < MAX_MAIL_SIZE) | |
376 | { | |
377 | $html .= $html_log_part; | |
378 | $text .= $text_log_part; | |
379 | } else { | |
380 | my $msg = "Log output was too long to be sent by mail. ". | |
381 | "See Task History for details!\n"; | |
382 | $text .= $msg; | |
383 | $html .= "<p>$msg</p>"; | |
384 | $html .= $html_end; | |
385 | } | |
386 | ||
7a0a8e67 | 387 | my $subject = "vzdump backup status ($hostname) : $stat"; |
aaeeeebe | 388 | |
7a0a8e67 TL |
389 | my $dcconf = PVE::Cluster::cfs_read_file('datacenter.cfg'); |
390 | my $mailfrom = $dcconf->{email_from} || "root"; | |
aaeeeebe | 391 | |
7a0a8e67 | 392 | PVE::Tools::sendmail($mailto, $subject, $text, $html, $mailfrom, "vzdump backup tool"); |
aaeeeebe DM |
393 | }; |
394 | ||
395 | sub new { | |
a7e42354 | 396 | my ($class, $cmdline, $opts, $skiplist) = @_; |
aaeeeebe DM |
397 | |
398 | mkpath $logdir; | |
399 | ||
400 | check_bin ('cp'); | |
401 | check_bin ('df'); | |
402 | check_bin ('sendmail'); | |
403 | check_bin ('rsync'); | |
404 | check_bin ('tar'); | |
405 | check_bin ('mount'); | |
406 | check_bin ('umount'); | |
407 | check_bin ('cstream'); | |
408 | check_bin ('ionice'); | |
409 | ||
47664cbe | 410 | if ($opts->{mode} && $opts->{mode} eq 'snapshot') { |
aaeeeebe DM |
411 | check_bin ('lvcreate'); |
412 | check_bin ('lvs'); | |
413 | check_bin ('lvremove'); | |
414 | } | |
415 | ||
416 | my $defaults = read_vzdump_defaults(); | |
417 | ||
899b8373 DM |
418 | $opts->{remove} = 1 if !defined($opts->{remove}); |
419 | ||
aaeeeebe | 420 | foreach my $k (keys %$defaults) { |
5b6b72e6 | 421 | next if $k eq 'exclude-path' || $k eq 'maxfiles'; # dealt with separately |
aaeeeebe DM |
422 | if ($k eq 'dumpdir' || $k eq 'storage') { |
423 | $opts->{$k} = $defaults->{$k} if !defined ($opts->{dumpdir}) && | |
424 | !defined ($opts->{storage}); | |
425 | } else { | |
426 | $opts->{$k} = $defaults->{$k} if !defined ($opts->{$k}); | |
427 | } | |
428 | } | |
429 | ||
aaeeeebe DM |
430 | $opts->{dumpdir} =~ s|/+$|| if ($opts->{dumpdir}); |
431 | $opts->{tmpdir} =~ s|/+$|| if ($opts->{tmpdir}); | |
432 | ||
a7e42354 DM |
433 | $skiplist = [] if !$skiplist; |
434 | my $self = bless { cmdline => $cmdline, opts => $opts, skiplist => $skiplist }; | |
aaeeeebe | 435 | |
45f16a06 | 436 | my $findexcl = $self->{findexcl} = []; |
f4a8bab4 | 437 | if ($defaults->{'exclude-path'}) { |
45f16a06 | 438 | push @$findexcl, @{$defaults->{'exclude-path'}}; |
f4a8bab4 DM |
439 | } |
440 | ||
aaeeeebe | 441 | if ($opts->{'exclude-path'}) { |
45f16a06 | 442 | push @$findexcl, @{$opts->{'exclude-path'}}; |
aaeeeebe DM |
443 | } |
444 | ||
445 | if ($opts->{stdexcludes}) { | |
e42ae622 | 446 | push @$findexcl, '/tmp/?*', |
45f16a06 | 447 | '/var/tmp/?*', |
0568e342 | 448 | '/var/run/?*.pid'; |
aaeeeebe DM |
449 | } |
450 | ||
451 | foreach my $p (@plugins) { | |
452 | ||
453 | my $pd = $p->new ($self); | |
454 | ||
455 | push @{$self->{plugins}}, $pd; | |
aaeeeebe DM |
456 | } |
457 | ||
1a87db9e | 458 | if (defined($opts->{storage}) && $opts->{stdout}) { |
a6fcd7d9 FE |
459 | die "cannot use options 'storage' and 'stdout' at the same time\n"; |
460 | } elsif (defined($opts->{storage}) && defined($opts->{dumpdir})) { | |
461 | die "cannot use options 'storage' and 'dumpdir' at the same time\n"; | |
1a87db9e DM |
462 | } |
463 | ||
aaeeeebe | 464 | if (!$opts->{dumpdir} && !$opts->{storage}) { |
19d5c0f2 | 465 | $opts->{storage} = 'local'; |
aaeeeebe DM |
466 | } |
467 | ||
403761c4 WB |
468 | my $errors = ''; |
469 | ||
aaeeeebe | 470 | if ($opts->{storage}) { |
1e4583d5 | 471 | my $info = eval { storage_info ($opts->{storage}) }; |
e6946086 FE |
472 | if (my $err = $@) { |
473 | $errors .= "could not get storage information for '$opts->{storage}': $err"; | |
474 | } else { | |
475 | $opts->{dumpdir} = $info->{dumpdir}; | |
476 | $opts->{scfg} = $info->{scfg}; | |
477 | $opts->{pbs} = $info->{pbs}; | |
478 | ||
479 | if (!defined($opts->{'prune-backups'}) && !defined($opts->{maxfiles})) { | |
480 | $opts->{'prune-backups'} = $info->{'prune-backups'}; | |
b15bdd86 | 481 | $opts->{maxfiles} = $info->{maxfiles}; |
e6946086 FE |
482 | } |
483 | } | |
aaeeeebe | 484 | } elsif ($opts->{dumpdir}) { |
403761c4 | 485 | $errors .= "dumpdir '$opts->{dumpdir}' does not exist" |
aaeeeebe DM |
486 | if ! -d $opts->{dumpdir}; |
487 | } else { | |
60e049c2 | 488 | die "internal error"; |
aaeeeebe DM |
489 | } |
490 | ||
f354fe47 | 491 | if (!defined($opts->{'prune-backups'})) { |
77aea6eb | 492 | my $maxfiles = delete $opts->{maxfiles} // $defaults->{maxfiles}; |
b15bdd86 FE |
493 | $maxfiles = int($maxfiles); # shouldn't be necessary, but be safe |
494 | if ($maxfiles) { | |
495 | $opts->{'prune-backups'} = { 'keep-last' => $maxfiles }; | |
496 | } else { | |
1db8529e | 497 | $opts->{'prune-backups'} = { 'keep-all' => 1 }; |
b15bdd86 | 498 | } |
e6946086 | 499 | } |
5b6b72e6 | 500 | |
1db8529e FE |
501 | # avoid triggering any remove code path if keep-all is set |
502 | $opts->{remove} = 0 if $opts->{'prune-backups'}->{'keep-all'}; | |
503 | ||
aaeeeebe | 504 | if ($opts->{tmpdir} && ! -d $opts->{tmpdir}) { |
403761c4 WB |
505 | $errors .= "\n" if $errors; |
506 | $errors .= "tmpdir '$opts->{tmpdir}' does not exist"; | |
507 | } | |
508 | ||
509 | if ($errors) { | |
510 | eval { $self->sendmail([], 0, $errors); }; | |
511 | debugmsg ('err', $@) if $@; | |
512 | die "$errors\n"; | |
aaeeeebe DM |
513 | } |
514 | ||
515 | return $self; | |
aaeeeebe DM |
516 | } |
517 | ||
aaeeeebe DM |
518 | sub get_mount_info { |
519 | my ($dir) = @_; | |
520 | ||
8572d646 DM |
521 | # Note: df 'available' can be negative, and percentage set to '-' |
522 | ||
d93d0459 | 523 | my $cmd = [ 'df', '-P', '-T', '-B', '1', $dir]; |
aaeeeebe | 524 | |
d93d0459 | 525 | my $res; |
aaeeeebe | 526 | |
d93d0459 DM |
527 | my $parser = sub { |
528 | my $line = shift; | |
8572d646 DM |
529 | if (my ($fsid, $fstype, undef, $mp) = $line =~ |
530 | m!(\S+.*)\s+(\S+)\s+\d+\s+\-?\d+\s+\d+\s+(\d+%|-)\s+(/.*)$!) { | |
d93d0459 DM |
531 | $res = { |
532 | device => $fsid, | |
533 | fstype => $fstype, | |
534 | mountpoint => $mp, | |
535 | }; | |
536 | } | |
aaeeeebe | 537 | }; |
d93d0459 DM |
538 | |
539 | eval { PVE::Tools::run_command($cmd, errfunc => sub {}, outfunc => $parser); }; | |
540 | warn $@ if $@; | |
541 | ||
542 | return $res; | |
aaeeeebe DM |
543 | } |
544 | ||
aaeeeebe | 545 | sub getlock { |
eab837c4 | 546 | my ($self, $upid) = @_; |
aaeeeebe | 547 | |
8682f6fc | 548 | my $fh; |
60e049c2 | 549 | |
aaeeeebe | 550 | my $maxwait = $self->{opts}->{lockwait} || $self->{lockwait}; |
60e049c2 | 551 | |
dc6e0a52 | 552 | die "missing UPID" if !$upid; # should not happen |
eab837c4 | 553 | |
aaeeeebe DM |
554 | if (!open (SERVER_FLCK, ">>$lockfile")) { |
555 | debugmsg ('err', "can't open lock on file '$lockfile' - $!", undef, 1); | |
6ec9de44 | 556 | die "can't open lock on file '$lockfile' - $!"; |
aaeeeebe DM |
557 | } |
558 | ||
eab837c4 | 559 | if (!flock (SERVER_FLCK, LOCK_EX|LOCK_NB)) { |
aaeeeebe | 560 | |
eab837c4 | 561 | if (!$maxwait) { |
76189130 TL |
562 | debugmsg ('err', "can't acquire lock '$lockfile' (wait = 0)", undef, 1); |
563 | die "can't acquire lock '$lockfile' (wait = 0)"; | |
eab837c4 | 564 | } |
aaeeeebe | 565 | |
eab837c4 | 566 | debugmsg('info', "trying to get global lock - waiting...", undef, 1); |
aaeeeebe | 567 | |
eab837c4 DM |
568 | eval { |
569 | alarm ($maxwait * 60); | |
60e049c2 | 570 | |
eab837c4 | 571 | local $SIG{ALRM} = sub { alarm (0); die "got timeout\n"; }; |
aaeeeebe | 572 | |
eab837c4 DM |
573 | if (!flock (SERVER_FLCK, LOCK_EX)) { |
574 | my $err = $!; | |
575 | close (SERVER_FLCK); | |
576 | alarm (0); | |
577 | die "$err\n"; | |
578 | } | |
aaeeeebe | 579 | alarm (0); |
eab837c4 | 580 | }; |
aaeeeebe | 581 | alarm (0); |
60e049c2 | 582 | |
eab837c4 | 583 | my $err = $@; |
60e049c2 | 584 | |
eab837c4 | 585 | if ($err) { |
76189130 TL |
586 | debugmsg ('err', "can't acquire lock '$lockfile' - $err", undef, 1); |
587 | die "can't acquire lock '$lockfile' - $err"; | |
eab837c4 | 588 | } |
aaeeeebe | 589 | |
eab837c4 | 590 | debugmsg('info', "got global lock", undef, 1); |
aaeeeebe DM |
591 | } |
592 | ||
eab837c4 | 593 | PVE::Tools::file_set_contents($pidfile, $upid); |
aaeeeebe DM |
594 | } |
595 | ||
596 | sub run_hook_script { | |
597 | my ($self, $phase, $task, $logfd) = @_; | |
598 | ||
599 | my $opts = $self->{opts}; | |
600 | ||
601 | my $script = $opts->{script}; | |
aaeeeebe DM |
602 | return if !$script; |
603 | ||
59798893 | 604 | if (!-x $script) { |
3d45fe51 | 605 | die "The hook script '$script' is not executable.\n"; |
59798893 MA |
606 | } |
607 | ||
aaeeeebe DM |
608 | my $cmd = "$script $phase"; |
609 | ||
610 | $cmd .= " $task->{mode} $task->{vmid}" if ($task); | |
611 | ||
612 | local %ENV; | |
28272ca2 DM |
613 | # set immutable opts directly (so they are available in all phases) |
614 | $ENV{STOREID} = $opts->{storage} if $opts->{storage}; | |
615 | $ENV{DUMPDIR} = $opts->{dumpdir} if $opts->{dumpdir}; | |
616 | ||
6cba1855 | 617 | foreach my $ek (qw(vmtype hostname target logfile)) { |
aaeeeebe DM |
618 | $ENV{uc($ek)} = $task->{$ek} if $task->{$ek}; |
619 | } | |
5d0d14f1 SI |
620 | # FIXME: for backwards compatibility - drop with PVE 7.0 |
621 | $ENV{TARFILE} = $task->{target} if $task->{target}; | |
aaeeeebe DM |
622 | |
623 | run_command ($logfd, $cmd); | |
624 | } | |
625 | ||
d7550e09 | 626 | sub compressor_info { |
778d5b6d TL |
627 | my ($opts) = @_; |
628 | my $opt_compress = $opts->{compress}; | |
d7550e09 DM |
629 | |
630 | if (!$opt_compress || $opt_compress eq '0') { | |
631 | return undef; | |
632 | } elsif ($opt_compress eq '1' || $opt_compress eq 'lzo') { | |
633 | return ('lzop', 'lzo'); | |
634 | } elsif ($opt_compress eq 'gzip') { | |
778d5b6d | 635 | if ($opts->{pigz} > 0) { |
8555b100 TL |
636 | my $pigz_threads = $opts->{pigz}; |
637 | if ($pigz_threads == 1) { | |
638 | my $cpuinfo = PVE::ProcFSTools::read_cpuinfo(); | |
639 | $pigz_threads = int(($cpuinfo->{cpus} + 1)/2); | |
640 | } | |
819e4ff5 | 641 | return ("pigz -p ${pigz_threads} --rsyncable", 'gz'); |
778d5b6d | 642 | } else { |
e953f92a | 643 | return ('gzip --rsyncable', 'gz'); |
778d5b6d | 644 | } |
3e2c5105 AA |
645 | } elsif ($opt_compress eq 'zstd') { |
646 | my $zstd_threads = $opts->{zstd} // 1; | |
647 | if ($zstd_threads == 0) { | |
648 | my $cpuinfo = PVE::ProcFSTools::read_cpuinfo(); | |
649 | $zstd_threads = int(($cpuinfo->{cpus} + 1)/2); | |
650 | } | |
7420d7ff | 651 | return ("zstd --rsyncable --threads=${zstd_threads}", 'zst'); |
d7550e09 DM |
652 | } else { |
653 | die "internal error - unknown compression option '$opt_compress'"; | |
654 | } | |
655 | } | |
899b8373 DM |
656 | |
657 | sub get_backup_file_list { | |
658 | my ($dir, $bkname, $exclude_fn) = @_; | |
659 | ||
660 | my $bklist = []; | |
661 | foreach my $fn (<$dir/${bkname}-*>) { | |
662 | next if $exclude_fn && $fn eq $exclude_fn; | |
de2d177e FE |
663 | |
664 | my $archive_info = eval { PVE::Storage::archive_info($fn) } // {}; | |
665 | if ($archive_info->{is_std_name}) { | |
666 | my $filename = $archive_info->{filename}; | |
667 | my $backup = { | |
668 | 'path' => "$dir/$filename", | |
669 | 'ctime' => $archive_info->{ctime}, | |
670 | }; | |
671 | push @{$bklist}, $backup; | |
899b8373 DM |
672 | } |
673 | } | |
674 | ||
675 | return $bklist; | |
676 | } | |
60e049c2 | 677 | |
aaeeeebe DM |
678 | sub exec_backup_task { |
679 | my ($self, $task) = @_; | |
60e049c2 | 680 | |
aaeeeebe DM |
681 | my $opts = $self->{opts}; |
682 | ||
e6946086 | 683 | my $cfg = PVE::Storage::config(); |
aaeeeebe DM |
684 | my $vmid = $task->{vmid}; |
685 | my $plugin = $task->{plugin}; | |
1a87db9e DM |
686 | |
687 | $task->{backup_time} = time(); | |
688 | ||
689 | my $pbs_group_name; | |
690 | my $pbs_snapshot_name; | |
691 | ||
aaeeeebe | 692 | my $vmstarttime = time (); |
60e049c2 | 693 | |
aaeeeebe DM |
694 | my $logfd; |
695 | ||
696 | my $cleanup = {}; | |
697 | ||
4e0947c8 TL |
698 | my $log_vm_online_again = sub { |
699 | return if !defined($task->{vmstoptime}); | |
700 | $task->{vmconttime} //= time(); | |
701 | my $delay = $task->{vmconttime} - $task->{vmstoptime}; | |
310cde8b | 702 | $delay = '<1' if $delay < 1; |
4e0947c8 TL |
703 | debugmsg ('info', "guest is online again after $delay seconds", $logfd); |
704 | }; | |
aaeeeebe DM |
705 | |
706 | eval { | |
707 | die "unable to find VM '$vmid'\n" if !$plugin; | |
708 | ||
d51f5a1e FE |
709 | my $vmtype = $plugin->type(); |
710 | ||
711 | if ($self->{opts}->{pbs}) { | |
712 | if ($vmtype eq 'lxc') { | |
713 | $pbs_group_name = "ct/$vmid"; | |
714 | } elsif ($vmtype eq 'qemu') { | |
715 | $pbs_group_name = "vm/$vmid"; | |
716 | } else { | |
717 | die "pbs backup not implemented for plugin type '$vmtype'\n"; | |
718 | } | |
719 | my $btime = strftime("%FT%TZ", gmtime($task->{backup_time})); | |
720 | $pbs_snapshot_name = "$pbs_group_name/$btime"; | |
721 | } | |
722 | ||
2de3ee58 | 723 | # for now we deny backups of a running ha managed service in *stop* mode |
83ec8f81 | 724 | # as it interferes with the HA stack (started services should not stop). |
2de3ee58 | 725 | if ($opts->{mode} eq 'stop' && |
83ec8f81 | 726 | PVE::HA::Config::vm_is_ha_managed($vmid, 'started')) |
2de3ee58 TL |
727 | { |
728 | die "Cannot execute a backup with stop mode on a HA managed and". | |
729 | " enabled Service. Use snapshot mode or disable the Service.\n"; | |
730 | } | |
731 | ||
aaeeeebe DM |
732 | my $tmplog = "$logdir/$vmtype-$vmid.log"; |
733 | ||
aaeeeebe | 734 | my $bkname = "vzdump-$vmtype-$vmid"; |
1a87db9e | 735 | my $basename = $bkname . strftime("-%Y_%m_%d-%H_%M_%S", localtime($task->{backup_time})); |
aaeeeebe | 736 | |
e6946086 FE |
737 | my $prune_options = $opts->{'prune-backups'}; |
738 | ||
739 | my $backup_limit = 0; | |
1db8529e FE |
740 | my $keep_all = delete $prune_options->{'keep-all'}; |
741 | if ($keep_all) { | |
742 | $prune_options = { 'keep-all' => 1 }; | |
743 | } else { | |
744 | foreach my $keep (values %{$prune_options}) { | |
745 | $backup_limit += $keep; | |
746 | } | |
e6946086 | 747 | } |
899b8373 | 748 | |
e6946086 | 749 | if ($backup_limit && !$opts->{remove}) { |
1a87db9e | 750 | my $count; |
7ddcb3af | 751 | if ($self->{opts}->{pbs}) { |
1a87db9e DM |
752 | my $res = PVE::Storage::PBSPlugin::run_client_cmd($opts->{scfg}, $opts->{storage}, 'snapshots', $pbs_group_name); |
753 | $count = scalar(@$res); | |
754 | } else { | |
755 | my $bklist = get_backup_file_list($opts->{dumpdir}, $bkname); | |
756 | $count = scalar(@$bklist); | |
757 | } | |
e6946086 | 758 | die "There is a max backup limit of $backup_limit enforced by the". |
1a87db9e DM |
759 | " target storage or the vzdump parameters.". |
760 | " Either increase the limit or delete old backup(s).\n" | |
e6946086 | 761 | if $count >= $backup_limit; |
899b8373 DM |
762 | } |
763 | ||
7ddcb3af | 764 | if (!$self->{opts}->{pbs}) { |
1a87db9e DM |
765 | $task->{logfile} = "$opts->{dumpdir}/$basename.log"; |
766 | } | |
aaeeeebe | 767 | |
757fd3d5 | 768 | my $ext = $vmtype eq 'qemu' ? '.vma' : '.tar'; |
778d5b6d | 769 | my ($comp, $comp_ext) = compressor_info($opts); |
d7550e09 DM |
770 | if ($comp && $comp_ext) { |
771 | $ext .= ".${comp_ext}"; | |
772 | } | |
aaeeeebe | 773 | |
7ddcb3af | 774 | if ($self->{opts}->{pbs}) { |
1a87db9e | 775 | die "unable to pipe backup to stdout\n" if $opts->{stdout}; |
b3c3304f | 776 | $task->{target} = $pbs_snapshot_name; |
aaeeeebe | 777 | } else { |
1a87db9e | 778 | if ($opts->{stdout}) { |
6cba1855 | 779 | $task->{target} = '-'; |
1a87db9e | 780 | } else { |
6cba1855 | 781 | $task->{target} = $task->{tmptar} = "$opts->{dumpdir}/$basename$ext"; |
1a87db9e DM |
782 | $task->{tmptar} =~ s/\.[^\.]+$/\.dat/; |
783 | unlink $task->{tmptar}; | |
784 | } | |
aaeeeebe DM |
785 | } |
786 | ||
787 | $task->{vmtype} = $vmtype; | |
788 | ||
dfb16115 | 789 | my $pid = $$; |
01fc3672 | 790 | if ($opts->{tmpdir}) { |
dfb16115 | 791 | $task->{tmpdir} = "$opts->{tmpdir}/vzdumptmp${pid}_$vmid/"; |
01fc3672 | 792 | } elsif ($self->{opts}->{pbs}) { |
dfb16115 | 793 | $task->{tmpdir} = "/var/tmp/vzdumptmp${pid}_$vmid"; |
aaeeeebe DM |
794 | } else { |
795 | # dumpdir is posix? then use it as temporary dir | |
5dc86eb8 | 796 | my $info = get_mount_info($opts->{dumpdir}); |
60e049c2 | 797 | if ($vmtype eq 'qemu' || |
aaeeeebe DM |
798 | grep ($_ eq $info->{fstype}, @posix_filesystems)) { |
799 | $task->{tmpdir} = "$opts->{dumpdir}/$basename.tmp"; | |
800 | } else { | |
dfb16115 | 801 | $task->{tmpdir} = "/var/tmp/vzdumptmp${pid}_$vmid"; |
aaeeeebe DM |
802 | debugmsg ('info', "filesystem type on dumpdir is '$info->{fstype}' -" . |
803 | "using $task->{tmpdir} for temporary files", $logfd); | |
804 | } | |
805 | } | |
806 | ||
807 | rmtree $task->{tmpdir}; | |
808 | mkdir $task->{tmpdir}; | |
809 | -d $task->{tmpdir} || | |
810 | die "unable to create temporary directory '$task->{tmpdir}'"; | |
811 | ||
812 | $logfd = IO::File->new (">$tmplog") || | |
813 | die "unable to create log file '$tmplog'"; | |
814 | ||
815 | $task->{dumpdir} = $opts->{dumpdir}; | |
ff00abe6 | 816 | $task->{storeid} = $opts->{storage}; |
1a87db9e | 817 | $task->{scfg} = $opts->{scfg}; |
aaeeeebe DM |
818 | $task->{tmplog} = $tmplog; |
819 | ||
1a87db9e | 820 | unlink $task->{logfile} if defined($task->{logfile}); |
aaeeeebe | 821 | |
29f26f6d DJ |
822 | debugmsg ('info', "Starting Backup of VM $vmid ($vmtype)", $logfd, 1); |
823 | debugmsg ('info', "Backup started at " . strftime("%F %H:%M:%S", localtime())); | |
aaeeeebe DM |
824 | |
825 | $plugin->set_logfd ($logfd); | |
826 | ||
827 | # test is VM is running | |
828 | my ($running, $status_text) = $plugin->vm_status ($vmid); | |
829 | ||
830 | debugmsg ('info', "status = ${status_text}", $logfd); | |
831 | ||
832 | # lock VM (prevent config changes) | |
833 | $plugin->lock_vm ($vmid); | |
834 | ||
835 | $cleanup->{unlock} = 1; | |
836 | ||
837 | # prepare | |
838 | ||
6acb632a | 839 | my $mode = $running ? $task->{mode} : 'stop'; |
aaeeeebe DM |
840 | |
841 | if ($mode eq 'snapshot') { | |
842 | my %saved_task = %$task; | |
843 | eval { $plugin->prepare ($task, $vmid, $mode); }; | |
844 | if (my $err = $@) { | |
845 | die $err if $err !~ m/^mode failure/; | |
846 | debugmsg ('info', $err, $logfd); | |
847 | debugmsg ('info', "trying 'suspend' mode instead", $logfd); | |
848 | $mode = 'suspend'; # so prepare is called again below | |
60e049c2 | 849 | %$task = %saved_task; |
aaeeeebe DM |
850 | } |
851 | } | |
852 | ||
6acb632a WB |
853 | $cleanup->{prepared} = 1; |
854 | ||
aaeeeebe DM |
855 | $task->{mode} = $mode; |
856 | ||
857 | debugmsg ('info', "backup mode: $mode", $logfd); | |
858 | ||
859 | debugmsg ('info', "bandwidth limit: $opts->{bwlimit} KB/s", $logfd) | |
860 | if $opts->{bwlimit}; | |
861 | ||
862 | debugmsg ('info', "ionice priority: $opts->{ionice}", $logfd); | |
863 | ||
864 | if ($mode eq 'stop') { | |
865 | ||
866 | $plugin->prepare ($task, $vmid, $mode); | |
867 | ||
868 | $self->run_hook_script ('backup-start', $task, $logfd); | |
869 | ||
870 | if ($running) { | |
871 | debugmsg ('info', "stopping vm", $logfd); | |
4e0947c8 | 872 | $task->{vmstoptime} = time(); |
aaeeeebe DM |
873 | $self->run_hook_script ('pre-stop', $task, $logfd); |
874 | $plugin->stop_vm ($task, $vmid); | |
875 | $cleanup->{restart} = 1; | |
876 | } | |
60e049c2 | 877 | |
aaeeeebe DM |
878 | |
879 | } elsif ($mode eq 'suspend') { | |
880 | ||
881 | $plugin->prepare ($task, $vmid, $mode); | |
882 | ||
883 | $self->run_hook_script ('backup-start', $task, $logfd); | |
884 | ||
8187f6b0 DM |
885 | if ($vmtype eq 'lxc') { |
886 | # pre-suspend rsync | |
887 | $plugin->copy_data_phase1($task, $vmid); | |
888 | } | |
889 | ||
ef7d3fdd | 890 | debugmsg ('info', "suspending guest", $logfd); |
4e0947c8 | 891 | $task->{vmstoptime} = time (); |
aaeeeebe DM |
892 | $self->run_hook_script ('pre-stop', $task, $logfd); |
893 | $plugin->suspend_vm ($task, $vmid); | |
894 | $cleanup->{resume} = 1; | |
895 | ||
8187f6b0 DM |
896 | if ($vmtype eq 'lxc') { |
897 | # post-suspend rsync | |
898 | $plugin->copy_data_phase2($task, $vmid); | |
899 | ||
ef7d3fdd | 900 | debugmsg ('info', "resuming guest", $logfd); |
8187f6b0 DM |
901 | $cleanup->{resume} = 0; |
902 | $self->run_hook_script('pre-restart', $task, $logfd); | |
903 | $plugin->resume_vm($task, $vmid); | |
d5047243 | 904 | $self->run_hook_script('post-restart', $task, $logfd); |
4e0947c8 | 905 | $log_vm_online_again->(); |
8187f6b0 | 906 | } |
60e049c2 | 907 | |
aaeeeebe DM |
908 | } elsif ($mode eq 'snapshot') { |
909 | ||
c5be0f8c DM |
910 | $self->run_hook_script ('backup-start', $task, $logfd); |
911 | ||
aaeeeebe DM |
912 | my $snapshot_count = $task->{snapshot_count} || 0; |
913 | ||
914 | $self->run_hook_script ('pre-stop', $task, $logfd); | |
915 | ||
916 | if ($snapshot_count > 1) { | |
917 | debugmsg ('info', "suspend vm to make snapshot", $logfd); | |
4e0947c8 | 918 | $task->{vmstoptime} = time (); |
aaeeeebe DM |
919 | $plugin->suspend_vm ($task, $vmid); |
920 | $cleanup->{resume} = 1; | |
921 | } | |
922 | ||
923 | $plugin->snapshot ($task, $vmid); | |
924 | ||
925 | $self->run_hook_script ('pre-restart', $task, $logfd); | |
926 | ||
927 | if ($snapshot_count > 1) { | |
928 | debugmsg ('info', "resume vm", $logfd); | |
929 | $cleanup->{resume} = 0; | |
930 | $plugin->resume_vm ($task, $vmid); | |
4e0947c8 | 931 | $log_vm_online_again->(); |
aaeeeebe DM |
932 | } |
933 | ||
d5047243 FG |
934 | $self->run_hook_script ('post-restart', $task, $logfd); |
935 | ||
aaeeeebe DM |
936 | } else { |
937 | die "internal error - unknown mode '$mode'\n"; | |
938 | } | |
939 | ||
940 | # assemble archive image | |
941 | $plugin->assemble ($task, $vmid); | |
60e049c2 TM |
942 | |
943 | # produce archive | |
aaeeeebe DM |
944 | |
945 | if ($opts->{stdout}) { | |
946 | debugmsg ('info', "sending archive to stdout", $logfd); | |
d7550e09 | 947 | $plugin->archive($task, $vmid, $task->{tmptar}, $comp); |
aaeeeebe DM |
948 | $self->run_hook_script ('backup-end', $task, $logfd); |
949 | return; | |
950 | } | |
951 | ||
b3c3304f TL |
952 | my $archive_txt = $self->{opts}->{pbs} ? 'Proxmox Backup Server' : 'vzdump'; |
953 | debugmsg('info', "creating $archive_txt archive '$task->{target}'", $logfd); | |
d7550e09 | 954 | $plugin->archive($task, $vmid, $task->{tmptar}, $comp); |
aaeeeebe | 955 | |
7ddcb3af | 956 | if ($self->{opts}->{pbs}) { |
e21a31a7 | 957 | # size is added to task struct in guest vzdump plugins |
1a87db9e | 958 | } else { |
6cba1855 TL |
959 | rename ($task->{tmptar}, $task->{target}) || |
960 | die "unable to rename '$task->{tmptar}' to '$task->{target}'\n"; | |
aaeeeebe | 961 | |
1a87db9e | 962 | # determine size |
6cba1855 | 963 | $task->{size} = (-s $task->{target}) || 0; |
1a87db9e DM |
964 | my $cs = format_size ($task->{size}); |
965 | debugmsg ('info', "archive file size: $cs", $logfd); | |
966 | } | |
aaeeeebe DM |
967 | |
968 | # purge older backup | |
e6946086 | 969 | if ($opts->{remove}) { |
f354fe47 FE |
970 | if (!defined($opts->{storage})) { |
971 | my $bklist = get_backup_file_list($opts->{dumpdir}, $bkname, $task->{target}); | |
972 | PVE::Storage::prune_mark_backup_group($bklist, $prune_options); | |
e6946086 | 973 | |
f354fe47 FE |
974 | foreach my $prune_entry (@{$bklist}) { |
975 | next if $prune_entry->{mark} ne 'remove'; | |
976 | ||
977 | my $archive_path = $prune_entry->{path}; | |
978 | debugmsg ('info', "delete old backup '$archive_path'", $logfd); | |
979 | PVE::Storage::archive_remove($archive_path); | |
1a87db9e | 980 | } |
f354fe47 | 981 | } else { |
e6946086 FE |
982 | my $logfunc = sub { debugmsg($_[0], $_[1], $logfd) }; |
983 | PVE::Storage::prune_backups($cfg, $opts->{storage}, $prune_options, $vmid, $vmtype, 0, $logfunc); | |
aaeeeebe DM |
984 | } |
985 | } | |
986 | ||
987 | $self->run_hook_script ('backup-end', $task, $logfd); | |
988 | }; | |
989 | my $err = $@; | |
990 | ||
991 | if ($plugin) { | |
992 | # clean-up | |
993 | ||
994 | if ($cleanup->{unlock}) { | |
995 | eval { $plugin->unlock_vm ($vmid); }; | |
996 | warn $@ if $@; | |
997 | } | |
998 | ||
6acb632a | 999 | if ($cleanup->{prepared}) { |
ca2605e7 DM |
1000 | # only call cleanup when necessary (when prepare was executed) |
1001 | eval { $plugin->cleanup ($task, $vmid) }; | |
1002 | warn $@ if $@; | |
1003 | } | |
aaeeeebe DM |
1004 | |
1005 | eval { $plugin->set_logfd (undef); }; | |
1006 | warn $@ if $@; | |
1007 | ||
60e049c2 TM |
1008 | if ($cleanup->{resume} || $cleanup->{restart}) { |
1009 | eval { | |
aaeeeebe DM |
1010 | $self->run_hook_script ('pre-restart', $task, $logfd); |
1011 | if ($cleanup->{resume}) { | |
1012 | debugmsg ('info', "resume vm", $logfd); | |
1013 | $plugin->resume_vm ($task, $vmid); | |
1014 | } else { | |
757fd3d5 DM |
1015 | my $running = $plugin->vm_status($vmid); |
1016 | if (!$running) { | |
1017 | debugmsg ('info', "restarting vm", $logfd); | |
1018 | $plugin->start_vm ($task, $vmid); | |
1019 | } | |
d5047243 FG |
1020 | } |
1021 | $self->run_hook_script ('post-restart', $task, $logfd); | |
aaeeeebe DM |
1022 | }; |
1023 | my $err = $@; | |
1024 | if ($err) { | |
1025 | warn $err; | |
1026 | } else { | |
4e0947c8 | 1027 | $log_vm_online_again->(); |
aaeeeebe DM |
1028 | } |
1029 | } | |
1030 | } | |
1031 | ||
1032 | eval { unlink $task->{tmptar} if $task->{tmptar} && -f $task->{tmptar}; }; | |
1033 | warn $@ if $@; | |
1034 | ||
fe6643b6 | 1035 | eval { rmtree $task->{tmpdir} if $task->{tmpdir} && -d $task->{tmpdir}; }; |
aaeeeebe DM |
1036 | warn $@ if $@; |
1037 | ||
1038 | my $delay = $task->{backuptime} = time () - $vmstarttime; | |
1039 | ||
1040 | if ($err) { | |
1041 | $task->{state} = 'err'; | |
1042 | $task->{msg} = $err; | |
1043 | debugmsg ('err', "Backup of VM $vmid failed - $err", $logfd, 1); | |
29f26f6d | 1044 | debugmsg ('info', "Failed at " . strftime("%F %H:%M:%S", localtime())); |
aaeeeebe DM |
1045 | |
1046 | eval { $self->run_hook_script ('backup-abort', $task, $logfd); }; | |
1047 | ||
1048 | } else { | |
1049 | $task->{state} = 'ok'; | |
1050 | my $tstr = format_time ($delay); | |
1051 | debugmsg ('info', "Finished Backup of VM $vmid ($tstr)", $logfd, 1); | |
29f26f6d | 1052 | debugmsg ('info', "Backup finished at " . strftime("%F %H:%M:%S", localtime())); |
aaeeeebe DM |
1053 | } |
1054 | ||
1055 | close ($logfd) if $logfd; | |
60e049c2 | 1056 | |
1a87db9e | 1057 | if ($task->{tmplog}) { |
7ddcb3af | 1058 | if ($self->{opts}->{pbs}) { |
1a87db9e DM |
1059 | if ($task->{state} eq 'ok') { |
1060 | my $param = [$pbs_snapshot_name, $task->{tmplog}]; | |
1061 | PVE::Storage::PBSPlugin::run_raw_client_cmd( | |
1062 | $opts->{scfg}, $opts->{storage}, 'upload-log', $param, errmsg => "upload log failed"); | |
1063 | } | |
1064 | } elsif ($task->{logfile}) { | |
1065 | system {'cp'} 'cp', $task->{tmplog}, $task->{logfile}; | |
1066 | } | |
aaeeeebe DM |
1067 | } |
1068 | ||
1069 | eval { $self->run_hook_script ('log-end', $task); }; | |
1070 | ||
1071 | die $err if $err && $err =~ m/^interrupted by signal$/; | |
1072 | } | |
1073 | ||
1074 | sub exec_backup { | |
d7550e09 | 1075 | my ($self, $rpcenv, $authuser) = @_; |
aaeeeebe DM |
1076 | |
1077 | my $opts = $self->{opts}; | |
1078 | ||
1079 | debugmsg ('info', "starting new backup job: $self->{cmdline}", undef, 1); | |
f647d0fc FE |
1080 | |
1081 | if (scalar(@{$self->{skiplist}})) { | |
1082 | my $skip_string = join(', ', sort { $a <=> $b } @{$self->{skiplist}}); | |
1083 | debugmsg ('info', "skip external VMs: $skip_string"); | |
1084 | } | |
60e049c2 | 1085 | |
aaeeeebe | 1086 | my $tasklist = []; |
df5875b4 AL |
1087 | my $vzdump_plugins = {}; |
1088 | foreach my $plugin (@{$self->{plugins}}) { | |
1089 | my $type = $plugin->type(); | |
1090 | next if exists $vzdump_plugins->{$type}; | |
1091 | $vzdump_plugins->{$type} = $plugin; | |
1092 | } | |
aaeeeebe | 1093 | |
df5875b4 | 1094 | my $vmlist = PVE::Cluster::get_vmlist(); |
4d73f0e2 FE |
1095 | my $vmids = [ sort { $a <=> $b } @{$opts->{vmids}} ]; |
1096 | foreach my $vmid (@{$vmids}) { | |
7f874148 FE |
1097 | my $plugin; |
1098 | if (defined($vmlist->{ids}->{$vmid})) { | |
1099 | my $guest_type = $vmlist->{ids}->{$vmid}->{type}; | |
1100 | $plugin = $vzdump_plugins->{$guest_type}; | |
1101 | next if !$rpcenv->check($authuser, "/vms/$vmid", [ 'VM.Backup' ], $opts->{all}); | |
1102 | } | |
df5875b4 | 1103 | push @$tasklist, { |
f839f3d1 | 1104 | mode => $opts->{mode}, |
df5875b4 | 1105 | plugin => $plugin, |
f839f3d1 TL |
1106 | state => 'todo', |
1107 | vmid => $vmid, | |
1108 | }; | |
aaeeeebe DM |
1109 | } |
1110 | ||
44b21574 WB |
1111 | # Use in-memory files for the outer hook logs to pass them to sendmail. |
1112 | my $job_start_log = ''; | |
1113 | my $job_end_log = ''; | |
1114 | open my $job_start_fd, '>', \$job_start_log; | |
1115 | open my $job_end_fd, '>', \$job_end_log; | |
1116 | ||
aaeeeebe DM |
1117 | my $starttime = time(); |
1118 | my $errcount = 0; | |
1119 | eval { | |
1120 | ||
44b21574 | 1121 | $self->run_hook_script ('job-start', undef, $job_start_fd); |
aaeeeebe DM |
1122 | |
1123 | foreach my $task (@$tasklist) { | |
1124 | $self->exec_backup_task ($task); | |
1125 | $errcount += 1 if $task->{state} ne 'ok'; | |
1126 | } | |
1127 | ||
44b21574 | 1128 | $self->run_hook_script ('job-end', undef, $job_end_fd); |
aaeeeebe DM |
1129 | }; |
1130 | my $err = $@; | |
1131 | ||
44b21574 | 1132 | $self->run_hook_script ('job-abort', undef, $job_end_fd) if $err; |
aaeeeebe DM |
1133 | |
1134 | if ($err) { | |
1135 | debugmsg ('err', "Backup job failed - $err", undef, 1); | |
1136 | } else { | |
1137 | if ($errcount) { | |
1138 | debugmsg ('info', "Backup job finished with errors", undef, 1); | |
1139 | } else { | |
61ca4432 | 1140 | debugmsg ('info', "Backup job finished successfully", undef, 1); |
aaeeeebe DM |
1141 | } |
1142 | } | |
1143 | ||
44b21574 WB |
1144 | close $job_start_fd; |
1145 | close $job_end_fd; | |
1146 | ||
aaeeeebe DM |
1147 | my $totaltime = time() - $starttime; |
1148 | ||
44b21574 | 1149 | eval { $self->sendmail ($tasklist, $totaltime, undef, $job_start_log, $job_end_log); }; |
aaeeeebe | 1150 | debugmsg ('err', $@) if $@; |
4a4051d8 DM |
1151 | |
1152 | die $err if $err; | |
1153 | ||
60e049c2 | 1154 | die "job errors\n" if $errcount; |
8682f6fc WL |
1155 | |
1156 | unlink $pidfile; | |
aaeeeebe DM |
1157 | } |
1158 | ||
ac27b58d | 1159 | |
47664cbe DM |
1160 | sub option_exists { |
1161 | my $key = shift; | |
1162 | return defined($confdesc->{$key}); | |
1163 | } | |
1164 | ||
31aef761 DM |
1165 | sub verify_vzdump_parameters { |
1166 | my ($param, $check_missing) = @_; | |
1167 | ||
1168 | raise_param_exc({ all => "option conflicts with option 'vmid'"}) | |
eab837c4 | 1169 | if $param->{all} && $param->{vmid}; |
31aef761 DM |
1170 | |
1171 | raise_param_exc({ exclude => "option conflicts with option 'vmid'"}) | |
1172 | if $param->{exclude} && $param->{vmid}; | |
1173 | ||
f3376261 TM |
1174 | raise_param_exc({ pool => "option conflicts with option 'vmid'"}) |
1175 | if $param->{pool} && $param->{vmid}; | |
1176 | ||
e6946086 FE |
1177 | raise_param_exc({ 'prune-backups' => "option conflicts with option 'maxfiles'"}) |
1178 | if defined($param->{'prune-backups'}) && defined($param->{maxfiles}); | |
1179 | ||
1180 | $param->{'prune-backups'} = PVE::JSONSchema::parse_property_string('prune-backups', $param->{'prune-backups'}) | |
1181 | if defined($param->{'prune-backups'}); | |
1182 | ||
f3376261 | 1183 | $param->{all} = 1 if (defined($param->{exclude}) && !$param->{pool}); |
31aef761 | 1184 | |
e2a2525e FG |
1185 | warn "option 'size' is deprecated and will be removed in a future " . |
1186 | "release, please update your script/configuration!\n" | |
1187 | if defined($param->{size}); | |
1188 | ||
31aef761 DM |
1189 | return if !$check_missing; |
1190 | ||
1191 | raise_param_exc({ vmid => "property is missing"}) | |
f3376261 | 1192 | if !($param->{all} || $param->{stop} || $param->{pool}) && !$param->{vmid}; |
8682f6fc WL |
1193 | |
1194 | } | |
1195 | ||
eab837c4 | 1196 | sub stop_running_backups { |
8682f6fc WL |
1197 | my($self) = @_; |
1198 | ||
eab837c4 DM |
1199 | my $upid = PVE::Tools::file_read_firstline($pidfile); |
1200 | return if !$upid; | |
8682f6fc | 1201 | |
eab837c4 | 1202 | my $task = PVE::Tools::upid_decode($upid); |
8682f6fc | 1203 | |
60e049c2 | 1204 | if (PVE::ProcFSTools::check_process_running($task->{pid}, $task->{pstart}) && |
eab837c4 DM |
1205 | PVE::ProcFSTools::read_proc_starttime($task->{pid}) == $task->{pstart}) { |
1206 | kill(15, $task->{pid}); | |
1207 | # wait max 15 seconds to shut down (else, do nothing for now) | |
1208 | my $i; | |
1209 | for ($i = 15; $i > 0; $i--) { | |
1210 | last if !PVE::ProcFSTools::check_process_running(($task->{pid}, $task->{pstart})); | |
1211 | sleep (1); | |
1212 | } | |
76189130 | 1213 | die "stopping backup process $task->{pid} failed\n" if $i == 0; |
8682f6fc | 1214 | } |
31aef761 DM |
1215 | } |
1216 | ||
5c4da4c3 AL |
1217 | sub get_included_guests { |
1218 | my ($job) = @_; | |
1219 | ||
5c4da4c3 | 1220 | my $vmids = []; |
df5875b4 AL |
1221 | my $vmids_per_node = {}; |
1222 | ||
1223 | my $vmlist = PVE::Cluster::get_vmlist(); | |
5c4da4c3 | 1224 | |
5c4da4c3 AL |
1225 | if ($job->{pool}) { |
1226 | $vmids = PVE::API2Tools::get_resource_pool_guest_members($job->{pool}); | |
df5875b4 | 1227 | } elsif ($job->{vmid}) { |
05447e04 | 1228 | $vmids = [ split_list($job->{vmid}) ]; |
da7cb515 | 1229 | } elsif ($job->{all}) { |
df5875b4 | 1230 | # all or exclude |
05447e04 TL |
1231 | my $exclude = check_vmids(split_list($job->{exclude})); |
1232 | my $excludehash = { map { $_ => 1 } @$exclude }; | |
df5875b4 | 1233 | |
05447e04 | 1234 | for my $id (keys %{$vmlist->{ids}}) { |
df5875b4 AL |
1235 | next if $excludehash->{$id}; |
1236 | push @$vmids, $id; | |
1237 | } | |
da7cb515 AL |
1238 | } else { |
1239 | return $vmids_per_node; | |
5c4da4c3 | 1240 | } |
50ba40ec | 1241 | $vmids = check_vmids(@$vmids); |
5c4da4c3 | 1242 | |
50ba40ec | 1243 | for my $vmid (@$vmids) { |
7f874148 FE |
1244 | if (defined($vmlist->{ids}->{$vmid})) { |
1245 | my $node = $vmlist->{ids}->{$vmid}->{node}; | |
1246 | next if (defined $job->{node} && $job->{node} ne $node); | |
50ba40ec | 1247 | |
7f874148 FE |
1248 | push @{$vmids_per_node->{$node}}, $vmid; |
1249 | } else { | |
1250 | push @{$vmids_per_node->{''}}, $vmid; | |
1251 | } | |
5c4da4c3 AL |
1252 | } |
1253 | ||
df5875b4 | 1254 | return $vmids_per_node; |
5c4da4c3 AL |
1255 | } |
1256 | ||
aaeeeebe | 1257 | 1; |