| | 1 | package PVE::VZDump; |
| | 2 | |
| | 3 | use strict; |
| | 4 | use warnings; |
| | 5 | |
| | 6 | use Fcntl ':flock'; |
| | 7 | use File::Path; |
| | 8 | use IO::File; |
| | 9 | use IO::Select; |
| | 10 | use IPC::Open3; |
| | 11 | use POSIX qw(strftime); |
| | 12 | use Time::Local; |
| | 13 | |
| | 14 | use PVE::Cluster qw(cfs_read_file); |
| | 15 | use PVE::DataCenterConfig; |
| | 16 | use PVE::Exception qw(raise_param_exc); |
| | 17 | use PVE::HA::Config; |
| | 18 | use PVE::HA::Env::PVE2; |
| | 19 | use PVE::JSONSchema qw(get_standard_option); |
| | 20 | use PVE::RPCEnvironment; |
| | 21 | use PVE::Storage; |
| | 22 | use PVE::VZDump::Common; |
| | 23 | use PVE::VZDump::Plugin; |
| | 24 | use PVE::Tools qw(extract_param split_list); |
| | 25 | use PVE::API2Tools; |
| | 26 | |
| | 27 | my @posix_filesystems = qw(ext3 ext4 nfs nfs4 reiserfs xfs); |
| | 28 | |
| | 29 | my $lockfile = '/var/run/vzdump.lock'; |
| | 30 | my $pidfile = '/var/run/vzdump.pid'; |
| | 31 | my $logdir = '/var/log/vzdump'; |
| | 32 | |
| | 33 | my @plugins = qw(); |
| | 34 | |
| | 35 | my $confdesc = PVE::VZDump::Common::get_confdesc(); |
| | 36 | |
| | 37 | # Load available plugins |
| | 38 | my @pve_vzdump_classes = qw(PVE::VZDump::QemuServer PVE::VZDump::LXC); |
| | 39 | foreach my $plug (@pve_vzdump_classes) { |
| | 40 | my $filename = "/usr/share/perl5/$plug.pm"; |
| | 41 | $filename =~ s!::!/!g; |
| | 42 | if (-f $filename) { |
| | 43 | eval { require $filename; }; |
| | 44 | if (!$@) { |
| | 45 | $plug->import (); |
| | 46 | push @plugins, $plug; |
| | 47 | } else { |
| | 48 | die $@; |
| | 49 | } |
| | 50 | } |
| | 51 | } |
| | 52 | |
| | 53 | # helper functions |
| | 54 | |
| | 55 | sub debugmsg { |
| | 56 | my ($mtype, $msg, $logfd, $syslog) = @_; |
| | 57 | |
| | 58 | PVE::VZDump::Plugin::debugmsg(@_); |
| | 59 | } |
| | 60 | |
| | 61 | sub run_command { |
| | 62 | my ($logfd, $cmdstr, %param) = @_; |
| | 63 | |
| | 64 | my $logfunc = sub { |
| | 65 | my $line = shift; |
| | 66 | debugmsg ('info', $line, $logfd); |
| | 67 | }; |
| | 68 | |
| | 69 | PVE::Tools::run_command($cmdstr, %param, logfunc => $logfunc); |
| | 70 | } |
| | 71 | |
| | 72 | my $parse_prune_backups_maxfiles = sub { |
| | 73 | my ($param, $kind) = @_; |
| | 74 | |
| | 75 | my $maxfiles = delete $param->{maxfiles}; |
| | 76 | my $prune_backups = $param->{'prune-backups'}; |
| | 77 | |
| | 78 | debugmsg('warn', "both 'maxfiles' and 'prune-backups' defined as ${kind} - ignoring 'maxfiles'") |
| | 79 | if defined($maxfiles) && defined($prune_backups); |
| | 80 | |
| | 81 | if (defined($prune_backups)) { |
| | 82 | return if ref($prune_backups) eq 'HASH'; # already parsed |
| | 83 | $param->{'prune-backups'} = PVE::JSONSchema::parse_property_string( |
| | 84 | 'prune-backups', |
| | 85 | $prune_backups |
| | 86 | ); |
| | 87 | } elsif (defined($maxfiles)) { |
| | 88 | if ($maxfiles) { |
| | 89 | $param->{'prune-backups'} = { 'keep-last' => $maxfiles }; |
| | 90 | } else { |
| | 91 | $param->{'prune-backups'} = { 'keep-all' => 1 }; |
| | 92 | } |
| | 93 | } |
| | 94 | }; |
| | 95 | |
| | 96 | sub storage_info { |
| | 97 | my $storage = shift; |
| | 98 | |
| | 99 | my $cfg = PVE::Storage::config(); |
| | 100 | my $scfg = PVE::Storage::storage_config($cfg, $storage); |
| | 101 | my $type = $scfg->{type}; |
| | 102 | |
| | 103 | die "can't use storage type '$type' for backup\n" |
| | 104 | if (!($type eq 'dir' || $type eq 'nfs' || $type eq 'glusterfs' |
| | 105 | || $type eq 'cifs' || $type eq 'cephfs' || $type eq 'pbs')); |
| | 106 | die "can't use storage '$storage' for backups - wrong content type\n" |
| | 107 | if (!$scfg->{content}->{backup}); |
| | 108 | |
| | 109 | PVE::Storage::activate_storage($cfg, $storage); |
| | 110 | |
| | 111 | my $info = { |
| | 112 | scfg => $scfg, |
| | 113 | }; |
| | 114 | |
| | 115 | $info->{'prune-backups'} = PVE::JSONSchema::parse_property_string('prune-backups', $scfg->{'prune-backups'}) |
| | 116 | if defined($scfg->{'prune-backups'}); |
| | 117 | |
| | 118 | if ($type eq 'pbs') { |
| | 119 | $info->{pbs} = 1; |
| | 120 | } else { |
| | 121 | $info->{dumpdir} = PVE::Storage::get_backup_dir($cfg, $storage); |
| | 122 | } |
| | 123 | |
| | 124 | return $info; |
| | 125 | } |
| | 126 | |
| | 127 | sub format_size { |
| | 128 | my $size = shift; |
| | 129 | |
| | 130 | my $kb = $size / 1024; |
| | 131 | |
| | 132 | if ($kb < 1024) { |
| | 133 | return int ($kb) . "KB"; |
| | 134 | } |
| | 135 | |
| | 136 | my $mb = $size / (1024*1024); |
| | 137 | if ($mb < 1024) { |
| | 138 | return int ($mb) . "MB"; |
| | 139 | } |
| | 140 | my $gb = $mb / 1024; |
| | 141 | if ($gb < 1024) { |
| | 142 | return sprintf ("%.2fGB", $gb); |
| | 143 | } |
| | 144 | my $tb = $gb / 1024; |
| | 145 | return sprintf ("%.2fTB", $tb); |
| | 146 | } |
| | 147 | |
| | 148 | sub format_time { |
| | 149 | my $seconds = shift; |
| | 150 | |
| | 151 | my $hours = int ($seconds/3600); |
| | 152 | $seconds = $seconds - $hours*3600; |
| | 153 | my $min = int ($seconds/60); |
| | 154 | $seconds = $seconds - $min*60; |
| | 155 | |
| | 156 | return sprintf ("%02d:%02d:%02d", $hours, $min, $seconds); |
| | 157 | } |
| | 158 | |
| | 159 | sub encode8bit { |
| | 160 | my ($str) = @_; |
| | 161 | |
| | 162 | $str =~ s/^(.{990})/$1\n/mg; # reduce line length |
| | 163 | |
| | 164 | return $str; |
| | 165 | } |
| | 166 | |
| | 167 | sub escape_html { |
| | 168 | my ($str) = @_; |
| | 169 | |
| | 170 | $str =~ s/&/&/g; |
| | 171 | $str =~ s/</</g; |
| | 172 | $str =~ s/>/>/g; |
| | 173 | |
| | 174 | return $str; |
| | 175 | } |
| | 176 | |
| | 177 | sub check_bin { |
| | 178 | my ($bin) = @_; |
| | 179 | |
| | 180 | foreach my $p (split (/:/, $ENV{PATH})) { |
| | 181 | my $fn = "$p/$bin"; |
| | 182 | if (-x $fn) { |
| | 183 | return $fn; |
| | 184 | } |
| | 185 | } |
| | 186 | |
| | 187 | die "unable to find command '$bin'\n"; |
| | 188 | } |
| | 189 | |
| | 190 | sub check_vmids { |
| | 191 | my (@vmids) = @_; |
| | 192 | |
| | 193 | my $res = []; |
| | 194 | for my $vmid (sort {$a <=> $b} @vmids) { |
| | 195 | die "ERROR: strange VM ID '${vmid}'\n" if $vmid !~ m/^\d+$/; |
| | 196 | $vmid = int ($vmid); # remove leading zeros |
| | 197 | next if !$vmid; |
| | 198 | push @$res, $vmid; |
| | 199 | } |
| | 200 | |
| | 201 | return $res; |
| | 202 | } |
| | 203 | |
| | 204 | |
| | 205 | sub read_vzdump_defaults { |
| | 206 | |
| | 207 | my $fn = "/etc/vzdump.conf"; |
| | 208 | |
| | 209 | my $defaults = { |
| | 210 | map { |
| | 211 | my $default = $confdesc->{$_}->{default}; |
| | 212 | defined($default) ? ($_ => $default) : () |
| | 213 | } keys %$confdesc |
| | 214 | }; |
| | 215 | $parse_prune_backups_maxfiles->($defaults, "defaults in VZDump schema"); |
| | 216 | |
| | 217 | my $raw; |
| | 218 | eval { $raw = PVE::Tools::file_get_contents($fn); }; |
| | 219 | return $defaults if $@; |
| | 220 | |
| | 221 | my $conf_schema = { type => 'object', properties => $confdesc, }; |
| | 222 | my $res = PVE::JSONSchema::parse_config($conf_schema, $fn, $raw); |
| | 223 | if (my $excludes = $res->{'exclude-path'}) { |
| | 224 | $res->{'exclude-path'} = PVE::Tools::split_args($excludes); |
| | 225 | } |
| | 226 | if (defined($res->{mailto})) { |
| | 227 | my @mailto = split_list($res->{mailto}); |
| | 228 | $res->{mailto} = [ @mailto ]; |
| | 229 | } |
| | 230 | $parse_prune_backups_maxfiles->($res, "options in '$fn'"); |
| | 231 | |
| | 232 | foreach my $key (keys %$defaults) { |
| | 233 | $res->{$key} = $defaults->{$key} if !defined($res->{$key}); |
| | 234 | } |
| | 235 | |
| | 236 | if (defined($res->{storage}) && defined($res->{dumpdir})) { |
| | 237 | debugmsg('warn', "both 'storage' and 'dumpdir' defined in '$fn' - ignoring 'dumpdir'"); |
| | 238 | delete $res->{dumpdir}; |
| | 239 | } |
| | 240 | |
| | 241 | return $res; |
| | 242 | } |
| | 243 | |
| | 244 | use constant MAX_MAIL_SIZE => 1024*1024; |
| | 245 | sub sendmail { |
| | 246 | my ($self, $tasklist, $totaltime, $err, $detail_pre, $detail_post) = @_; |
| | 247 | |
| | 248 | my $opts = $self->{opts}; |
| | 249 | |
| | 250 | my $mailto = $opts->{mailto}; |
| | 251 | |
| | 252 | return if !($mailto && scalar(@$mailto)); |
| | 253 | |
| | 254 | my $cmdline = $self->{cmdline}; |
| | 255 | |
| | 256 | my $ecount = 0; |
| | 257 | foreach my $task (@$tasklist) { |
| | 258 | $ecount++ if $task->{state} ne 'ok'; |
| | 259 | chomp $task->{msg} if $task->{msg}; |
| | 260 | $task->{backuptime} = 0 if !$task->{backuptime}; |
| | 261 | $task->{size} = 0 if !$task->{size}; |
| | 262 | $task->{target} = 'unknown' if !$task->{target}; |
| | 263 | $task->{hostname} = "VM $task->{vmid}" if !$task->{hostname}; |
| | 264 | |
| | 265 | if ($task->{state} eq 'todo') { |
| | 266 | $task->{msg} = 'aborted'; |
| | 267 | } |
| | 268 | } |
| | 269 | |
| | 270 | my $notify = $opts->{mailnotification} || 'always'; |
| | 271 | return if (!$ecount && !$err && ($notify eq 'failure')); |
| | 272 | |
| | 273 | my $stat = ($ecount || $err) ? 'backup failed' : 'backup successful'; |
| | 274 | if ($err) { |
| | 275 | if ($err =~ /\n/) { |
| | 276 | $stat .= ": multiple problems"; |
| | 277 | } else { |
| | 278 | $stat .= ": $err"; |
| | 279 | $err = undef; |
| | 280 | } |
| | 281 | } |
| | 282 | |
| | 283 | my $hostname = `hostname -f` || PVE::INotify::nodename(); |
| | 284 | chomp $hostname; |
| | 285 | |
| | 286 | # text part |
| | 287 | my $text = $err ? "$err\n\n" : ''; |
| | 288 | my $namelength = 20; |
| | 289 | $text .= sprintf ( |
| | 290 | "%-10s %-${namelength}s %-6s %10s %10s %s\n", |
| | 291 | qw(VMID NAME STATUS TIME SIZE FILENAME) |
| | 292 | ); |
| | 293 | foreach my $task (@$tasklist) { |
| | 294 | my $name = substr($task->{hostname}, 0, $namelength); |
| | 295 | my $successful = $task->{state} eq 'ok'; |
| | 296 | my $size = $successful ? format_size ($task->{size}) : 0; |
| | 297 | my $filename = $successful ? $task->{target} : '-'; |
| | 298 | my $size_fmt = $successful ? "%10s": "%8.2fMB"; |
| | 299 | $text .= sprintf( |
| | 300 | "%-10s %-${namelength}s %-6s %10s $size_fmt %s\n", |
| | 301 | $task->{vmid}, |
| | 302 | $name, |
| | 303 | $task->{state}, |
| | 304 | format_time($task->{backuptime}), |
| | 305 | $size, |
| | 306 | $filename, |
| | 307 | ); |
| | 308 | } |
| | 309 | |
| | 310 | my $text_log_part; |
| | 311 | $text_log_part .= "\nDetailed backup logs:\n\n"; |
| | 312 | $text_log_part .= "$cmdline\n\n"; |
| | 313 | |
| | 314 | $text_log_part .= $detail_pre . "\n" if defined($detail_pre); |
| | 315 | foreach my $task (@$tasklist) { |
| | 316 | my $vmid = $task->{vmid}; |
| | 317 | my $log = $task->{tmplog}; |
| | 318 | if (!$log) { |
| | 319 | $text_log_part .= "$vmid: no log available\n\n"; |
| | 320 | next; |
| | 321 | } |
| | 322 | if (open (my $TMP, '<', "$log")) { |
| | 323 | while (my $line = <$TMP>) { |
| | 324 | next if $line =~ /^status: \d+/; # not useful in mails |
| | 325 | $text_log_part .= encode8bit ("$vmid: $line"); |
| | 326 | } |
| | 327 | close ($TMP); |
| | 328 | } else { |
| | 329 | $text_log_part .= "$vmid: Could not open log file\n\n"; |
| | 330 | } |
| | 331 | $text_log_part .= "\n"; |
| | 332 | } |
| | 333 | $text_log_part .= $detail_post if defined($detail_post); |
| | 334 | |
| | 335 | # html part |
| | 336 | my $html = "<html><body>\n"; |
| | 337 | $html .= "<p>" . (escape_html($err) =~ s/\n/<br>/gr) . "</p>\n" if $err; |
| | 338 | $html .= "<table border=1 cellpadding=3>\n"; |
| | 339 | $html .= "<tr><td>VMID<td>NAME<td>STATUS<td>TIME<td>SIZE<td>FILENAME</tr>\n"; |
| | 340 | |
| | 341 | my $ssize = 0; |
| | 342 | foreach my $task (@$tasklist) { |
| | 343 | my $vmid = $task->{vmid}; |
| | 344 | my $name = $task->{hostname}; |
| | 345 | |
| | 346 | if ($task->{state} eq 'ok') { |
| | 347 | $ssize += $task->{size}; |
| | 348 | |
| | 349 | $html .= sprintf ( |
| | 350 | "<tr><td>%s<td>%s<td>OK<td>%s<td align=right>%s<td>%s</tr>\n", |
| | 351 | $vmid, |
| | 352 | $name, |
| | 353 | format_time($task->{backuptime}), |
| | 354 | format_size ($task->{size}), |
| | 355 | escape_html ($task->{target}), |
| | 356 | ); |
| | 357 | } else { |
| | 358 | $html .= sprintf ( |
| | 359 | "<tr><td>%s<td>%s<td><font color=red>FAILED<td>%s<td colspan=2>%s</tr>\n", |
| | 360 | $vmid, |
| | 361 | $name, |
| | 362 | format_time($task->{backuptime}), |
| | 363 | escape_html ($task->{msg}), |
| | 364 | ); |
| | 365 | } |
| | 366 | } |
| | 367 | |
| | 368 | $html .= sprintf ("<tr><td align=left colspan=3>TOTAL<td>%s<td>%s<td></tr>", |
| | 369 | format_time ($totaltime), format_size ($ssize)); |
| | 370 | |
| | 371 | $html .= "\n</table><br><br>\n"; |
| | 372 | my $html_log_part; |
| | 373 | $html_log_part .= "Detailed backup logs:<br /><br />\n"; |
| | 374 | $html_log_part .= "<pre>\n"; |
| | 375 | $html_log_part .= escape_html($cmdline) . "\n\n"; |
| | 376 | |
| | 377 | $html_log_part .= escape_html($detail_pre) . "\n" if defined($detail_pre); |
| | 378 | foreach my $task (@$tasklist) { |
| | 379 | my $vmid = $task->{vmid}; |
| | 380 | my $log = $task->{tmplog}; |
| | 381 | if (!$log) { |
| | 382 | $html_log_part .= "$vmid: no log available\n\n"; |
| | 383 | next; |
| | 384 | } |
| | 385 | if (open (my $TMP, '<', "$log")) { |
| | 386 | while (my $line = <$TMP>) { |
| | 387 | next if $line =~ /^status: \d+/; # not useful in mails |
| | 388 | if ($line =~ m/^\S+\s\d+\s+\d+:\d+:\d+\s+(ERROR|WARN):/) { |
| | 389 | $html_log_part .= encode8bit ("$vmid: <font color=red>". |
| | 390 | escape_html ($line) . "</font>"); |
| | 391 | } else { |
| | 392 | $html_log_part .= encode8bit ("$vmid: " . escape_html ($line)); |
| | 393 | } |
| | 394 | } |
| | 395 | close ($TMP); |
| | 396 | } else { |
| | 397 | $html_log_part .= "$vmid: Could not open log file\n\n"; |
| | 398 | } |
| | 399 | $html_log_part .= "\n"; |
| | 400 | } |
| | 401 | $html_log_part .= escape_html($detail_post) if defined($detail_post); |
| | 402 | $html_log_part .= "</pre>"; |
| | 403 | my $html_end = "\n</body></html>\n"; |
| | 404 | # end html part |
| | 405 | |
| | 406 | if (length($text) + length($text_log_part) + |
| | 407 | length($html) + length($html_log_part) + |
| | 408 | length($html_end) < MAX_MAIL_SIZE) |
| | 409 | { |
| | 410 | $html .= $html_log_part; |
| | 411 | $html .= $html_end; |
| | 412 | $text .= $text_log_part; |
| | 413 | } else { |
| | 414 | my $msg = "Log output was too long to be sent by mail. ". |
| | 415 | "See Task History for details!\n"; |
| | 416 | $text .= $msg; |
| | 417 | $html .= "<p>$msg</p>"; |
| | 418 | $html .= $html_end; |
| | 419 | } |
| | 420 | |
| | 421 | my $subject = "vzdump backup status ($hostname) : $stat"; |
| | 422 | |
| | 423 | my $dcconf = PVE::Cluster::cfs_read_file('datacenter.cfg'); |
| | 424 | my $mailfrom = $dcconf->{email_from} || "root"; |
| | 425 | |
| | 426 | PVE::Tools::sendmail($mailto, $subject, $text, $html, $mailfrom, "vzdump backup tool"); |
| | 427 | }; |
| | 428 | |
| | 429 | sub new { |
| | 430 | my ($class, $cmdline, $opts, $skiplist) = @_; |
| | 431 | |
| | 432 | mkpath $logdir; |
| | 433 | |
| | 434 | check_bin ('cp'); |
| | 435 | check_bin ('df'); |
| | 436 | check_bin ('sendmail'); |
| | 437 | check_bin ('rsync'); |
| | 438 | check_bin ('tar'); |
| | 439 | check_bin ('mount'); |
| | 440 | check_bin ('umount'); |
| | 441 | check_bin ('cstream'); |
| | 442 | check_bin ('ionice'); |
| | 443 | |
| | 444 | if ($opts->{mode} && $opts->{mode} eq 'snapshot') { |
| | 445 | check_bin ('lvcreate'); |
| | 446 | check_bin ('lvs'); |
| | 447 | check_bin ('lvremove'); |
| | 448 | } |
| | 449 | |
| | 450 | my $defaults = read_vzdump_defaults(); |
| | 451 | |
| | 452 | foreach my $k (keys %$defaults) { |
| | 453 | next if $k eq 'exclude-path' || $k eq 'prune-backups'; # dealt with separately |
| | 454 | if ($k eq 'dumpdir' || $k eq 'storage') { |
| | 455 | $opts->{$k} = $defaults->{$k} if !defined ($opts->{dumpdir}) && |
| | 456 | !defined ($opts->{storage}); |
| | 457 | } else { |
| | 458 | $opts->{$k} = $defaults->{$k} if !defined ($opts->{$k}); |
| | 459 | } |
| | 460 | } |
| | 461 | |
| | 462 | $opts->{dumpdir} =~ s|/+$|| if ($opts->{dumpdir}); |
| | 463 | $opts->{tmpdir} =~ s|/+$|| if ($opts->{tmpdir}); |
| | 464 | |
| | 465 | $skiplist = [] if !$skiplist; |
| | 466 | my $self = bless { |
| | 467 | cmdline => $cmdline, |
| | 468 | opts => $opts, |
| | 469 | skiplist => $skiplist, |
| | 470 | }, $class; |
| | 471 | |
| | 472 | my $findexcl = $self->{findexcl} = []; |
| | 473 | if ($defaults->{'exclude-path'}) { |
| | 474 | push @$findexcl, @{$defaults->{'exclude-path'}}; |
| | 475 | } |
| | 476 | |
| | 477 | if ($opts->{'exclude-path'}) { |
| | 478 | push @$findexcl, @{$opts->{'exclude-path'}}; |
| | 479 | } |
| | 480 | |
| | 481 | if ($opts->{stdexcludes}) { |
| | 482 | push @$findexcl, |
| | 483 | '/tmp/?*', |
| | 484 | '/var/tmp/?*', |
| | 485 | '/var/run/?*.pid', |
| | 486 | ; |
| | 487 | } |
| | 488 | |
| | 489 | foreach my $p (@plugins) { |
| | 490 | my $pd = $p->new($self); |
| | 491 | |
| | 492 | push @{$self->{plugins}}, $pd; |
| | 493 | } |
| | 494 | |
| | 495 | if (defined($opts->{storage}) && $opts->{stdout}) { |
| | 496 | die "cannot use options 'storage' and 'stdout' at the same time\n"; |
| | 497 | } elsif (defined($opts->{storage}) && defined($opts->{dumpdir})) { |
| | 498 | die "cannot use options 'storage' and 'dumpdir' at the same time\n"; |
| | 499 | } |
| | 500 | |
| | 501 | if (!$opts->{dumpdir} && !$opts->{storage}) { |
| | 502 | $opts->{storage} = 'local'; |
| | 503 | } |
| | 504 | |
| | 505 | my $errors = ''; |
| | 506 | |
| | 507 | if ($opts->{storage}) { |
| | 508 | my $info = eval { storage_info ($opts->{storage}) }; |
| | 509 | if (my $err = $@) { |
| | 510 | chomp($err); |
| | 511 | $errors .= "could not get storage information for '$opts->{storage}': $err"; |
| | 512 | } else { |
| | 513 | $opts->{dumpdir} = $info->{dumpdir}; |
| | 514 | $opts->{scfg} = $info->{scfg}; |
| | 515 | $opts->{pbs} = $info->{pbs}; |
| | 516 | $opts->{'prune-backups'} //= $info->{'prune-backups'}; |
| | 517 | } |
| | 518 | } elsif ($opts->{dumpdir}) { |
| | 519 | $errors .= "dumpdir '$opts->{dumpdir}' does not exist" |
| | 520 | if ! -d $opts->{dumpdir}; |
| | 521 | } else { |
| | 522 | die "internal error"; |
| | 523 | } |
| | 524 | |
| | 525 | $opts->{'prune-backups'} //= $defaults->{'prune-backups'}; |
| | 526 | |
| | 527 | # avoid triggering any remove code path if keep-all is set |
| | 528 | $opts->{remove} = 0 if $opts->{'prune-backups'}->{'keep-all'}; |
| | 529 | |
| | 530 | if ($opts->{tmpdir} && ! -d $opts->{tmpdir}) { |
| | 531 | $errors .= "\n" if $errors; |
| | 532 | $errors .= "tmpdir '$opts->{tmpdir}' does not exist"; |
| | 533 | } |
| | 534 | |
| | 535 | if ($errors) { |
| | 536 | eval { $self->sendmail([], 0, $errors); }; |
| | 537 | debugmsg ('err', $@) if $@; |
| | 538 | die "$errors\n"; |
| | 539 | } |
| | 540 | |
| | 541 | return $self; |
| | 542 | } |
| | 543 | |
| | 544 | sub get_mount_info { |
| | 545 | my ($dir) = @_; |
| | 546 | |
| | 547 | # Note: df 'available' can be negative, and percentage set to '-' |
| | 548 | |
| | 549 | my $cmd = [ 'df', '-P', '-T', '-B', '1', $dir]; |
| | 550 | |
| | 551 | my $res; |
| | 552 | |
| | 553 | my $parser = sub { |
| | 554 | my $line = shift; |
| | 555 | if (my ($fsid, $fstype, undef, $mp) = $line =~ |
| | 556 | m!(\S+.*)\s+(\S+)\s+\d+\s+\-?\d+\s+\d+\s+(\d+%|-)\s+(/.*)$!) { |
| | 557 | $res = { |
| | 558 | device => $fsid, |
| | 559 | fstype => $fstype, |
| | 560 | mountpoint => $mp, |
| | 561 | }; |
| | 562 | } |
| | 563 | }; |
| | 564 | |
| | 565 | eval { PVE::Tools::run_command($cmd, errfunc => sub {}, outfunc => $parser); }; |
| | 566 | warn $@ if $@; |
| | 567 | |
| | 568 | return $res; |
| | 569 | } |
| | 570 | |
| | 571 | sub getlock { |
| | 572 | my ($self, $upid) = @_; |
| | 573 | |
| | 574 | my $fh; |
| | 575 | |
| | 576 | my $maxwait = $self->{opts}->{lockwait} || $self->{lockwait}; |
| | 577 | |
| | 578 | die "missing UPID" if !$upid; # should not happen |
| | 579 | |
| | 580 | my $SERVER_FLCK; |
| | 581 | if (!open ($SERVER_FLCK, '>>', "$lockfile")) { |
| | 582 | debugmsg ('err', "can't open lock on file '$lockfile' - $!", undef, 1); |
| | 583 | die "can't open lock on file '$lockfile' - $!"; |
| | 584 | } |
| | 585 | |
| | 586 | if (!flock ($SERVER_FLCK, LOCK_EX|LOCK_NB)) { |
| | 587 | if (!$maxwait) { |
| | 588 | debugmsg ('err', "can't acquire lock '$lockfile' (wait = 0)", undef, 1); |
| | 589 | die "can't acquire lock '$lockfile' (wait = 0)"; |
| | 590 | } |
| | 591 | |
| | 592 | debugmsg('info', "trying to get global lock - waiting...", undef, 1); |
| | 593 | eval { |
| | 594 | alarm ($maxwait * 60); |
| | 595 | |
| | 596 | local $SIG{ALRM} = sub { alarm (0); die "got timeout\n"; }; |
| | 597 | |
| | 598 | if (!flock ($SERVER_FLCK, LOCK_EX)) { |
| | 599 | my $err = $!; |
| | 600 | close ($SERVER_FLCK); |
| | 601 | alarm (0); |
| | 602 | die "$err\n"; |
| | 603 | } |
| | 604 | alarm (0); |
| | 605 | }; |
| | 606 | alarm (0); |
| | 607 | |
| | 608 | my $err = $@; |
| | 609 | |
| | 610 | if ($err) { |
| | 611 | debugmsg ('err', "can't acquire lock '$lockfile' - $err", undef, 1); |
| | 612 | die "can't acquire lock '$lockfile' - $err"; |
| | 613 | } |
| | 614 | |
| | 615 | debugmsg('info', "got global lock", undef, 1); |
| | 616 | } |
| | 617 | |
| | 618 | PVE::Tools::file_set_contents($pidfile, $upid); |
| | 619 | } |
| | 620 | |
| | 621 | sub run_hook_script { |
| | 622 | my ($self, $phase, $task, $logfd) = @_; |
| | 623 | |
| | 624 | my $opts = $self->{opts}; |
| | 625 | |
| | 626 | my $script = $opts->{script}; |
| | 627 | return if !$script; |
| | 628 | |
| | 629 | if (!-x $script) { |
| | 630 | die "The hook script '$script' is not executable.\n"; |
| | 631 | } |
| | 632 | |
| | 633 | my $cmd = "$script $phase"; |
| | 634 | |
| | 635 | $cmd .= " $task->{mode} $task->{vmid}" if ($task); |
| | 636 | |
| | 637 | local %ENV; |
| | 638 | # set immutable opts directly (so they are available in all phases) |
| | 639 | $ENV{STOREID} = $opts->{storage} if $opts->{storage}; |
| | 640 | $ENV{DUMPDIR} = $opts->{dumpdir} if $opts->{dumpdir}; |
| | 641 | |
| | 642 | foreach my $ek (qw(vmtype hostname target logfile)) { |
| | 643 | $ENV{uc($ek)} = $task->{$ek} if $task->{$ek}; |
| | 644 | } |
| | 645 | # FIXME: for backwards compatibility - drop with PVE 7.0 |
| | 646 | $ENV{TARFILE} = $task->{target} if $task->{target}; |
| | 647 | |
| | 648 | run_command ($logfd, $cmd); |
| | 649 | } |
| | 650 | |
| | 651 | sub compressor_info { |
| | 652 | my ($opts) = @_; |
| | 653 | my $opt_compress = $opts->{compress}; |
| | 654 | |
| | 655 | if (!$opt_compress || $opt_compress eq '0') { |
| | 656 | return undef; |
| | 657 | } elsif ($opt_compress eq '1' || $opt_compress eq 'lzo') { |
| | 658 | return ('lzop', 'lzo'); |
| | 659 | } elsif ($opt_compress eq 'gzip') { |
| | 660 | if ($opts->{pigz} > 0) { |
| | 661 | my $pigz_threads = $opts->{pigz}; |
| | 662 | if ($pigz_threads == 1) { |
| | 663 | my $cpuinfo = PVE::ProcFSTools::read_cpuinfo(); |
| | 664 | $pigz_threads = int(($cpuinfo->{cpus} + 1)/2); |
| | 665 | } |
| | 666 | return ("pigz -p ${pigz_threads} --rsyncable", 'gz'); |
| | 667 | } else { |
| | 668 | return ('gzip --rsyncable', 'gz'); |
| | 669 | } |
| | 670 | } elsif ($opt_compress eq 'zstd') { |
| | 671 | my $zstd_threads = $opts->{zstd} // 1; |
| | 672 | if ($zstd_threads == 0) { |
| | 673 | my $cpuinfo = PVE::ProcFSTools::read_cpuinfo(); |
| | 674 | $zstd_threads = int(($cpuinfo->{cpus} + 1)/2); |
| | 675 | } |
| | 676 | return ("zstd --rsyncable --threads=${zstd_threads}", 'zst'); |
| | 677 | } else { |
| | 678 | die "internal error - unknown compression option '$opt_compress'"; |
| | 679 | } |
| | 680 | } |
| | 681 | |
| | 682 | sub get_backup_file_list { |
| | 683 | my ($dir, $bkname, $exclude_fn) = @_; |
| | 684 | |
| | 685 | my $bklist = []; |
| | 686 | foreach my $fn (<$dir/${bkname}-*>) { |
| | 687 | next if $exclude_fn && $fn eq $exclude_fn; |
| | 688 | |
| | 689 | my $archive_info = eval { PVE::Storage::archive_info($fn) } // {}; |
| | 690 | if ($archive_info->{is_std_name}) { |
| | 691 | my $filename = $archive_info->{filename}; |
| | 692 | my $backup = { |
| | 693 | 'path' => "$dir/$filename", |
| | 694 | 'ctime' => $archive_info->{ctime}, |
| | 695 | }; |
| | 696 | push @{$bklist}, $backup; |
| | 697 | } |
| | 698 | } |
| | 699 | |
| | 700 | return $bklist; |
| | 701 | } |
| | 702 | |
| | 703 | sub exec_backup_task { |
| | 704 | my ($self, $task) = @_; |
| | 705 | |
| | 706 | my $opts = $self->{opts}; |
| | 707 | |
| | 708 | my $cfg = PVE::Storage::config(); |
| | 709 | my $vmid = $task->{vmid}; |
| | 710 | my $plugin = $task->{plugin}; |
| | 711 | |
| | 712 | $task->{backup_time} = time(); |
| | 713 | |
| | 714 | my $pbs_group_name; |
| | 715 | my $pbs_snapshot_name; |
| | 716 | |
| | 717 | my $vmstarttime = time (); |
| | 718 | |
| | 719 | my $logfd; |
| | 720 | |
| | 721 | my $cleanup = {}; |
| | 722 | |
| | 723 | my $log_vm_online_again = sub { |
| | 724 | return if !defined($task->{vmstoptime}); |
| | 725 | $task->{vmconttime} //= time(); |
| | 726 | my $delay = $task->{vmconttime} - $task->{vmstoptime}; |
| | 727 | $delay = '<1' if $delay < 1; |
| | 728 | debugmsg ('info', "guest is online again after $delay seconds", $logfd); |
| | 729 | }; |
| | 730 | |
| | 731 | eval { |
| | 732 | die "unable to find VM '$vmid'\n" if !$plugin; |
| | 733 | |
| | 734 | my $vmtype = $plugin->type(); |
| | 735 | |
| | 736 | if ($self->{opts}->{pbs}) { |
| | 737 | if ($vmtype eq 'lxc') { |
| | 738 | $pbs_group_name = "ct/$vmid"; |
| | 739 | } elsif ($vmtype eq 'qemu') { |
| | 740 | $pbs_group_name = "vm/$vmid"; |
| | 741 | } else { |
| | 742 | die "pbs backup not implemented for plugin type '$vmtype'\n"; |
| | 743 | } |
| | 744 | my $btime = strftime("%FT%TZ", gmtime($task->{backup_time})); |
| | 745 | $pbs_snapshot_name = "$pbs_group_name/$btime"; |
| | 746 | } |
| | 747 | |
| | 748 | # for now we deny backups of a running ha managed service in *stop* mode |
| | 749 | # as it interferes with the HA stack (started services should not stop). |
| | 750 | if ($opts->{mode} eq 'stop' && |
| | 751 | PVE::HA::Config::vm_is_ha_managed($vmid, 'started')) |
| | 752 | { |
| | 753 | die "Cannot execute a backup with stop mode on a HA managed and". |
| | 754 | " enabled Service. Use snapshot mode or disable the Service.\n"; |
| | 755 | } |
| | 756 | |
| | 757 | my $tmplog = "$logdir/$vmtype-$vmid.log"; |
| | 758 | |
| | 759 | my $bkname = "vzdump-$vmtype-$vmid"; |
| | 760 | my $basename = $bkname . strftime("-%Y_%m_%d-%H_%M_%S", localtime($task->{backup_time})); |
| | 761 | |
| | 762 | my $prune_options = $opts->{'prune-backups'}; |
| | 763 | |
| | 764 | my $backup_limit = 0; |
| | 765 | if (!$prune_options->{'keep-all'}) { |
| | 766 | foreach my $keep (values %{$prune_options}) { |
| | 767 | $backup_limit += $keep; |
| | 768 | } |
| | 769 | } |
| | 770 | |
| | 771 | if ($backup_limit && !$opts->{remove}) { |
| | 772 | my $count; |
| | 773 | if ($self->{opts}->{pbs}) { |
| | 774 | my $res = PVE::Storage::PBSPlugin::run_client_cmd($opts->{scfg}, $opts->{storage}, 'snapshots', $pbs_group_name); |
| | 775 | $count = scalar(@$res); |
| | 776 | } else { |
| | 777 | my $bklist = get_backup_file_list($opts->{dumpdir}, $bkname); |
| | 778 | $count = scalar(@$bklist); |
| | 779 | } |
| | 780 | die "There is a max backup limit of $backup_limit enforced by the". |
| | 781 | " target storage or the vzdump parameters.". |
| | 782 | " Either increase the limit or delete old backup(s).\n" |
| | 783 | if $count >= $backup_limit; |
| | 784 | } |
| | 785 | |
| | 786 | if (!$self->{opts}->{pbs}) { |
| | 787 | $task->{logfile} = "$opts->{dumpdir}/$basename.log"; |
| | 788 | } |
| | 789 | |
| | 790 | my $ext = $vmtype eq 'qemu' ? '.vma' : '.tar'; |
| | 791 | my ($comp, $comp_ext) = compressor_info($opts); |
| | 792 | if ($comp && $comp_ext) { |
| | 793 | $ext .= ".${comp_ext}"; |
| | 794 | } |
| | 795 | |
| | 796 | if ($self->{opts}->{pbs}) { |
| | 797 | die "unable to pipe backup to stdout\n" if $opts->{stdout}; |
| | 798 | $task->{target} = $pbs_snapshot_name; |
| | 799 | } else { |
| | 800 | if ($opts->{stdout}) { |
| | 801 | $task->{target} = '-'; |
| | 802 | } else { |
| | 803 | $task->{target} = $task->{tmptar} = "$opts->{dumpdir}/$basename$ext"; |
| | 804 | $task->{tmptar} =~ s/\.[^\.]+$/\.dat/; |
| | 805 | unlink $task->{tmptar}; |
| | 806 | } |
| | 807 | } |
| | 808 | |
| | 809 | $task->{vmtype} = $vmtype; |
| | 810 | |
| | 811 | my $pid = $$; |
| | 812 | if ($opts->{tmpdir}) { |
| | 813 | $task->{tmpdir} = "$opts->{tmpdir}/vzdumptmp${pid}_$vmid/"; |
| | 814 | } elsif ($self->{opts}->{pbs}) { |
| | 815 | $task->{tmpdir} = "/var/tmp/vzdumptmp${pid}_$vmid"; |
| | 816 | } else { |
| | 817 | # dumpdir is posix? then use it as temporary dir |
| | 818 | my $info = get_mount_info($opts->{dumpdir}); |
| | 819 | if ($vmtype eq 'qemu' || |
| | 820 | grep ($_ eq $info->{fstype}, @posix_filesystems)) { |
| | 821 | $task->{tmpdir} = "$opts->{dumpdir}/$basename.tmp"; |
| | 822 | } else { |
| | 823 | $task->{tmpdir} = "/var/tmp/vzdumptmp${pid}_$vmid"; |
| | 824 | debugmsg ('info', "filesystem type on dumpdir is '$info->{fstype}' -" . |
| | 825 | "using $task->{tmpdir} for temporary files", $logfd); |
| | 826 | } |
| | 827 | } |
| | 828 | |
| | 829 | rmtree $task->{tmpdir}; |
| | 830 | mkdir $task->{tmpdir}; |
| | 831 | -d $task->{tmpdir} || |
| | 832 | die "unable to create temporary directory '$task->{tmpdir}'"; |
| | 833 | |
| | 834 | $logfd = IO::File->new (">$tmplog") || |
| | 835 | die "unable to create log file '$tmplog'"; |
| | 836 | |
| | 837 | $task->{dumpdir} = $opts->{dumpdir}; |
| | 838 | $task->{storeid} = $opts->{storage}; |
| | 839 | $task->{scfg} = $opts->{scfg}; |
| | 840 | $task->{tmplog} = $tmplog; |
| | 841 | |
| | 842 | unlink $task->{logfile} if defined($task->{logfile}); |
| | 843 | |
| | 844 | debugmsg ('info', "Starting Backup of VM $vmid ($vmtype)", $logfd, 1); |
| | 845 | debugmsg ('info', "Backup started at " . strftime("%F %H:%M:%S", localtime())); |
| | 846 | |
| | 847 | $plugin->set_logfd ($logfd); |
| | 848 | |
| | 849 | # test is VM is running |
| | 850 | my ($running, $status_text) = $plugin->vm_status ($vmid); |
| | 851 | |
| | 852 | debugmsg ('info', "status = ${status_text}", $logfd); |
| | 853 | |
| | 854 | # lock VM (prevent config changes) |
| | 855 | $plugin->lock_vm ($vmid); |
| | 856 | |
| | 857 | $cleanup->{unlock} = 1; |
| | 858 | |
| | 859 | # prepare |
| | 860 | |
| | 861 | my $mode = $running ? $task->{mode} : 'stop'; |
| | 862 | |
| | 863 | if ($mode eq 'snapshot') { |
| | 864 | my %saved_task = %$task; |
| | 865 | eval { $plugin->prepare ($task, $vmid, $mode); }; |
| | 866 | if (my $err = $@) { |
| | 867 | die $err if $err !~ m/^mode failure/; |
| | 868 | debugmsg ('info', $err, $logfd); |
| | 869 | debugmsg ('info', "trying 'suspend' mode instead", $logfd); |
| | 870 | $mode = 'suspend'; # so prepare is called again below |
| | 871 | %$task = %saved_task; |
| | 872 | } |
| | 873 | } |
| | 874 | |
| | 875 | $cleanup->{prepared} = 1; |
| | 876 | |
| | 877 | $task->{mode} = $mode; |
| | 878 | |
| | 879 | debugmsg ('info', "backup mode: $mode", $logfd); |
| | 880 | |
| | 881 | debugmsg ('info', "bandwidth limit: $opts->{bwlimit} KB/s", $logfd) |
| | 882 | if $opts->{bwlimit}; |
| | 883 | |
| | 884 | debugmsg ('info', "ionice priority: $opts->{ionice}", $logfd); |
| | 885 | |
| | 886 | if ($mode eq 'stop') { |
| | 887 | |
| | 888 | $plugin->prepare ($task, $vmid, $mode); |
| | 889 | |
| | 890 | $self->run_hook_script ('backup-start', $task, $logfd); |
| | 891 | |
| | 892 | if ($running) { |
| | 893 | debugmsg ('info', "stopping vm", $logfd); |
| | 894 | $task->{vmstoptime} = time(); |
| | 895 | $self->run_hook_script ('pre-stop', $task, $logfd); |
| | 896 | $plugin->stop_vm ($task, $vmid); |
| | 897 | $cleanup->{restart} = 1; |
| | 898 | } |
| | 899 | |
| | 900 | |
| | 901 | } elsif ($mode eq 'suspend') { |
| | 902 | |
| | 903 | $plugin->prepare ($task, $vmid, $mode); |
| | 904 | |
| | 905 | $self->run_hook_script ('backup-start', $task, $logfd); |
| | 906 | |
| | 907 | if ($vmtype eq 'lxc') { |
| | 908 | # pre-suspend rsync |
| | 909 | $plugin->copy_data_phase1($task, $vmid); |
| | 910 | } |
| | 911 | |
| | 912 | debugmsg ('info', "suspending guest", $logfd); |
| | 913 | $task->{vmstoptime} = time (); |
| | 914 | $self->run_hook_script ('pre-stop', $task, $logfd); |
| | 915 | $plugin->suspend_vm ($task, $vmid); |
| | 916 | $cleanup->{resume} = 1; |
| | 917 | |
| | 918 | if ($vmtype eq 'lxc') { |
| | 919 | # post-suspend rsync |
| | 920 | $plugin->copy_data_phase2($task, $vmid); |
| | 921 | |
| | 922 | debugmsg ('info', "resuming guest", $logfd); |
| | 923 | $cleanup->{resume} = 0; |
| | 924 | $self->run_hook_script('pre-restart', $task, $logfd); |
| | 925 | $plugin->resume_vm($task, $vmid); |
| | 926 | $self->run_hook_script('post-restart', $task, $logfd); |
| | 927 | $log_vm_online_again->(); |
| | 928 | } |
| | 929 | |
| | 930 | } elsif ($mode eq 'snapshot') { |
| | 931 | |
| | 932 | $self->run_hook_script ('backup-start', $task, $logfd); |
| | 933 | |
| | 934 | my $snapshot_count = $task->{snapshot_count} || 0; |
| | 935 | |
| | 936 | $self->run_hook_script ('pre-stop', $task, $logfd); |
| | 937 | |
| | 938 | if ($snapshot_count > 1) { |
| | 939 | debugmsg ('info', "suspend vm to make snapshot", $logfd); |
| | 940 | $task->{vmstoptime} = time (); |
| | 941 | $plugin->suspend_vm ($task, $vmid); |
| | 942 | $cleanup->{resume} = 1; |
| | 943 | } |
| | 944 | |
| | 945 | $plugin->snapshot ($task, $vmid); |
| | 946 | |
| | 947 | $self->run_hook_script ('pre-restart', $task, $logfd); |
| | 948 | |
| | 949 | if ($snapshot_count > 1) { |
| | 950 | debugmsg ('info', "resume vm", $logfd); |
| | 951 | $cleanup->{resume} = 0; |
| | 952 | $plugin->resume_vm ($task, $vmid); |
| | 953 | $log_vm_online_again->(); |
| | 954 | } |
| | 955 | |
| | 956 | $self->run_hook_script ('post-restart', $task, $logfd); |
| | 957 | |
| | 958 | } else { |
| | 959 | die "internal error - unknown mode '$mode'\n"; |
| | 960 | } |
| | 961 | |
| | 962 | # assemble archive image |
| | 963 | $plugin->assemble ($task, $vmid); |
| | 964 | |
| | 965 | # produce archive |
| | 966 | |
| | 967 | if ($opts->{stdout}) { |
| | 968 | debugmsg ('info', "sending archive to stdout", $logfd); |
| | 969 | $plugin->archive($task, $vmid, $task->{tmptar}, $comp); |
| | 970 | $self->run_hook_script ('backup-end', $task, $logfd); |
| | 971 | return; |
| | 972 | } |
| | 973 | |
| | 974 | my $archive_txt = $self->{opts}->{pbs} ? 'Proxmox Backup Server' : 'vzdump'; |
| | 975 | debugmsg('info', "creating $archive_txt archive '$task->{target}'", $logfd); |
| | 976 | $plugin->archive($task, $vmid, $task->{tmptar}, $comp); |
| | 977 | |
| | 978 | if ($self->{opts}->{pbs}) { |
| | 979 | # size is added to task struct in guest vzdump plugins |
| | 980 | } else { |
| | 981 | rename ($task->{tmptar}, $task->{target}) || |
| | 982 | die "unable to rename '$task->{tmptar}' to '$task->{target}'\n"; |
| | 983 | |
| | 984 | # determine size |
| | 985 | $task->{size} = (-s $task->{target}) || 0; |
| | 986 | my $cs = format_size ($task->{size}); |
| | 987 | debugmsg ('info', "archive file size: $cs", $logfd); |
| | 988 | } |
| | 989 | |
| | 990 | # purge older backup |
| | 991 | if ($opts->{remove}) { |
| | 992 | if (!defined($opts->{storage})) { |
| | 993 | my $bklist = get_backup_file_list($opts->{dumpdir}, $bkname, $task->{target}); |
| | 994 | PVE::Storage::prune_mark_backup_group($bklist, $prune_options); |
| | 995 | |
| | 996 | foreach my $prune_entry (@{$bklist}) { |
| | 997 | next if $prune_entry->{mark} ne 'remove'; |
| | 998 | |
| | 999 | my $archive_path = $prune_entry->{path}; |
| | 1000 | debugmsg ('info', "delete old backup '$archive_path'", $logfd); |
| | 1001 | PVE::Storage::archive_remove($archive_path); |
| | 1002 | } |
| | 1003 | } else { |
| | 1004 | my $logfunc = sub { debugmsg($_[0], $_[1], $logfd) }; |
| | 1005 | PVE::Storage::prune_backups($cfg, $opts->{storage}, $prune_options, $vmid, $vmtype, 0, $logfunc); |
| | 1006 | } |
| | 1007 | } |
| | 1008 | |
| | 1009 | $self->run_hook_script ('backup-end', $task, $logfd); |
| | 1010 | }; |
| | 1011 | my $err = $@; |
| | 1012 | |
| | 1013 | if ($plugin) { |
| | 1014 | # clean-up |
| | 1015 | |
| | 1016 | if ($cleanup->{unlock}) { |
| | 1017 | eval { $plugin->unlock_vm ($vmid); }; |
| | 1018 | warn $@ if $@; |
| | 1019 | } |
| | 1020 | |
| | 1021 | if ($cleanup->{prepared}) { |
| | 1022 | # only call cleanup when necessary (when prepare was executed) |
| | 1023 | eval { $plugin->cleanup ($task, $vmid) }; |
| | 1024 | warn $@ if $@; |
| | 1025 | } |
| | 1026 | |
| | 1027 | eval { $plugin->set_logfd (undef); }; |
| | 1028 | warn $@ if $@; |
| | 1029 | |
| | 1030 | if ($cleanup->{resume} || $cleanup->{restart}) { |
| | 1031 | eval { |
| | 1032 | $self->run_hook_script ('pre-restart', $task, $logfd); |
| | 1033 | if ($cleanup->{resume}) { |
| | 1034 | debugmsg ('info', "resume vm", $logfd); |
| | 1035 | $plugin->resume_vm ($task, $vmid); |
| | 1036 | } else { |
| | 1037 | my $running = $plugin->vm_status($vmid); |
| | 1038 | if (!$running) { |
| | 1039 | debugmsg ('info', "restarting vm", $logfd); |
| | 1040 | $plugin->start_vm ($task, $vmid); |
| | 1041 | } |
| | 1042 | } |
| | 1043 | $self->run_hook_script ('post-restart', $task, $logfd); |
| | 1044 | }; |
| | 1045 | my $err = $@; |
| | 1046 | if ($err) { |
| | 1047 | warn $err; |
| | 1048 | } else { |
| | 1049 | $log_vm_online_again->(); |
| | 1050 | } |
| | 1051 | } |
| | 1052 | } |
| | 1053 | |
| | 1054 | eval { unlink $task->{tmptar} if $task->{tmptar} && -f $task->{tmptar}; }; |
| | 1055 | warn $@ if $@; |
| | 1056 | |
| | 1057 | eval { rmtree $task->{tmpdir} if $task->{tmpdir} && -d $task->{tmpdir}; }; |
| | 1058 | warn $@ if $@; |
| | 1059 | |
| | 1060 | my $delay = $task->{backuptime} = time () - $vmstarttime; |
| | 1061 | |
| | 1062 | if ($err) { |
| | 1063 | $task->{state} = 'err'; |
| | 1064 | $task->{msg} = $err; |
| | 1065 | debugmsg ('err', "Backup of VM $vmid failed - $err", $logfd, 1); |
| | 1066 | debugmsg ('info', "Failed at " . strftime("%F %H:%M:%S", localtime())); |
| | 1067 | |
| | 1068 | eval { $self->run_hook_script ('backup-abort', $task, $logfd); }; |
| | 1069 | |
| | 1070 | } else { |
| | 1071 | $task->{state} = 'ok'; |
| | 1072 | my $tstr = format_time ($delay); |
| | 1073 | debugmsg ('info', "Finished Backup of VM $vmid ($tstr)", $logfd, 1); |
| | 1074 | debugmsg ('info', "Backup finished at " . strftime("%F %H:%M:%S", localtime())); |
| | 1075 | } |
| | 1076 | |
| | 1077 | close ($logfd) if $logfd; |
| | 1078 | |
| | 1079 | if ($task->{tmplog}) { |
| | 1080 | if ($self->{opts}->{pbs}) { |
| | 1081 | if ($task->{state} eq 'ok') { |
| | 1082 | eval { |
| | 1083 | PVE::Storage::PBSPlugin::run_raw_client_cmd( |
| | 1084 | $opts->{scfg}, |
| | 1085 | $opts->{storage}, |
| | 1086 | 'upload-log', |
| | 1087 | [ $pbs_snapshot_name, $task->{tmplog} ], |
| | 1088 | errmsg => "uploading backup task log failed", |
| | 1089 | ); |
| | 1090 | }; |
| | 1091 | debugmsg('warn', "$@") if $@; # $@ contains already error prefix |
| | 1092 | } |
| | 1093 | } elsif ($task->{logfile}) { |
| | 1094 | system {'cp'} 'cp', $task->{tmplog}, $task->{logfile}; |
| | 1095 | } |
| | 1096 | } |
| | 1097 | |
| | 1098 | eval { $self->run_hook_script ('log-end', $task); }; |
| | 1099 | |
| | 1100 | die $err if $err && $err =~ m/^interrupted by signal$/; |
| | 1101 | } |
| | 1102 | |
| | 1103 | sub exec_backup { |
| | 1104 | my ($self, $rpcenv, $authuser) = @_; |
| | 1105 | |
| | 1106 | my $opts = $self->{opts}; |
| | 1107 | |
| | 1108 | debugmsg ('info', "starting new backup job: $self->{cmdline}", undef, 1); |
| | 1109 | |
| | 1110 | if (scalar(@{$self->{skiplist}})) { |
| | 1111 | my $skip_string = join(', ', sort { $a <=> $b } @{$self->{skiplist}}); |
| | 1112 | debugmsg ('info', "skip external VMs: $skip_string"); |
| | 1113 | } |
| | 1114 | |
| | 1115 | my $tasklist = []; |
| | 1116 | my $vzdump_plugins = {}; |
| | 1117 | foreach my $plugin (@{$self->{plugins}}) { |
| | 1118 | my $type = $plugin->type(); |
| | 1119 | next if exists $vzdump_plugins->{$type}; |
| | 1120 | $vzdump_plugins->{$type} = $plugin; |
| | 1121 | } |
| | 1122 | |
| | 1123 | my $vmlist = PVE::Cluster::get_vmlist(); |
| | 1124 | my $vmids = [ sort { $a <=> $b } @{$opts->{vmids}} ]; |
| | 1125 | foreach my $vmid (@{$vmids}) { |
| | 1126 | my $plugin; |
| | 1127 | if (defined($vmlist->{ids}->{$vmid})) { |
| | 1128 | my $guest_type = $vmlist->{ids}->{$vmid}->{type}; |
| | 1129 | $plugin = $vzdump_plugins->{$guest_type}; |
| | 1130 | next if !$rpcenv->check($authuser, "/vms/$vmid", [ 'VM.Backup' ], $opts->{all}); |
| | 1131 | } |
| | 1132 | push @$tasklist, { |
| | 1133 | mode => $opts->{mode}, |
| | 1134 | plugin => $plugin, |
| | 1135 | state => 'todo', |
| | 1136 | vmid => $vmid, |
| | 1137 | }; |
| | 1138 | } |
| | 1139 | |
| | 1140 | # Use in-memory files for the outer hook logs to pass them to sendmail. |
| | 1141 | my $job_start_log = ''; |
| | 1142 | my $job_end_log = ''; |
| | 1143 | open my $job_start_fd, '>', \$job_start_log; |
| | 1144 | open my $job_end_fd, '>', \$job_end_log; |
| | 1145 | |
| | 1146 | my $starttime = time(); |
| | 1147 | my $errcount = 0; |
| | 1148 | eval { |
| | 1149 | |
| | 1150 | $self->run_hook_script ('job-start', undef, $job_start_fd); |
| | 1151 | |
| | 1152 | foreach my $task (@$tasklist) { |
| | 1153 | $self->exec_backup_task ($task); |
| | 1154 | $errcount += 1 if $task->{state} ne 'ok'; |
| | 1155 | } |
| | 1156 | |
| | 1157 | $self->run_hook_script ('job-end', undef, $job_end_fd); |
| | 1158 | }; |
| | 1159 | my $err = $@; |
| | 1160 | |
| | 1161 | $self->run_hook_script ('job-abort', undef, $job_end_fd) if $err; |
| | 1162 | |
| | 1163 | if ($err) { |
| | 1164 | debugmsg ('err', "Backup job failed - $err", undef, 1); |
| | 1165 | } else { |
| | 1166 | if ($errcount) { |
| | 1167 | debugmsg ('info', "Backup job finished with errors", undef, 1); |
| | 1168 | } else { |
| | 1169 | debugmsg ('info', "Backup job finished successfully", undef, 1); |
| | 1170 | } |
| | 1171 | } |
| | 1172 | |
| | 1173 | close $job_start_fd; |
| | 1174 | close $job_end_fd; |
| | 1175 | |
| | 1176 | my $totaltime = time() - $starttime; |
| | 1177 | |
| | 1178 | eval { $self->sendmail ($tasklist, $totaltime, undef, $job_start_log, $job_end_log); }; |
| | 1179 | debugmsg ('err', $@) if $@; |
| | 1180 | |
| | 1181 | die $err if $err; |
| | 1182 | |
| | 1183 | die "job errors\n" if $errcount; |
| | 1184 | |
| | 1185 | unlink $pidfile; |
| | 1186 | } |
| | 1187 | |
| | 1188 | |
| | 1189 | sub option_exists { |
| | 1190 | my $key = shift; |
| | 1191 | return defined($confdesc->{$key}); |
| | 1192 | } |
| | 1193 | |
| | 1194 | # NOTE it might make sense to merge this and verify_vzdump_parameters(), but one |
| | 1195 | # needs to adapt command_line() in guest-common's PVE/VZDump/Common.pm and detect |
| | 1196 | # a second parsing attempt, because verify_vzdump_parameters() is called twice |
| | 1197 | # during the update_job API call. |
| | 1198 | sub parse_mailto_exclude_path { |
| | 1199 | my ($param) = @_; |
| | 1200 | |
| | 1201 | # exclude-path list need to be 0 separated |
| | 1202 | if (defined($param->{'exclude-path'})) { |
| | 1203 | my @expaths = split(/\0/, $param->{'exclude-path'} || ''); |
| | 1204 | $param->{'exclude-path'} = [ @expaths ]; |
| | 1205 | } |
| | 1206 | |
| | 1207 | if (defined($param->{mailto})) { |
| | 1208 | my @mailto = PVE::Tools::split_list(extract_param($param, 'mailto')); |
| | 1209 | $param->{mailto} = [ @mailto ]; |
| | 1210 | } |
| | 1211 | |
| | 1212 | return; |
| | 1213 | } |
| | 1214 | |
| | 1215 | sub verify_vzdump_parameters { |
| | 1216 | my ($param, $check_missing) = @_; |
| | 1217 | |
| | 1218 | raise_param_exc({ all => "option conflicts with option 'vmid'"}) |
| | 1219 | if $param->{all} && $param->{vmid}; |
| | 1220 | |
| | 1221 | raise_param_exc({ exclude => "option conflicts with option 'vmid'"}) |
| | 1222 | if $param->{exclude} && $param->{vmid}; |
| | 1223 | |
| | 1224 | raise_param_exc({ pool => "option conflicts with option 'vmid'"}) |
| | 1225 | if $param->{pool} && $param->{vmid}; |
| | 1226 | |
| | 1227 | raise_param_exc({ 'prune-backups' => "option conflicts with option 'maxfiles'"}) |
| | 1228 | if defined($param->{'prune-backups'}) && defined($param->{maxfiles}); |
| | 1229 | |
| | 1230 | $parse_prune_backups_maxfiles->($param, 'CLI parameters'); |
| | 1231 | |
| | 1232 | $param->{all} = 1 if (defined($param->{exclude}) && !$param->{pool}); |
| | 1233 | |
| | 1234 | warn "option 'size' is deprecated and will be removed in a future " . |
| | 1235 | "release, please update your script/configuration!\n" |
| | 1236 | if defined($param->{size}); |
| | 1237 | |
| | 1238 | return if !$check_missing; |
| | 1239 | |
| | 1240 | raise_param_exc({ vmid => "property is missing"}) |
| | 1241 | if !($param->{all} || $param->{stop} || $param->{pool}) && !$param->{vmid}; |
| | 1242 | |
| | 1243 | } |
| | 1244 | |
| | 1245 | sub stop_running_backups { |
| | 1246 | my($self) = @_; |
| | 1247 | |
| | 1248 | my $upid = PVE::Tools::file_read_firstline($pidfile); |
| | 1249 | return if !$upid; |
| | 1250 | |
| | 1251 | my $task = PVE::Tools::upid_decode($upid); |
| | 1252 | |
| | 1253 | if (PVE::ProcFSTools::check_process_running($task->{pid}, $task->{pstart}) && |
| | 1254 | PVE::ProcFSTools::read_proc_starttime($task->{pid}) == $task->{pstart}) { |
| | 1255 | kill(15, $task->{pid}); |
| | 1256 | # wait max 15 seconds to shut down (else, do nothing for now) |
| | 1257 | my $i; |
| | 1258 | for ($i = 15; $i > 0; $i--) { |
| | 1259 | last if !PVE::ProcFSTools::check_process_running(($task->{pid}, $task->{pstart})); |
| | 1260 | sleep (1); |
| | 1261 | } |
| | 1262 | die "stopping backup process $task->{pid} failed\n" if $i == 0; |
| | 1263 | } |
| | 1264 | } |
| | 1265 | |
| | 1266 | sub get_included_guests { |
| | 1267 | my ($job) = @_; |
| | 1268 | |
| | 1269 | my $vmids = []; |
| | 1270 | my $vmids_per_node = {}; |
| | 1271 | |
| | 1272 | my $vmlist = PVE::Cluster::get_vmlist(); |
| | 1273 | |
| | 1274 | if ($job->{pool}) { |
| | 1275 | $vmids = PVE::API2Tools::get_resource_pool_guest_members($job->{pool}); |
| | 1276 | } elsif ($job->{vmid}) { |
| | 1277 | $vmids = [ split_list($job->{vmid}) ]; |
| | 1278 | } elsif ($job->{all}) { |
| | 1279 | # all or exclude |
| | 1280 | my $exclude = check_vmids(split_list($job->{exclude})); |
| | 1281 | my $excludehash = { map { $_ => 1 } @$exclude }; |
| | 1282 | |
| | 1283 | for my $id (keys %{$vmlist->{ids}}) { |
| | 1284 | next if $excludehash->{$id}; |
| | 1285 | push @$vmids, $id; |
| | 1286 | } |
| | 1287 | } else { |
| | 1288 | return $vmids_per_node; |
| | 1289 | } |
| | 1290 | $vmids = check_vmids(@$vmids); |
| | 1291 | |
| | 1292 | for my $vmid (@$vmids) { |
| | 1293 | if (defined($vmlist->{ids}->{$vmid})) { |
| | 1294 | my $node = $vmlist->{ids}->{$vmid}->{node}; |
| | 1295 | next if (defined $job->{node} && $job->{node} ne $node); |
| | 1296 | |
| | 1297 | push @{$vmids_per_node->{$node}}, $vmid; |
| | 1298 | } else { |
| | 1299 | push @{$vmids_per_node->{''}}, $vmid; |
| | 1300 | } |
| | 1301 | } |
| | 1302 | |
| | 1303 | return $vmids_per_node; |
| | 1304 | } |
| | 1305 | |
| | 1306 | 1; |
projects / pve-manager.git / blame_incremental