PVE/VZDump.pm

   1 package PVE::VZDump;
   2
   3 use strict;
   4 use warnings;
   5 use Fcntl ':flock';
   6 use PVE::Exception qw(raise_param_exc);
   7 use PVE::SafeSyslog;
   8 use IO::File;
   9 use IO::Select;
  10 use IPC::Open3;
  11 use POSIX qw(strftime);
  12 use File::Path;
  13 use PVE::RPCEnvironment;
  14 use PVE::Storage;
  15 use PVE::Cluster qw(cfs_read_file);
  16 use Time::localtime;
  17 use Time::Local;
  18 use PVE::JSONSchema qw(get_standard_option);
  19
  20 my @posix_filesystems = qw(ext3 ext4 nfs nfs4 reiserfs xfs);
  21
  22 my $lockfile = '/var/run/vzdump.lock';
  23
  24 my $pidfile = '/var/run/vzdump.pid';
  25
  26 my $logdir = '/var/log/vzdump';
  27
  28 my @plugins = qw();
  29
  30 # Load available plugins
  31 my $pveplug = "/usr/share/perl5/PVE/VZDump/QemuServer.pm";
  32 if (-f $pveplug) {
  33     eval { require $pveplug; };
  34     if (!$@) {
  35         PVE::VZDump::QemuServer->import ();
  36           push @plugins, "PVE::VZDump::QemuServer";
  37       } else {
  38           warn $@;
  39       }
  40 }
  41
  42 # helper functions
  43
  44 my $debugstattxt = {
  45     err =>  'ERROR:',
  46     info => 'INFO:',
  47     warn => 'WARN:',
  48 };
  49
  50 sub debugmsg {
  51     my ($mtype, $msg, $logfd, $syslog) = @_;
  52
  53     chomp $msg;
  54
  55     return if !$msg;
  56
  57     my $pre = $debugstattxt->{$mtype} || $debugstattxt->{'err'};
  58
  59     my $timestr = strftime ("%b %d %H:%M:%S", CORE::localtime);
  60
  61     syslog ($mtype eq 'info' ? 'info' : 'err', "$pre $msg") if $syslog;
  62
  63     foreach my $line (split (/\n/, $msg)) {
  64         print STDERR "$pre $line\n";
  65         print $logfd "$timestr $pre $line\n" if $logfd;
  66     }
  67 }
  68
  69 sub run_command {
  70     my ($logfd, $cmdstr, %param) = @_;
  71
  72     my $logfunc = sub {
  73         my $line = shift;
  74         debugmsg ('info', $line, $logfd);
  75     };
  76
  77     PVE::Tools::run_command($cmdstr, %param, logfunc => $logfunc);
  78 }
  79
  80 sub storage_info {
  81     my $storage = shift;
  82
  83     my $cfg = cfs_read_file('storage.cfg');
  84     my $scfg = PVE::Storage::storage_config($cfg, $storage);
  85     my $type = $scfg->{type};
  86
  87     die "can't use storage type '$type' for backup\n"
  88         if (!($type eq 'dir' || $type eq 'nfs' || $type eq 'glusterfs'));
  89     die "can't use storage for backups - wrong content type\n"
  90         if (!$scfg->{content}->{backup});
  91
  92     PVE::Storage::activate_storage($cfg, $storage);
  93
  94     return {
  95         dumpdir => PVE::Storage::get_backup_dir($cfg, $storage),
  96         maxfiles => $scfg->{maxfiles},
  97     };
  98 }
  99
 100 sub format_size {
 101     my $size = shift;
 102
 103     my $kb = $size / 1024;
 104
 105     if ($kb < 1024) {
 106         return int ($kb) . "KB";
 107     }
 108
 109     my $mb = $size / (1024*1024);
 110
 111     if ($mb < 1024) {
 112         return int ($mb) . "MB";
 113     } else {
 114         my $gb = $mb / 1024;
 115         return sprintf ("%.2fGB", $gb);
 116     }
 117 }
 118
 119 sub format_time {
 120     my $seconds = shift;
 121
 122     my $hours = int ($seconds/3600);
 123     $seconds = $seconds - $hours*3600;
 124     my $min = int ($seconds/60);
 125     $seconds = $seconds - $min*60;
 126
 127     return sprintf ("%02d:%02d:%02d", $hours, $min, $seconds);
 128 }
 129
 130 sub encode8bit {
 131     my ($str) = @_;
 132
 133     $str =~ s/^(.{990})/$1\n/mg; # reduce line length
 134
 135     return $str;
 136 }
 137
 138 sub escape_html {
 139     my ($str) = @_;
 140
 141     $str =~ s/&/&amp;/g;
 142     $str =~ s/</&lt;/g;
 143     $str =~ s/>/&gt;/g;
 144
 145     return $str;
 146 }
 147
 148 sub check_bin {
 149     my ($bin)  = @_;
 150
 151     foreach my $p (split (/:/, $ENV{PATH})) {
 152         my $fn = "$p/$bin";
 153         if (-x $fn) {
 154             return $fn;
 155         }
 156     }
 157
 158     die "unable to find command '$bin'\n";
 159 }
 160
 161 sub check_vmids {
 162     my (@vmids) = @_;
 163
 164     my $res = [];
 165     foreach my $vmid (@vmids) {
 166         die "ERROR: strange VM ID '${vmid}'\n" if $vmid !~ m/^\d+$/;
 167         $vmid = int ($vmid); # remove leading zeros
 168         next if !$vmid;
 169         push @$res, $vmid;
 170     }
 171
 172     return $res;
 173 }
 174
 175
 176 sub read_vzdump_defaults {
 177
 178     my $fn = "/etc/vzdump.conf";
 179
 180     my $res = {
 181         bwlimit => 0,
 182         ionice => 7,
 183         size => 1024,
 184         lockwait => 3*60, # 3 hours
 185         stopwait => 10, # 10 minutes
 186         mode => 'snapshot',
 187         maxfiles => 1,
 188     };
 189
 190     my $fh = IO::File->new ("<$fn");
 191     return $res if !$fh;
 192
 193     my $line;
 194     while (defined ($line = <$fh>)) {
 195         next if $line =~ m/^\s*$/;
 196         next if $line =~ m/^\#/;
 197
 198         if ($line =~ m/tmpdir:\s*(.*\S)\s*$/) {
 199             $res->{tmpdir} = $1;
 200         } elsif ($line =~ m/dumpdir:\s*(.*\S)\s*$/) {
 201             $res->{dumpdir} = $1;
 202         } elsif ($line =~ m/storage:\s*(\S+)\s*$/) {
 203             $res->{storage} = $1;
 204         } elsif ($line =~ m/script:\s*(.*\S)\s*$/) {
 205             $res->{script} = $1;
 206         } elsif ($line =~ m/bwlimit:\s*(\d+)\s*$/) {
 207             $res->{bwlimit} = int($1);
 208         } elsif ($line =~ m/ionice:\s*([0-8])\s*$/) {
 209             $res->{ionice} = int($1);
 210         } elsif ($line =~ m/lockwait:\s*(\d+)\s*$/) {
 211             $res->{lockwait} = int($1);
 212         } elsif ($line =~ m/stopwait:\s*(\d+)\s*$/) {
 213             $res->{stopwait} = int($1);
 214         } elsif ($line =~ m/stop:\s*(\d+)\s*$/) {
 215             $res->{stop} = int($1);
 216         } elsif ($line =~ m/size:\s*(\d+)\s*$/) {
 217             $res->{size} = int($1);
 218         } elsif ($line =~ m/maxfiles:\s*(\d+)\s*$/) {
 219             $res->{maxfiles} = int($1);
 220         } elsif ($line =~ m/exclude-path:\s*(.*)\s*$/) {
 221             $res->{'exclude-path'} = PVE::Tools::split_args($1);
 222         } elsif ($line =~ m/mode:\s*(stop|snapshot|suspend)\s*$/) {
 223             $res->{mode} = $1;
 224         } else {
 225             debugmsg ('warn', "unable to parse configuration file '$fn' - error at line " . $., undef, 1);
 226         }
 227
 228     }
 229     close ($fh);
 230
 231     return $res;
 232 }
 233
 234
 235 sub find_add_exclude {
 236     my ($self, $excltype, $value) = @_;
 237
 238     if (($excltype eq '-regex') || ($excltype eq '-files')) {
 239         $value = "\.$value";
 240     }
 241
 242     if ($excltype eq '-files') {
 243         push @{$self->{findexcl}}, "'('", '-not', '-type', 'd', '-regex' , "'$value'", "')'", '-o';
 244     } else {
 245         push @{$self->{findexcl}}, "'('", $excltype , "'$value'", '-prune', "')'", '-o';
 246     }
 247 }
 248
 249 sub sendmail {
 250     my ($self, $tasklist, $totaltime, $err) = @_;
 251
 252     my $opts = $self->{opts};
 253
 254     my $mailto = $opts->{mailto};
 255
 256     return if !($mailto && scalar(@$mailto));
 257
 258     my $cmdline = $self->{cmdline};
 259
 260     my $ecount = 0;
 261     foreach my $task (@$tasklist) {
 262         $ecount++ if $task->{state} ne 'ok';
 263         chomp $task->{msg} if $task->{msg};
 264         $task->{backuptime} = 0 if !$task->{backuptime};
 265         $task->{size} = 0 if !$task->{size};
 266         $task->{tarfile} = 'unknown' if !$task->{tarfile};
 267         $task->{hostname} = "VM $task->{vmid}" if !$task->{hostname};
 268
 269         if ($task->{state} eq 'todo') {
 270             $task->{msg} = 'aborted';
 271         }
 272     }
 273
 274     my $notify = $opts->{mailnotification} || 'always';
 275     return if (!$ecount && !$err && ($notify eq 'failure'));
 276
 277     my $stat = ($ecount || $err) ? 'backup failed' : 'backup successful';
 278     $stat .= ": $err" if $err;
 279
 280     my $hostname = `hostname -f` || PVE::INotify::nodename();
 281     chomp $hostname;
 282
 283     my $boundary = "----_=_NextPart_001_".int(time).$$;
 284
 285     my $rcvrarg = '';
 286     foreach my $r (@$mailto) {
 287         $rcvrarg .= " '$r'";
 288     }
 289     my $dcconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
 290     my $mailfrom = $dcconf->{email_from} || "root";
 291
 292     open (MAIL,"|sendmail -B 8BITMIME -f $mailfrom $rcvrarg") ||
 293         die "unable to open 'sendmail' - $!";
 294
 295     my $rcvrtxt = join (', ', @$mailto);
 296
 297     print MAIL "Content-Type: multipart/alternative;\n";
 298     print MAIL "\tboundary=\"$boundary\"\n";
 299     print MAIL "MIME-Version: 1.0\n";
 300
 301     print MAIL "FROM: vzdump backup tool <$mailfrom>\n";
 302     print MAIL "TO: $rcvrtxt\n";
 303     print MAIL "SUBJECT: vzdump backup status ($hostname) : $stat\n";
 304     print MAIL "\n";
 305     print MAIL "This is a multi-part message in MIME format.\n\n";
 306     print MAIL "--$boundary\n";
 307
 308     print MAIL "Content-Type: text/plain;\n";
 309     print MAIL "\tcharset=\"UTF8\"\n";
 310     print MAIL "Content-Transfer-Encoding: 8bit\n";
 311     print MAIL "\n";
 312
 313     # text part
 314
 315     my $fill = '  '; # Avoid The Remove Extra Line Breaks Issue (MS Outlook)
 316
 317     print MAIL sprintf ("${fill}%-10s %-6s %10s %10s  %s\n", qw(VMID STATUS TIME SIZE FILENAME));
 318     foreach my $task (@$tasklist) {
 319         my $vmid = $task->{vmid};
 320         if  ($task->{state} eq 'ok') {
 321
 322             print MAIL sprintf ("${fill}%-10s %-6s %10s %10s  %s\n", $vmid,
 323                                 $task->{state},
 324                                 format_time($task->{backuptime}),
 325                                 format_size ($task->{size}),
 326                                 $task->{tarfile});
 327         } else {
 328             print MAIL sprintf ("${fill}%-10s %-6s %10s %8.2fMB  %s\n", $vmid,
 329                                 $task->{state},
 330                                 format_time($task->{backuptime}),
 331                                 0, '-');
 332         }
 333     }
 334     print MAIL "${fill}\n";
 335     print MAIL "${fill}Detailed backup logs:\n";
 336     print MAIL "${fill}\n";
 337     print MAIL "$fill$cmdline\n";
 338     print MAIL "${fill}\n";
 339
 340     foreach my $task (@$tasklist) {
 341         my $vmid = $task->{vmid};
 342         my $log = $task->{tmplog};
 343         if (!$log) {
 344             print MAIL "${fill}$vmid: no log available\n\n";
 345             next;
 346         }
 347         open (TMP, "$log");
 348         while (my $line = <TMP>) { print MAIL encode8bit ("${fill}$vmid: $line"); }
 349         close (TMP);
 350         print MAIL "${fill}\n";
 351     }
 352
 353     # end text part
 354     print MAIL "\n--$boundary\n";
 355
 356     print MAIL "Content-Type: text/html;\n";
 357     print MAIL "\tcharset=\"UTF8\"\n";
 358     print MAIL "Content-Transfer-Encoding: 8bit\n";
 359     print MAIL "\n";
 360
 361     # html part
 362
 363     print MAIL "<html><body>\n";
 364
 365     print MAIL "<table border=1 cellpadding=3>\n";
 366
 367     print MAIL "<tr><td>VMID<td>NAME<td>STATUS<td>TIME<td>SIZE<td>FILENAME</tr>\n";
 368
 369     my $ssize = 0;
 370
 371     foreach my $task (@$tasklist) {
 372         my $vmid = $task->{vmid};
 373         my $name = $task->{hostname};
 374
 375         if  ($task->{state} eq 'ok') {
 376
 377             $ssize += $task->{size};
 378
 379             print MAIL sprintf ("<tr><td>%s<td>%s<td>OK<td>%s<td align=right>%s<td>%s</tr>\n",
 380                                 $vmid, $name,
 381                                 format_time($task->{backuptime}),
 382                                 format_size ($task->{size}),
 383                                 escape_html ($task->{tarfile}));
 384         } else {
 385             print MAIL sprintf ("<tr><td>%s<td>%s<td><font color=red>FAILED<td>%s<td colspan=2>%s</tr>\n",
 386
 387                                 $vmid, $name, format_time($task->{backuptime}),
 388                                 escape_html ($task->{msg}));
 389         }
 390     }
 391
 392     print MAIL sprintf ("<tr><td align=left colspan=3>TOTAL<td>%s<td>%s<td></tr>",
 393  format_time ($totaltime), format_size ($ssize));
 394
 395     print MAIL "</table><br><br>\n";
 396     print MAIL "Detailed backup logs:<br>\n";
 397     print MAIL "<br>\n";
 398     print MAIL "<pre>\n";
 399     print MAIL escape_html($cmdline) . "\n";
 400     print MAIL "\n";
 401
 402     foreach my $task (@$tasklist) {
 403         my $vmid = $task->{vmid};
 404         my $log = $task->{tmplog};
 405         if (!$log) {
 406             print MAIL "$vmid: no log available\n\n";
 407             next;
 408         }
 409         open (TMP, "$log");
 410         while (my $line = <TMP>) {
 411             if ($line =~ m/^\S+\s\d+\s+\d+:\d+:\d+\s+(ERROR|WARN):/) {
 412                 print MAIL encode8bit ("$vmid: <font color=red>".
 413                                        escape_html ($line) . "</font>");
 414             } else {
 415                 print MAIL encode8bit ("$vmid: " . escape_html ($line));
 416             }
 417         }
 418         close (TMP);
 419         print MAIL "\n";
 420     }
 421     print MAIL "</pre>\n";
 422
 423     print MAIL "</body></html>\n";
 424
 425     # end html part
 426     print MAIL "\n--$boundary--\n";
 427
 428     close(MAIL);
 429 };
 430
 431 sub new {
 432     my ($class, $cmdline, $opts, $skiplist) = @_;
 433
 434     mkpath $logdir;
 435
 436     check_bin ('cp');
 437     check_bin ('df');
 438     check_bin ('sendmail');
 439     check_bin ('rsync');
 440     check_bin ('tar');
 441     check_bin ('mount');
 442     check_bin ('umount');
 443     check_bin ('cstream');
 444     check_bin ('ionice');
 445
 446     if ($opts->{mode} && $opts->{mode} eq 'snapshot') {
 447         check_bin ('lvcreate');
 448         check_bin ('lvs');
 449         check_bin ('lvremove');
 450     }
 451
 452     my $defaults = read_vzdump_defaults();
 453
 454     my $maxfiles = $opts->{maxfiles}; # save here, because we overwrite with default
 455
 456     $opts->{remove} = 1 if !defined($opts->{remove});
 457
 458     foreach my $k (keys %$defaults) {
 459         if ($k eq 'dumpdir' || $k eq 'storage') {
 460             $opts->{$k} = $defaults->{$k} if !defined ($opts->{dumpdir}) &&
 461                 !defined ($opts->{storage});
 462         } else {
 463             $opts->{$k} = $defaults->{$k} if !defined ($opts->{$k});
 464         }
 465     }
 466
 467     $opts->{dumpdir} =~ s|/+$|| if ($opts->{dumpdir});
 468     $opts->{tmpdir} =~ s|/+$|| if ($opts->{tmpdir});
 469
 470     $skiplist = [] if !$skiplist;
 471     my $self = bless { cmdline => $cmdline, opts => $opts, skiplist => $skiplist };
 472
 473     #always skip '.'
 474     push @{$self->{findexcl}}, "'('", '-regex' , "'^\\.\$'", "')'", '-o';
 475
 476     $self->find_add_exclude ('-type', 's'); # skip sockets
 477
 478     if ($defaults->{'exclude-path'}) {
 479         foreach my $path (@{$defaults->{'exclude-path'}}) {
 480             $self->find_add_exclude ('-regex', $path);
 481         }
 482     }
 483
 484     if ($opts->{'exclude-path'}) {
 485         foreach my $path (@{$opts->{'exclude-path'}}) {
 486             $self->find_add_exclude ('-regex', $path);
 487         }
 488     }
 489
 490     if ($opts->{stdexcludes}) {
 491         $self->find_add_exclude ('-files', '/var/log/.+');
 492         $self->find_add_exclude ('-regex', '/tmp/.+');
 493         $self->find_add_exclude ('-regex', '/var/tmp/.+');
 494         $self->find_add_exclude ('-regex', '/var/run/.+pid');
 495     }
 496
 497     foreach my $p (@plugins) {
 498
 499         my $pd = $p->new ($self);
 500
 501         push @{$self->{plugins}}, $pd;
 502     }
 503
 504     if (!$opts->{dumpdir} && !$opts->{storage}) {
 505         $opts->{storage} = 'local';
 506     }
 507
 508     if ($opts->{storage}) {
 509         my $info = storage_info ($opts->{storage});
 510         $opts->{dumpdir} = $info->{dumpdir};
 511         $maxfiles = $info->{maxfiles} if !defined($maxfiles) && defined($info->{maxfiles});
 512     } elsif ($opts->{dumpdir}) {
 513         die "dumpdir '$opts->{dumpdir}' does not exist\n"
 514             if ! -d $opts->{dumpdir};
 515     } else {
 516         die "internal error";
 517     }
 518
 519     if ($opts->{tmpdir} && ! -d $opts->{tmpdir}) {
 520         die "tmpdir '$opts->{tmpdir}' does not exist\n";
 521     }
 522
 523     $opts->{maxfiles} = $maxfiles if defined($maxfiles);
 524
 525     return $self;
 526
 527 }
 528
 529 sub get_lvm_mapping {
 530
 531     my $devmapper;
 532
 533     my $cmd = ['lvs', '--units', 'm', '--separator', ':', '--noheadings',
 534                '-o', 'vg_name,lv_name,lv_size' ];
 535
 536     my $parser = sub {
 537         my $line = shift;
 538         if ($line =~ m|^\s*(\S+):(\S+):(\d+(\.\d+))[Mm]$|) {
 539             my $vg = $1;
 540             my $lv = $2;
 541             $devmapper->{"/dev/$vg/$lv"} = [$vg, $lv];
 542             my $qlv = $lv;
 543             $qlv =~ s/-/--/g;
 544             my $qvg = $vg;
 545             $qvg =~ s/-/--/g;
 546             $devmapper->{"/dev/mapper/$qvg-$qlv"} = [$vg, $lv];
 547         }
 548     };
 549
 550     eval { PVE::Tools::run_command($cmd, errfunc => sub {}, outfunc => $parser); };
 551     warn $@ if $@;
 552
 553     return $devmapper;
 554 }
 555
 556 sub get_mount_info {
 557     my ($dir) = @_;
 558
 559     # Note: df 'available' can be negative, and percentage set to '-'
 560
 561     my $cmd = [ 'df', '-P', '-T', '-B', '1', $dir];
 562
 563     my $res;
 564
 565     my $parser = sub {
 566         my $line = shift;
 567         if (my ($fsid, $fstype, undef, $mp) = $line =~
 568             m!(\S+.*)\s+(\S+)\s+\d+\s+\-?\d+\s+\d+\s+(\d+%|-)\s+(/.*)$!) {
 569             $res = {
 570                 device => $fsid,
 571                 fstype => $fstype,
 572                 mountpoint => $mp,
 573             };
 574         }
 575     };
 576
 577     eval { PVE::Tools::run_command($cmd, errfunc => sub {}, outfunc => $parser); };
 578     warn $@ if $@;
 579
 580     return $res;
 581 }
 582
 583 sub get_lvm_device {
 584     my ($dir, $mapping) = @_;
 585
 586     my $info = get_mount_info($dir);
 587
 588     return undef if !$info;
 589
 590     my $dev = $info->{device};
 591
 592     my ($vg, $lv);
 593
 594     ($vg, $lv) = @{$mapping->{$dev}} if defined $mapping->{$dev};
 595
 596     return wantarray ? ($dev, $info->{mountpoint}, $vg, $lv, $info->{fstype}) : $dev;
 597 }
 598
 599 sub getlock {
 600     my ($self, $upid) = @_;
 601
 602     my $fh;
 603
 604     my $maxwait = $self->{opts}->{lockwait} || $self->{lockwait};
 605
 606     die "missimg UPID" if !$upid; # should not happen
 607
 608     if (!open (SERVER_FLCK, ">>$lockfile")) {
 609         debugmsg ('err', "can't open lock on file '$lockfile' - $!", undef, 1);
 610         die "can't open lock on file '$lockfile' - $!";
 611     }
 612
 613     if (!flock (SERVER_FLCK, LOCK_EX|LOCK_NB)) {
 614
 615         if (!$maxwait) {
 616             debugmsg ('err', "can't aquire lock '$lockfile' (wait = 0)", undef, 1);
 617             die "can't aquire lock '$lockfile' (wait = 0)";
 618         }
 619
 620         debugmsg('info', "trying to get global lock - waiting...", undef, 1);
 621
 622         eval {
 623             alarm ($maxwait * 60);
 624
 625             local $SIG{ALRM} = sub { alarm (0); die "got timeout\n"; };
 626
 627             if (!flock (SERVER_FLCK, LOCK_EX)) {
 628                 my $err = $!;
 629                 close (SERVER_FLCK);
 630                 alarm (0);
 631                 die "$err\n";
 632             }
 633             alarm (0);
 634         };
 635         alarm (0);
 636
 637         my $err = $@;
 638
 639         if ($err) {
 640             debugmsg ('err', "can't aquire lock '$lockfile' - $err", undef, 1);
 641             die "can't aquire lock '$lockfile' - $err";
 642         }
 643
 644         debugmsg('info', "got global lock", undef, 1);
 645     }
 646
 647     PVE::Tools::file_set_contents($pidfile, $upid);
 648 }
 649
 650 sub run_hook_script {
 651     my ($self, $phase, $task, $logfd) = @_;
 652
 653     my $opts = $self->{opts};
 654
 655     my $script = $opts->{script};
 656
 657     return if !$script;
 658
 659     my $cmd = "$script $phase";
 660
 661     $cmd .= " $task->{mode} $task->{vmid}" if ($task);
 662
 663     local %ENV;
 664
 665     # set immutable opts directly (so they are available in all phases)
 666     $ENV{STOREID} = $opts->{storage} if $opts->{storage};
 667     $ENV{DUMPDIR} = $opts->{dumpdir} if $opts->{dumpdir};
 668
 669     foreach my $ek (qw(vmtype hostname tarfile logfile)) {
 670         $ENV{uc($ek)} = $task->{$ek} if $task->{$ek};
 671     }
 672
 673     run_command ($logfd, $cmd);
 674 }
 675
 676 sub compressor_info {
 677     my ($opt_compress) = @_;
 678
 679     if (!$opt_compress || $opt_compress eq '0') {
 680         return undef;
 681     } elsif ($opt_compress eq '1' || $opt_compress eq 'lzo') {
 682         return ('lzop', 'lzo');
 683     } elsif ($opt_compress eq 'gzip') {
 684         return ('gzip', 'gz');
 685     } else {
 686         die "internal error - unknown compression option '$opt_compress'";
 687     }
 688 }
 689
 690 sub get_backup_file_list {
 691     my ($dir, $bkname, $exclude_fn) = @_;
 692
 693     my $bklist = [];
 694     foreach my $fn (<$dir/${bkname}-*>) {
 695         next if $exclude_fn && $fn eq $exclude_fn;
 696         if ($fn =~ m!/(${bkname}-(\d{4})_(\d{2})_(\d{2})-(\d{2})_(\d{2})_(\d{2})\.(tgz|((tar|vma)(\.(gz|lzo))?)))$!) {
 697             $fn = "$dir/$1"; # untaint
 698             my $t = timelocal ($7, $6, $5, $4, $3 - 1, $2 - 1900);
 699             push @$bklist, [$fn, $t];
 700         }
 701     }
 702
 703     return $bklist;
 704 }
 705
 706 sub exec_backup_task {
 707     my ($self, $task) = @_;
 708
 709     my $opts = $self->{opts};
 710
 711     my $vmid = $task->{vmid};
 712     my $plugin = $task->{plugin};
 713
 714     my $vmstarttime = time ();
 715
 716     my $logfd;
 717
 718     my $cleanup = {};
 719
 720     my $vmstoptime = 0;
 721
 722     eval {
 723         die "unable to find VM '$vmid'\n" if !$plugin;
 724
 725         my $vmtype = $plugin->type();
 726
 727         my $tmplog = "$logdir/$vmtype-$vmid.log";
 728
 729         my $lt = localtime();
 730
 731         my $bkname = "vzdump-$vmtype-$vmid";
 732         my $basename = sprintf "${bkname}-%04d_%02d_%02d-%02d_%02d_%02d",
 733         $lt->year + 1900, $lt->mon + 1, $lt->mday,
 734         $lt->hour, $lt->min, $lt->sec;
 735
 736         my $maxfiles = $opts->{maxfiles};
 737
 738         if ($maxfiles && !$opts->{remove}) {
 739             my $bklist = get_backup_file_list($opts->{dumpdir}, $bkname);
 740             die "only $maxfiles backup(s) allowed - please consider to remove old backup files.\n"
 741                 if scalar(@$bklist) >= $maxfiles;
 742         }
 743
 744         my $logfile = $task->{logfile} = "$opts->{dumpdir}/$basename.log";
 745
 746         my $ext = $vmtype eq 'qemu' ? '.vma' : '.tar';
 747         my ($comp, $comp_ext) = compressor_info($opts->{compress});
 748         if ($comp && $comp_ext) {
 749             $ext .= ".${comp_ext}";
 750         }
 751
 752         if ($opts->{stdout}) {
 753             $task->{tarfile} = '-';
 754         } else {
 755             my $tarfile = $task->{tarfile} = "$opts->{dumpdir}/$basename$ext";
 756             $task->{tmptar} = $task->{tarfile};
 757             $task->{tmptar} =~ s/\.[^\.]+$/\.dat/;
 758             unlink $task->{tmptar};
 759         }
 760
 761         $task->{vmtype} = $vmtype;
 762
 763         if ($opts->{tmpdir}) {
 764             $task->{tmpdir} = "$opts->{tmpdir}/vzdumptmp$$";
 765         } else {
 766             # dumpdir is posix? then use it as temporary dir
 767             my $info = get_mount_info($opts->{dumpdir});
 768             if ($vmtype eq 'qemu' ||
 769                 grep ($_ eq $info->{fstype}, @posix_filesystems)) {
 770                 $task->{tmpdir} = "$opts->{dumpdir}/$basename.tmp";
 771             } else {
 772                 $task->{tmpdir} = "/var/tmp/vzdumptmp$$";
 773                 debugmsg ('info', "filesystem type on dumpdir is '$info->{fstype}' -" .
 774                           "using $task->{tmpdir} for temporary files", $logfd);
 775             }
 776         }
 777
 778         rmtree $task->{tmpdir};
 779         mkdir $task->{tmpdir};
 780         -d $task->{tmpdir} ||
 781             die "unable to create temporary directory '$task->{tmpdir}'";
 782
 783         $logfd = IO::File->new (">$tmplog") ||
 784             die "unable to create log file '$tmplog'";
 785
 786         $task->{dumpdir} = $opts->{dumpdir};
 787         $task->{storeid} = $opts->{storage};
 788         $task->{tmplog} = $tmplog;
 789
 790         unlink $logfile;
 791
 792         debugmsg ('info',  "Starting Backup of VM $vmid ($vmtype)", $logfd, 1);
 793
 794         $plugin->set_logfd ($logfd);
 795
 796         # test is VM is running
 797         my ($running, $status_text) = $plugin->vm_status ($vmid);
 798
 799         debugmsg ('info', "status = ${status_text}", $logfd);
 800
 801         # lock VM (prevent config changes)
 802         $plugin->lock_vm ($vmid);
 803
 804         $cleanup->{unlock} = 1;
 805
 806         # prepare
 807
 808         my $mode = $running ? $opts->{mode} : 'stop';
 809
 810         if ($mode eq 'snapshot') {
 811             my %saved_task = %$task;
 812             eval { $plugin->prepare ($task, $vmid, $mode); };
 813             if (my $err = $@) {
 814                 die $err if $err !~ m/^mode failure/;
 815                 debugmsg ('info',  $err, $logfd);
 816                 debugmsg ('info',  "trying 'suspend' mode instead", $logfd);
 817                 $mode = 'suspend'; # so prepare is called again below
 818                 %$task = %saved_task;
 819             }
 820         }
 821
 822         $task->{mode} = $mode;
 823
 824         debugmsg ('info', "backup mode: $mode", $logfd);
 825
 826         debugmsg ('info', "bandwidth limit: $opts->{bwlimit} KB/s", $logfd)
 827             if $opts->{bwlimit};
 828
 829         debugmsg ('info', "ionice priority: $opts->{ionice}", $logfd);
 830
 831         if ($mode eq 'stop') {
 832
 833             $plugin->prepare ($task, $vmid, $mode);
 834
 835             $self->run_hook_script ('backup-start', $task, $logfd);
 836
 837             if ($running) {
 838                 debugmsg ('info', "stopping vm", $logfd);
 839                 $vmstoptime = time ();
 840                 $self->run_hook_script ('pre-stop', $task, $logfd);
 841                 $plugin->stop_vm ($task, $vmid);
 842                 $cleanup->{restart} = 1;
 843             }
 844
 845
 846         } elsif ($mode eq 'suspend') {
 847
 848             $plugin->prepare ($task, $vmid, $mode);
 849
 850             $self->run_hook_script ('backup-start', $task, $logfd);
 851
 852             debugmsg ('info', "suspend vm", $logfd);
 853             $vmstoptime = time ();
 854             $self->run_hook_script ('pre-stop', $task, $logfd);
 855             $plugin->suspend_vm ($task, $vmid);
 856             $cleanup->{resume} = 1;
 857
 858         } elsif ($mode eq 'snapshot') {
 859
 860             $self->run_hook_script ('backup-start', $task, $logfd);
 861
 862             my $snapshot_count = $task->{snapshot_count} || 0;
 863
 864             $self->run_hook_script ('pre-stop', $task, $logfd);
 865
 866             if ($snapshot_count > 1) {
 867                 debugmsg ('info', "suspend vm to make snapshot", $logfd);
 868                 $vmstoptime = time ();
 869                 $plugin->suspend_vm ($task, $vmid);
 870                 $cleanup->{resume} = 1;
 871             }
 872
 873             $plugin->snapshot ($task, $vmid);
 874
 875             $self->run_hook_script ('pre-restart', $task, $logfd);
 876
 877             if ($snapshot_count > 1) {
 878                 debugmsg ('info', "resume vm", $logfd);
 879                 $cleanup->{resume} = 0;
 880                 $plugin->resume_vm ($task, $vmid);
 881                 my $delay = time () - $vmstoptime;
 882                 debugmsg ('info', "vm is online again after $delay seconds", $logfd);
 883             }
 884
 885         } else {
 886             die "internal error - unknown mode '$mode'\n";
 887         }
 888
 889         # assemble archive image
 890         $plugin->assemble ($task, $vmid);
 891
 892         # produce archive
 893
 894         if ($opts->{stdout}) {
 895             debugmsg ('info', "sending archive to stdout", $logfd);
 896             $plugin->archive($task, $vmid, $task->{tmptar}, $comp);
 897             $self->run_hook_script ('backup-end', $task, $logfd);
 898             return;
 899         }
 900
 901         debugmsg ('info', "creating archive '$task->{tarfile}'", $logfd);
 902         $plugin->archive($task, $vmid, $task->{tmptar}, $comp);
 903
 904         rename ($task->{tmptar}, $task->{tarfile}) ||
 905             die "unable to rename '$task->{tmptar}' to '$task->{tarfile}'\n";
 906
 907         # determine size
 908         $task->{size} = (-s $task->{tarfile}) || 0;
 909         my $cs = format_size ($task->{size});
 910         debugmsg ('info', "archive file size: $cs", $logfd);
 911
 912         # purge older backup
 913
 914         if ($maxfiles && $opts->{remove}) {
 915             my $bklist = get_backup_file_list($opts->{dumpdir}, $bkname, $task->{tarfile});
 916             $bklist = [ sort { $b->[1] <=> $a->[1] } @$bklist ];
 917
 918             while (scalar (@$bklist) >= $maxfiles) {
 919                 my $d = pop @$bklist;
 920                 debugmsg ('info', "delete old backup '$d->[0]'", $logfd);
 921                 unlink $d->[0];
 922                 my $logfn = $d->[0];
 923                 $logfn =~ s/\.(tgz|((tar|vma)(\.(gz|lzo))?))$/\.log/;
 924                 unlink $logfn;
 925             }
 926         }
 927
 928         $self->run_hook_script ('backup-end', $task, $logfd);
 929     };
 930     my $err = $@;
 931
 932     if ($plugin) {
 933         # clean-up
 934
 935         if ($cleanup->{unlock}) {
 936             eval { $plugin->unlock_vm ($vmid); };
 937             warn $@ if $@;
 938         }
 939
 940         eval { $plugin->cleanup ($task, $vmid) };
 941         warn $@ if $@;
 942
 943         eval { $plugin->set_logfd (undef); };
 944         warn $@ if $@;
 945
 946         if ($cleanup->{resume} || $cleanup->{restart}) {
 947             eval {
 948                 $self->run_hook_script ('pre-restart', $task, $logfd);
 949                 if ($cleanup->{resume}) {
 950                     debugmsg ('info', "resume vm", $logfd);
 951                     $plugin->resume_vm ($task, $vmid);
 952                 } else {
 953                     my $running = $plugin->vm_status($vmid);
 954                     if (!$running) {
 955                         debugmsg ('info', "restarting vm", $logfd);
 956                         $plugin->start_vm ($task, $vmid);
 957                     }
 958                 }
 959             };
 960             my $err = $@;
 961             if ($err) {
 962                 warn $err;
 963             } else {
 964                 my $delay = time () - $vmstoptime;
 965                 debugmsg ('info', "vm is online again after $delay seconds", $logfd);
 966             }
 967         }
 968     }
 969
 970     eval { unlink $task->{tmptar} if $task->{tmptar} && -f $task->{tmptar}; };
 971     warn $@ if $@;
 972
 973     eval { rmtree $task->{tmpdir} if $task->{tmpdir} && -d $task->{tmpdir}; };
 974     warn $@ if $@;
 975
 976     my $delay = $task->{backuptime} = time () - $vmstarttime;
 977
 978     if ($err) {
 979         $task->{state} = 'err';
 980         $task->{msg} = $err;
 981         debugmsg ('err', "Backup of VM $vmid failed - $err", $logfd, 1);
 982
 983         eval { $self->run_hook_script ('backup-abort', $task, $logfd); };
 984
 985     } else {
 986         $task->{state} = 'ok';
 987         my $tstr = format_time ($delay);
 988         debugmsg ('info', "Finished Backup of VM $vmid ($tstr)", $logfd, 1);
 989     }
 990
 991     close ($logfd) if $logfd;
 992
 993     if ($task->{tmplog} && $task->{logfile}) {
 994         system ("cp '$task->{tmplog}' '$task->{logfile}'");
 995     }
 996
 997     eval { $self->run_hook_script ('log-end', $task); };
 998
 999     die $err if $err && $err =~ m/^interrupted by signal$/;
1000 }
1001
1002 sub exec_backup {
1003     my ($self, $rpcenv, $authuser) = @_;
1004
1005     my $opts = $self->{opts};
1006
1007     debugmsg ('info', "starting new backup job: $self->{cmdline}", undef, 1);
1008     debugmsg ('info', "skip external VMs: " . join(', ', @{$self->{skiplist}}))
1009         if scalar(@{$self->{skiplist}});
1010
1011     my $tasklist = [];
1012
1013     if ($opts->{all}) {
1014         foreach my $plugin (@{$self->{plugins}}) {
1015             my $vmlist = $plugin->vmlist();
1016             foreach my $vmid (sort @$vmlist) {
1017                 next if grep { $_ eq  $vmid } @{$opts->{exclude}};
1018                 next if !$rpcenv->check($authuser, "/vms/$vmid", [ 'VM.Backup' ], 1);
1019                 push @$tasklist, { vmid => $vmid,  state => 'todo', plugin => $plugin };
1020             }
1021         }
1022     } else {
1023         foreach my $vmid (sort @{$opts->{vmids}}) {
1024             my $plugin;
1025             foreach my $pg (@{$self->{plugins}}) {
1026                 my $vmlist = $pg->vmlist();
1027                 if (grep { $_ eq  $vmid } @$vmlist) {
1028                     $plugin = $pg;
1029                     last;
1030                 }
1031             }
1032             $rpcenv->check($authuser, "/vms/$vmid", [ 'VM.Backup' ]);
1033             push @$tasklist, { vmid => $vmid,  state => 'todo', plugin => $plugin };
1034         }
1035     }
1036
1037     my $starttime = time();
1038     my $errcount = 0;
1039     eval {
1040
1041         $self->run_hook_script ('job-start');
1042
1043         foreach my $task (@$tasklist) {
1044             $self->exec_backup_task ($task);
1045             $errcount += 1 if $task->{state} ne 'ok';
1046         }
1047
1048         $self->run_hook_script ('job-end');
1049     };
1050     my $err = $@;
1051
1052     $self->run_hook_script ('job-abort') if $err;
1053
1054     if ($err) {
1055         debugmsg ('err', "Backup job failed - $err", undef, 1);
1056     } else {
1057         if ($errcount) {
1058             debugmsg ('info', "Backup job finished with errors", undef, 1);
1059         } else {
1060             debugmsg ('info', "Backup job finished successfully", undef, 1);
1061         }
1062     }
1063
1064     my $totaltime = time() - $starttime;
1065
1066     eval { $self->sendmail ($tasklist, $totaltime); };
1067     debugmsg ('err', $@) if $@;
1068
1069     die $err if $err;
1070
1071     die "job errors\n" if $errcount;
1072
1073     unlink $pidfile;
1074 }
1075
1076 my $confdesc = {
1077     vmid => {
1078         type => 'string', format => 'pve-vmid-list',
1079         description => "The ID of the VM you want to backup.",
1080         optional => 1,
1081     },
1082     node => get_standard_option('pve-node', {
1083         description => "Only run if executed on this node.",
1084         optional => 1,
1085     }),
1086     all => {
1087         type => 'boolean',
1088         description => "Backup all known VMs on this host.",
1089         optional => 1,
1090         default => 0,
1091     },
1092     stdexcludes => {
1093         type => 'boolean',
1094         description => "Exclude temorary files and logs.",
1095         optional => 1,
1096         default => 1,
1097     },
1098     compress => {
1099         type => 'string',
1100         description => "Compress dump file.",
1101         optional => 1,
1102         enum => ['0', '1', 'gzip', 'lzo'],
1103         default => 'lzo',
1104     },
1105     quiet => {
1106         type => 'boolean',
1107         description => "Be quiet.",
1108         optional => 1,
1109         default => 0,
1110     },
1111     mode => {
1112         type => 'string',
1113         description => "Backup mode.",
1114         optional => 1,
1115         default => 'stop',
1116         enum => [ 'snapshot', 'suspend', 'stop' ],
1117     },
1118     exclude => {
1119         type => 'string', format => 'pve-vmid-list',
1120         description => "exclude specified VMs (assumes --all)",
1121         optional => 1,
1122     },
1123     'exclude-path' => {
1124         type => 'string', format => 'string-alist',
1125         description => "exclude certain files/directories (regex).",
1126         optional => 1,
1127     },
1128     mailto => {
1129         type => 'string', format => 'string-list',
1130         description => "",
1131         optional => 1,
1132     },
1133     mailnotification => {
1134         type => 'string',
1135         description => "Specify when to send an email",
1136         optional => 1,
1137         enum => [ 'always', 'failure' ],
1138         default => 'always',
1139     },
1140     tmpdir => {
1141         type => 'string',
1142         description => "Store temporary files to specified directory.",
1143         optional => 1,
1144     },
1145     dumpdir => {
1146         type => 'string',
1147         description => "Store resulting files to specified directory.",
1148         optional => 1,
1149     },
1150     script => {
1151         type => 'string',
1152         description => "Use specified hook script.",
1153         optional => 1,
1154     },
1155     storage => get_standard_option('pve-storage-id', {
1156         description => "Store resulting file to this storage.",
1157         optional => 1,
1158     }),
1159     stop => {
1160         type => 'boolean',
1161         description => "Stop runnig backup jobs on this host.",
1162         optional => 1,
1163         default => 0,
1164     },
1165     size => {
1166         type => 'integer',
1167         description => "LVM snapshot size in MB.",
1168         optional => 1,
1169         minimum => 500,
1170     },
1171     bwlimit => {
1172         type => 'integer',
1173         description => "Limit I/O bandwidth (KBytes per second).",
1174         optional => 1,
1175         minimum => 0,
1176     },
1177     ionice => {
1178         type => 'integer',
1179         description => "Set CFQ ionice priority.",
1180         optional => 1,
1181         minimum => 0,
1182         maximum => 8,
1183     },
1184     lockwait => {
1185         type => 'integer',
1186         description => "Maximal time to wait for the global lock (minutes).",
1187         optional => 1,
1188         minimum => 0,
1189     },
1190     stopwait => {
1191         type => 'integer',
1192         description => "Maximal time to wait until a VM is stopped (minutes).",
1193         optional => 1,
1194         minimum => 0,
1195     },
1196     maxfiles => {
1197         type => 'integer',
1198         description => "Maximal number of backup files per VM.",
1199         optional => 1,
1200         minimum => 1,
1201     },
1202     remove => {
1203         type => 'boolean',
1204         description => "Remove old backup files if there are more than 'maxfiles' backup files.",
1205         optional => 1,
1206         default => 1,
1207     },
1208 };
1209
1210 sub option_exists {
1211     my $key = shift;
1212     return defined($confdesc->{$key});
1213 }
1214
1215 # add JSON properties for create and set function
1216 sub json_config_properties {
1217     my $prop = shift;
1218
1219     foreach my $opt (keys %$confdesc) {
1220         $prop->{$opt} = $confdesc->{$opt};
1221     }
1222
1223     return $prop;
1224 }
1225
1226 sub verify_vzdump_parameters {
1227     my ($param, $check_missing) = @_;
1228
1229     raise_param_exc({ all => "option conflicts with option 'vmid'"})
1230         if $param->{all} && $param->{vmid};
1231
1232     raise_param_exc({ exclude => "option conflicts with option 'vmid'"})
1233         if $param->{exclude} && $param->{vmid};
1234
1235     $param->{all} = 1 if defined($param->{exclude});
1236
1237     return if !$check_missing;
1238
1239     raise_param_exc({ vmid => "property is missing"})
1240         if !($param->{all} || $param->{stop}) && !$param->{vmid};
1241
1242 }
1243
1244 sub stop_running_backups {
1245     my($self) = @_;
1246
1247     my $upid = PVE::Tools::file_read_firstline($pidfile);
1248     return if !$upid;
1249
1250     my $task = PVE::Tools::upid_decode($upid);
1251
1252     if (PVE::ProcFSTools::check_process_running($task->{pid}, $task->{pstart}) &&
1253         PVE::ProcFSTools::read_proc_starttime($task->{pid}) == $task->{pstart}) {
1254         kill(15, $task->{pid});
1255         # wait max 15 seconds to shut down (else, do nothing for now)
1256         my $i;
1257         for ($i = 15; $i > 0; $i--) {
1258             last if !PVE::ProcFSTools::check_process_running(($task->{pid}, $task->{pstart}));
1259             sleep (1);
1260         }
1261         die "stoping backup process $task->{pid} failed\n" if $i == 0;
1262     }
1263 }
1264
1265 sub command_line {
1266     my ($param) = @_;
1267
1268     my $cmd = "vzdump";
1269
1270     if ($param->{vmid}) {
1271         $cmd .= " " . join(' ', PVE::Tools::split_list($param->{vmid}));
1272     }
1273
1274     foreach my $p (keys %$param) {
1275         next if $p eq 'id' || $p eq 'vmid' || $p eq 'starttime' || $p eq 'dow' || $p eq 'stdout';
1276         my $v = $param->{$p};
1277         my $pd = $confdesc->{$p} || die "no such vzdump option '$p'\n";
1278         if ($p eq 'exclude-path') {
1279             foreach my $path (split(/\0/, $v || '')) {
1280                 $cmd .= " --$p " . PVE::Tools::shellquote($path);
1281             }
1282         } else {
1283             $cmd .= " --$p " . PVE::Tools::shellquote($v) if defined($v) && $v ne '';
1284         }
1285     }
1286
1287     return $cmd;
1288 }
1289
1290 1;