PVE/VZDump.pm

   1 package PVE::VZDump;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Fcntl ':flock';
   7 use File::Path;
   8 use IO::File;
   9 use IO::Select;
  10 use IPC::Open3;
  11 use POSIX qw(strftime);
  12 use Time::Local;
  13
  14 use PVE::Cluster qw(cfs_read_file);
  15 use PVE::DataCenterConfig;
  16 use PVE::Exception qw(raise_param_exc);
  17 use PVE::HA::Config;
  18 use PVE::HA::Env::PVE2;
  19 use PVE::JSONSchema qw(get_standard_option);
  20 use PVE::RPCEnvironment;
  21 use PVE::Storage;
  22 use PVE::VZDump::Common;
  23 use PVE::VZDump::Plugin;
  24 use PVE::Tools qw(extract_param split_list);
  25 use PVE::API2Tools;
  26
  27 my @posix_filesystems = qw(ext3 ext4 nfs nfs4 reiserfs xfs);
  28
  29 my $lockfile = '/var/run/vzdump.lock';
  30 my $pidfile = '/var/run/vzdump.pid';
  31 my $logdir = '/var/log/vzdump';
  32
  33 my @plugins = qw();
  34
  35 my $confdesc = PVE::VZDump::Common::get_confdesc();
  36
  37 # Load available plugins
  38 my @pve_vzdump_classes = qw(PVE::VZDump::QemuServer PVE::VZDump::LXC);
  39 foreach my $plug (@pve_vzdump_classes) {
  40     my $filename = "/usr/share/perl5/$plug.pm";
  41     $filename =~ s!::!/!g;
  42     if (-f $filename) {
  43         eval { require $filename; };
  44         if (!$@) {
  45             $plug->import ();
  46             push @plugins, $plug;
  47         } else {
  48             die $@;
  49         }
  50     }
  51 }
  52
  53 # helper functions
  54
  55 sub debugmsg {
  56     my ($mtype, $msg, $logfd, $syslog) = @_;
  57
  58     PVE::VZDump::Plugin::debugmsg(@_);
  59 }
  60
  61 sub run_command {
  62     my ($logfd, $cmdstr, %param) = @_;
  63
  64     my $logfunc = sub {
  65         my $line = shift;
  66         debugmsg ('info', $line, $logfd);
  67     };
  68
  69     PVE::Tools::run_command($cmdstr, %param, logfunc => $logfunc);
  70 }
  71
  72 my $parse_prune_backups_maxfiles = sub {
  73     my ($param, $kind) = @_;
  74
  75     my $maxfiles = delete $param->{maxfiles};
  76     my $prune_backups = $param->{'prune-backups'};
  77
  78     debugmsg('warn', "both 'maxfiles' and 'prune-backups' defined as ${kind} - ignoring 'maxfiles'")
  79         if defined($maxfiles) && defined($prune_backups);
  80
  81     if (defined($prune_backups)) {
  82         return if ref($prune_backups) eq 'HASH'; # already parsed
  83         $param->{'prune-backups'} = PVE::JSONSchema::parse_property_string(
  84             'prune-backups',
  85             $prune_backups
  86         );
  87     } elsif (defined($maxfiles)) {
  88         if ($maxfiles) {
  89             $param->{'prune-backups'} = { 'keep-last' => $maxfiles };
  90         } else {
  91             $param->{'prune-backups'} = { 'keep-all' => 1 };
  92         }
  93     }
  94 };
  95
  96 sub storage_info {
  97     my $storage = shift;
  98
  99     my $cfg = PVE::Storage::config();
 100     my $scfg = PVE::Storage::storage_config($cfg, $storage);
 101     my $type = $scfg->{type};
 102
 103     die "can't use storage type '$type' for backup\n"
 104         if (!($type eq 'dir' || $type eq 'nfs' || $type eq 'glusterfs'
 105               || $type eq 'cifs' || $type eq 'cephfs' || $type eq 'pbs'));
 106     die "can't use storage '$storage' for backups - wrong content type\n"
 107         if (!$scfg->{content}->{backup});
 108
 109     my $info = {
 110         scfg => $scfg,
 111     };
 112
 113     $info->{'prune-backups'} = PVE::JSONSchema::parse_property_string('prune-backups', $scfg->{'prune-backups'})
 114         if defined($scfg->{'prune-backups'});
 115
 116     if ($type eq 'pbs') {
 117         $info->{pbs} = 1;
 118     } else {
 119         $info->{dumpdir} = PVE::Storage::get_backup_dir($cfg, $storage);
 120     }
 121
 122     return $info;
 123 }
 124
 125 sub format_size {
 126     my $size = shift;
 127
 128     my $kb = $size / 1024;
 129
 130     if ($kb < 1024) {
 131         return int ($kb) . "KB";
 132     }
 133
 134     my $mb = $size / (1024*1024);
 135     if ($mb < 1024) {
 136         return int ($mb) . "MB";
 137     }
 138     my $gb = $mb / 1024;
 139     if ($gb < 1024) {
 140         return sprintf ("%.2fGB", $gb);
 141     }
 142     my $tb = $gb / 1024;
 143     return sprintf ("%.2fTB", $tb);
 144 }
 145
 146 sub format_time {
 147     my $seconds = shift;
 148
 149     my $hours = int ($seconds/3600);
 150     $seconds = $seconds - $hours*3600;
 151     my $min = int ($seconds/60);
 152     $seconds = $seconds - $min*60;
 153
 154     return sprintf ("%02d:%02d:%02d", $hours, $min, $seconds);
 155 }
 156
 157 sub encode8bit {
 158     my ($str) = @_;
 159
 160     $str =~ s/^(.{990})/$1\n/mg; # reduce line length
 161
 162     return $str;
 163 }
 164
 165 sub escape_html {
 166     my ($str) = @_;
 167
 168     $str =~ s/&/&amp;/g;
 169     $str =~ s/</&lt;/g;
 170     $str =~ s/>/&gt;/g;
 171
 172     return $str;
 173 }
 174
 175 sub check_bin {
 176     my ($bin)  = @_;
 177
 178     foreach my $p (split (/:/, $ENV{PATH})) {
 179         my $fn = "$p/$bin";
 180         if (-x $fn) {
 181             return $fn;
 182         }
 183     }
 184
 185     die "unable to find command '$bin'\n";
 186 }
 187
 188 sub check_vmids {
 189     my (@vmids) = @_;
 190
 191     my $res = [];
 192     for my $vmid (sort {$a <=> $b} @vmids) {
 193         die "ERROR: strange VM ID '${vmid}'\n" if $vmid !~ m/^\d+$/;
 194         $vmid = int ($vmid); # remove leading zeros
 195         next if !$vmid;
 196         push @$res, $vmid;
 197     }
 198
 199     return $res;
 200 }
 201
 202
 203 sub read_vzdump_defaults {
 204
 205     my $fn = "/etc/vzdump.conf";
 206
 207     my $defaults = {
 208         map {
 209             my $default = $confdesc->{$_}->{default};
 210              defined($default) ? ($_ => $default) : ()
 211         } keys %$confdesc
 212     };
 213     $parse_prune_backups_maxfiles->($defaults, "defaults in VZDump schema");
 214
 215     my $raw;
 216     eval { $raw = PVE::Tools::file_get_contents($fn); };
 217     return $defaults if $@;
 218
 219     my $conf_schema = { type => 'object', properties => $confdesc, };
 220     my $res = PVE::JSONSchema::parse_config($conf_schema, $fn, $raw);
 221     if (my $excludes = $res->{'exclude-path'}) {
 222         $res->{'exclude-path'} = PVE::Tools::split_args($excludes);
 223     }
 224     if (defined($res->{mailto})) {
 225         my @mailto = split_list($res->{mailto});
 226         $res->{mailto} = [ @mailto ];
 227     }
 228     $parse_prune_backups_maxfiles->($res, "options in '$fn'");
 229
 230     foreach my $key (keys %$defaults) {
 231         $res->{$key} = $defaults->{$key} if !defined($res->{$key});
 232     }
 233
 234     if (defined($res->{storage}) && defined($res->{dumpdir})) {
 235         debugmsg('warn', "both 'storage' and 'dumpdir' defined in '$fn' - ignoring 'dumpdir'");
 236         delete $res->{dumpdir};
 237     }
 238
 239     return $res;
 240 }
 241
 242 use constant MAX_MAIL_SIZE => 1024*1024;
 243 sub sendmail {
 244     my ($self, $tasklist, $totaltime, $err, $detail_pre, $detail_post) = @_;
 245
 246     my $opts = $self->{opts};
 247
 248     my $mailto = $opts->{mailto};
 249
 250     return if !($mailto && scalar(@$mailto));
 251
 252     my $cmdline = $self->{cmdline};
 253
 254     my $ecount = 0;
 255     foreach my $task (@$tasklist) {
 256         $ecount++ if $task->{state} ne 'ok';
 257         chomp $task->{msg} if $task->{msg};
 258         $task->{backuptime} = 0 if !$task->{backuptime};
 259         $task->{size} = 0 if !$task->{size};
 260         $task->{target} = 'unknown' if !$task->{target};
 261         $task->{hostname} = "VM $task->{vmid}" if !$task->{hostname};
 262
 263         if ($task->{state} eq 'todo') {
 264             $task->{msg} = 'aborted';
 265         }
 266     }
 267
 268     my $notify = $opts->{mailnotification} || 'always';
 269     return if (!$ecount && !$err && ($notify eq 'failure'));
 270
 271     my $stat = ($ecount || $err) ? 'backup failed' : 'backup successful';
 272     if ($err) {
 273         if ($err =~ /\n/) {
 274             $stat .= ": multiple problems";
 275         } else {
 276             $stat .= ": $err";
 277             $err = undef;
 278         }
 279     }
 280
 281     my $hostname = `hostname -f` || PVE::INotify::nodename();
 282     chomp $hostname;
 283
 284     # text part
 285     my $text = $err ? "$err\n\n" : '';
 286     my $namelength = 20;
 287     $text .= sprintf (
 288         "%-10s %-${namelength}s %-6s %10s %10s  %s\n",
 289         qw(VMID NAME STATUS TIME SIZE FILENAME)
 290     );
 291     foreach my $task (@$tasklist) {
 292         my $name = substr($task->{hostname}, 0, $namelength);
 293         my $successful = $task->{state} eq 'ok';
 294         my $size = $successful ? format_size ($task->{size}) : 0;
 295         my $filename = $successful ? $task->{target} : '-';
 296         my $size_fmt = $successful ? "%10s": "%8.2fMB";
 297         $text .= sprintf(
 298             "%-10s %-${namelength}s %-6s %10s $size_fmt  %s\n",
 299             $task->{vmid},
 300             $name,
 301             $task->{state},
 302             format_time($task->{backuptime}),
 303             $size,
 304             $filename,
 305         );
 306     }
 307
 308     my $text_log_part;
 309     $text_log_part .= "\nDetailed backup logs:\n\n";
 310     $text_log_part .= "$cmdline\n\n";
 311
 312     $text_log_part .= $detail_pre . "\n" if defined($detail_pre);
 313     foreach my $task (@$tasklist) {
 314         my $vmid = $task->{vmid};
 315         my $log = $task->{tmplog};
 316         if (!$log) {
 317             $text_log_part .= "$vmid: no log available\n\n";
 318             next;
 319         }
 320         if (open (my $TMP, '<', "$log")) {
 321             while (my $line = <$TMP>) {
 322                 next if $line =~ /^status: \d+/; # not useful in mails
 323                 $text_log_part .= encode8bit ("$vmid: $line");
 324             }
 325             close ($TMP);
 326         } else {
 327             $text_log_part .= "$vmid: Could not open log file\n\n";
 328         }
 329         $text_log_part .= "\n";
 330     }
 331     $text_log_part .= $detail_post if defined($detail_post);
 332
 333     # html part
 334     my $html = "<html><body>\n";
 335     $html .= "<p>" . (escape_html($err) =~ s/\n/<br>/gr) . "</p>\n" if $err;
 336     $html .= "<table border=1 cellpadding=3>\n";
 337     $html .= "<tr><td>VMID<td>NAME<td>STATUS<td>TIME<td>SIZE<td>FILENAME</tr>\n";
 338
 339     my $ssize = 0;
 340     foreach my $task (@$tasklist) {
 341         my $vmid = $task->{vmid};
 342         my $name = $task->{hostname};
 343
 344         if  ($task->{state} eq 'ok') {
 345             $ssize += $task->{size};
 346
 347             $html .= sprintf (
 348                 "<tr><td>%s<td>%s<td>OK<td>%s<td align=right>%s<td>%s</tr>\n",
 349                 $vmid,
 350                 $name,
 351                 format_time($task->{backuptime}),
 352                 format_size ($task->{size}),
 353                 escape_html ($task->{target}),
 354             );
 355         } else {
 356             $html .= sprintf (
 357                 "<tr><td>%s<td>%s<td><font color=red>FAILED<td>%s<td colspan=2>%s</tr>\n",
 358                 $vmid,
 359                 $name,
 360                 format_time($task->{backuptime}),
 361                 escape_html ($task->{msg}),
 362             );
 363         }
 364     }
 365
 366     $html .= sprintf ("<tr><td align=left colspan=3>TOTAL<td>%s<td>%s<td></tr>",
 367  format_time ($totaltime), format_size ($ssize));
 368
 369     $html .= "\n</table><br><br>\n";
 370     my $html_log_part;
 371     $html_log_part .= "Detailed backup logs:<br /><br />\n";
 372     $html_log_part .= "<pre>\n";
 373     $html_log_part .= escape_html($cmdline) . "\n\n";
 374
 375     $html_log_part .= escape_html($detail_pre) . "\n" if defined($detail_pre);
 376     foreach my $task (@$tasklist) {
 377         my $vmid = $task->{vmid};
 378         my $log = $task->{tmplog};
 379         if (!$log) {
 380             $html_log_part .= "$vmid: no log available\n\n";
 381             next;
 382         }
 383         if (open (my $TMP, '<', "$log")) {
 384             while (my $line = <$TMP>) {
 385                 next if $line =~ /^status: \d+/; # not useful in mails
 386                 if ($line =~ m/^\S+\s\d+\s+\d+:\d+:\d+\s+(ERROR|WARN):/) {
 387                     $html_log_part .= encode8bit ("$vmid: <font color=red>".
 388                         escape_html ($line) . "</font>");
 389                 } else {
 390                     $html_log_part .= encode8bit ("$vmid: " . escape_html ($line));
 391                 }
 392             }
 393             close ($TMP);
 394         } else {
 395             $html_log_part .= "$vmid: Could not open log file\n\n";
 396         }
 397         $html_log_part .= "\n";
 398     }
 399     $html_log_part .= escape_html($detail_post) if defined($detail_post);
 400     $html_log_part .= "</pre>";
 401     my $html_end = "\n</body></html>\n";
 402     # end html part
 403
 404     if (length($text) + length($text_log_part) +
 405         length($html) + length($html_log_part) +
 406         length($html_end) < MAX_MAIL_SIZE)
 407     {
 408         $html .= $html_log_part;
 409         $html .= $html_end;
 410         $text .= $text_log_part;
 411     } else {
 412         my $msg = "Log output was too long to be sent by mail. ".
 413             "See Task History for details!\n";
 414         $text .= $msg;
 415         $html .= "<p>$msg</p>";
 416         $html .= $html_end;
 417     }
 418
 419     my $subject = "vzdump backup status ($hostname) : $stat";
 420
 421     my $dcconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
 422     my $mailfrom = $dcconf->{email_from} || "root";
 423
 424     PVE::Tools::sendmail($mailto, $subject, $text, $html, $mailfrom, "vzdump backup tool");
 425 };
 426
 427 sub new {
 428     my ($class, $cmdline, $opts, $skiplist) = @_;
 429
 430     mkpath $logdir;
 431
 432     check_bin ('cp');
 433     check_bin ('df');
 434     check_bin ('sendmail');
 435     check_bin ('rsync');
 436     check_bin ('tar');
 437     check_bin ('mount');
 438     check_bin ('umount');
 439     check_bin ('cstream');
 440     check_bin ('ionice');
 441
 442     if ($opts->{mode} && $opts->{mode} eq 'snapshot') {
 443         check_bin ('lvcreate');
 444         check_bin ('lvs');
 445         check_bin ('lvremove');
 446     }
 447
 448     my $defaults = read_vzdump_defaults();
 449
 450     foreach my $k (keys %$defaults) {
 451         next if $k eq 'exclude-path' || $k eq 'prune-backups'; # dealt with separately
 452         if ($k eq 'dumpdir' || $k eq 'storage') {
 453             $opts->{$k} = $defaults->{$k} if !defined ($opts->{dumpdir}) &&
 454                 !defined ($opts->{storage});
 455         } else {
 456             $opts->{$k} = $defaults->{$k} if !defined ($opts->{$k});
 457         }
 458     }
 459
 460     $opts->{dumpdir} =~ s|/+$|| if ($opts->{dumpdir});
 461     $opts->{tmpdir} =~ s|/+$|| if ($opts->{tmpdir});
 462
 463     $skiplist = [] if !$skiplist;
 464     my $self = bless {
 465         cmdline => $cmdline,
 466         opts => $opts,
 467         skiplist => $skiplist,
 468     }, $class;
 469
 470     my $findexcl = $self->{findexcl} = [];
 471     if ($defaults->{'exclude-path'}) {
 472         push @$findexcl, @{$defaults->{'exclude-path'}};
 473     }
 474
 475     if ($opts->{'exclude-path'}) {
 476         push @$findexcl, @{$opts->{'exclude-path'}};
 477     }
 478
 479     if ($opts->{stdexcludes}) {
 480         push @$findexcl,
 481             '/tmp/?*',
 482             '/var/tmp/?*',
 483             '/var/run/?*.pid',
 484             ;
 485     }
 486
 487     foreach my $p (@plugins) {
 488         my $pd = $p->new($self);
 489
 490         push @{$self->{plugins}}, $pd;
 491     }
 492
 493     if (defined($opts->{storage}) && $opts->{stdout}) {
 494         die "cannot use options 'storage' and 'stdout' at the same time\n";
 495     } elsif (defined($opts->{storage}) && defined($opts->{dumpdir})) {
 496         die "cannot use options 'storage' and 'dumpdir' at the same time\n";
 497     }
 498
 499     if (!$opts->{dumpdir} && !$opts->{storage}) {
 500         $opts->{storage} = 'local';
 501     }
 502
 503     my $errors = '';
 504
 505     if ($opts->{storage}) {
 506         my $storage_cfg = PVE::Storage::config();
 507         eval { PVE::Storage::activate_storage($storage_cfg, $opts->{storage}) };
 508         if (my $err = $@) {
 509             chomp($err);
 510             $errors .= "could not activate storage '$opts->{storage}': $err";
 511         }
 512
 513         my $info = eval { storage_info ($opts->{storage}) };
 514         if (my $err = $@) {
 515             chomp($err);
 516             $errors .= "could not get storage information for '$opts->{storage}': $err";
 517         } else {
 518             $opts->{dumpdir} = $info->{dumpdir};
 519             $opts->{scfg} = $info->{scfg};
 520             $opts->{pbs} = $info->{pbs};
 521             $opts->{'prune-backups'} //= $info->{'prune-backups'};
 522         }
 523     } elsif ($opts->{dumpdir}) {
 524         $errors .= "dumpdir '$opts->{dumpdir}' does not exist"
 525             if ! -d $opts->{dumpdir};
 526     } else {
 527         die "internal error";
 528     }
 529
 530     $opts->{'prune-backups'} //= $defaults->{'prune-backups'};
 531
 532     # avoid triggering any remove code path if keep-all is set
 533     $opts->{remove} = 0 if $opts->{'prune-backups'}->{'keep-all'};
 534
 535     if ($opts->{tmpdir} && ! -d $opts->{tmpdir}) {
 536         $errors .= "\n" if $errors;
 537         $errors .= "tmpdir '$opts->{tmpdir}' does not exist";
 538     }
 539
 540     if ($errors) {
 541         eval { $self->sendmail([], 0, $errors); };
 542         debugmsg ('err', $@) if $@;
 543         die "$errors\n";
 544     }
 545
 546     return $self;
 547 }
 548
 549 sub get_mount_info {
 550     my ($dir) = @_;
 551
 552     # Note: df 'available' can be negative, and percentage set to '-'
 553
 554     my $cmd = [ 'df', '-P', '-T', '-B', '1', $dir];
 555
 556     my $res;
 557
 558     my $parser = sub {
 559         my $line = shift;
 560         if (my ($fsid, $fstype, undef, $mp) = $line =~
 561             m!(\S+.*)\s+(\S+)\s+\d+\s+\-?\d+\s+\d+\s+(\d+%|-)\s+(/.*)$!) {
 562             $res = {
 563                 device => $fsid,
 564                 fstype => $fstype,
 565                 mountpoint => $mp,
 566             };
 567         }
 568     };
 569
 570     eval { PVE::Tools::run_command($cmd, errfunc => sub {}, outfunc => $parser); };
 571     warn $@ if $@;
 572
 573     return $res;
 574 }
 575
 576 sub getlock {
 577     my ($self, $upid) = @_;
 578
 579     my $fh;
 580
 581     my $maxwait = $self->{opts}->{lockwait} || $self->{lockwait};
 582
 583     die "missing UPID" if !$upid; # should not happen
 584
 585     my $SERVER_FLCK;
 586     if (!open ($SERVER_FLCK, '>>', "$lockfile")) {
 587         debugmsg ('err', "can't open lock on file '$lockfile' - $!", undef, 1);
 588         die "can't open lock on file '$lockfile' - $!";
 589     }
 590
 591     if (!flock ($SERVER_FLCK, LOCK_EX|LOCK_NB)) {
 592         if (!$maxwait) {
 593             debugmsg ('err', "can't acquire lock '$lockfile' (wait = 0)", undef, 1);
 594             die "can't acquire lock '$lockfile' (wait = 0)";
 595         }
 596
 597         debugmsg('info', "trying to get global lock - waiting...", undef, 1);
 598         eval {
 599             alarm ($maxwait * 60);
 600
 601             local $SIG{ALRM} = sub { alarm (0); die "got timeout\n"; };
 602
 603             if (!flock ($SERVER_FLCK, LOCK_EX)) {
 604                 my $err = $!;
 605                 close ($SERVER_FLCK);
 606                 alarm (0);
 607                 die "$err\n";
 608             }
 609             alarm (0);
 610         };
 611         alarm (0);
 612
 613         my $err = $@;
 614
 615         if ($err) {
 616             debugmsg ('err', "can't acquire lock '$lockfile' - $err", undef, 1);
 617             die "can't acquire lock '$lockfile' - $err";
 618         }
 619
 620         debugmsg('info', "got global lock", undef, 1);
 621     }
 622
 623     PVE::Tools::file_set_contents($pidfile, $upid);
 624 }
 625
 626 sub run_hook_script {
 627     my ($self, $phase, $task, $logfd) = @_;
 628
 629     my $opts = $self->{opts};
 630
 631     my $script = $opts->{script};
 632     return if !$script;
 633
 634     if (!-x $script) {
 635         die "The hook script '$script' is not executable.\n";
 636     }
 637
 638     my $cmd = "$script $phase";
 639
 640     $cmd .= " $task->{mode} $task->{vmid}" if ($task);
 641
 642     local %ENV;
 643     # set immutable opts directly (so they are available in all phases)
 644     $ENV{STOREID} = $opts->{storage} if $opts->{storage};
 645     $ENV{DUMPDIR} = $opts->{dumpdir} if $opts->{dumpdir};
 646
 647     foreach my $ek (qw(vmtype hostname target logfile)) {
 648         $ENV{uc($ek)} = $task->{$ek} if $task->{$ek};
 649     }
 650     # FIXME: for backwards compatibility - drop with PVE 7.0
 651     $ENV{TARFILE} = $task->{target} if $task->{target};
 652
 653     run_command ($logfd, $cmd);
 654 }
 655
 656 sub compressor_info {
 657     my ($opts) = @_;
 658     my $opt_compress = $opts->{compress};
 659
 660     if (!$opt_compress || $opt_compress eq '0') {
 661         return undef;
 662     } elsif ($opt_compress eq '1' || $opt_compress eq 'lzo') {
 663         return ('lzop', 'lzo');
 664     } elsif ($opt_compress eq 'gzip') {
 665         if ($opts->{pigz} > 0) {
 666             my $pigz_threads = $opts->{pigz};
 667             if ($pigz_threads == 1) {
 668                 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
 669                 $pigz_threads = int(($cpuinfo->{cpus} + 1)/2);
 670             }
 671             return ("pigz -p ${pigz_threads} --rsyncable", 'gz');
 672         } else {
 673             return ('gzip --rsyncable', 'gz');
 674         }
 675     } elsif ($opt_compress eq 'zstd') {
 676         my $zstd_threads = $opts->{zstd} // 1;
 677         if ($zstd_threads == 0) {
 678             my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
 679             $zstd_threads = int(($cpuinfo->{cpus} + 1)/2);
 680         }
 681         return ("zstd --rsyncable --threads=${zstd_threads}", 'zst');
 682     } else {
 683         die "internal error - unknown compression option '$opt_compress'";
 684     }
 685 }
 686
 687 sub get_backup_file_list {
 688     my ($dir, $bkname, $exclude_fn) = @_;
 689
 690     my $bklist = [];
 691     foreach my $fn (<$dir/${bkname}-*>) {
 692         next if $exclude_fn && $fn eq $exclude_fn;
 693
 694         my $archive_info = eval { PVE::Storage::archive_info($fn) } // {};
 695         if ($archive_info->{is_std_name}) {
 696             my $filename = $archive_info->{filename};
 697             my $backup = {
 698                 'path' => "$dir/$filename",
 699                 'ctime' => $archive_info->{ctime},
 700             };
 701             push @{$bklist}, $backup;
 702         }
 703     }
 704
 705     return $bklist;
 706 }
 707
 708 sub exec_backup_task {
 709     my ($self, $task) = @_;
 710
 711     my $opts = $self->{opts};
 712
 713     my $cfg = PVE::Storage::config();
 714     my $vmid = $task->{vmid};
 715     my $plugin = $task->{plugin};
 716
 717     $task->{backup_time} = time();
 718
 719     my $pbs_group_name;
 720     my $pbs_snapshot_name;
 721
 722     my $vmstarttime = time ();
 723
 724     my $logfd;
 725
 726     my $cleanup = {};
 727
 728     my $log_vm_online_again = sub {
 729         return if !defined($task->{vmstoptime});
 730         $task->{vmconttime} //= time();
 731         my $delay = $task->{vmconttime} - $task->{vmstoptime};
 732         $delay = '<1' if $delay < 1;
 733         debugmsg ('info', "guest is online again after $delay seconds", $logfd);
 734     };
 735
 736     eval {
 737         die "unable to find VM '$vmid'\n" if !$plugin;
 738
 739         my $vmtype = $plugin->type();
 740
 741         if ($self->{opts}->{pbs}) {
 742             if ($vmtype eq 'lxc') {
 743                 $pbs_group_name = "ct/$vmid";
 744             } elsif  ($vmtype eq 'qemu') {
 745                 $pbs_group_name = "vm/$vmid";
 746             } else {
 747                 die "pbs backup not implemented for plugin type '$vmtype'\n";
 748             }
 749             my $btime = strftime("%FT%TZ", gmtime($task->{backup_time}));
 750             $pbs_snapshot_name = "$pbs_group_name/$btime";
 751         }
 752
 753         # for now we deny backups of a running ha managed service in *stop* mode
 754         # as it interferes with the HA stack (started services should not stop).
 755         if ($opts->{mode} eq 'stop' &&
 756             PVE::HA::Config::vm_is_ha_managed($vmid, 'started'))
 757         {
 758             die "Cannot execute a backup with stop mode on a HA managed and".
 759                 " enabled Service. Use snapshot mode or disable the Service.\n";
 760         }
 761
 762         my $tmplog = "$logdir/$vmtype-$vmid.log";
 763
 764         my $bkname = "vzdump-$vmtype-$vmid";
 765         my $basename = $bkname . strftime("-%Y_%m_%d-%H_%M_%S", localtime($task->{backup_time}));
 766
 767         my $prune_options = $opts->{'prune-backups'};
 768
 769         my $backup_limit = 0;
 770         if (!$prune_options->{'keep-all'}) {
 771             foreach my $keep (values %{$prune_options}) {
 772                 $backup_limit += $keep;
 773             }
 774         }
 775
 776         if ($backup_limit && !$opts->{remove}) {
 777             my $count;
 778             if ($self->{opts}->{pbs}) {
 779                 my $res = PVE::Storage::PBSPlugin::run_client_cmd($opts->{scfg}, $opts->{storage}, 'snapshots', $pbs_group_name);
 780                 $count = scalar(@$res);
 781             } else {
 782                 my $bklist = get_backup_file_list($opts->{dumpdir}, $bkname);
 783                 $count = scalar(@$bklist);
 784             }
 785             die "There is a max backup limit of $backup_limit enforced by the".
 786                 " target storage or the vzdump parameters.".
 787                 " Either increase the limit or delete old backup(s).\n"
 788                 if $count >= $backup_limit;
 789         }
 790
 791         if (!$self->{opts}->{pbs}) {
 792             $task->{logfile} = "$opts->{dumpdir}/$basename.log";
 793         }
 794
 795         my $ext = $vmtype eq 'qemu' ? '.vma' : '.tar';
 796         my ($comp, $comp_ext) = compressor_info($opts);
 797         if ($comp && $comp_ext) {
 798             $ext .= ".${comp_ext}";
 799         }
 800
 801         if ($self->{opts}->{pbs}) {
 802             die "unable to pipe backup to stdout\n" if $opts->{stdout};
 803             $task->{target} = $pbs_snapshot_name;
 804         } else {
 805             if ($opts->{stdout}) {
 806                 $task->{target} = '-';
 807             } else {
 808                 $task->{target} = $task->{tmptar} = "$opts->{dumpdir}/$basename$ext";
 809                 $task->{tmptar} =~ s/\.[^\.]+$/\.dat/;
 810                 unlink $task->{tmptar};
 811             }
 812         }
 813
 814         $task->{vmtype} = $vmtype;
 815
 816         my $pid = $$;
 817         if ($opts->{tmpdir}) {
 818             $task->{tmpdir} = "$opts->{tmpdir}/vzdumptmp${pid}_$vmid/";
 819         } elsif ($self->{opts}->{pbs}) {
 820             $task->{tmpdir} = "/var/tmp/vzdumptmp${pid}_$vmid";
 821         } else {
 822             # dumpdir is posix? then use it as temporary dir
 823             my $info = get_mount_info($opts->{dumpdir});
 824             if ($vmtype eq 'qemu' ||
 825                 grep ($_ eq $info->{fstype}, @posix_filesystems)) {
 826                 $task->{tmpdir} = "$opts->{dumpdir}/$basename.tmp";
 827             } else {
 828                 $task->{tmpdir} = "/var/tmp/vzdumptmp${pid}_$vmid";
 829                 debugmsg ('info', "filesystem type on dumpdir is '$info->{fstype}' -" .
 830                           "using $task->{tmpdir} for temporary files", $logfd);
 831             }
 832         }
 833
 834         rmtree $task->{tmpdir};
 835         mkdir $task->{tmpdir};
 836         -d $task->{tmpdir} ||
 837             die "unable to create temporary directory '$task->{tmpdir}'";
 838
 839         $logfd = IO::File->new (">$tmplog") ||
 840             die "unable to create log file '$tmplog'";
 841
 842         $task->{dumpdir} = $opts->{dumpdir};
 843         $task->{storeid} = $opts->{storage};
 844         $task->{scfg} = $opts->{scfg};
 845         $task->{tmplog} = $tmplog;
 846
 847         unlink $task->{logfile} if defined($task->{logfile});
 848
 849         debugmsg ('info', "Starting Backup of VM $vmid ($vmtype)", $logfd, 1);
 850         debugmsg ('info', "Backup started at " . strftime("%F %H:%M:%S", localtime()));
 851
 852         $plugin->set_logfd ($logfd);
 853
 854         # test is VM is running
 855         my ($running, $status_text) = $plugin->vm_status ($vmid);
 856
 857         debugmsg ('info', "status = ${status_text}", $logfd);
 858
 859         # lock VM (prevent config changes)
 860         $plugin->lock_vm ($vmid);
 861
 862         $cleanup->{unlock} = 1;
 863
 864         # prepare
 865
 866         my $mode = $running ? $task->{mode} : 'stop';
 867
 868         if ($mode eq 'snapshot') {
 869             my %saved_task = %$task;
 870             eval { $plugin->prepare ($task, $vmid, $mode); };
 871             if (my $err = $@) {
 872                 die $err if $err !~ m/^mode failure/;
 873                 debugmsg ('info',  $err, $logfd);
 874                 debugmsg ('info',  "trying 'suspend' mode instead", $logfd);
 875                 $mode = 'suspend'; # so prepare is called again below
 876                 %$task = %saved_task;
 877             }
 878         }
 879
 880         $cleanup->{prepared} = 1;
 881
 882         $task->{mode} = $mode;
 883
 884         debugmsg ('info', "backup mode: $mode", $logfd);
 885
 886         debugmsg ('info', "bandwidth limit: $opts->{bwlimit} KB/s", $logfd)
 887             if $opts->{bwlimit};
 888
 889         debugmsg ('info', "ionice priority: $opts->{ionice}", $logfd);
 890
 891         if ($mode eq 'stop') {
 892
 893             $plugin->prepare ($task, $vmid, $mode);
 894
 895             $self->run_hook_script ('backup-start', $task, $logfd);
 896
 897             if ($running) {
 898                 debugmsg ('info', "stopping vm", $logfd);
 899                 $task->{vmstoptime} = time();
 900                 $self->run_hook_script ('pre-stop', $task, $logfd);
 901                 $plugin->stop_vm ($task, $vmid);
 902                 $cleanup->{restart} = 1;
 903             }
 904
 905
 906         } elsif ($mode eq 'suspend') {
 907
 908             $plugin->prepare ($task, $vmid, $mode);
 909
 910             $self->run_hook_script ('backup-start', $task, $logfd);
 911
 912             if ($vmtype eq 'lxc') {
 913                 # pre-suspend rsync
 914                 $plugin->copy_data_phase1($task, $vmid);
 915             }
 916
 917             debugmsg ('info', "suspending guest", $logfd);
 918             $task->{vmstoptime} = time ();
 919             $self->run_hook_script ('pre-stop', $task, $logfd);
 920             $plugin->suspend_vm ($task, $vmid);
 921             $cleanup->{resume} = 1;
 922
 923             if ($vmtype eq 'lxc') {
 924                 # post-suspend rsync
 925                 $plugin->copy_data_phase2($task, $vmid);
 926
 927                 debugmsg ('info', "resuming guest", $logfd);
 928                 $cleanup->{resume} = 0;
 929                 $self->run_hook_script('pre-restart', $task, $logfd);
 930                 $plugin->resume_vm($task, $vmid);
 931                 $self->run_hook_script('post-restart', $task, $logfd);
 932                 $log_vm_online_again->();
 933             }
 934
 935         } elsif ($mode eq 'snapshot') {
 936
 937             $self->run_hook_script ('backup-start', $task, $logfd);
 938
 939             my $snapshot_count = $task->{snapshot_count} || 0;
 940
 941             $self->run_hook_script ('pre-stop', $task, $logfd);
 942
 943             if ($snapshot_count > 1) {
 944                 debugmsg ('info', "suspend vm to make snapshot", $logfd);
 945                 $task->{vmstoptime} = time ();
 946                 $plugin->suspend_vm ($task, $vmid);
 947                 $cleanup->{resume} = 1;
 948             }
 949
 950             $plugin->snapshot ($task, $vmid);
 951
 952             $self->run_hook_script ('pre-restart', $task, $logfd);
 953
 954             if ($snapshot_count > 1) {
 955                 debugmsg ('info', "resume vm", $logfd);
 956                 $cleanup->{resume} = 0;
 957                 $plugin->resume_vm ($task, $vmid);
 958                 $log_vm_online_again->();
 959             }
 960
 961             $self->run_hook_script ('post-restart', $task, $logfd);
 962
 963         } else {
 964             die "internal error - unknown mode '$mode'\n";
 965         }
 966
 967         # assemble archive image
 968         $plugin->assemble ($task, $vmid);
 969
 970         # produce archive
 971
 972         if ($opts->{stdout}) {
 973             debugmsg ('info', "sending archive to stdout", $logfd);
 974             $plugin->archive($task, $vmid, $task->{tmptar}, $comp);
 975             $self->run_hook_script ('backup-end', $task, $logfd);
 976             return;
 977         }
 978
 979         my $archive_txt = $self->{opts}->{pbs} ? 'Proxmox Backup Server' : 'vzdump';
 980         debugmsg('info', "creating $archive_txt archive '$task->{target}'", $logfd);
 981         $plugin->archive($task, $vmid, $task->{tmptar}, $comp);
 982
 983         if ($self->{opts}->{pbs}) {
 984             # size is added to task struct in guest vzdump plugins
 985         } else {
 986             rename ($task->{tmptar}, $task->{target}) ||
 987                 die "unable to rename '$task->{tmptar}' to '$task->{target}'\n";
 988
 989             # determine size
 990             $task->{size} = (-s $task->{target}) || 0;
 991             my $cs = format_size ($task->{size});
 992             debugmsg ('info', "archive file size: $cs", $logfd);
 993         }
 994
 995         # purge older backup
 996         if ($opts->{remove}) {
 997             if (!defined($opts->{storage})) {
 998                 my $bklist = get_backup_file_list($opts->{dumpdir}, $bkname, $task->{target});
 999                 PVE::Storage::prune_mark_backup_group($bklist, $prune_options);
1000
1001                 foreach my $prune_entry (@{$bklist}) {
1002                     next if $prune_entry->{mark} ne 'remove';
1003
1004                     my $archive_path = $prune_entry->{path};
1005                     debugmsg ('info', "delete old backup '$archive_path'", $logfd);
1006                     PVE::Storage::archive_remove($archive_path);
1007                 }
1008             } else {
1009                 my $logfunc = sub { debugmsg($_[0], $_[1], $logfd) };
1010                 PVE::Storage::prune_backups($cfg, $opts->{storage}, $prune_options, $vmid, $vmtype, 0, $logfunc);
1011             }
1012         }
1013
1014         $self->run_hook_script ('backup-end', $task, $logfd);
1015     };
1016     my $err = $@;
1017
1018     if ($plugin) {
1019         # clean-up
1020
1021         if ($cleanup->{unlock}) {
1022             eval { $plugin->unlock_vm ($vmid); };
1023             warn $@ if $@;
1024         }
1025
1026         if ($cleanup->{prepared}) {
1027             # only call cleanup when necessary (when prepare was executed)
1028             eval { $plugin->cleanup ($task, $vmid) };
1029             warn $@ if $@;
1030         }
1031
1032         eval { $plugin->set_logfd (undef); };
1033         warn $@ if $@;
1034
1035         if ($cleanup->{resume} || $cleanup->{restart}) {
1036             eval {
1037                 $self->run_hook_script ('pre-restart', $task, $logfd);
1038                 if ($cleanup->{resume}) {
1039                     debugmsg ('info', "resume vm", $logfd);
1040                     $plugin->resume_vm ($task, $vmid);
1041                 } else {
1042                     my $running = $plugin->vm_status($vmid);
1043                     if (!$running) {
1044                         debugmsg ('info', "restarting vm", $logfd);
1045                         $plugin->start_vm ($task, $vmid);
1046                     }
1047                 }
1048                 $self->run_hook_script ('post-restart', $task, $logfd);
1049             };
1050             my $err = $@;
1051             if ($err) {
1052                 warn $err;
1053             } else {
1054                 $log_vm_online_again->();
1055             }
1056         }
1057     }
1058
1059     eval { unlink $task->{tmptar} if $task->{tmptar} && -f $task->{tmptar}; };
1060     warn $@ if $@;
1061
1062     eval { rmtree $task->{tmpdir} if $task->{tmpdir} && -d $task->{tmpdir}; };
1063     warn $@ if $@;
1064
1065     my $delay = $task->{backuptime} = time () - $vmstarttime;
1066
1067     if ($err) {
1068         $task->{state} = 'err';
1069         $task->{msg} = $err;
1070         debugmsg ('err', "Backup of VM $vmid failed - $err", $logfd, 1);
1071         debugmsg ('info', "Failed at " . strftime("%F %H:%M:%S", localtime()));
1072
1073         eval { $self->run_hook_script ('backup-abort', $task, $logfd); };
1074
1075     } else {
1076         $task->{state} = 'ok';
1077         my $tstr = format_time ($delay);
1078         debugmsg ('info', "Finished Backup of VM $vmid ($tstr)", $logfd, 1);
1079         debugmsg ('info', "Backup finished at " . strftime("%F %H:%M:%S", localtime()));
1080     }
1081
1082     close ($logfd) if $logfd;
1083
1084     if ($task->{tmplog}) {
1085         if ($self->{opts}->{pbs}) {
1086             if ($task->{state} eq 'ok') {
1087                 eval {
1088                     PVE::Storage::PBSPlugin::run_raw_client_cmd(
1089                         $opts->{scfg},
1090                         $opts->{storage},
1091                         'upload-log',
1092                         [ $pbs_snapshot_name, $task->{tmplog} ],
1093                         errmsg => "uploading backup task log failed",
1094                     );
1095                 };
1096                 debugmsg('warn', "$@") if $@; # $@ contains already error prefix
1097             }
1098         } elsif ($task->{logfile}) {
1099             system {'cp'} 'cp', $task->{tmplog}, $task->{logfile};
1100         }
1101     }
1102
1103     eval { $self->run_hook_script ('log-end', $task); };
1104
1105     die $err if $err && $err =~ m/^interrupted by signal$/;
1106 }
1107
1108 sub exec_backup {
1109     my ($self, $rpcenv, $authuser) = @_;
1110
1111     my $opts = $self->{opts};
1112
1113     debugmsg ('info', "starting new backup job: $self->{cmdline}", undef, 1);
1114
1115     if (scalar(@{$self->{skiplist}})) {
1116         my $skip_string = join(', ', sort { $a <=> $b } @{$self->{skiplist}});
1117         debugmsg ('info', "skip external VMs: $skip_string");
1118     }
1119
1120     my $tasklist = [];
1121     my $vzdump_plugins =  {};
1122     foreach my $plugin (@{$self->{plugins}}) {
1123         my $type = $plugin->type();
1124         next if exists $vzdump_plugins->{$type};
1125         $vzdump_plugins->{$type} = $plugin;
1126     }
1127
1128     my $vmlist = PVE::Cluster::get_vmlist();
1129     my $vmids = [ sort { $a <=> $b } @{$opts->{vmids}} ];
1130     foreach my $vmid (@{$vmids}) {
1131         my $plugin;
1132         if (defined($vmlist->{ids}->{$vmid})) {
1133             my $guest_type = $vmlist->{ids}->{$vmid}->{type};
1134             $plugin = $vzdump_plugins->{$guest_type};
1135             next if !$rpcenv->check($authuser, "/vms/$vmid", [ 'VM.Backup' ], $opts->{all});
1136         }
1137         push @$tasklist, {
1138             mode => $opts->{mode},
1139             plugin => $plugin,
1140             state => 'todo',
1141             vmid => $vmid,
1142         };
1143     }
1144
1145     # Use in-memory files for the outer hook logs to pass them to sendmail.
1146     my $job_start_log = '';
1147     my $job_end_log = '';
1148     open my $job_start_fd, '>', \$job_start_log;
1149     open my $job_end_fd, '>', \$job_end_log;
1150
1151     my $starttime = time();
1152     my $errcount = 0;
1153     eval {
1154
1155         $self->run_hook_script ('job-start', undef, $job_start_fd);
1156
1157         foreach my $task (@$tasklist) {
1158             $self->exec_backup_task ($task);
1159             $errcount += 1 if $task->{state} ne 'ok';
1160         }
1161
1162         $self->run_hook_script ('job-end', undef, $job_end_fd);
1163     };
1164     my $err = $@;
1165
1166     $self->run_hook_script ('job-abort', undef, $job_end_fd) if $err;
1167
1168     if ($err) {
1169         debugmsg ('err', "Backup job failed - $err", undef, 1);
1170     } else {
1171         if ($errcount) {
1172             debugmsg ('info', "Backup job finished with errors", undef, 1);
1173         } else {
1174             debugmsg ('info', "Backup job finished successfully", undef, 1);
1175         }
1176     }
1177
1178     close $job_start_fd;
1179     close $job_end_fd;
1180
1181     my $totaltime = time() - $starttime;
1182
1183     eval { $self->sendmail ($tasklist, $totaltime, undef, $job_start_log, $job_end_log); };
1184     debugmsg ('err', $@) if $@;
1185
1186     die $err if $err;
1187
1188     die "job errors\n" if $errcount;
1189
1190     unlink $pidfile;
1191 }
1192
1193
1194 sub option_exists {
1195     my $key = shift;
1196     return defined($confdesc->{$key});
1197 }
1198
1199 # NOTE it might make sense to merge this and verify_vzdump_parameters(), but one
1200 # needs to adapt command_line() in guest-common's PVE/VZDump/Common.pm and detect
1201 # a second parsing attempt, because verify_vzdump_parameters() is called twice
1202 # during the update_job API call.
1203 sub parse_mailto_exclude_path {
1204     my ($param) = @_;
1205
1206     # exclude-path list need to be 0 separated
1207     if (defined($param->{'exclude-path'})) {
1208         my @expaths = split(/\0/, $param->{'exclude-path'} || '');
1209         $param->{'exclude-path'} = [ @expaths ];
1210     }
1211
1212     if (defined($param->{mailto})) {
1213         my @mailto = PVE::Tools::split_list(extract_param($param, 'mailto'));
1214         $param->{mailto} = [ @mailto ];
1215     }
1216
1217     return;
1218 }
1219
1220 sub verify_vzdump_parameters {
1221     my ($param, $check_missing) = @_;
1222
1223     raise_param_exc({ all => "option conflicts with option 'vmid'"})
1224         if $param->{all} && $param->{vmid};
1225
1226     raise_param_exc({ exclude => "option conflicts with option 'vmid'"})
1227         if $param->{exclude} && $param->{vmid};
1228
1229     raise_param_exc({ pool => "option conflicts with option 'vmid'"})
1230         if $param->{pool} && $param->{vmid};
1231
1232     raise_param_exc({ 'prune-backups' => "option conflicts with option 'maxfiles'"})
1233         if defined($param->{'prune-backups'}) && defined($param->{maxfiles});
1234
1235     $parse_prune_backups_maxfiles->($param, 'CLI parameters');
1236
1237     $param->{all} = 1 if (defined($param->{exclude}) && !$param->{pool});
1238
1239     warn "option 'size' is deprecated and will be removed in a future " .
1240          "release, please update your script/configuration!\n"
1241         if defined($param->{size});
1242
1243     return if !$check_missing;
1244
1245     raise_param_exc({ vmid => "property is missing"})
1246         if !($param->{all} || $param->{stop} || $param->{pool}) && !$param->{vmid};
1247
1248 }
1249
1250 sub stop_running_backups {
1251     my($self) = @_;
1252
1253     my $upid = PVE::Tools::file_read_firstline($pidfile);
1254     return if !$upid;
1255
1256     my $task = PVE::Tools::upid_decode($upid);
1257
1258     if (PVE::ProcFSTools::check_process_running($task->{pid}, $task->{pstart}) &&
1259         PVE::ProcFSTools::read_proc_starttime($task->{pid}) == $task->{pstart}) {
1260         kill(15, $task->{pid});
1261         # wait max 15 seconds to shut down (else, do nothing for now)
1262         my $i;
1263         for ($i = 15; $i > 0; $i--) {
1264             last if !PVE::ProcFSTools::check_process_running(($task->{pid}, $task->{pstart}));
1265             sleep (1);
1266         }
1267         die "stopping backup process $task->{pid} failed\n" if $i == 0;
1268     }
1269 }
1270
1271 sub get_included_guests {
1272     my ($job) = @_;
1273
1274     my $vmids = [];
1275     my $vmids_per_node = {};
1276
1277     my $vmlist = PVE::Cluster::get_vmlist();
1278
1279     if ($job->{pool}) {
1280         $vmids = PVE::API2Tools::get_resource_pool_guest_members($job->{pool});
1281     } elsif ($job->{vmid}) {
1282         $vmids = [ split_list($job->{vmid}) ];
1283     } elsif ($job->{all}) {
1284         # all or exclude
1285         my $exclude = check_vmids(split_list($job->{exclude}));
1286         my $excludehash = { map { $_ => 1 } @$exclude };
1287
1288         for my $id (keys %{$vmlist->{ids}}) {
1289             next if $excludehash->{$id};
1290             push @$vmids, $id;
1291         }
1292     } else {
1293         return $vmids_per_node;
1294     }
1295     $vmids = check_vmids(@$vmids);
1296
1297     for my $vmid (@$vmids) {
1298         if (defined($vmlist->{ids}->{$vmid})) {
1299             my $node = $vmlist->{ids}->{$vmid}->{node};
1300             next if (defined $job->{node} && $job->{node} ne $node);
1301
1302             push @{$vmids_per_node->{$node}}, $vmid;
1303         } else {
1304             push @{$vmids_per_node->{''}}, $vmid;
1305         }
1306     }
1307
1308     return $vmids_per_node;
1309 }
1310
1311 1;