src/PVE/VZDump/Common.pm

   1 package PVE::VZDump::Common;
   2
   3 use strict;
   4 use warnings;
   5 use Digest::SHA;
   6
   7 use PVE::Tools;
   8 use PVE::SafeSyslog qw(syslog);
   9 use PVE::Storage;
  10 use PVE::Cluster qw(cfs_register_file);
  11 use PVE::JSONSchema qw(get_standard_option);
  12
  13 # NOTE: this is the legacy config, nowadays jobs.cfg is used (handled in pve-manager)
  14 cfs_register_file(
  15     'vzdump.cron',
  16     \&parse_vzdump_cron_config,
  17     \&write_vzdump_cron_config,
  18 );
  19
  20 my $dowhash_to_dow = sub {
  21     my ($d, $num) = @_;
  22
  23     my @da = ();
  24     push @da, $num ? 1 : 'mon' if $d->{mon};
  25     push @da, $num ? 2 : 'tue' if $d->{tue};
  26     push @da, $num ? 3 : 'wed' if $d->{wed};
  27     push @da, $num ? 4 : 'thu' if $d->{thu};
  28     push @da, $num ? 5 : 'fri' if $d->{fri};
  29     push @da, $num ? 6 : 'sat' if $d->{sat};
  30     push @da, $num ? 7 : 'sun' if $d->{sun};
  31
  32     return join ',', @da;
  33 };
  34
  35 our $PROPERTY_STRINGS = {
  36     'performance' => 'backup-performance',
  37     'prune-backups' => 'prune-backups',
  38 };
  39
  40 my sub parse_property_strings {
  41     my ($opts) = @_;
  42
  43     for my $opt (keys $PROPERTY_STRINGS->%*) {
  44         next if !defined($opts->{$opt});
  45
  46         my $format = $PROPERTY_STRINGS->{$opt};
  47         $opts->{$opt} = PVE::JSONSchema::parse_property_string($format, $opts->{$opt});
  48     }
  49 }
  50
  51 # parse crontab style day of week
  52 sub parse_dow {
  53     my ($dowstr, $noerr) = @_;
  54
  55     my $dowmap = {mon => 1, tue => 2, wed => 3, thu => 4,
  56                   fri => 5, sat => 6, sun => 7};
  57     my $rdowmap = { '1' => 'mon', '2' => 'tue', '3' => 'wed', '4' => 'thu',
  58                     '5' => 'fri', '6' => 'sat', '7' => 'sun', '0' => 'sun'};
  59
  60     my $res = {};
  61
  62     $dowstr = '1,2,3,4,5,6,7' if $dowstr eq '*';
  63
  64     foreach my $day (PVE::Tools::split_list($dowstr)) {
  65         if ($day =~ m/^(mon|tue|wed|thu|fri|sat|sun)-(mon|tue|wed|thu|fri|sat|sun)$/i) {
  66             for (my $i = $dowmap->{lc($1)}; $i <= $dowmap->{lc($2)}; $i++) {
  67                 my $r = $rdowmap->{$i};
  68                 $res->{$r} = 1;
  69             }
  70         } elsif ($day =~ m/^(mon|tue|wed|thu|fri|sat|sun|[0-7])$/i) {
  71             $day = $rdowmap->{$day} if $day =~ m/\d/;
  72             $res->{lc($day)} = 1;
  73         } else {
  74             return undef if $noerr;
  75             die "unable to parse day of week '$dowstr'\n";
  76         }
  77     }
  78
  79     return $res;
  80 };
  81
  82 PVE::JSONSchema::register_format('backup-performance', {
  83     'max-workers' => {
  84         description => "Applies to VMs. Allow up to this many IO workers at the same time.",
  85         type => 'integer',
  86         minimum => 1,
  87         maximum => 256,
  88         default => 16,
  89         optional => 1,
  90     },
  91     'pbs-entries-max' => {
  92         description => "Applies to container backups sent to PBS. Limits the number of entries "
  93             ."allowed in memory at a given time to avoid unintended OOM situations. Increase it to "
  94             ."enable backups of containers with a large amount of files.",
  95         type => 'integer',
  96         minimum => 1,
  97         default => 1048576,
  98         optional => 1,
  99     },
 100 });
 101
 102 my $confdesc = {
 103     vmid => {
 104         type => 'string', format => 'pve-vmid-list',
 105         description => "The ID of the guest system you want to backup.",
 106         completion => \&PVE::Cluster::complete_local_vmid,
 107         optional => 1,
 108     },
 109     node => get_standard_option('pve-node', {
 110         description => "Only run if executed on this node.",
 111         completion => \&PVE::Cluster::get_nodelist,
 112         optional => 1,
 113     }),
 114     all => {
 115         type => 'boolean',
 116         description => "Backup all known guest systems on this host.",
 117         optional => 1,
 118         default => 0,
 119     },
 120     stdexcludes => {
 121         type => 'boolean',
 122         description => "Exclude temporary files and logs.",
 123         optional => 1,
 124         default => 1,
 125     },
 126     compress => {
 127         type => 'string',
 128         description => "Compress dump file.",
 129         optional => 1,
 130         enum => ['0', '1', 'gzip', 'lzo', 'zstd'],
 131         default => '0',
 132     },
 133     pigz=> {
 134         type => "integer",
 135         description => "Use pigz instead of gzip when N>0.".
 136             " N=1 uses half of cores, N>1 uses N as thread count.",
 137         optional => 1,
 138         default => 0,
 139     },
 140     zstd => {
 141         type => "integer",
 142         description => "Zstd threads. N=0 uses half of the available cores,".
 143             " N>0 uses N as thread count.",
 144         optional => 1,
 145         default => 1,
 146     },
 147     quiet => {
 148         type => 'boolean',
 149         description => "Be quiet.",
 150         optional => 1,
 151         default => 0,
 152     },
 153     mode => {
 154         type => 'string',
 155         description => "Backup mode.",
 156         optional => 1,
 157         default => 'snapshot',
 158         enum => [ 'snapshot', 'suspend', 'stop' ],
 159     },
 160     exclude => {
 161         type => 'string', format => 'pve-vmid-list',
 162         description => "Exclude specified guest systems (assumes --all)",
 163         optional => 1,
 164     },
 165     'exclude-path' => {
 166         type => 'array',
 167         description => "Exclude certain files/directories (shell globs)." .
 168             " Paths starting with '/' are anchored to the container's root, " .
 169             " other paths match relative to each subdirectory.",
 170         optional => 1,
 171         items => {
 172             type => 'string',
 173         },
 174     },
 175     mailto => {
 176         type => 'string',
 177         format => 'email-or-username-list',
 178         description => "Deprecated: Use notification targets/matchers instead." .
 179             " Comma-separated list of email addresses or users that should" .
 180             " receive email notifications.",
 181         optional => 1,
 182     },
 183     mailnotification => {
 184         type => 'string',
 185         description => "Deprecated: use notification targets/matchers instead." .
 186             " Specify when to send a notification mail",
 187         optional => 1,
 188         enum => [ 'always', 'failure' ],
 189         default => 'always',
 190     },
 191     'notification-mode' => {
 192         type => 'string',
 193         description => "Determine which notification system to use." .
 194             " If set to 'legacy-sendmail', vzdump will consider the" .
 195             " mailto/mailnotification parameters and send emails to the" .
 196             " specified address(es) via the 'sendmail' command." .
 197             " If set to 'notification-system', a notification will be sent via PVE's" .
 198             " notification system and mailto/mailnotification will be ignored" .
 199             " If set to 'auto' (default setting), an email will be sent if " .
 200             " mailto is set, and the notification system will be used if not.",
 201         optional => 1,
 202         enum => [ 'auto', 'legacy-sendmail', 'notification-system'],
 203         default => 'auto',
 204     },
 205     'notification-policy' => {
 206         type => 'string',
 207         description => "Deprecated: Do not use",
 208         optional => 1,
 209         enum => [ 'always', 'failure', 'never'],
 210         default => 'always',
 211     },
 212     'notification-target' => {
 213         type => 'string',
 214         format => 'pve-configid',
 215         description => "Deprecated: Do not use",
 216         optional => 1,
 217     },
 218     tmpdir => {
 219         type => 'string',
 220         description => "Store temporary files to specified directory.",
 221         optional => 1,
 222     },
 223     dumpdir => {
 224         type => 'string',
 225         description => "Store resulting files to specified directory.",
 226         optional => 1,
 227     },
 228     script => {
 229         type => 'string',
 230         description => "Use specified hook script.",
 231         optional => 1,
 232     },
 233     storage => get_standard_option('pve-storage-id', {
 234         description => "Store resulting file to this storage.",
 235         completion => \&complete_backup_storage,
 236         optional => 1,
 237     }),
 238     stop => {
 239         type => 'boolean',
 240         description => "Stop running backup jobs on this host.",
 241         optional => 1,
 242         default => 0,
 243     },
 244     bwlimit => {
 245         type => 'integer',
 246         description => "Limit I/O bandwidth (in KiB/s).",
 247         optional => 1,
 248         minimum => 0,
 249         default => 0,
 250     },
 251     ionice => {
 252         type => 'integer',
 253         description => "Set IO priority when using the BFQ scheduler. For snapshot and suspend "
 254             ."mode backups of VMs, this only affects the compressor. A value of 8 means the idle "
 255             ."priority is used, otherwise the best-effort priority is used with the specified "
 256             ."value.",
 257         optional => 1,
 258         minimum => 0,
 259         maximum => 8,
 260         default => 7,
 261     },
 262     performance => {
 263         type => 'string',
 264         description => "Other performance-related settings.",
 265         format => 'backup-performance',
 266         optional => 1,
 267     },
 268     lockwait => {
 269         type => 'integer',
 270         description => "Maximal time to wait for the global lock (minutes).",
 271         optional => 1,
 272         minimum => 0,
 273         default => 3*60, # 3 hours
 274     },
 275     stopwait => {
 276         type => 'integer',
 277         description => "Maximal time to wait until a guest system is stopped (minutes).",
 278         optional => 1,
 279         minimum => 0,
 280         default => 10, # 10 minutes
 281     },
 282     # FIXME remove with PVE 8.0 or PVE 9.0
 283     maxfiles => {
 284         type => 'integer',
 285         description => "Deprecated: use 'prune-backups' instead. " .
 286             "Maximal number of backup files per guest system.",
 287         optional => 1,
 288         minimum => 1,
 289     },
 290     'prune-backups' => get_standard_option('prune-backups', {
 291         description => "Use these retention options instead of those from the storage configuration.",
 292         optional => 1,
 293         default => "keep-all=1",
 294     }),
 295     remove => {
 296         type => 'boolean',
 297         description => "Prune older backups according to 'prune-backups'.",
 298         optional => 1,
 299         default => 1,
 300     },
 301     pool => {
 302         type => 'string',
 303         description => 'Backup all known guest systems included in the specified pool.',
 304         optional => 1,
 305     },
 306     'notes-template' => {
 307         type => 'string',
 308         description => "Template string for generating notes for the backup(s). It can contain ".
 309             "variables which will be replaced by their values. Currently supported are ".
 310             "{{cluster}}, {{guestname}}, {{node}}, and {{vmid}}, but more might be added in the ".
 311             "future. Needs to be a single line, newline and backslash need to be escaped as '\\n' ".
 312             "and '\\\\' respectively.",
 313         requires => 'storage',
 314         maxLength => 1024,
 315         optional => 1,
 316     },
 317     protected => {
 318         type => 'boolean',
 319         description => "If true, mark backup(s) as protected.",
 320         requires => 'storage',
 321         optional => 1,
 322     },
 323 };
 324
 325 sub get_confdesc {
 326     return $confdesc;
 327 }
 328
 329 # add JSON properties for create and set function
 330 sub json_config_properties {
 331     my $prop = shift;
 332
 333     foreach my $opt (keys %$confdesc) {
 334         $prop->{$opt} = $confdesc->{$opt};
 335     }
 336
 337     return $prop;
 338 }
 339
 340 my $vzdump_properties = {
 341     additionalProperties => 0,
 342     properties => json_config_properties({}),
 343 };
 344
 345 sub parse_vzdump_cron_config {
 346     my ($filename, $raw) = @_;
 347
 348     my $jobs = []; # correct jobs
 349
 350     my $ejobs = []; # mailfomerd lines
 351
 352     my $jid = 1; # we start at 1
 353
 354     my $digest = Digest::SHA::sha1_hex(defined($raw) ? $raw : '');
 355
 356     while ($raw && $raw =~ s/^(.*?)(\n|$)//) {
 357         my $line = $1;
 358
 359         next if $line =~ m/^\#/;
 360         next if $line =~ m/^\s*$/;
 361         next if $line =~ m/^PATH\s*=/; # we always overwrite path
 362
 363         if ($line =~ m|^(\d+)\s+(\d+)\s+\*\s+\*\s+(\S+)\s+root\s+(/\S+/)?(#)?vzdump(\s+(.*))?$|) {
 364             eval {
 365                 my $minute = int($1);
 366                 my $hour = int($2);
 367                 my $dow = $3;
 368                 my $param = $7;
 369                 my $enabled = $5;
 370
 371                 my $dowhash = parse_dow($dow, 1);
 372                 die "unable to parse day of week '$dow' in '$filename'\n" if !$dowhash;
 373
 374                 my $args = PVE::Tools::split_args($param);
 375                 my $opts = PVE::JSONSchema::get_options($vzdump_properties, $args, 'vmid');
 376
 377                 $opts->{enabled} = !defined($enabled);
 378                 $opts->{id} = "$digest:$jid";
 379                 $jid++;
 380                 $opts->{starttime} = sprintf "%02d:%02d", $hour, $minute;
 381                 $opts->{dow} = &$dowhash_to_dow($dowhash);
 382
 383                 parse_property_strings($opts);
 384
 385                 push @$jobs, $opts;
 386             };
 387             my $err = $@;
 388             if ($err) {
 389                 syslog ('err', "parse error in '$filename': $err");
 390                 push @$ejobs, { line => $line };
 391             }
 392         } elsif ($line =~ m|^\S+\s+(\S+)\s+\S+\s+\S+\s+\S+\s+\S+\s+(\S.*)$|) {
 393             syslog ('err', "warning: malformed line in '$filename'");
 394             push @$ejobs, { line => $line };
 395         } else {
 396             syslog ('err', "ignoring malformed line in '$filename'");
 397         }
 398     }
 399
 400     my $res = {};
 401     $res->{digest} = $digest;
 402     $res->{jobs} = $jobs;
 403     $res->{ejobs} = $ejobs;
 404
 405     return $res;
 406 }
 407
 408 sub write_vzdump_cron_config {
 409     my ($filename, $cfg) = @_;
 410
 411     my $out = "# cluster wide vzdump cron schedule\n";
 412     $out .= "# Automatically generated file - do not edit\n\n";
 413     $out .= "PATH=\"/usr/sbin:/usr/bin:/sbin:/bin\"\n\n";
 414
 415     my $jobs = $cfg->{jobs} || [];
 416     foreach my $job (@$jobs) {
 417         my $enabled = ($job->{enabled}) ? '' : '#';
 418         my $dh = parse_dow($job->{dow});
 419         my $dow;
 420         if ($dh->{mon} && $dh->{tue} && $dh->{wed} && $dh->{thu} &&
 421             $dh->{fri} && $dh->{sat} && $dh->{sun}) {
 422             $dow = '*';
 423         } else {
 424             $dow = &$dowhash_to_dow($dh, 1);
 425             $dow = '*' if !$dow;
 426         }
 427
 428         my ($hour, $minute);
 429
 430         die "no job start time specified\n" if !$job->{starttime};
 431         if ($job->{starttime} =~ m/^(\d{1,2}):(\d{1,2})$/) {
 432             ($hour, $minute) = (int($1), int($2));
 433             die "hour '$hour' out of range\n" if $hour < 0 || $hour > 23;
 434             die "minute '$minute' out of range\n" if $minute < 0 || $minute > 59;
 435         } else {
 436             die "unable to parse job start time\n";
 437         }
 438
 439         $job->{quiet} = 1; # we do not want messages from cron
 440
 441         my $cmd = command_line($job);
 442
 443         $out .= sprintf "$minute $hour * * %-11s root $enabled$cmd\n", $dow;
 444     }
 445
 446     my $ejobs = $cfg->{ejobs} || [];
 447     foreach my $job (@$ejobs) {
 448         $out .= "$job->{line}\n" if $job->{line};
 449     }
 450
 451     return $out;
 452 }
 453
 454 sub command_line {
 455     my ($param) = @_;
 456
 457     my $cmd = "vzdump";
 458
 459     if ($param->{vmid}) {
 460         $cmd .= " " . join(' ', PVE::Tools::split_list($param->{vmid}));
 461     }
 462
 463     foreach my $p (keys %$param) {
 464         next if $p eq 'id' || $p eq 'vmid' || $p eq 'starttime' ||
 465                 $p eq 'dow' || $p eq 'stdout' || $p eq 'enabled';
 466         my $v = $param->{$p};
 467         my $pd = $confdesc->{$p} || die "no such vzdump option '$p'\n";
 468         if ($p eq 'exclude-path') {
 469             foreach my $path (@$v) {
 470                 $cmd .= " --$p " . PVE::Tools::shellquote($path);
 471             }
 472         } else {
 473             $v = join(",", PVE::Tools::split_list($v)) if $p eq 'mailto';
 474             $v = PVE::JSONSchema::print_property_string($v, $PROPERTY_STRINGS->{$p})
 475                 if $PROPERTY_STRINGS->{$p};
 476
 477             $cmd .= " --$p " . PVE::Tools::shellquote($v) if defined($v) && $v ne '';
 478         }
 479     }
 480
 481     return $cmd;
 482 }
 483
 484 # bash completion helpers
 485 sub complete_backup_storage {
 486
 487     my $cfg = PVE::Storage::config();
 488     my $ids = $cfg->{ids};
 489
 490     my $nodename = PVE::INotify::nodename();
 491
 492     my $res = [];
 493     foreach my $sid (keys %$ids) {
 494         my $scfg = $ids->{$sid};
 495         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $nodename, 1);
 496         next if !$scfg->{content}->{backup};
 497         push @$res, $sid;
 498     }
 499
 500     return $res;
 501 }
 502
 503 1;