src/PVE/VZDump/Common.pm

   1 package PVE::VZDump::Common;
   2
   3 use strict;
   4 use warnings;
   5 use Digest::SHA;
   6
   7 use PVE::Tools;
   8 use PVE::SafeSyslog qw(syslog);
   9 use PVE::Storage;
  10 use PVE::Cluster qw(cfs_register_file);
  11 use PVE::JSONSchema qw(get_standard_option);
  12
  13 # NOTE: this is the legacy config, nowadays jobs.cfg is used (handled in pve-manager)
  14 cfs_register_file(
  15     'vzdump.cron',
  16     \&parse_vzdump_cron_config,
  17     \&write_vzdump_cron_config,
  18 );
  19
  20 my $dowhash_to_dow = sub {
  21     my ($d, $num) = @_;
  22
  23     my @da = ();
  24     push @da, $num ? 1 : 'mon' if $d->{mon};
  25     push @da, $num ? 2 : 'tue' if $d->{tue};
  26     push @da, $num ? 3 : 'wed' if $d->{wed};
  27     push @da, $num ? 4 : 'thu' if $d->{thu};
  28     push @da, $num ? 5 : 'fri' if $d->{fri};
  29     push @da, $num ? 6 : 'sat' if $d->{sat};
  30     push @da, $num ? 7 : 'sun' if $d->{sun};
  31
  32     return join ',', @da;
  33 };
  34
  35 our $PROPERTY_STRINGS = {
  36     'fleecing' => 'backup-fleecing',
  37     'performance' => 'backup-performance',
  38     'prune-backups' => 'prune-backups',
  39 };
  40
  41 my sub parse_property_strings {
  42     my ($opts) = @_;
  43
  44     for my $opt (keys $PROPERTY_STRINGS->%*) {
  45         next if !defined($opts->{$opt});
  46
  47         my $format = $PROPERTY_STRINGS->{$opt};
  48         $opts->{$opt} = PVE::JSONSchema::parse_property_string($format, $opts->{$opt});
  49     }
  50 }
  51
  52 # parse crontab style day of week
  53 sub parse_dow {
  54     my ($dowstr, $noerr) = @_;
  55
  56     my $dowmap = {mon => 1, tue => 2, wed => 3, thu => 4,
  57                   fri => 5, sat => 6, sun => 7};
  58     my $rdowmap = { '1' => 'mon', '2' => 'tue', '3' => 'wed', '4' => 'thu',
  59                     '5' => 'fri', '6' => 'sat', '7' => 'sun', '0' => 'sun'};
  60
  61     my $res = {};
  62
  63     $dowstr = '1,2,3,4,5,6,7' if $dowstr eq '*';
  64
  65     foreach my $day (PVE::Tools::split_list($dowstr)) {
  66         if ($day =~ m/^(mon|tue|wed|thu|fri|sat|sun)-(mon|tue|wed|thu|fri|sat|sun)$/i) {
  67             for (my $i = $dowmap->{lc($1)}; $i <= $dowmap->{lc($2)}; $i++) {
  68                 my $r = $rdowmap->{$i};
  69                 $res->{$r} = 1;
  70             }
  71         } elsif ($day =~ m/^(mon|tue|wed|thu|fri|sat|sun|[0-7])$/i) {
  72             $day = $rdowmap->{$day} if $day =~ m/\d/;
  73             $res->{lc($day)} = 1;
  74         } else {
  75             return undef if $noerr;
  76             die "unable to parse day of week '$dowstr'\n";
  77         }
  78     }
  79
  80     return $res;
  81 };
  82
  83 PVE::JSONSchema::register_format('backup-fleecing', {
  84     enabled => {
  85         description => "Enable backup fleecing. Cache backup data from blocks where new guest"
  86             ." writes happen on specified storage instead of copying them directly to the backup"
  87             ." target. This can help guest IO performance and even prevent hangs, at the cost of"
  88             ." requiring more storage space.",
  89         type => 'boolean',
  90         default => 0,
  91         optional => 1,
  92         default_key => 1,
  93     },
  94     storage => get_standard_option('pve-storage-id', {
  95         description => "Use this storage to storage fleecing images. For efficient space usage,"
  96             ." it's best to use a local storage that supports discard and either thin provisioning"
  97             ." or sparse files.",
  98         optional => 1,
  99     }),
 100 }, \&verify_backup_fleecing);
 101
 102 sub verify_backup_fleecing {
 103     my ($param, $noerr) = @_;
 104
 105     if (!$param->{storage} && $param->{enabled}) {
 106         return if $noerr;
 107         die "'storage' parameter is required when 'enabled' is set\n";
 108     }
 109
 110     return $param;
 111 }
 112
 113 PVE::JSONSchema::register_format('backup-performance', {
 114     'max-workers' => {
 115         description => "Applies to VMs. Allow up to this many IO workers at the same time.",
 116         type => 'integer',
 117         minimum => 1,
 118         maximum => 256,
 119         default => 16,
 120         optional => 1,
 121     },
 122     'pbs-entries-max' => {
 123         description => "Applies to container backups sent to PBS. Limits the number of entries"
 124             ." allowed in memory at a given time to avoid unintended OOM situations. Increase it to"
 125             ." enable backups of containers with a large amount of files.",
 126         type => 'integer',
 127         minimum => 1,
 128         default => 1048576,
 129         optional => 1,
 130     },
 131 });
 132
 133 my $confdesc = {
 134     vmid => {
 135         type => 'string', format => 'pve-vmid-list',
 136         description => "The ID of the guest system you want to backup.",
 137         completion => \&PVE::Cluster::complete_local_vmid,
 138         optional => 1,
 139     },
 140     node => get_standard_option('pve-node', {
 141         description => "Only run if executed on this node.",
 142         completion => \&PVE::Cluster::get_nodelist,
 143         optional => 1,
 144     }),
 145     all => {
 146         type => 'boolean',
 147         description => "Backup all known guest systems on this host.",
 148         optional => 1,
 149         default => 0,
 150     },
 151     stdexcludes => {
 152         type => 'boolean',
 153         description => "Exclude temporary files and logs.",
 154         optional => 1,
 155         default => 1,
 156     },
 157     compress => {
 158         type => 'string',
 159         description => "Compress dump file.",
 160         optional => 1,
 161         enum => ['0', '1', 'gzip', 'lzo', 'zstd'],
 162         default => '0',
 163     },
 164     pigz=> {
 165         type => "integer",
 166         description => "Use pigz instead of gzip when N>0. N=1 uses half of cores, N>1 uses N as"
 167             ." thread count.",
 168         optional => 1,
 169         default => 0,
 170     },
 171     zstd => {
 172         type => "integer",
 173         description => "Zstd threads. N=0 uses half of the available cores, if N is set to a value"
 174             ." bigger than 0, N is used as thread count.",
 175         optional => 1,
 176         default => 1,
 177     },
 178     quiet => {
 179         type => 'boolean',
 180         description => "Be quiet.",
 181         optional => 1,
 182         default => 0,
 183     },
 184     mode => {
 185         type => 'string',
 186         description => "Backup mode.",
 187         optional => 1,
 188         default => 'snapshot',
 189         enum => [ 'snapshot', 'suspend', 'stop' ],
 190     },
 191     exclude => {
 192         type => 'string', format => 'pve-vmid-list',
 193         description => "Exclude specified guest systems (assumes --all)",
 194         optional => 1,
 195     },
 196     'exclude-path' => {
 197         type => 'array',
 198         description => "Exclude certain files/directories (shell globs). Paths starting with '/'"
 199             ." are anchored to the container's root, other paths match relative to each"
 200             ." subdirectory.",
 201         optional => 1,
 202         items => {
 203             type => 'string',
 204         },
 205     },
 206     mailto => {
 207         type => 'string',
 208         format => 'email-or-username-list',
 209         description => "Deprecated: Use notification targets/matchers instead. Comma-separated list"
 210             ." of email addresses or users that should receive email notifications.",
 211         optional => 1,
 212     },
 213     mailnotification => {
 214         type => 'string',
 215         description => "Deprecated: use notification targets/matchers instead." .
 216             " Specify when to send a notification mail",
 217         optional => 1,
 218         enum => [ 'always', 'failure' ],
 219         default => 'always',
 220     },
 221     'notification-mode' => {
 222         type => 'string',
 223         description => "Determine which notification system to use. If set to 'legacy-sendmail',"
 224             ." vzdump will consider the mailto/mailnotification parameters and send emails to the"
 225             ." specified address(es) via the 'sendmail' command. If set to 'notification-system',"
 226             ." a notification will be sent via PVE's notification system, and the mailto and"
 227             ." mailnotification will be ignored. If set to 'auto' (default setting), an email will"
 228             ." be sent if mailto is set, and the notification system will be used if not.",
 229         optional => 1,
 230         enum => ['auto', 'legacy-sendmail', 'notification-system'],
 231         default => 'auto',
 232     },
 233     'notification-policy' => {
 234         type => 'string',
 235         description => "Deprecated: Do not use",
 236         optional => 1,
 237         enum => [ 'always', 'failure', 'never'],
 238         default => 'always',
 239     },
 240     'notification-target' => {
 241         type => 'string',
 242         format => 'pve-configid',
 243         description => "Deprecated: Do not use",
 244         optional => 1,
 245     },
 246     tmpdir => {
 247         type => 'string',
 248         description => "Store temporary files to specified directory.",
 249         optional => 1,
 250     },
 251     dumpdir => {
 252         type => 'string',
 253         description => "Store resulting files to specified directory.",
 254         optional => 1,
 255     },
 256     script => {
 257         type => 'string',
 258         description => "Use specified hook script.",
 259         optional => 1,
 260     },
 261     storage => get_standard_option('pve-storage-id', {
 262         description => "Store resulting file to this storage.",
 263         completion => \&complete_backup_storage,
 264         optional => 1,
 265     }),
 266     stop => {
 267         type => 'boolean',
 268         description => "Stop running backup jobs on this host.",
 269         optional => 1,
 270         default => 0,
 271     },
 272     bwlimit => {
 273         type => 'integer',
 274         description => "Limit I/O bandwidth (in KiB/s).",
 275         optional => 1,
 276         minimum => 0,
 277         default => 0,
 278     },
 279     ionice => {
 280         type => 'integer',
 281         description => "Set IO priority when using the BFQ scheduler. For snapshot and suspend"
 282             ." mode backups of VMs, this only affects the compressor. A value of 8 means the idle"
 283             ." priority is used, otherwise the best-effort priority is used with the specified"
 284             ." value.",
 285         optional => 1,
 286         minimum => 0,
 287         maximum => 8,
 288         default => 7,
 289     },
 290     performance => {
 291         type => 'string',
 292         description => "Other performance-related settings.",
 293         format => 'backup-performance',
 294         optional => 1,
 295     },
 296     fleecing => {
 297         type => 'string',
 298         description => "Options for backup fleecing (VM only).",
 299         format => 'backup-fleecing',
 300         optional => 1,
 301     },
 302     lockwait => {
 303         type => 'integer',
 304         description => "Maximal time to wait for the global lock (minutes).",
 305         optional => 1,
 306         minimum => 0,
 307         default => 3*60, # 3 hours
 308     },
 309     stopwait => {
 310         type => 'integer',
 311         description => "Maximal time to wait until a guest system is stopped (minutes).",
 312         optional => 1,
 313         minimum => 0,
 314         default => 10, # 10 minutes
 315     },
 316     # FIXME remove with PVE 8.0 or PVE 9.0
 317     maxfiles => {
 318         type => 'integer',
 319         description => "Deprecated: use 'prune-backups' instead. " .
 320             "Maximal number of backup files per guest system.",
 321         optional => 1,
 322         minimum => 1,
 323     },
 324     'prune-backups' => get_standard_option('prune-backups', {
 325         description => "Use these retention options instead of those from the storage configuration.",
 326         optional => 1,
 327         default => "keep-all=1",
 328     }),
 329     remove => {
 330         type => 'boolean',
 331         description => "Prune older backups according to 'prune-backups'.",
 332         optional => 1,
 333         default => 1,
 334     },
 335     pool => {
 336         type => 'string',
 337         description => 'Backup all known guest systems included in the specified pool.',
 338         optional => 1,
 339     },
 340     'notes-template' => {
 341         type => 'string',
 342         description => "Template string for generating notes for the backup(s). It can contain"
 343             ." variables which will be replaced by their values. Currently supported are"
 344             ." {{cluster}}, {{guestname}}, {{node}}, and {{vmid}}, but more might be added in the"
 345             ." future. Needs to be a single line, newline and backslash need to be escaped as '\\n'"
 346             ." and '\\\\' respectively.",
 347         requires => 'storage',
 348         maxLength => 1024,
 349         optional => 1,
 350     },
 351     protected => {
 352         type => 'boolean',
 353         description => "If true, mark backup(s) as protected.",
 354         requires => 'storage',
 355         optional => 1,
 356     },
 357 };
 358
 359 sub get_confdesc {
 360     return $confdesc;
 361 }
 362
 363 # add JSON properties for create and set function
 364 sub json_config_properties {
 365     my $prop = shift;
 366
 367     foreach my $opt (keys %$confdesc) {
 368         $prop->{$opt} = $confdesc->{$opt};
 369     }
 370
 371     return $prop;
 372 }
 373
 374 my $vzdump_properties = {
 375     additionalProperties => 0,
 376     properties => json_config_properties({}),
 377 };
 378
 379 sub parse_vzdump_cron_config {
 380     my ($filename, $raw) = @_;
 381
 382     my $jobs = []; # correct jobs
 383
 384     my $ejobs = []; # mailfomerd lines
 385
 386     my $jid = 1; # we start at 1
 387
 388     my $digest = Digest::SHA::sha1_hex(defined($raw) ? $raw : '');
 389
 390     while ($raw && $raw =~ s/^(.*?)(\n|$)//) {
 391         my $line = $1;
 392
 393         next if $line =~ m/^\#/;
 394         next if $line =~ m/^\s*$/;
 395         next if $line =~ m/^PATH\s*=/; # we always overwrite path
 396
 397         if ($line =~ m|^(\d+)\s+(\d+)\s+\*\s+\*\s+(\S+)\s+root\s+(/\S+/)?(#)?vzdump(\s+(.*))?$|) {
 398             eval {
 399                 my $minute = int($1);
 400                 my $hour = int($2);
 401                 my $dow = $3;
 402                 my $param = $7;
 403                 my $enabled = $5;
 404
 405                 my $dowhash = parse_dow($dow, 1);
 406                 die "unable to parse day of week '$dow' in '$filename'\n" if !$dowhash;
 407
 408                 my $args = PVE::Tools::split_args($param);
 409                 my $opts = PVE::JSONSchema::get_options($vzdump_properties, $args, 'vmid');
 410
 411                 $opts->{enabled} = !defined($enabled);
 412                 $opts->{id} = "$digest:$jid";
 413                 $jid++;
 414                 $opts->{starttime} = sprintf "%02d:%02d", $hour, $minute;
 415                 $opts->{dow} = &$dowhash_to_dow($dowhash);
 416
 417                 parse_property_strings($opts);
 418
 419                 push @$jobs, $opts;
 420             };
 421             my $err = $@;
 422             if ($err) {
 423                 syslog ('err', "parse error in '$filename': $err");
 424                 push @$ejobs, { line => $line };
 425             }
 426         } elsif ($line =~ m|^\S+\s+(\S+)\s+\S+\s+\S+\s+\S+\s+\S+\s+(\S.*)$|) {
 427             syslog ('err', "warning: malformed line in '$filename'");
 428             push @$ejobs, { line => $line };
 429         } else {
 430             syslog ('err', "ignoring malformed line in '$filename'");
 431         }
 432     }
 433
 434     my $res = {};
 435     $res->{digest} = $digest;
 436     $res->{jobs} = $jobs;
 437     $res->{ejobs} = $ejobs;
 438
 439     return $res;
 440 }
 441
 442 sub write_vzdump_cron_config {
 443     my ($filename, $cfg) = @_;
 444
 445     my $out = "# cluster wide vzdump cron schedule\n";
 446     $out .= "# Automatically generated file - do not edit\n\n";
 447     $out .= "PATH=\"/usr/sbin:/usr/bin:/sbin:/bin\"\n\n";
 448
 449     my $jobs = $cfg->{jobs} || [];
 450     foreach my $job (@$jobs) {
 451         my $enabled = ($job->{enabled}) ? '' : '#';
 452         my $dh = parse_dow($job->{dow});
 453         my $dow;
 454         if ($dh->{mon} && $dh->{tue} && $dh->{wed} && $dh->{thu} &&
 455             $dh->{fri} && $dh->{sat} && $dh->{sun}) {
 456             $dow = '*';
 457         } else {
 458             $dow = &$dowhash_to_dow($dh, 1);
 459             $dow = '*' if !$dow;
 460         }
 461
 462         my ($hour, $minute);
 463
 464         die "no job start time specified\n" if !$job->{starttime};
 465         if ($job->{starttime} =~ m/^(\d{1,2}):(\d{1,2})$/) {
 466             ($hour, $minute) = (int($1), int($2));
 467             die "hour '$hour' out of range\n" if $hour < 0 || $hour > 23;
 468             die "minute '$minute' out of range\n" if $minute < 0 || $minute > 59;
 469         } else {
 470             die "unable to parse job start time\n";
 471         }
 472
 473         $job->{quiet} = 1; # we do not want messages from cron
 474
 475         my $cmd = command_line($job);
 476
 477         $out .= sprintf "$minute $hour * * %-11s root $enabled$cmd\n", $dow;
 478     }
 479
 480     my $ejobs = $cfg->{ejobs} || [];
 481     foreach my $job (@$ejobs) {
 482         $out .= "$job->{line}\n" if $job->{line};
 483     }
 484
 485     return $out;
 486 }
 487
 488 sub command_line {
 489     my ($param) = @_;
 490
 491     my $cmd = "vzdump";
 492
 493     if ($param->{vmid}) {
 494         $cmd .= " " . join(' ', PVE::Tools::split_list($param->{vmid}));
 495     }
 496
 497     foreach my $p (keys %$param) {
 498         next if $p eq 'id' || $p eq 'vmid' || $p eq 'starttime' ||
 499                 $p eq 'dow' || $p eq 'stdout' || $p eq 'enabled';
 500         my $v = $param->{$p};
 501         my $pd = $confdesc->{$p} || die "no such vzdump option '$p'\n";
 502         if ($p eq 'exclude-path') {
 503             foreach my $path (@$v) {
 504                 $cmd .= " --$p " . PVE::Tools::shellquote($path);
 505             }
 506         } else {
 507             $v = join(",", PVE::Tools::split_list($v)) if $p eq 'mailto';
 508             $v = PVE::JSONSchema::print_property_string($v, $PROPERTY_STRINGS->{$p})
 509                 if $PROPERTY_STRINGS->{$p};
 510
 511             $cmd .= " --$p " . PVE::Tools::shellquote($v) if defined($v) && $v ne '';
 512         }
 513     }
 514
 515     return $cmd;
 516 }
 517
 518 # bash completion helpers
 519 sub complete_backup_storage {
 520
 521     my $cfg = PVE::Storage::config();
 522     my $ids = $cfg->{ids};
 523
 524     my $nodename = PVE::INotify::nodename();
 525
 526     my $res = [];
 527     foreach my $sid (keys %$ids) {
 528         my $scfg = $ids->{$sid};
 529         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $nodename, 1);
 530         next if !$scfg->{content}->{backup};
 531         push @$res, $sid;
 532     }
 533
 534     return $res;
 535 }
 536
 537 1;