src/PMG/Backup.pm

   1 package PMG::Backup;
   2
   3 use strict;
   4 use warnings;
   5 use Data::Dumper;
   6 use File::Basename;
   7 use File::Find;
   8 use File::Path;
   9 use POSIX qw(strftime);
  10
  11 use PVE::JSONSchema qw(get_standard_option);
  12 use PVE::Tools;
  13
  14 use PMG::pmgcfg;
  15 use PMG::AtomicFile;
  16 use PMG::Utils qw(postgres_admin_cmd);
  17
  18 my $sa_configs = [
  19     "/etc/mail/spamassassin/custom.cf",
  20     "/etc/mail/spamassassin/pmg-scores.cf",
  21 ];
  22
  23 sub get_restore_options {
  24     return (
  25         node => get_standard_option('pve-node'),
  26         config => {
  27             description => "Restore system configuration.",
  28             type => 'boolean',
  29             optional => 1,
  30             default => 0,
  31         },
  32         database => {
  33             description => "Restore the rule database. This is the default.",
  34             type => 'boolean',
  35             optional => 1,
  36             default => 1,
  37         },
  38         statistic => {
  39             description => "Restore statistic databases. Only considered when you restore the 'database'.",
  40             type => 'boolean',
  41             optional => 1,
  42             default => 0,
  43         });
  44 }
  45
  46 sub dump_table {
  47     my ($dbh, $table, $ofh, $seq, $seqcol) = @_;
  48
  49     my $sth = $dbh->column_info(undef, undef, $table, undef);
  50
  51     my $attrs = $sth->fetchall_arrayref({});
  52
  53     my @col_arr;
  54     foreach my $ref (@$attrs) {
  55         push @col_arr, $ref->{COLUMN_NAME};
  56     }
  57
  58     $sth->finish();
  59
  60     my $cols = join (', ', @col_arr);
  61     $cols || die "unable to fetch column definitions: ERROR";
  62
  63     print $ofh "COPY $table ($cols) FROM stdin;\n";
  64
  65     my $cmd = "COPY $table ($cols) TO STDOUT";
  66     $dbh->do($cmd);
  67
  68     my $data = '';
  69     while ($dbh->pg_getcopydata($data) >= 0) {
  70         print $ofh $data;
  71     }
  72
  73     print $ofh "\\.\n\n";
  74
  75     if ($seq && $seqcol) {
  76         print $ofh "SELECT setval('$seq', max($seqcol)) FROM $table;\n\n";
  77     }
  78 }
  79
  80 sub dumpdb {
  81     my ($ofh) = @_;
  82
  83     print $ofh "SET client_encoding = 'SQL_ASCII';\n";
  84     print $ofh "SET check_function_bodies = false;\n\n";
  85
  86     my $dbh = PMG::DBTools::open_ruledb();
  87
  88     print $ofh "BEGIN TRANSACTION;\n\n";
  89
  90     eval {
  91         $dbh->begin_work;
  92
  93         # read a consistent snapshot
  94         $dbh->do("SET TRANSACTION ISOLATION LEVEL SERIALIZABLE");
  95
  96         dump_table($dbh, 'attribut', $ofh);
  97         dump_table($dbh, 'object', $ofh, 'object_id_seq', 'id');
  98         dump_table($dbh, 'objectgroup', $ofh, 'objectgroup_id_seq', 'id');
  99         dump_table($dbh, 'rule', $ofh, 'rule_id_seq', 'id');
 100         dump_table($dbh, 'rulegroup', $ofh);
 101         dump_table($dbh, 'userprefs', $ofh);
 102
 103         # we do not save the following tables: cgreylist, cmailstore, cmsreceivers, clusterinfo
 104     };
 105     my $err = $@;
 106
 107     $dbh->rollback(); # end read-only transaction
 108
 109     $dbh->disconnect();
 110
 111     die $err if $err;
 112
 113     print $ofh "COMMIT TRANSACTION;\n\n";
 114 }
 115
 116 sub dumpstatdb {
 117     my ($ofh) = @_;
 118
 119     print $ofh "SET client_encoding = 'SQL_ASCII';\n";
 120     print $ofh "SET check_function_bodies = false;\n\n";
 121
 122     my $dbh = PMG::DBTools::open_ruledb();
 123
 124     eval {
 125         $dbh->begin_work;
 126
 127         # read a consistent snapshot
 128         $dbh->do("SET TRANSACTION ISOLATION LEVEL SERIALIZABLE");
 129
 130         print $ofh "BEGIN TRANSACTION;\n\n";
 131
 132         dump_table($dbh, 'dailystat', $ofh);
 133         dump_table($dbh, 'domainstat', $ofh);
 134         dump_table($dbh, 'virusinfo', $ofh);
 135         dump_table($dbh, 'localstat', $ofh);
 136
 137         # drop/create the index is a little bit faster (20%)
 138
 139         print $ofh "DROP INDEX cstatistic_time_index;\n\n";
 140         print $ofh "ALTER TABLE cstatistic DROP CONSTRAINT cstatistic_id_key;\n\n";
 141         print $ofh "ALTER TABLE cstatistic DROP CONSTRAINT cstatistic_pkey;\n\n";
 142         dump_table($dbh, 'cstatistic', $ofh, 'cstatistic_id_seq', 'id');
 143         print $ofh "ALTER TABLE ONLY cstatistic ADD CONSTRAINT cstatistic_pkey PRIMARY KEY (cid, rid);\n\n";
 144         print $ofh "ALTER TABLE ONLY cstatistic ADD CONSTRAINT cstatistic_id_key UNIQUE (id);\n\n";
 145         print $ofh "CREATE INDEX CStatistic_Time_Index ON CStatistic (Time);\n\n";
 146
 147         print $ofh "DROP INDEX CStatistic_ID_Index;\n\n";
 148         dump_table($dbh, 'creceivers', $ofh);
 149         print $ofh "CREATE INDEX CStatistic_ID_Index ON CReceivers (CStatistic_CID, CStatistic_RID);\n\n";
 150
 151         dump_table($dbh, 'statinfo', $ofh);
 152
 153         print $ofh "COMMIT TRANSACTION;\n\n";
 154     };
 155     my $err = $@;
 156
 157     $dbh->rollback(); # end read-only transaction
 158
 159     $dbh->disconnect();
 160
 161     die $err if $err;
 162 }
 163
 164 # this function assumes that directory $dirname exists and is empty
 165 sub pmg_backup {
 166     my ($dirname, $include_statistics) = @_;
 167
 168     die "No backupdir provided!\n" if !defined($dirname);
 169
 170     my $time = time;
 171     my $dbfn = "Proxmox_ruledb.sql";
 172     my $statfn = "Proxmox_statdb.sql";
 173     my $tarfn = "config_backup.tar";
 174     my $sigfn = "proxmox_backup_v1.md5";
 175     my $verfn = "version.txt";
 176
 177     eval {
 178
 179         # dump the database first
 180         my $fh = PMG::AtomicFile->open("$dirname/$dbfn", "w") ||
 181             die "can't open '$dirname/$dbfn' - $! :ERROR";
 182
 183         dumpdb($fh);
 184
 185         $fh->close(1);
 186
 187         if ($include_statistics) {
 188             # dump the statistic db
 189             my $sfh = PMG::AtomicFile->open("$dirname/$statfn", "w") ||
 190                 die "can't open '$dirname/$statfn' - $! :ERROR";
 191
 192             dumpstatdb($sfh);
 193
 194             $sfh->close(1);
 195         }
 196
 197         my $pkg = PMG::pmgcfg::package();
 198         my $release = PMG::pmgcfg::release();
 199
 200         my $vfh = PMG::AtomicFile->open ("$dirname/$verfn", "w") ||
 201             die "can't open '$dirname/$verfn' - $! :ERROR";
 202
 203         $time = time;
 204         my $now = localtime;
 205         print $vfh "product: $pkg\nversion: $release\nbackuptime:$time:$now\n";
 206         $vfh->close(1);
 207
 208         my $extra_cfgs =  [];
 209
 210         push @$extra_cfgs, @{$sa_configs};
 211
 212         my $extradb = $include_statistics ? $statfn : '';
 213
 214         my $extra = join(' ', @$extra_cfgs);
 215
 216         system("/bin/tar cf $dirname/$tarfn -C / " .
 217                "/etc/pmg $extra>/dev/null 2>&1") == 0 ||
 218                die "unable to create system configuration backup: ERROR";
 219
 220         system("cd $dirname; md5sum $tarfn $dbfn $extradb $verfn> $sigfn") == 0 ||
 221             die "unable to create backup signature: ERROR";
 222
 223     };
 224     my $err = $@;
 225
 226     if ($err) {
 227         die $err;
 228     }
 229 }
 230
 231 sub pmg_backup_pack {
 232     my ($filename, $include_statistics) = @_;
 233
 234     my $time = time;
 235     my $dirname = "/tmp/proxbackup_$$.$time";
 236
 237     eval {
 238
 239         my $targetdir = dirname($filename);
 240         mkdir $targetdir; # try to create target dir
 241         -d $targetdir ||
 242             die "unable to access target directory '$targetdir'\n";
 243
 244         rmtree $dirname;
 245         # create backup directory
 246         mkdir $dirname;
 247
 248         pmg_backup($dirname, $include_statistics);
 249
 250         system("rm -f $filename; tar czf $filename --strip-components=1 -C $dirname .") == 0 ||
 251             die "unable to create backup archive: ERROR\n";
 252     };
 253     my $err = $@;
 254
 255     rmtree $dirname;
 256
 257     if ($err) {
 258         unlink $filename;
 259         die $err;
 260     }
 261 }
 262
 263 sub pmg_restore {
 264     my ($filename, $restore_database, $restore_config, $restore_statistics) = @_;
 265
 266     my $dbname = 'Proxmox_ruledb';
 267
 268     my $time = time;
 269     my $dirname = "/tmp/proxrestore_$$.$time";
 270     my $dbfn = "Proxmox_ruledb.sql";
 271     my $statfn = "Proxmox_statdb.sql";
 272     my $tarfn = "config_backup.tar";
 273     my $sigfn = "proxmox_backup_v1.md5";
 274
 275     my $untar = 1;
 276
 277     # directory indicates that the files were restored from a PBS remote
 278     if ( -d $filename ) {
 279         $dirname = $filename;
 280         $untar = 0;
 281     }
 282
 283     eval {
 284
 285         if ($untar) {
 286             # remove any leftovers
 287             rmtree $dirname;
 288             # create a temporary directory
 289             mkdir $dirname;
 290
 291             system("cd $dirname; tar xzf $filename >/dev/null 2>&1") == 0 ||
 292                 die "unable to extract backup archive: ERROR";
 293         }
 294
 295         system("cd $dirname; md5sum -c $sigfn") == 0 ||
 296             die "proxmox backup signature check failed: ERROR";
 297
 298         if ($restore_config) {
 299             # restore the tar file
 300             mkdir "$dirname/config/";
 301             system("tar xpf $dirname/$tarfn -C $dirname/config/") == 0 ||
 302                 die "unable to restore configuration tar archive: ERROR";
 303
 304             -d "$dirname/config/etc/pmg" ||
 305                 die "backup does not contain a valid system configuration directory (/etc/pmg)\n";
 306             # unlink unneeded files
 307             unlink "$dirname/config/etc/pmg/cluster.conf"; # never restore cluster config
 308             rmtree "$dirname/config/etc/pmg/master";
 309
 310             # remove current config, but keep directories for INotify
 311             File::Find::find(
 312                 sub {
 313                     my $file = $File::Find::name;
 314                     return if -d $file;
 315                     unlink($file) || $! == POSIX::ENOENT || die "removing $file failed: $!\n";
 316                 },
 317                 '/etc/pmg',
 318             );
 319
 320             # copy files
 321             system("cp -a $dirname/config/etc/pmg/* /etc/pmg/") == 0 ||
 322                 die "unable to restore system configuration: ERROR";
 323
 324             for my $sa_cfg (@{$sa_configs}) {
 325                 if (-f "$dirname/config/${sa_cfg}") {
 326                     my $data = PVE::Tools::file_get_contents(
 327                         "$dirname/config/${sa_cfg}", 1024*1024);
 328                     PVE::Tools::file_set_contents($sa_cfg, $data);
 329                 }
 330             }
 331
 332             my $cfg = PMG::Config->new();
 333             my $ruledb = PMG::RuleDB->new();
 334             my $rulecache = PMG::RuleCache->new($ruledb);
 335             $cfg->rewrite_config($rulecache, 1);
 336         }
 337
 338         if ($restore_database) {
 339             # recreate the database
 340
 341             # stop all services accessing the database
 342             PMG::Utils::service_wait_stopped(40, $PMG::Utils::db_service_list);
 343
 344             print "Destroy existing rule database\n";
 345             PMG::DBTools::delete_ruledb($dbname);
 346
 347             print "Create new database\n";
 348             my $dbh = PMG::DBTools::create_ruledb($dbname);
 349
 350             system("cat $dirname/$dbfn|psql $dbname >/dev/null 2>&1") == 0 ||
 351                 die "unable to restore rule database: ERROR";
 352
 353             if ($restore_statistics) {
 354                 if (-f "$dirname/$statfn") {
 355                     system("cat $dirname/$statfn|psql $dbname >/dev/null 2>&1") == 0 ||
 356                         die "unable to restore statistic database: ERROR";
 357                 }
 358             }
 359
 360             print STDERR "run analyze to speed up database queries\n";
 361             postgres_admin_cmd('psql', { input => 'analyze;' }, $dbname);
 362
 363             print "Analyzing/Upgrading existing Databases...";
 364             my $ruledb = PMG::RuleDB->new($dbh);
 365             PMG::DBTools::upgradedb($ruledb);
 366             print "done\n";
 367
 368             # cleanup old spam/virus storage
 369             PMG::MailQueue::create_spooldirs(0, 1);
 370
 371             my $cfg = PMG::Config->new();
 372             my $rulecache = PMG::RuleCache->new($ruledb);
 373             $cfg->rewrite_config($rulecache, 1);
 374
 375             # and restart services as soon as possible
 376             foreach my $service (reverse @$PMG::Utils::db_service_list) {
 377                 eval { PVE::Tools::run_command(['systemctl', 'start', $service]); };
 378                 warn $@ if $@;
 379             }
 380         }
 381     };
 382     my $err = $@;
 383
 384     rmtree $dirname;
 385
 386     die $err if $err;
 387 }
 388
 389 sub send_backup_notification {
 390     my ($notify_on, $target, $log, $err) = @_;
 391
 392     return if !$notify_on;
 393     return if $notify_on eq 'never';
 394     return if $notify_on eq 'error' && !$err;
 395
 396     my $cfg = PMG::Config->new();
 397     my $email = $cfg->get ('admin', 'email');
 398     if (!$email) {
 399         warn "not sending notification: no admin email configured\n";
 400         return;
 401     }
 402
 403     my $nodename = PVE::INotify::nodename();
 404     my $fqdn = PVE::Tools::get_fqdn($nodename);
 405
 406
 407     my $vars = {
 408         hostname => $nodename,
 409         fqdn => $fqdn,
 410         date => strftime("%F", localtime()),
 411         target => $target,
 412         log => $log,
 413         err => $err,
 414     };
 415
 416     my $tt = PMG::Config::get_template_toolkit();
 417
 418     my $mailfrom = "Proxmox Mail Gateway <postmaster>";
 419     PMG::Utils::finalize_report($tt, 'backup-notification.tt', $vars, $mailfrom, $email);
 420
 421 }
 422
 423 1;