src/PMG/Backup.pm

   1 package PMG::Backup;
   2
   3 use strict;
   4 use warnings;
   5 use Data::Dumper;
   6 use File::Basename;
   7 use File::Path;
   8 use POSIX qw(strftime);
   9
  10 use PVE::JSONSchema qw(get_standard_option);
  11 use PVE::Tools;
  12
  13 use PMG::pmgcfg;
  14 use PMG::AtomicFile;
  15 use PMG::Utils qw(postgres_admin_cmd);
  16
  17 my $sa_configs = [
  18     "/etc/mail/spamassassin/custom.cf",
  19     "/etc/mail/spamassassin/pmg-scores.cf",
  20 ];
  21
  22 sub get_restore_options {
  23     return (
  24         node => get_standard_option('pve-node'),
  25         config => {
  26             description => "Restore system configuration.",
  27             type => 'boolean',
  28             optional => 1,
  29             default => 0,
  30         },
  31         database => {
  32             description => "Restore the rule database. This is the default.",
  33             type => 'boolean',
  34             optional => 1,
  35             default => 1,
  36         },
  37         statistic => {
  38             description => "Restore statistic databases. Only considered when you restore the 'database'.",
  39             type => 'boolean',
  40             optional => 1,
  41             default => 0,
  42         });
  43 }
  44
  45 sub dump_table {
  46     my ($dbh, $table, $ofh, $seq, $seqcol) = @_;
  47
  48     my $sth = $dbh->column_info(undef, undef, $table, undef);
  49
  50     my $attrs = $sth->fetchall_arrayref({});
  51
  52     my @col_arr;
  53     foreach my $ref (@$attrs) {
  54         push @col_arr, $ref->{COLUMN_NAME};
  55     }
  56
  57     $sth->finish();
  58
  59     my $cols = join (', ', @col_arr);
  60     $cols || die "unable to fetch column definitions: ERROR";
  61
  62     print $ofh "COPY $table ($cols) FROM stdin;\n";
  63
  64     my $cmd = "COPY $table ($cols) TO STDOUT";
  65     $dbh->do($cmd);
  66
  67     my $data = '';
  68     while ($dbh->pg_getcopydata($data) >= 0) {
  69         print $ofh $data;
  70     }
  71
  72     print $ofh "\\.\n\n";
  73
  74     if ($seq && $seqcol) {
  75         print $ofh "SELECT setval('$seq', max($seqcol)) FROM $table;\n\n";
  76     }
  77 }
  78
  79 sub dumpdb {
  80     my ($ofh) = @_;
  81
  82     print $ofh "SET client_encoding = 'SQL_ASCII';\n";
  83     print $ofh "SET check_function_bodies = false;\n\n";
  84
  85     my $dbh = PMG::DBTools::open_ruledb();
  86
  87     print $ofh "BEGIN TRANSACTION;\n\n";
  88
  89     eval {
  90         $dbh->begin_work;
  91
  92         # read a consistent snapshot
  93         $dbh->do("SET TRANSACTION ISOLATION LEVEL SERIALIZABLE");
  94
  95         dump_table($dbh, 'attribut', $ofh);
  96         dump_table($dbh, 'object', $ofh, 'object_id_seq', 'id');
  97         dump_table($dbh, 'objectgroup', $ofh, 'objectgroup_id_seq', 'id');
  98         dump_table($dbh, 'rule', $ofh, 'rule_id_seq', 'id');
  99         dump_table($dbh, 'rulegroup', $ofh);
 100         dump_table($dbh, 'userprefs', $ofh);
 101
 102         # we do not save the following tables: cgreylist, cmailstore, cmsreceivers, clusterinfo
 103     };
 104     my $err = $@;
 105
 106     $dbh->rollback(); # end read-only transaction
 107
 108     $dbh->disconnect();
 109
 110     die $err if $err;
 111
 112     print $ofh "COMMIT TRANSACTION;\n\n";
 113 }
 114
 115 sub dumpstatdb {
 116     my ($ofh) = @_;
 117
 118     print $ofh "SET client_encoding = 'SQL_ASCII';\n";
 119     print $ofh "SET check_function_bodies = false;\n\n";
 120
 121     my $dbh = PMG::DBTools::open_ruledb();
 122
 123     eval {
 124         $dbh->begin_work;
 125
 126         # read a consistent snapshot
 127         $dbh->do("SET TRANSACTION ISOLATION LEVEL SERIALIZABLE");
 128
 129         print $ofh "BEGIN TRANSACTION;\n\n";
 130
 131         dump_table($dbh, 'dailystat', $ofh);
 132         dump_table($dbh, 'domainstat', $ofh);
 133         dump_table($dbh, 'virusinfo', $ofh);
 134         dump_table($dbh, 'localstat', $ofh);
 135
 136         # drop/create the index is a little bit faster (20%)
 137
 138         print $ofh "DROP INDEX cstatistic_time_index;\n\n";
 139         print $ofh "ALTER TABLE cstatistic DROP CONSTRAINT cstatistic_id_key;\n\n";
 140         print $ofh "ALTER TABLE cstatistic DROP CONSTRAINT cstatistic_pkey;\n\n";
 141         dump_table($dbh, 'cstatistic', $ofh, 'cstatistic_id_seq', 'id');
 142         print $ofh "ALTER TABLE ONLY cstatistic ADD CONSTRAINT cstatistic_pkey PRIMARY KEY (cid, rid);\n\n";
 143         print $ofh "ALTER TABLE ONLY cstatistic ADD CONSTRAINT cstatistic_id_key UNIQUE (id);\n\n";
 144         print $ofh "CREATE INDEX CStatistic_Time_Index ON CStatistic (Time);\n\n";
 145
 146         print $ofh "DROP INDEX CStatistic_ID_Index;\n\n";
 147         dump_table($dbh, 'creceivers', $ofh);
 148         print $ofh "CREATE INDEX CStatistic_ID_Index ON CReceivers (CStatistic_CID, CStatistic_RID);\n\n";
 149
 150         dump_table($dbh, 'statinfo', $ofh);
 151
 152         print $ofh "COMMIT TRANSACTION;\n\n";
 153     };
 154     my $err = $@;
 155
 156     $dbh->rollback(); # end read-only transaction
 157
 158     $dbh->disconnect();
 159
 160     die $err if $err;
 161 }
 162
 163 # this function assumes that directory $dirname exists and is empty
 164 sub pmg_backup {
 165     my ($dirname, $include_statistics) = @_;
 166
 167     die "No backupdir provided!\n" if !defined($dirname);
 168
 169     my $time = time;
 170     my $dbfn = "Proxmox_ruledb.sql";
 171     my $statfn = "Proxmox_statdb.sql";
 172     my $tarfn = "config_backup.tar";
 173     my $sigfn = "proxmox_backup_v1.md5";
 174     my $verfn = "version.txt";
 175
 176     eval {
 177
 178         # dump the database first
 179         my $fh = PMG::AtomicFile->open("$dirname/$dbfn", "w") ||
 180             die "cant open '$dirname/$dbfn' - $! :ERROR";
 181
 182         dumpdb($fh);
 183
 184         $fh->close(1);
 185
 186         if ($include_statistics) {
 187             # dump the statistic db
 188             my $sfh = PMG::AtomicFile->open("$dirname/$statfn", "w") ||
 189                 die "cant open '$dirname/$statfn' - $! :ERROR";
 190
 191             dumpstatdb($sfh);
 192
 193             $sfh->close(1);
 194         }
 195
 196         my $pkg = PMG::pmgcfg::package();
 197         my $release = PMG::pmgcfg::release();
 198
 199         my $vfh = PMG::AtomicFile->open ("$dirname/$verfn", "w") ||
 200             die "cant open '$dirname/$verfn' - $! :ERROR";
 201
 202         $time = time;
 203         my $now = localtime;
 204         print $vfh "product: $pkg\nversion: $release\nbackuptime:$time:$now\n";
 205         $vfh->close(1);
 206
 207         my $extra_cfgs =  [];
 208
 209         push @$extra_cfgs, @{$sa_configs};
 210
 211         my $extradb = $include_statistics ? $statfn : '';
 212
 213         my $extra = join(' ', @$extra_cfgs);
 214
 215         system("/bin/tar cf $dirname/$tarfn -C / " .
 216                "/etc/pmg $extra>/dev/null 2>&1") == 0 ||
 217                die "unable to create system configuration backup: ERROR";
 218
 219         system("cd $dirname; md5sum $tarfn $dbfn $extradb $verfn> $sigfn") == 0 ||
 220             die "unable to create backup signature: ERROR";
 221
 222     };
 223     my $err = $@;
 224
 225     if ($err) {
 226         die $err;
 227     }
 228 }
 229
 230 sub pmg_backup_pack {
 231     my ($filename, $include_statistics) = @_;
 232
 233     my $time = time;
 234     my $dirname = "/tmp/proxbackup_$$.$time";
 235
 236     eval {
 237
 238         my $targetdir = dirname($filename);
 239         mkdir $targetdir; # try to create target dir
 240         -d $targetdir ||
 241             die "unable to access target directory '$targetdir'\n";
 242
 243         rmtree $dirname;
 244         # create backup directory
 245         mkdir $dirname;
 246
 247         pmg_backup($dirname, $include_statistics);
 248
 249         system("rm -f $filename; tar czf $filename --strip-components=1 -C $dirname .") == 0 ||
 250             die "unable to create backup archive: ERROR\n";
 251     };
 252     my $err = $@;
 253
 254     rmtree $dirname;
 255
 256     if ($err) {
 257         unlink $filename;
 258         die $err;
 259     }
 260 }
 261
 262 sub pmg_restore {
 263     my ($filename, $restore_database, $restore_config, $restore_statistics) = @_;
 264
 265     my $dbname = 'Proxmox_ruledb';
 266
 267     my $time = time;
 268     my $dirname = "/tmp/proxrestore_$$.$time";
 269     my $dbfn = "Proxmox_ruledb.sql";
 270     my $statfn = "Proxmox_statdb.sql";
 271     my $tarfn = "config_backup.tar";
 272     my $sigfn = "proxmox_backup_v1.md5";
 273
 274     my $untar = 1;
 275
 276     # directory indicates that the files were restored from a PBS remote
 277     if ( -d $filename ) {
 278         $dirname = $filename;
 279         $untar = 0;
 280     }
 281
 282     eval {
 283
 284         if ($untar) {
 285             # remove any leftovers
 286             rmtree $dirname;
 287             # create a temporary directory
 288             mkdir $dirname;
 289
 290             system("cd $dirname; tar xzf $filename >/dev/null 2>&1") == 0 ||
 291                 die "unable to extract backup archive: ERROR";
 292         }
 293
 294         system("cd $dirname; md5sum -c $sigfn") == 0 ||
 295             die "proxmox backup signature check failed: ERROR";
 296
 297         if ($restore_config) {
 298             # restore the tar file
 299             mkdir "$dirname/config/";
 300             system("tar xpf $dirname/$tarfn -C $dirname/config/") == 0 ||
 301                 die "unable to restore configuration tar archive: ERROR";
 302
 303             -d "$dirname/config/etc/pmg" ||
 304                 die "backup does not contain a valid system configuration directory (/etc/pmg)\n";
 305             # unlink unneeded files
 306             unlink "$dirname/config/etc/pmg/cluster.conf"; # never restore cluster config
 307             rmtree "$dirname/config/etc/pmg/master";
 308
 309             # remove current config, but keep directory for INotify
 310             rmtree("/etc/pmg", { keep_root => 1 });
 311             # copy files
 312             system("cp -a $dirname/config/etc/pmg/* /etc/pmg/") == 0 ||
 313                 die "unable to restore system configuration: ERROR";
 314
 315             for my $sa_cfg (@{$sa_configs}) {
 316                 if (-f "$dirname/config/${sa_cfg}") {
 317                     my $data = PVE::Tools::file_get_contents(
 318                         "$dirname/config/${sa_cfg}", 1024*1024);
 319                     PVE::Tools::file_set_contents($sa_cfg, $data);
 320                 }
 321             }
 322
 323             my $cfg = PMG::Config->new();
 324             my $ruledb = PMG::RuleDB->new();
 325             my $rulecache = PMG::RuleCache->new($ruledb);
 326             $cfg->rewrite_config($rulecache, 1);
 327         }
 328
 329         if ($restore_database) {
 330             # recreate the database
 331
 332             # stop all services accessing the database
 333             PMG::Utils::service_wait_stopped(40, $PMG::Utils::db_service_list);
 334
 335             print "Destroy existing rule database\n";
 336             PMG::DBTools::delete_ruledb($dbname);
 337
 338             print "Create new database\n";
 339             my $dbh = PMG::DBTools::create_ruledb($dbname);
 340
 341             system("cat $dirname/$dbfn|psql $dbname >/dev/null 2>&1") == 0 ||
 342                 die "unable to restore rule database: ERROR";
 343
 344             if ($restore_statistics) {
 345                 if (-f "$dirname/$statfn") {
 346                     system("cat $dirname/$statfn|psql $dbname >/dev/null 2>&1") == 0 ||
 347                         die "unable to restore statistic database: ERROR";
 348                 }
 349             }
 350
 351             print STDERR "run analyze to speed up database queries\n";
 352             postgres_admin_cmd('psql', { input => 'analyze;' }, $dbname);
 353
 354             print "Analyzing/Upgrading existing Databases...";
 355             my $ruledb = PMG::RuleDB->new($dbh);
 356             PMG::DBTools::upgradedb($ruledb);
 357             print "done\n";
 358
 359             # cleanup old spam/virus storage
 360             PMG::MailQueue::create_spooldirs(0, 1);
 361
 362             my $cfg = PMG::Config->new();
 363             my $rulecache = PMG::RuleCache->new($ruledb);
 364             $cfg->rewrite_config($rulecache, 1);
 365
 366             # and restart services as soon as possible
 367             foreach my $service (reverse @$PMG::Utils::db_service_list) {
 368                 eval { PVE::Tools::run_command(['systemctl', 'start', $service]); };
 369                 warn $@ if $@;
 370             }
 371         }
 372     };
 373     my $err = $@;
 374
 375     rmtree $dirname;
 376
 377     die $err if $err;
 378 }
 379
 380 sub send_backup_notification {
 381     my ($notify_on, $target, $log, $err) = @_;
 382
 383     return if !$notify_on;
 384     return if $notify_on eq 'never';
 385     return if $notify_on eq 'error' && !$err;
 386
 387     my $cfg = PMG::Config->new();
 388     my $email = $cfg->get ('admin', 'email');
 389     if (!$email) {
 390         warn "not sending notifcation: no admin email configured\n";
 391         return;
 392     }
 393
 394     my $nodename = PVE::INotify::nodename();
 395     my $fqdn = PVE::Tools::get_fqdn($nodename);
 396
 397
 398     my $vars = {
 399         hostname => $nodename,
 400         fqdn => $fqdn,
 401         date => strftime("%F", localtime()),
 402         target => $target,
 403         log => $log,
 404         err => $err,
 405     };
 406
 407     my $tt = PMG::Config::get_template_toolkit();
 408
 409     my $mailfrom = "Proxmox Mail Gateway <postmaster>";
 410     PMG::Utils::finalize_report($tt, 'backup-notification.tt', $vars, $mailfrom, $email);
 411
 412 }
 413
 414 1;