[pmg-api.git] / src / PMG / Backup.pm

package PMG::Backup;

use strict;
use warnings;
use Data::Dumper;
use File::Basename;
use File::Path;

use PVE::Tools;

use PMG::pmgcfg;
use PMG::AtomicFile;
use PMG::Utils qw(postgres_admin_cmd);

my $sa_custom_config_fn = "/etc/mail/spamassassin/custom.cf";

sub dump_table {
    my ($dbh, $table, $ofh, $seq, $seqcol) = @_;

    my $sth = $dbh->column_info(undef, undef, $table, undef);

    my $attrs = $sth->fetchall_arrayref({});

    my @col_arr;
    foreach my $ref (@$attrs) {
	push @col_arr, $ref->{COLUMN_NAME};
    }

    $sth->finish();

    my $cols = join (', ', @col_arr);
    $cols || die "unable to fetch column definitions: ERROR";

    print $ofh "COPY $table ($cols) FROM stdin;\n";

    my $cmd = "COPY $table ($cols) TO STDOUT";
    $dbh->do($cmd);

    my $data = '';
    while ($dbh->pg_getcopydata($data) >= 0) {
	print $ofh $data;
    }

    print $ofh "\\.\n\n";

    if ($seq && $seqcol) {
	print $ofh "SELECT setval('$seq', max($seqcol)) FROM $table;\n\n";
    }
}

sub dumpdb {
    my ($ofh) = @_;

    print $ofh "SET client_encoding = 'SQL_ASCII';\n";
    print $ofh "SET check_function_bodies = false;\n\n";

    my $dbh = PMG::DBTools::open_ruledb();

    print $ofh "BEGIN TRANSACTION;\n\n";

    eval {
	$dbh->begin_work;

	# read a consistent snapshot
	$dbh->do("SET TRANSACTION ISOLATION LEVEL SERIALIZABLE");

	dump_table($dbh, 'attribut', $ofh);
	dump_table($dbh, 'object', $ofh, 'object_id_seq', 'id');
	dump_table($dbh, 'objectgroup', $ofh, 'objectgroup_id_seq', 'id');
	dump_table($dbh, 'rule', $ofh, 'rule_id_seq', 'id');
	dump_table($dbh, 'rulegroup', $ofh);
	dump_table($dbh, 'userprefs', $ofh);

	# we do not save the following tables: cgreylist, cmailstore, cmsreceivers, clusterinfo
    };
    my $err = $@;

    $dbh->rollback(); # end read-only transaction

    $dbh->disconnect();

    die $err if $err;

    print $ofh "COMMIT TRANSACTION;\n\n";
}

sub dumpstatdb {
    my ($ofh) = @_;

    print $ofh "SET client_encoding = 'SQL_ASCII';\n";
    print $ofh "SET check_function_bodies = false;\n\n";

    my $dbh = PMG::DBTools::open_ruledb();

    eval {
	$dbh->begin_work;

	# read a consistent snapshot
	$dbh->do("SET TRANSACTION ISOLATION LEVEL SERIALIZABLE");

	print $ofh "BEGIN TRANSACTION;\n\n";

	dump_table($dbh, 'dailystat', $ofh);
	dump_table($dbh, 'domainstat', $ofh);
	dump_table($dbh, 'virusinfo', $ofh);
	dump_table($dbh, 'localstat', $ofh);

	# drop/create the index is a little bit faster (20%)

	print $ofh "DROP INDEX cstatistic_time_index;\n\n";
	print $ofh "ALTER TABLE cstatistic DROP CONSTRAINT cstatistic_id_key;\n\n";
	print $ofh "ALTER TABLE cstatistic DROP CONSTRAINT cstatistic_pkey;\n\n";
	dump_table($dbh, 'cstatistic', $ofh, 'cstatistic_id_seq', 'id');
	print $ofh "ALTER TABLE ONLY cstatistic ADD CONSTRAINT cstatistic_pkey PRIMARY KEY (cid, rid);\n\n";
	print $ofh "ALTER TABLE ONLY cstatistic ADD CONSTRAINT cstatistic_id_key UNIQUE (id);\n\n";
	print $ofh "CREATE INDEX CStatistic_Time_Index ON CStatistic (Time);\n\n";

	print $ofh "DROP INDEX CStatistic_ID_Index;\n\n";
	dump_table($dbh, 'creceivers', $ofh);
	print $ofh "CREATE INDEX CStatistic_ID_Index ON CReceivers (CStatistic_CID, CStatistic_RID);\n\n";

	dump_table($dbh, 'statinfo', $ofh);

	print $ofh "COMMIT TRANSACTION;\n\n";
    };
    my $err = $@;

    $dbh->rollback(); # end read-only transaction

    $dbh->disconnect();

    die $err if $err;
}

sub pmg_backup {
    my ($filename, $include_statistics) = @_;

    my $time = time;
    my $dirname = "/tmp/proxbackup_$$.$time";
    my $dbfn = "Proxmox_ruledb.sql";
    my $statfn = "Proxmox_statdb.sql";
    my $tarfn = "config_backup.tar";
    my $sigfn = "proxmox_backup_v1.md5";
    my $verfn = "version.txt";

    eval {

	my $targetdir = dirname($filename);
	mkdir $targetdir; # try to create target dir
	-d $targetdir ||
	    die "unable to access target directory '$targetdir'\n";

	# create a temporary directory
	mkdir $dirname;

	# dump the database first
	my $fh = PMG::AtomicFile->open("$dirname/$dbfn", "w") ||
	    die "cant open '$dirname/$dbfn' - $! :ERROR";

	dumpdb($fh);

	$fh->close(1);

	if ($include_statistics) {
	    # dump the statistic db
	    my $sfh = PMG::AtomicFile->open("$dirname/$statfn", "w") ||
		die "cant open '$dirname/$statfn' - $! :ERROR";

	    dumpstatdb($sfh);

	    $sfh->close(1);
	}

	my $pkg = PMG::pmgcfg::package();
	my $release = PMG::pmgcfg::release();

	my $vfh = PMG::AtomicFile->open ("$dirname/$verfn", "w") ||
	    die "cant open '$dirname/$verfn' - $! :ERROR";

	$time = time;
	my $now = localtime;
	print $vfh "product: $pkg\nversion: $release\nbackuptime:$time:$now\n";
	$vfh->close(1);

	my $extra_cfgs =  [];

	push @$extra_cfgs, $sa_custom_config_fn;

	#push @$extra_cfgs, '/etc/postfix/tls_policy';

	my $extradb = $include_statistics ? $statfn : '';

	my $extra = join(' ', @$extra_cfgs);

	system("/bin/tar cf $dirname/$tarfn -C / " .
	       "/etc/pmg $extra>/dev/null 2>&1") == 0 ||
	       die "unable to create system configuration backup: ERROR";

	system("cd $dirname; md5sum $tarfn $dbfn $extradb $verfn> $sigfn") == 0 ||
	    die "unable to create backup signature: ERROR";

	system("rm -f $filename; tar czf $filename -C $dirname $verfn $sigfn $dbfn $extradb $tarfn") == 0 ||
	    die "unable to create backup archive: ERROR";
    };
    my $err = $@;

    rmtree $dirname;

    if ($err) {
	unlink $filename;
	die $err;
    }
}

sub pmg_restore {
    my ($filename, $restore_database, $restore_config, $restore_statistics) = @_;

    my $dbname = 'Proxmox_ruledb';

    my $time = time;
    my $dirname = "/tmp/proxrestore_$$.$time";
    my $dbfn = "Proxmox_ruledb.sql";
    my $statfn = "Proxmox_statdb.sql";
    my $tarfn = "config_backup.tar";
    my $sigfn = "proxmox_backup_v1.md5";

    eval {
	# create a temporary directory
	mkdir $dirname;

	system("cd $dirname; tar xzf $filename >/dev/null 2>&1") == 0 ||
	    die "unable to extract backup archive: ERROR";

	system("cd $dirname; md5sum -c $sigfn") == 0 ||
	    die "proxmox backup signature check failed: ERROR";

	if ($restore_config) {
	    # restore the tar file
	    mkdir "$dirname/config/";
	    system("tar xpf $dirname/$tarfn -C $dirname/config/") == 0 ||
		die "unable to restore configuration tar archive: ERROR";

	    -d "$dirname/config/etc/pmg" ||
		die "backup does not contain a valid system configuration directory (/etc/pmg)\n";
	    # unlink unneeded files
	    unlink "$dirname/config/etc/pmg/cluster.conf"; # never restore cluster config
	    rmtree "$dirname/config/etc/pmg/master";

	    # remove current config, but keep directory for INotify
	    rmtree("/etc/pmg", { keep_root => 1 });
	    # copy files
	    system("cp -a $dirname/config/etc/pmg/* /etc/pmg/") == 0 ||
		die "unable to restore system configuration: ERROR";

	    if (-f "$dirname/config/${sa_custom_config_fn}") {
		my $data = PVE::Tools::file_get_contents(
		    "$dirname/config/${sa_custom_config_fn}", 1024*1024);
		PVE::Tools::file_set_contents($sa_custom_config_fn, $data);
	    }

	    my $cfg = PMG::Config->new();
	    my $ruledb = PMG::RuleDB->new();
	    my $rulecache = PMG::RuleCache->new($ruledb);
	    $cfg->rewrite_config($rulecache, 1);
	}

	if ($restore_database) {
	    # recreate the database

	    # stop all services accessing the database
	    PMG::Utils::service_wait_stopped(40, $PMG::Utils::db_service_list);

	    print "Destroy existing rule database\n";
	    PMG::DBTools::delete_ruledb($dbname);

	    print "Create new database\n";
	    my $dbh = PMG::DBTools::create_ruledb($dbname);

	    system("cat $dirname/$dbfn|psql $dbname >/dev/null 2>&1") == 0 ||
		die "unable to restore rule database: ERROR";

	    if ($restore_statistics) {
		if (-f "$dirname/$statfn") {
		    system("cat $dirname/$statfn|psql $dbname >/dev/null 2>&1") == 0 ||
			die "unable to restore statistic database: ERROR";
		}
	    }

	    print STDERR "run analyze to speed up database queries\n";
	    postgres_admin_cmd('psql', { input => 'analyze;' }, $dbname);

	    print "Analyzing/Upgrading existing Databases...";
	    my $ruledb = PMG::RuleDB->new($dbh);
	    PMG::DBTools::upgradedb($ruledb);
	    print "done\n";

	    # cleanup old spam/virus storage
	    PMG::MailQueue::create_spooldirs(0, 1);

	    my $cfg = PMG::Config->new();
	    my $rulecache = PMG::RuleCache->new($ruledb);
	    $cfg->rewrite_config($rulecache, 1);

	    # and restart services as soon as possible
	    foreach my $service (reverse @$PMG::Utils::db_service_list) {
		eval { PVE::Tools::run_command(['systemctl', 'start', $service]); };
		warn $@ if $@;
	    }
	}
    };
    my $err = $@;

    rmtree $dirname;

    die $err if $err;
}

1;
Commit	Line	Data
9a8d51a4 DM	1	package PMG::Backup;
	2
	3	use strict;
	4	use warnings;
	5	use Data::Dumper;
	6	use File::Basename;
	7	use File::Path;
	8
	9	use PVE::Tools;
	10
	11	use PMG::pmgcfg;
	12	use PMG::AtomicFile;
6529020a	13	use PMG::Utils qw(postgres_admin_cmd);
9a8d51a4	14
29b263e5	15	my $sa_custom_config_fn = "/etc/mail/spamassassin/custom.cf";
520f7717	16
9a8d51a4 DM	17	sub dump_table {
	18	my ($dbh, $table, $ofh, $seq, $seqcol) = @_;
	19
	20	my $sth = $dbh->column_info(undef, undef, $table, undef);
	21
	22	my $attrs = $sth->fetchall_arrayref({});
	23
	24	my @col_arr;
	25	foreach my $ref (@$attrs) {
	26	push @col_arr, $ref->{COLUMN_NAME};
	27	}
	28
	29	$sth->finish();
	30
	31	my $cols = join (', ', @col_arr);
	32	$cols \|\| die "unable to fetch column definitions: ERROR";
	33
	34	print $ofh "COPY $table ($cols) FROM stdin;\n";
	35
	36	my $cmd = "COPY $table ($cols) TO STDOUT";
	37	$dbh->do($cmd);
	38
	39	my $data = '';
	40	while ($dbh->pg_getcopydata($data) >= 0) {
	41	print $ofh $data;
	42	}
	43
	44	print $ofh "\\.\n\n";
	45
	46	if ($seq && $seqcol) {
	47	print $ofh "SELECT setval('$seq', max($seqcol)) FROM $table;\n\n";
	48	}
	49	}
	50
	51	sub dumpdb {
	52	my ($ofh) = @_;
	53
	54	print $ofh "SET client_encoding = 'SQL_ASCII';\n";
	55	print $ofh "SET check_function_bodies = false;\n\n";
	56
	57	my $dbh = PMG::DBTools::open_ruledb();
	58
	59	print $ofh "BEGIN TRANSACTION;\n\n";
	60
	61	eval {
	62	$dbh->begin_work;
	63
	64	# read a consistent snapshot
	65	$dbh->do("SET TRANSACTION ISOLATION LEVEL SERIALIZABLE");
	66
	67	dump_table($dbh, 'attribut', $ofh);
	68	dump_table($dbh, 'object', $ofh, 'object_id_seq', 'id');
	69	dump_table($dbh, 'objectgroup', $ofh, 'objectgroup_id_seq', 'id');
	70	dump_table($dbh, 'rule', $ofh, 'rule_id_seq', 'id');
	71	dump_table($dbh, 'rulegroup', $ofh);
	72	dump_table($dbh, 'userprefs', $ofh);
	73
	74	# we do not save the following tables: cgreylist, cmailstore, cmsreceivers, clusterinfo
	75	};
	76	my $err = $@;
	77
	78	$dbh->rollback(); # end read-only transaction
	79
	80	$dbh->disconnect();
81
82	die $err if $err;
83
84	print $ofh "COMMIT TRANSACTION;\n\n";
85	}
86
87	sub dumpstatdb {
88	my ($ofh) = @_;
89
90	print $ofh "SET client_encoding = 'SQL_ASCII';\n";
91	print $ofh "SET check_function_bodies = false;\n\n";
92
93	my $dbh = PMG::DBTools::open_ruledb();
94
95	eval {
96	$dbh->begin_work;
97
98	# read a consistent snapshot
99	$dbh->do("SET TRANSACTION ISOLATION LEVEL SERIALIZABLE");
100
101	print $ofh "BEGIN TRANSACTION;\n\n";
102
103	dump_table($dbh, 'dailystat', $ofh);
104	dump_table($dbh, 'domainstat', $ofh);
105	dump_table($dbh, 'virusinfo', $ofh);
106	dump_table($dbh, 'localstat', $ofh);
107
108	# drop/create the index is a little bit faster (20%)
109
110	print $ofh "DROP INDEX cstatistic_time_index;\n\n";
111	print $ofh "ALTER TABLE cstatistic DROP CONSTRAINT cstatistic_id_key;\n\n";
112	print $ofh "ALTER TABLE cstatistic DROP CONSTRAINT cstatistic_pkey;\n\n";
113	dump_table($dbh, 'cstatistic', $ofh, 'cstatistic_id_seq', 'id');
114	print $ofh "ALTER TABLE ONLY cstatistic ADD CONSTRAINT cstatistic_pkey PRIMARY KEY (cid, rid);\n\n";
115	print $ofh "ALTER TABLE ONLY cstatistic ADD CONSTRAINT cstatistic_id_key UNIQUE (id);\n\n";
116	print $ofh "CREATE INDEX CStatistic_Time_Index ON CStatistic (Time);\n\n";
117
118	print $ofh "DROP INDEX CStatistic_ID_Index;\n\n";
119	dump_table($dbh, 'creceivers', $ofh);
120	print $ofh "CREATE INDEX CStatistic_ID_Index ON CReceivers (CStatistic_CID, CStatistic_RID);\n\n";
121
122	dump_table($dbh, 'statinfo', $ofh);
123
124	print $ofh "COMMIT TRANSACTION;\n\n";
125	};
126	my $err = $@;
127
128	$dbh->rollback(); # end read-only transaction
129
130	$dbh->disconnect();
131
132	die $err if $err;
133	}
134
135	sub pmg_backup {
136	my ($filename, $include_statistics) = @_;
137
138	my $time = time;
139	my $dirname = "/tmp/proxbackup_$$.$time";
140	my $dbfn = "Proxmox_ruledb.sql";
141	my $statfn = "Proxmox_statdb.sql";
142	my $tarfn = "config_backup.tar";
143	my $sigfn = "proxmox_backup_v1.md5";
144	my $verfn = "version.txt";
145
146	eval {
147
148	my $targetdir = dirname($filename);
149	mkdir $targetdir; # try to create target dir
150	-d $targetdir \|\|
fb9e46a0	151	die "unable to access target directory '$targetdir'\n";
9a8d51a4 DM	152
	153	# create a temporary directory
	154	mkdir $dirname;
	155
	156	# dump the database first
	157	my $fh = PMG::AtomicFile->open("$dirname/$dbfn", "w") \|\|
	158	die "cant open '$dirname/$dbfn' - $! :ERROR";
	159
	160	dumpdb($fh);
	161
	162	$fh->close(1);
	163
	164	if ($include_statistics) {
	165	# dump the statistic db
	166	my $sfh = PMG::AtomicFile->open("$dirname/$statfn", "w") \|\|
	167	die "cant open '$dirname/$statfn' - $! :ERROR";
	168
	169	dumpstatdb($sfh);
	170
	171	$sfh->close(1);
	172	}
	173
	174	my $pkg = PMG::pmgcfg::package();
1c15ce0f	175	my $release = PMG::pmgcfg::release();
9a8d51a4 DM	176
	177	my $vfh = PMG::AtomicFile->open ("$dirname/$verfn", "w") \|\|
	178	die "cant open '$dirname/$verfn' - $! :ERROR";
	179
	180	$time = time;
	181	my $now = localtime;
1c15ce0f	182	print $vfh "product: $pkg\nversion: $release\nbackuptime:$time:$now\n";
9a8d51a4 DM	183	$vfh->close(1);
9a8d51a4 DM	184
bd1325ed	185	my $extra_cfgs = [];
9a8d51a4	186
29b263e5	187	push @$extra_cfgs, $sa_custom_config_fn;
9a8d51a4	188
bd1325ed	189	#push @$extra_cfgs, '/etc/postfix/tls_policy';
9a8d51a4 DM	190
	191	my $extradb = $include_statistics ? $statfn : '';
	192
bd1325ed	193	my $extra = join(' ', @$extra_cfgs);
9a8d51a4 DM	194
9a8d51a4 DM	195	system("/bin/tar cf $dirname/$tarfn -C / " .
bd1325ed	196	"/etc/pmg $extra>/dev/null 2>&1") == 0 \|\|
9a8d51a4 DM	197	die "unable to create system configuration backup: ERROR";
	198
	199	system("cd $dirname; md5sum $tarfn $dbfn $extradb $verfn> $sigfn") == 0 \|\|
	200	die "unable to create backup signature: ERROR";
	201
	202	system("rm -f $filename; tar czf $filename -C $dirname $verfn $sigfn $dbfn $extradb $tarfn") == 0 \|\|
	203	die "unable to create backup archive: ERROR";
	204	};
	205	my $err = $@;
	206
	207	rmtree $dirname;
	208
	209	if ($err) {
	210	unlink $filename;
	211	die $err;
	212	}
	213	}
	214
a6d276e9 DM	215	sub pmg_restore {
	216	my ($filename, $restore_database, $restore_config, $restore_statistics) = @_;
	217
	218	my $dbname = 'Proxmox_ruledb';
	219
	220	my $time = time;
	221	my $dirname = "/tmp/proxrestore_$$.$time";
	222	my $dbfn = "Proxmox_ruledb.sql";
	223	my $statfn = "Proxmox_statdb.sql";
	224	my $tarfn = "config_backup.tar";
	225	my $sigfn = "proxmox_backup_v1.md5";
	226
	227	eval {
	228	# create a temporary directory
	229	mkdir $dirname;
	230
	231	system("cd $dirname; tar xzf $filename >/dev/null 2>&1") == 0 \|\|
	232	die "unable to extract backup archive: ERROR";
	233
	234	system("cd $dirname; md5sum -c $sigfn") == 0 \|\|
	235	die "proxmox backup signature check failed: ERROR";
	236
	237	if ($restore_config) {
	238	# restore the tar file
	239	mkdir "$dirname/config/";
	240	system("tar xpf $dirname/$tarfn -C $dirname/config/") == 0 \|\|
	241	die "unable to restore configuration tar archive: ERROR";
	242
	243	-d "$dirname/config/etc/pmg" \|\|
	244	die "backup does not contain a valid system configuration directory (/etc/pmg)\n";
	245	# unlink unneeded files
	246	unlink "$dirname/config/etc/pmg/cluster.conf"; # never restore cluster config
	247	rmtree "$dirname/config/etc/pmg/master";
	248
8624613e DM	249	# remove current config, but keep directory for INotify
8624613e DM	250	rmtree("/etc/pmg", { keep_root => 1 });
a6d276e9 DM	251	# copy files
	252	system("cp -a $dirname/config/etc/pmg/* /etc/pmg/") == 0 \|\|
	253	die "unable to restore system configuration: ERROR";
	254
29b263e5	255	if (-f "$dirname/config/${sa_custom_config_fn}") {
520f7717	256	my $data = PVE::Tools::file_get_contents(
29b263e5 DM	257	"$dirname/config/${sa_custom_config_fn}", 1024*1024);
29b263e5 DM	258	PVE::Tools::file_set_contents($sa_custom_config_fn, $data);
520f7717 DM	259	}
520f7717 DM	260
a6d276e9 DM	261	my $cfg = PMG::Config->new();
	262	my $ruledb = PMG::RuleDB->new();
	263	my $rulecache = PMG::RuleCache->new($ruledb);
	264	$cfg->rewrite_config($rulecache, 1);
	265	}
	266
	267	if ($restore_database) {
	268	# recreate the database
	269
	270	# stop all services accessing the database
	271	PMG::Utils::service_wait_stopped(40, $PMG::Utils::db_service_list);
	272
	273	print "Destroy existing rule database\n";
	274	PMG::DBTools::delete_ruledb($dbname);
	275
	276	print "Create new database\n";
	277	my $dbh = PMG::DBTools::create_ruledb($dbname);
a6d276e9 DM	278
	279	system("cat $dirname/$dbfn\|psql $dbname >/dev/null 2>&1") == 0 \|\|
	280	die "unable to restore rule database: ERROR";
	281
	282	if ($restore_statistics) {
	283	if (-f "$dirname/$statfn") {
	284	system("cat $dirname/$statfn\|psql $dbname >/dev/null 2>&1") == 0 \|\|
	285	die "unable to restore statistic database: ERROR";
	286	}
	287	}
	288
	289	print STDERR "run analyze to speed up database queries\n";
6529020a	290	postgres_admin_cmd('psql', { input => 'analyze;' }, $dbname);
a6d276e9 DM	291
a6d276e9 DM	292	print "Analyzing/Upgrading existing Databases...";
9882b397	293	my $ruledb = PMG::RuleDB->new($dbh);
a6d276e9 DM	294	PMG::DBTools::upgradedb($ruledb);
	295	print "done\n";
	296
	297	# cleanup old spam/virus storage
	298	PMG::MailQueue::create_spooldirs(0, 1);
	299
	300	my $cfg = PMG::Config->new();
	301	my $rulecache = PMG::RuleCache->new($ruledb);
	302	$cfg->rewrite_config($rulecache, 1);
	303
	304	# and restart services as soon as possible
	305	foreach my $service (reverse @$PMG::Utils::db_service_list) {
	306	eval { PVE::Tools::run_command(['systemctl', 'start', $service]); };
	307	warn $@ if $@;
	308	}
	309	}
	310	};
	311	my $err = $@;
	312
	313	rmtree $dirname;
	314
	315	die $err if $err;
	316	}
	317
9a8d51a4	318	1;