[pmg-api.git] / PMG / Backup.pm

package PMG::Backup;

use strict;
use warnings;
use Data::Dumper;
use File::Basename;
use File::Path;

use PVE::Tools;

use PMG::pmgcfg;
use PMG::AtomicFile;

sub dump_table {
    my ($dbh, $table, $ofh, $seq, $seqcol) = @_;

    my $sth = $dbh->column_info(undef, undef, $table, undef);

    my $attrs = $sth->fetchall_arrayref({});

    my @col_arr;
    foreach my $ref (@$attrs) {
	push @col_arr, $ref->{COLUMN_NAME};
    }

    $sth->finish();

    my $cols = join (', ', @col_arr);
    $cols || die "unable to fetch column definitions: ERROR";

    print $ofh "COPY $table ($cols) FROM stdin;\n";

    my $cmd = "COPY $table ($cols) TO STDOUT";
    $dbh->do($cmd);

    my $data = '';
    while ($dbh->pg_getcopydata($data) >= 0) {
	print $ofh $data;
    }

    print $ofh "\\.\n\n";

    if ($seq && $seqcol) {
	print $ofh "SELECT setval('$seq', max($seqcol)) FROM $table;\n\n";
    }
}

sub dumpdb {
    my ($ofh) = @_;

    print $ofh "SET client_encoding = 'SQL_ASCII';\n";
    print $ofh "SET check_function_bodies = false;\n\n";

    my $dbh = PMG::DBTools::open_ruledb();

    print $ofh "BEGIN TRANSACTION;\n\n";

    eval {
	$dbh->begin_work;

	# read a consistent snapshot
	$dbh->do("SET TRANSACTION ISOLATION LEVEL SERIALIZABLE");

	dump_table($dbh, 'attribut', $ofh);
	dump_table($dbh, 'object', $ofh, 'object_id_seq', 'id');
	dump_table($dbh, 'objectgroup', $ofh, 'objectgroup_id_seq', 'id');
	dump_table($dbh, 'rule', $ofh, 'rule_id_seq', 'id');
	dump_table($dbh, 'rulegroup', $ofh);
	dump_table($dbh, 'userprefs', $ofh);

	# we do not save the following tables: cgreylist, cmailstore, cmsreceivers, clusterinfo
    };
    my $err = $@;

    $dbh->rollback(); # end read-only transaction

    $dbh->disconnect();

    die $err if $err;

    print $ofh "COMMIT TRANSACTION;\n\n";
}

sub dumpstatdb {
    my ($ofh) = @_;

    print $ofh "SET client_encoding = 'SQL_ASCII';\n";
    print $ofh "SET check_function_bodies = false;\n\n";

    my $dbh = PMG::DBTools::open_ruledb();

    eval {
	$dbh->begin_work;

	# read a consistent snapshot
	$dbh->do("SET TRANSACTION ISOLATION LEVEL SERIALIZABLE");

	print $ofh "BEGIN TRANSACTION;\n\n";

	dump_table($dbh, 'dailystat', $ofh);
	dump_table($dbh, 'domainstat', $ofh);
	dump_table($dbh, 'virusinfo', $ofh);
	dump_table($dbh, 'localstat', $ofh);

	# drop/create the index is a little bit faster (20%)

	print $ofh "DROP INDEX cstatistic_time_index;\n\n";
	print $ofh "ALTER TABLE cstatistic DROP CONSTRAINT cstatistic_id_key;\n\n";
	print $ofh "ALTER TABLE cstatistic DROP CONSTRAINT cstatistic_pkey;\n\n";
	dump_table($dbh, 'cstatistic', $ofh, 'cstatistic_id_seq', 'id');
	print $ofh "ALTER TABLE ONLY cstatistic ADD CONSTRAINT cstatistic_pkey PRIMARY KEY (cid, rid);\n\n";
	print $ofh "ALTER TABLE ONLY cstatistic ADD CONSTRAINT cstatistic_id_key UNIQUE (id);\n\n";
	print $ofh "CREATE INDEX CStatistic_Time_Index ON CStatistic (Time);\n\n";

	print $ofh "DROP INDEX CStatistic_ID_Index;\n\n";
	dump_table($dbh, 'creceivers', $ofh);
	print $ofh "CREATE INDEX CStatistic_ID_Index ON CReceivers (CStatistic_CID, CStatistic_RID);\n\n";

	dump_table($dbh, 'statinfo', $ofh);

	print $ofh "COMMIT TRANSACTION;\n\n";
    };
    my $err = $@;

    $dbh->rollback(); # end read-only transaction

    $dbh->disconnect();

    die $err if $err;
}

sub pmg_backup {
    my ($filename, $include_statistics) = @_;

    my $time = time;
    my $dirname = "/tmp/proxbackup_$$.$time";
    my $dbfn = "Proxmox_ruledb.sql";
    my $statfn = "Proxmox_statdb.sql";
    my $tarfn = "config_backup.tar";
    my $sigfn = "proxmox_backup_v1.md5";
    my $verfn = "version.txt";

    eval {

	my $targetdir = dirname($filename);
	mkdir $targetdir; # try to create target dir
	-d $targetdir ||
	    die "unable to access target directory '$targetdir'\n";

	# create a temporary directory
	mkdir $dirname;

	# dump the database first
	my $fh = PMG::AtomicFile->open("$dirname/$dbfn", "w") ||
	    die "cant open '$dirname/$dbfn' - $! :ERROR";

	dumpdb($fh);

	$fh->close(1);

	if ($include_statistics) {
	    # dump the statistic db
	    my $sfh = PMG::AtomicFile->open("$dirname/$statfn", "w") ||
		die "cant open '$dirname/$statfn' - $! :ERROR";

	    dumpstatdb($sfh);

	    $sfh->close(1);
	}

	my $pkg = PMG::pmgcfg::package();
	my $ver = PMG::pmgcfg::version();

	my $vfh = PMG::AtomicFile->open ("$dirname/$verfn", "w") ||
	    die "cant open '$dirname/$verfn' - $! :ERROR";

	$time = time;
	my $now = localtime;
	print $vfh "product: $pkg\nversion: $ver\nbackuptime:$time:$now\n";
	$vfh->close(1);

	my $extra_cfgs =  [];

	push @$extra_cfgs, '/etc/mail/spamassassin/custom.cf';

	#push @$extra_cfgs, '/etc/postfix/tls_policy';

	my $extradb = $include_statistics ? $statfn : '';

	my $extra = join(' ', @$extra_cfgs);

	system("/bin/tar cf $dirname/$tarfn -C / " .
	       "/etc/pmg $extra>/dev/null 2>&1") == 0 ||
	       die "unable to create system configuration backup: ERROR";

	system("cd $dirname; md5sum $tarfn $dbfn $extradb $verfn> $sigfn") == 0 ||
	    die "unable to create backup signature: ERROR";

	system("rm -f $filename; tar czf $filename -C $dirname $verfn $sigfn $dbfn $extradb $tarfn") == 0 ||
	    die "unable to create backup archive: ERROR";
    };
    my $err = $@;

    rmtree $dirname;

    if ($err) {
	unlink $filename;
	die $err;
    }
}

sub pmg_restore {
    my ($filename, $restore_database, $restore_config, $restore_statistics) = @_;

    my $dbname = 'Proxmox_ruledb';

    my $time = time;
    my $dirname = "/tmp/proxrestore_$$.$time";
    my $dbfn = "Proxmox_ruledb.sql";
    my $statfn = "Proxmox_statdb.sql";
    my $tarfn = "config_backup.tar";
    my $sigfn = "proxmox_backup_v1.md5";

    eval {
	# create a temporary directory
	mkdir $dirname;

	system("cd $dirname; tar xzf $filename >/dev/null 2>&1") == 0 ||
	    die "unable to extract backup archive: ERROR";

	system("cd $dirname; md5sum -c $sigfn") == 0 ||
	    die "proxmox backup signature check failed: ERROR";

	if ($restore_config) {
	    # restore the tar file
	    mkdir "$dirname/config/";
	    system("tar xpf $dirname/$tarfn -C $dirname/config/") == 0 ||
		die "unable to restore configuration tar archive: ERROR";

	    -d "$dirname/config/etc/pmg" ||
		die "backup does not contain a valid system configuration directory (/etc/pmg)\n";
	    # unlink unneeded files
	    unlink "$dirname/config/etc/pmg/cluster.conf"; # never restore cluster config
	    rmtree "$dirname/config/etc/pmg/master";

	    # backup old config to /etc/pmg.oldremove current config
	    rmtree "/etc/pmg";
	    mkdir "/etc/pmg";
	    # copy files
	    system("cp -a $dirname/config/etc/pmg/* /etc/pmg/") == 0 ||
		die "unable to restore system configuration: ERROR";

	    my $cfg = PMG::Config->new();
	    my $ruledb = PMG::RuleDB->new();
	    my $rulecache = PMG::RuleCache->new($ruledb);
	    $cfg->rewrite_config($rulecache, 1);
	}

	if ($restore_database) {
	    # recreate the database

	    # stop all services accessing the database
	    PMG::Utils::service_wait_stopped(40, $PMG::Utils::db_service_list);

	    print "Destroy existing rule database\n";
	    PMG::DBTools::delete_ruledb($dbname);

	    print "Create new database\n";
	    my $dbh = PMG::DBTools::create_ruledb($dbname);
	    my $ruledb = PMG::RuleDB->new($dbh);
	    PMG::DBTools::init_ruledb($ruledb);

	    system("cat $dirname/$dbfn|psql $dbname >/dev/null 2>&1") == 0 ||
		die "unable to restore rule database: ERROR";

	    if ($restore_statistics) {
		if (-f "$dirname/$statfn") {
		    system("cat $dirname/$statfn|psql $dbname >/dev/null 2>&1") == 0 ||
			die "unable to restore statistic database: ERROR";
		}
	    }

	    print STDERR "run analyze to speed up database queries\n";
	    PMG::DBTools::postgres_admin_cmd('psql', { input => 'analyze;' }, $dbname);

	    print "Analyzing/Upgrading existing Databases...";
	    PMG::DBTools::upgradedb($ruledb);
	    print "done\n";

	    # cleanup old spam/virus storage
	    PMG::MailQueue::create_spooldirs(0, 1);

	    my $cfg = PMG::Config->new();
	    my $rulecache = PMG::RuleCache->new($ruledb);
	    $cfg->rewrite_config($rulecache, 1);

	    # and restart services as soon as possible
	    foreach my $service (reverse @$PMG::Utils::db_service_list) {
		eval { PVE::Tools::run_command(['systemctl', 'start', $service]); };
		warn $@ if $@;
	    }
	}
    };
    my $err = $@;

    rmtree $dirname;

    die $err if $err;
}

1;
Commit	Line	Data
9a8d51a4 DM	1	package PMG::Backup;
	2
	3	use strict;
	4	use warnings;
	5	use Data::Dumper;
	6	use File::Basename;
	7	use File::Path;
	8
	9	use PVE::Tools;
	10
	11	use PMG::pmgcfg;
	12	use PMG::AtomicFile;
	13
	14	sub dump_table {
	15	my ($dbh, $table, $ofh, $seq, $seqcol) = @_;
	16
	17	my $sth = $dbh->column_info(undef, undef, $table, undef);
	18
	19	my $attrs = $sth->fetchall_arrayref({});
	20
	21	my @col_arr;
	22	foreach my $ref (@$attrs) {
	23	push @col_arr, $ref->{COLUMN_NAME};
	24	}
	25
	26	$sth->finish();
	27
	28	my $cols = join (', ', @col_arr);
	29	$cols \|\| die "unable to fetch column definitions: ERROR";
	30
	31	print $ofh "COPY $table ($cols) FROM stdin;\n";
	32
	33	my $cmd = "COPY $table ($cols) TO STDOUT";
	34	$dbh->do($cmd);
	35
	36	my $data = '';
	37	while ($dbh->pg_getcopydata($data) >= 0) {
	38	print $ofh $data;
	39	}
	40
	41	print $ofh "\\.\n\n";
	42
	43	if ($seq && $seqcol) {
	44	print $ofh "SELECT setval('$seq', max($seqcol)) FROM $table;\n\n";
	45	}
	46	}
	47
	48	sub dumpdb {
	49	my ($ofh) = @_;
	50
	51	print $ofh "SET client_encoding = 'SQL_ASCII';\n";
	52	print $ofh "SET check_function_bodies = false;\n\n";
	53
	54	my $dbh = PMG::DBTools::open_ruledb();
	55
	56	print $ofh "BEGIN TRANSACTION;\n\n";
	57
	58	eval {
	59	$dbh->begin_work;
	60
	61	# read a consistent snapshot
	62	$dbh->do("SET TRANSACTION ISOLATION LEVEL SERIALIZABLE");
	63
	64	dump_table($dbh, 'attribut', $ofh);
65	dump_table($dbh, 'object', $ofh, 'object_id_seq', 'id');
66	dump_table($dbh, 'objectgroup', $ofh, 'objectgroup_id_seq', 'id');
67	dump_table($dbh, 'rule', $ofh, 'rule_id_seq', 'id');
68	dump_table($dbh, 'rulegroup', $ofh);
69	dump_table($dbh, 'userprefs', $ofh);
70
71	# we do not save the following tables: cgreylist, cmailstore, cmsreceivers, clusterinfo
72	};
73	my $err = $@;
74
75	$dbh->rollback(); # end read-only transaction
76
77	$dbh->disconnect();
78
79	die $err if $err;
80
81	print $ofh "COMMIT TRANSACTION;\n\n";
82	}
83
84	sub dumpstatdb {
85	my ($ofh) = @_;
86
87	print $ofh "SET client_encoding = 'SQL_ASCII';\n";
88	print $ofh "SET check_function_bodies = false;\n\n";
89
90	my $dbh = PMG::DBTools::open_ruledb();
91
92	eval {
93	$dbh->begin_work;
94
95	# read a consistent snapshot
96	$dbh->do("SET TRANSACTION ISOLATION LEVEL SERIALIZABLE");
97
98	print $ofh "BEGIN TRANSACTION;\n\n";
99
100	dump_table($dbh, 'dailystat', $ofh);
101	dump_table($dbh, 'domainstat', $ofh);
102	dump_table($dbh, 'virusinfo', $ofh);
103	dump_table($dbh, 'localstat', $ofh);
104
105	# drop/create the index is a little bit faster (20%)
106
107	print $ofh "DROP INDEX cstatistic_time_index;\n\n";
108	print $ofh "ALTER TABLE cstatistic DROP CONSTRAINT cstatistic_id_key;\n\n";
109	print $ofh "ALTER TABLE cstatistic DROP CONSTRAINT cstatistic_pkey;\n\n";
110	dump_table($dbh, 'cstatistic', $ofh, 'cstatistic_id_seq', 'id');
111	print $ofh "ALTER TABLE ONLY cstatistic ADD CONSTRAINT cstatistic_pkey PRIMARY KEY (cid, rid);\n\n";
112	print $ofh "ALTER TABLE ONLY cstatistic ADD CONSTRAINT cstatistic_id_key UNIQUE (id);\n\n";
113	print $ofh "CREATE INDEX CStatistic_Time_Index ON CStatistic (Time);\n\n";
114
115	print $ofh "DROP INDEX CStatistic_ID_Index;\n\n";
116	dump_table($dbh, 'creceivers', $ofh);
117	print $ofh "CREATE INDEX CStatistic_ID_Index ON CReceivers (CStatistic_CID, CStatistic_RID);\n\n";
118
119	dump_table($dbh, 'statinfo', $ofh);
120
121	print $ofh "COMMIT TRANSACTION;\n\n";
122	};
123	my $err = $@;
124
125	$dbh->rollback(); # end read-only transaction
126
127	$dbh->disconnect();
128
129	die $err if $err;
130	}
131
132	sub pmg_backup {
133	my ($filename, $include_statistics) = @_;
134
135	my $time = time;
136	my $dirname = "/tmp/proxbackup_$$.$time";
137	my $dbfn = "Proxmox_ruledb.sql";
138	my $statfn = "Proxmox_statdb.sql";
139	my $tarfn = "config_backup.tar";
140	my $sigfn = "proxmox_backup_v1.md5";
141	my $verfn = "version.txt";
142
143	eval {
144
145	my $targetdir = dirname($filename);
146	mkdir $targetdir; # try to create target dir
147	-d $targetdir \|\|
fb9e46a0	148	die "unable to access target directory '$targetdir'\n";
9a8d51a4 DM	149
	150	# create a temporary directory
	151	mkdir $dirname;
	152
	153	# dump the database first
	154	my $fh = PMG::AtomicFile->open("$dirname/$dbfn", "w") \|\|
	155	die "cant open '$dirname/$dbfn' - $! :ERROR";
	156
	157	dumpdb($fh);
	158
	159	$fh->close(1);
	160
	161	if ($include_statistics) {
	162	# dump the statistic db
	163	my $sfh = PMG::AtomicFile->open("$dirname/$statfn", "w") \|\|
	164	die "cant open '$dirname/$statfn' - $! :ERROR";
	165
	166	dumpstatdb($sfh);
	167
	168	$sfh->close(1);
	169	}
	170
	171	my $pkg = PMG::pmgcfg::package();
	172	my $ver = PMG::pmgcfg::version();
	173
	174	my $vfh = PMG::AtomicFile->open ("$dirname/$verfn", "w") \|\|
	175	die "cant open '$dirname/$verfn' - $! :ERROR";
	176
	177	$time = time;
	178	my $now = localtime;
	179	print $vfh "product: $pkg\nversion: $ver\nbackuptime:$time:$now\n";
	180	$vfh->close(1);
	181
bd1325ed	182	my $extra_cfgs = [];
9a8d51a4	183
bd1325ed	184	push @$extra_cfgs, '/etc/mail/spamassassin/custom.cf';
9a8d51a4	185
bd1325ed	186	#push @$extra_cfgs, '/etc/postfix/tls_policy';
9a8d51a4 DM	187
	188	my $extradb = $include_statistics ? $statfn : '';
	189
bd1325ed	190	my $extra = join(' ', @$extra_cfgs);
9a8d51a4 DM	191
9a8d51a4 DM	192	system("/bin/tar cf $dirname/$tarfn -C / " .
bd1325ed	193	"/etc/pmg $extra>/dev/null 2>&1") == 0 \|\|
9a8d51a4 DM	194	die "unable to create system configuration backup: ERROR";
	195
	196	system("cd $dirname; md5sum $tarfn $dbfn $extradb $verfn> $sigfn") == 0 \|\|
	197	die "unable to create backup signature: ERROR";
	198
	199	system("rm -f $filename; tar czf $filename -C $dirname $verfn $sigfn $dbfn $extradb $tarfn") == 0 \|\|
	200	die "unable to create backup archive: ERROR";
	201	};
	202	my $err = $@;
	203
	204	rmtree $dirname;
	205
	206	if ($err) {
	207	unlink $filename;
	208	die $err;
	209	}
	210	}
	211
a6d276e9 DM	212	sub pmg_restore {
	213	my ($filename, $restore_database, $restore_config, $restore_statistics) = @_;
	214
	215	my $dbname = 'Proxmox_ruledb';
	216
	217	my $time = time;
	218	my $dirname = "/tmp/proxrestore_$$.$time";
	219	my $dbfn = "Proxmox_ruledb.sql";
	220	my $statfn = "Proxmox_statdb.sql";
	221	my $tarfn = "config_backup.tar";
	222	my $sigfn = "proxmox_backup_v1.md5";
	223
	224	eval {
	225	# create a temporary directory
	226	mkdir $dirname;
	227
	228	system("cd $dirname; tar xzf $filename >/dev/null 2>&1") == 0 \|\|
	229	die "unable to extract backup archive: ERROR";
	230
	231	system("cd $dirname; md5sum -c $sigfn") == 0 \|\|
	232	die "proxmox backup signature check failed: ERROR";
	233
	234	if ($restore_config) {
	235	# restore the tar file
	236	mkdir "$dirname/config/";
	237	system("tar xpf $dirname/$tarfn -C $dirname/config/") == 0 \|\|
	238	die "unable to restore configuration tar archive: ERROR";
	239
	240	-d "$dirname/config/etc/pmg" \|\|
	241	die "backup does not contain a valid system configuration directory (/etc/pmg)\n";
	242	# unlink unneeded files
	243	unlink "$dirname/config/etc/pmg/cluster.conf"; # never restore cluster config
	244	rmtree "$dirname/config/etc/pmg/master";
	245
	246	# backup old config to /etc/pmg.oldremove current config
	247	rmtree "/etc/pmg";
	248	mkdir "/etc/pmg";
	249	# copy files
	250	system("cp -a $dirname/config/etc/pmg/* /etc/pmg/") == 0 \|\|
	251	die "unable to restore system configuration: ERROR";
	252
	253	my $cfg = PMG::Config->new();
	254	my $ruledb = PMG::RuleDB->new();
	255	my $rulecache = PMG::RuleCache->new($ruledb);
	256	$cfg->rewrite_config($rulecache, 1);
	257	}
	258
	259	if ($restore_database) {
	260	# recreate the database
	261
	262	# stop all services accessing the database
	263	PMG::Utils::service_wait_stopped(40, $PMG::Utils::db_service_list);
	264
	265	print "Destroy existing rule database\n";
	266	PMG::DBTools::delete_ruledb($dbname);
	267
	268	print "Create new database\n";
	269	my $dbh = PMG::DBTools::create_ruledb($dbname);
	270	my $ruledb = PMG::RuleDB->new($dbh);
	271	PMG::DBTools::init_ruledb($ruledb);
	272
	273	system("cat $dirname/$dbfn\|psql $dbname >/dev/null 2>&1") == 0 \|\|
	274	die "unable to restore rule database: ERROR";
	275
276	if ($restore_statistics) {
277	if (-f "$dirname/$statfn") {
278	system("cat $dirname/$statfn\|psql $dbname >/dev/null 2>&1") == 0 \|\|
279	die "unable to restore statistic database: ERROR";
280	}
281	}
282
283	print STDERR "run analyze to speed up database queries\n";
284	PMG::DBTools::postgres_admin_cmd('psql', { input => 'analyze;' }, $dbname);
285
286	print "Analyzing/Upgrading existing Databases...";
287	PMG::DBTools::upgradedb($ruledb);
288	print "done\n";
289
290	# cleanup old spam/virus storage
291	PMG::MailQueue::create_spooldirs(0, 1);
292
293	my $cfg = PMG::Config->new();
294	my $rulecache = PMG::RuleCache->new($ruledb);
295	$cfg->rewrite_config($rulecache, 1);
296
297	# and restart services as soon as possible
298	foreach my $service (reverse @$PMG::Utils::db_service_list) {
299	eval { PVE::Tools::run_command(['systemctl', 'start', $service]); };
300	warn $@ if $@;
301	}
302	}
303	};
304	my $err = $@;
305
306	rmtree $dirname;
307
308	die $err if $err;
309	}
310
9a8d51a4	311	1;