[pmg-api.git] / src / PMG / Backup.pm

package PMG::Backup;

use strict;
use warnings;
use Data::Dumper;
use File::Basename;
use File::Path;

use PVE::Tools;

use PMG::pmgcfg;
use PMG::AtomicFile;

my $sa_custom_config_fn = "/etc/mail/spamassassin/custom.cf";

sub dump_table {
    my ($dbh, $table, $ofh, $seq, $seqcol) = @_;

    my $sth = $dbh->column_info(undef, undef, $table, undef);

    my $attrs = $sth->fetchall_arrayref({});

    my @col_arr;
    foreach my $ref (@$attrs) {
	push @col_arr, $ref->{COLUMN_NAME};
    }

    $sth->finish();

    my $cols = join (', ', @col_arr);
    $cols || die "unable to fetch column definitions: ERROR";

    print $ofh "COPY $table ($cols) FROM stdin;\n";

    my $cmd = "COPY $table ($cols) TO STDOUT";
    $dbh->do($cmd);

    my $data = '';
    while ($dbh->pg_getcopydata($data) >= 0) {
	print $ofh $data;
    }

    print $ofh "\\.\n\n";

    if ($seq && $seqcol) {
	print $ofh "SELECT setval('$seq', max($seqcol)) FROM $table;\n\n";
    }
}

sub dumpdb {
    my ($ofh) = @_;

    print $ofh "SET client_encoding = 'SQL_ASCII';\n";
    print $ofh "SET check_function_bodies = false;\n\n";

    my $dbh = PMG::DBTools::open_ruledb();

    print $ofh "BEGIN TRANSACTION;\n\n";

    eval {
	$dbh->begin_work;

	# read a consistent snapshot
	$dbh->do("SET TRANSACTION ISOLATION LEVEL SERIALIZABLE");

	dump_table($dbh, 'attribut', $ofh);
	dump_table($dbh, 'object', $ofh, 'object_id_seq', 'id');
	dump_table($dbh, 'objectgroup', $ofh, 'objectgroup_id_seq', 'id');
	dump_table($dbh, 'rule', $ofh, 'rule_id_seq', 'id');
	dump_table($dbh, 'rulegroup', $ofh);
	dump_table($dbh, 'userprefs', $ofh);

	# we do not save the following tables: cgreylist, cmailstore, cmsreceivers, clusterinfo
    };
    my $err = $@;

    $dbh->rollback(); # end read-only transaction

    $dbh->disconnect();

    die $err if $err;

    print $ofh "COMMIT TRANSACTION;\n\n";
}

sub dumpstatdb {
    my ($ofh) = @_;

    print $ofh "SET client_encoding = 'SQL_ASCII';\n";
    print $ofh "SET check_function_bodies = false;\n\n";

    my $dbh = PMG::DBTools::open_ruledb();

    eval {
	$dbh->begin_work;

	# read a consistent snapshot
	$dbh->do("SET TRANSACTION ISOLATION LEVEL SERIALIZABLE");

	print $ofh "BEGIN TRANSACTION;\n\n";

	dump_table($dbh, 'dailystat', $ofh);
	dump_table($dbh, 'domainstat', $ofh);
	dump_table($dbh, 'virusinfo', $ofh);
	dump_table($dbh, 'localstat', $ofh);

	# drop/create the index is a little bit faster (20%)

	print $ofh "DROP INDEX cstatistic_time_index;\n\n";
	print $ofh "ALTER TABLE cstatistic DROP CONSTRAINT cstatistic_id_key;\n\n";
	print $ofh "ALTER TABLE cstatistic DROP CONSTRAINT cstatistic_pkey;\n\n";
	dump_table($dbh, 'cstatistic', $ofh, 'cstatistic_id_seq', 'id');
	print $ofh "ALTER TABLE ONLY cstatistic ADD CONSTRAINT cstatistic_pkey PRIMARY KEY (cid, rid);\n\n";
	print $ofh "ALTER TABLE ONLY cstatistic ADD CONSTRAINT cstatistic_id_key UNIQUE (id);\n\n";
	print $ofh "CREATE INDEX CStatistic_Time_Index ON CStatistic (Time);\n\n";

	print $ofh "DROP INDEX CStatistic_ID_Index;\n\n";
	dump_table($dbh, 'creceivers', $ofh);
	print $ofh "CREATE INDEX CStatistic_ID_Index ON CReceivers (CStatistic_CID, CStatistic_RID);\n\n";

	dump_table($dbh, 'statinfo', $ofh);

	print $ofh "COMMIT TRANSACTION;\n\n";
    };
    my $err = $@;

    $dbh->rollback(); # end read-only transaction

    $dbh->disconnect();

    die $err if $err;
}

sub pmg_backup {
    my ($filename, $include_statistics) = @_;

    my $time = time;
    my $dirname = "/tmp/proxbackup_$$.$time";
    my $dbfn = "Proxmox_ruledb.sql";
    my $statfn = "Proxmox_statdb.sql";
    my $tarfn = "config_backup.tar";
    my $sigfn = "proxmox_backup_v1.md5";
    my $verfn = "version.txt";

    eval {

	my $targetdir = dirname($filename);
	mkdir $targetdir; # try to create target dir
	-d $targetdir ||
	    die "unable to access target directory '$targetdir'\n";

	# create a temporary directory
	mkdir $dirname;

	# dump the database first
	my $fh = PMG::AtomicFile->open("$dirname/$dbfn", "w") ||
	    die "cant open '$dirname/$dbfn' - $! :ERROR";

	dumpdb($fh);

	$fh->close(1);

	if ($include_statistics) {
	    # dump the statistic db
	    my $sfh = PMG::AtomicFile->open("$dirname/$statfn", "w") ||
		die "cant open '$dirname/$statfn' - $! :ERROR";

	    dumpstatdb($sfh);

	    $sfh->close(1);
	}

	my $pkg = PMG::pmgcfg::package();
	my $release = PMG::pmgcfg::release();

	my $vfh = PMG::AtomicFile->open ("$dirname/$verfn", "w") ||
	    die "cant open '$dirname/$verfn' - $! :ERROR";

	$time = time;
	my $now = localtime;
	print $vfh "product: $pkg\nversion: $release\nbackuptime:$time:$now\n";
	$vfh->close(1);

	my $extra_cfgs =  [];

	push @$extra_cfgs, $sa_custom_config_fn;

	#push @$extra_cfgs, '/etc/postfix/tls_policy';

	my $extradb = $include_statistics ? $statfn : '';

	my $extra = join(' ', @$extra_cfgs);

	system("/bin/tar cf $dirname/$tarfn -C / " .
	       "/etc/pmg $extra>/dev/null 2>&1") == 0 ||
	       die "unable to create system configuration backup: ERROR";

	system("cd $dirname; md5sum $tarfn $dbfn $extradb $verfn> $sigfn") == 0 ||
	    die "unable to create backup signature: ERROR";

	system("rm -f $filename; tar czf $filename -C $dirname $verfn $sigfn $dbfn $extradb $tarfn") == 0 ||
	    die "unable to create backup archive: ERROR";
    };
    my $err = $@;

    rmtree $dirname;

    if ($err) {
	unlink $filename;
	die $err;
    }
}

sub pmg_restore {
    my ($filename, $restore_database, $restore_config, $restore_statistics) = @_;

    my $dbname = 'Proxmox_ruledb';

    my $time = time;
    my $dirname = "/tmp/proxrestore_$$.$time";
    my $dbfn = "Proxmox_ruledb.sql";
    my $statfn = "Proxmox_statdb.sql";
    my $tarfn = "config_backup.tar";
    my $sigfn = "proxmox_backup_v1.md5";

    eval {
	# create a temporary directory
	mkdir $dirname;

	system("cd $dirname; tar xzf $filename >/dev/null 2>&1") == 0 ||
	    die "unable to extract backup archive: ERROR";

	system("cd $dirname; md5sum -c $sigfn") == 0 ||
	    die "proxmox backup signature check failed: ERROR";

	if ($restore_config) {
	    # restore the tar file
	    mkdir "$dirname/config/";
	    system("tar xpf $dirname/$tarfn -C $dirname/config/") == 0 ||
		die "unable to restore configuration tar archive: ERROR";

	    -d "$dirname/config/etc/pmg" ||
		die "backup does not contain a valid system configuration directory (/etc/pmg)\n";
	    # unlink unneeded files
	    unlink "$dirname/config/etc/pmg/cluster.conf"; # never restore cluster config
	    rmtree "$dirname/config/etc/pmg/master";

	    # remove current config, but keep directory for INotify
	    rmtree("/etc/pmg", { keep_root => 1 });
	    # copy files
	    system("cp -a $dirname/config/etc/pmg/* /etc/pmg/") == 0 ||
		die "unable to restore system configuration: ERROR";

	    if (-f "$dirname/config/${sa_custom_config_fn}") {
		my $data = PVE::Tools::file_get_contents(
		    "$dirname/config/${sa_custom_config_fn}", 1024*1024);
		PVE::Tools::file_set_contents($sa_custom_config_fn, $data);
	    }

	    my $cfg = PMG::Config->new();
	    my $ruledb = PMG::RuleDB->new();
	    my $rulecache = PMG::RuleCache->new($ruledb);
	    $cfg->rewrite_config($rulecache, 1);
	}

	if ($restore_database) {
	    # recreate the database

	    # stop all services accessing the database
	    PMG::Utils::service_wait_stopped(40, $PMG::Utils::db_service_list);

	    print "Destroy existing rule database\n";
	    PMG::DBTools::delete_ruledb($dbname);

	    print "Create new database\n";
	    my $dbh = PMG::DBTools::create_ruledb($dbname);

	    system("cat $dirname/$dbfn|psql $dbname >/dev/null 2>&1") == 0 ||
		die "unable to restore rule database: ERROR";

	    if ($restore_statistics) {
		if (-f "$dirname/$statfn") {
		    system("cat $dirname/$statfn|psql $dbname >/dev/null 2>&1") == 0 ||
			die "unable to restore statistic database: ERROR";
		}
	    }

	    print STDERR "run analyze to speed up database queries\n";
	    PMG::DBTools::postgres_admin_cmd('psql', { input => 'analyze;' }, $dbname);

	    print "Analyzing/Upgrading existing Databases...";
	    my $ruledb = PMG::RuleDB->new($dbh);
	    PMG::DBTools::upgradedb($ruledb);
	    print "done\n";

	    # cleanup old spam/virus storage
	    PMG::MailQueue::create_spooldirs(0, 1);

	    my $cfg = PMG::Config->new();
	    my $rulecache = PMG::RuleCache->new($ruledb);
	    $cfg->rewrite_config($rulecache, 1);

	    # and restart services as soon as possible
	    foreach my $service (reverse @$PMG::Utils::db_service_list) {
		eval { PVE::Tools::run_command(['systemctl', 'start', $service]); };
		warn $@ if $@;
	    }
	}
    };
    my $err = $@;

    rmtree $dirname;

    die $err if $err;
}

1;
Commit	Line	Data
9a8d51a4 DM	1	package PMG::Backup;
	2
	3	use strict;
	4	use warnings;
	5	use Data::Dumper;
	6	use File::Basename;
	7	use File::Path;
	8
	9	use PVE::Tools;
	10
	11	use PMG::pmgcfg;
	12	use PMG::AtomicFile;
	13
29b263e5	14	my $sa_custom_config_fn = "/etc/mail/spamassassin/custom.cf";
520f7717	15
9a8d51a4 DM	16	sub dump_table {
	17	my ($dbh, $table, $ofh, $seq, $seqcol) = @_;
	18
	19	my $sth = $dbh->column_info(undef, undef, $table, undef);
	20
	21	my $attrs = $sth->fetchall_arrayref({});
	22
	23	my @col_arr;
	24	foreach my $ref (@$attrs) {
	25	push @col_arr, $ref->{COLUMN_NAME};
	26	}
	27
	28	$sth->finish();
	29
	30	my $cols = join (', ', @col_arr);
	31	$cols \|\| die "unable to fetch column definitions: ERROR";
	32
	33	print $ofh "COPY $table ($cols) FROM stdin;\n";
	34
	35	my $cmd = "COPY $table ($cols) TO STDOUT";
	36	$dbh->do($cmd);
	37
	38	my $data = '';
	39	while ($dbh->pg_getcopydata($data) >= 0) {
	40	print $ofh $data;
	41	}
	42
	43	print $ofh "\\.\n\n";
	44
	45	if ($seq && $seqcol) {
	46	print $ofh "SELECT setval('$seq', max($seqcol)) FROM $table;\n\n";
	47	}
	48	}
	49
	50	sub dumpdb {
	51	my ($ofh) = @_;
	52
	53	print $ofh "SET client_encoding = 'SQL_ASCII';\n";
	54	print $ofh "SET check_function_bodies = false;\n\n";
	55
	56	my $dbh = PMG::DBTools::open_ruledb();
	57
	58	print $ofh "BEGIN TRANSACTION;\n\n";
	59
	60	eval {
	61	$dbh->begin_work;
	62
	63	# read a consistent snapshot
	64	$dbh->do("SET TRANSACTION ISOLATION LEVEL SERIALIZABLE");
	65
	66	dump_table($dbh, 'attribut', $ofh);
	67	dump_table($dbh, 'object', $ofh, 'object_id_seq', 'id');
	68	dump_table($dbh, 'objectgroup', $ofh, 'objectgroup_id_seq', 'id');
	69	dump_table($dbh, 'rule', $ofh, 'rule_id_seq', 'id');
	70	dump_table($dbh, 'rulegroup', $ofh);
	71	dump_table($dbh, 'userprefs', $ofh);
	72
	73	# we do not save the following tables: cgreylist, cmailstore, cmsreceivers, clusterinfo
	74	};
	75	my $err = $@;
	76
	77	$dbh->rollback(); # end read-only transaction
	78
	79	$dbh->disconnect();
80
81	die $err if $err;
82
83	print $ofh "COMMIT TRANSACTION;\n\n";
84	}
85
86	sub dumpstatdb {
87	my ($ofh) = @_;
88
89	print $ofh "SET client_encoding = 'SQL_ASCII';\n";
90	print $ofh "SET check_function_bodies = false;\n\n";
91
92	my $dbh = PMG::DBTools::open_ruledb();
93
94	eval {
95	$dbh->begin_work;
96
97	# read a consistent snapshot
98	$dbh->do("SET TRANSACTION ISOLATION LEVEL SERIALIZABLE");
99
100	print $ofh "BEGIN TRANSACTION;\n\n";
101
102	dump_table($dbh, 'dailystat', $ofh);
103	dump_table($dbh, 'domainstat', $ofh);
104	dump_table($dbh, 'virusinfo', $ofh);
105	dump_table($dbh, 'localstat', $ofh);
106
107	# drop/create the index is a little bit faster (20%)
108
109	print $ofh "DROP INDEX cstatistic_time_index;\n\n";
110	print $ofh "ALTER TABLE cstatistic DROP CONSTRAINT cstatistic_id_key;\n\n";
111	print $ofh "ALTER TABLE cstatistic DROP CONSTRAINT cstatistic_pkey;\n\n";
112	dump_table($dbh, 'cstatistic', $ofh, 'cstatistic_id_seq', 'id');
113	print $ofh "ALTER TABLE ONLY cstatistic ADD CONSTRAINT cstatistic_pkey PRIMARY KEY (cid, rid);\n\n";
114	print $ofh "ALTER TABLE ONLY cstatistic ADD CONSTRAINT cstatistic_id_key UNIQUE (id);\n\n";
115	print $ofh "CREATE INDEX CStatistic_Time_Index ON CStatistic (Time);\n\n";
116
117	print $ofh "DROP INDEX CStatistic_ID_Index;\n\n";
118	dump_table($dbh, 'creceivers', $ofh);
119	print $ofh "CREATE INDEX CStatistic_ID_Index ON CReceivers (CStatistic_CID, CStatistic_RID);\n\n";
120
121	dump_table($dbh, 'statinfo', $ofh);
122
123	print $ofh "COMMIT TRANSACTION;\n\n";
124	};
125	my $err = $@;
126
127	$dbh->rollback(); # end read-only transaction
128
129	$dbh->disconnect();
130
131	die $err if $err;
132	}
133
134	sub pmg_backup {
135	my ($filename, $include_statistics) = @_;
136
137	my $time = time;
138	my $dirname = "/tmp/proxbackup_$$.$time";
139	my $dbfn = "Proxmox_ruledb.sql";
140	my $statfn = "Proxmox_statdb.sql";
141	my $tarfn = "config_backup.tar";
142	my $sigfn = "proxmox_backup_v1.md5";
143	my $verfn = "version.txt";
144
145	eval {
146
147	my $targetdir = dirname($filename);
148	mkdir $targetdir; # try to create target dir
149	-d $targetdir \|\|
fb9e46a0	150	die "unable to access target directory '$targetdir'\n";
9a8d51a4 DM	151
	152	# create a temporary directory
	153	mkdir $dirname;
	154
	155	# dump the database first
	156	my $fh = PMG::AtomicFile->open("$dirname/$dbfn", "w") \|\|
	157	die "cant open '$dirname/$dbfn' - $! :ERROR";
	158
	159	dumpdb($fh);
	160
	161	$fh->close(1);
	162
	163	if ($include_statistics) {
	164	# dump the statistic db
	165	my $sfh = PMG::AtomicFile->open("$dirname/$statfn", "w") \|\|
	166	die "cant open '$dirname/$statfn' - $! :ERROR";
	167
	168	dumpstatdb($sfh);
	169
	170	$sfh->close(1);
	171	}
	172
	173	my $pkg = PMG::pmgcfg::package();
1c15ce0f	174	my $release = PMG::pmgcfg::release();
9a8d51a4 DM	175
	176	my $vfh = PMG::AtomicFile->open ("$dirname/$verfn", "w") \|\|
	177	die "cant open '$dirname/$verfn' - $! :ERROR";
	178
	179	$time = time;
	180	my $now = localtime;
1c15ce0f	181	print $vfh "product: $pkg\nversion: $release\nbackuptime:$time:$now\n";
9a8d51a4 DM	182	$vfh->close(1);
9a8d51a4 DM	183
bd1325ed	184	my $extra_cfgs = [];
9a8d51a4	185
29b263e5	186	push @$extra_cfgs, $sa_custom_config_fn;
9a8d51a4	187
bd1325ed	188	#push @$extra_cfgs, '/etc/postfix/tls_policy';
9a8d51a4 DM	189
	190	my $extradb = $include_statistics ? $statfn : '';
	191
bd1325ed	192	my $extra = join(' ', @$extra_cfgs);
9a8d51a4 DM	193
9a8d51a4 DM	194	system("/bin/tar cf $dirname/$tarfn -C / " .
bd1325ed	195	"/etc/pmg $extra>/dev/null 2>&1") == 0 \|\|
9a8d51a4 DM	196	die "unable to create system configuration backup: ERROR";
	197
	198	system("cd $dirname; md5sum $tarfn $dbfn $extradb $verfn> $sigfn") == 0 \|\|
	199	die "unable to create backup signature: ERROR";
	200
	201	system("rm -f $filename; tar czf $filename -C $dirname $verfn $sigfn $dbfn $extradb $tarfn") == 0 \|\|
	202	die "unable to create backup archive: ERROR";
	203	};
	204	my $err = $@;
	205
	206	rmtree $dirname;
	207
	208	if ($err) {
	209	unlink $filename;
	210	die $err;
	211	}
	212	}
	213
a6d276e9 DM	214	sub pmg_restore {
	215	my ($filename, $restore_database, $restore_config, $restore_statistics) = @_;
	216
	217	my $dbname = 'Proxmox_ruledb';
	218
	219	my $time = time;
	220	my $dirname = "/tmp/proxrestore_$$.$time";
	221	my $dbfn = "Proxmox_ruledb.sql";
	222	my $statfn = "Proxmox_statdb.sql";
	223	my $tarfn = "config_backup.tar";
	224	my $sigfn = "proxmox_backup_v1.md5";
	225
	226	eval {
	227	# create a temporary directory
	228	mkdir $dirname;
	229
	230	system("cd $dirname; tar xzf $filename >/dev/null 2>&1") == 0 \|\|
	231	die "unable to extract backup archive: ERROR";
	232
	233	system("cd $dirname; md5sum -c $sigfn") == 0 \|\|
	234	die "proxmox backup signature check failed: ERROR";
	235
	236	if ($restore_config) {
	237	# restore the tar file
	238	mkdir "$dirname/config/";
	239	system("tar xpf $dirname/$tarfn -C $dirname/config/") == 0 \|\|
	240	die "unable to restore configuration tar archive: ERROR";
	241
	242	-d "$dirname/config/etc/pmg" \|\|
	243	die "backup does not contain a valid system configuration directory (/etc/pmg)\n";
	244	# unlink unneeded files
	245	unlink "$dirname/config/etc/pmg/cluster.conf"; # never restore cluster config
	246	rmtree "$dirname/config/etc/pmg/master";
	247
8624613e DM	248	# remove current config, but keep directory for INotify
8624613e DM	249	rmtree("/etc/pmg", { keep_root => 1 });
a6d276e9 DM	250	# copy files
	251	system("cp -a $dirname/config/etc/pmg/* /etc/pmg/") == 0 \|\|
	252	die "unable to restore system configuration: ERROR";
	253
29b263e5	254	if (-f "$dirname/config/${sa_custom_config_fn}") {
520f7717	255	my $data = PVE::Tools::file_get_contents(
29b263e5 DM	256	"$dirname/config/${sa_custom_config_fn}", 1024*1024);
29b263e5 DM	257	PVE::Tools::file_set_contents($sa_custom_config_fn, $data);
520f7717 DM	258	}
520f7717 DM	259
a6d276e9 DM	260	my $cfg = PMG::Config->new();
	261	my $ruledb = PMG::RuleDB->new();
	262	my $rulecache = PMG::RuleCache->new($ruledb);
	263	$cfg->rewrite_config($rulecache, 1);
	264	}
	265
	266	if ($restore_database) {
	267	# recreate the database
	268
	269	# stop all services accessing the database
	270	PMG::Utils::service_wait_stopped(40, $PMG::Utils::db_service_list);
	271
	272	print "Destroy existing rule database\n";
	273	PMG::DBTools::delete_ruledb($dbname);
	274
	275	print "Create new database\n";
	276	my $dbh = PMG::DBTools::create_ruledb($dbname);
a6d276e9 DM	277
	278	system("cat $dirname/$dbfn\|psql $dbname >/dev/null 2>&1") == 0 \|\|
	279	die "unable to restore rule database: ERROR";
	280
	281	if ($restore_statistics) {
	282	if (-f "$dirname/$statfn") {
	283	system("cat $dirname/$statfn\|psql $dbname >/dev/null 2>&1") == 0 \|\|
	284	die "unable to restore statistic database: ERROR";
	285	}
	286	}
	287
	288	print STDERR "run analyze to speed up database queries\n";
	289	PMG::DBTools::postgres_admin_cmd('psql', { input => 'analyze;' }, $dbname);
	290
	291	print "Analyzing/Upgrading existing Databases...";
9882b397	292	my $ruledb = PMG::RuleDB->new($dbh);
a6d276e9 DM	293	PMG::DBTools::upgradedb($ruledb);
	294	print "done\n";
	295
	296	# cleanup old spam/virus storage
	297	PMG::MailQueue::create_spooldirs(0, 1);
	298
	299	my $cfg = PMG::Config->new();
	300	my $rulecache = PMG::RuleCache->new($ruledb);
	301	$cfg->rewrite_config($rulecache, 1);
	302
	303	# and restart services as soon as possible
	304	foreach my $service (reverse @$PMG::Utils::db_service_list) {
	305	eval { PVE::Tools::run_command(['systemctl', 'start', $service]); };
	306	warn $@ if $@;
	307	}
	308	}
	309	};
	310	my $err = $@;
	311
	312	rmtree $dirname;
	313
	314	die $err if $err;
	315	}
	316
9a8d51a4	317	1;