]>
Commit | Line | Data |
---|---|---|
132089d5 DM |
1 | package PMG::Service::pmgmirror; |
2 | ||
3 | use strict; | |
4 | use warnings; | |
5 | use Data::Dumper; | |
9430b6d4 | 6 | use Time::HiRes qw (gettimeofday tv_interval); |
132089d5 DM |
7 | |
8 | use PVE::SafeSyslog; | |
9 | use PVE::Tools qw(extract_param); | |
10 | use PVE::INotify; | |
11 | use PVE::Daemon; | |
83ad364b | 12 | use PVE::ProcFSTools; |
132089d5 | 13 | |
9430b6d4 | 14 | use PMG::Ticket; |
132089d5 DM |
15 | use PMG::RESTEnvironment; |
16 | use PMG::DBTools; | |
17 | use PMG::RuleDB; | |
18 | use PMG::Cluster; | |
19 | use PMG::ClusterConfig; | |
20 | use PMG::Statistic; | |
5314a3a9 | 21 | use PMG::Utils; |
132089d5 DM |
22 | |
23 | use base qw(PVE::Daemon); | |
24 | ||
25 | my $cmdline = [$0, @ARGV]; | |
26 | ||
27 | my %daemon_options = (restart_on_error => 5, stop_wait_time => 5); | |
28 | ||
29 | my $daemon = __PACKAGE__->new('pmgmirror', $cmdline, %daemon_options); | |
30 | ||
31 | my $restart_request = 0; | |
32 | my $next_update = 0; | |
33 | ||
34 | my $cycle = 0; | |
2c26c08d | 35 | my $updatetime = 60*2; |
9430b6d4 | 36 | my $maxtimediff = 5; |
132089d5 | 37 | |
83ad364b | 38 | my $initial_memory_usage; |
132089d5 DM |
39 | |
40 | sub init { | |
83ad364b | 41 | # syslog('INIT'); |
132089d5 DM |
42 | } |
43 | ||
44 | sub hup { | |
45 | my ($self) = @_; | |
46 | ||
47 | $restart_request = 1; | |
48 | } | |
49 | ||
9430b6d4 DM |
50 | sub sync_data_from_node { |
51 | my ($dbh, $rdb, $cinfo, $ni, $ticket, $rsynctime_ref) = @_; | |
2c26c08d | 52 | |
9430b6d4 DM |
53 | my $ctime = PMG::DBTools::get_remote_time($rdb); |
54 | my $ltime = time(); | |
55 | ||
56 | my $timediff = abs($ltime - $ctime); | |
57 | if ($timediff > $maxtimediff) { | |
58 | die "large time difference (> $timediff seconds) - not syncing\n"; | |
59 | } | |
60 | ||
61 | if ($ni->{type} eq 'master') { | |
62 | PMG::Cluster::sync_ruledb_from_master($dbh, $rdb, $ni, $ticket); | |
63 | PMG::Cluster::sync_deleted_nodes_from_master($dbh, $rdb, $cinfo, $ni, $rsynctime_ref); | |
64 | } | |
65 | ||
66 | PMG::Cluster::sync_quarantine_db($dbh, $rdb, $ni, $rsynctime_ref); | |
67 | ||
68 | PMG::Cluster::sync_greylist_db($dbh, $rdb, $ni); | |
69 | ||
70 | PMG::Cluster::sync_userprefs_db($dbh, $rdb, $ni); | |
71 | ||
72 | PMG::Cluster::sync_statistic_db($dbh, $rdb, $ni); | |
73 | ||
74 | if ($ni->{type} eq 'master') { | |
75 | PMG::Cluster::sync_domainstat_db($dbh, $rdb, $ni); | |
76 | ||
77 | PMG::Cluster::sync_dailystat_db($dbh, $rdb, $ni); | |
78 | ||
79 | PMG::Cluster::sync_virusinfo_db($dbh, $rdb, $ni); | |
80 | } | |
5e1408fd DM |
81 | |
82 | PMG::Cluster::sync_localstat_db($dbh, $rdb, $ni); | |
9430b6d4 DM |
83 | } |
84 | ||
85 | sub cluster_sync { | |
2c26c08d DM |
86 | |
87 | my $cinfo = PMG::ClusterConfig->new(); # reload | |
88 | my $role = $cinfo->{local}->{type} // '-'; | |
89 | ||
90 | return if $role eq '-'; | |
91 | return if !$cinfo->{master}; # just to be sure | |
92 | ||
9430b6d4 | 93 | my $start_time = [ gettimeofday() ]; |
2c26c08d | 94 | |
1359baef | 95 | syslog ('info', "starting cluster synchronization"); |
2c26c08d DM |
96 | |
97 | my $master_ip = $cinfo->{master}->{ip}; | |
98 | my $master_name = $cinfo->{master}->{name}; | |
99 | ||
5314a3a9 | 100 | my $force_restart = {}; |
9430b6d4 | 101 | if ($role ne 'master') { |
5314a3a9 | 102 | $force_restart = PMG::Cluster::sync_config_from_master($master_name, $master_ip); |
9430b6d4 | 103 | } |
2c26c08d | 104 | |
9430b6d4 | 105 | my $csynctime = tv_interval($start_time); |
2c26c08d DM |
106 | |
107 | $cinfo = PMG::ClusterConfig->new(); # reload | |
108 | $role = $cinfo->{local}->{type} // '-'; | |
109 | ||
110 | return if $role eq '-'; | |
111 | return if !$cinfo->{master}; # just to be sure | |
112 | ||
9430b6d4 DM |
113 | my $ticket = PMG::Ticket::assemble_ticket('root@pam'); |
114 | ||
115 | my $dbh = PMG::DBTools::open_ruledb(); | |
116 | ||
117 | my $errcount = 0; | |
118 | ||
119 | my $rsynctime = 0; | |
120 | ||
121 | my $sync_node = sub { | |
122 | my ($ni) = @_; | |
123 | ||
124 | my $rdb; | |
125 | eval { | |
83e9f427 | 126 | $rdb = PMG::DBTools::open_ruledb(undef, '/run/pmgtunnel', $ni->{cid}); |
9430b6d4 DM |
127 | sync_data_from_node($dbh, $rdb, $cinfo, $ni, $ticket, \$rsynctime); |
128 | }; | |
129 | my $err = $@; | |
130 | ||
131 | $rdb->disconnect() if $rdb; | |
132 | ||
133 | if ($err) { | |
134 | $errcount++; | |
135 | syslog ('err', "database sync '$ni->{name}' failed - $err"); | |
964c7db5 DM |
136 | } else { |
137 | PMG::DBTools::create_clusterinfo_default($dbh, $ni->{cid}, 'lastsync', 0, undef); | |
138 | PMG::DBTools::write_maxint_clusterinfo($dbh, $ni->{cid}, 'lastsync', time()); | |
9430b6d4 DM |
139 | } |
140 | }; | |
141 | ||
142 | # sync data from master first | |
143 | if ($cinfo->{master}->{cid} ne $cinfo->{local}->{cid}) { | |
144 | $sync_node->($cinfo->{master}); | |
d6750ad4 DM |
145 | |
146 | # rewrite config after sync from master | |
147 | my $cfg = PMG::Config->new(); | |
148 | my $ruledb = PMG::RuleDB->new($dbh); | |
149 | my $rulecache = PMG::RuleCache->new($ruledb); | |
150 | $cfg->rewrite_config($rulecache, 1); | |
9430b6d4 DM |
151 | } |
152 | ||
153 | foreach my $ni (values %{$cinfo->{ids}}) { | |
154 | next if $ni->{cid} eq $cinfo->{local}->{cid}; | |
155 | next if $ni->{cid} eq $cinfo->{master}->{cid}; | |
156 | $sync_node->($ni); | |
157 | } | |
158 | ||
159 | $dbh->disconnect(); | |
160 | ||
161 | my $cptime = tv_interval($start_time); | |
2c26c08d | 162 | |
9430b6d4 | 163 | my $dbtime = $cptime - $rsynctime - $csynctime; |
2c26c08d | 164 | |
1359baef | 165 | syslog('info', sprintf("cluster synchronization finished (%d errors, %.2f seconds " . |
2c26c08d | 166 | "(files %.2f, database %.2f, config %.2f))", |
9430b6d4 | 167 | $errcount, $cptime, $rsynctime, $dbtime, $csynctime)); |
2c26c08d | 168 | |
5314a3a9 DC |
169 | foreach my $service (keys %$force_restart) { |
170 | PMG::Utils::service_cmd($service, 'restart'); | |
171 | } | |
2c26c08d DM |
172 | } |
173 | ||
132089d5 DM |
174 | sub run { |
175 | my ($self) = @_; | |
176 | ||
177 | for (;;) { # forever | |
178 | ||
179 | $next_update = time() + $updatetime; | |
180 | ||
9430b6d4 DM |
181 | eval { |
182 | # Note: do nothing in first cycle (give pmgtunnel some time to startup) | |
183 | cluster_sync() if $cycle > 0; | |
184 | }; | |
132089d5 | 185 | if (my $err = $@) { |
b902c0b8 | 186 | syslog('err', "sync error: $err"); |
132089d5 DM |
187 | } |
188 | ||
83ad364b DM |
189 | $cycle++; |
190 | ||
191 | last if $self->{terminate}; | |
192 | ||
193 | my $mem = PVE::ProcFSTools::read_memory_usage(); | |
194 | ||
195 | if (!defined($initial_memory_usage) || ($cycle < 10)) { | |
196 | $initial_memory_usage = $mem->{resident}; | |
197 | } else { | |
198 | my $diff = $mem->{resident} - $initial_memory_usage; | |
199 | if ($diff > 5*1024*1024) { | |
200 | syslog ('info', "restarting server after $cycle cycles to " . | |
201 | "reduce memory usage (free $mem->{resident} ($diff) bytes)"); | |
202 | $self->restart_daemon(); | |
203 | } | |
204 | } | |
205 | ||
132089d5 DM |
206 | my $wcount = 0; |
207 | while ((time() < $next_update) && | |
208 | ($wcount < $updatetime) && # protect against time wrap | |
209 | !$restart_request && !$self->{terminate}) { | |
210 | ||
211 | $wcount++; sleep (1); | |
212 | }; | |
213 | ||
214 | last if $self->{terminate}; | |
215 | ||
216 | $self->restart_daemon() if $restart_request; | |
217 | } | |
218 | } | |
219 | ||
220 | $daemon->register_start_command("Start the Database Mirror Daemon"); | |
221 | $daemon->register_stop_command("Stop the Database Mirror Daemon"); | |
222 | $daemon->register_restart_command(1, "Restart the Database Mirror Daemon"); | |
223 | ||
224 | our $cmddef = { | |
225 | start => [ __PACKAGE__, 'start', []], | |
226 | restart => [ __PACKAGE__, 'restart', []], | |
227 | stop => [ __PACKAGE__, 'stop', []], | |
228 | }; | |
229 | ||
230 | 1; |