]>
Commit | Line | Data |
---|---|---|
d0920c29 DM |
1 | package ReplicationTestEnv; |
2 | ||
3 | use strict; | |
4 | use warnings; | |
d59f1fa8 | 5 | |
c5014e65 DM |
6 | use Clone 'clone'; |
7 | use File::Basename; | |
d59f1fa8 | 8 | use JSON; |
d0920c29 DM |
9 | |
10 | use lib ('.', '../..'); | |
11 | ||
d0920c29 | 12 | use PVE::Cluster; |
d59f1fa8 TL |
13 | use PVE::INotify; |
14 | use PVE::LXC::Config; | |
15 | use PVE::QemuConfig; | |
d0920c29 | 16 | use PVE::Storage; |
d59f1fa8 | 17 | |
810c6776 | 18 | use PVE::API2::Replication; |
d0920c29 | 19 | use PVE::Replication; |
d59f1fa8 TL |
20 | use PVE::ReplicationConfig; |
21 | use PVE::ReplicationState; | |
810c6776 | 22 | |
d0920c29 DM |
23 | use Test::MockModule; |
24 | ||
25 | our $mocked_nodename = 'node1'; | |
26 | ||
27 | our $mocked_replication_jobs = {}; | |
28 | ||
d092dc4f DM |
29 | my $pve_replication_config_module = Test::MockModule->new('PVE::ReplicationConfig'); |
30 | my $pve_replication_state_module = Test::MockModule->new('PVE::ReplicationState'); | |
d0920c29 DM |
31 | |
32 | our $mocked_vm_configs = {}; | |
33 | ||
34 | our $mocked_ct_configs = {}; | |
35 | ||
12c206a5 WB |
36 | my $mocked_get_members = sub { |
37 | return { | |
38 | node1 => { online => 1 }, | |
39 | node2 => { online => 1 }, | |
40 | node3 => { online => 1 }, | |
41 | }; | |
42 | }; | |
43 | ||
d0920c29 DM |
44 | my $mocked_vmlist = sub { |
45 | my $res = {}; | |
46 | ||
47 | foreach my $id (keys %$mocked_ct_configs) { | |
48 | my $d = $mocked_ct_configs->{$id}; | |
49 | $res->{$id} = { 'type' => 'lxc', 'node' => $d->{node}, 'version' => 1 }; | |
50 | } | |
51 | foreach my $id (keys %$mocked_vm_configs) { | |
52 | my $d = $mocked_vm_configs->{$id}; | |
53 | $res->{$id} = { 'type' => 'qemu', 'node' => $d->{node}, 'version' => 1 }; | |
54 | } | |
55 | ||
56 | return { 'ids' => $res }; | |
57 | }; | |
58 | ||
331025d9 DM |
59 | my $mocked_get_ssh_info = sub { |
60 | my ($node, $network_cidr) = @_; | |
61 | ||
62 | return { node => $node }; | |
63 | }; | |
64 | ||
65 | my $mocked_ssh_info_to_command = sub { | |
66 | my ($info, @extra_options) = @_; | |
67 | ||
68 | return ['fake_ssh', $info->{name}, @extra_options]; | |
69 | }; | |
d0920c29 | 70 | |
492d4039 | 71 | my $statefile = ".mocked_repl_state.$$"; |
d0920c29 DM |
72 | |
73 | unlink $statefile; | |
d255af01 | 74 | $PVE::ReplicationState::state_path = $statefile; |
492d4039 TL |
75 | $PVE::ReplicationState::state_lock = ".mocked_repl_state_lock.$$"; |
76 | $PVE::API2::Replication::pvesr_lock_path = ".mocked_pvesr_lock.$$"; | |
77 | $PVE::GuestHelpers::lockdir = ".mocked_pve-manager_lock.$$"; | |
72741c0b WB |
78 | |
79 | if (!mkdir($PVE::GuestHelpers::lockdir) && !$!{EEXIST}) { | |
80 | # If we cannot create the guest helper lockdir we'll loop endlessly, so die | |
81 | # if it fails. | |
82 | die "mkdir($PVE::GuestHelpers::lockdir): $!\n"; | |
83 | } | |
d0920c29 | 84 | |
fe57e096 FG |
85 | my $pve_sshinfo_module = Test::MockModule->new('PVE::SSHInfo'); |
86 | ||
d0920c29 DM |
87 | my $pve_cluster_module = Test::MockModule->new('PVE::Cluster'); |
88 | ||
89 | my $pve_inotify_module = Test::MockModule->new('PVE::INotify'); | |
90 | ||
91 | my $mocked_qemu_load_conf = sub { | |
92 | my ($class, $vmid, $node) = @_; | |
93 | ||
94 | $node = $mocked_nodename if !$node; | |
95 | ||
96 | my $conf = $mocked_vm_configs->{$vmid}; | |
97 | ||
98 | die "no such vm '$vmid'" if !defined($conf); | |
99 | die "vm '$vmid' on wrong node" if $conf->{node} ne $node; | |
100 | ||
101 | return $conf; | |
102 | }; | |
103 | ||
104 | my $pve_qemuserver_module = Test::MockModule->new('PVE::QemuServer'); | |
105 | ||
106 | my $pve_qemuconfig_module = Test::MockModule->new('PVE::QemuConfig'); | |
107 | ||
108 | my $mocked_lxc_load_conf = sub { | |
109 | my ($class, $vmid, $node) = @_; | |
110 | ||
111 | $node = $mocked_nodename if !$node; | |
112 | ||
113 | my $conf = $mocked_ct_configs->{$vmid}; | |
114 | ||
115 | die "no such ct '$vmid'" if !defined($conf); | |
116 | die "ct '$vmid' on wrong node" if $conf->{node} ne $node; | |
117 | ||
118 | return $conf; | |
119 | }; | |
120 | ||
121 | my $pve_lxc_config_module = Test::MockModule->new('PVE::LXC::Config'); | |
122 | ||
d092dc4f | 123 | my $mocked_replication_config_new = sub { |
d0920c29 | 124 | |
c5014e65 DM |
125 | my $res = clone($mocked_replication_jobs); |
126 | ||
d0920c29 DM |
127 | return bless { ids => $res }, 'PVE::ReplicationConfig'; |
128 | }; | |
129 | ||
130 | my $mocked_storage_config = { | |
131 | ids => { | |
132 | local => { | |
133 | type => 'dir', | |
134 | shared => 0, | |
135 | content => { | |
136 | 'iso' => 1, | |
137 | 'backup' => 1, | |
138 | }, | |
139 | path => "/var/lib/vz", | |
140 | }, | |
141 | 'local-zfs' => { | |
142 | type => 'zfspool', | |
143 | pool => 'nonexistent-testpool', | |
144 | shared => 0, | |
145 | content => { | |
146 | 'images' => 1, | |
147 | 'rootdir' => 1 | |
148 | }, | |
149 | }, | |
150 | }, | |
151 | }; | |
152 | ||
153 | my $pve_storage_module = Test::MockModule->new('PVE::Storage'); | |
c5014e65 | 154 | |
4550bb78 DM |
155 | my $mocked_storage_content = {}; |
156 | ||
157 | sub register_mocked_volid { | |
158 | my ($volid, $snapname) = @_; | |
159 | ||
160 | my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid); | |
331025d9 | 161 | my $scfg = $mocked_storage_config->{ids}->{$storeid} || |
4550bb78 DM |
162 | die "no such storage '$storeid'\n"; |
163 | ||
164 | my $d = $mocked_storage_content->{$storeid}->{$volname} //= {}; | |
165 | ||
166 | $d->{$snapname} = 1 if $snapname; | |
167 | } | |
168 | ||
169 | my $mocked_volume_snapshot_list = sub { | |
170 | my ($cfg, $volid, $prefix) = @_; | |
171 | ||
172 | my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid); | |
173 | my $snaps = []; | |
174 | ||
175 | if (my $d = $mocked_storage_content->{$storeid}->{$volname}) { | |
176 | $snaps = [keys %$d]; | |
177 | } | |
178 | ||
179 | return $snaps; | |
180 | }; | |
181 | ||
182 | my $mocked_volume_snapshot = sub { | |
183 | my ($cfg, $volid, $snap) = @_; | |
184 | ||
185 | my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid); | |
186 | ||
187 | my $d = $mocked_storage_content->{$storeid}->{$volname}; | |
188 | die "no such volid '$volid'\n" if !$d; | |
189 | $d->{$snap} = 1; | |
0a7bd2d2 FE |
190 | |
191 | return; | |
4550bb78 DM |
192 | }; |
193 | ||
194 | my $mocked_volume_snapshot_delete = sub { | |
195 | my ($cfg, $volid, $snap, $running) = @_; | |
196 | ||
197 | my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid); | |
198 | my $d = $mocked_storage_content->{$storeid}->{$volname}; | |
199 | die "no such volid '$volid'\n" if !$d; | |
200 | delete $d->{$snap} || die "no such snapshot '$snap' on '$volid'\n"; | |
201 | }; | |
202 | ||
f842e812 DM |
203 | my $pve_replication_module = Test::MockModule->new('PVE::Replication'); |
204 | ||
205 | my $mocked_job_logfile_name = sub { | |
206 | my ($jobid) = @_; | |
207 | ||
208 | return ".mocked_replication_log_$jobid"; | |
209 | }; | |
210 | ||
211 | my $mocked_log_time = 0; | |
212 | ||
213 | my $mocked_get_log_time = sub { | |
214 | return $mocked_log_time; | |
215 | }; | |
216 | ||
0d7c2683 FG |
217 | my $locks = {}; |
218 | ||
219 | my $mocked_cfs_lock_file = sub { | |
220 | my ($filename, $timeout, $code, @param) = @_; | |
221 | ||
222 | die "$filename already locked\n" if ($locks->{$filename}); | |
223 | ||
224 | $locks->{$filename} = 1; | |
225 | ||
226 | my $res = $code->(@param); | |
227 | ||
228 | delete $locks->{$filename}; | |
229 | ||
230 | return $res; | |
231 | }; | |
232 | ||
33cd5dfe FG |
233 | my $mocked_cfs_read_file = sub { |
234 | my ($filename) = @_; | |
235 | ||
236 | return {} if $filename eq 'datacenter.cfg'; | |
237 | return PVE::Cluster::cfs_read_file($filename); | |
238 | }; | |
239 | ||
0d7c2683 FG |
240 | my $mocked_cfs_write_file = sub { |
241 | my ($filename, $cfg) = @_; | |
242 | ||
243 | die "wrong file - $filename\n" if $filename ne 'replication.cfg'; | |
244 | ||
245 | $cfg->write_config(); # checks but no actual write to pmxcfs | |
246 | }; | |
247 | ||
d0920c29 | 248 | sub setup { |
d092dc4f | 249 | $pve_replication_state_module->mock(job_logfile_name => $mocked_job_logfile_name); |
f842e812 DM |
250 | $pve_replication_module->mock(get_log_time => $mocked_get_log_time); |
251 | ||
d0920c29 | 252 | $pve_storage_module->mock(config => sub { return $mocked_storage_config; }); |
4550bb78 DM |
253 | $pve_storage_module->mock(volume_snapshot_list => $mocked_volume_snapshot_list); |
254 | $pve_storage_module->mock(volume_snapshot => $mocked_volume_snapshot); | |
255 | $pve_storage_module->mock(volume_snapshot_delete => $mocked_volume_snapshot_delete); | |
d0920c29 | 256 | |
0d7c2683 FG |
257 | $pve_replication_config_module->mock( |
258 | new => $mocked_replication_config_new, | |
259 | lock => sub { $mocked_cfs_lock_file->('replication.cfg', undef, $_[0]); }, | |
260 | write => sub { $mocked_cfs_write_file->('replication.cfg', $_[0]); }, | |
261 | ); | |
d0920c29 DM |
262 | $pve_qemuserver_module->mock(check_running => sub { return 0; }); |
263 | $pve_qemuconfig_module->mock(load_config => $mocked_qemu_load_conf); | |
264 | ||
265 | $pve_lxc_config_module->mock(load_config => $mocked_lxc_load_conf); | |
266 | ||
fe57e096 | 267 | $pve_sshinfo_module->mock( |
331025d9 DM |
268 | get_ssh_info => $mocked_get_ssh_info, |
269 | ssh_info_to_command => $mocked_ssh_info_to_command, | |
fe57e096 FG |
270 | ); |
271 | ||
272 | $pve_cluster_module->mock( | |
12c206a5 | 273 | get_vmlist => sub { return $mocked_vmlist->(); }, |
b03b4749 | 274 | get_members => $mocked_get_members, |
0d7c2683 FG |
275 | cfs_update => sub {}, |
276 | cfs_lock_file => $mocked_cfs_lock_file, | |
277 | cfs_write_file => $mocked_cfs_write_file, | |
33cd5dfe | 278 | cfs_read_file => $mocked_cfs_read_file, |
0d7c2683 | 279 | ); |
d0920c29 DM |
280 | $pve_inotify_module->mock('nodename' => sub { return $mocked_nodename; }); |
281 | }; | |
282 | ||
c5014e65 DM |
283 | # code to generate/conpare test logs |
284 | ||
285 | my $logname; | |
286 | my $logfh; | |
287 | ||
288 | sub openlog { | |
289 | my ($filename) = @_; | |
290 | ||
291 | if (!$filename) { | |
292 | # compute from $0 | |
293 | $filename = basename($0); | |
294 | if ($filename =~ m/^(\S+)\.pl$/) { | |
295 | $filename = "$1.log"; | |
296 | } else { | |
297 | die "unable to compute log name for $0"; | |
298 | } | |
299 | } | |
300 | ||
301 | die "log already open" if defined($logname); | |
302 | ||
303 | open (my $fh, ">", "$filename.tmp") || | |
304 | die "unable to open log - $!"; | |
305 | ||
306 | $logname = $filename; | |
307 | $logfh = $fh; | |
308 | } | |
309 | ||
c5014e65 DM |
310 | sub commit_log { |
311 | ||
312 | close($logfh); | |
313 | ||
314 | if (-f $logname) { | |
315 | my $diff = `diff -u '$logname' '$logname.tmp'`; | |
316 | if ($diff) { | |
58d46211 | 317 | warn "got unexpected output\n"; |
c5014e65 DM |
318 | print "# diff -u '$logname' '$logname.tmp'\n"; |
319 | print $diff; | |
320 | exit(-1); | |
321 | } | |
322 | } else { | |
323 | rename("$logname.tmp", $logname) || die "rename log failed - $!"; | |
324 | } | |
325 | } | |
326 | ||
327 | my $status; | |
328 | ||
329 | # helper to track job status | |
330 | sub track_jobs { | |
331 | my ($ctime) = @_; | |
332 | ||
f842e812 DM |
333 | $mocked_log_time = $ctime; |
334 | ||
c364b61f DM |
335 | my $logmsg = sub { |
336 | my ($msg) = @_; | |
337 | ||
f842e812 DM |
338 | print "$msg\n"; |
339 | print $logfh "$msg\n"; | |
c364b61f DM |
340 | }; |
341 | ||
c5014e65 | 342 | if (!$status) { |
d092dc4f | 343 | $status = PVE::ReplicationState::job_status(); |
c5014e65 DM |
344 | foreach my $jobid (sort keys %$status) { |
345 | my $jobcfg = $status->{$jobid}; | |
f842e812 | 346 | $logmsg->("$ctime $jobid: new job next_sync => $jobcfg->{next_sync}"); |
c5014e65 DM |
347 | } |
348 | } | |
349 | ||
2aa02957 | 350 | PVE::API2::Replication::run_jobs($ctime, $logmsg, 1); |
c5014e65 | 351 | |
d092dc4f | 352 | my $new = PVE::ReplicationState::job_status(); |
c5014e65 DM |
353 | |
354 | # detect removed jobs | |
355 | foreach my $jobid (sort keys %$status) { | |
356 | if (!$new->{$jobid}) { | |
f842e812 | 357 | $logmsg->("$ctime $jobid: vanished job"); |
c5014e65 DM |
358 | } |
359 | } | |
360 | ||
361 | foreach my $jobid (sort keys %$new) { | |
362 | my $jobcfg = $new->{$jobid}; | |
363 | my $oldcfg = $status->{$jobid}; | |
364 | if (!$oldcfg) { | |
f842e812 | 365 | $logmsg->("$ctime $jobid: new job next_sync => $jobcfg->{next_sync}"); |
c5014e65 DM |
366 | next; # no old state to compare |
367 | } else { | |
368 | foreach my $k (qw(target guest vmtype next_sync)) { | |
369 | my $changes = ''; | |
370 | if ($oldcfg->{$k} ne $jobcfg->{$k}) { | |
371 | $changes .= ', ' if $changes; | |
372 | $changes .= "$k => $jobcfg->{$k}"; | |
373 | } | |
f842e812 | 374 | $logmsg->("$ctime $jobid: changed config $changes") if $changes; |
c5014e65 DM |
375 | } |
376 | } | |
377 | ||
378 | my $oldstate = $oldcfg->{state}; | |
356fbf79 | 379 | |
c5014e65 DM |
380 | my $state = $jobcfg->{state}; |
381 | ||
382 | my $changes = ''; | |
e137f69f | 383 | foreach my $k (qw(last_node last_try last_sync fail_count error)) { |
c5014e65 | 384 | if (($oldstate->{$k} // '') ne ($state->{$k} // '')) { |
39c41c9d | 385 | my $value = $state->{$k} // ''; |
c5014e65 DM |
386 | chomp $value; |
387 | $changes .= ', ' if $changes; | |
388 | $changes .= "$k => $value"; | |
389 | } | |
390 | } | |
f842e812 | 391 | $logmsg->("$ctime $jobid: changed state $changes") if $changes; |
c5014e65 | 392 | |
356fbf79 DM |
393 | my $old_storeid_list = $oldstate->{storeid_list}; |
394 | my $storeid_list = $state->{storeid_list}; | |
395 | ||
396 | my $storeid_list_changes = 0; | |
397 | foreach my $storeid (@$storeid_list) { | |
398 | next if grep { $_ eq $storeid } @$old_storeid_list; | |
399 | $storeid_list_changes = 1; | |
400 | } | |
401 | ||
402 | foreach my $storeid (@$old_storeid_list) { | |
403 | next if grep { $_ eq $storeid } @$storeid_list; | |
404 | $storeid_list_changes = 1; | |
405 | } | |
406 | ||
f842e812 | 407 | $logmsg->("$ctime $jobid: changed storeid list " . join(',', @$storeid_list)) |
356fbf79 | 408 | if $storeid_list_changes; |
c5014e65 DM |
409 | } |
410 | $status = $new; | |
411 | } | |
d0920c29 DM |
412 | |
413 | ||
414 | 1; |