test/ReplicationTestEnv.pm

   1 package ReplicationTestEnv;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Clone 'clone';
   7 use File::Basename;
   8 use JSON;
   9
  10 use lib ('.', '../..');
  11
  12 use PVE::Cluster;
  13 use PVE::INotify;
  14 use PVE::LXC::Config;
  15 use PVE::QemuConfig;
  16 use PVE::Storage;
  17
  18 use PVE::API2::Replication;
  19 use PVE::Replication;
  20 use PVE::ReplicationConfig;
  21 use PVE::ReplicationState;
  22
  23 use Test::MockModule;
  24
  25 our $mocked_nodename = 'node1';
  26
  27 our $mocked_replication_jobs = {};
  28
  29 my $pve_replication_config_module = Test::MockModule->new('PVE::ReplicationConfig');
  30 my $pve_replication_state_module = Test::MockModule->new('PVE::ReplicationState');
  31
  32 our $mocked_vm_configs = {};
  33
  34 our $mocked_ct_configs = {};
  35
  36 my $mocked_get_members = sub {
  37     return {
  38         node1 => { online => 1 },
  39         node2 => { online => 1 },
  40         node3 => { online => 1 },
  41     };
  42 };
  43
  44 my $mocked_vmlist = sub {
  45     my $res = {};
  46
  47     foreach my $id (keys %$mocked_ct_configs) {
  48         my $d = $mocked_ct_configs->{$id};
  49         $res->{$id} = { 'type' => 'lxc', 'node' => $d->{node}, 'version' => 1 };
  50     }
  51     foreach my $id (keys %$mocked_vm_configs) {
  52         my $d = $mocked_vm_configs->{$id};
  53         $res->{$id} = { 'type' => 'qemu', 'node' => $d->{node}, 'version' => 1 };
  54     }
  55
  56     return { 'ids' => $res };
  57 };
  58
  59 my $mocked_get_ssh_info  = sub {
  60     my ($node, $network_cidr) = @_;
  61
  62     return { node => $node };
  63 };
  64
  65 my $mocked_ssh_info_to_command = sub {
  66     my ($info, @extra_options) = @_;
  67
  68     return ['fake_ssh', $info->{name}, @extra_options];
  69 };
  70
  71 my $statefile = ".mocked_repl_state.$$";
  72
  73 unlink $statefile;
  74 $PVE::ReplicationState::state_path = $statefile;
  75 $PVE::ReplicationState::state_lock = ".mocked_repl_state_lock.$$";
  76 $PVE::API2::Replication::pvesr_lock_path = ".mocked_pvesr_lock.$$";
  77 $PVE::GuestHelpers::lockdir = ".mocked_pve-manager_lock.$$";
  78
  79 if (!mkdir($PVE::GuestHelpers::lockdir) && !$!{EEXIST}) {
  80     # If we cannot create the guest helper lockdir we'll loop endlessly, so die
  81     # if it fails.
  82     die "mkdir($PVE::GuestHelpers::lockdir): $!\n";
  83 }
  84
  85 my $pve_sshinfo_module = Test::MockModule->new('PVE::SSHInfo');
  86
  87 my $pve_cluster_module = Test::MockModule->new('PVE::Cluster');
  88
  89 my $pve_inotify_module = Test::MockModule->new('PVE::INotify');
  90
  91 my $mocked_qemu_load_conf = sub {
  92     my ($class, $vmid, $node) = @_;
  93
  94     $node = $mocked_nodename if !$node;
  95
  96     my $conf = $mocked_vm_configs->{$vmid};
  97
  98     die "no such vm '$vmid'" if !defined($conf);
  99     die "vm '$vmid' on wrong node" if $conf->{node} ne $node;
 100
 101     return $conf;
 102 };
 103
 104 my $pve_qemuserver_module = Test::MockModule->new('PVE::QemuServer');
 105
 106 my $pve_qemuconfig_module = Test::MockModule->new('PVE::QemuConfig');
 107
 108 my $mocked_lxc_load_conf = sub {
 109     my ($class, $vmid, $node) = @_;
 110
 111     $node = $mocked_nodename if !$node;
 112
 113     my $conf = $mocked_ct_configs->{$vmid};
 114
 115     die "no such ct '$vmid'" if !defined($conf);
 116     die "ct '$vmid' on wrong node" if $conf->{node} ne $node;
 117
 118     return $conf;
 119 };
 120
 121 my $pve_lxc_config_module = Test::MockModule->new('PVE::LXC::Config');
 122
 123 my $mocked_replication_config_new = sub {
 124
 125     my $res = clone($mocked_replication_jobs);
 126
 127     return bless { ids => $res }, 'PVE::ReplicationConfig';
 128 };
 129
 130 my $mocked_storage_config = {
 131     ids => {
 132         local => {
 133             type => 'dir',
 134             shared => 0,
 135             content => {
 136                 'iso' => 1,
 137                 'backup' => 1,
 138             },
 139             path => "/var/lib/vz",
 140         },
 141         'local-zfs' => {
 142             type => 'zfspool',
 143             pool => 'nonexistent-testpool',
 144             shared => 0,
 145             content => {
 146                 'images' => 1,
 147                 'rootdir' => 1
 148             },
 149         },
 150     },
 151 };
 152
 153 my $pve_storage_module = Test::MockModule->new('PVE::Storage');
 154
 155 my $mocked_storage_content = {};
 156
 157 my $timestamp_counter = 0;
 158
 159 sub generate_snapshot_info {
 160     $timestamp_counter++;
 161
 162     return {
 163         id => $timestamp_counter,
 164         timestamp => $timestamp_counter,
 165     };
 166 }
 167
 168 sub register_mocked_volid {
 169     my ($volid, $snapname) = @_;
 170
 171     my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
 172     my $scfg = $mocked_storage_config->{ids}->{$storeid} ||
 173         die "no such storage '$storeid'\n";
 174
 175     my $d = $mocked_storage_content->{$storeid}->{$volname} //= {};
 176
 177     $d->{$snapname} = generate_snapshot_info() if $snapname;
 178 }
 179
 180 my $mocked_volume_snapshot = sub {
 181     my ($cfg, $volid, $snap) = @_;
 182
 183     my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
 184
 185     my $d = $mocked_storage_content->{$storeid}->{$volname};
 186     die "no such volid '$volid'\n" if !$d;
 187     $d->{$snap} = generate_snapshot_info();
 188
 189     return;
 190 };
 191
 192 my $mocked_volume_snapshot_delete = sub {
 193     my ($cfg, $volid, $snap, $running) = @_;
 194
 195     my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
 196     my $d = $mocked_storage_content->{$storeid}->{$volname};
 197     die "no such volid '$volid'\n" if !$d;
 198     delete $d->{$snap} || die "no such snapshot '$snap' on '$volid'\n";
 199 };
 200
 201 my $mocked_volume_snapshot_info = sub {
 202     my ($cfg, $volid) = @_;
 203
 204     my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
 205
 206     return $mocked_storage_content->{$storeid}->{$volname} // {};
 207 };
 208
 209 my $pve_replication_module = Test::MockModule->new('PVE::Replication');
 210
 211 my $mocked_job_logfile_name = sub {
 212     my ($jobid) = @_;
 213
 214     return ".mocked_replication_log_$jobid";
 215 };
 216
 217 my $mocked_log_time = 0;
 218
 219 my $mocked_get_log_time = sub {
 220     return $mocked_log_time;
 221 };
 222
 223 my $locks = {};
 224
 225 my $mocked_cfs_lock_file = sub {
 226     my ($filename, $timeout, $code, @param) = @_;
 227
 228     die "$filename already locked\n" if ($locks->{$filename});
 229
 230     $locks->{$filename} = 1;
 231
 232     my $res = $code->(@param);
 233
 234     delete $locks->{$filename};
 235
 236     return $res;
 237 };
 238
 239 my $mocked_cfs_read_file = sub {
 240     my ($filename) = @_;
 241
 242     return {} if $filename eq 'datacenter.cfg';
 243     return PVE::Cluster::cfs_read_file($filename);
 244 };
 245
 246 my $mocked_cfs_write_file = sub {
 247     my ($filename, $cfg) = @_;
 248
 249     die "wrong file - $filename\n" if $filename ne 'replication.cfg';
 250
 251     $cfg->write_config(); # checks but no actual write to pmxcfs
 252 };
 253
 254 sub setup {
 255     $pve_replication_state_module->mock(job_logfile_name => $mocked_job_logfile_name);
 256     $pve_replication_module->mock(get_log_time => $mocked_get_log_time);
 257
 258     $pve_storage_module->mock(config => sub { return $mocked_storage_config; });
 259     $pve_storage_module->mock(volume_snapshot => $mocked_volume_snapshot);
 260     $pve_storage_module->mock(volume_snapshot_delete => $mocked_volume_snapshot_delete);
 261     $pve_storage_module->mock(volume_snapshot_info => $mocked_volume_snapshot_info);
 262
 263     $pve_replication_config_module->mock(
 264         new => $mocked_replication_config_new,
 265         lock => sub { $mocked_cfs_lock_file->('replication.cfg', undef, $_[0]); },
 266         write => sub { $mocked_cfs_write_file->('replication.cfg', $_[0]); },
 267     );
 268     $pve_qemuserver_module->mock(check_running => sub { return 0; });
 269     $pve_qemuconfig_module->mock(load_config => $mocked_qemu_load_conf);
 270
 271     $pve_lxc_config_module->mock(load_config => $mocked_lxc_load_conf);
 272
 273     $pve_sshinfo_module->mock(
 274         get_ssh_info => $mocked_get_ssh_info,
 275         ssh_info_to_command => $mocked_ssh_info_to_command,
 276     );
 277
 278     $pve_cluster_module->mock(
 279         get_vmlist => sub { return $mocked_vmlist->(); },
 280         get_members => $mocked_get_members,
 281         cfs_update => sub {},
 282         cfs_lock_file => $mocked_cfs_lock_file,
 283         cfs_write_file => $mocked_cfs_write_file,
 284         cfs_read_file => $mocked_cfs_read_file,
 285     );
 286     $pve_inotify_module->mock('nodename' => sub { return $mocked_nodename; });
 287 };
 288
 289 # code to generate/conpare test logs
 290
 291 my $logname;
 292 my $logfh;
 293
 294 sub openlog {
 295     my ($filename) = @_;
 296
 297     if (!$filename) {
 298         # compute from $0
 299         $filename = basename($0);
 300         if ($filename =~ m/^(\S+)\.pl$/) {
 301             $filename = "$1.log";
 302         } else {
 303             die "unable to compute log name for $0";
 304         }
 305     }
 306
 307     die "log already open" if defined($logname);
 308
 309     open (my $fh, ">", "$filename.tmp") ||
 310         die "unable to open log  - $!";
 311
 312     $logname = $filename;
 313     $logfh = $fh;
 314 }
 315
 316 sub commit_log {
 317
 318     close($logfh);
 319
 320     if (-f $logname) {
 321         my $diff = `diff -u '$logname' '$logname.tmp'`;
 322         if ($diff) {
 323             warn "got unexpected output\n";
 324             print "# diff -u '$logname' '$logname.tmp'\n";
 325             print $diff;
 326             exit(-1);
 327         }
 328     } else {
 329         rename("$logname.tmp", $logname) || die "rename log failed - $!";
 330     }
 331 }
 332
 333 my $status;
 334
 335 # helper to track job status
 336 sub track_jobs {
 337     my ($ctime) = @_;
 338
 339     $mocked_log_time = $ctime;
 340
 341     my $logmsg = sub {
 342         my ($msg) = @_;
 343
 344         print "$msg\n";
 345         print $logfh "$msg\n";
 346     };
 347
 348     if (!$status) {
 349         $status = PVE::ReplicationState::job_status();
 350         foreach my $jobid (sort keys %$status) {
 351             my $jobcfg = $status->{$jobid};
 352             $logmsg->("$ctime $jobid: new job next_sync => $jobcfg->{next_sync}");
 353         }
 354     }
 355
 356     PVE::API2::Replication::run_jobs($ctime, $logmsg, 1);
 357
 358     my $new = PVE::ReplicationState::job_status();
 359
 360     # detect removed jobs
 361     foreach my $jobid (sort keys %$status) {
 362         if (!$new->{$jobid}) {
 363             $logmsg->("$ctime $jobid: vanished job");
 364         }
 365     }
 366
 367     foreach my $jobid (sort keys %$new) {
 368         my $jobcfg = $new->{$jobid};
 369         my $oldcfg = $status->{$jobid};
 370         if (!$oldcfg) {
 371             $logmsg->("$ctime $jobid: new job next_sync => $jobcfg->{next_sync}");
 372             next; # no old state to compare
 373         } else {
 374             foreach my $k (qw(target guest vmtype next_sync)) {
 375                 my $changes = '';
 376                 if ($oldcfg->{$k} ne $jobcfg->{$k}) {
 377                     $changes .= ', ' if $changes;
 378                     $changes .= "$k => $jobcfg->{$k}";
 379                 }
 380                 $logmsg->("$ctime $jobid: changed config $changes") if $changes;
 381             }
 382         }
 383
 384         my $oldstate = $oldcfg->{state};
 385
 386         my $state = $jobcfg->{state};
 387
 388         my $changes = '';
 389         foreach my $k (qw(last_node last_try last_sync fail_count error)) {
 390             if (($oldstate->{$k} // '') ne ($state->{$k} // '')) {
 391                 my $value = $state->{$k} // '';
 392                 chomp $value;
 393                 $changes .= ', ' if $changes;
 394                 $changes .= "$k => $value";
 395             }
 396         }
 397         $logmsg->("$ctime $jobid: changed state $changes") if $changes;
 398
 399         my $old_storeid_list = $oldstate->{storeid_list};
 400         my $storeid_list = $state->{storeid_list};
 401
 402         my $storeid_list_changes = 0;
 403         foreach my $storeid (@$storeid_list) {
 404             next if grep { $_ eq $storeid } @$old_storeid_list;
 405             $storeid_list_changes = 1;
 406         }
 407
 408         foreach my $storeid (@$old_storeid_list) {
 409             next if grep { $_ eq $storeid } @$storeid_list;
 410             $storeid_list_changes = 1;
 411         }
 412
 413         $logmsg->("$ctime $jobid: changed storeid list " . join(',', @$storeid_list))
 414             if $storeid_list_changes;
 415     }
 416     $status = $new;
 417 }
 418
 419
 420 1;