bin/test/ReplicationTestEnv.pm

   1 package ReplicationTestEnv;
   2
   3 use strict;
   4 use warnings;
   5 use JSON;
   6 use Clone 'clone';
   7 use File::Basename;
   8
   9 use lib ('.', '../..');
  10
  11 use Data::Dumper;
  12
  13 use PVE::INotify;
  14 use PVE::Cluster;
  15 use PVE::Storage;
  16 use PVE::ReplicationConfig;
  17 use PVE::ReplicationState;
  18 use PVE::Replication;
  19 use PVE::QemuConfig;
  20 use PVE::LXC::Config;
  21
  22 use Test::MockModule;
  23
  24 our $mocked_nodename = 'node1';
  25
  26 our $mocked_replication_jobs = {};
  27
  28 my $pve_replicationconfig = Test::MockModule->new('PVE::ReplicationConfig');
  29
  30 our $mocked_vm_configs = {};
  31
  32 our $mocked_ct_configs = {};
  33
  34 my $mocked_vmlist = sub {
  35     my $res = {};
  36
  37     foreach my $id (keys %$mocked_ct_configs) {
  38         my $d = $mocked_ct_configs->{$id};
  39         $res->{$id} = { 'type' => 'lxc', 'node' => $d->{node}, 'version' => 1 };
  40     }
  41     foreach my $id (keys %$mocked_vm_configs) {
  42         my $d = $mocked_vm_configs->{$id};
  43         $res->{$id} = { 'type' => 'qemu', 'node' => $d->{node}, 'version' => 1 };
  44     }
  45
  46     return { 'ids' => $res };
  47 };
  48
  49 my $mocked_get_ssh_info  = sub {
  50     my ($node, $network_cidr) = @_;
  51
  52     return { node => $node };
  53 };
  54
  55 my $mocked_ssh_info_to_command = sub {
  56     my ($info, @extra_options) = @_;
  57
  58     return ['fake_ssh', $info->{name}, @extra_options];
  59 };
  60
  61 my $statefile = ".mocked_repl_state";
  62
  63 unlink $statefile;
  64 $PVE::ReplicationState::state_path = $statefile;
  65 $PVE::ReplicationState::state_lock = ".mocked_repl_state_lock";
  66 $PVE::Replication::pvesr_lock_path = ".mocked_pvesr_lock";
  67 $PVE::GuestHelpers::lockdir = ".mocked_pve-manager_lock";
  68
  69 if (!mkdir($PVE::GuestHelpers::lockdir) && !$!{EEXIST}) {
  70     # If we cannot create the guest helper lockdir we'll loop endlessly, so die
  71     # if it fails.
  72     die "mkdir($PVE::GuestHelpers::lockdir): $!\n";
  73 }
  74
  75 my $pve_cluster_module = Test::MockModule->new('PVE::Cluster');
  76
  77 my $pve_inotify_module = Test::MockModule->new('PVE::INotify');
  78
  79 my $mocked_qemu_load_conf = sub {
  80     my ($class, $vmid, $node) = @_;
  81
  82     $node = $mocked_nodename if !$node;
  83
  84     my $conf = $mocked_vm_configs->{$vmid};
  85
  86     die "no such vm '$vmid'" if !defined($conf);
  87     die "vm '$vmid' on wrong node" if $conf->{node} ne $node;
  88
  89     return $conf;
  90 };
  91
  92 my $pve_qemuserver_module = Test::MockModule->new('PVE::QemuServer');
  93
  94 my $pve_qemuconfig_module = Test::MockModule->new('PVE::QemuConfig');
  95
  96 my $mocked_lxc_load_conf = sub {
  97     my ($class, $vmid, $node) = @_;
  98
  99     $node = $mocked_nodename if !$node;
 100
 101     my $conf = $mocked_ct_configs->{$vmid};
 102
 103     die "no such ct '$vmid'" if !defined($conf);
 104     die "ct '$vmid' on wrong node" if $conf->{node} ne $node;
 105
 106     return $conf;
 107 };
 108
 109 my $pve_lxc_config_module = Test::MockModule->new('PVE::LXC::Config');
 110
 111 my $mocked_replication_config = sub {
 112
 113     my $res = clone($mocked_replication_jobs);
 114
 115     return bless { ids => $res }, 'PVE::ReplicationConfig';
 116 };
 117
 118 my $mocked_storage_config = {
 119     ids => {
 120         local => {
 121             type => 'dir',
 122             shared => 0,
 123             content => {
 124                 'iso' => 1,
 125                 'backup' => 1,
 126             },
 127             path => "/var/lib/vz",
 128         },
 129         'local-zfs' => {
 130             type => 'zfspool',
 131             pool => 'nonexistent-testpool',
 132             shared => 0,
 133             content => {
 134                 'images' => 1,
 135                 'rootdir' => 1
 136             },
 137         },
 138     },
 139 };
 140
 141 my $pve_storage_module = Test::MockModule->new('PVE::Storage');
 142
 143 my $mocked_storage_content = {};
 144
 145 sub register_mocked_volid {
 146     my ($volid, $snapname) = @_;
 147
 148     my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
 149     my $scfg = $mocked_storage_config->{ids}->{$storeid} ||
 150         die "no such storage '$storeid'\n";
 151
 152     my $d = $mocked_storage_content->{$storeid}->{$volname} //= {};
 153
 154     $d->{$snapname} = 1 if $snapname;
 155 }
 156
 157 my $mocked_volume_snapshot_list = sub {
 158     my ($cfg, $volid, $prefix) = @_;
 159
 160     my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
 161     my $snaps = [];
 162
 163     if (my $d = $mocked_storage_content->{$storeid}->{$volname}) {
 164         $snaps = [keys %$d];
 165     }
 166
 167     return $snaps;
 168 };
 169
 170 my $mocked_volume_snapshot = sub {
 171     my ($cfg, $volid, $snap) = @_;
 172
 173     my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
 174
 175     my $d = $mocked_storage_content->{$storeid}->{$volname};
 176     die "no such volid '$volid'\n" if !$d;
 177     $d->{$snap} = 1;
 178 };
 179
 180 my $mocked_volume_snapshot_delete = sub {
 181     my ($cfg, $volid, $snap, $running) = @_;
 182
 183     my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
 184     my $d = $mocked_storage_content->{$storeid}->{$volname};
 185     die "no such volid '$volid'\n" if !$d;
 186     delete $d->{$snap} || die "no such snapshot '$snap' on '$volid'\n";
 187 };
 188
 189 my $pve_replication_module = Test::MockModule->new('PVE::Replication');
 190
 191 my $mocked_job_logfile_name = sub {
 192     my ($jobid) = @_;
 193
 194     return ".mocked_replication_log_$jobid";
 195 };
 196
 197 my $mocked_log_time = 0;
 198
 199 my $mocked_get_log_time = sub {
 200     return $mocked_log_time;
 201 };
 202
 203 sub setup {
 204     $pve_replication_module->mock(job_logfile_name => $mocked_job_logfile_name);
 205     $pve_replication_module->mock(get_log_time => $mocked_get_log_time);
 206
 207     $pve_storage_module->mock(config => sub { return $mocked_storage_config; });
 208     $pve_storage_module->mock(volume_snapshot_list => $mocked_volume_snapshot_list);
 209     $pve_storage_module->mock(volume_snapshot => $mocked_volume_snapshot);
 210     $pve_storage_module->mock(volume_snapshot_delete => $mocked_volume_snapshot_delete);
 211
 212     $pve_replicationconfig->mock(new => $mocked_replication_config);
 213     $pve_qemuserver_module->mock(check_running => sub { return 0; });
 214     $pve_qemuconfig_module->mock(load_config => $mocked_qemu_load_conf);
 215
 216     $pve_lxc_config_module->mock(load_config => $mocked_lxc_load_conf);
 217
 218
 219     $pve_cluster_module->mock(
 220         get_ssh_info => $mocked_get_ssh_info,
 221         ssh_info_to_command => $mocked_ssh_info_to_command,
 222         get_vmlist => sub { return $mocked_vmlist->(); });
 223     $pve_inotify_module->mock('nodename' => sub { return $mocked_nodename; });
 224 };
 225
 226 # code to generate/conpare test logs
 227
 228 my $logname;
 229 my $logfh;
 230
 231 sub openlog {
 232     my ($filename) = @_;
 233
 234     if (!$filename) {
 235         # compute from $0
 236         $filename = basename($0);
 237         if ($filename =~ m/^(\S+)\.pl$/) {
 238             $filename = "$1.log";
 239         } else {
 240             die "unable to compute log name for $0";
 241         }
 242     }
 243
 244     die "log already open" if defined($logname);
 245
 246     open (my $fh, ">", "$filename.tmp") ||
 247         die "unable to open log  - $!";
 248
 249     $logname = $filename;
 250     $logfh = $fh;
 251 }
 252
 253 sub commit_log {
 254
 255     close($logfh);
 256
 257     if (-f $logname) {
 258         my $diff = `diff -u '$logname' '$logname.tmp'`;
 259         if ($diff) {
 260             warn "got unexpeted output\n";
 261             print "# diff -u '$logname' '$logname.tmp'\n";
 262             print $diff;
 263             exit(-1);
 264         }
 265     } else {
 266         rename("$logname.tmp", $logname) || die "rename log failed - $!";
 267     }
 268 }
 269
 270 my $status;
 271
 272 # helper to track job status
 273 sub track_jobs {
 274     my ($ctime) = @_;
 275
 276     $mocked_log_time = $ctime;
 277
 278     my $logmsg = sub {
 279         my ($msg) = @_;
 280
 281         print "$msg\n";
 282         print $logfh "$msg\n";
 283     };
 284
 285     if (!$status) {
 286         $status = PVE::Replication::job_status();
 287         foreach my $jobid (sort keys %$status) {
 288             my $jobcfg = $status->{$jobid};
 289             $logmsg->("$ctime $jobid: new job next_sync => $jobcfg->{next_sync}");
 290         }
 291     }
 292
 293     PVE::Replication::run_jobs($ctime, $logmsg);
 294
 295     my $new = PVE::Replication::job_status();
 296
 297     # detect removed jobs
 298     foreach my $jobid (sort keys %$status) {
 299         if (!$new->{$jobid}) {
 300             $logmsg->("$ctime $jobid: vanished job");
 301         }
 302     }
 303
 304     foreach my $jobid (sort keys %$new) {
 305         my $jobcfg = $new->{$jobid};
 306         my $oldcfg = $status->{$jobid};
 307         if (!$oldcfg) {
 308             $logmsg->("$ctime $jobid: new job next_sync => $jobcfg->{next_sync}");
 309             next; # no old state to compare
 310         } else {
 311             foreach my $k (qw(target guest vmtype next_sync)) {
 312                 my $changes = '';
 313                 if ($oldcfg->{$k} ne $jobcfg->{$k}) {
 314                     $changes .= ', ' if $changes;
 315                     $changes .= "$k => $jobcfg->{$k}";
 316                 }
 317                 $logmsg->("$ctime $jobid: changed config $changes") if $changes;
 318             }
 319         }
 320
 321         my $oldstate = $oldcfg->{state};
 322
 323         my $state = $jobcfg->{state};
 324
 325         my $changes = '';
 326         foreach my $k (qw(last_node last_try last_sync fail_count error)) {
 327             if (($oldstate->{$k} // '') ne ($state->{$k} // '')) {
 328                 my $value = $state->{$k} // '';
 329                 chomp $value;
 330                 $changes .= ', ' if $changes;
 331                 $changes .= "$k => $value";
 332             }
 333         }
 334         $logmsg->("$ctime $jobid: changed state $changes") if $changes;
 335
 336         my $old_storeid_list = $oldstate->{storeid_list};
 337         my $storeid_list = $state->{storeid_list};
 338
 339         my $storeid_list_changes = 0;
 340         foreach my $storeid (@$storeid_list) {
 341             next if grep { $_ eq $storeid } @$old_storeid_list;
 342             $storeid_list_changes = 1;
 343         }
 344
 345         foreach my $storeid (@$old_storeid_list) {
 346             next if grep { $_ eq $storeid } @$storeid_list;
 347             $storeid_list_changes = 1;
 348         }
 349
 350         $logmsg->("$ctime $jobid: changed storeid list " . join(',', @$storeid_list))
 351             if $storeid_list_changes;
 352     }
 353     $status = $new;
 354 }
 355
 356
 357 1;