test/ReplicationTestEnv.pm

   1 package ReplicationTestEnv;
   2
   3 use strict;
   4 use warnings;
   5 use JSON;
   6 use Clone 'clone';
   7 use File::Basename;
   8
   9 use lib ('.', '../..');
  10
  11 use Data::Dumper;
  12
  13 use PVE::INotify;
  14 use PVE::Cluster;
  15 use PVE::Storage;
  16 use PVE::ReplicationConfig;
  17 use PVE::ReplicationState;
  18 use PVE::API2::Replication;
  19 use PVE::Replication;
  20 use PVE::QemuConfig;
  21 use PVE::LXC::Config;
  22
  23
  24 use Test::MockModule;
  25
  26 our $mocked_nodename = 'node1';
  27
  28 our $mocked_replication_jobs = {};
  29
  30 my $pve_replication_config_module = Test::MockModule->new('PVE::ReplicationConfig');
  31 my $pve_replication_state_module = Test::MockModule->new('PVE::ReplicationState');
  32
  33 our $mocked_vm_configs = {};
  34
  35 our $mocked_ct_configs = {};
  36
  37 my $mocked_get_members = sub {
  38     return {
  39         node1 => { online => 1 },
  40         node2 => { online => 1 },
  41         node3 => { online => 1 },
  42     };
  43 };
  44
  45 my $mocked_vmlist = sub {
  46     my $res = {};
  47
  48     foreach my $id (keys %$mocked_ct_configs) {
  49         my $d = $mocked_ct_configs->{$id};
  50         $res->{$id} = { 'type' => 'lxc', 'node' => $d->{node}, 'version' => 1 };
  51     }
  52     foreach my $id (keys %$mocked_vm_configs) {
  53         my $d = $mocked_vm_configs->{$id};
  54         $res->{$id} = { 'type' => 'qemu', 'node' => $d->{node}, 'version' => 1 };
  55     }
  56
  57     return { 'ids' => $res };
  58 };
  59
  60 my $mocked_get_ssh_info  = sub {
  61     my ($node, $network_cidr) = @_;
  62
  63     return { node => $node };
  64 };
  65
  66 my $mocked_ssh_info_to_command = sub {
  67     my ($info, @extra_options) = @_;
  68
  69     return ['fake_ssh', $info->{name}, @extra_options];
  70 };
  71
  72 my $statefile = ".mocked_repl_state";
  73
  74 unlink $statefile;
  75 $PVE::ReplicationState::state_path = $statefile;
  76 $PVE::ReplicationState::state_lock = ".mocked_repl_state_lock";
  77 $PVE::API2::Replication::pvesr_lock_path = ".mocked_pvesr_lock";
  78 $PVE::GuestHelpers::lockdir = ".mocked_pve-manager_lock";
  79
  80 if (!mkdir($PVE::GuestHelpers::lockdir) && !$!{EEXIST}) {
  81     # If we cannot create the guest helper lockdir we'll loop endlessly, so die
  82     # if it fails.
  83     die "mkdir($PVE::GuestHelpers::lockdir): $!\n";
  84 }
  85
  86 my $pve_sshinfo_module = Test::MockModule->new('PVE::SSHInfo');
  87
  88 my $pve_cluster_module = Test::MockModule->new('PVE::Cluster');
  89
  90 my $pve_inotify_module = Test::MockModule->new('PVE::INotify');
  91
  92 my $mocked_qemu_load_conf = sub {
  93     my ($class, $vmid, $node) = @_;
  94
  95     $node = $mocked_nodename if !$node;
  96
  97     my $conf = $mocked_vm_configs->{$vmid};
  98
  99     die "no such vm '$vmid'" if !defined($conf);
 100     die "vm '$vmid' on wrong node" if $conf->{node} ne $node;
 101
 102     return $conf;
 103 };
 104
 105 my $pve_qemuserver_module = Test::MockModule->new('PVE::QemuServer');
 106
 107 my $pve_qemuconfig_module = Test::MockModule->new('PVE::QemuConfig');
 108
 109 my $mocked_lxc_load_conf = sub {
 110     my ($class, $vmid, $node) = @_;
 111
 112     $node = $mocked_nodename if !$node;
 113
 114     my $conf = $mocked_ct_configs->{$vmid};
 115
 116     die "no such ct '$vmid'" if !defined($conf);
 117     die "ct '$vmid' on wrong node" if $conf->{node} ne $node;
 118
 119     return $conf;
 120 };
 121
 122 my $pve_lxc_config_module = Test::MockModule->new('PVE::LXC::Config');
 123
 124 my $mocked_replication_config_new = sub {
 125
 126     my $res = clone($mocked_replication_jobs);
 127
 128     return bless { ids => $res }, 'PVE::ReplicationConfig';
 129 };
 130
 131 my $mocked_storage_config = {
 132     ids => {
 133         local => {
 134             type => 'dir',
 135             shared => 0,
 136             content => {
 137                 'iso' => 1,
 138                 'backup' => 1,
 139             },
 140             path => "/var/lib/vz",
 141         },
 142         'local-zfs' => {
 143             type => 'zfspool',
 144             pool => 'nonexistent-testpool',
 145             shared => 0,
 146             content => {
 147                 'images' => 1,
 148                 'rootdir' => 1
 149             },
 150         },
 151     },
 152 };
 153
 154 my $pve_storage_module = Test::MockModule->new('PVE::Storage');
 155
 156 my $mocked_storage_content = {};
 157
 158 sub register_mocked_volid {
 159     my ($volid, $snapname) = @_;
 160
 161     my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
 162     my $scfg = $mocked_storage_config->{ids}->{$storeid} ||
 163         die "no such storage '$storeid'\n";
 164
 165     my $d = $mocked_storage_content->{$storeid}->{$volname} //= {};
 166
 167     $d->{$snapname} = 1 if $snapname;
 168 }
 169
 170 my $mocked_volume_snapshot_list = sub {
 171     my ($cfg, $volid, $prefix) = @_;
 172
 173     my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
 174     my $snaps = [];
 175
 176     if (my $d = $mocked_storage_content->{$storeid}->{$volname}) {
 177         $snaps = [keys %$d];
 178     }
 179
 180     return $snaps;
 181 };
 182
 183 my $mocked_volume_snapshot = sub {
 184     my ($cfg, $volid, $snap) = @_;
 185
 186     my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
 187
 188     my $d = $mocked_storage_content->{$storeid}->{$volname};
 189     die "no such volid '$volid'\n" if !$d;
 190     $d->{$snap} = 1;
 191 };
 192
 193 my $mocked_volume_snapshot_delete = sub {
 194     my ($cfg, $volid, $snap, $running) = @_;
 195
 196     my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
 197     my $d = $mocked_storage_content->{$storeid}->{$volname};
 198     die "no such volid '$volid'\n" if !$d;
 199     delete $d->{$snap} || die "no such snapshot '$snap' on '$volid'\n";
 200 };
 201
 202 my $pve_replication_module = Test::MockModule->new('PVE::Replication');
 203
 204 my $mocked_job_logfile_name = sub {
 205     my ($jobid) = @_;
 206
 207     return ".mocked_replication_log_$jobid";
 208 };
 209
 210 my $mocked_log_time = 0;
 211
 212 my $mocked_get_log_time = sub {
 213     return $mocked_log_time;
 214 };
 215
 216 my $locks = {};
 217
 218 my $mocked_cfs_lock_file = sub {
 219     my ($filename, $timeout, $code, @param) = @_;
 220
 221     die "$filename already locked\n" if ($locks->{$filename});
 222
 223     $locks->{$filename} = 1;
 224
 225     my $res = $code->(@param);
 226
 227     delete $locks->{$filename};
 228
 229     return $res;
 230 };
 231
 232 my $mocked_cfs_read_file = sub {
 233     my ($filename) = @_;
 234
 235     return {} if $filename eq 'datacenter.cfg';
 236     return PVE::Cluster::cfs_read_file($filename);
 237 };
 238
 239 my $mocked_cfs_write_file = sub {
 240     my ($filename, $cfg) = @_;
 241
 242     die "wrong file - $filename\n" if $filename ne 'replication.cfg';
 243
 244     $cfg->write_config(); # checks but no actual write to pmxcfs
 245 };
 246
 247 sub setup {
 248     $pve_replication_state_module->mock(job_logfile_name => $mocked_job_logfile_name);
 249     $pve_replication_module->mock(get_log_time => $mocked_get_log_time);
 250
 251     $pve_storage_module->mock(config => sub { return $mocked_storage_config; });
 252     $pve_storage_module->mock(volume_snapshot_list => $mocked_volume_snapshot_list);
 253     $pve_storage_module->mock(volume_snapshot => $mocked_volume_snapshot);
 254     $pve_storage_module->mock(volume_snapshot_delete => $mocked_volume_snapshot_delete);
 255
 256     $pve_replication_config_module->mock(
 257         new => $mocked_replication_config_new,
 258         lock => sub { $mocked_cfs_lock_file->('replication.cfg', undef, $_[0]); },
 259         write => sub { $mocked_cfs_write_file->('replication.cfg', $_[0]); },
 260     );
 261     $pve_qemuserver_module->mock(check_running => sub { return 0; });
 262     $pve_qemuconfig_module->mock(load_config => $mocked_qemu_load_conf);
 263
 264     $pve_lxc_config_module->mock(load_config => $mocked_lxc_load_conf);
 265
 266     $pve_sshinfo_module->mock(
 267         get_ssh_info => $mocked_get_ssh_info,
 268         ssh_info_to_command => $mocked_ssh_info_to_command,
 269     );
 270
 271     $pve_cluster_module->mock(
 272         get_vmlist => sub { return $mocked_vmlist->(); },
 273         get_members => $mocked_get_members,
 274         cfs_update => sub {},
 275         cfs_lock_file => $mocked_cfs_lock_file,
 276         cfs_write_file => $mocked_cfs_write_file,
 277         cfs_read_file => $mocked_cfs_read_file,
 278     );
 279     $pve_inotify_module->mock('nodename' => sub { return $mocked_nodename; });
 280 };
 281
 282 # code to generate/conpare test logs
 283
 284 my $logname;
 285 my $logfh;
 286
 287 sub openlog {
 288     my ($filename) = @_;
 289
 290     if (!$filename) {
 291         # compute from $0
 292         $filename = basename($0);
 293         if ($filename =~ m/^(\S+)\.pl$/) {
 294             $filename = "$1.log";
 295         } else {
 296             die "unable to compute log name for $0";
 297         }
 298     }
 299
 300     die "log already open" if defined($logname);
 301
 302     open (my $fh, ">", "$filename.tmp") ||
 303         die "unable to open log  - $!";
 304
 305     $logname = $filename;
 306     $logfh = $fh;
 307 }
 308
 309 sub commit_log {
 310
 311     close($logfh);
 312
 313     if (-f $logname) {
 314         my $diff = `diff -u '$logname' '$logname.tmp'`;
 315         if ($diff) {
 316             warn "got unexpected output\n";
 317             print "# diff -u '$logname' '$logname.tmp'\n";
 318             print $diff;
 319             exit(-1);
 320         }
 321     } else {
 322         rename("$logname.tmp", $logname) || die "rename log failed - $!";
 323     }
 324 }
 325
 326 my $status;
 327
 328 # helper to track job status
 329 sub track_jobs {
 330     my ($ctime) = @_;
 331
 332     $mocked_log_time = $ctime;
 333
 334     my $logmsg = sub {
 335         my ($msg) = @_;
 336
 337         print "$msg\n";
 338         print $logfh "$msg\n";
 339     };
 340
 341     if (!$status) {
 342         $status = PVE::ReplicationState::job_status();
 343         foreach my $jobid (sort keys %$status) {
 344             my $jobcfg = $status->{$jobid};
 345             $logmsg->("$ctime $jobid: new job next_sync => $jobcfg->{next_sync}");
 346         }
 347     }
 348
 349     PVE::API2::Replication::run_jobs($ctime, $logmsg, 1);
 350
 351     my $new = PVE::ReplicationState::job_status();
 352
 353     # detect removed jobs
 354     foreach my $jobid (sort keys %$status) {
 355         if (!$new->{$jobid}) {
 356             $logmsg->("$ctime $jobid: vanished job");
 357         }
 358     }
 359
 360     foreach my $jobid (sort keys %$new) {
 361         my $jobcfg = $new->{$jobid};
 362         my $oldcfg = $status->{$jobid};
 363         if (!$oldcfg) {
 364             $logmsg->("$ctime $jobid: new job next_sync => $jobcfg->{next_sync}");
 365             next; # no old state to compare
 366         } else {
 367             foreach my $k (qw(target guest vmtype next_sync)) {
 368                 my $changes = '';
 369                 if ($oldcfg->{$k} ne $jobcfg->{$k}) {
 370                     $changes .= ', ' if $changes;
 371                     $changes .= "$k => $jobcfg->{$k}";
 372                 }
 373                 $logmsg->("$ctime $jobid: changed config $changes") if $changes;
 374             }
 375         }
 376
 377         my $oldstate = $oldcfg->{state};
 378
 379         my $state = $jobcfg->{state};
 380
 381         my $changes = '';
 382         foreach my $k (qw(last_node last_try last_sync fail_count error)) {
 383             if (($oldstate->{$k} // '') ne ($state->{$k} // '')) {
 384                 my $value = $state->{$k} // '';
 385                 chomp $value;
 386                 $changes .= ', ' if $changes;
 387                 $changes .= "$k => $value";
 388             }
 389         }
 390         $logmsg->("$ctime $jobid: changed state $changes") if $changes;
 391
 392         my $old_storeid_list = $oldstate->{storeid_list};
 393         my $storeid_list = $state->{storeid_list};
 394
 395         my $storeid_list_changes = 0;
 396         foreach my $storeid (@$storeid_list) {
 397             next if grep { $_ eq $storeid } @$old_storeid_list;
 398             $storeid_list_changes = 1;
 399         }
 400
 401         foreach my $storeid (@$old_storeid_list) {
 402             next if grep { $_ eq $storeid } @$storeid_list;
 403             $storeid_list_changes = 1;
 404         }
 405
 406         $logmsg->("$ctime $jobid: changed storeid list " . join(',', @$storeid_list))
 407             if $storeid_list_changes;
 408     }
 409     $status = $new;
 410 }
 411
 412
 413 1;