test/ReplicationTestEnv.pm

   1 package ReplicationTestEnv;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Clone 'clone';
   7 use File::Basename;
   8 use JSON;
   9
  10 use lib ('.', '../..');
  11
  12 use PVE::Cluster;
  13 use PVE::INotify;
  14 use PVE::LXC::Config;
  15 use PVE::QemuConfig;
  16 use PVE::Storage;
  17
  18 use PVE::API2::Replication;
  19 use PVE::Replication;
  20 use PVE::ReplicationConfig;
  21 use PVE::ReplicationState;
  22
  23 use Test::MockModule;
  24
  25 our $mocked_nodename = 'node1';
  26
  27 our $mocked_replication_jobs = {};
  28
  29 my $pve_replication_config_module = Test::MockModule->new('PVE::ReplicationConfig');
  30 my $pve_replication_state_module = Test::MockModule->new('PVE::ReplicationState');
  31
  32 our $mocked_vm_configs = {};
  33
  34 our $mocked_ct_configs = {};
  35
  36 my $mocked_get_members = sub {
  37     return {
  38         node1 => { online => 1 },
  39         node2 => { online => 1 },
  40         node3 => { online => 1 },
  41     };
  42 };
  43
  44 my $mocked_vmlist = sub {
  45     my $res = {};
  46
  47     foreach my $id (keys %$mocked_ct_configs) {
  48         my $d = $mocked_ct_configs->{$id};
  49         $res->{$id} = { 'type' => 'lxc', 'node' => $d->{node}, 'version' => 1 };
  50     }
  51     foreach my $id (keys %$mocked_vm_configs) {
  52         my $d = $mocked_vm_configs->{$id};
  53         $res->{$id} = { 'type' => 'qemu', 'node' => $d->{node}, 'version' => 1 };
  54     }
  55
  56     return { 'ids' => $res };
  57 };
  58
  59 my $mocked_get_ssh_info  = sub {
  60     my ($node, $network_cidr) = @_;
  61
  62     return { node => $node };
  63 };
  64
  65 my $mocked_ssh_info_to_command = sub {
  66     my ($info, @extra_options) = @_;
  67
  68     return ['fake_ssh', $info->{name}, @extra_options];
  69 };
  70
  71 my $statefile = ".mocked_repl_state.$$";
  72
  73 unlink $statefile;
  74 $PVE::ReplicationState::state_path = $statefile;
  75 $PVE::ReplicationState::state_lock = ".mocked_repl_state_lock.$$";
  76 $PVE::API2::Replication::pvesr_lock_path = ".mocked_pvesr_lock.$$";
  77 $PVE::GuestHelpers::lockdir = ".mocked_pve-manager_lock.$$";
  78
  79 if (!mkdir($PVE::GuestHelpers::lockdir) && !$!{EEXIST}) {
  80     # If we cannot create the guest helper lockdir we'll loop endlessly, so die
  81     # if it fails.
  82     die "mkdir($PVE::GuestHelpers::lockdir): $!\n";
  83 }
  84
  85 my $pve_sshinfo_module = Test::MockModule->new('PVE::SSHInfo');
  86
  87 my $pve_cluster_module = Test::MockModule->new('PVE::Cluster');
  88
  89 my $pve_inotify_module = Test::MockModule->new('PVE::INotify');
  90
  91 my $mocked_qemu_load_conf = sub {
  92     my ($class, $vmid, $node) = @_;
  93
  94     $node = $mocked_nodename if !$node;
  95
  96     my $conf = $mocked_vm_configs->{$vmid};
  97
  98     die "no such vm '$vmid'" if !defined($conf);
  99     die "vm '$vmid' on wrong node" if $conf->{node} ne $node;
 100
 101     return $conf;
 102 };
 103
 104 my $pve_qemuserver_module = Test::MockModule->new('PVE::QemuServer');
 105
 106 my $pve_qemuconfig_module = Test::MockModule->new('PVE::QemuConfig');
 107
 108 my $mocked_lxc_load_conf = sub {
 109     my ($class, $vmid, $node) = @_;
 110
 111     $node = $mocked_nodename if !$node;
 112
 113     my $conf = $mocked_ct_configs->{$vmid};
 114
 115     die "no such ct '$vmid'" if !defined($conf);
 116     die "ct '$vmid' on wrong node" if $conf->{node} ne $node;
 117
 118     return $conf;
 119 };
 120
 121 my $pve_lxc_config_module = Test::MockModule->new('PVE::LXC::Config');
 122
 123 my $mocked_replication_config_new = sub {
 124
 125     my $res = clone($mocked_replication_jobs);
 126
 127     return bless { ids => $res }, 'PVE::ReplicationConfig';
 128 };
 129
 130 my $mocked_storage_config = {
 131     ids => {
 132         local => {
 133             type => 'dir',
 134             shared => 0,
 135             content => {
 136                 'iso' => 1,
 137                 'backup' => 1,
 138             },
 139             path => "/var/lib/vz",
 140         },
 141         'local-zfs' => {
 142             type => 'zfspool',
 143             pool => 'nonexistent-testpool',
 144             shared => 0,
 145             content => {
 146                 'images' => 1,
 147                 'rootdir' => 1
 148             },
 149         },
 150     },
 151 };
 152
 153 my $pve_storage_module = Test::MockModule->new('PVE::Storage');
 154
 155 my $mocked_storage_content = {};
 156
 157 sub register_mocked_volid {
 158     my ($volid, $snapname) = @_;
 159
 160     my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
 161     my $scfg = $mocked_storage_config->{ids}->{$storeid} ||
 162         die "no such storage '$storeid'\n";
 163
 164     my $d = $mocked_storage_content->{$storeid}->{$volname} //= {};
 165
 166     $d->{$snapname} = 1 if $snapname;
 167 }
 168
 169 my $mocked_volume_snapshot_list = sub {
 170     my ($cfg, $volid, $prefix) = @_;
 171
 172     my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
 173     my $snaps = [];
 174
 175     if (my $d = $mocked_storage_content->{$storeid}->{$volname}) {
 176         $snaps = [keys %$d];
 177     }
 178
 179     return $snaps;
 180 };
 181
 182 my $mocked_volume_snapshot = sub {
 183     my ($cfg, $volid, $snap) = @_;
 184
 185     my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
 186
 187     my $d = $mocked_storage_content->{$storeid}->{$volname};
 188     die "no such volid '$volid'\n" if !$d;
 189     $d->{$snap} = 1;
 190 };
 191
 192 my $mocked_volume_snapshot_delete = sub {
 193     my ($cfg, $volid, $snap, $running) = @_;
 194
 195     my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
 196     my $d = $mocked_storage_content->{$storeid}->{$volname};
 197     die "no such volid '$volid'\n" if !$d;
 198     delete $d->{$snap} || die "no such snapshot '$snap' on '$volid'\n";
 199 };
 200
 201 my $pve_replication_module = Test::MockModule->new('PVE::Replication');
 202
 203 my $mocked_job_logfile_name = sub {
 204     my ($jobid) = @_;
 205
 206     return ".mocked_replication_log_$jobid";
 207 };
 208
 209 my $mocked_log_time = 0;
 210
 211 my $mocked_get_log_time = sub {
 212     return $mocked_log_time;
 213 };
 214
 215 my $locks = {};
 216
 217 my $mocked_cfs_lock_file = sub {
 218     my ($filename, $timeout, $code, @param) = @_;
 219
 220     die "$filename already locked\n" if ($locks->{$filename});
 221
 222     $locks->{$filename} = 1;
 223
 224     my $res = $code->(@param);
 225
 226     delete $locks->{$filename};
 227
 228     return $res;
 229 };
 230
 231 my $mocked_cfs_read_file = sub {
 232     my ($filename) = @_;
 233
 234     return {} if $filename eq 'datacenter.cfg';
 235     return PVE::Cluster::cfs_read_file($filename);
 236 };
 237
 238 my $mocked_cfs_write_file = sub {
 239     my ($filename, $cfg) = @_;
 240
 241     die "wrong file - $filename\n" if $filename ne 'replication.cfg';
 242
 243     $cfg->write_config(); # checks but no actual write to pmxcfs
 244 };
 245
 246 sub setup {
 247     $pve_replication_state_module->mock(job_logfile_name => $mocked_job_logfile_name);
 248     $pve_replication_module->mock(get_log_time => $mocked_get_log_time);
 249
 250     $pve_storage_module->mock(config => sub { return $mocked_storage_config; });
 251     $pve_storage_module->mock(volume_snapshot_list => $mocked_volume_snapshot_list);
 252     $pve_storage_module->mock(volume_snapshot => $mocked_volume_snapshot);
 253     $pve_storage_module->mock(volume_snapshot_delete => $mocked_volume_snapshot_delete);
 254
 255     $pve_replication_config_module->mock(
 256         new => $mocked_replication_config_new,
 257         lock => sub { $mocked_cfs_lock_file->('replication.cfg', undef, $_[0]); },
 258         write => sub { $mocked_cfs_write_file->('replication.cfg', $_[0]); },
 259     );
 260     $pve_qemuserver_module->mock(check_running => sub { return 0; });
 261     $pve_qemuconfig_module->mock(load_config => $mocked_qemu_load_conf);
 262
 263     $pve_lxc_config_module->mock(load_config => $mocked_lxc_load_conf);
 264
 265     $pve_sshinfo_module->mock(
 266         get_ssh_info => $mocked_get_ssh_info,
 267         ssh_info_to_command => $mocked_ssh_info_to_command,
 268     );
 269
 270     $pve_cluster_module->mock(
 271         get_vmlist => sub { return $mocked_vmlist->(); },
 272         get_members => $mocked_get_members,
 273         cfs_update => sub {},
 274         cfs_lock_file => $mocked_cfs_lock_file,
 275         cfs_write_file => $mocked_cfs_write_file,
 276         cfs_read_file => $mocked_cfs_read_file,
 277     );
 278     $pve_inotify_module->mock('nodename' => sub { return $mocked_nodename; });
 279 };
 280
 281 # code to generate/conpare test logs
 282
 283 my $logname;
 284 my $logfh;
 285
 286 sub openlog {
 287     my ($filename) = @_;
 288
 289     if (!$filename) {
 290         # compute from $0
 291         $filename = basename($0);
 292         if ($filename =~ m/^(\S+)\.pl$/) {
 293             $filename = "$1.log";
 294         } else {
 295             die "unable to compute log name for $0";
 296         }
 297     }
 298
 299     die "log already open" if defined($logname);
 300
 301     open (my $fh, ">", "$filename.tmp") ||
 302         die "unable to open log  - $!";
 303
 304     $logname = $filename;
 305     $logfh = $fh;
 306 }
 307
 308 sub commit_log {
 309
 310     close($logfh);
 311
 312     if (-f $logname) {
 313         my $diff = `diff -u '$logname' '$logname.tmp'`;
 314         if ($diff) {
 315             warn "got unexpected output\n";
 316             print "# diff -u '$logname' '$logname.tmp'\n";
 317             print $diff;
 318             exit(-1);
 319         }
 320     } else {
 321         rename("$logname.tmp", $logname) || die "rename log failed - $!";
 322     }
 323 }
 324
 325 my $status;
 326
 327 # helper to track job status
 328 sub track_jobs {
 329     my ($ctime) = @_;
 330
 331     $mocked_log_time = $ctime;
 332
 333     my $logmsg = sub {
 334         my ($msg) = @_;
 335
 336         print "$msg\n";
 337         print $logfh "$msg\n";
 338     };
 339
 340     if (!$status) {
 341         $status = PVE::ReplicationState::job_status();
 342         foreach my $jobid (sort keys %$status) {
 343             my $jobcfg = $status->{$jobid};
 344             $logmsg->("$ctime $jobid: new job next_sync => $jobcfg->{next_sync}");
 345         }
 346     }
 347
 348     PVE::API2::Replication::run_jobs($ctime, $logmsg, 1);
 349
 350     my $new = PVE::ReplicationState::job_status();
 351
 352     # detect removed jobs
 353     foreach my $jobid (sort keys %$status) {
 354         if (!$new->{$jobid}) {
 355             $logmsg->("$ctime $jobid: vanished job");
 356         }
 357     }
 358
 359     foreach my $jobid (sort keys %$new) {
 360         my $jobcfg = $new->{$jobid};
 361         my $oldcfg = $status->{$jobid};
 362         if (!$oldcfg) {
 363             $logmsg->("$ctime $jobid: new job next_sync => $jobcfg->{next_sync}");
 364             next; # no old state to compare
 365         } else {
 366             foreach my $k (qw(target guest vmtype next_sync)) {
 367                 my $changes = '';
 368                 if ($oldcfg->{$k} ne $jobcfg->{$k}) {
 369                     $changes .= ', ' if $changes;
 370                     $changes .= "$k => $jobcfg->{$k}";
 371                 }
 372                 $logmsg->("$ctime $jobid: changed config $changes") if $changes;
 373             }
 374         }
 375
 376         my $oldstate = $oldcfg->{state};
 377
 378         my $state = $jobcfg->{state};
 379
 380         my $changes = '';
 381         foreach my $k (qw(last_node last_try last_sync fail_count error)) {
 382             if (($oldstate->{$k} // '') ne ($state->{$k} // '')) {
 383                 my $value = $state->{$k} // '';
 384                 chomp $value;
 385                 $changes .= ', ' if $changes;
 386                 $changes .= "$k => $value";
 387             }
 388         }
 389         $logmsg->("$ctime $jobid: changed state $changes") if $changes;
 390
 391         my $old_storeid_list = $oldstate->{storeid_list};
 392         my $storeid_list = $state->{storeid_list};
 393
 394         my $storeid_list_changes = 0;
 395         foreach my $storeid (@$storeid_list) {
 396             next if grep { $_ eq $storeid } @$old_storeid_list;
 397             $storeid_list_changes = 1;
 398         }
 399
 400         foreach my $storeid (@$old_storeid_list) {
 401             next if grep { $_ eq $storeid } @$storeid_list;
 402             $storeid_list_changes = 1;
 403         }
 404
 405         $logmsg->("$ctime $jobid: changed storeid list " . join(',', @$storeid_list))
 406             if $storeid_list_changes;
 407     }
 408     $status = $new;
 409 }
 410
 411
 412 1;