src/PVE/HA/Sim/TestHardware.pm

   1 package PVE::HA::Sim::TestHardware;
   2
   3 # Simulate Hardware resources
   4
   5 # power supply for nodes: on/off
   6 # network connection to nodes: on/off
   7 # watchdog devices for nodes
   8
   9 use strict;
  10 use warnings;
  11 use POSIX qw(strftime EINTR);
  12 use Data::Dumper;
  13 use JSON;
  14 use IO::File;
  15 use Fcntl qw(:DEFAULT :flock);
  16 use File::Copy;
  17 use File::Path qw(make_path remove_tree);
  18
  19 use PVE::HA::CRM;
  20 use PVE::HA::LRM;
  21
  22 use PVE::HA::Sim::TestEnv;
  23 use base qw(PVE::HA::Sim::Hardware);
  24
  25 my $max_sim_time = 10000;
  26
  27 sub new {
  28     my ($this, $testdir) = @_;
  29
  30     my $class = ref($this) || $this;
  31
  32     my $self = $class->SUPER::new($testdir);
  33
  34     my $raw = PVE::Tools::file_get_contents("$testdir/cmdlist");
  35     $self->{cmdlist} = decode_json($raw);
  36
  37     $self->{loop_count} = 0;
  38     $self->{cur_time} = 0;
  39
  40     my $statusdir = $self->statusdir();
  41     my $logfile = "$statusdir/log";
  42     $self->{logfh} = IO::File->new(">>$logfile") ||
  43         die "unable to open '$logfile' - $!";
  44
  45     foreach my $node (sort keys %{$self->{nodes}}) {
  46
  47         my $d = $self->{nodes}->{$node};
  48
  49         $d->{crm_env} =
  50             PVE::HA::Env->new('PVE::HA::Sim::TestEnv', $node, $self, 'crm');
  51
  52         $d->{lrm_env} =
  53             PVE::HA::Env->new('PVE::HA::Sim::TestEnv', $node, $self, 'lrm');
  54
  55         $d->{crm} = undef; # create on power on
  56         $d->{lrm} = undef; # create on power on
  57     }
  58
  59     return $self;
  60 }
  61
  62 sub get_time {
  63     my ($self) = @_;
  64
  65     return $self->{cur_time};
  66 }
  67
  68 sub log {
  69     my ($self, $level, $msg, $id) = @_;
  70
  71     chomp $msg;
  72
  73     my $time = $self->get_time();
  74
  75     $id = 'hardware' if !$id;
  76
  77     my $line = sprintf("%-5s %5d %12s: $msg\n", $level, $time, $id);
  78     print $line;
  79
  80     $self->{logfh}->print($line);
  81     $self->{logfh}->flush();
  82 }
  83
  84 # simulate hardware commands
  85 # power <node> <on|off>
  86 # network <node> <on|off>
  87 # reboot <node>
  88 # shutdown <node>
  89 # restart-lrm <node>
  90 # service <sid> <enabled|disabled>
  91 # service <sid> <migrate|relocate> <target>
  92
  93 sub sim_hardware_cmd {
  94     my ($self, $cmdstr, $logid) = @_;
  95
  96     my $code = sub {
  97
  98         my $cstatus = $self->read_hardware_status_nolock();
  99
 100         my ($cmd, $objid, $action, $target) = split(/\s+/, $cmdstr);
 101
 102         die "sim_hardware_cmd: no node or service for command specified"
 103             if !$objid;
 104
 105         my ($node, $sid, $d);
 106
 107         if ($cmd eq 'service') {
 108             $sid = PVE::HA::Tools::pve_verify_ha_resource_id($objid);
 109         } else {
 110             $node = $objid;
 111             $d = $self->{nodes}->{$node} ||
 112                 die "sim_hardware_cmd: no such node '$node'\n";
 113         }
 114
 115         $self->log('info', "execute $cmdstr", $logid);
 116
 117         if ($cmd eq 'power') {
 118             die "sim_hardware_cmd: unknown action '$action'" if $action !~ m/^(on|off)$/;
 119             if ($cstatus->{$node}->{power} ne $action) {
 120                 if ($action eq 'on') {
 121                     $d->{crm} = PVE::HA::CRM->new($d->{crm_env}) if !$d->{crm};
 122                     $d->{lrm} = PVE::HA::LRM->new($d->{lrm_env}) if !$d->{lrm};
 123                     $d->{lrm_restart} = undef;
 124                 } else {
 125                     if ($d->{crm}) {
 126                         $d->{crm_env}->log('info', "killed by poweroff");
 127                         $d->{crm} = undef;
 128                     }
 129                     if ($d->{lrm}) {
 130                         $d->{lrm_env}->log('info', "killed by poweroff");
 131                         $d->{lrm} = undef;
 132                         $d->{lrm_restart} = undef;
 133                     }
 134                     $self->watchdog_reset_nolock($node);
 135                     $self->write_service_status($node, {});
 136                 }
 137             }
 138
 139             $cstatus->{$node}->{power} = $action;
 140             $cstatus->{$node}->{network} = $action;
 141             $cstatus->{$node}->{shutdown} = undef;
 142
 143             $self->write_hardware_status_nolock($cstatus);
 144
 145         } elsif ($cmd eq 'network') {
 146             die "sim_hardware_cmd: unknown network action '$action'"
 147                 if $action !~ m/^(on|off)$/;
 148             $cstatus->{$node}->{network} = $action;
 149
 150             $self->write_hardware_status_nolock($cstatus);
 151
 152         } elsif ($cmd eq 'reboot' || $cmd eq 'shutdown') {
 153             $cstatus->{$node}->{shutdown} = $cmd;
 154
 155             $self->write_hardware_status_nolock($cstatus);
 156
 157             $d->{lrm}->shutdown_request() if $d->{lrm};
 158         } elsif ($cmd eq 'restart-lrm') {
 159             if ($d->{lrm}) {
 160                 $d->{lrm_restart} = 1;
 161                 $d->{lrm}->shutdown_request();
 162             }
 163         } elsif ($cmd eq 'service') {
 164             if ($action eq 'enabled' || $action eq 'disabled') {
 165
 166                 $self->set_service_state($sid, $action);
 167
 168             } elsif ($action eq 'migrate' || $action eq 'relocate') {
 169
 170                 die "sim_hardware_cmd: missing target node for '$action' command"
 171                     if !$target;
 172
 173                 $self->queue_crm_commands_nolock("$action $sid $target");
 174
 175             } else {
 176                 die "sim_hardware_cmd: unknown service action '$action' " .
 177                     "- not implemented\n"
 178             }
 179         } else {
 180             die "sim_hardware_cmd: unknown command '$cmdstr'\n";
 181         }
 182
 183     };
 184
 185     return $self->global_lock($code);
 186 }
 187
 188 sub run {
 189     my ($self) = @_;
 190
 191     my $last_command_time = 0;
 192     my $next_cmd_at = 0;
 193
 194     for (;;) {
 195
 196         my $starttime = $self->get_time();
 197
 198         my @nodes = sort keys %{$self->{nodes}};
 199
 200         my $nodecount = scalar(@nodes);
 201
 202         my $looptime = $nodecount*2;
 203         $looptime = 20 if $looptime < 20;
 204
 205         die "unable to simulate so many nodes. You need to increate watchdog/lock timeouts.\n"
 206             if $looptime >= 60;
 207
 208         foreach my $node (@nodes) {
 209
 210             my $d = $self->{nodes}->{$node};
 211
 212             if (my $crm = $d->{crm}) {
 213
 214                 $d->{crm_env}->loop_start_hook($self->get_time());
 215
 216                 die "implement me (CRM exit)" if !$crm->do_one_iteration();
 217
 218                 $d->{crm_env}->loop_end_hook();
 219
 220                 my $nodetime = $d->{crm_env}->get_time();
 221                 $self->{cur_time} = $nodetime if $nodetime > $self->{cur_time};
 222             }
 223
 224             if (my $lrm = $d->{lrm}) {
 225
 226                 $d->{lrm_env}->loop_start_hook($self->get_time());
 227
 228                 my $exit_lrm = !$lrm->do_one_iteration();
 229
 230                 $d->{lrm_env}->loop_end_hook();
 231
 232                 my $nodetime = $d->{lrm_env}->get_time();
 233                 $self->{cur_time} = $nodetime if $nodetime > $self->{cur_time};
 234
 235                 if ($exit_lrm) {
 236                     $d->{lrm_env}->log('info', "exit (loop end)");
 237                     $d->{lrm} = undef;
 238                     my $cstatus = $self->read_hardware_status_nolock();
 239                     my $nstatus = $cstatus->{$node} || die "no node status for node '$node'";
 240                     my $shutdown = $nstatus->{shutdown} || '';
 241                     if ($d->{lrm_restart}) {
 242                         die "lrm restart during shutdown - not implemented" if $shutdown;
 243                         $d->{lrm_restart} = undef;
 244                         $d->{lrm} = PVE::HA::LRM->new($d->{lrm_env});
 245                     } elsif ($shutdown eq 'reboot') {
 246                         $self->sim_hardware_cmd("power $node off", 'reboot');
 247                         $self->sim_hardware_cmd("power $node on", 'reboot');
 248                     } elsif ($shutdown eq 'shutdown') {
 249                         $self->sim_hardware_cmd("power $node off", 'shutdown');
 250                     } else {
 251                         die "unexpected LRM exit - not implemented"
 252                     }
 253                 }
 254             }
 255
 256             foreach my $n (@nodes) {
 257                 if (!$self->watchdog_check($n)) {
 258                     $self->sim_hardware_cmd("power $n off", 'watchdog');
 259                     $self->log('info', "server '$n' stopped by poweroff (watchdog)");
 260                     $self->{nodes}->{$n}->{crm} = undef;
 261                     $self->{nodes}->{$n}->{lrm} = undef;
 262                 }
 263             }
 264         }
 265
 266
 267         $self->{cur_time} = $starttime + $looptime
 268             if ($self->{cur_time} - $starttime) < $looptime;
 269
 270         die "simulation end\n" if $self->{cur_time} > $max_sim_time;
 271
 272         foreach my $node (@nodes) {
 273             my $d = $self->{nodes}->{$node};
 274             # forced time update
 275             $d->{lrm_env}->loop_start_hook($self->get_time());
 276             $d->{crm_env}->loop_start_hook($self->get_time());
 277         }
 278
 279         next if $self->{cur_time} < $next_cmd_at;
 280
 281         # apply new comand after 5 loop iterations
 282
 283         if (($self->{loop_count} % 5) == 0) {
 284             my $list = shift @{$self->{cmdlist}};
 285             if (!$list) {
 286                 # end sumulation (500 seconds after last command)
 287                 return if (($self->{cur_time} - $last_command_time) > 500);
 288             }
 289
 290             foreach my $cmd (@$list) {
 291                 $last_command_time = $self->{cur_time};
 292
 293                 if ($cmd =~ m/^delay\s+(\d+)\s*$/) {
 294                     $next_cmd_at = $self->{cur_time} + $1;
 295                 } else {
 296                     $self->sim_hardware_cmd($cmd, 'cmdlist');
 297                 }
 298             }
 299         }
 300
 301         ++$self->{loop_count};
 302     }
 303 }
 304
 305 1;